From fea7db33071fa968fc597e63507099a66d86a5e9 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Thu, 5 Nov 2026 09:00:40 -0700 Subject: [PATCH 1/5] fix(durable): lock the Handle and refuse steps after a failed Append Fire, Tick, RunService and DeliverToActor now hold a lock for the whole step, so a host's timer loop no longer loses acknowledged steps. A step that applied to the kernel but wasn't appended leaves the instance ahead of the journal, so the Handle refuses further steps with ErrHandleFailed until it's recovered. Relates to #214. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 10 + durable/CHANGELOG.md | 16 ++ durable/README.md | 10 + durable/actor.go | 20 +- durable/errors.go | 10 +- durable/runner.go | 85 ++++++-- durable/runner_test.go | 244 ++++++++++++++++++++++ durable/service.go | 20 +- 8 files changed, 391 insertions(+), 24 deletions(-) diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index 8dfd056b..4680e2a8 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -125,6 +125,16 @@ Time-travel needs the full record history through the target step. A `Store` opt in by implementing `HistoryStore` (the in-tree `MemStore` does so under `WithHistory`); `StateAt` otherwise falls back to the latest checkpoint plus tail. +## Handles across goroutines + +`Fire`, `Tick`, `RunService` and `DeliverToActor` are safe to call from different +goroutines: each holds the Handle's lock for the whole step. A host callback must not +call back into its Handle. `Instance()` is outside the lock. + +A step that may not have been recorded fails the Handle with `ErrHandleFailed`. `Recover` +a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a +newer owner holds the instance, so stop. + ## A note on serialized payloads Events, service done-data, actor done-data, and actor messages are recorded as their diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 543dc021..5067e0d0 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -25,9 +25,23 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. a refused payload, and `ErrRecordTooLarge` a Record too large for a `FileStore` line. `Record.EventData` and `Record.EventDataError` hold the payload. +- **`ErrHandleFailed`** reports a Handle out of step with its journal. + ### Changed - Requires Go 1.27.3. +- **A Handle's steps are serialized across goroutines.** `Fire`, `Tick`, `RunService` and + `DeliverToActor` hold a lock for the whole step, so a host's timer loop no longer loses + acknowledged steps. A host callback must not call back into its Handle, or it + deadlocks. `Instance()` isn't covered: read it only while no step is running. A `Tick`, + `RunService` or `DeliverToActor` whose context ended while it waited returns that error + and changes nothing. +- **A step that may not have been recorded fails the Handle.** That call and every later + one return `ErrHandleFailed`, wrapping the cause. `Recover` a new Handle with the same + options to continue. A service whose result wasn't recorded is in flight again, so + `RunService` runs it a second time. An `InvokeActor` actor restarts from its initial + state, and a `Spawn` actor doesn't return. If the cause is `ErrFenced`, a newer owner + holds the instance: stop. - **A `WithEventData` payload reaches `Assign` decoded, live as well as on replay.** A struct arrives as a map and an integer as a `float64`, which loses precision above 2^53, so an assertion to the original @@ -59,6 +73,8 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Fixed +- **A failed checkpoint no longer makes the next step conflict.** `Tick`, `RunService` + and `DeliverToActor` now advance the next step as soon as the append lands. - **The next step survives a checkpoint that compacts tick barriers.** A checkpoint now records the step that follows it, in a `nextStep` field of its envelope. Recovery used the Trace count, which runs low past a tick barrier, so the next Fire could reuse a diff --git a/durable/README.md b/durable/README.md index 9b9858c6..f720c52f 100644 --- a/durable/README.md +++ b/durable/README.md @@ -136,6 +136,16 @@ opts in by implementing `HistoryStore` (the in-tree `MemStore` does so under `WithHistory`); `StateAt` falls back to `Load` (latest checkpoint plus tail) otherwise. +## Handles across goroutines + +`Fire`, `Tick`, `RunService` and `DeliverToActor` are safe to call from different +goroutines: each holds the Handle's lock for the whole step. A host callback must not +call back into its Handle. `Instance()` is outside the lock. + +A step that may not have been recorded fails the Handle with `ErrHandleFailed`. `Recover` +a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a +newer owner holds the instance, so stop. + ## Caveat: serialized payloads Events, service done-data, actor done-data, and actor messages are recorded as their diff --git a/durable/actor.go b/durable/actor.go index 792ba00d..41abf380 100644 --- a/durable/actor.go +++ b/durable/actor.go @@ -99,10 +99,21 @@ type actorMessagePayload struct { // re-instantiating the actor. // // It returns whether the actor was found running (false for a ref that names no -// live actor — already settled, stopped, or never spawned), and a persistence -// failure as the error. The kernel transitions the delivery drove do not -// themselves fail. +// live actor — already settled, stopped, or never spawned — or with an error when the +// Handle has failed or ctx has ended), and a persistence failure as the error. The +// kernel transitions the delivery drove do not themselves fail. After a failed Append +// the result wasn't recorded: on a recovered Handle an InvokeActor actor restarts from its +// initial state, and a Spawn actor doesn't return. Resending a recorded delivery repeats +// any message it sends the parent. func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef, event any) (bool, error) { + h.mu.Lock() + defer h.mu.Unlock() + if err := h.refuse(); err != nil { + return false, err + } + if err := ctx.Err(); err != nil { + return false, err + } if h.actors == nil { return false, fmt.Errorf("durable: no actor palette wired for %q (supply WithActorPalette)", h.id) } @@ -116,7 +127,7 @@ func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef output, settled, settleErr := h.actorOutcome() entries, err := recordActorTransitions(ref.ID, h.inst.History(), before, output, settled, settleErr) if err != nil { - return true, fmt.Errorf("durable: recording actor %q result for %q: %w", ref.ID, h.id, err) + return true, h.fail(fmt.Errorf("durable: recording actor %q result for %q: %w", ref.ID, h.id, err)) } if h.actorBuf != nil { *h.actorBuf = append(*h.actorBuf, entries...) @@ -127,7 +138,6 @@ func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef if err := h.persistStep(ctx, step, &rec); err != nil { return true, err } - h.nextStep++ return true, nil } diff --git a/durable/errors.go b/durable/errors.go index d08e5f47..5ac19d7e 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -16,6 +16,7 @@ const ( errorStepOutOfRange = "time-travel step out of range" errorEventDataEncoding = "event data can't be recorded" errorRecordTooLarge = "record too large for the journal" + errorHandleFailed = "handle is out of step with the journal" ) // Sentinel errors a Store reports. Callers match them with errors.Is. @@ -49,9 +50,16 @@ var ( // ErrRecordTooLarge is reported by FileStore.Append for a Record whose journal line // exceeds the store's line limit, which a later Load could not read back, even // without its Snapshot. No journal line is written, but a Handle's kernel has already - // applied the step. + // applied the step, so the Handle fails with ErrHandleFailed. ErrRecordTooLarge = errors.New(errorPrefix + errorRecordTooLarge) + // ErrHandleFailed is reported by a Handle's Fire, Tick, RunService and + // DeliverToActor once a step applied to the live instance but may not have been + // recorded, on that call and every later one. It wraps the cause. Recover a new + // Handle with the same options to continue, unless the cause is ErrFenced: then a + // newer owner holds the instance, so stop. + ErrHandleFailed = errors.New(errorPrefix + errorHandleFailed) + // ErrCheckpointNotAdvancing is reported by Store.Checkpoint when throughStep // does not advance beyond the instance's current checkpoint. ErrCheckpointNotAdvancing = errors.New(errorPrefix + errorCheckpointNotAdvancing) diff --git a/durable/runner.go b/durable/runner.go index e77e5a11..442fec6f 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "sync" "time" "github.com/stablekernel/crucible/state" @@ -76,11 +77,33 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // clock that journals every reading the scheduler consumes, so timer-driven // transitions are durable and replay wall-clock-independent. Obtain a Handle from // Runner.Start or Recover. +// +// Fire, Tick, RunService and DeliverToActor are safe to call from different +// goroutines: each holds the Handle's lock for the whole step, including the Store +// write, effect dispatch and any service it runs. Any host callback run during a step +// (an effect handler, service, guard, action, reducer or actor behavior) must not call +// back into the same Handle, or it deadlocks. A step waits for the lock without watching +// its context, so a slow service delays other steps past their deadlines; a Tick, +// RunService or DeliverToActor whose context has ended by then returns that error and +// changes nothing, and the kernel refuses a Fire. Instance returns the +// live instance outside that lock, so read it only while no step is running. A +// stateless Runner.Fire builds its own Handle, so it isn't serialized with this one. +// +// A step that applied to the kernel but may not have been recorded leaves the Handle out +// of step with the journal. That call and every later one return ErrHandleFailed, +// wrapping the cause. Recover a new Handle with the same options to continue. A service +// whose result wasn't recorded is in flight again, so RunService runs it a second time. +// A running InvokeActor actor restarts from its initial state on any Recover, losing every +// delivery so far, and a Spawn actor doesn't return. If the cause is ErrFenced, a +// newer owner holds the instance: stop, because recovering won't help. type Handle[S comparable, E comparable, C any] struct { - runner *Runner[S, E, C] - id InstanceID - inst *state.Instance[S, E, C] + runner *Runner[S, E, C] + id InstanceID + inst *state.Instance[S, E, C] + + mu sync.Mutex // serializes the steps, and guards the instance and every field below nextStep int + failed error // the error that took the Handle out of step with the journal, nil until then sched *state.Scheduler[S, E, C] recClock *recordingClock // owns the clock-read accumulator and its mutex @@ -93,6 +116,21 @@ type Handle[S comparable, E comparable, C any] struct { actorBuf *[]state.JournalEntry // recording actor-transition accumulator, flushed per step } +// refuse reports ErrHandleFailed once the Handle is out of step with the journal. +func (h *Handle[S, E, C]) refuse() error { + if h.failed == nil { + return nil + } + return fmt.Errorf("%w: %q: %w", ErrHandleFailed, h.id, h.failed) +} + +// fail records err as the reason the Handle is out of step with the journal, and +// returns the same refusal every later step gets. +func (h *Handle[S, E, C]) fail(err error) error { + h.failed = err + return h.refuse() +} + // drainClock returns and clears the clock readings accumulated since the last // drain, so each recorded step carries exactly the readings consumed during it. // The drain is taken under the recording clock's mutex, so closing a step's @@ -227,7 +265,8 @@ func (h *Handle[S, E, C]) dropFiredTimers(now time.Time) { // Instance returns the underlying kernel Instance the Handle wraps, for reads // such as Configuration, Snapshot, or Current. Drive it through the Handle's Fire // (or the Runner's) so steps continue to be recorded; firing the bare Instance -// bypasses durability. +// bypasses durability. Read it only while no step is running: it's outside the Handle's +// lock. After ErrHandleFailed it shows state the journal may not hold. func (h *Handle[S, E, C]) Instance() *state.Instance[S, E, C] { return h.inst } // ID returns the InstanceID the Handle records under. @@ -343,7 +382,8 @@ func (r *Runner[S, E, C]) newActorSystem(inst *state.Instance[S, E, C]) *state.A // how a state.WithEventData payload is handled, and when it's refused. It is the // stateless entry point (no Handle required); for a hot path that fires many // events in sequence, hold a Handle from Start or Recover and use Handle.Fire to -// avoid reloading between steps. +// avoid reloading between steps. Calls on one id aren't serialized, and must not overlap +// each other or a live Handle. func (r *Runner[S, E, C]) Fire(ctx context.Context, id InstanceID, event E, opts ...state.FireOption) (state.FireResult[S], error) { h, err := r.recover(ctx, id) if err != nil { @@ -370,6 +410,11 @@ func (r *Runner[S, E, C]) Fire(ctx context.Context, id InstanceID, event E, opts // kernel runs. The payload is read on the calling goroutine with no lock of its own, so // don't mutate it concurrently with Fire. func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireOption) (state.FireResult[S], error) { + h.mu.Lock() + defer h.mu.Unlock() + if err := h.refuse(); err != nil { + return state.FireResult[S]{}, err + } // Encode the WithEventData payload before the kernel runs, so a payload that // can't be recorded refuses the Fire rather than applying a step it can't record. payload, opts, err := encodeEventData(opts, h.runner.cfg.maxEventData) @@ -395,7 +440,7 @@ func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireO des := dispatchableEffects(step, res.Effects) effEnvs, err := recordEffects(des) if err != nil { - return res, err + return res, h.fail(err) } rec := Record{Step: step, Event: []byte(res.Trace.EventPayload), EventData: payload.data, EventDataError: payload.errMsg, Entries: h.drainClock(), Effects: effEnvs} // Write-ahead append the step Record (with its effect ids) BEFORE dispatching, but @@ -403,7 +448,7 @@ func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireO // a crash mid-dispatch leaves the effect recorded in the still-present tail for // recovery to redispatch — never orphaned behind the compaction. if err := h.appendStep(ctx, step, &rec); err != nil { - return res, err + return res, h.fail(err) } h.nextStep++ // Dispatch AFTER the append: a crash in this window leaves the effect recorded @@ -429,6 +474,16 @@ func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireO // count of timers fired — so recovery re-derives the same timers at their // recorded instants. It returns the FireResults of the timers it fired, in order. func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], error) { + h.mu.Lock() + defer h.mu.Unlock() + if err := h.refuse(); err != nil { + return nil, err + } + // A context that ended in the lock queue must stop here: the scheduler would consume + // a due timer whose Fire the kernel then refuses. + if err := ctx.Err(); err != nil { + return nil, err + } results := h.sched.Tick(ctx) step := h.nextStep @@ -450,20 +505,24 @@ func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], erro if len(rec.Entries) == 0 && rec.TickSteps == 0 { return results, nil } - if err := h.persistStep(ctx, step, &rec); err != nil { + if err := h.appendStep(ctx, step, &rec); err != nil { + return results, h.fail(err) + } + h.nextStep = rec.followingStep() + if err := h.checkpointStep(ctx, step, &rec); err != nil { return results, err } - h.nextStep += len(results) + 1 return results, nil } -// persistStep checkpoints if the policy is due, then write-ahead appends rec -// before the step is acknowledged to the caller. The checkpoint is taken at the -// barrier's own step so a Load after it returns this Snapshot plus the later tail. +// persistStep write-ahead appends rec, advances the next step, then checkpoints if +// the policy is due. A failed append fails the Handle, because the kernel already +// applied the step; a failed checkpoint doesn't, because the Record is durable. func (h *Handle[S, E, C]) persistStep(ctx context.Context, step int, rec *Record) error { if err := h.appendStep(ctx, step, rec); err != nil { - return err + return h.fail(err) } + h.nextStep = rec.followingStep() return h.checkpointStep(ctx, step, rec) } diff --git a/durable/runner_test.go b/durable/runner_test.go index 2d33819a..bc5b3f7b 100644 --- a/durable/runner_test.go +++ b/durable/runner_test.go @@ -4,6 +4,10 @@ import ( "context" "encoding/json" "errors" + "runtime" + "strings" + "sync" + "sync/atomic" "testing" "time" @@ -511,6 +515,21 @@ func quantityMachine() *state.Machine[string, string, *runCtx] { Quench() } +// flakyAppendStore fails the first Append of one step, then delegates. +type flakyAppendStore struct { + durable.Store + failStep int + failed bool +} + +func (s *flakyAppendStore) Append(ctx context.Context, id durable.InstanceID, rec durable.Record, opts ...durable.AppendOption) (int64, error) { + if rec.Step == s.failStep && !s.failed { + s.failed = true + return 0, errors.New("store unavailable") + } + return s.Store.Append(ctx, id, rec, opts...) +} + // TestRecover_TwoHandlesOnOneInstance_JournalMatchesAnAcknowledgedHistory proves // two Handles on one instance (a stalled worker, and the worker that recovered the // instance after it) cannot both have their Fires acknowledged while the journal @@ -552,6 +571,71 @@ func TestRecover_TwoHandlesOnOneInstance_JournalMatchesAnAcknowledgedHistory(t * } } +// TestRunner_FireDuringTimerLoop_LosesNoAcknowledgedSteps proves the "Tick from its +// own timer loop" pattern that drainClock documents holds while requests Fire on +// the same Handle: the recovered instance matches the live one, and every +// acknowledged Fire survives. +func TestRunner_FireDuringTimerLoop_LosesNoAcknowledgedSteps(t *testing.T) { + if runtime.GOMAXPROCS(0) < 2 { + t.Skip("needs two CPUs to interleave Fire and Tick") + } + ctx := context.Background() + m := pollingMachine() + store := durable.NewMemStore() + id := durable.InstanceID("poll-1") + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *runCtx]{durable.WithRunnerClock[string, string, *runCtx](clk)} + runner := durable.NewRunner(m, store, opts...) + h, err := runner.Start(ctx, id, &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + + const n = 200 + var acked, fired, failed atomic.Int64 + var wg sync.WaitGroup + wg.Add(2) + go func() { // the host's timer loop + defer wg.Done() + for range n { + clk.Advance(time.Second) + results, terr := h.Tick(ctx) + if terr != nil { + failed.Add(1) + } + fired.Add(int64(len(results))) + } + }() + go func() { // request traffic + defer wg.Done() + for range n { + if _, ferr := h.Fire(ctx, "poke"); ferr != nil { + failed.Add(1) + continue + } + acked.Add(1) + } + }() + wg.Wait() + if failed.Load() != 0 || acked.Load() != n || fired.Load() == 0 { + t.Fatalf("failed %d calls, acknowledged %d of %d Fires, fired %d timers", failed.Load(), acked.Load(), n, fired.Load()) + } + + recovered, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got, want := recovered.Instance().Entity().Count, int(acked.Load()); got != want { + t.Fatalf("recovered %d pokes, want %d acknowledged", got, want) + } + if live, got := handleSnapshot(t, h), handleSnapshot(t, recovered); got != live { + t.Fatalf("recovered instance differs from live:\n live: %s\n got: %s", live, got) + } +} + // TestRecover_FireWithEventData_MatchesLive proves an instance driven with // state.WithEventData recovers to the live state, because the Handle journals the data. func TestRecover_FireWithEventData_MatchesLive(t *testing.T) { @@ -629,6 +713,36 @@ func TestRecover_StepAfterTickCheckpoint_IsPersisted(t *testing.T) { } } +// TestRunner_FireAfterFailedAppend_MatchesJournal proves a Fire whose Append failed +// acknowledges nothing further: the Handle refuses, and recovery matches the journal. +func TestRunner_FireAfterFailedAppend_MatchesJournal(t *testing.T) { + ctx := context.Background() + m := pollingMachine() + store := &flakyAppendStore{Store: durable.NewMemStore(), failStep: 1} + id := durable.InstanceID("poll-1") + h, err := durable.NewRunner(m, store).Start(ctx, id, &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + if _, err = h.Fire(ctx, "poke"); !errors.Is(err, durable.ErrHandleFailed) { // step 1: the Append fails + t.Fatalf("Fire(poke) = %v, want ErrHandleFailed", err) + } + if _, err = h.Fire(ctx, "poke"); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("second Fire(poke) = %v, want ErrHandleFailed", err) + } + + recovered, err := durable.Recover(ctx, m, store, id) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got := recovered.Instance().Entity().Count; got != 0 { + t.Fatalf("recovered %d pokes, want 0: none was recorded", got) + } +} + // lastStep returns the Step of the newest Record in a history-keeping store. func lastStep(t *testing.T, store *durable.MemStore, id durable.InstanceID) int { t.Helper() @@ -725,3 +839,133 @@ func TestRecover_CheckpointWithoutNextStep_UsesTheTraceCount(t *testing.T) { t.Fatalf("poke recorded at step %d, want 3", got) } } + +// TestHandle_FailedAppend_RefusesEveryStep proves the failed Handle stays refused, +// names the cause, and that a recovered Handle continues from the journal. +func TestHandle_FailedAppend_RefusesEveryStep(t *testing.T) { + ctx := context.Background() + m := pollingMachine() + store := &flakyAppendStore{Store: durable.NewMemStore(), failStep: 1} + id := durable.InstanceID("poll-1") + h, err := durable.NewRunner(m, store).Start(ctx, id, &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + if _, err = h.Fire(ctx, "poke"); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire(poke) = %v, want ErrHandleFailed", err) + } + if _, err = h.Fire(ctx, "poke"); !errors.Is(err, durable.ErrHandleFailed) || !strings.Contains(err.Error(), "store unavailable") { + t.Fatalf("Fire after the failure = %v, want ErrHandleFailed naming the cause", err) + } + if _, err = h.Tick(ctx); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Tick after the failure = %v, want ErrHandleFailed", err) + } + recovered, err := durable.Recover(ctx, m, store, id) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if _, err = recovered.Fire(ctx, "poke"); err != nil { + t.Fatalf("Fire on the recovered Handle: %v", err) + } +} + +// failingCheckpointStore fails the next Checkpoint after armed is set. +type failingCheckpointStore struct { + durable.Store + armed bool +} + +func (s *failingCheckpointStore) Checkpoint(ctx context.Context, id durable.InstanceID, snapshot []byte, throughStep int, opts ...durable.CheckpointOption) error { + if s.armed { + s.armed = false + return errors.New("checkpoint unavailable") + } + return s.Store.Checkpoint(ctx, id, snapshot, throughStep, opts...) +} + +// TestHandle_FailedCheckpointOnTick_KeepsGoing proves a failed checkpoint doesn't fail +// the Handle or make the next step conflict: the Record is durable. +func TestHandle_FailedCheckpointOnTick_KeepsGoing(t *testing.T) { + ctx := context.Background() + store := &failingCheckpointStore{Store: durable.NewMemStore()} + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *runCtx]{ + durable.WithRunnerClock[string, string, *runCtx](clk), + durable.WithCheckpointEvery[string, string, *runCtx](2), + } + h, err := durable.NewRunner(pollingMachine(), store, opts...).Start(ctx, "poll-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + store.armed = true + clk.Advance(time.Second) + if _, err = h.Tick(ctx); err == nil || errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Tick = %v, want the plain checkpoint error", err) + } + if _, err = h.Fire(ctx, "poke"); err != nil { + t.Fatalf("Fire after the failed checkpoint: %v", err) + } +} + +// TestHandle_FencedOwner_Stops proves a Handle that lost its instance reports the fence +// through ErrHandleFailed, so a host can tell it must stop rather than recover. +func TestHandle_FencedOwner_Stops(t *testing.T) { + ctx := context.Background() + m := pollingMachine() + store := durable.NewMemStore() + const id = durable.InstanceID("poll-1") + owner := func(token int64) durable.Option[string, string, *runCtx] { + return durable.WithOwnerFence[string, string, *runCtx](token) + } + old, err := durable.NewRunner(m, store, owner(1)).Start(ctx, id, &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = old.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + newer, err := durable.Recover(ctx, m, store, id, owner(2)) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if _, err = newer.Fire(ctx, "poke"); err != nil { + t.Fatalf("newer Fire: %v", err) + } + _, err = old.Fire(ctx, "poke") + if !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, durable.ErrFenced) { + t.Fatalf("old Fire = %v, want ErrHandleFailed wrapping ErrFenced", err) + } +} + +// TestHandle_ContextEndedBeforeTheStep_ChangesNothing proves an ended context stops a +// Tick before the scheduler consumes a due timer. +func TestHandle_ContextEndedBeforeTheStep_ChangesNothing(t *testing.T) { + ctx := context.Background() + m := pollingMachine() + store := durable.NewMemStore() + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *runCtx]{durable.WithRunnerClock[string, string, *runCtx](clk)} + h, err := durable.NewRunner(m, store, opts...).Start(ctx, "poll-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + clk.Advance(time.Second) + ended, cancel := context.WithCancel(ctx) + cancel() + if _, err = h.Tick(ended); !errors.Is(err, context.Canceled) { + t.Fatalf("Tick = %v, want context.Canceled", err) + } + results, err := h.Tick(ctx) + if err != nil || len(results) != 1 { + t.Fatalf("Tick after = (%d results, %v), want the timer still due", len(results), err) + } +} diff --git a/durable/service.go b/durable/service.go index fedfe318..aacb91bc 100644 --- a/durable/service.go +++ b/durable/service.go @@ -84,10 +84,21 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // re-invoking the service. // // It returns the routed FireResult and true, or the zero result and false when id -// names no in-flight service (already settled, stopped, or never started). A -// persistence failure is returned as the error; the kernel transition itself does -// not fail (a settled service always routes onDone or onError). +// names no in-flight service (already settled, stopped, or never started), or false +// and the error when the Handle has failed or ctx has ended. A persistence failure is +// returned as the error; the kernel transition itself does not fail (a settled service +// always routes onDone or onError). After a failed Append the result wasn't recorded, so +// a recovered Handle has the service in flight again, and RunService runs it a second +// time. func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.FireResult[S], bool, error) { + h.mu.Lock() + defer h.mu.Unlock() + if err := h.refuse(); err != nil { + return state.FireResult[S]{}, false, err + } + if err := ctx.Err(); err != nil { + return state.FireResult[S]{}, false, err + } if h.svc == nil { return state.FireResult[S]{}, false, fmt.Errorf("durable: no service registry wired for %q (supply WithServiceRegistry)", h.id) } @@ -109,7 +120,7 @@ func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.Fire entry, err := recordServiceOutcome(id, h.svc.LastResult, h.svc.LastError()) if err != nil { - return res, true, fmt.Errorf("durable: recording service %q result for %q: %w", id, h.id, err) + return res, true, h.fail(fmt.Errorf("durable: recording service %q result for %q: %w", id, h.id, err)) } if h.svcBuf != nil { *h.svcBuf = append(*h.svcBuf, entry) @@ -120,7 +131,6 @@ func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.Fire if err := h.persistStep(ctx, step, &rec); err != nil { return res, true, err } - h.nextStep++ return res, true, nil } From 2f8b57c886d7f85c4937611d3c48deefe600e005 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 9 Nov 2026 11:04:01 -0700 Subject: [PATCH 2/5] test(durable): cover the failed-Handle latch on RunService and DeliverToActor RunService and DeliverToActor refuse steps after a failed Append and return an ended context's error, but only Fire and Tick were tested. New tests fail an Append under each and end the context before each, and check nothing ran or was journaled. The Tick context test now also checks that nothing was appended. The RunService and DeliverToActor docs said a failed step returns false; they return true with ErrHandleFailed once the service settled or the actor received the event. The docs now say so, name a context that ends during the Store write as a possible cause, and note that the latch relies on the Append running inline. Signed-off-by: Ray Karnes --- durable/actor.go | 14 +-- durable/errors.go | 2 +- durable/handle_failure_test.go | 164 +++++++++++++++++++++++++++++++++ durable/runner.go | 10 +- durable/runner_test.go | 6 +- durable/service.go | 10 +- 6 files changed, 190 insertions(+), 16 deletions(-) create mode 100644 durable/handle_failure_test.go diff --git a/durable/actor.go b/durable/actor.go index 41abf380..17bed76b 100644 --- a/durable/actor.go +++ b/durable/actor.go @@ -98,12 +98,14 @@ type actorMessagePayload struct { // JournalActorMessage correlated by the actor id so recovery replays it without // re-instantiating the actor. // -// It returns whether the actor was found running (false for a ref that names no -// live actor — already settled, stopped, or never spawned — or with an error when the -// Handle has failed or ctx has ended), and a persistence failure as the error. The -// kernel transitions the delivery drove do not themselves fail. After a failed Append -// the result wasn't recorded: on a recovered Handle an InvokeActor actor restarts from its -// initial state, and a Spawn actor doesn't return. Resending a recorded delivery repeats +// It returns false for a ref that names no live actor (already settled, stopped, or +// never spawned), and false with the error when the Handle had already failed or ctx +// has ended. If the actor received the event but the delivery couldn't be recorded or +// appended, it returns true and ErrHandleFailed: the Handle has failed. A failed +// checkpoint returns true and that error, and the Handle carries on. The kernel +// transitions the delivery drove do not themselves fail. When a delivery wasn't +// recorded, on a recovered Handle an InvokeActor actor restarts from its initial state, +// and a Spawn actor doesn't return. Resending a recorded delivery repeats // any message it sends the parent. func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef, event any) (bool, error) { h.mu.Lock() diff --git a/durable/errors.go b/durable/errors.go index 5ac19d7e..ea357b31 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -19,7 +19,7 @@ const ( errorHandleFailed = "handle is out of step with the journal" ) -// Sentinel errors a Store reports. Callers match them with errors.Is. +// Sentinel errors a Store or a Handle reports. Callers match them with errors.Is. var ( // ErrInstanceNotFound is reported by Store.Load for an instance that has // never been written. diff --git a/durable/handle_failure_test.go b/durable/handle_failure_test.go new file mode 100644 index 00000000..ed57686c --- /dev/null +++ b/durable/handle_failure_test.go @@ -0,0 +1,164 @@ +package durable_test + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// fetchID is the invocation id of the fetch service the single-service machine runs. +var fetchID = state.InvokeID("fetch", "loading", 0) + +// armedAppendStore counts Appends and, once armed, fails the next one. +type armedAppendStore struct { + durable.Store + armed bool + appends int +} + +func (s *armedAppendStore) Append(ctx context.Context, id durable.InstanceID, rec durable.Record, opts ...durable.AppendOption) (int64, error) { + if s.armed { + s.armed = false + return 0, errors.New("store unavailable") + } + s.appends++ + return s.Store.Append(ctx, id, rec, opts...) +} + +// startFetch starts the single-service machine over store and moves it into the +// state that invokes the service. +func startFetch(t *testing.T, store durable.Store, calls *int64) *durable.Handle[string, string, svcCtx] { + t.Helper() + ctx := context.Background() + fn := counterService(calls) + h, err := durable.NewRunner(fetchMachine(fn), store, + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))). + Start(ctx, "fetch-1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + return h +} + +// startTwin starts the two-actor machine over store, with childA running. +func startTwin(t *testing.T, store durable.Store, runs *int64) (*durable.Handle[string, string, actorCtx], state.ActorRef) { + t.Helper() + palette := map[string]state.ActorBehavior{"childA": counterChild(runs), "childB": counterChild(runs)} + h, err := durable.NewRunner(twoActorMachine(), store, + durable.WithActorPalette[string, string, actorCtx](palette)). + Start(context.Background(), "twin-1", actorCtx{}, state.WithInitialState("first")) + if err != nil { + t.Fatalf("Start: %v", err) + } + ref, ok := h.ActorRef(state.ActorID("twin", "first", 0)) + if !ok { + t.Fatal("no ref for childA") + } + return h, ref +} + +// TestHandle_RunService_FailedAppend_FailsHandle proves a failed Append of a service +// outcome fails the Handle: RunService reports the settle with ErrHandleFailed, every +// later step is refused, and a recovered Handle runs the service again. +func TestHandle_RunService_FailedAppend_FailsHandle(t *testing.T) { + ctx := context.Background() + var calls int64 + store := &armedAppendStore{Store: durable.NewMemStore()} + h := startFetch(t, store, &calls) + + store.armed = true + _, ok, err := h.RunService(ctx, fetchID) + if !ok || !errors.Is(err, durable.ErrHandleFailed) || !strings.Contains(err.Error(), "store unavailable") { + t.Fatalf("RunService = (%v, %v), want true and ErrHandleFailed naming the cause", ok, err) + } + if _, ok, err = h.RunService(ctx, fetchID); ok || !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("RunService after the failure = (%v, %v), want false and ErrHandleFailed", ok, err) + } + if _, err = h.Fire(ctx, "start"); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire after the failure = %v, want ErrHandleFailed", err) + } + + fn := counterService(&calls) + rh, err := durable.Recover(ctx, fetchMachine(fn), store, "fetch-1", + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got := rh.Instance().Current(); got != "loading" { + t.Fatalf("recovered state = %q, want loading (the outcome wasn't recorded)", got) + } + mustRunService(t, rh, fetchID) + if got := atomic.LoadInt64(&calls); got != 2 { + t.Fatalf("service ran %d times, want 2 (once live, once after recovery)", got) + } +} + +// TestHandle_DeliverToActor_FailedAppend_FailsHandle proves a failed Append of an +// actor delivery fails the Handle: DeliverToActor reports the delivery with +// ErrHandleFailed, and every later step is refused. +func TestHandle_DeliverToActor_FailedAppend_FailsHandle(t *testing.T) { + ctx := context.Background() + var runs int64 + store := &armedAppendStore{Store: durable.NewMemStore()} + h, ref := startTwin(t, store, &runs) + + store.armed = true + delivered, err := h.DeliverToActor(ctx, ref, "finish") + if !delivered || !errors.Is(err, durable.ErrHandleFailed) || !strings.Contains(err.Error(), "store unavailable") { + t.Fatalf("DeliverToActor = (%v, %v), want true and ErrHandleFailed naming the cause", delivered, err) + } + if delivered, err = h.DeliverToActor(ctx, ref, "finish"); delivered || !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("DeliverToActor after the failure = (%v, %v), want false and ErrHandleFailed", delivered, err) + } + if _, err = h.Tick(ctx); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Tick after the failure = %v, want ErrHandleFailed", err) + } +} + +// TestHandle_RunService_ContextEnded_ChangesNothing proves an ended context stops +// RunService before the service runs or anything is journaled. +func TestHandle_RunService_ContextEnded_ChangesNothing(t *testing.T) { + var calls int64 + store := &armedAppendStore{Store: durable.NewMemStore()} + h := startFetch(t, store, &calls) + before := store.appends + + ended, cancel := context.WithCancel(context.Background()) + cancel() + if _, ok, err := h.RunService(ended, fetchID); ok || !errors.Is(err, context.Canceled) { + t.Fatalf("RunService = (%v, %v), want false and context.Canceled", ok, err) + } + if got := atomic.LoadInt64(&calls); got != 0 || store.appends != before { + t.Fatalf("after an ended context: %d service runs and %d new appends, want 0 and 0", got, store.appends-before) + } + mustRunService(t, h, fetchID) +} + +// TestHandle_DeliverToActor_ContextEnded_ChangesNothing proves an ended context stops +// DeliverToActor before the actor sees the event or anything is journaled. +func TestHandle_DeliverToActor_ContextEnded_ChangesNothing(t *testing.T) { + var runs int64 + store := &armedAppendStore{Store: durable.NewMemStore()} + h, ref := startTwin(t, store, &runs) + before, state0 := store.appends, h.Instance().Current() + + ended, cancel := context.WithCancel(context.Background()) + cancel() + if delivered, err := h.DeliverToActor(ended, ref, "finish"); delivered || !errors.Is(err, context.Canceled) { + t.Fatalf("DeliverToActor = (%v, %v), want false and context.Canceled", delivered, err) + } + if store.appends != before || h.Instance().Current() != state0 { + t.Fatalf("after an ended context: %d new appends, state %q, want 0 and %q", store.appends-before, h.Instance().Current(), state0) + } + if delivered, err := h.DeliverToActor(context.Background(), ref, "finish"); !delivered || err != nil { + t.Fatalf("DeliverToActor after = (%v, %v), want the actor still running", delivered, err) + } +} diff --git a/durable/runner.go b/durable/runner.go index 442fec6f..9fe3f0ce 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -91,7 +91,8 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // // A step that applied to the kernel but may not have been recorded leaves the Handle out // of step with the journal. That call and every later one return ErrHandleFailed, -// wrapping the cause. Recover a new Handle with the same options to continue. A service +// wrapping the cause, which can be a context that ended during the Store write. Recover +// a new Handle with the same options to continue. A service // whose result wasn't recorded is in flight again, so RunService runs it a second time. // A running InvokeActor actor restarts from its initial state on any Recover, losing every // delivery so far, and a Spawn actor doesn't return. If the cause is ErrFenced, a @@ -396,7 +397,7 @@ func (r *Runner[S, E, C]) Fire(ctx context.Context, id InstanceID, event E, opts // it Fires the kernel, appends a Record carrying the driving event at the // step's ordinal (write-ahead, before returning), and — when the // checkpoint policy is due — persists a full Snapshot and compacts the tail. A -// kernel transition error is recorded as a no-op (no step was produced) and +// kernel transition error journals nothing (no step was produced) and is // returned to the caller. // // A state.WithEventData payload reaches the transition's Assign in its JSON-decoded @@ -517,7 +518,10 @@ func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], erro // persistStep write-ahead appends rec, advances the next step, then checkpoints if // the policy is due. A failed append fails the Handle, because the kernel already -// applied the step; a failed checkpoint doesn't, because the Record is durable. +// applied the step; a failed checkpoint doesn't, because the Record is durable. This +// relies on the Append running inline, under the Handle's lock, before the step +// returns: an Append deferred off this goroutine would acknowledge a step whose +// outcome isn't known, and the Handle couldn't fail. func (h *Handle[S, E, C]) persistStep(ctx context.Context, step int, rec *Record) error { if err := h.appendStep(ctx, step, rec); err != nil { return h.fail(err) diff --git a/durable/runner_test.go b/durable/runner_test.go index bc5b3f7b..5318f305 100644 --- a/durable/runner_test.go +++ b/durable/runner_test.go @@ -948,7 +948,7 @@ func TestHandle_FencedOwner_Stops(t *testing.T) { func TestHandle_ContextEndedBeforeTheStep_ChangesNothing(t *testing.T) { ctx := context.Background() m := pollingMachine() - store := durable.NewMemStore() + store := &armedAppendStore{Store: durable.NewMemStore()} clk := state.NewFakeClock(epoch) opts := []durable.Option[string, string, *runCtx]{durable.WithRunnerClock[string, string, *runCtx](clk)} h, err := durable.NewRunner(m, store, opts...).Start(ctx, "poll-1", &runCtx{}, state.WithInitialState("idle")) @@ -959,11 +959,15 @@ func TestHandle_ContextEndedBeforeTheStep_ChangesNothing(t *testing.T) { t.Fatalf("Fire(start): %v", err) } clk.Advance(time.Second) + before := store.appends ended, cancel := context.WithCancel(ctx) cancel() if _, err = h.Tick(ended); !errors.Is(err, context.Canceled) { t.Fatalf("Tick = %v, want context.Canceled", err) } + if store.appends != before { + t.Fatalf("Tick on an ended context appended %d Records, want 0", store.appends-before) + } results, err := h.Tick(ctx) if err != nil || len(results) != 1 { t.Fatalf("Tick after = (%d results, %v), want the timer still due", len(results), err) diff --git a/durable/service.go b/durable/service.go index aacb91bc..e0927377 100644 --- a/durable/service.go +++ b/durable/service.go @@ -85,11 +85,11 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // // It returns the routed FireResult and true, or the zero result and false when id // names no in-flight service (already settled, stopped, or never started), or false -// and the error when the Handle has failed or ctx has ended. A persistence failure is -// returned as the error; the kernel transition itself does not fail (a settled service -// always routes onDone or onError). After a failed Append the result wasn't recorded, so -// a recovered Handle has the service in flight again, and RunService runs it a second -// time. +// and the error when the Handle had already failed or ctx has ended. If the service +// settled but its outcome couldn't be recorded or appended, it returns the result, +// true and ErrHandleFailed: the Handle has failed. A failed checkpoint returns the +// result, true and that error, and the Handle carries on. A recovered Handle has an +// unrecorded service in flight again, and RunService runs it a second time. func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.FireResult[S], bool, error) { h.mu.Lock() defer h.mu.Unlock() From 0dee0b571e38bda4f07ba929a0f098c0b3f010be Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Thu, 5 Nov 2026 09:18:22 -0700 Subject: [PATCH 3/5] fix(durable): don't journal a step whose context ended partway A context ended during RunService, Tick or DeliverToActor stops the kernel's Fire, but the result was journaled anyway. Recovery then committed something the live instance never applied, or for an actor, failed on every attempt. Each now returns ErrHandleFailed and records nothing. Recover and StateAt fail on a canceled context instead of returning state short of the journal. Relates to #214. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 7 +- durable/CHANGELOG.md | 7 + durable/README.md | 7 +- durable/actor.go | 40 +++-- durable/actor_test.go | 42 +++++ durable/doc.go | 23 +-- durable/errors.go | 3 +- durable/runner.go | 44 +++-- durable/runner_options.go | 18 +- durable/service.go | 41 +++-- durable/service_test.go | 201 ++++++++++++++++++++++ durable/timetravel.go | 12 +- 12 files changed, 371 insertions(+), 74 deletions(-) diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index 4680e2a8..cec71a8f 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -72,8 +72,8 @@ the first time, and replayed verbatim on recovery: | Seam | Wire with | Recorded as | | --- | --- | --- | | **Clock** (timers) | `WithRunnerClock` | `JournalClockRead`, replayed so timers fire at the same recorded instants, wall-clock-independent; armed deadlines survive checkpoint compaction | -| **Invoked services** | `WithServiceRegistry` + `Handle.RunService` | `JournalServiceResult`; the service runs once, then recovery replays its result through the kernel's settle seam | -| **Child-machine actors** | `WithActorPalette` + `Handle.DeliverToActor` | `JournalActorMessage`; the behavior runs once, then recovery re-fires the recorded parent transition | +| **Invoked services** | `WithServiceRegistry` + `Handle.RunService` | `JournalServiceResult`; a recorded result replays through the kernel's settle seam without re-running the service, and an unrecorded one (a context canceled during the service, a failed `Append`, a crash) runs again when the host calls `RunService` | +| **Child-machine actors** | `WithActorPalette` + `Handle.DeliverToActor` | `JournalActorMessage`; recovery re-fires a recorded parent transition; a settled actor doesn't re-run, and a running one restarts from its initial state on any `Recover` | | **Domain effects** | `WithEffectHandler` | dispatch set, applied exactly once via deterministic `EffectID` dedup | Use `WithCheckpointEvery(n)` to tune how often a full snapshot is written: a @@ -131,7 +131,8 @@ in by implementing `HistoryStore` (the in-tree `MemStore` does so under goroutines: each holds the Handle's lock for the whole step. A host callback must not call back into its Handle. `Instance()` is outside the lock. -A step that may not have been recorded fails the Handle with `ErrHandleFailed`. `Recover` +A step that may not have been recorded, or a `Tick`, `RunService` or `DeliverToActor` whose +context ended partway, fails the Handle with `ErrHandleFailed`. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the instance, so stop. diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 5067e0d0..6d9a73df 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -75,6 +75,13 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **A failed checkpoint no longer makes the next step conflict.** `Tick`, `RunService` and `DeliverToActor` now advance the next step as soon as the append lands. +- **A step whose context ended partway isn't journaled.** A canceled `RunService`, `Tick` + or `DeliverToActor` used to record a result the live instance never applied, which + recovery then committed, or for an actor, failed on. Each now returns `ErrHandleFailed` + wrapping the error and records nothing. After `Recover` the service stays in flight and + must tolerate running again, the timer is re-armed, and the actor restarts. `Recover` + and `StateAt` now fail on a canceled context, rather than returning state short of the + journal. - **The next step survives a checkpoint that compacts tick barriers.** A checkpoint now records the step that follows it, in a `nextStep` field of its envelope. Recovery used the Trace count, which runs low past a tick barrier, so the next Fire could reuse a diff --git a/durable/README.md b/durable/README.md index f720c52f..156b62a1 100644 --- a/durable/README.md +++ b/durable/README.md @@ -82,8 +82,8 @@ the first time, and replayed verbatim on recovery: | Seam | Wire with | Recorded as | | --- | --- | --- | | **Clock** (timers) | `WithRunnerClock` | `JournalClockRead`, replayed so timers fire at the same recorded instants, wall-clock-independent; armed deadlines survive checkpoint compaction | -| **Invoked services** | `WithServiceRegistry` + `Handle.RunService` | `JournalServiceResult`; the service runs once, then recovery replays its result through the kernel's settle seam | -| **Child-machine actors** | `WithActorPalette` + `Handle.DeliverToActor` | `JournalActorMessage`; the behavior runs once, then recovery re-fires the recorded parent transition | +| **Invoked services** | `WithServiceRegistry` + `Handle.RunService` | `JournalServiceResult`; a recorded result replays through the kernel's settle seam without re-running the service, and an unrecorded one (a context canceled during the service, a failed `Append`, a crash) runs again when the host calls `RunService` | +| **Child-machine actors** | `WithActorPalette` + `Handle.DeliverToActor` | `JournalActorMessage`; recovery re-fires a recorded parent transition; a settled actor doesn't re-run, and a running one restarts from its initial state on any `Recover` | | **Domain effects** | `WithEffectHandler` | dispatch set, applied exactly once via deterministic `EffectID` dedup | Use `WithCheckpointEvery(n)` to tune how often a full snapshot is written (a @@ -142,7 +142,8 @@ otherwise. goroutines: each holds the Handle's lock for the whole step. A host callback must not call back into its Handle. `Instance()` is outside the lock. -A step that may not have been recorded fails the Handle with `ErrHandleFailed`. `Recover` +A step that may not have been recorded, or a `Tick`, `RunService` or `DeliverToActor` whose +context ended partway, fails the Handle with `ErrHandleFailed`. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the instance, so stop. diff --git a/durable/actor.go b/durable/actor.go index 17bed76b..716c4dd5 100644 --- a/durable/actor.go +++ b/durable/actor.go @@ -17,15 +17,14 @@ import ( // NOT a pure function of (configuration, context, event payload, machine // definition) and MUST be recorded to reproduce a run. // -// # Run exactly once, replay the result +// # Run once per recorded result, replay the result // -// A durable actor's behavior runs exactly once, on the live path, and the result -// it routes into the parent is recorded; on recovery the recorded result is -// replayed back through the SAME parent transition the live run drove, and the -// actor behavior is NEVER re-instantiated. This is the standard durable-execution -// guarantee: the child machine (with its side effects and its nondeterministic -// output) runs once, and recovery is a pure replay against the parent kernel rather -// than a re-execution of the actor tree. +// A durable actor's behavior runs on the live path and the result it routes into the +// parent is recorded; on recovery the recorded result is replayed back through the SAME +// parent transition the live run drove, so a settled actor never re-runs. This is the +// standard durable-execution guarantee: recovery is a pure replay against the parent +// kernel rather than a re-execution of the actor tree. An actor still running at recovery +// restarts from its initial state, because its own state isn't journaled. // // The Runner wraps the kernel's reusable host driver, state.ActorSystem, which // turns SpawnActor / StopActor effects into running child actors, owns each actor's @@ -90,23 +89,21 @@ type actorMessagePayload struct { HasData bool `json:"hasData,omitempty"` } -// DeliverToActor routes event into the running actor identified by ref exactly -// once, runs the actor (settling it when it reaches its final state), records each +// DeliverToActor routes event into the running actor identified by ref, runs the actor (settling it when it reaches its final state), records each // parent transition the delivery drove, and persists the produced step. The actor // behavior is resolved and run against the palette supplied with WithActorPalette; // the done-data, error, or parent-driving message it routes is journaled as a // JournalActorMessage correlated by the actor id so recovery replays it without // re-instantiating the actor. // -// It returns false for a ref that names no live actor (already settled, stopped, or -// never spawned), and false with the error when the Handle had already failed or ctx -// has ended. If the actor received the event but the delivery couldn't be recorded or -// appended, it returns true and ErrHandleFailed: the Handle has failed. A failed -// checkpoint returns true and that error, and the Handle carries on. The kernel -// transitions the delivery drove do not themselves fail. When a delivery wasn't -// recorded, on a recovered Handle an InvokeActor actor restarts from its initial state, -// and a Spawn actor doesn't return. Resending a recorded delivery repeats -// any message it sends the parent. +// It returns whether the actor was found running: false for a ref that names no live +// actor (already settled, stopped, or never spawned), or false and an error when no +// palette is wired, the Handle has failed, or ctx ended before the delivery. Once the +// actor ran it returns true with any error. A failed Append, a result that won't encode, +// or a ctx that ended during the delivery returns ErrHandleFailed and records nothing: on +// a recovered Handle an InvokeActor actor restarts from its initial state, and a Spawn +// actor doesn't return. A failed Checkpoint returns the plain error, and the delivery is +// recorded. Resending a recorded delivery repeats any message it sends the parent. func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef, event any) (bool, error) { h.mu.Lock() defer h.mu.Unlock() @@ -125,6 +122,11 @@ func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef if !delivered { return false, nil } + // A context that ended during the delivery can stop a parent Fire partway, and a + // refused Fire leaves a Trace replay can't re-fire. Record nothing. + if err := ctx.Err(); err != nil { + return true, h.fail(fmt.Errorf("durable: delivering to actor %q for %q: %w", ref.ID, h.id, err)) + } output, settled, settleErr := h.actorOutcome() entries, err := recordActorTransitions(ref.ID, h.inst.History(), before, output, settled, settleErr) diff --git a/durable/actor_test.go b/durable/actor_test.go index f8af0cd5..ef3cd6f5 100644 --- a/durable/actor_test.go +++ b/durable/actor_test.go @@ -2,6 +2,7 @@ package durable_test import ( "context" + "errors" "fmt" "sync/atomic" "testing" @@ -704,3 +705,44 @@ func TestActor_Determinism(t *testing.T) { t.Fatalf("two live runs diverged:\n run1=%s\n run2=%s", s1, s2) } } + +// TestDeliverToActor_ContextEndedPartway_RecordsNothing proves a delivery whose context +// ended records nothing, so every later Recover still works. +func TestDeliverToActor_ContextEndedPartway_RecordsNothing(t *testing.T) { + ctx := context.Background() + m := supervisorMachine() + store := durable.NewMemStore() + id := durable.InstanceID("sup-cancel") + turn, cancel := context.WithCancel(ctx) + defer cancel() + behavior := func(map[string]any) (state.ActorInstance, error) { + child := state.ForgeFor[actorCtx]("child"). + State("working"). + State("done").Final(). + Initial("working"). + Transition("working").On("finish").GoTo("done"). + Quench() + inst := child.Cast(actorCtx{}, state.WithInitialState[string]("working")) + return state.NewActor(inst, func(*state.Instance[string, string, actorCtx]) any { + cancel() // the turn ends after the child finishes, before the parent settles + return "late" + }), nil + } + opts := []durable.Option[string, string, actorCtx]{ + durable.WithActorPalette[string, string, actorCtx](map[string]state.ActorBehavior{"spawnChild": behavior}), + } + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, actorCtx{}, state.WithInitialState("supervising")) + if err != nil { + t.Fatalf("Start: %v", err) + } + ref, ok := h.ActorRef(state.ActorID("supervisor", "supervising", 0)) + if !ok { + t.Fatal("no actor ref for spawned child") + } + if _, err = h.DeliverToActor(turn, ref, "finish"); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("DeliverToActor = %v, want ErrHandleFailed", err) + } + if _, err = durable.Recover(ctx, m, store, id, opts...); err != nil { + t.Fatalf("Recover: %v", err) + } +} diff --git a/durable/doc.go b/durable/doc.go index 00e5a8fa..93355b2f 100644 --- a/durable/doc.go +++ b/durable/doc.go @@ -99,17 +99,18 @@ // was compacted out of the tail is re-armed from its persisted deadline rather // than lost. // -// Invoked services (WithServiceRegistry): a service (`invoke`) runs exactly once -// on the live path; its result is journaled as a JournalServiceResult. On -// recovery the recorded result is replayed back through the same kernel settle -// seam — the service is never re-invoked and the same onDone or onError event -// re-fires with the same data. -// -// Child-machine actors (WithActorPalette): an actor's behavior runs exactly once -// on the live path; each parent transition the delivery drives is journaled as a -// JournalActorMessage. On recovery the recorded transition is re-fired directly -// through the parent with the recorded done-data — the actor behavior is never -// re-instantiated. +// Invoked services (WithServiceRegistry): a service (`invoke`) runs on the live path; +// its result is journaled as a JournalServiceResult. On recovery the recorded result is +// replayed back through the same kernel settle seam — that service is never re-invoked +// and the same onDone or onError event re-fires with the same data. A result that +// wasn't recorded has nothing to replay, so the service runs again when the host calls +// RunService. +// +// Child-machine actors (WithActorPalette): an actor's behavior runs on the live path; +// each parent transition the delivery drives is journaled as a JournalActorMessage. On +// recovery the recorded transition is re-fired directly through the parent with the +// recorded done-data, so a settled actor never re-runs. An actor still running at +// recovery restarts from its initial state, because its own state isn't journaled. // // # Idempotent effect dispatch // diff --git a/durable/errors.go b/durable/errors.go index ea357b31..928739f7 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -55,7 +55,8 @@ var ( // ErrHandleFailed is reported by a Handle's Fire, Tick, RunService and // DeliverToActor once a step applied to the live instance but may not have been - // recorded, on that call and every later one. It wraps the cause. Recover a new + // recorded, or a Tick, RunService or DeliverToActor's context ended partway, on that + // call and every later one. It wraps the cause. Recover a new // Handle with the same options to continue, unless the cause is ErrFenced: then a // newer owner holds the instance, so stop. ErrHandleFailed = errors.New(errorPrefix + errorHandleFailed) diff --git a/durable/runner.go b/durable/runner.go index 9fe3f0ce..73ea9af3 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -39,15 +39,16 @@ import ( // deadline or testing dueness — is journaled and returned verbatim on recovery, // making timer-driven transitions durable and replay wall-clock-independent. // -// Invoked services are the second recorded seam: a service runs exactly once on -// the live path (Handle.RunService, against the registry supplied with -// WithServiceRegistry) and its result is journaled as a JournalServiceResult; on -// recovery the recorded result is replayed back through the kernel's settle seam, -// so the service is never re-invoked and the same onDone / onError event re-fires -// with the same data. +// Invoked services are the second recorded seam: a service runs on the live path +// (Handle.RunService, against the registry supplied with WithServiceRegistry) and its +// result is journaled as a JournalServiceResult; on recovery the recorded result is +// replayed back through the kernel's settle seam, so that service is never re-invoked +// and the same onDone / onError event re-fires with the same data. A result that wasn't +// recorded has nothing to replay, so the service runs again when the host calls +// RunService. // // Child-machine actors are the third recorded seam: an actor's behavior runs -// exactly once on the live path (Handle.DeliverToActor, against the palette +// on the live path (Handle.DeliverToActor, against the palette // supplied with WithActorPalette) and each parent transition the delivery drives — // the actor's onDone / onError, or a message it sends — is journaled as a // JournalActorMessage; on recovery the recorded transition is replayed by re-firing @@ -89,10 +90,11 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // live instance outside that lock, so read it only while no step is running. A // stateless Runner.Fire builds its own Handle, so it isn't serialized with this one. // -// A step that applied to the kernel but may not have been recorded leaves the Handle out -// of step with the journal. That call and every later one return ErrHandleFailed, -// wrapping the cause, which can be a context that ended during the Store write. Recover -// a new Handle with the same options to continue. A service +// A step that applied to the kernel but may not have been recorded, or a Tick, RunService +// or DeliverToActor whose context ended partway, leaves the Handle out of step with the +// journal. That call and every later one return ErrHandleFailed, wrapping the cause, +// which can be a context that ended during the Store write. Recover a new Handle with +// the same options to continue. A service // whose result wasn't recorded is in flight again, so RunService runs it a second time. // A running InvokeActor actor restarts from its initial state on any Recover, losing every // delivery so far, and a Spawn actor doesn't return. If the cause is ErrFenced, a @@ -486,6 +488,16 @@ func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], erro return nil, err } results := h.sched.Tick(ctx) + // A context that ended partway makes the kernel refuse a due timer's Fire after the + // scheduler consumed it. Journaling that would commit on recovery what live never + // applied, so record nothing: Recover re-arms the timer. + if err := ctx.Err(); err != nil { + for i := range results { + if results[i].Err != nil { + return results, h.fail(fmt.Errorf("durable: ticking %q: %w", h.id, err)) + } + } + } step := h.nextStep rec := Record{Step: step, Tick: true, TickSteps: len(results), Entries: h.drainClock()} @@ -581,7 +593,7 @@ func (h *Handle[S, E, C]) checkpointStep(ctx context.Context, step int, rec *Rec // snapshot (firing nothing, no IO), and replays the tail's recorded driving // events through the kernel to reach the instance's live state. The returned // Handle continues recording subsequent Fires. Recover reports ErrInstanceNotFound -// for an instance that was never started. +// for an instance that was never started, and the context's error if ctx ends. func Recover[S comparable, E comparable, C any](ctx context.Context, m *state.Machine[S, E, C], st Store, id InstanceID, opts ...Option[S, E, C]) (*Handle[S, E, C], error) { r := NewRunner(m, st, opts...) return r.recover(ctx, id) @@ -652,6 +664,11 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S armed := map[string]pendingTimer{} for i := range tail { + // A canceled context can stop a replayed settle or tick with no error, leaving + // state short of the journal, so check it before each Record. + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("durable: recovering %q: %w", id, err) + } rec := &tail[i] switch { case hasActorEntry(rec): @@ -790,6 +807,9 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S if n := len(tail); n > 0 { next = tail[n-1].followingStep() } + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("durable: recovering %q: %w", id, err) + } return &Handle[S, E, C]{ runner: r, id: id, diff --git a/durable/runner_options.go b/durable/runner_options.go index a84886e4..624051c5 100644 --- a/durable/runner_options.go +++ b/durable/runner_options.go @@ -91,10 +91,10 @@ func WithRunnerClock[S comparable, E comparable, C any](c state.Clock) Option[S, } // WithServiceRegistry binds the invoked-service implementations the Runner runs on -// the live path and resolves on recovery. A durable service runs exactly once — -// live — and its result is recorded; on recovery the recorded result is replayed -// through the same settle seam and the service is never re-invoked, so the registry -// is consulted only to execute a service the first time. Supply it for any machine +// the live path and resolves on recovery. A durable service runs live and its result is +// recorded; on recovery the recorded result is replayed through the same settle seam and +// that service is never re-invoked, so the registry is consulted only to execute a +// service whose result isn't recorded. Supply it for any machine // that declares `invoke` states; a purely event-driven or timer-driven machine // needs none. The registry binds the same ServiceFns the machine declares, by name // (state.NewRegistry().Service(name, fn)). @@ -108,11 +108,11 @@ func WithServiceRegistry[S comparable, E comparable, C any](reg *state.Registry[ // WithActorPalette binds the child-machine actor behaviors the Runner spawns and // runs on the live path and resolves on recovery, keyed by the actor src name (the -// name passed to InvokeActor or the Spawn built-in). A durable actor's behavior -// runs exactly once — live — and its done-data, error, or parent-driving message is -// recorded; on recovery the recorded result is replayed back through the same parent -// transition and the behavior is never re-instantiated, so the palette is consulted -// only to run an actor the first time. Supply it for any machine that spawns child +// name passed to InvokeActor or the Spawn built-in). A durable actor's behavior runs +// live and its done-data, error, or parent-driving message is recorded; on recovery the +// recorded result is replayed back through the same parent transition and that behavior +// is never re-instantiated, so the palette is consulted only to run an actor whose +// result isn't recorded. Supply it for any machine that spawns child // actors; a machine that spawns none needs none. Each behavior is the actor-model // analog of a ServiceFn, registered by the same src name the machine declares. func WithActorPalette[S comparable, E comparable, C any](palette map[string]state.ActorBehavior) Option[S, E, C] { diff --git a/durable/service.go b/durable/service.go index e0927377..d61b1f73 100644 --- a/durable/service.go +++ b/durable/service.go @@ -16,14 +16,15 @@ import ( // (configuration, context, event payload, machine definition) and MUST be recorded // to reproduce a run. // -// # Run exactly once, replay the result +// # Run once per recorded result, replay the result // -// A durable service runs exactly once, on the live path, and its result is -// recorded; on recovery the recorded result is replayed back through the same -// settle seam, and the service is NEVER re-invoked. This is the standard -// durable-execution guarantee: the external call (with its side effects and its -// nondeterministic output) happens once, and recovery is a pure replay against the -// kernel rather than a re-execution of the host's services. +// A durable service runs on the live path and its result is recorded; on recovery the +// recorded result is replayed back through the same settle seam, and that service is +// NEVER re-invoked. This is the standard durable-execution guarantee: recovery is a pure +// replay against the kernel rather than a re-execution of the host's services. A result +// that wasn't recorded, after a context canceled during the service, a failed Append or a +// crash, has nothing to replay, so it stays in flight, runs again when the host calls +// RunService, and must tolerate a second run. // // The Runner wraps the kernel's reusable host driver, state.ServiceRunner, which // turns StartService / StopService effects into real ServiceFn executions and @@ -75,7 +76,7 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { return out } -// RunService runs the in-flight invoked service identified by id exactly once, +// RunService runs the in-flight invoked service identified by id, // records its outcome, and settles it — firing the invocation's onDone (on // success) or onError (on failure) event through the durable instance and // recording the produced step. The service is resolved and executed against the @@ -84,12 +85,16 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // re-invoking the service. // // It returns the routed FireResult and true, or the zero result and false when id -// names no in-flight service (already settled, stopped, or never started), or false -// and the error when the Handle had already failed or ctx has ended. If the service -// settled but its outcome couldn't be recorded or appended, it returns the result, -// true and ErrHandleFailed: the Handle has failed. A failed checkpoint returns the -// result, true and that error, and the Handle carries on. A recovered Handle has an -// unrecorded service in flight again, and RunService runs it a second time. +// names no in-flight service (already settled, stopped, or never started). It returns +// false and an error when no registry is wired, the Handle has failed, or ctx ended +// before the service started. +// +// Once the service has run, it returns true with any error. A failed Append, a result +// that won't encode, or a ctx that ended during the service returns ErrHandleFailed and +// records nothing: a recovered Handle has the service in flight again, and RunService +// runs it a second time. That error also matches the cause, so test ErrHandleFailed +// first: it alone means Recover is needed. A failed Checkpoint returns the plain error, +// and the result is recorded. func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.FireResult[S], bool, error) { h.mu.Lock() defer h.mu.Unlock() @@ -112,6 +117,14 @@ func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.Fire } res := results[0] + // A context canceled during the service stops the settle's Fire before it applies, + // so the instance never saw this result: journaling it would commit it on recovery. + // The runner has already dropped the service, so this Handle can't run it again; + // Recover re-arms it. + if res.Err != nil && ctx.Err() != nil { + return res, true, h.fail(fmt.Errorf("durable: settling service %q for %q: %w", id, h.id, res.Err)) + } + // The settle re-fired the invocation's onDone / onError, which may enter a state // that arms a timer, starts another service, or spawns an actor. Absorb those // effects into every driver so the next seam method finds them armed — the service diff --git a/durable/service_test.go b/durable/service_test.go index df1d4db8..bf4fb5f0 100644 --- a/durable/service_test.go +++ b/durable/service_test.go @@ -6,6 +6,7 @@ import ( "fmt" "sync/atomic" "testing" + "time" "github.com/stablekernel/crucible/durable" "github.com/stablekernel/crucible/state" @@ -470,3 +471,203 @@ func TestService_Determinism(t *testing.T) { t.Fatalf("two live runs diverged:\n run1=%s\n run2=%s", s1, s2) } } + +// TestService_ResultFromCanceledTurn_NotCommittedOnRecover proves a service result +// whose settle is canceled with the turn's context, and so never reaches the live +// instance, is not committed by recovery either. +func TestService_ResultFromCanceledTurn_NotCommittedOnRecover(t *testing.T) { + ctx := context.Background() + turnCtx, cancel := context.WithCancel(ctx) + defer cancel() + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { + cancel() // the turn's deadline passes while the call is in flight + return "late", nil + } + m := fetchMachine(fn) + store := durable.NewMemStore() + id := durable.InstanceID("fetch-1") + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))} + runner := durable.NewRunner(m, store, opts...) + h, err := runner.Start(ctx, id, svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + _, _, _ = h.RunService(turnCtx, state.InvokeID("fetch", "loading", 0)) + + recovered, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + // Compare state, not bytes: live keeps the Trace of the refused settle. + if live, got := h.Instance().Current(), recovered.Instance().Current(); got != live || live != "loading" { + t.Fatalf("recovered %q, live %q, want both loading: the canceled turn's result was committed", got, live) + } + if notes := recovered.Instance().Entity().Notes; len(notes) != 0 { + t.Fatalf("recovered notes %v, want none", notes) + } +} + +// TestService_CanceledTurn_RecoverRerunsTheService proves the canceled result isn't +// journaled, the Handle refuses further steps, and a recovered Handle can run the +// service to completion. +func TestService_CanceledTurn_RecoverRerunsTheService(t *testing.T) { + ctx := context.Background() + turnCtx, cancel := context.WithCancel(ctx) + defer cancel() + calls := 0 + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { + calls++ + cancel() + return "late", nil + } + m := fetchMachine(fn) + store := durable.NewMemStore() + id := durable.InstanceID("fetch-1") + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))} + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + invoke := state.InvokeID("fetch", "loading", 0) + if _, _, err = h.RunService(turnCtx, invoke); !errors.Is(err, context.Canceled) { + t.Fatalf("RunService = %v, want context.Canceled", err) + } + if _, err = h.Fire(ctx, "start"); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire after the cancel = %v, want ErrHandleFailed", err) + } + recovered, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if _, ok, err := recovered.RunService(ctx, invoke); err != nil || !ok { + t.Fatalf("RunService after Recover = (%v, %v)", ok, err) + } + if calls != 2 { + t.Fatalf("service ran %d times, want 2", calls) + } +} + +// TestRecover_CanceledContext_DoesNotLoseARecordedResult proves a canceled Recover +// reports the cancellation rather than a Handle short of the journal. +func TestRecover_CanceledContext_DoesNotLoseARecordedResult(t *testing.T) { + ctx := context.Background() + var calls int64 + fn := counterService(&calls) + m := fetchMachine(fn) + store := durable.NewMemStore() + id := durable.InstanceID("fetch-1") + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))} + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("loading")) + if err != nil { + t.Fatalf("Start: %v", err) + } + mustRunService(t, h, state.InvokeID("fetch", "loading", 0)) + canceled, cancel := context.WithCancel(ctx) + cancel() + if _, err = durable.Recover(canceled, m, store, id, opts...); !errors.Is(err, context.Canceled) { + t.Fatalf("Recover = %v, want context.Canceled", err) + } +} + +// TestStateAt_CanceledContext_ReportsIt proves a canceled historical read reports the +// cancellation, not a view short of the journal. +func TestStateAt_CanceledContext_ReportsIt(t *testing.T) { + ctx := context.Background() + var calls int64 + fn := counterService(&calls) + m := fetchMachine(fn) + store := durable.NewMemStore(durable.WithHistory()) + id := durable.InstanceID("fetch-1") + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))} + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("loading")) + if err != nil { + t.Fatalf("Start: %v", err) + } + mustRunService(t, h, state.InvokeID("fetch", "loading", 0)) + canceled, cancel := context.WithCancel(ctx) + cancel() + if _, err = durable.StateAt(canceled, m, store, id, 0, opts...); !errors.Is(err, context.Canceled) { + t.Fatalf("StateAt = %v, want context.Canceled", err) + } +} + +// TestRecover_CanceledContext_ChainedServices_ReportsIt proves a canceled replay of two +// chained services reports the cancellation, not a journal that looks corrupt. +func TestRecover_CanceledContext_ChainedServices_ReportsIt(t *testing.T) { + ctx := context.Background() + var calls int64 + fn := counterService(&calls) + m := chainMachine(fn, fn) + store := durable.NewMemStore() + id := durable.InstanceID("chain-1") + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](chainRegistry(fn, fn))} + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("first")) + if err != nil { + t.Fatalf("Start: %v", err) + } + mustRunService(t, h, state.InvokeID("chain", "first", 0)) + mustRunService(t, h, state.InvokeID("chain", "second", 0)) + canceled, cancel := context.WithCancel(ctx) + cancel() + if _, err = durable.Recover(canceled, m, store, id, opts...); !errors.Is(err, context.Canceled) { + t.Fatalf("Recover = %v, want context.Canceled", err) + } +} + +// TestTick_ContextEndedPartway_RecordsNothing proves a due timer the kernel refused isn't +// journaled: the Handle fails and the journal doesn't grow. +func TestTick_ContextEndedPartway_RecordsNothing(t *testing.T) { + ctx := context.Background() + m := timerMachine() + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + turn, cancel := context.WithCancel(ctx) + defer cancel() + clk := &cancelingClock{Clock: state.NewFakeClock(epoch)} + opts := []durable.Option[string, string, *timerCtx]{durable.WithRunnerClock[string, string, *timerCtx](clk)} + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, &timerCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "arm"); err != nil { + t.Fatalf("Fire(arm): %v", err) + } + _, before, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + clk.Clock.(*state.FakeClock).Advance(6 * time.Second) + clk.onNow = cancel // the turn ends as the Tick reads the clock + if _, err = h.Tick(turn); !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, context.Canceled) { + t.Fatalf("Tick = %v, want ErrHandleFailed wrapping context.Canceled", err) + } + clk.onNow = nil + _, after, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(after) != len(before) { + t.Fatalf("journal grew from %d to %d records", len(before), len(after)) + } +} + +// cancelingClock runs onNow once on the next clock read. +type cancelingClock struct { + state.Clock + onNow func() +} + +func (c *cancelingClock) Now() time.Time { + if c.onNow != nil { + f := c.onNow + c.onNow = nil + f() + } + return c.Clock.Now() +} diff --git a/durable/timetravel.go b/durable/timetravel.go index 4d5be559..654a28e7 100644 --- a/durable/timetravel.go +++ b/durable/timetravel.go @@ -101,7 +101,8 @@ func (v *TimeTravelView[S, E, C]) Step() int { return v.step } // historical read applies no side effect. // // step must be in [BaselineStep, lastRecordedStep]; a step outside that range -// reports ErrStepOutOfRange, and an unknown instance reports ErrInstanceNotFound. +// reports ErrStepOutOfRange, an unknown instance reports ErrInstanceNotFound, and an +// ended ctx reports its error. func StateAt[S comparable, E comparable, C any]( ctx context.Context, m *state.Machine[S, E, C], @@ -241,6 +242,11 @@ func replayThrough[S comparable, E comparable, C any]( } for i := range bounded { + // A canceled context can stop a replayed settle or tick with no error, leaving + // state short of the journal, so check it before each Record. + if err := ctx.Err(); err != nil { + return err + } rec := &bounded[i] switch { case hasActorEntry(rec): @@ -282,7 +288,9 @@ func replayThrough[S comparable, E comparable, C any]( // applies no side effect. } } - return nil + // A canceled context can stop a replayed settle or tick without an error reaching + // here, which would hand back a view short of the journal. + return ctx.Err() } // boundRecords returns the prefix of records whose first ordinal falls at or below From 76f85127afb3091dc9a315aa4102edc941a22c88 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Thu, 5 Nov 2026 09:20:12 -0700 Subject: [PATCH 4/5] fix(durable): dispatch effects from Start, Tick and RunService Only Fire dispatched domain effects. Every step path now commits through one helper that appends, dispatches, then checkpoints. A failed dispatch fails the Handle before its checkpoint, so a later checkpoint can't compact the effect away, and Recover retries it. Start applies the initial entry's effects before the baseline, so a handler failure leaves no instance. EffectInstance gives a handler its instance id, and state.ServiceRunner gains WithResultTransform so a service's result reaches the machine in the form a replay supplies. Effects from a parent transition an actor drives are still not dispatched. Relates to #214. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 28 ++-- durable/CHANGELOG.md | 24 +++ durable/README.md | 28 ++-- durable/actor.go | 6 +- durable/doc.go | 9 +- durable/effect.go | 64 ++++++-- durable/effect_test.go | 179 ++++++++++++++++++++++ durable/errors.go | 9 +- durable/runner.go | 137 ++++++++++------- durable/runner_options.go | 17 +- durable/service.go | 82 +++++++--- durable/service_test.go | 55 +++++++ durable/store.go | 2 +- durable/timetravel.go | 2 +- examples/fooddelivery/fooddelivery.go | 13 +- state/CHANGELOG.md | 2 + state/runner.go | 23 ++- 17 files changed, 545 insertions(+), 135 deletions(-) diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index cec71a8f..afaa543a 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -30,10 +30,12 @@ implement the `Store` interface out of tree. - **Deterministic replay.** A recovered instance is byte-identical to one that never crashed, because recovery replays the recorded driving events and nondeterministic results rather than re-executing their sources. -- **Exactly-once effects.** A domain effect is applied exactly once over the +- **Exactly-once effects.** A domain effect from a step (not an initial entry, which is at-least-once, or a parent transition an actor drives, which is not dispatched) is applied exactly once over the instance's lifetime (the live run plus any number of recoveries), even though - the replay loop is at-least-once. Each effect carries a deterministic - `EffectID` and is deduplicated through the `Store`'s dispatch set. + the replay loop is at-least-once. The exception is a crash or failed mark after the + handler returned, or a handoff between owners, where it can apply twice, so keep + handlers idempotent on `EffectInstance` plus the `EffectID`. Each effect carries a + deterministic `EffectID` and is deduplicated through the `Store`'s dispatch set. - **Durability across restart.** Every `Fire` step is write-ahead appended to the `Store` before it is acknowledged, so a crash after a successful `Fire` never loses the step. Periodic checkpoints bound the tail that recovery replays. @@ -74,7 +76,7 @@ the first time, and replayed verbatim on recovery: | **Clock** (timers) | `WithRunnerClock` | `JournalClockRead`, replayed so timers fire at the same recorded instants, wall-clock-independent; armed deadlines survive checkpoint compaction | | **Invoked services** | `WithServiceRegistry` + `Handle.RunService` | `JournalServiceResult`; a recorded result replays through the kernel's settle seam without re-running the service, and an unrecorded one (a context canceled during the service, a failed `Append`, a crash) runs again when the host calls `RunService` | | **Child-machine actors** | `WithActorPalette` + `Handle.DeliverToActor` | `JournalActorMessage`; recovery re-fires a recorded parent transition; a settled actor doesn't re-run, and a running one restarts from its initial state on any `Recover` | -| **Domain effects** | `WithEffectHandler` | dispatch set, applied exactly once via deterministic `EffectID` dedup | +| **Domain effects** | `WithEffectHandler` | dispatch set, a step's effects applied exactly once via deterministic `EffectID` dedup, with the exceptions under Exactly-once effects | Use `WithCheckpointEvery(n)` to tune how often a full snapshot is written: a shorter interval bounds recovery replay, a longer one cuts checkpoint cost. @@ -131,19 +133,21 @@ in by implementing `HistoryStore` (the in-tree `MemStore` does so under goroutines: each holds the Handle's lock for the whole step. A host callback must not call back into its Handle. `Instance()` is outside the lock. -A step that may not have been recorded, or a `Tick`, `RunService` or `DeliverToActor` whose -context ended partway, fails the Handle with `ErrHandleFailed`. `Recover` -a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a -newer owner holds the instance, so stop. +A step that may not have been recorded, a failed effect dispatch, or a `Tick`, `RunService` +or `DeliverToActor` whose context ended partway, fails the Handle with `ErrHandleFailed`. +`Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: +then a newer owner holds the instance, so stop. `Recover` retries an unapplied effect, and +returns `ErrEffectDispatch` while the handler still fails. ## A note on serialized payloads Events, service done-data, actor done-data, and actor messages are recorded as their JSON form. A parent reducer that type-asserts a non-JSON Go type from `AssignCtx.Event` -observes the JSON-decoded shape on a replayed `onDone`. A `WithEventData` payload is -JSON-decoded live as well as on replay, so a reducer gets the same value both ways: a -struct is a map and an integer a `float64`. An error stays an error with the same -message, with any invalid UTF-8 replaced. `Handle.Fire` refuses a payload it can't +observes the JSON-decoded shape of actor done-data on a replayed `onDone`. A +`WithEventData` payload and a service's result are JSON-decoded live as well as on replay, +so a reducer gets the same value both ways: a struct is a map and an integer a `float64`, +which loses precision above 2^53. An error stays an error with the same message, with any +invalid UTF-8 replaced, and `errors.Is` no longer matches it. `Handle.Fire` refuses a payload it can't record, or one over `WithMaxEventDataBytes` (8 MiB by default), with `ErrEventDataEncoding`, and nothing applies. A typed-codec option to carry the concrete Go value across the journal boundary is diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 6d9a73df..9c3ce2b1 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -9,6 +9,8 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Added +- **`EffectInstance`** reads the instance from the ctx an `EffectHandler` receives, because + effect ids repeat across instances. - **Fencing.** `WithFence` on `Store.Append` and `WithCheckpointFence` on `Store.Checkpoint` carry a caller-issued token, and a Store refuses a token lower than the highest it has seen for the instance with `ErrFenced`, writing nothing. @@ -30,6 +32,20 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed - Requires Go 1.27.3. +- **A failed effect dispatch fails the Handle.** The call returns `ErrHandleFailed` + wrapping `ErrEffectDispatch`, before any checkpoint, so the effect can't be compacted + away. `Recover` retries it, and fails with `ErrEffectDispatch` while the handler still + does. A `Fire` after a failed dispatch used to be accepted. A failed `MarkDispatched` + fails the Handle too, and `Recover` applies that effect again. The handler's error now + matches through `errors.Is`. +- **A `KindedEffect` that can't marshal fails `Tick` and `RunService`,** as it already + failed `Fire`, even with no handler wired. +- **A service result reaches `Assign` decoded, live as well as on replay,** so a retried + effect can't differ from the first attempt: a struct arrives as a map, an integer as a + `float64`, which loses precision above 2^53, and an error as an error with the same + message, any invalid UTF-8 replaced, which `errors.Is` no longer matches. An `Error` + method that panics settles with a message saying so. The journal records this decoded + value. - **A Handle's steps are serialized across goroutines.** `Fire`, `Tick`, `RunService` and `DeliverToActor` hold a lock for the whole step, so a host's timer loop no longer loses acknowledged steps. A host callback must not call back into its Handle, or it @@ -75,6 +91,14 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **A failed checkpoint no longer makes the next step conflict.** `Tick`, `RunService` and `DeliverToActor` now advance the next step as soon as the append lands. +- **Effects reach the handler from `Start`, `Tick` and `RunService`.** They were only + dispatched from `Fire`. `Start` applies the initial entry's effects after encoding the + baseline and before writing it, and only if the entry succeeded. Those effects are + at-least-once: a handler failure, a failed baseline write or overlapping `Start` calls + re-apply them, so key any dedupe on `EffectInstance` plus the effect id. Effects from a + parent transition an actor drives, including during `Fire`, are still not dispatched. + The first `Recover` of an older journal applies effects from uncompacted `Tick` and + service steps. - **A step whose context ended partway isn't journaled.** A canceled `RunService`, `Tick` or `DeliverToActor` used to record a result the live instance never applied, which recovery then committed, or for an actor, failed on. Each now returns `ErrHandleFailed` diff --git a/durable/README.md b/durable/README.md index 156b62a1..8fc48890 100644 --- a/durable/README.md +++ b/durable/README.md @@ -31,10 +31,12 @@ backends implement the `Store` interface out of tree. - **Deterministic replay**: a recovered instance is byte-identical to one that never crashed, because recovery replays the recorded driving events and nondeterministic results rather than re-executing their sources. -- **Exactly-once effects**: a domain effect is applied exactly once over the +- **Exactly-once effects**: a domain effect from a step (not an initial entry, which is at-least-once, or a parent transition an actor drives, which is not dispatched) is applied exactly once over the instance's lifetime (the live run plus any number of recoveries), even though - the replay loop is at-least-once. Each effect carries a deterministic - `EffectID` and is deduplicated through the `Store`'s dispatch set. + the replay loop is at-least-once. The exception is a crash or failed mark after the + handler returned, or a handoff between owners, where it can apply twice, so keep + handlers idempotent on `EffectInstance` plus the `EffectID`. Each effect carries a + deterministic `EffectID` and is deduplicated through the `Store`'s dispatch set. - **Durability across restart**: every `Fire` step is write-ahead appended to the `Store` before it is acknowledged, so a crash after a successful `Fire` never loses the step. Periodic checkpoints bound the tail recovery replays. @@ -84,7 +86,7 @@ the first time, and replayed verbatim on recovery: | **Clock** (timers) | `WithRunnerClock` | `JournalClockRead`, replayed so timers fire at the same recorded instants, wall-clock-independent; armed deadlines survive checkpoint compaction | | **Invoked services** | `WithServiceRegistry` + `Handle.RunService` | `JournalServiceResult`; a recorded result replays through the kernel's settle seam without re-running the service, and an unrecorded one (a context canceled during the service, a failed `Append`, a crash) runs again when the host calls `RunService` | | **Child-machine actors** | `WithActorPalette` + `Handle.DeliverToActor` | `JournalActorMessage`; recovery re-fires a recorded parent transition; a settled actor doesn't re-run, and a running one restarts from its initial state on any `Recover` | -| **Domain effects** | `WithEffectHandler` | dispatch set, applied exactly once via deterministic `EffectID` dedup | +| **Domain effects** | `WithEffectHandler` | dispatch set, a step's effects applied exactly once via deterministic `EffectID` dedup, with the exceptions under Exactly-once effects | Use `WithCheckpointEvery(n)` to tune how often a full snapshot is written (a shorter interval bounds recovery replay; a longer one cuts checkpoint cost). @@ -142,19 +144,21 @@ otherwise. goroutines: each holds the Handle's lock for the whole step. A host callback must not call back into its Handle. `Instance()` is outside the lock. -A step that may not have been recorded, or a `Tick`, `RunService` or `DeliverToActor` whose -context ended partway, fails the Handle with `ErrHandleFailed`. `Recover` -a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a -newer owner holds the instance, so stop. +A step that may not have been recorded, a failed effect dispatch, or a `Tick`, `RunService` +or `DeliverToActor` whose context ended partway, fails the Handle with `ErrHandleFailed`. +`Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: +then a newer owner holds the instance, so stop. `Recover` retries an unapplied effect, and +returns `ErrEffectDispatch` while the handler still fails. ## Caveat: serialized payloads Events, service done-data, actor done-data, and actor messages are recorded as their JSON form. A parent reducer that type-asserts a non-JSON Go type from `AssignCtx.Event` -observes the JSON-decoded shape on a replayed `onDone`. A `WithEventData` payload is -JSON-decoded live as well as on replay, so a reducer gets the same value both ways: a -struct is a map and an integer a `float64`. An error stays an error with the same -message, with any invalid UTF-8 replaced. `Handle.Fire` refuses a payload it can't +observes the JSON-decoded shape of actor done-data on a replayed `onDone`. A +`WithEventData` payload and a service's result are JSON-decoded live as well as on replay, +so a reducer gets the same value both ways: a struct is a map and an integer a `float64`, +which loses precision above 2^53. An error stays an error with the same message, with any +invalid UTF-8 replaced, and `errors.Is` no longer matches it. `Handle.Fire` refuses a payload it can't record, or one over `WithMaxEventDataBytes` (8 MiB by default), with `ErrEventDataEncoding`, and nothing applies. A typed-codec option to carry the concrete Go value across the journal boundary is diff --git a/durable/actor.go b/durable/actor.go index 716c4dd5..ecf89db3 100644 --- a/durable/actor.go +++ b/durable/actor.go @@ -137,9 +137,9 @@ func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef *h.actorBuf = append(*h.actorBuf, entries...) } - step := h.nextStep - rec := Record{Step: step, Entries: h.drainActors()} - if err := h.persistStep(ctx, step, &rec); err != nil { + rec := Record{Step: h.nextStep, Entries: h.drainActors()} + // nil: a parent transition an actor drives isn't dispatched (see doc.go). + if err := h.commitStep(ctx, &rec, nil); err != nil { return true, err } return true, nil diff --git a/durable/doc.go b/durable/doc.go index 93355b2f..99ead028 100644 --- a/durable/doc.go +++ b/durable/doc.go @@ -17,9 +17,12 @@ // recovery replays the exact recorded driving events and nondeterministic results // through the same pure transition function. // -// Exactly-once effects: a domain effect emitted by a transition is applied +// Exactly-once effects: a domain effect emitted by a step (Runner.Start's initial entry +// is at-least-once, and a parent transition an actor drives isn't dispatched) is applied // exactly once over the instance's lifetime — the live run plus any number of -// recoveries — even though the runtime's replay loop is at-least-once. Each +// recoveries — even though the runtime's replay loop is at-least-once. The exception is +// a crash or failed mark after the handler returned, or a handoff between owners, where +// an effect can apply twice, so keep handlers idempotent on EffectInstance plus the EffectID. Each // effect is stamped with a deterministic EffectID (step, ordinal, kind) and // deduplicated through the Store's dispatch set. // @@ -118,7 +121,7 @@ // publishing a message — as an Effect value the kernel routes out through // FireResult.Effects without applying. The Runner applies it through the caller- // supplied EffectHandler (WithEffectHandler), exactly once over the instance's -// lifetime. +// lifetime, with the exceptions under Exactly-once effects above. // // Each emitted effect is stamped with a deterministic EffectID derived from its // step, its emission ordinal within that step, and its kind — every component a diff --git a/durable/effect.go b/durable/effect.go index 73a002c0..cd828610 100644 --- a/durable/effect.go +++ b/durable/effect.go @@ -34,8 +34,8 @@ import ( // Each dispatchable effect is stamped with an EffectID that is a pure function of // the step it was emitted at, its ordinal among the dispatchable effects of that // step, and its kind: "##". Every component is deterministic -// — the step is the Fire ordinal, emission order is fixed by the machine -// definition, and the kind is the effect's stable discriminant — so the same +// — the step is its journal ordinal (-1 for Start's entry effects), emission order is +// fixed by the machine definition, and the kind is the effect's stable discriminant — so the same // effect is stamped with the identical id on the live run and on every recovery // replay, which re-fires the same events and re-emits the same effects in the // same order. That stable identity is what makes the dedup set meaningful across @@ -49,6 +49,8 @@ import ( // (write-ahead: the intent to dispatch is durable before any dispatch). // 2. Dispatch each effect whose id is not already in the dispatched set, calling // the handler, and mark each id dispatched in the Store as it succeeds. +// 3. Checkpoint, if one is due. This comes last so a checkpoint never compacts an +// effect that hasn't landed. // // This yields exactly-once effect application over the at-least-once delivery the // runtime is built on: @@ -59,20 +61,44 @@ import ( // once) and, because it is then marked, never again. // - Crash AFTER dispatch (id marked): recovery re-emits the effect but finds its // id already marked and skips it — so it is not redispatched (exactly once). -// - Handler error: the effect is left un-marked and the error surfaces wrapped in -// ErrEffectDispatch, so a later recovery retries it (at-least-once until it -// succeeds; exactly-once thereafter). +// - Crash or failed mark AFTER the handler returned: the effect was applied but its +// id isn't marked, so recovery applies it again. The handler should be idempotent +// on EffectInstance plus the id for this window, and for a handoff, when two owners +// can run it at once. +// - Handler error: the effect is left un-marked, the error surfaces wrapped in +// ErrEffectDispatch and the Handle fails with ErrHandleFailed, so a later recovery +// retries it (at-least-once until it succeeds; exactly-once thereafter). // // The dedup set is held by the Store (DispatchStore) so it shares the Record's // durability and survives checkpoint compaction — a delayed redispatch of an // already-applied, since-compacted effect is still recognized and skipped. // EffectHandler applies one emitted domain effect, identified by its stamped -// EffectID. The Runner calls it exactly once per EffectID over the lifetime of an -// instance (see WithEffectHandler). A non-nil error leaves the effect un-marked -// for a later retry and is surfaced to the caller wrapped in ErrEffectDispatch. +// EffectID. The Runner calls it once per EffectID over the lifetime of an instance +// (see WithEffectHandler), with three exceptions: a repeated Start can apply the initial +// entry's effects again (see Runner.Start), an effect applies again if the handler +// returned but the mark didn't land, and two owners can run it at once during a handoff. +// An id is unique only within its instance: EffectInstance reads the instance from ctx. +// A non-nil error leaves the effect un-marked for a later retry and is surfaced to the +// caller wrapped in ErrEffectDispatch. type EffectHandler func(ctx context.Context, effectID string, effect state.Effect) error +// effectInstanceKey is the ctx key for the instance an EffectHandler call belongs to. +type effectInstanceKey struct{} + +// withEffectInstance returns ctx carrying id for the handler. +func withEffectInstance(ctx context.Context, id InstanceID) context.Context { + return context.WithValue(ctx, effectInstanceKey{}, id) +} + +// EffectInstance returns the instance an EffectHandler call belongs to, from the ctx the +// handler received. A handler that keeps its own dedupe record must key on this plus the +// effect id, because effect ids repeat across instances. +func EffectInstance(ctx context.Context) (InstanceID, bool) { + id, ok := ctx.Value(effectInstanceKey{}).(InstanceID) + return id, ok +} + // DispatchStore is the dedup seam for idempotent effect dispatch: the Store // tracks which effect ids an instance has applied, so a (re)dispatch skips the // ones that already landed. It is an additive, optional capability — the core @@ -193,6 +219,24 @@ func (h *Handle[S, E, C]) dispatchEffects(ctx context.Context, des []dispatchabl return h.runner.dispatch(ctx, h.id, des) } +// applyStartEffects invokes the handler for the initial entry's domain effects. Like +// dispatch it is a no-op without a handler or a DispatchStore. It marks nothing, because +// the instance doesn't exist yet and nothing replays these effects. +func (r *Runner[S, E, C]) applyStartEffects(ctx context.Context, id InstanceID, des []dispatchableEffect) error { + if len(des) == 0 || r.cfg.effectHandler == nil { + return nil + } + if _, ok := r.store.(DispatchStore); !ok { + return nil + } + for _, de := range des { + if err := r.cfg.effectHandler(withEffectInstance(ctx, id), de.id, de.effect); err != nil { + return fmt.Errorf("%w: effect %q for %q: %w", ErrEffectDispatch, de.id, id, err) + } + } + return nil +} + // dispatchReplayEffects re-dispatches a replayed step's domain effects during // recovery: it re-stamps the re-emitted effects with the same deterministic ids // (step and emission order reproduce exactly) and applies only those not already @@ -225,8 +269,8 @@ func (r *Runner[S, E, C]) dispatch(ctx context.Context, id InstanceID, des []dis if dispatched[de.id] { continue } - if err := r.cfg.effectHandler(ctx, de.id, de.effect); err != nil { - return fmt.Errorf("%w: effect %q for %q: %v", ErrEffectDispatch, de.id, id, err) + if err := r.cfg.effectHandler(withEffectInstance(ctx, id), de.id, de.effect); err != nil { + return fmt.Errorf("%w: effect %q for %q: %w", ErrEffectDispatch, de.id, id, err) } if err := ds.MarkDispatched(ctx, id, de.id); err != nil { return fmt.Errorf("durable: marking effect %q dispatched for %q: %w", de.id, id, err) diff --git a/durable/effect_test.go b/durable/effect_test.go index bb065564..1be259b5 100644 --- a/durable/effect_test.go +++ b/durable/effect_test.go @@ -507,3 +507,182 @@ func TestMemStore_DispatchSet_MarkAndQuery(t *testing.T) { t.Fatalf("unmarked id reported dispatched") } } + +// entryNotifyMachine emits a host effect on entering the state an instance starts +// in, so the effect originates in Start rather than Fire. +func entryNotifyMachine() *state.Machine[string, string, *runCtx] { + return state.ForgeFor[*runCtx]("entryNotify"). + Action("notify", func(state.ActionCtx[*runCtx]) (state.Effect, error) { + return notifyEffect{To: "ops@example.com"}, nil + }). + State("ready").OnEntry("notify"). + Initial("ready"). + Quench() +} + +// TestRunner_EffectDispatch_Start_AppliesEntryEffect proves a domain effect emitted +// on entering the initial state reaches the handler. +func TestRunner_EffectDispatch_Start_AppliesEntryEffect(t *testing.T) { + ctx := context.Background() + rec := newEffectRecorder() + r := durable.NewRunner(entryNotifyMachine(), durable.NewMemStore(), + durable.WithEffectHandler[string, string, *runCtx](rec.handler())) + if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); err != nil { + t.Fatalf("Start: %v", err) + } + if n := rec.totalApplies(); n != 1 { + t.Fatalf("handler applied %d effects, want 1 (the initial state's entry notify)", n) + } +} + +// TestRunner_EffectDispatch_TimerTransition_AppliesEffect proves a domain effect +// emitted by a timer-driven transition reaches the handler. +func TestRunner_EffectDispatch_TimerTransition_AppliesEffect(t *testing.T) { + ctx := context.Background() + rec := newEffectRecorder() + clk := state.NewFakeClock(epoch) + r := durable.NewRunner(timerNotifyMachine(), durable.NewMemStore(), + durable.WithEffectHandler[string, string, *runCtx](rec.handler()), + durable.WithRunnerClock[string, string, *runCtx](clk)) + h, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); err != nil { + t.Fatalf("Fire go: %v", err) + } + clk.Advance(2 * time.Second) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + if n := rec.totalApplies(); n != 2 { + t.Fatalf("handler applied %d effects, want 2 (the timer transition's notify)", n) + } +} + +// TestRunner_EffectDispatch_ServiceSettle_AppliesEffect proves a domain effect +// emitted by the transition a service result drives reaches the handler. +func TestRunner_EffectDispatch_ServiceSettle_AppliesEffect(t *testing.T) { + ctx := context.Background() + rec := newEffectRecorder() + var calls int64 + fn := counterService(&calls) + r := durable.NewRunner(fetchThenWaitMachine(fn), durable.NewMemStore(), + durable.WithEffectHandler[string, string, svcCtx](rec.handler()), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))) + h, err := r.Start(ctx, "i1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire start: %v", err) + } + mustRunService(t, h, state.InvokeID("fetchThenWait", "loading", 0)) + if n := rec.totalApplies(); n != 1 { + t.Fatalf("handler applied %d effects, want 1 (the settle transition's notify)", n) + } +} + +// TestRunner_EffectDispatch_FailedThenCheckpointed_AppliedOnce proves an effect +// whose dispatch failed is still applied exactly once, live or on recovery, after +// a later checkpoint compacts its step. +func TestRunner_EffectDispatch_FailedThenCheckpointed_AppliedOnce(t *testing.T) { + ctx := context.Background() + st := durable.NewMemStore() + live := newEffectRecorder() + live.failKinds["demo.notify"] = errors.New("downstream unavailable") + opts := []durable.Option[string, string, *runCtx]{ + durable.WithEffectHandler[string, string, *runCtx](live.handler()), + durable.WithCheckpointEvery[string, string, *runCtx](2), + } + h, err := durable.NewRunner(effectMachine(), st, opts...).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); err == nil { // step 0: its notify fails + t.Fatalf("Fire go: want the handler error") + } + delete(live.failKinds, "demo.notify") + if _, err = h.Fire(ctx, "bill"); err != nil { // step 1: checkpoint compacts step 0 + // A Handle that refuses Fires after a failed dispatch is resumed by Recover. + if h, err = durable.Recover(ctx, effectMachine(), st, "i1", opts...); err != nil { + t.Fatalf("Recover: %v", err) + } + if _, err = h.Fire(ctx, "bill"); err != nil { + t.Fatalf("Fire bill: %v", err) + } + } + + healthy := newEffectRecorder() + if _, err = durable.Recover(ctx, effectMachine(), st, "i1", + durable.WithEffectHandler[string, string, *runCtx](healthy.handler())); err != nil { + t.Fatalf("Recover: %v", err) + } + // go emits notify; bill emits charge and notify. Each applies exactly once. + ids := append(live.ids(), healthy.ids()...) + if len(ids) != 3 { + t.Fatalf("applied %d effect ids across live and recovery, want 3: %v", len(ids), ids) + } + for _, id := range ids { + if n := live.count(id) + healthy.count(id); n != 1 { + t.Fatalf("effect %q applied %d times, want once", id, n) + } + } +} + +// failingEntryMachine emits a host effect on entry, then fails the entry. +func failingEntryMachine() *state.Machine[string, string, *runCtx] { + return state.ForgeFor[*runCtx]("failingEntry"). + Action("notify", func(state.ActionCtx[*runCtx]) (state.Effect, error) { + return notifyEffect{To: "ops@example.com"}, nil + }). + Action("boom", func(state.ActionCtx[*runCtx]) (state.Effect, error) { + return nil, errors.New("boom") + }). + State("ready").OnEntry("notify").OnEntry("boom"). + Initial("ready"). + Quench() +} + +// TestRunner_EffectDispatch_Start_FailedEntryAppliesNothing proves an effect emitted +// before a failed entry action doesn't reach the handler, as with a failed Fire. +func TestRunner_EffectDispatch_Start_FailedEntryAppliesNothing(t *testing.T) { + ctx := context.Background() + rec := newEffectRecorder() + r := durable.NewRunner(failingEntryMachine(), durable.NewMemStore(), + durable.WithEffectHandler[string, string, *runCtx](rec.handler())) + if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); err != nil { + t.Fatalf("Start: %v", err) + } + if n := rec.totalApplies(); n != 0 { + t.Fatalf("handler applied %d effects from a failed entry, want 0", n) + } +} + +// TestEffectInstance_ReachesTheHandler proves a handler can tell instances apart, on +// Start and on a step. +func TestEffectInstance_ReachesTheHandler(t *testing.T) { + ctx := context.Background() + var seen []durable.InstanceID + handler := func(ctx context.Context, _ string, _ state.Effect) error { + id, ok := durable.EffectInstance(ctx) + if !ok { + t.Error("no instance on the handler's ctx") + } + seen = append(seen, id) + return nil + } + r := durable.NewRunner(entryNotifyMachine(), durable.NewMemStore(), + durable.WithEffectHandler[string, string, *runCtx](handler)) + for _, id := range []durable.InstanceID{"a", "b"} { + if _, err := r.Start(ctx, id, &runCtx{}, state.WithInitialState("ready")); err != nil { + t.Fatalf("Start: %v", err) + } + } + if len(seen) != 2 || seen[0] != "a" || seen[1] != "b" { + t.Fatalf("handler saw %v, want [a b]", seen) + } + if _, ok := durable.EffectInstance(ctx); ok { + t.Fatal("instance on a ctx the runner never touched") + } +} diff --git a/durable/errors.go b/durable/errors.go index 928739f7..50d04c80 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -55,8 +55,8 @@ var ( // ErrHandleFailed is reported by a Handle's Fire, Tick, RunService and // DeliverToActor once a step applied to the live instance but may not have been - // recorded, or a Tick, RunService or DeliverToActor's context ended partway, on that - // call and every later one. It wraps the cause. Recover a new + // recorded, a Tick, RunService or DeliverToActor's context ended partway, or an + // effect's dispatch failed, on that call and every later one. It wraps the cause. Recover a new // Handle with the same options to continue, unless the cause is ErrFenced: then a // newer owner holds the instance, so stop. ErrHandleFailed = errors.New(errorPrefix + errorHandleFailed) @@ -70,9 +70,10 @@ var ( ErrInstanceExists = errors.New(errorPrefix + errorInstanceExists) // ErrEffectDispatch wraps a failure returned by the caller-supplied effect - // handler (WithEffectHandler) while applying an emitted effect. The effect was + // handler (WithEffectHandler) while applying an emitted effect. A step's effect was // persisted (write-ahead) but not marked dispatched, so a later recovery - // retries it; callers match this with errors.Is to distinguish a dispatch + // retries it, and the Handle fails with ErrHandleFailed; a failure in Runner.Start + // leaves no instance. Recover fails with this error while the handler still does; callers match this with errors.Is to distinguish a dispatch // failure from a kernel transition or persistence error. ErrEffectDispatch = errors.New(errorPrefix + errorEffectDispatch) ) diff --git a/durable/runner.go b/durable/runner.go index 73ea9af3..826a7f53 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -98,7 +98,8 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // whose result wasn't recorded is in flight again, so RunService runs it a second time. // A running InvokeActor actor restarts from its initial state on any Recover, losing every // delivery so far, and a Spawn actor doesn't return. If the cause is ErrFenced, a -// newer owner holds the instance: stop, because recovering won't help. +// newer owner holds the instance: stop, because recovering won't help. A failed effect +// dispatch fails the Handle the same way, and Recover retries the effect. type Handle[S comparable, E comparable, C any] struct { runner *Runner[S, E, C] id InstanceID @@ -150,10 +151,9 @@ func (h *Handle[S, E, C]) drainClock() []state.JournalEntry { // durable seams compose: the scheduler arms or cancels delayed (`after`) timers (and // the Handle's timer table mirrors their deadlines for checkpointing), the service // runner starts or stops invoked services, and the actor system spawns or stops -// child-machine actors. It is the shared absorb the live seam methods (Fire, -// RunService, DeliverToActor, Tick) call for the effects they produced, so a state a -// settle or delivery enters arms whatever it declares — a service whose onDone spawns -// an actor, an actor whose onDone arms a timer, and so on. Absorbing a driver's +// child-machine actors. It is the shared absorb Fire and RunService call for the +// effects they produced, so a state a settle enters arms whatever it declares, such as +// a service whose onDone spawns an actor. Absorbing a driver's // effects is a no-op when that driver is not wired or the effects name none of its // kind. func (h *Handle[S, E, C]) absorbDrivers(ctx context.Context, effects []state.Effect) { @@ -281,6 +281,14 @@ func (h *Handle[S, E, C]) ID() InstanceID { return h.id } // state.WithInitialState) configure the initial configuration. Starting an // InstanceID that already exists in the Store reports ErrInstanceExists rather // than clobbering its recorded baseline. +// +// Start applies the initial entry's domain effects before the baseline write, and only +// if the entry succeeded: a failed entry still returns a Handle and a nil error, and +// Instance().InitialErr reports it. They're at-least-once, unlike a step's: a handler failure or a +// failed baseline write leaves no instance, so calling Start again re-applies every +// effect, as does a crash before the baseline. Overlapping Starts on one id, including a +// stale owner's, each apply them before the Store refuses all but one. To dedupe, key on +// EffectInstance plus the effect id. A handler wired only at Recover never sees them. func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opts ...state.CastOption[S]) (*Handle[S, E, C], error) { if _, _, err := r.store.Load(ctx, id); err == nil { return nil, fmt.Errorf("%w: %q", ErrInstanceExists, id) @@ -301,10 +309,10 @@ func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opt castOpts := append([]state.CastOption[S]{state.WithClock[S](recClock), state.WithUnboundedHistory[S]()}, opts...) inst := r.machine.Cast(input, castOpts...) - // Persist a baseline checkpoint at baselineStep (below the first fired step) - // so the instance is reconstructable from the Store before any event is fired. - // The baseline is the cast snapshot; the tail then accumulates fired steps on - // top of it. + // Encode a baseline checkpoint at baselineStep (below the first fired step), so the + // instance is reconstructable from the Store before any event is fired. The baseline + // is the cast snapshot; the tail then accumulates fired steps on top of it. Encoding + // comes before the effects, so an instance that can't be encoded applies none. snap, err := state.MarshalSnapshot(inst.Snapshot()) if err != nil { return nil, fmt.Errorf("durable: marshaling start snapshot for %q: %w", id, err) @@ -315,6 +323,16 @@ func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opt if err != nil { return nil, fmt.Errorf("durable: marshaling start checkpoint envelope for %q: %w", id, err) } + // Apply the initial entry's domain effects before the instance exists: a handler + // failure then leaves nothing behind, so the caller can Start again. Nothing can + // re-derive these effects after the baseline, so they're at-least-once. An entry that + // failed applies none, as a failed Fire dispatches none. + if inst.InitialErr() == nil { + if err := r.applyStartEffects(ctx, id, dispatchableEffects(baselineStep, inst.InitialEffects())); err != nil { + return nil, err + } + } + if err := r.store.Checkpoint(ctx, id, baseEnv, baselineStep, WithCheckpointFence(r.cfg.fence)); err != nil { return nil, fmt.Errorf("durable: checkpointing start baseline for %q: %w", id, err) } @@ -363,7 +381,7 @@ func (r *Runner[S, E, C]) newServiceRunner(inst *state.Instance[S, E, C]) *state if r.cfg.serviceReg == nil { return nil } - return state.NewServiceRunner(inst, r.cfg.serviceReg) + return state.NewServiceRunner(inst, r.cfg.serviceReg).WithResultTransform(replayableOutcome) } // newActorSystem builds the kernel host driver for child-machine actors bound to @@ -382,7 +400,8 @@ func (r *Runner[S, E, C]) newActorSystem(inst *state.Instance[S, E, C]) *state.A // Fire drives one event through a durable instance identified by id, loading and // replaying it from the Store first, then recording the step. Handle.Fire describes -// how a state.WithEventData payload is handled, and when it's refused. It is the +// how a state.WithEventData payload is handled, and when it's refused. Loading can +// return ErrEffectDispatch before the event fires, as Recover can. It is the // stateless entry point (no Handle required); for a hot path that fires many // events in sequence, hold a Handle from Start or Recover and use Handle.Fire to // avoid reloading between steps. Calls on one id aren't serialized, and must not overlap @@ -400,7 +419,8 @@ func (r *Runner[S, E, C]) Fire(ctx context.Context, id InstanceID, event E, opts // step's ordinal (write-ahead, before returning), and — when the // checkpoint policy is due — persists a full Snapshot and compacts the tail. A // kernel transition error journals nothing (no step was produced) and is -// returned to the caller. +// returned to the caller. Between the append and any checkpoint it applies the step's +// domain effects; a failed dispatch fails the Handle with ErrHandleFailed. // // A state.WithEventData payload reaches the transition's Assign in its JSON-decoded // form, live and on replay alike, so both see the same value: a struct arrives as a @@ -437,32 +457,8 @@ func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireO h.absorbDrivers(ctx, res.Effects) step := h.nextStep - // Stamp this step's domain effects with their deterministic ids and carry them - // in the Record so the write-ahead append durably names every effect that must - // be dispatched before any dispatch happens. - des := dispatchableEffects(step, res.Effects) - effEnvs, err := recordEffects(des) - if err != nil { - return res, h.fail(err) - } - rec := Record{Step: step, Event: []byte(res.Trace.EventPayload), EventData: payload.data, EventDataError: payload.errMsg, Entries: h.drainClock(), Effects: effEnvs} - // Write-ahead append the step Record (with its effect ids) BEFORE dispatching, but - // hold off compacting it: dispatch happens between the append and the checkpoint so - // a crash mid-dispatch leaves the effect recorded in the still-present tail for - // recovery to redispatch — never orphaned behind the compaction. - if err := h.appendStep(ctx, step, &rec); err != nil { - return res, h.fail(err) - } - h.nextStep++ - // Dispatch AFTER the append: a crash in this window leaves the effect recorded - // but un-marked, so recovery redispatches it (at-least-once), deduped by id to - // exactly-once once it lands. - if err := h.dispatchEffects(ctx, des); err != nil { - return res, err - } - // Compact through this step only now that its effects are dispatched, so the - // checkpoint never discards a not-yet-dispatched effect. - if err := h.checkpointStep(ctx, step, &rec); err != nil { + rec := Record{Step: step, Event: []byte(res.Trace.EventPayload), EventData: payload.data, EventDataError: payload.errMsg, Entries: h.drainClock()} + if err := h.commitStep(ctx, &rec, res.Effects); err != nil { return res, err } return res, nil @@ -518,28 +514,40 @@ func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], erro if len(rec.Entries) == 0 && rec.TickSteps == 0 { return results, nil } - if err := h.appendStep(ctx, step, &rec); err != nil { - return results, h.fail(err) + var effects []state.Effect + for i := range results { + effects = append(effects, results[i].Effects...) } - h.nextStep = rec.followingStep() - if err := h.checkpointStep(ctx, step, &rec); err != nil { - return results, err - } - return results, nil + return results, h.commitStep(ctx, &rec, effects) } -// persistStep write-ahead appends rec, advances the next step, then checkpoints if -// the policy is due. A failed append fails the Handle, because the kernel already -// applied the step; a failed checkpoint doesn't, because the Record is durable. This -// relies on the Append running inline, under the Handle's lock, before the step -// returns: an Append deferred off this goroutine would acknowledge a step whose -// outcome isn't known, and the Handle couldn't fail. -func (h *Handle[S, E, C]) persistStep(ctx context.Context, step int, rec *Record) error { - if err := h.appendStep(ctx, step, rec); err != nil { +// commitStep makes a step durable, in write-ahead order. It stamps the step's domain +// effects and carries them in rec, appends rec, advances the next step, dispatches the +// effects, and only then checkpoints, so a checkpoint never compacts an effect that +// hasn't landed: a crash between append and dispatch leaves it in the tail for recovery +// to apply, at-least-once and deduped by id. +// +// The kernel has already applied the step, so a failed encode or append fails the +// Handle. So does a failed dispatch: the effect stays in the tail, and Recover retries +// it. A failed checkpoint doesn't, because the Record is durable. This relies on the +// Append running inline, under the Handle's lock, before the step returns: an Append +// deferred off this goroutine would acknowledge a step whose outcome isn't known, and +// the Handle couldn't fail. +func (h *Handle[S, E, C]) commitStep(ctx context.Context, rec *Record, effects []state.Effect) error { + des := dispatchableEffects(rec.Step, effects) + envs, err := recordEffects(des) + if err != nil { + return h.fail(err) + } + rec.Effects = envs + if err := h.appendStep(ctx, rec.Step, rec); err != nil { return h.fail(err) } h.nextStep = rec.followingStep() - return h.checkpointStep(ctx, step, rec) + if err := h.dispatchEffects(ctx, des); err != nil { + return h.fail(err) + } + return h.checkpointStep(ctx, rec.Step, rec) } // checkpointDue reports whether the checkpoint policy lands a checkpoint at step. @@ -549,7 +557,7 @@ func (h *Handle[S, E, C]) checkpointDue(step int) bool { // appendStep marshals the due checkpoint snapshot into rec (so the Record carries it) // and write-ahead appends rec, WITHOUT compacting the tail yet. Splitting the append -// from the compaction lets the effect-bearing Fire path dispatch a step's domain +// from the compaction lets commitStep dispatch a step's domain // effects after the durable append but before the checkpoint compacts that step away // — so a crash between append and dispatch leaves the effect recoverable from the // still-present tail rather than orphaned behind the compaction. @@ -593,7 +601,9 @@ func (h *Handle[S, E, C]) checkpointStep(ctx context.Context, step int, rec *Rec // snapshot (firing nothing, no IO), and replays the tail's recorded driving // events through the kernel to reach the instance's live state. The returned // Handle continues recording subsequent Fires. Recover reports ErrInstanceNotFound -// for an instance that was never started, and the context's error if ctx ends. +// for an instance that was never started, and the context's error if ctx ends. It +// applies any recorded effect not yet marked dispatched, so it returns ErrEffectDispatch +// and no Handle while the effect handler keeps failing. func Recover[S comparable, E comparable, C any](ctx context.Context, m *state.Machine[S, E, C], st Store, id InstanceID, opts ...Option[S, E, C]) (*Handle[S, E, C], error) { r := NewRunner(m, st, opts...) return r.recover(ctx, id) @@ -687,10 +697,16 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S // re-fires the identical onDone / onError event with the identical data — // running no service. The settle absorbs its own follow-on StartService // effects, so a chained invoke arms its successor for the next entry. + var settled []state.Effect for _, entry := range serviceEntries(rec) { - if err := replayService(ctx, svc, entry); err != nil { + effects, err := replayService(ctx, svc, entry) + if err != nil { return nil, fmt.Errorf("durable: at step %d for %q: %w", rec.Step, id, err) } + settled = append(settled, effects...) + } + if err := r.dispatchReplayEffects(ctx, id, rec.Step, settled); err != nil { + return nil, err } case rec.Tick: // Re-derive the timers the live tick fired: ticking the replay clock @@ -709,6 +725,13 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S mirrorArmed(armed, tickRes[ri].Effects, now) } } + var ticked []state.Effect + for ri := range tickRes { + ticked = append(ticked, tickRes[ri].Effects...) + } + if err := r.dispatchReplayEffects(ctx, id, rec.Step, ticked); err != nil { + return nil, err + } case len(rec.Event) == 0: continue // a checkpoint-only Record drives no event default: diff --git a/durable/runner_options.go b/durable/runner_options.go index 624051c5..f6a62451 100644 --- a/durable/runner_options.go +++ b/durable/runner_options.go @@ -124,14 +124,18 @@ func WithActorPalette[S comparable, E comparable, C any](palette map[string]stat } // WithEffectHandler binds the seam the Runner calls to apply each emitted domain -// effect — an email, a charge, a published message: any at-most-once side effect -// a transition emits as a domain Effect value. A durable effect is applied +// effect — an email, a charge, a published message: any side effect +// a transition emits as a domain Effect value. Effects from a parent transition an actor +// drives aren't dispatched, and a Store that isn't a DispatchStore applies none. A +// durable effect from a step is applied // exactly once over the whole lifetime of an instance (the live run plus any -// number of recoveries), deduped by a deterministic EffectID: the Runner stamps +// number of recoveries), except that it can apply twice after a crash or failed mark +// once the handler returned, or during a handoff, so keep the handler idempotent on +// EffectInstance plus the id. It is deduped by a deterministic EffectID: the Runner stamps // each emitted effect, write-ahead persists the step Record carrying those ids, // then invokes the handler for every id not already marked dispatched, marking -// each as it succeeds. A handler error is surfaced (wrapped in ErrEffectDispatch) -// and the failing effect is left un-marked, so a later recovery retries it +// each as it succeeds. A handler error fails the Handle with ErrHandleFailed, wrapping +// ErrEffectDispatch, and the failing effect is left un-marked, so a later recovery retries it // (at-least-once until it succeeds; exactly-once once it does). // // The handler receives the stamped EffectID and the live effect value. It is nil @@ -170,7 +174,8 @@ func WithMaxEventDataBytes[S comparable, E comparable, C any](n int) Option[S, E // Checkpoint, so a Handle that has lost the instance to a newer owner gets ErrFenced // instead of committing. Pass the token the host's lease issued; a retry by the same // owner keeps the same token. A non-positive token is the same as no token, which a -// fenced instance refuses. MarkDispatched is not fenced. +// fenced instance refuses. MarkDispatched is not fenced, so during a handoff an effect +// handler can run concurrently for one effect id. // // The token applies to every instance the Runner drives, so a host with a token per // instance passes it to Recover, or builds a Runner per instance. No fence token is written diff --git a/durable/service.go b/durable/service.go index d61b1f73..6cbf4308 100644 --- a/durable/service.go +++ b/durable/service.go @@ -29,8 +29,8 @@ import ( // The Runner wraps the kernel's reusable host driver, state.ServiceRunner, which // turns StartService / StopService effects into real ServiceFn executions and // re-fires each result through the invocation's onDone / onError event. On the live -// path RunService delegates to ServiceRunner.Run (resolve + run + settle), reads -// the raw outcome the service produced, and journals it as a JournalServiceResult +// path RunService delegates to ServiceRunner.Tick (resolve + run + settle), reads +// the outcome in its JSON-decoded form, and journals it as a JournalServiceResult // correlated by the invocation id. On recovery the recorded outcomes are replayed // in order through ServiceRunner.SettleDone / SettleError — the same settle methods // the live run drove — so the kernel re-fires the identical onDone / onError event @@ -55,7 +55,7 @@ type serviceResultPayload struct { // Result is the JSON-encoded successful return of the ServiceFn, present for a // SettleDone outcome. It is decoded to a Go value and re-fired through onDone so // the transition's Assign reads it from AssignCtx.Event exactly as on the live - // run. + // run, which settles with the same decoded form (see replayableOutcome). Result json.RawMessage `json:"result,omitempty"` // Error is the failure message for a SettleError outcome. It is reconstructed // into an error and re-fired through onError so the transition's Assign reads it @@ -80,9 +80,12 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // records its outcome, and settles it — firing the invocation's onDone (on // success) or onError (on failure) event through the durable instance and // recording the produced step. The service is resolved and executed against the -// registry supplied with WithServiceRegistry; its raw result or error is journaled +// registry supplied with WithServiceRegistry; its result or error is journaled // as a JournalServiceResult correlated by id so recovery replays it without -// re-invoking the service. +// re-invoking the service. The outcome reaches the machine JSON-decoded, live as on +// replay, and the journal holds that form: a struct is a map, an integer a float64 that +// loses precision above 2^53, and an error an error with the same message, which +// errors.Is no longer matches. // // It returns the routed FireResult and true, or the zero result and false when id // names no in-flight service (already settled, stopped, or never started). It returns @@ -90,11 +93,13 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // before the service started. // // Once the service has run, it returns true with any error. A failed Append, a result -// that won't encode, or a ctx that ended during the service returns ErrHandleFailed and -// records nothing: a recovered Handle has the service in flight again, and RunService +// or KindedEffect that won't encode, or a ctx that ended during the service returns +// ErrHandleFailed and records nothing: a recovered Handle has the service in flight again, and RunService // runs it a second time. That error also matches the cause, so test ErrHandleFailed // first: it alone means Recover is needed. A failed Checkpoint returns the plain error, // and the result is recorded. +// A failed effect dispatch also returns ErrHandleFailed, but the result is recorded: +// Recover retries the effect and the service doesn't re-run. func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.FireResult[S], bool, error) { h.mu.Lock() defer h.mu.Unlock() @@ -139,14 +144,45 @@ func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.Fire *h.svcBuf = append(*h.svcBuf, entry) } - step := h.nextStep - rec := Record{Step: step, Entries: h.drainServices()} - if err := h.persistStep(ctx, step, &rec); err != nil { + rec := Record{Step: h.nextStep, Entries: h.drainServices()} + if err := h.commitStep(ctx, &rec, res.Effects); err != nil { return res, true, err } return res, true, nil } +// replayableOutcome gives a service's outcome the form a replay supplies, so the live +// settle and a replay can't differ: a result in its JSON-decoded form, an error as an +// error with the same message once any invalid UTF-8 is replaced, which is what the +// round trip through JSON does. An Error method that panics settles with a message +// saying so. A result with no JSON form is left as it is, and fails +// the Handle when recording it fails. +func replayableOutcome(result any, err error) (any, error) { + if err != nil { + msg, merr := errorMessage(err) + if merr != nil { + return nil, merr + } + b, jerr := marshalPayload(msg) + if jerr != nil { + return nil, errors.New(msg) + } + if jerr = json.Unmarshal(b, &msg); jerr != nil { + return nil, errors.New(msg) + } + return nil, errors.New(msg) + } + b, jerr := marshalPayload(result) + if jerr != nil { + return result, nil + } + var decoded any + if jerr = json.Unmarshal(b, &decoded); jerr != nil { + return result, nil + } + return decoded, nil +} + // recordServiceOutcome builds the JournalServiceResult entry for a just-settled // service. It reads the success result through lastResult (valid only during the // settle's synchronous window) and the error through lastErr, encoding whichever @@ -157,7 +193,7 @@ func recordServiceOutcome(id string, lastResult func() (any, bool), lastErr erro msg := lastErr.Error() sp.Error = &msg } else if result, ok := lastResult(); ok { - raw, err := json.Marshal(result) + raw, err := marshalPayload(result) if err != nil { return state.JournalEntry{}, fmt.Errorf("marshaling service result: %w", err) } @@ -179,32 +215,34 @@ func recordServiceOutcome(id string, lastResult func() (any, bool), lastErr erro // the recorded payload and calls SettleDone (with the recorded result) or // SettleError (with the reconstructed error), so the kernel re-fires the same // onDone / onError event with the same data and re-derives identical context. It -// reports an error only when the recorded entry cannot be decoded or names no +// returns the effects the settle emitted, and reports an error only when the recorded entry cannot be decoded or names no // in-flight service (a corrupt or mis-ordered journal). -func replayService[S comparable, E comparable, C any](ctx context.Context, svc *state.ServiceRunner[S, E, C], entry state.JournalEntry) error { +func replayService[S comparable, E comparable, C any](ctx context.Context, svc *state.ServiceRunner[S, E, C], entry state.JournalEntry) ([]state.Effect, error) { if svc == nil { - return errors.New("durable: replaying a service result requires a service registry (supply WithServiceRegistry)") + return nil, errors.New("durable: replaying a service result requires a service registry (supply WithServiceRegistry)") } var sp serviceResultPayload if err := json.Unmarshal(entry.Payload, &sp); err != nil { - return fmt.Errorf("durable: decoding recorded service %q outcome: %w", entry.CorrelationID, err) + return nil, fmt.Errorf("durable: decoding recorded service %q outcome: %w", entry.CorrelationID, err) } if sp.Error != nil { - if _, ok := svc.SettleError(ctx, entry.CorrelationID, errors.New(*sp.Error)); !ok { - return fmt.Errorf("durable: replaying service %q: no in-flight service to settle", entry.CorrelationID) + res, ok := svc.SettleError(ctx, entry.CorrelationID, errors.New(*sp.Error)) + if !ok { + return nil, fmt.Errorf("durable: replaying service %q: no in-flight service to settle", entry.CorrelationID) } - return nil + return res.Effects, nil } var result any if len(sp.Result) > 0 { if err := json.Unmarshal(sp.Result, &result); err != nil { - return fmt.Errorf("durable: decoding recorded service %q result: %w", entry.CorrelationID, err) + return nil, fmt.Errorf("durable: decoding recorded service %q result: %w", entry.CorrelationID, err) } } - if _, ok := svc.SettleDone(ctx, entry.CorrelationID, result); !ok { - return fmt.Errorf("durable: replaying service %q: no in-flight service to settle", entry.CorrelationID) + res, ok := svc.SettleDone(ctx, entry.CorrelationID, result) + if !ok { + return nil, fmt.Errorf("durable: replaying service %q: no in-flight service to settle", entry.CorrelationID) } - return nil + return res.Effects, nil } // serviceEntries returns the recorded service outcomes carried by a single diff --git a/durable/service_test.go b/durable/service_test.go index bf4fb5f0..9d9223b1 100644 --- a/durable/service_test.go +++ b/durable/service_test.go @@ -472,6 +472,25 @@ func TestService_Determinism(t *testing.T) { } } +// fetchThenWaitMachine routes its service result into a state that arms a timer, +// emitting a host effect on the way, so both originate in RunService. +func fetchThenWaitMachine(fn state.ServiceFn[svcCtx]) *state.Machine[string, string, svcCtx] { + return state.ForgeFor[svcCtx]("fetchThenWait"). + Service("fetch", fn). + Action("notify", func(state.ActionCtx[svcCtx]) (state.Effect, error) { + return notifyEffect{To: "ops@example.com"}, nil + }). + State("idle"). + State("loading").Invoke("fetch", state.WithInvokeOnDone("ok")). + State("waiting"). + State("done").Final(). + Initial("idle"). + Transition("idle").On("start").GoTo("loading"). + Transition("loading").On("ok").GoTo("waiting").Do("notify"). + Transition("waiting").After(5 * time.Second).On("elapsed").GoTo("done"). + Quench() +} + // TestService_ResultFromCanceledTurn_NotCommittedOnRecover proves a service result // whose settle is canceled with the turn's context, and so never reaches the live // instance, is not committed by recovery either. @@ -671,3 +690,39 @@ func (c *cancelingClock) Now() time.Time { } return c.Clock.Now() } + +// TestRunService_AssignSeesTheDecodedResult proves a typed result reaches Assign in the +// form a replay supplies, so the two can't differ. +func TestRunService_AssignSeesTheDecodedResult(t *testing.T) { + type user struct{ Email string } + ctx := context.Background() + var seen []any + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { return user{Email: "a@b.c"}, nil } + m := state.ForgeFor[svcCtx]("decoded"). + Service("fetch", fn). + Reducer("note", func(in state.AssignCtx[svcCtx]) svcCtx { + seen = append(seen, in.Event) + return in.Entity + }). + State("loading").Invoke("fetch", state.WithInvokeOnDone("done"), state.WithInvokeOnError("fail")). + State("ready").Final(). + State("failed").Final(). + Initial("loading"). + Transition("loading").On("done").GoTo("ready").Assign("note"). + Transition("loading").On("fail").GoTo("failed"). + Quench() + store := durable.NewMemStore() + id := durable.InstanceID("d-1") + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))} + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("loading")) + if err != nil { + t.Fatalf("Start: %v", err) + } + mustRunService(t, h, state.InvokeID("decoded", "loading", 0)) + if _, err = durable.Recover(ctx, m, store, id, opts...); err != nil { + t.Fatalf("Recover: %v", err) + } + if len(seen) != 2 || fmt.Sprintf("%T:%v", seen[0], seen[0]) != fmt.Sprintf("%T:%v", seen[1], seen[1]) { + t.Fatalf("live and replay saw %#v", seen) + } +} diff --git a/durable/store.go b/durable/store.go index 31b12a79..b57254f7 100644 --- a/durable/store.go +++ b/durable/store.go @@ -59,7 +59,7 @@ type Record struct { // Effects are the effects the step emitted, each with its deterministically // stamped EffectID, persisted before dispatch so a crash between persist and - // dispatch is recoverable: Resume re-dispatches, deduped by EffectID. + // dispatch is recoverable: Recover re-dispatches, deduped by EffectID. Effects []state.EffectEnvelope // Snapshot, when non-empty, is a full marshaled state.Snapshot captured at diff --git a/durable/timetravel.go b/durable/timetravel.go index 654a28e7..24a4eea1 100644 --- a/durable/timetravel.go +++ b/durable/timetravel.go @@ -257,7 +257,7 @@ func replayThrough[S comparable, E comparable, C any]( } case hasServiceEntry(rec): for _, entry := range serviceEntries(rec) { - if err := replayService(ctx, svc, entry); err != nil { + if _, err := replayService(ctx, svc, entry); err != nil { return fmt.Errorf("at step %d: %w", rec.Step, err) } } diff --git a/examples/fooddelivery/fooddelivery.go b/examples/fooddelivery/fooddelivery.go index f7b50b8b..9c005ae0 100644 --- a/examples/fooddelivery/fooddelivery.go +++ b/examples/fooddelivery/fooddelivery.go @@ -388,10 +388,17 @@ func settleReducer(in state.AssignCtx[Order]) Order { // onDone event's payload — the saga's compensation made observable. func recordRefund(in state.AssignCtx[Order]) Order { c := in.Entity - if amount, ok := in.Event.(int64); ok { - c.Refund = amount - c.Log = append(c.Log, fmt.Sprintf("refunded:%d", amount)) + var amount int64 + switch v := in.Event.(type) { + case int64: + amount = v + case float64: // a durable Runner delivers a service result JSON-decoded + amount = int64(v) + default: + return c } + c.Refund = amount + c.Log = append(c.Log, fmt.Sprintf("refunded:%d", amount)) return c } diff --git a/state/CHANGELOG.md b/state/CHANGELOG.md index e4894a80..562df93d 100644 --- a/state/CHANGELOG.md +++ b/state/CHANGELOG.md @@ -13,6 +13,8 @@ counts as an additive (minor) versus breaking (major) change. Use the ### Added +- **`ServiceRunner.WithResultTransform`** rewrites a service's outcome before `Tick` + settles it, so a host that records outcomes can give the live run what a replay will see. - **`FireEventData`** reports the payload a `WithEventData` option carries, so a host that records a Fire can persist it. It's additive, so the next release is v1.1.0. diff --git a/state/runner.go b/state/runner.go index 699ebeb7..1d8d8a29 100644 --- a/state/runner.go +++ b/state/runner.go @@ -97,6 +97,9 @@ type ServiceRunner[S comparable, E comparable, C any] struct { // transition's action. It is atomic so a concurrent reader sees a consistent // snapshot. lastResult atomic.Value // serviceOutcome + + // transform, when set, rewrites a ServiceFn's outcome before Tick settles it. + transform func(any, error) (any, error) } // serviceOutcome is the result/error pair a settled service exposes through @@ -217,6 +220,17 @@ func (r *ServiceRunner[S, E, C]) settle(ctx context.Context, id string, result a return res, true } +// WithResultTransform sets a function that rewrites a ServiceFn's outcome before Tick +// settles it, and returns the runner. A host that records outcomes uses it to give the +// live run the form a replay will see. LastResult and LastError then report the rewritten +// outcome. It also runs on an UnboundRefError, where only the returned error is used. It +// doesn't apply to SettleDone or SettleError, which take the outcome as given. Set it +// before the first Tick: the field isn't guarded. +func (r *ServiceRunner[S, E, C]) WithResultTransform(f func(result any, err error) (any, error)) *ServiceRunner[S, E, C] { + r.transform = f + return r +} + // Tick resolves and runs the in-flight service id against the bound registry, // settling it with the ServiceFn's result or error. It is the ServiceRunner's // advance verb — the host-driver counterpart of Scheduler.Tick and @@ -237,10 +251,17 @@ func (r *ServiceRunner[S, E, C]) Tick(ctx context.Context, id string) []FireResu } fn := r.resolve(rs.src.Name) if fn == nil { - res, _ := r.SettleError(ctx, id, &UnboundRefError{Kind: "service", Name: rs.src.Name}) + var err error = &UnboundRefError{Kind: "service", Name: rs.src.Name} + if r.transform != nil { + _, err = r.transform(nil, err) + } + res, _ := r.SettleError(ctx, id, err) return []FireResult[S]{res} } out, err := fn(ctx, ServiceCtx[C]{Entity: r.inst.entity, Params: rs.src.Params, Input: rs.input}) + if r.transform != nil { + out, err = r.transform(out, err) + } if err != nil { res, _ := r.SettleError(ctx, id, err) return []FireResult[S]{res} From faeac084e118ff503d6d468b62b1a4af90f5d947 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Thu, 5 Nov 2026 09:22:29 -0700 Subject: [PATCH 5/5] fix(durable): keep timers across recovery and skip idle ticks Start now arms the initial state's timers and carries them in the baseline. Recovery re-arms checkpointed timers before the tail replays, so one fired in the tail isn't missed. RunService records the clock read that arms a timer, and replay arms it. A tick that fires nothing records nothing, so a timer loop no longer grows the journal. Relates to #214. Signed-off-by: Ray Karnes --- durable/CHANGELOG.md | 15 + durable/checkpoint.go | 29 +- durable/checkpoint_internal_test.go | 38 +- durable/clock.go | 121 +++++- durable/clock_internal_test.go | 23 ++ durable/clock_test.go | 94 +++++ durable/doc.go | 4 +- durable/runner.go | 205 ++++++---- durable/service.go | 3 +- durable/service_test.go | 562 ++++++++++++++++++++++++++++ durable/store.go | 2 +- durable/timer_checkpoint_test.go | 539 ++++++++++++++++++++++++++ durable/timetravel.go | 53 ++- 13 files changed, 1545 insertions(+), 143 deletions(-) diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 9c3ce2b1..89e0bbf6 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -91,6 +91,21 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **A failed checkpoint no longer makes the next step conflict.** `Tick`, `RunService` and `DeliverToActor` now advance the next step as soon as the append lands. +- **Timers survive recovery.** `Start` now arms the initial state's timers, which it never + did, so a new instance starting in a timed state fires a timer it didn't before. It + carries them in the baseline; an instance started earlier keeps its initial timer + unarmed. Recovery re-arms checkpointed timers before the tail replays, so one fired in + the tail isn't missed, and a timer armed by a service result now replays. `StateAt` + re-arms baseline timers too, so its result changes when one fires in the window, and it + fails if a timer's event won't decode with the options passed. +- **A tick that fires nothing records nothing.** It used to append a Record every time, + so a timer loop grew the journal while a timer was pending. An idle `Tick` doesn't touch + the Store, so it can't report `ErrFenced` or a Store failure: a stale owner finds out on + its next real step. Replay still reads older journals, whose idle Records and misplaced + settle clock reads it aligns on. Two kinds of older journal can't be repaired, and can + recover differently or fail `Recover`: an instance that continued after an older + recovery missed a fired timer, and a checkpoint taken after a tick that followed a settle + or whose clock moved mid-tick, which can hold a wrong timer table. - **Effects reach the handler from `Start`, `Tick` and `RunService`.** They were only dispatched from `Fire`. `Start` applies the initial entry's effects after encoding the baseline and before writing it, and only if the entry succeeded. Those effects are diff --git a/durable/checkpoint.go b/durable/checkpoint.go index c65eb830..bec5eab9 100644 --- a/durable/checkpoint.go +++ b/durable/checkpoint.go @@ -23,10 +23,10 @@ import ( // The declared `after` Delay re-arms a fresh timer at now+Delay; a timer pending at // a checkpoint has already burned part of its delay, so the durable layer persists // each armed timer's absolute deadline alongside the checkpoint and, on recovery, -// re-arms the remainder (deadline − resumed now) rather than the full declared -// Delay. The deadline is itself a recorded clock read, so the re-armed timer fires -// at the same recorded instant regardless of the wall clock at recovery time, and -// the recovered run is byte-identical to one that never crashed. +// re-arms it at that deadline rather than for the full declared Delay. The deadline +// is itself a recorded clock read, so the re-armed timer fires at the same recorded +// instant regardless of the wall clock at recovery time, and the recovered run is +// byte-identical to one that never crashed. // pendingTimer is one armed delayed (`after`) timer the durable Handle tracks // alongside the kernel scheduler, so its absolute deadline can be persisted at a @@ -40,7 +40,7 @@ type pendingTimer struct { // scheduler's pending set key identically. id string // due is the timer's absolute fire deadline (the recorded arming clock read plus - // the declared Delay), the value persisted so recovery re-arms the remainder. + // the declared Delay), the value persisted so recovery re-arms it at that deadline. due time.Time // event is the delayed event the timer re-fires through Fire when it elapses, // type-erased for the abstract ScheduleAfter surface and re-typed by the @@ -158,27 +158,20 @@ func mirrorArmed(armed map[string]pendingTimer, effects []state.Effect, now time } } -// reArmEffects builds the ScheduleAfter effects that re-arm persisted timers at -// their recorded absolute deadlines, relative to now: each timer's remaining delay -// is its deadline minus now, so absorbing the effects through the live scheduler -// reproduces the same absolute deadline the live run held. A deadline already at or -// before now yields a non-positive delay, which the scheduler arms as immediately -// due, so an elapsed-while-down timer fires on the next Tick. Timers whose id is -// already armed (re-established by tail replay) are skipped, so a timer the tail -// still carries is not double-armed. Returns the effects and the durable-side -// pendingTimer table to seed the Handle, both in scheduleID order. +// reArmEffects builds the ScheduleAfter effects that re-arm persisted timers at their +// recorded absolute deadlines, relative to now. now can be any fixed instant: the caller +// holds the scheduler's clock on it while absorbing the effects, so each timer's deadline +// comes out exact, and an elapsed one fires on the next Tick. Recovery calls it before +// the tail replays, because the tail can fire these timers. Returns the effects and the +// durable-side pending-timer table. func reArmEffects[E comparable]( codec EventCodec[E], wires []pendingTimerWire, - already map[string]pendingTimer, now time.Time, ) ([]state.Effect, map[string]pendingTimer, error) { var effects []state.Effect seeded := map[string]pendingTimer{} for _, w := range wires { - if _, armed := already[w.ID]; armed { - continue // tail replay already re-armed this timer; do not double-arm. - } ev, err := decodeTimerEvent(codec, w.Event) if err != nil { return nil, nil, fmt.Errorf("durable: decoding pending timer %q event: %w", w.ID, err) diff --git a/durable/checkpoint_internal_test.go b/durable/checkpoint_internal_test.go index 5ae66b4f..45ea9cae 100644 --- a/durable/checkpoint_internal_test.go +++ b/durable/checkpoint_internal_test.go @@ -90,22 +90,19 @@ func TestUnmarshalCheckpoint_Garbage(t *testing.T) { } } -// TestReArmEffects_SkipsAlreadyArmed confirms a persisted timer already armed by -// tail replay is not re-armed (no double-arm), while an un-armed one is re-armed at -// its remaining delay. -func TestReArmEffects_SkipsAlreadyArmed(t *testing.T) { +// TestReArmEffects_RemainingDelay confirms a persisted timer is re-armed at its +// remaining delay. +func TestReArmEffects_RemainingDelay(t *testing.T) { now := time.Unix(0, 1_000_000_000).UTC() wires := []pendingTimerWire{ - {ID: "armed", DueUnixNano: now.Add(5 * time.Second).UnixNano(), Event: json.RawMessage(`"x"`)}, {ID: "compacted", DueUnixNano: now.Add(8 * time.Second).UnixNano(), Event: json.RawMessage(`"y"`)}, } - already := map[string]pendingTimer{"armed": {id: "armed"}} - effects, seeded, err := reArmEffects[string](jsonEventCodec[string]{}, wires, already, now) + effects, seeded, err := reArmEffects[string](jsonEventCodec[string]{}, wires, now) if err != nil { t.Fatalf("reArmEffects: %v", err) } if len(effects) != 1 || len(seeded) != 1 { - t.Fatalf("want exactly the compacted timer re-armed, got effects=%d seeded=%d", len(effects), len(seeded)) + t.Fatalf("want the timer re-armed, got effects=%d seeded=%d", len(effects), len(seeded)) } sa, ok := effects[0].(state.ScheduleAfter) if !ok || sa.ID != "compacted" { @@ -124,7 +121,7 @@ func TestReArmEffects_ElapsedDeadlineImmediate(t *testing.T) { wires := []pendingTimerWire{ {ID: "past", DueUnixNano: now.Add(-3 * time.Second).UnixNano(), Event: json.RawMessage(`"z"`)}, } - effects, _, err := reArmEffects[string](jsonEventCodec[string]{}, wires, map[string]pendingTimer{}, now) + effects, _, err := reArmEffects[string](jsonEventCodec[string]{}, wires, now) if err != nil { t.Fatalf("reArmEffects: %v", err) } @@ -142,7 +139,7 @@ func TestReArmEffects_BadEventErrors(t *testing.T) { wires := []pendingTimerWire{ {ID: "bad", DueUnixNano: 1, Event: json.RawMessage(`{not valid`)}, } - _, _, err := reArmEffects[string](jsonEventCodec[string]{}, wires, map[string]pendingTimer{}, time.Unix(0, 0)) + _, _, err := reArmEffects[string](jsonEventCodec[string]{}, wires, time.Unix(0, 0)) if err == nil { t.Fatal("want decode error for corrupt timer event, got nil") } @@ -160,18 +157,21 @@ func TestMarshalCheckpoint_BadEventErrors(t *testing.T) { } } -// TestTickInstant covers the no-clock-read branch (a tick that fired nothing -// records no read) alongside the happy path. -func TestTickInstant(t *testing.T) { - if _, ok := tickInstant(nil); ok { +// TestAbsorbRead confirms each result's instant comes from the tick's own reads, the last +// in the Record, past a read an older journal left at the front. +func TestAbsorbRead(t *testing.T) { + if _, ok := absorbRead(nil, 0, 1); ok { t.Fatal("empty entries should report no instant") } - entries := []state.JournalEntry{ - {Kind: state.JournalClockRead, ClockUnixNano: 42}, + read := func(n int64) state.JournalEntry { + return state.JournalEntry{Kind: state.JournalClockRead, ClockUnixNano: n} } - got, ok := tickInstant(entries) - if !ok || got.UnixNano() != 42 { - t.Fatalf("tickInstant: ok=%v got=%d", ok, got.UnixNano()) + entries := []state.JournalEntry{read(1), read(2), read(3), read(4), read(5), read(6)} + for i, want := range []int64{3, 5} { + got, ok := absorbRead(entries, i, 2) + if !ok || got.UnixNano() != want { + t.Fatalf("absorbRead(%d): ok=%v got=%d, want %d", i, ok, got.UnixNano(), want) + } } } diff --git a/durable/clock.go b/durable/clock.go index ad5081f4..b794cdac 100644 --- a/durable/clock.go +++ b/durable/clock.go @@ -31,9 +31,9 @@ import ( // step may absorb several effects. The recordingClock accumulates reads into a // shared buffer the Handle flushes into the Record.Entries of the step during // which they occurred (the step whose Fire/Tick produced them), in read order. -// Replay is order-driven, not step-keyed: the replayClock walks every recorded -// JournalClockRead across the loaded tail in recorded order and returns each -// reading once, so the scheduler re-derives identical deadlines. +// Replay walks every recorded JournalClockRead across the loaded tail in recorded +// order, aligning on each Record because a step's own readings are the last in it, so +// the scheduler re-derives identical deadlines. // recordingClock wraps a real state.Clock and records every Now() reading as a // JournalClockRead into a shared buffer, in read order. After delegates to the @@ -117,18 +117,20 @@ func (c *recordingClock) armReadingAt(i int) (time.Time, bool) { // readings and returns each once. // // When the cursor is exhausted — a Now() beyond the recorded readings, which -// happens on the live re-arm after replay — it falls through to a real clock so +// happens on live continuation after replay — it falls through to a real clock so // post-recovery timing continues against wall time. The fallthrough never affects // the replayed prefix, which is fully determined by the journal. type replayClock struct { mu sync.Mutex readings []int64 cursor int + held *time.Time // a fixed reading set by hold, nil otherwise + last time.Time // the last reading Now returned, see lastRead fallback state.Clock } // newReplayClock builds a replayClock over the recorded readings, falling through -// to fallback once the readings are exhausted (the live re-arm after replay). +// to fallback once the readings are exhausted (live continuation after replay). func newReplayClock(readings []int64, fallback state.Clock) *replayClock { return &replayClock{readings: readings, fallback: fallback} } @@ -138,12 +140,115 @@ func newReplayClock(readings []int64, fallback state.Clock) *replayClock { func (c *replayClock) Now() time.Time { c.mu.Lock() defer c.mu.Unlock() + if c.held != nil { + return *c.held + } if c.cursor < len(c.readings) { - t := time.Unix(0, c.readings[c.cursor]).UTC() + c.last = time.Unix(0, c.readings[c.cursor]).UTC() c.cursor++ - return t + return c.last + } + c.last = c.fallback.Now() + return c.last +} + +// skipTo moves the cursor forward to reading n and never back. Replay aligns on each +// Record this way, because a step's own readings are the last ones in its Record: an +// older journal left a settle's reading at the front of the next Record, which replay +// either consumed at the settle or must pass over. +func (c *replayClock) skipTo(n int) { + c.mu.Lock() + defer c.mu.Unlock() + if c.cursor < n { + c.cursor = n + } +} + +// lastRead returns the instant the last Now returned outside a hold: the one a +// Scheduler.Absorb just armed its timers at. +func (c *replayClock) lastRead() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.last +} + +// readingAt returns reading i without moving the cursor. +func (c *replayClock) readingAt(i int) (time.Time, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if i < 0 || i >= len(c.readings) { + return time.Time{}, false + } + return time.Unix(0, c.readings[i]).UTC(), true +} + +// settleClock aligns a replayed settle's Absorb. A settle's own reading ends its Record. +// An older journal's settle Record holds none: its reading opens the next Record, if the +// process that took the settle lived to take the next step. If not, that Record holds +// only its own readings, so a settle must not consume them. It peeks instead, and each +// Record still aligns from its end. +type settleClock struct { + clock *replayClock + older int // read-less settles since the last Record that held a reading +} + +// next records that replay reached rec. +func (s *settleClock) next(rec *Record) { + if clockReadCount(rec) > 0 { + s.older = 0 + } +} + +// absorb runs absorb for a settle's timers at the reading they were armed at, and returns +// that instant. lost is true when the journal holds no such reading, so the fallback +// clock supplied it, and the caller should carry that read into the next step's Record. +func (s *settleClock) absorb(rec *Record, end int, absorb func()) (at time.Time, lost bool) { + if clockReadCount(rec) > 0 { + s.clock.skipTo(end - 1) + absorb() + return s.clock.lastRead(), false } - return c.fallback.Now() + if t, ok := s.clock.readingAt(end + s.older); ok { + s.older++ + s.clock.hold(t) + absorb() + s.clock.release() + return t, false + } + t := s.clock.fallback.Now() + s.clock.hold(t) + absorb() + s.clock.release() + return t, true +} + +// tickReads returns how many clock readings a tick that fired n timers takes: one to pick +// each timer, one to absorb its effects, and a last read that finds nothing due. +func tickReads(n int) int { return 2*n + 1 } + +// clockReadCount returns how many clock readings rec holds. +func clockReadCount(rec *Record) int { + n := 0 + for _, e := range rec.Entries { + if e.Kind == state.JournalClockRead { + n++ + } + } + return n +} + +// hold makes Now return t without consuming a reading, until release. +func (c *replayClock) hold(t time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + c.held = &t +} + +// release ends a hold. +func (c *replayClock) release() { + c.mu.Lock() + defer c.mu.Unlock() + c.held = nil } // After delegates to the fallback clock; replay drives elapses through Now + Tick diff --git a/durable/clock_internal_test.go b/durable/clock_internal_test.go index a0a01241..827c27c3 100644 --- a/durable/clock_internal_test.go +++ b/durable/clock_internal_test.go @@ -167,3 +167,26 @@ func TestHasTimerEffect(t *testing.T) { t.Fatal("no effects must not count as a timer effect") } } + +// TestSettleClock_ReadLessSettlesPeekWithoutConsuming confirms read-less settles take the +// readings after their Record in order and leave the cursor, so the next Record can still +// read its own, and that a settle past the last reading reports it lost. +func TestSettleClock_ReadLessSettlesPeekWithoutConsuming(t *testing.T) { + at := func(n int64) time.Time { return time.Unix(0, n).UTC() } + c := newReplayClock([]int64{10, 20}, state.NewFakeClock(clockEpoch)) + s := settleClock{clock: c} + rec := &Record{} + for _, want := range []int64{10, 20} { + got, lost := s.absorb(rec, 0, func() { c.Now() }) + if lost || !got.Equal(at(want)) { + t.Fatalf("absorb: got %v lost %v, want %d", got, lost, want) + } + } + got, lost := s.absorb(rec, 0, func() { c.Now() }) + if !lost || !got.Equal(clockEpoch) { + t.Fatalf("a settle past the last reading: got %v lost %v, want the fallback clock", got, lost) + } + if c.cursor != 0 { + t.Fatalf("cursor at %d, want 0: a settle must leave the next Record's readings", c.cursor) + } +} diff --git a/durable/clock_test.go b/durable/clock_test.go index 85d24db4..ab339ac5 100644 --- a/durable/clock_test.go +++ b/durable/clock_test.go @@ -414,3 +414,97 @@ func recoverTimerBytes(t *testing.T, ctx context.Context, m *state.Machine[strin } return string(b) } + +// TestClock_StartInTimedState_ArmsTimer proves Start arms the `after` timer of the +// state an instance starts in, as a host absorbing InitialEffects does, and that recovery +// reaches the same state. +func TestClock_StartInTimedState_ArmsTimer(t *testing.T) { + ctx := context.Background() + m := timerMachine() + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *timerCtx]{durable.WithRunnerClock[string, string, *timerCtx](clk)} + runner := durable.NewRunner(m, store, opts...) + h, err := runner.Start(ctx, id, &timerCtx{}, state.WithInitialState("arming")) + if err != nil { + t.Fatalf("Start: %v", err) + } + clk.Advance(6 * time.Second) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + if got := h.Instance().Current(); got != "done" { + t.Fatalf("state %q 6s after starting in arming, want done", got) + } + + // A clock still at the start proves the timer replays at its recorded instant. + recovered, err := durable.Recover(ctx, m, store, id, durable.WithRunnerClock[string, string, *timerCtx](state.NewFakeClock(epoch))) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if live, got := handleSnapshot(t, h), handleSnapshot(t, recovered); got != live { + t.Fatalf("recovered instance differs from live:\n live: %s\n got: %s", live, got) + } +} + +// TestClock_TickWithNothingDue_AppendsNoRecord proves a timer loop ticking while a +// timer is pending but not yet due does not grow the journal. +func TestClock_TickWithNothingDue_AppendsNoRecord(t *testing.T) { + ctx := context.Background() + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + clk := state.NewFakeClock(epoch) + runner := durable.NewRunner(timerMachine(), store, durable.WithRunnerClock[string, string, *timerCtx](clk)) + h, err := runner.Start(ctx, id, &timerCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "arm"); err != nil { + t.Fatalf("Fire(arm): %v", err) + } + _, before, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + for range 100 { + clk.Advance(10 * time.Millisecond) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + } + _, after, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(after) != len(before) { + t.Fatalf("100 ticks before the 5s timer was due grew the journal from %d to %d records", len(before), len(after)) + } +} + +// TestClock_TickWithNoTimer_AppendsNoRecord proves a timer loop ticking an instance with no +// timer armed does not grow the journal. +func TestClock_TickWithNoTimer_AppendsNoRecord(t *testing.T) { + ctx := context.Background() + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + clk := state.NewFakeClock(epoch) + runner := durable.NewRunner(timerMachine(), store, durable.WithRunnerClock[string, string, *timerCtx](clk)) + h, err := runner.Start(ctx, id, &timerCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + for range 100 { + clk.Advance(10 * time.Millisecond) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + } + _, tail, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(tail) != 0 { + t.Fatalf("100 ticks with no timer armed appended %d records", len(tail)) + } +} diff --git a/durable/doc.go b/durable/doc.go index 99ead028..d5b21ff8 100644 --- a/durable/doc.go +++ b/durable/doc.go @@ -92,8 +92,8 @@ // // Clock (WithRunnerClock): a running instance reads time only through its host // scheduler, which arms and ticks delayed `after` transitions. The Runner wraps -// the real clock in a recording clock on the live path so every Now() reading the -// scheduler consumes is journaled as a JournalClockRead in the step's Record. +// the real clock in a recording clock on the live path so the Now() readings of a step +// are journaled as JournalClockRead entries in its Record. An idle Tick records nothing. // On recovery a replay clock returns those readings in order, making timer-driven // transitions wall-clock-independent: the same timers fire at the same recorded // instants regardless of the wall clock at recovery time. A timer also survives diff --git a/durable/runner.go b/durable/runner.go index 826a7f53..42f8ff96 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -111,6 +111,7 @@ type Handle[S comparable, E comparable, C any] struct { sched *state.Scheduler[S, E, C] recClock *recordingClock // owns the clock-read accumulator and its mutex + carried []state.JournalEntry // a read recovery took for an older settle, for the next step's Record timers map[string]pendingTimer // armed `after` timers with absolute deadlines, persisted per checkpoint svc *state.ServiceRunner[S, E, C] // host driver for invoked services, nil when none wired @@ -144,7 +145,9 @@ func (h *Handle[S, E, C]) drainClock() []state.JournalEntry { if h.recClock == nil { return nil } - return h.recClock.drain() + out := append(h.carried, h.recClock.drain()...) + h.carried = nil + return out } // absorbDrivers feeds a transition's effects into every wired host driver so the @@ -221,10 +224,10 @@ func (h *Handle[S, E, C]) armNow(before int) time.Time { } // armTimersFrom mirrors the ScheduleAfter/CancelScheduled effects a chained timer -// transition emitted into the Handle's timer table at the tick instant now, so a -// timer a fired transition re-armed (the next link in an `after` chain) is tracked -// for the next checkpoint. It is the Tick-side analog of absorbTimers, sharing the -// tick's single recorded instant rather than reading the clock again. +// transition emitted into the Handle's timer table at now, the instant the scheduler +// armed them, so a timer a fired transition re-armed (the next link in an `after` chain) +// is tracked for the next checkpoint. It is the Tick-side analog of absorbTimers, using a +// recorded instant rather than reading the clock again. func (h *Handle[S, E, C]) armTimersFrom(effects []state.Effect, now time.Time) { for _, eff := range effects { switch e := eff.(type) { @@ -239,28 +242,32 @@ func (h *Handle[S, E, C]) armTimersFrom(effects []state.Effect, now time.Time) { } } -// tickInstant returns the single clock instant a tick's recorded reads share (the -// FakeClock does not advance within a tick; a SystemClock's intra-tick drift is -// sub-timer). It reads the first recorded JournalClockRead. ok is false when the -// tick recorded no clock read (it fired nothing), so the caller skips reconciliation. -func tickInstant(entries []state.JournalEntry) (time.Time, bool) { +// absorbRead returns the clock instant the scheduler armed result i's timers at, in a +// tick that fired n timers. A Scheduler.Tick reads the clock once to pick each due timer +// and once to absorb its effects, and a transition reads no clock, so result i's instant +// is read 2i+1 of the tick's own reads, the last tickReads(n) in entries. +func absorbRead(entries []state.JournalEntry, i, n int) (time.Time, bool) { + var reads []int64 for _, e := range entries { if e.Kind == state.JournalClockRead { - return time.Unix(0, e.ClockUnixNano).UTC(), true + reads = append(reads, e.ClockUnixNano) } } - return time.Time{}, false + at := len(reads) - tickReads(n) + 2*i + 1 + if at < 0 || at >= len(reads) { + return time.Time{}, false + } + return time.Unix(0, reads[at]).UTC(), true } -// dropFiredTimers removes from the Handle's timer table the timers a Tick fired, so -// a checkpoint taken after the tick does not persist an already-fired timer. The -// scheduler removes a due timer before re-firing it; the Handle mirrors that by -// dropping every timer whose deadline is at or before now, matching the scheduler's -// dueLocked selection. -func (h *Handle[S, E, C]) dropFiredTimers(now time.Time) { - for id, pt := range h.timers { - if !pt.due.After(now) { - delete(h.timers, id) +// dropFiredTimers removes from table every timer the scheduler no longer holds, so a +// checkpoint taken after a tick does not persist one that fired. It asks the scheduler +// rather than comparing deadlines, because a timer can come due partway through a tick, +// or arm and fire within one. +func dropFiredTimers[S comparable, E comparable, C any](table map[string]pendingTimer, sched *state.Scheduler[S, E, C]) { + for id := range table { + if !sched.HasPending(id) { + delete(table, id) } } } @@ -282,6 +289,9 @@ func (h *Handle[S, E, C]) ID() InstanceID { return h.id } // InstanceID that already exists in the Store reports ErrInstanceExists rather // than clobbering its recorded baseline. // +// Start arms the initial state's `after` timers and carries them in the baseline, so a +// Tick fires them and recovery re-arms them. +// // Start applies the initial entry's domain effects before the baseline write, and only // if the entry succeeded: a failed entry still returns a Handle and a nil error, and // Instance().InitialErr reports it. They're at-least-once, unlike a step's: a handler failure or a @@ -309,6 +319,20 @@ func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opt castOpts := append([]state.CastOption[S]{state.WithClock[S](recClock), state.WithUnboundedHistory[S]()}, opts...) inst := r.machine.Cast(input, castOpts...) + // Arm the timers of the state the instance starts in, and carry their deadlines in + // the baseline, which is what a recovery re-arms them from. The arming clock read + // belongs to no step, so it doesn't reach the first Record. + initial := inst.InitialEffects() + sched := state.NewScheduler(inst) + timers := map[string]pendingTimer{} + if hasTimerEffect(initial) { + sched.Absorb(ctx, initial) + reads := recClock.drain() + if n := len(reads); n > 0 { + mirrorArmed(timers, initial, time.Unix(0, reads[n-1].ClockUnixNano).UTC()) + } + } + // Encode a baseline checkpoint at baselineStep (below the first fired step), so the // instance is reconstructable from the Store before any event is fired. The baseline // is the cast snapshot; the tail then accumulates fired steps on top of it. Encoding @@ -317,9 +341,9 @@ func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opt if err != nil { return nil, fmt.Errorf("durable: marshaling start snapshot for %q: %w", id, err) } - // Wrap the baseline snapshot in a checkpoint envelope (no timers armed yet) so - // every checkpoint a recover loads carries the same durable envelope shape. - baseEnv, err := marshalCheckpoint(snap, nil, 0) + // Wrap the baseline snapshot in a checkpoint envelope so every checkpoint a recover + // loads carries the same durable envelope shape. + baseEnv, err := marshalCheckpoint(snap, timers, 0) if err != nil { return nil, fmt.Errorf("durable: marshaling start checkpoint envelope for %q: %w", id, err) } @@ -328,7 +352,7 @@ func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opt // re-derive these effects after the baseline, so they're at-least-once. An entry that // failed applies none, as a failed Fire dispatches none. if inst.InitialErr() == nil { - if err := r.applyStartEffects(ctx, id, dispatchableEffects(baselineStep, inst.InitialEffects())); err != nil { + if err := r.applyStartEffects(ctx, id, dispatchableEffects(baselineStep, initial)); err != nil { return nil, err } } @@ -364,9 +388,9 @@ func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opt id: id, inst: inst, nextStep: 0, - sched: state.NewScheduler(inst), + sched: sched, recClock: recClock, - timers: map[string]pendingTimer{}, + timers: timers, svc: svc, svcBuf: &svcBuf, actors: actors, @@ -469,9 +493,13 @@ func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireO // due order, re-firing the delayed events through the durable instance and // recording the clock readings the tick consumed. A host calls it from its own // timer loop (with a real clock) or a test calls it after advancing a fake clock. -// It records one tick barrier — carrying the consumed clock readings and the -// count of timers fired — so recovery re-derives the same timers at their -// recorded instants. It returns the FireResults of the timers it fired, in order. +// When a timer fired it records one tick barrier — carrying the consumed clock readings +// and the count of timers fired — so recovery re-derives the same timers at their +// recorded instants. A tick that fires nothing records nothing. It returns the +// FireResults of the timers it fired, in order. A timer whose transition fails is +// consumed and reported only in its FireResult's Err, not in Tick's error. Tick applies +// the step's domain effects: a failed dispatch, or a KindedEffect that won't marshal, +// fails the Handle. func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], error) { h.mu.Lock() defer h.mu.Unlock() @@ -495,25 +523,25 @@ func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], erro } } + // A tick that fired nothing changes nothing, so it records nothing, and its clock + // read goes with it: replay runs no tick for it, so it consumes no recorded reading. + if len(results) == 0 { + if h.recClock != nil { + h.recClock.drain() + } + return results, nil + } step := h.nextStep rec := Record{Step: step, Tick: true, TickSteps: len(results), Entries: h.drainClock()} // Reconcile the Handle's timer table with the tick: drop the timers it fired and // arm any a chained transition scheduled, so a checkpoint after the tick persists - // exactly the still-pending deadlines. The clock does not advance within one tick - // (a FakeClock advances only between ticks; a SystemClock's intra-tick drift is - // sub-timer), so the tick's recorded reads share one instant: use it for both the - // drop (timers due at or before it fired) and any chained re-arm. - if now, ok := tickInstant(rec.Entries); ok { - h.dropFiredTimers(now) - for i := range results { + // exactly the still-pending deadlines. + for i := range results { + if now, ok := absorbRead(rec.Entries, i, len(results)); ok { h.armTimersFrom(results[i].Effects, now) } } - // A tick that read no clock and fired nothing records nothing: there is no - // nondeterminism to journal and no step was produced. - if len(rec.Entries) == 0 && rec.TickSteps == 0 { - return results, nil - } + dropFiredTimers(h.timers, h.sched) var effects []state.Effect for i := range results { effects = append(effects, results[i].Effects...) @@ -667,12 +695,32 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S actorBuf := make([]state.JournalEntry, 0) actors := r.newActorSystem(inst) - // armed mirrors the scheduler's pending timers re-established by tail replay, - // with their recorded deadlines, so a persisted-but-compacted timer is re-armed - // only when the tail did not already re-arm it, and the Handle's table is seeded - // with every still-pending timer's deadline for the next checkpoint. + // armed mirrors the scheduler's pending timers with their recorded deadlines. It starts + // from the timers the checkpoint carries, tail replay updates it, and it seeds the + // Handle's table for the next checkpoint. armed := map[string]pendingTimer{} + // Re-arm the timers pending at the checkpoint before the tail replays, because the + // tail can fire them. Their deadlines are absolute, so any fixed instant works as the + // arming clock read, and holding the replay clock on one keeps this read from + // consuming a recorded reading. + const armedAt = 0 + reEffects, reSeeded, reErr := reArmEffects(r.cfg.eventCodec, persistedTimers, time.Unix(armedAt, 0).UTC()) + if reErr != nil { + return nil, fmt.Errorf("durable: re-arming pending timers for %q: %w", id, reErr) + } + if len(reEffects) > 0 { + repClock.hold(time.Unix(armedAt, 0).UTC()) + sched.Absorb(ctx, reEffects) + repClock.release() + } + for id, pt := range reSeeded { + armed[id] = pt + } + + reads := 0 // clock readings through the current Record + settles := settleClock{clock: repClock} + carry := false for i := range tail { // A canceled context can stop a replayed settle or tick with no error, leaving // state short of the journal, so check it before each Record. @@ -680,6 +728,8 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S return nil, fmt.Errorf("durable: recovering %q: %w", id, err) } rec := &tail[i] + reads += clockReadCount(rec) + settles.next(rec) switch { case hasActorEntry(rec): // Re-fire each recorded parent transition the live delivery drove, in @@ -705,26 +755,32 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S } settled = append(settled, effects...) } + // Arm the timers the settle scheduled, as the live RunService did. + if hasTimerEffect(settled) { + at, lost := settles.absorb(rec, reads, func() { sched.Absorb(ctx, settled) }) + mirrorArmed(armed, settled, at) + carry = carry || lost + } if err := r.dispatchReplayEffects(ctx, id, rec.Step, settled); err != nil { return nil, err } + case rec.Tick && rec.TickSteps == 0: + // Only an older journal holds a tick that fired nothing. Don't tick: a timer + // re-armed from a checkpoint could be due by now, and the Record says otherwise. case rec.Tick: // Re-derive the timers the live tick fired: ticking the replay clock // returns the recorded readings, so the same deadlines come due and the // same timers fire, at their recorded instants. Mirror the tick into the // armed table — drop the timers it fired, arm any a chained transition // re-scheduled — so the deadline table tracks the still-pending set. + repClock.skipTo(reads - tickReads(rec.TickSteps)) tickRes := sched.Tick(ctx) - if now, ok := tickInstant(rec.Entries); ok { - for id, pt := range armed { - if !pt.due.After(now) { - delete(armed, id) - } - } - for ri := range tickRes { + for ri := range tickRes { + if now, ok := absorbRead(rec.Entries, ri, rec.TickSteps); ok { mirrorArmed(armed, tickRes[ri].Effects, now) } } + dropFiredTimers(armed, sched) var ticked []state.Effect for ri := range tickRes { ticked = append(ticked, tickRes[ri].Effects...) @@ -753,13 +809,11 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S // the recorded clock read the Absorb consumed (captured at the recorded // step rather than the live wall clock). if hasTimerEffect(res.Effects) { + repClock.skipTo(reads - 1) sched.Absorb(ctx, res.Effects) - // The step's recorded clock read (the one the live Absorb consumed) is - // the instant to derive each re-armed deadline from, so the mirror uses - // recorded time, not the live wall clock. - if now, ok := tickInstant(rec.Entries); ok { - mirrorArmed(armed, res.Effects, now) - } + // The recorded read the Absorb consumed is the instant to derive each + // deadline from, so the mirror uses recorded time, not the wall clock. + mirrorArmed(armed, res.Effects, repClock.lastRead()) } // Arm this step's services so a service it entered is in flight for the // recorded settlement that follows, mirroring the live Fire path. @@ -778,7 +832,15 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S // Drain any clock readings the replay re-recorded after the cursor exhausted // (none in the common case where replay consumes exactly the recorded - // readings): they belong to no new step and must not leak into the next Fire. + // readings): they belong to no new step and must not leak into the next Fire. The + // exception is a read an older settle lost: the next step's Record carries it, so a + // later replay arms that settle's timers at the same instant. Skip the cursor to the + // end, so no reading left in the tail reaches the live clock. + repClock.skipTo(reads) + var carried []state.JournalEntry + if carry { + carried = append(carried, buf...) + } buf = buf[:0] // Arm the live actors at the resume point from the final configuration's @@ -791,28 +853,6 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S actors.Absorb(ctx, inst.StartEffects()) } - // Re-arm any timer that was pending at the checkpoint but whose arming - // ScheduleAfter was compacted out of the tail, so it survives compaction: the - // tail replay could not re-arm it (its arm is gone), so re-arm it here at its - // persisted absolute deadline. Already-armed timers (re-established by the tail) - // are skipped, so a timer the tail still carried is never double-armed. The - // remaining delay is the persisted deadline minus the recovery clock's now, so - // the re-armed timer fires at its recorded instant — an already-elapsed deadline - // arms as immediately due and fires on the next Tick. - reEffects, reSeeded, reErr := reArmEffects(r.cfg.eventCodec, persistedTimers, armed, repClock.Now()) - if reErr != nil { - return nil, fmt.Errorf("durable: re-arming pending timers for %q: %w", id, reErr) - } - if len(reEffects) > 0 { - sched.Absorb(ctx, reEffects) - } - for id, pt := range reSeeded { - armed[id] = pt - } - // The re-arm reads the recovery clock to compute remaining delays; that reading - // belongs to no recorded step and must not leak into the next Fire's Record. - buf = buf[:0] - // The next step ordinal continues past the highest assigned step. Step ordinals // are assigned monotonically: each Record consumes one ordinal regardless of how // many parent Traces it produced (an actor delivery can drive several), except a @@ -840,6 +880,7 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S nextStep: next, sched: sched, recClock: recClock, + carried: carried, timers: armed, svc: svc, svcBuf: &svcBuf, diff --git a/durable/service.go b/durable/service.go index 6cbf4308..170602e4 100644 --- a/durable/service.go +++ b/durable/service.go @@ -144,7 +144,8 @@ func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.Fire *h.svcBuf = append(*h.svcBuf, entry) } - rec := Record{Step: h.nextStep, Entries: h.drainServices()} + // The clock read belongs to this step when the settle armed a timer. + rec := Record{Step: h.nextStep, Entries: append(h.drainServices(), h.drainClock()...)} if err := h.commitStep(ctx, &rec, res.Effects); err != nil { return res, true, err } diff --git a/durable/service_test.go b/durable/service_test.go index 9d9223b1..7d7387c0 100644 --- a/durable/service_test.go +++ b/durable/service_test.go @@ -487,10 +487,572 @@ func fetchThenWaitMachine(fn state.ServiceFn[svcCtx]) *state.Machine[string, str Initial("idle"). Transition("idle").On("start").GoTo("loading"). Transition("loading").On("ok").GoTo("waiting").Do("notify"). + Transition("waiting").On("poke").GoTo("waiting"). Transition("waiting").After(5 * time.Second).On("elapsed").GoTo("done"). Quench() } +// TestService_TimerArmedBySettle_ReplaysOnRecover proves a timer armed by a service +// result survives recovery, with an idle tick between the settle and the tick that +// fires it. +func TestService_TimerArmedBySettle_ReplaysOnRecover(t *testing.T) { + ctx := context.Background() + var calls int64 + fn := counterService(&calls) + m := fetchThenWaitMachine(fn) + store := durable.NewMemStore() + id := durable.InstanceID("fetch-1") + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, svcCtx]{ + durable.WithRunnerClock[string, string, svcCtx](clk), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + } + runner := durable.NewRunner(m, store, opts...) + h, err := runner.Start(ctx, id, svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + clk.Advance(time.Second) // the settle's read differs from the earlier ones + mustRunService(t, h, state.InvokeID("fetchThenWait", "loading", 0)) + for _, d := range []time.Duration{2 * time.Second, 4 * time.Second} { // idle, then due + clk.Advance(d) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + } + if got := h.Instance().Current(); got != "done" { + t.Fatalf("live state %q, want done", got) + } + + // A clock still at the start proves the timer replays at its recorded instant. + recovered, err := durable.Recover(ctx, m, store, id, append(opts, durable.WithRunnerClock[string, string, svcCtx](state.NewFakeClock(epoch)))...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if live, got := handleSnapshot(t, h), handleSnapshot(t, recovered); got != live { + t.Fatalf("recovered instance differs from live:\n live: %s\n got: %s", live, got) + } + if got := atomic.LoadInt64(&calls); got != 1 { + t.Fatalf("service ran %d times, want 1: recovery replays the recorded result", got) + } +} + +// TestStateAt_TimerArmedBySettle_Fires proves a historical read arms a timer from a +// recorded service result, so it fires in the replayed window as it did live. +func TestStateAt_TimerArmedBySettle_Fires(t *testing.T) { + ctx := context.Background() + var calls int64 + fn := counterService(&calls) + m := fetchThenWaitMachine(fn) + store := durable.NewMemStore(durable.WithHistory()) + id := durable.InstanceID("fetch-1") + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, svcCtx]{ + durable.WithRunnerClock[string, string, svcCtx](clk), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + } + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + mustRunService(t, h, state.InvokeID("fetchThenWait", "loading", 0)) + clk.Advance(6 * time.Second) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + view, err := durable.StateAt(ctx, m, store, id, 2, opts...) // the tick's barrier + if err != nil { + t.Fatalf("StateAt: %v", err) + } + if got := view.Instance().Current(); got != "done" { + t.Fatalf("StateAt state %q, want done", got) + } +} + +// TestRecover_TimerArmedBySettle_SurvivesALaterCheckpoint proves recovery keeps a timer +// armed by a replayed service result in the table the next checkpoint persists, so a +// second recovery from that checkpoint still fires it. +func TestRecover_TimerArmedBySettle_SurvivesALaterCheckpoint(t *testing.T) { + ctx := context.Background() + var calls int64 + fn := counterService(&calls) + m := fetchThenWaitMachine(fn) + store := durable.NewMemStore() + id := durable.InstanceID("fetch-1") + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, svcCtx]{ + durable.WithRunnerClock[string, string, svcCtx](clk), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + durable.WithCheckpointEvery[string, string, svcCtx](3), + } + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + mustRunService(t, h, state.InvokeID("fetchThenWait", "loading", 0)) + + first, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("first Recover: %v", err) + } + for range 3 { // cross a checkpoint boundary after the settle has replayed + if _, err = first.Fire(ctx, "poke"); err != nil { + t.Fatalf("Fire(poke): %v", err) + } + } + + second, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("second Recover: %v", err) + } + clk.Advance(6 * time.Second) + if _, err = second.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + if got := second.Instance().Current(); got != "done" { + t.Fatalf("state %q after the second recovery, want done", got) + } +} + +// fetchChainMachine arms a timer on a service result, then chains a second one. +func fetchChainMachine(fn state.ServiceFn[svcCtx]) *state.Machine[string, string, svcCtx] { + return state.ForgeFor[svcCtx]("fetchChain"). + Service("fetch", fn). + State("idle"). + State("loading").Invoke("fetch", state.WithInvokeOnDone("ok")). + State("waiting"). + State("next"). + State("done"). + Initial("idle"). + Transition("idle").On("start").GoTo("loading"). + Transition("loading").On("ok").GoTo("waiting"). + Transition("waiting").On("poke").GoTo("waiting"). + Transition("waiting").After(5 * time.Second).On("first").GoTo("next"). + Transition("next").After(5 * time.Second).On("second").GoTo("done"). + Quench() +} + +// olderSettle splits a settle Record as an older journal wrote it: the Record without its +// clock read, and that read, which the older code left for the next Record to carry. +func olderSettle(settle durable.Record) (durable.Record, state.JournalEntry) { + var read state.JournalEntry + var kept []state.JournalEntry + for _, e := range settle.Entries { + if e.Kind == state.JournalClockRead { + read = e + continue + } + kept = append(kept, e) + } + settle.Entries = kept + return settle, read +} + +// clockReadAt is a recorded clock read d after epoch. +func clockReadAt(d time.Duration) state.JournalEntry { + return state.JournalEntry{Kind: state.JournalClockRead, ClockUnixNano: epoch.Add(d).UnixNano()} +} + +// startFetchChain starts a fetchChainMachine and settles its service. +func startFetchChain( + t *testing.T, + m *state.Machine[string, string, svcCtx], + store durable.Store, + id durable.InstanceID, + opts []durable.Option[string, string, svcCtx], +) *durable.Handle[string, string, svcCtx] { + t.Helper() + ctx := context.Background() + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + mustRunService(t, h, state.InvokeID("fetchChain", "loading", 0)) + return h +} + +// appendAll appends recs to store, failing the test on any error. +func appendAll(t *testing.T, store durable.Store, id durable.InstanceID, recs ...durable.Record) { + t.Helper() + for _, rec := range recs { + if _, err := store.Append(context.Background(), id, rec); err != nil { + t.Fatalf("Append: %v", err) + } + } +} + +// TestRecover_OlderJournal_SettleReadInIdleTick_KeepsReplayClockAligned proves replay stays +// on the recorded readings when an older journal carries a settle's clock read in the idle +// tick Record after it. Replay consumes that read at the settle, so skipping the idle tick +// must not consume it again, or the last tick reads the live clock and fires a timer no +// Record holds. +func TestRecover_OlderJournal_SettleReadInIdleTick_KeepsReplayClockAligned(t *testing.T) { + ctx := context.Background() + var calls int64 + fn := counterService(&calls) + m := fetchChainMachine(fn) + clk := state.NewFakeClock(epoch) + id := durable.InstanceID("fetch-1") + opts := []durable.Option[string, string, svcCtx]{ + durable.WithRunnerClock[string, string, svcCtx](clk), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + } + store := durable.NewMemStore() + h := startFetchChain(t, m, store, id, opts) + clk.Advance(6 * time.Second) + if _, err := h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + _, tail, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + + settle, settleRead := olderSettle(tail[1]) + firstIdle := durable.Record{Step: 2, Tick: true, Entries: []state.JournalEntry{settleRead, clockReadAt(time.Second)}} + secondIdle := durable.Record{Step: 3, Tick: true, Entries: []state.JournalEntry{clockReadAt(2 * time.Second)}} + tick := tail[2] + tick.Step = 4 + older := durable.NewMemStore() + if _, err = durable.NewRunner(m, older, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("idle")); err != nil { + t.Fatalf("Start: %v", err) + } + appendAll(t, older, id, tail[0], settle, firstIdle, secondIdle, tick) + + clk.Advance(94 * time.Second) // the wall clock is far past every timer at recovery + recovered, err := durable.Recover(ctx, m, older, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got := recovered.Instance().Current(); got != "next" { + t.Fatalf("recovered state %q, want next", got) + } +} + +// TestRecover_OlderJournal_TickPassesAnEarlierSettleRead proves a tail tick fires its +// timer when an older journal left a read at the front of its Record that no replayed +// settle consumed, because a checkpoint compacted the settle away. +func TestRecover_OlderJournal_TickPassesAnEarlierSettleRead(t *testing.T) { + ctx := context.Background() + var calls int64 + fn := counterService(&calls) + m := fetchChainMachine(fn) + clk := state.NewFakeClock(epoch) + id := durable.InstanceID("fetch-1") + opts := []durable.Option[string, string, svcCtx]{ + durable.WithRunnerClock[string, string, svcCtx](clk), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + } + // One run takes the checkpoint at the settle; a second, with none, gives the tick Record. + checkpointed := durable.NewMemStore() + startFetchChain(t, m, checkpointed, id, append(opts[:2:2], durable.WithCheckpointEvery[string, string, svcCtx](2))) + snapshot, compacted, err := checkpointed.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(compacted) != 0 { + t.Fatalf("tail holds %d records, want the settle checkpointed", len(compacted)) + } + store := durable.NewMemStore() + h := startFetchChain(t, m, store, id, opts) + clk.Advance(6 * time.Second) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + _, tail, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + + tick := tail[2] + tick.Entries = append([]state.JournalEntry{clockReadAt(0)}, tick.Entries...) + older := durable.NewMemStore() + if _, err = durable.NewRunner(m, older, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("idle")); err != nil { + t.Fatalf("Start: %v", err) + } + appendAll(t, older, id, durable.Record{Step: 0}, durable.Record{Step: 1}) + if err = older.Checkpoint(ctx, id, snapshot, 1); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + appendAll(t, older, id, tick) + + recovered, err := durable.Recover(ctx, m, older, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got := recovered.Instance().Current(); got != "next" { + t.Fatalf("recovered state %q, want next", got) + } + view, err := durable.StateAt(ctx, m, older, id, tick.Step, opts...) + if err != nil { + t.Fatalf("StateAt: %v", err) + } + if got := view.Instance().Current(); got != "next" { + t.Fatalf("StateAt state %q, want next", got) + } +} + +// TestRecover_OlderJournal_SettleSkipsAnUnusedRead proves a replayed settle arms from its +// own reading, not a stale one an older journal left in an earlier Record that no Absorb +// consumed. +func TestRecover_OlderJournal_SettleSkipsAnUnusedRead(t *testing.T) { + ctx := context.Background() + var calls int64 + fn := counterService(&calls) + m := fetchChainMachine(fn) + clk := state.NewFakeClock(epoch) + id := durable.InstanceID("fetch-1") + opts := []durable.Option[string, string, svcCtx]{ + durable.WithRunnerClock[string, string, svcCtx](clk), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + } + store := durable.NewMemStore() + startFetchChain(t, m, store, id, opts) + _, tail, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + + fire := tail[0] + fire.Entries = append([]state.JournalEntry{clockReadAt(0)}, fire.Entries...) // unused + settle, _ := olderSettle(tail[1]) + older := durable.NewMemStore() + if _, err = durable.NewRunner(m, older, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("idle")); err != nil { + t.Fatalf("Start: %v", err) + } + appendAll(t, older, id, fire, settle, + durable.Record{Step: 2, Tick: true, Entries: []state.JournalEntry{clockReadAt(time.Second)}}) + + recovered, err := durable.Recover(ctx, m, older, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + clk.Advance(5*time.Second + 500*time.Millisecond) // the timer is due at 6s, from the settle's read + for range 2 { // a misaligned replay leaves a reading for the first + fired, err := recovered.Tick(ctx) + if err != nil { + t.Fatalf("Tick: %v", err) + } + if len(fired) != 0 { + t.Fatalf("Tick fired %d timers before the deadline, want 0", len(fired)) + } + } +} + +// TestRecover_OlderJournal_SettleReadLostToARestart_LeavesTheNextStepsReads proves a +// read-less settle doesn't consume the next Record's reading when a restart lost its own. +// That reading stands in for the lost one, and the tick after it still fires what it +// recorded. +func TestRecover_OlderJournal_SettleReadLostToARestart_LeavesTheNextStepsReads(t *testing.T) { + ctx := context.Background() + var calls int64 + fn := counterService(&calls) + m := fetchChainMachine(fn) + clk := state.NewFakeClock(epoch) + id := durable.InstanceID("fetch-1") + opts := []durable.Option[string, string, svcCtx]{ + durable.WithRunnerClock[string, string, svcCtx](clk), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + } + store := durable.NewMemStore() + h := startFetchChain(t, m, store, id, opts) + clk.Advance(6 * time.Second) + if _, err := h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + _, tail, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + + settle, _ := olderSettle(tail[1]) // the restart lost its read + older := durable.NewMemStore() + if _, err = durable.NewRunner(m, older, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("idle")); err != nil { + t.Fatalf("Start: %v", err) + } + tick := tail[2] + tick.Step = 3 + appendAll(t, older, id, tail[0], settle, + durable.Record{Step: 2, Tick: true, Entries: []state.JournalEntry{clockReadAt(time.Second)}}, tick) + + recovered, err := durable.Recover(ctx, m, older, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got := recovered.Instance().Current(); got != "next" { + t.Fatalf("recovered state %q, want next", got) + } + view, err := durable.StateAt(ctx, m, older, id, tick.Step, opts...) + if err != nil { + t.Fatalf("StateAt: %v", err) + } + if got := view.Instance().Current(); got != "next" { + t.Fatalf("StateAt state %q, want next", got) + } +} + +// TestRecover_OlderJournal_TailEndingOnASettle_CarriesItsRead proves recovery keeps the +// read it took for a settle that ended an older journal: the next step's Record carries +// it, so a later recovery arms that timer at the same instant. +func TestRecover_OlderJournal_TailEndingOnASettle_CarriesItsRead(t *testing.T) { + ctx := context.Background() + var calls int64 + fn := counterService(&calls) + m := fetchChainMachine(fn) + clk := state.NewFakeClock(epoch) + id := durable.InstanceID("fetch-1") + opts := []durable.Option[string, string, svcCtx]{ + durable.WithRunnerClock[string, string, svcCtx](clk), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + } + store := durable.NewMemStore() + startFetchChain(t, m, store, id, opts) + _, tail, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + settle, _ := olderSettle(tail[1]) + older := durable.NewMemStore() + if _, err = durable.NewRunner(m, older, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("idle")); err != nil { + t.Fatalf("Start: %v", err) + } + appendAll(t, older, id, tail[0], settle) + + clk.Advance(3 * time.Second) + first, err := durable.Recover(ctx, m, older, id, opts...) + if err != nil { + t.Fatalf("first Recover: %v", err) + } + clk.Advance(6 * time.Second) // due at 8s, from the recovery's read + if _, err = first.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + if got := first.Instance().Current(); got != "next" { + t.Fatalf("live state %q, want next", got) + } + second, err := durable.Recover(ctx, m, older, id, opts...) + if err != nil { + t.Fatalf("second Recover: %v", err) + } + if got := second.Instance().Current(); got != "next" { + t.Fatalf("recovered state %q, want next", got) + } +} + +// TestRecover_OlderJournal_LeftoverReadStaysOutOfTheLiveClock proves a reading no step +// consumed at the end of a tail doesn't reach the live clock, so a timer armed after +// recovery measures from the real time. +func TestRecover_OlderJournal_LeftoverReadStaysOutOfTheLiveClock(t *testing.T) { + ctx := context.Background() + var calls int64 + fn := counterService(&calls) + m := fetchChainMachine(fn) + clk := state.NewFakeClock(epoch) + id := durable.InstanceID("fetch-1") + opts := []durable.Option[string, string, svcCtx]{ + durable.WithRunnerClock[string, string, svcCtx](clk), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + } + store := durable.NewMemStore() + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire: %v", err) + } + _, tail, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + fire := tail[0] + fire.Entries = append([]state.JournalEntry{clockReadAt(0)}, fire.Entries...) // no Absorb used it + older := durable.NewMemStore() + if _, err = durable.NewRunner(m, older, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("idle")); err != nil { + t.Fatalf("Start: %v", err) + } + appendAll(t, older, id, fire) + + clk.Advance(94 * time.Second) + recovered, err := durable.Recover(ctx, m, older, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + mustRunService(t, recovered, state.InvokeID("fetchChain", "loading", 0)) // arms 5s, due at 99s + clk.Advance(time.Second) + fired, err := recovered.Tick(ctx) + if err != nil { + t.Fatalf("Tick: %v", err) + } + if len(fired) != 0 { + t.Fatalf("Tick fired %d timers before the deadline, want 0", len(fired)) + } +} + +// TestRecover_OlderJournal_SettleTimerReachesTheNextCheckpoint proves a timer armed by an +// older settle Record, which holds no clock read, still reaches the Handle's table, so the +// next checkpoint persists it and a second recovery fires it. +func TestRecover_OlderJournal_SettleTimerReachesTheNextCheckpoint(t *testing.T) { + ctx := context.Background() + var calls int64 + fn := counterService(&calls) + m := fetchChainMachine(fn) + clk := state.NewFakeClock(epoch) + id := durable.InstanceID("fetch-1") + opts := []durable.Option[string, string, svcCtx]{ + durable.WithRunnerClock[string, string, svcCtx](clk), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + durable.WithCheckpointEvery[string, string, svcCtx](6), + } + store := durable.NewMemStore() + startFetchChain(t, m, store, id, opts) + _, tail, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + + settle, settleRead := olderSettle(tail[1]) + older := durable.NewMemStore() + if _, err = durable.NewRunner(m, older, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("idle")); err != nil { + t.Fatalf("Start: %v", err) + } + appendAll(t, older, id, tail[0], settle, + durable.Record{Step: 2, Tick: true, Entries: []state.JournalEntry{settleRead, clockReadAt(time.Second)}}) + + first, err := durable.Recover(ctx, m, older, id, opts...) + if err != nil { + t.Fatalf("first Recover: %v", err) + } + for range 3 { // cross a checkpoint boundary after the settle has replayed + if _, err = first.Fire(ctx, "poke"); err != nil { + t.Fatalf("Fire(poke): %v", err) + } + } + second, err := durable.Recover(ctx, m, older, id, opts...) + if err != nil { + t.Fatalf("second Recover: %v", err) + } + clk.Advance(6 * time.Second) + if _, err = second.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + if got := second.Instance().Current(); got != "next" { + t.Fatalf("state %q after the second recovery, want next", got) + } +} + // TestService_ResultFromCanceledTurn_NotCommittedOnRecover proves a service result // whose settle is canceled with the turn's context, and so never reaches the live // instance, is not committed by recovery either. diff --git a/durable/store.go b/durable/store.go index b57254f7..cbeda795 100644 --- a/durable/store.go +++ b/durable/store.go @@ -80,7 +80,7 @@ type Record struct { Tick bool // TickSteps is the number of timer transitions the tick at this barrier fired - // (zero for a tick that found nothing due). It lets replay advance its step + // (zero only in a journal from before idle ticks stopped recording). It lets replay advance its step // accounting across a re-derived tick without a recorded Event per timer. TickSteps int } diff --git a/durable/timer_checkpoint_test.go b/durable/timer_checkpoint_test.go index 3834a216..dee74767 100644 --- a/durable/timer_checkpoint_test.go +++ b/durable/timer_checkpoint_test.go @@ -381,3 +381,542 @@ func TestTimerCheckpoint_RecoveryDeterministic(t *testing.T) { t.Fatalf("recovery across checkpoint nondeterministic\n first: %s\n second: %s", first, second) } } + +// TestTimerCheckpoint_FiredAfterCheckpoint_ByteIdentical proves a timer persisted by +// a checkpoint and fired later in the tail, after an idle tick, replays at its +// recorded instant instead of being missed. +func TestTimerCheckpoint_FiredAfterCheckpoint_ByteIdentical(t *testing.T) { + ctx := context.Background() + m := pendingTimerMachine() + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *timerCtx]{ + durable.WithRunnerClock[string, string, *timerCtx](clk), + durable.WithCheckpointEvery[string, string, *timerCtx](3), + } + runner := durable.NewRunner(m, store, opts...) + h, err := runner.Start(ctx, id, &timerCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + for _, ev := range []string{"arm", "poke", "poke"} { // checkpoint lands with the timer pending + if _, err = h.Fire(ctx, ev); err != nil { + t.Fatalf("Fire(%s): %v", ev, err) + } + } + _, tail, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(tail) != 0 { + t.Fatalf("tail holds %d records, want the checkpoint to have compacted them", len(tail)) + } + for _, d := range []time.Duration{5 * time.Second, 6 * time.Second} { // idle, then due + clk.Advance(d) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + } + + if got := h.Instance().Current(); got != "done" { + t.Fatalf("live state %q, want done", got) + } + + // A clock still at the start proves the timer replays at its recorded instant. + recovered, err := durable.Recover(ctx, m, store, id, append(opts, durable.WithRunnerClock[string, string, *timerCtx](state.NewFakeClock(epoch)))...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if live, got := handleSnapshot(t, h), handleSnapshot(t, recovered); got != live { + t.Fatalf("recovered instance differs from live:\n live: %s\n got: %s", live, got) + } +} + +// TestStateAt_TimerArmedAtStart_Fires proves a historical read re-arms the timers a +// baseline carries, so a timer armed by Start fires in the replayed window. +func TestStateAt_TimerArmedAtStart_Fires(t *testing.T) { + ctx := context.Background() + m := timerMachine() + store := durable.NewMemStore(durable.WithHistory()) + id := durable.InstanceID("timer-1") + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *timerCtx]{durable.WithRunnerClock[string, string, *timerCtx](clk)} + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, &timerCtx{}, state.WithInitialState("arming")) + if err != nil { + t.Fatalf("Start: %v", err) + } + clk.Advance(6 * time.Second) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + view, err := durable.StateAt(ctx, m, store, id, 0, opts...) + if err != nil { + t.Fatalf("StateAt: %v", err) + } + if got := view.Instance().Current(); got != "done" { + t.Fatalf("StateAt state %q, want done", got) + } +} + +// TestRecover_OlderIdleTickRecord_FiresNothing proves an idle tick Record from an older +// journal replays as nothing fired, even with a due re-armed timer. +func TestRecover_OlderIdleTickRecord_FiresNothing(t *testing.T) { + ctx := context.Background() + m := pendingTimerMachine() + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *timerCtx]{ + durable.WithRunnerClock[string, string, *timerCtx](clk), + durable.WithCheckpointEvery[string, string, *timerCtx](3), + } + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, &timerCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + for _, ev := range []string{"arm", "poke", "poke"} { // the checkpoint carries the timer + if _, err = h.Fire(ctx, ev); err != nil { + t.Fatalf("Fire(%s): %v", ev, err) + } + } + // An older build recorded this idle tick with a reading past the timer's deadline. + late := epoch.Add(time.Hour).UnixNano() + idle := durable.Record{Step: 3, Tick: true, Entries: []state.JournalEntry{{Kind: state.JournalClockRead, ClockUnixNano: late}}} + if _, err = store.Append(ctx, id, idle); err != nil { + t.Fatalf("Append: %v", err) + } + recovered, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got := recovered.Instance().Current(); got != "waiting" { + t.Fatalf("recovered in %q, want waiting: the Record says nothing fired", got) + } +} + +// driftingClock moves by step on every read, as a real clock moves during a tick. +type driftingClock struct { + now time.Time + step time.Duration +} + +func (c *driftingClock) Now() time.Time { + t := c.now + c.now = c.now.Add(c.step) + return t +} + +func (c *driftingClock) After(time.Duration) <-chan time.Time { return nil } + +// twoTimerMachine holds two timers on a parent state. Each fires into one of parent's +// children, so firing one cancels neither. +func twoTimerMachine() *state.Machine[string, string, *timerCtx] { + return state.ForgeFor[*timerCtx]("twoTimers"). + State("idle"). + SuperState("parent"). + Initial("p1"). + SubState("p1"). + SubState("p2"). + SubState("p3"). + EndSuperState(). + Initial("idle"). + Transition("idle").On("go").GoTo("parent"). + Transition("parent").On("poke").GoTo("p1"). + Transition("parent").After(10 * time.Second).On("first").GoTo("p2"). + Transition("parent").After(10*time.Second + 10*time.Millisecond).On("second").GoTo("p3"). + Quench() +} + +// tickAcrossTwoDeadlines starts a twoTimerMachine and ticks it so the tick's first read +// finds one timer due and its second read finds the other. +func tickAcrossTwoDeadlines( + t *testing.T, + m *state.Machine[string, string, *timerCtx], + store durable.Store, + id durable.InstanceID, + clk *driftingClock, + opts []durable.Option[string, string, *timerCtx], +) { + t.Helper() + ctx := context.Background() + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, &timerCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); err != nil { + t.Fatalf("Fire: %v", err) + } + clk.now, clk.step = epoch.Add(10*time.Second+5*time.Millisecond), 20*time.Millisecond + fired, err := h.Tick(ctx) + if err != nil { + t.Fatalf("Tick: %v", err) + } + if len(fired) != 2 { + t.Fatalf("Tick fired %d timers, want 2", len(fired)) + } + clk.step = 0 +} + +// slowChainMachine has a chained timer armed after a slow action, and a second region +// whose timer fires between the chained timer's start and its real deadline. +func slowChainMachine(clk *driftingClock, other time.Duration) *state.Machine[string, string, *timerCtx] { + return state.ForgeFor[*timerCtx]("slowChain"). + Action("slow", func(state.ActionCtx[*timerCtx]) (state.Effect, error) { + clk.now = clk.now.Add(200 * time.Millisecond) + return nil, nil + }). + State("idle"). + SuperState("live"). + Region("first"). + Initial("a1"). + SubState("a1"). + SubState("a2"). + SubState("a3"). + EndRegion(). + Region("second"). + Initial("b1"). + SubState("b1"). + SubState("b2"). + EndRegion(). + EndSuperState(). + Initial("idle"). + Transition("idle").On("arm").GoTo("live"). + Transition("a1").After(10 * time.Second).On("x").GoTo("a2").Do("slow"). + Transition("a2").After(5 * time.Second).On("c").GoTo("a3"). + Transition("b1").After(other).On("y").GoTo("b2"). + Transition("b1").On("poke").GoTo("b1"). + Quench() +} + +// TestTick_ChainedTimerAfterSlowAction_SurvivesRecovery proves the Handle holds a chained +// timer at the deadline the scheduler armed, after the action ran. An earlier deadline +// would drop it when a later tick passes that instant, and recovery would lose it. +func TestTick_ChainedTimerAfterSlowAction_SurvivesRecovery(t *testing.T) { + ctx := context.Background() + clk := &driftingClock{now: epoch, step: 100 * time.Microsecond} + other := 15*time.Second - 100*time.Microsecond + m := slowChainMachine(clk, other) + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + opts := []durable.Option[string, string, *timerCtx]{ + durable.WithRunnerClock[string, string, *timerCtx](clk), + durable.WithCheckpointEvery[string, string, *timerCtx](1), + } + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, &timerCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "arm"); err != nil { + t.Fatalf("Fire: %v", err) + } + for _, at := range []time.Duration{10 * time.Second, other} { + clk.now = epoch.Add(at) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + } + recovered, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + clk.now = epoch.Add(16 * time.Second) + fired, err := recovered.Tick(ctx) + if err != nil { + t.Fatalf("recovered Tick: %v", err) + } + if len(fired) != 1 { + t.Fatalf("recovered Tick fired %d timers, want the chained timer", len(fired)) + } +} + +// TestRecover_ChainedTimerAfterSlowAction_NotFiredEarly proves a checkpointed chained +// timer keeps its real deadline, so a tail tick doesn't fire it before live did. +func TestRecover_ChainedTimerAfterSlowAction_NotFiredEarly(t *testing.T) { + ctx := context.Background() + clk := &driftingClock{now: epoch, step: 100 * time.Microsecond} + other := 15*time.Second - 100*time.Microsecond + m := slowChainMachine(clk, other) + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + opts := []durable.Option[string, string, *timerCtx]{ + durable.WithRunnerClock[string, string, *timerCtx](clk), + durable.WithCheckpointEvery[string, string, *timerCtx](2), + } + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, &timerCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "arm"); err != nil { + t.Fatalf("Fire: %v", err) + } + clk.now = epoch.Add(10 * time.Second) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + if _, err = h.Fire(ctx, "poke"); err != nil { + t.Fatalf("Fire(poke): %v", err) + } + clk.now = epoch.Add(other) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + recovered, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if live, got := handleSnapshot(t, h), handleSnapshot(t, recovered); got != live { + t.Fatalf("recovered instance differs from live:\n live: %s\n got: %s", live, got) + } +} + +// TestRecover_ReplayedChainedTimer_KeepsItsDeadline proves a replayed tick mirrors a +// chained timer at the deadline live armed, so the next checkpoint persists that deadline. +func TestRecover_ReplayedChainedTimer_KeepsItsDeadline(t *testing.T) { + ctx := context.Background() + clk := &driftingClock{now: epoch, step: 100 * time.Microsecond} + m := slowChainMachine(clk, time.Minute) + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + opts := []durable.Option[string, string, *timerCtx]{ + durable.WithRunnerClock[string, string, *timerCtx](clk), + durable.WithCheckpointEvery[string, string, *timerCtx](6), + } + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, &timerCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "arm"); err != nil { + t.Fatalf("Fire: %v", err) + } + clk.now = epoch.Add(10 * time.Second) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + first, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("first Recover: %v", err) + } + for range 3 { // cross a checkpoint boundary after the tick has replayed + if _, err = first.Fire(ctx, "poke"); err != nil { + t.Fatalf("Fire(poke): %v", err) + } + } + second, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("second Recover: %v", err) + } + clk.now = epoch.Add(15*time.Second + 100*time.Millisecond) // before 15.2s, the real deadline + fired, err := second.Tick(ctx) + if err != nil { + t.Fatalf("recovered Tick: %v", err) + } + if len(fired) != 0 { + t.Fatalf("recovered Tick fired %d timers before the deadline, want 0", len(fired)) + } +} + +// TestRecover_OlderJournal_TailEndingOnIdleTicks_LeavesNoReadings proves a tail ending on +// idle tick Records leaves none of their readings for the live clock, so the first Tick +// after recovery reads the real time and fires what is due. +func TestRecover_OlderJournal_TailEndingOnIdleTicks_LeavesNoReadings(t *testing.T) { + ctx := context.Background() + m := pendingTimerMachine() + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *timerCtx]{durable.WithRunnerClock[string, string, *timerCtx](clk)} + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, &timerCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "arm"); err != nil { + t.Fatalf("Fire: %v", err) + } + for step := 1; step <= 3; step++ { + read := state.JournalEntry{Kind: state.JournalClockRead, ClockUnixNano: epoch.Add(time.Duration(step) * time.Second).UnixNano()} + idle := durable.Record{Step: step, Tick: true, Entries: []state.JournalEntry{read}} + if _, err = store.Append(ctx, id, idle); err != nil { + t.Fatalf("Append: %v", err) + } + } + + clk.Advance(20 * time.Second) + recovered, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + fired, err := recovered.Tick(ctx) + if err != nil { + t.Fatalf("Tick: %v", err) + } + if len(fired) != 1 { + t.Fatalf("first Tick after recovery fired %d timers, want 1", len(fired)) + } +} + +// TestRecover_OlderJournal_FireArmsFromItsOwnRead proves a replayed Fire arms its timer +// from its own reading, the last in its Record, past a read an older journal left in front. +func TestRecover_OlderJournal_FireArmsFromItsOwnRead(t *testing.T) { + ctx := context.Background() + m := pendingTimerMachine() + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *timerCtx]{durable.WithRunnerClock[string, string, *timerCtx](clk)} + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, &timerCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + clk.Advance(time.Second) + if _, err = h.Fire(ctx, "arm"); err != nil { + t.Fatalf("Fire: %v", err) + } + _, tail, err := store.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + + stale := state.JournalEntry{Kind: state.JournalClockRead, ClockUnixNano: epoch.UnixNano()} + fire := tail[0] + fire.Entries = append([]state.JournalEntry{stale}, fire.Entries...) + older := durable.NewMemStore() + if _, err = durable.NewRunner(m, older, opts...).Start(ctx, id, &timerCtx{}, state.WithInitialState("idle")); err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = older.Append(ctx, id, fire); err != nil { + t.Fatalf("Append: %v", err) + } + + recovered, err := durable.Recover(ctx, m, older, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + clk.Advance(9*time.Second + 500*time.Millisecond) // the timer is due at 11s + for range 2 { // a misaligned replay leaves a reading for the first + fired, err := recovered.Tick(ctx) + if err != nil { + t.Fatalf("Tick: %v", err) + } + if len(fired) != 0 { + t.Fatalf("Tick fired %d timers before the deadline, want 0", len(fired)) + } + } +} + +// TestTick_TimerArmedAndFiredInOneTick_NotPersisted proves a timer a tick both arms and +// fires leaves the table, so a checkpoint doesn't persist it for recovery to fire again. +func TestTick_TimerArmedAndFiredInOneTick_NotPersisted(t *testing.T) { + ctx := context.Background() + m := state.ForgeFor[*timerCtx]("zeroDelay"). + State("idle"). + State("wait"). + SuperState("parent"). + Initial("p1"). + SubState("p1"). + SubState("p2"). + SubState("p3"). + EndSuperState(). + Initial("idle"). + Transition("idle").On("arm").GoTo("wait"). + Transition("wait").After(10 * time.Second).On("go").GoTo("parent"). + Transition("parent").After(0).On("zero").GoTo("p2"). + Transition("p2").On("poke").GoTo("p3"). + Quench() + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *timerCtx]{ + durable.WithRunnerClock[string, string, *timerCtx](clk), + durable.WithCheckpointEvery[string, string, *timerCtx](1), + } + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, &timerCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "arm"); err != nil { + t.Fatalf("Fire: %v", err) + } + clk.Advance(10 * time.Second) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + if _, err = h.Fire(ctx, "poke"); err != nil { + t.Fatalf("Fire(poke): %v", err) + } + recovered, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + fired, err := recovered.Tick(ctx) + if err != nil { + t.Fatalf("recovered Tick: %v", err) + } + if len(fired) != 0 { + t.Fatalf("recovered Tick fired %d timers, want 0", len(fired)) + } +} + +// TestTick_TimerDueAfterTheFirstRead_NotPersisted proves a timer that comes due between a +// tick's clock reads leaves the table with the others it fired, so a checkpoint doesn't +// persist it and recovery doesn't fire it again. +func TestTick_TimerDueAfterTheFirstRead_NotPersisted(t *testing.T) { + ctx := context.Background() + m := twoTimerMachine() + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + clk := &driftingClock{now: epoch} + opts := []durable.Option[string, string, *timerCtx]{ + durable.WithRunnerClock[string, string, *timerCtx](clk), + durable.WithCheckpointEvery[string, string, *timerCtx](1), + } + tickAcrossTwoDeadlines(t, m, store, id, clk, opts) + + recovered, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + again, err := recovered.Tick(ctx) + if err != nil { + t.Fatalf("recovered Tick: %v", err) + } + if len(again) != 0 { + t.Fatalf("recovered Tick fired %d timers, want 0", len(again)) + } +} + +// TestRecover_TimerDueAfterTheFirstRead_NotPersisted proves the same when the tick is +// replayed: the replayed tick drops the timer, so the next checkpoint doesn't persist it. +func TestRecover_TimerDueAfterTheFirstRead_NotPersisted(t *testing.T) { + ctx := context.Background() + m := twoTimerMachine() + store := durable.NewMemStore() + id := durable.InstanceID("timer-1") + clk := &driftingClock{now: epoch} + opts := []durable.Option[string, string, *timerCtx]{ + durable.WithRunnerClock[string, string, *timerCtx](clk), + durable.WithCheckpointEvery[string, string, *timerCtx](6), + } + tickAcrossTwoDeadlines(t, m, store, id, clk, opts) + + first, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("first Recover: %v", err) + } + for range 3 { // cross a checkpoint boundary after the tick has replayed + if _, err = first.Fire(ctx, "poke"); err != nil { + t.Fatalf("Fire(poke): %v", err) + } + } + second, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("second Recover: %v", err) + } + again, err := second.Tick(ctx) + if err != nil { + t.Fatalf("recovered Tick: %v", err) + } + if len(again) != 0 { + t.Fatalf("recovered Tick fired %d timers, want 0", len(again)) + } +} diff --git a/durable/timetravel.go b/durable/timetravel.go index 24a4eea1..354b54f4 100644 --- a/durable/timetravel.go +++ b/durable/timetravel.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" "github.com/stablekernel/crucible/state" ) @@ -133,12 +134,12 @@ func StateAt[S comparable, E comparable, C any]( bounded := boundRecords(records, step) repClock := newReplayClock(clockReadings(bounded), cfg.clock) - inst, err := restoreBaseline(m, baseline, repClock) + inst, timers, err := restoreBaseline(m, baseline, repClock) if err != nil { return nil, fmt.Errorf("durable: reconstructing %q at step %d: %w", id, step, err) } - if err := replayThrough(ctx, inst, &cfg, bounded); err != nil { + if err := replayThrough(ctx, inst, &cfg, bounded, timers, repClock); err != nil { return nil, fmt.Errorf("durable: reconstructing %q at step %d: %w", id, step, err) } return &TimeTravelView[S, E, C]{inst: inst, step: step}, nil @@ -195,30 +196,29 @@ func lastStep(records []Record) int { // restoreBaseline unwraps the baseline checkpoint envelope and restores a detached // kernel instance from it under the bounded replay clock, so the reconstruction -// reads recorded instants rather than the wall clock. The baseline carries no armed -// timers (it is the pre-event cast), so its envelope's timer table is empty and only -// the snapshot matters here. +// reads recorded instants rather than the wall clock. It also returns the timers the +// baseline carries, which the replay must re-arm before it runs. func restoreBaseline[S comparable, E comparable, C any]( m *state.Machine[S, E, C], baseline []byte, repClock state.Clock, -) (*state.Instance[S, E, C], error) { - kernelSnap, _, _, err := unmarshalCheckpoint(baseline) +) (*state.Instance[S, E, C], []pendingTimerWire, error) { + kernelSnap, timers, _, err := unmarshalCheckpoint(baseline) if err != nil { - return nil, fmt.Errorf("unmarshaling baseline envelope: %w", err) + return nil, nil, fmt.Errorf("unmarshaling baseline envelope: %w", err) } snap, err := state.UnmarshalSnapshot[S, E, C](kernelSnap) if err != nil { - return nil, fmt.Errorf("unmarshaling baseline snapshot: %w", err) + return nil, nil, fmt.Errorf("unmarshaling baseline snapshot: %w", err) } // Reconstruct in the same full-trace, unbounded-history mode the runner records // in, so a time-travel reconstruction's Trace history is byte-identical to the // live run it replays. inst, err := m.Restore(snap, state.WithRestoreClock[S](repClock), state.WithRestoreUnboundedHistory[S]()) if err != nil { - return nil, fmt.Errorf("restoring baseline snapshot: %w", err) + return nil, nil, fmt.Errorf("restoring baseline snapshot: %w", err) } - return inst, nil + return inst, timers, nil } // replayThrough replays the bounded record window forward into inst, reproducing the @@ -233,14 +233,32 @@ func replayThrough[S comparable, E comparable, C any]( inst *state.Instance[S, E, C], cfg *runnerConfig[S, E, C], bounded []Record, + timers []pendingTimerWire, + repClock *replayClock, ) error { sched := state.NewScheduler(inst) + // Re-arm the baseline's timers before the window replays, because it can fire them. + // As in Recover, the deadlines are absolute, so a held instant keeps this read from + // consuming a recorded reading. + armedAt := time.Unix(0, 0).UTC() + reEffects, _, err := reArmEffects(cfg.eventCodec, timers, armedAt) + if err != nil { + return fmt.Errorf("re-arming baseline timers: %w", err) + } + if len(reEffects) > 0 { + repClock.hold(armedAt) + sched.Absorb(ctx, reEffects) + repClock.release() + } + svc := newReplayServiceRunner(inst, cfg) if svc != nil { svc.Absorb(ctx, inst.StartEffects()) } + reads := 0 // clock readings through the current Record + settles := settleClock{clock: repClock} for i := range bounded { // A canceled context can stop a replayed settle or tick with no error, leaving // state short of the journal, so check it before each Record. @@ -248,6 +266,8 @@ func replayThrough[S comparable, E comparable, C any]( return err } rec := &bounded[i] + reads += clockReadCount(rec) + settles.next(rec) switch { case hasActorEntry(rec): for _, entry := range actorEntries(rec) { @@ -257,11 +277,19 @@ func replayThrough[S comparable, E comparable, C any]( } case hasServiceEntry(rec): for _, entry := range serviceEntries(rec) { - if _, err := replayService(ctx, svc, entry); err != nil { + effects, err := replayService(ctx, svc, entry) + if err != nil { return fmt.Errorf("at step %d: %w", rec.Step, err) } + if hasTimerEffect(effects) { + settles.absorb(rec, reads, func() { sched.Absorb(ctx, effects) }) + } } + case rec.Tick && rec.TickSteps == 0: + // Only an older journal holds a tick that fired nothing. Don't tick: a timer + // re-armed from a checkpoint could be due by now, and the Record says otherwise. case rec.Tick: + repClock.skipTo(reads - tickReads(rec.TickSteps)) sched.Tick(ctx) case len(rec.Event) == 0: continue // a checkpoint-only Record drives no event @@ -279,6 +307,7 @@ func replayThrough[S comparable, E comparable, C any]( return fmt.Errorf("replaying step %d: %w", rec.Step, res.Err) } if hasTimerEffect(res.Effects) { + repClock.skipTo(reads - 1) sched.Absorb(ctx, res.Effects) } if svc != nil {