diff --git a/sei-db/config/checkpoint_config.go b/sei-db/config/checkpoint_config.go new file mode 100644 index 0000000000..35a7ea1c76 --- /dev/null +++ b/sei-db/config/checkpoint_config.go @@ -0,0 +1,29 @@ +package config + +import ( + "time" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" +) + +// CheckpointConfig configures a CheckpointScheduler: how far apart the heights it picks are. +// +// A height has to clear every interval set above 0, so with both set the tighter one paces the +// cadence. A value of 0 or less is unused, and with neither set checkpointing is off. +type CheckpointConfig struct { + // TimeInterval is the wall-clock gap between checkpoints, measured from the last one completing. + TimeInterval time.Duration + + // BlockInterval places checkpoints on multiples of itself: at 1000, heights 1000, 2000, 3000 + // and so on are eligible. + BlockInterval int64 +} + +// DefaultCheckpointConfig returns a cadence mirroring the state-commit snapshot settings: a +// checkpoint every 10,000 blocks, and no more than one an hour. +func DefaultCheckpointConfig() CheckpointConfig { + return CheckpointConfig{ + TimeInterval: time.Duration(memiavl.DefaultSnapshotMinTimeInterval) * time.Second, + BlockInterval: memiavl.DefaultSnapshotInterval, + } +} diff --git a/sei-db/controller/checkpoint_scheduler.go b/sei-db/controller/checkpoint_scheduler.go index bfc3dcdaa4..da48207977 100644 --- a/sei-db/controller/checkpoint_scheduler.go +++ b/sei-db/controller/checkpoint_scheduler.go @@ -4,91 +4,168 @@ package controller import ( - "errors" - "fmt" "sync" + "time" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + "github.com/sei-protocol/seilog" + + "github.com/sei-protocol/sei-chain/sei-db/config" ) -// CheckpointScheduler coordinates checkpoints for stores with in-flight writes. +var checkpointLogger = seilog.NewLogger("db", "checkpoint") + +// CheckpointScheduler picks the heights every store on a node checkpoints at. +// +// Stores ask ShouldCheckpoint at each live commit and call MarkCheckpointComplete when that +// checkpoint finishes, success or failure. Replay, WAL catch-up, and state-sync must not ask. +// ShouldCheckpoint will register the store. +// +// The goal is that every registered store either checkpoints on the exact same height, or none of them do. +// That is achieved by holding a yes until every registered store has taken the checkpoint on the same height. +// So a lagging store is still told yes at that height, or no if the faster stores rejected that height already. +// +// A store that passes a held height without taking the checkpoint — one whose version jumped over it — is +// released from that height when it asks next time, so it costs that store one checkpoint rather than +// stopping the whole schedule. A store that stops asking altogether could hold the height indefinitely. // -// The engine-side capabilities this builds on — types.Checkpointable, -// types.DrainBarrier and types.CheckpointVersionSetter — stay with the engines -// that implement them. What lives here is the decision of when a checkpoint runs -// and what version it is labeled with. -type CheckpointScheduler interface { - SupportsCheckpoint() bool - ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) - SetCheckpointVersion(destDir string, version int64) error +// A time interval, a block interval, or both may be set. Both set means both must hold; neither disables checkpointing. +type CheckpointScheduler struct { + mu sync.Mutex + config config.CheckpointConfig + + // registered holds every store that has asked, which is how a store joins the schedule. + registered map[string]struct{} + // awaiting maps each store nextCheckpointVersion is held for to whether that store has taken + // the height yet. Empty means every store has checkpointed this height. + awaiting map[string]bool + + // nextCheckpointVersion is the height stores checkpoint at, 0 before the first one is picked. + // It stays set once complete, so a store that has yet to reach it is still given that height. + nextCheckpointVersion int64 + // checkpointedAt is when the last checkpoint completed, the scheduler's creation before the first. + checkpointedAt time.Time + // rejectedVersion is the highest height answered no. A later ask at it is answered no again + // rather than turning yes once an interval elapses under it. + rejectedVersion int64 } -// ErrCheckpointCanceled reports that a queued checkpoint was canceled before -// it started. -var ErrCheckpointCanceled = errors.New("state store checkpoint canceled") - -// SupportsCheckpoint reports whether db carries every engine capability a scheduled checkpoint needs. -// A store built from several engines answers for the set: one engine short of the capabilities makes -// the whole snapshot unpublishable. -func SupportsCheckpoint(db types.StateStore) bool { - _, checkpointable := db.(types.Checkpointable) - _, barrier := db.(types.DrainBarrier) - _, versionSetter := db.(types.CheckpointVersionSetter) - return checkpointable && barrier && versionSetter +// NewCheckpointScheduler returns a scheduler holding every store that asks it to one cadence. +func NewCheckpointScheduler(cfg config.CheckpointConfig) *CheckpointScheduler { + checkpointLogger.Info("checkpoint scheduler created", + "timeInterval", cfg.TimeInterval, "blockInterval", cfg.BlockInterval) + return &CheckpointScheduler{ + config: cfg, + registered: make(map[string]struct{}), + awaiting: make(map[string]bool), + checkpointedAt: time.Now(), + } } -// FanIn returns a report callback for n parallel branches. Each branch calls it once, and the last -// call passes done the first error any branch reported, or nil. -func FanIn(n int, done func(error)) func(error) { - var ( - mu sync.Mutex - remaining = n - firstErr error - ) - return func(err error) { - mu.Lock() - if err != nil && firstErr == nil { - firstErr = err - } - remaining-- - isLast := remaining == 0 - // Read under the lock: a peer branch may report between the unlock and the call to done. - outcome := firstErr - mu.Unlock() - if isLast { - done(outcome) - } +// ShouldCheckpoint reports whether version is a height for store to checkpoint at, registering +// store with the schedule when this is its first ask. Call it only from the live commit path, +// never during replay, WAL catch-up, or state-sync forward-fill. +func (s *CheckpointScheduler) ShouldCheckpoint(store string, version int64) bool { + if !s.checkpointEnabled() || version <= 0 { + return false } + + s.mu.Lock() + defer s.mu.Unlock() + s.registered[store] = struct{}{} + + if version == s.nextCheckpointVersion { + s.holdFor(store) + return true + } + if version > s.nextCheckpointVersion { + s.skipPastHeight(store) + } + if s.alreadyRejected(version) { + return false + } + if !s.allStoresCheckpointed() || !s.hasReachedNextInterval(version) { + s.rejectedVersion = version + return false + } + s.pickCheckpointHeight(store, version) + return true } -// ScheduleCheckpoint checkpoints an engine after all writes already enqueued -// on it have been applied. -func ScheduleCheckpoint(db types.StateStore, destDir string, shouldRun func() bool, done func(error)) { - cp, ok := db.(types.Checkpointable) - if !ok { - done(fmt.Errorf("state store backend %T does not support checkpoints", db)) +// MarkCheckpointComplete records that store has finished the height it was given. Both intervals +// start once every store that height is held for has reported it. Any other version is ignored, as +// is a repeat from the same store. +// +// A store that took a height must report it on every path out of the checkpoint, a failed one +// included, which in practice means deferring the call. No further height is picked while one store +// is unreported, so a missed call stops the node checkpointing until it restarts. +func (s *CheckpointScheduler) MarkCheckpointComplete(store string, version int64) { + s.mu.Lock() + defer s.mu.Unlock() + + if version != s.nextCheckpointVersion { return } - barrier, ok := db.(types.DrainBarrier) - if !ok { - done(fmt.Errorf("state store backend %T does not support ordered checkpoint barriers", db)) + if _, awaited := s.awaiting[store]; !awaited { + return + } + delete(s.awaiting, store) + s.updateCheckpointTime() +} + +func (s *CheckpointScheduler) checkpointEnabled() bool { + return s.config.TimeInterval > 0 || s.config.BlockInterval > 0 +} + +func (s *CheckpointScheduler) allStoresCheckpointed() bool { + return len(s.awaiting) == 0 +} + +// alreadyRejected reports whether version has been turned down already, either by trailing +// nextCheckpointVersion or by being asked and refused. Standing by that no is what stops two stores +// asking at different moments from splitting on one version. +func (s *CheckpointScheduler) alreadyRejected(version int64) bool { + return version < s.nextCheckpointVersion || version <= s.rejectedVersion +} + +// hasReachedNextInterval reports whether version satisfies every configured interval. +func (s *CheckpointScheduler) hasReachedNextInterval(version int64) bool { + timeElapsed := s.config.TimeInterval <= 0 || time.Since(s.checkpointedAt) >= s.config.TimeInterval + onBlockBoundary := s.config.BlockInterval <= 0 || version%s.config.BlockInterval == 0 + return timeElapsed && onBlockBoundary +} + +// pickCheckpointHeight sets nextCheckpointVersion to version and holds it for the stores registered +// now, of which store is the one that has taken it. One registering later is not held for here, at +// a version it may already have passed, and is held for from the moment it takes a height. +func (s *CheckpointScheduler) pickCheckpointHeight(store string, version int64) { + s.nextCheckpointVersion = version + s.awaiting = make(map[string]bool, len(s.registered)) + for name := range s.registered { + s.awaiting[name] = false + } + s.awaiting[store] = true +} + +// holdFor records that store has taken nextCheckpointVersion, which is held until store reports it. +func (s *CheckpointScheduler) holdFor(store string) { + s.awaiting[store] = true +} + +// skipPastHeight releases the hold store has on nextCheckpointVersion, for a store asking above +// that height and so past it for good. A store that took the height keeps its hold: it may be +// writing that checkpoint while it commits later versions, and the intervals wait for it. +func (s *CheckpointScheduler) skipPastHeight(store string) { + if took, awaited := s.awaiting[store]; !awaited || took { return } - barrier.ScheduleAtDrain(func() { - if shouldRun != nil && !shouldRun() { - done(ErrCheckpointCanceled) - return - } - done(cp.Checkpoint(destDir)) - }) + delete(s.awaiting, store) + s.updateCheckpointTime() } -// SetCheckpointVersion makes a completed checkpoint self-describing without -// changing the live database. -func SetCheckpointVersion(db types.StateStore, destDir string, version int64) error { - setter, ok := db.(types.CheckpointVersionSetter) - if !ok { - return fmt.Errorf("state store backend %T cannot set checkpoint version", db) +// updateCheckpointTime records when nextCheckpointVersion completed, once every store has +// checkpointed this height. +func (s *CheckpointScheduler) updateCheckpointTime() { + if s.allStoresCheckpointed() { + s.checkpointedAt = time.Now() } - return setter.SetCheckpointVersion(destDir, version) } diff --git a/sei-db/controller/checkpoint_scheduler_test.go b/sei-db/controller/checkpoint_scheduler_test.go new file mode 100644 index 0000000000..2110667b05 --- /dev/null +++ b/sei-db/controller/checkpoint_scheduler_test.go @@ -0,0 +1,396 @@ +package controller + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/config" +) + +// newScheduler spells the two intervals out positionally, which reads better than a config literal +// in tests that vary nothing else. +func newScheduler(timeInterval time.Duration, blockInterval int64) *CheckpointScheduler { + return NewCheckpointScheduler(config.CheckpointConfig{ + TimeInterval: timeInterval, + BlockInterval: blockInterval, + }) +} + +// elapseTimeInterval moves the interval's starting point back rather than waiting it out. +func (s *CheckpointScheduler) elapseTimeInterval() { + s.checkpointedAt = time.Now().Add(-s.config.TimeInterval) +} + +// --------------------------------------------------------------------------- +// Which intervals are in use +// --------------------------------------------------------------------------- + +func TestNeitherIntervalSetDisablesCheckpointing(t *testing.T) { + for _, scheduler := range []*CheckpointScheduler{ + newScheduler(0, 0), + newScheduler(-time.Second, 0), + newScheduler(0, -1), + } { + require.False(t, scheduler.ShouldCheckpoint("sc", 1)) + require.False(t, scheduler.ShouldCheckpoint("sc", 1_000_000)) + } +} + +func TestNonPositiveVersionsAreNeverCheckpointHeights(t *testing.T) { + scheduler := newScheduler(time.Hour, 10) + scheduler.elapseTimeInterval() + + require.False(t, scheduler.ShouldCheckpoint("sc", 0)) + require.False(t, scheduler.ShouldCheckpoint("sc", -1)) +} + +// The first height clears the same intervals as every later one, measured from the scheduler's +// creation. Answering yes to whichever version happens to ask first would ignore the cadence. +func TestTheFirstHeightWaitsForTheTimeInterval(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + + require.False(t, scheduler.ShouldCheckpoint("sc", 10)) + + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 11)) +} + +func TestTheFirstHeightWaitsForTheBlockInterval(t *testing.T) { + scheduler := newScheduler(0, 100) + + require.False(t, scheduler.ShouldCheckpoint("sc", 99)) + require.True(t, scheduler.ShouldCheckpoint("sc", 100)) +} + +// Heights sit on multiples of the block interval rather than a count from the last one, so a node +// that starts mid-grid still checkpoints at the same heights as one that did not. +func TestBlockIntervalHeightsAreMultiplesOfTheInterval(t *testing.T) { + scheduler := newScheduler(0, 100) + + for _, offGrid := range []int64{4001, 4050, 4099} { + require.False(t, scheduler.ShouldCheckpoint("sc", offGrid)) + } + require.True(t, scheduler.ShouldCheckpoint("sc", 4100)) +} + +// With both set a height has to clear both, so the tighter one paces the cadence. +func TestBothIntervalsMustElapseWhenBothAreSet(t *testing.T) { + scheduler := newScheduler(time.Hour, 10) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 10)) + scheduler.MarkCheckpointComplete("sc", 10) + + require.False(t, scheduler.ShouldCheckpoint("sc", 15), "neither interval has elapsed") + + scheduler.elapseTimeInterval() + require.False(t, scheduler.ShouldCheckpoint("sc", 19), "the block interval has not elapsed") + require.True(t, scheduler.ShouldCheckpoint("sc", 20)) +} + +// --------------------------------------------------------------------------- +// One answer per height +// --------------------------------------------------------------------------- + +func TestAPickedHeightIsYesForEveryStore(t *testing.T) { + scheduler := newScheduler(0, 10) + require.True(t, scheduler.ShouldCheckpoint("sc", 10)) + + require.True(t, scheduler.ShouldCheckpoint("ss", 10)) + require.True(t, scheduler.ShouldCheckpoint("receipt", 10)) +} + +// The height stays available after it completes: a store that reaches it later than the store that +// finished it is handed the same height rather than the next one. +func TestAPickedHeightStaysYesForALaggingStore(t *testing.T) { + scheduler := newScheduler(0, 10) + require.True(t, scheduler.ShouldCheckpoint("sc", 10)) + scheduler.MarkCheckpointComplete("sc", 10) + + require.True(t, scheduler.ShouldCheckpoint("sc", 10)) +} + +// A height answered no stays no once an interval elapses under it, so two stores asking either side +// of that moment cannot split on the same version. +func TestARejectedHeightStaysNoForEveryStore(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 10)) + scheduler.MarkCheckpointComplete("sc", 10) + + require.False(t, scheduler.ShouldCheckpoint("sc", 11)) + + scheduler.elapseTimeInterval() + require.False(t, scheduler.ShouldCheckpoint("ss", 11), "11 was already answered no") + require.True(t, scheduler.ShouldCheckpoint("sc", 12)) +} + +// The refusal covers every height at or under the one refused, not just that exact height. A store +// that asks just short of the interval pushes the floor up to the height it was at, so a store +// lagging behind it cannot walk in under that floor the moment the interval elapses and take a +// height the store ahead of it was already refused. +func TestALaggingStoreCannotTakeAHeightUnderARefusedOne(t *testing.T) { + scheduler := newScheduler(5*time.Minute, 0) + + require.False(t, scheduler.ShouldCheckpoint("sc", 100), "the interval has not elapsed") + + scheduler.elapseTimeInterval() + for _, lagging := range []int64{98, 99, 100} { + require.False(t, scheduler.ShouldCheckpoint("ss", lagging), "height %d is under the refused 100", lagging) + } + + require.True(t, scheduler.ShouldCheckpoint("sc", 101)) + require.True(t, scheduler.ShouldCheckpoint("ss", 101), "the lagging store reaches the same height") +} + +func TestAHeightBelowTheCurrentOneIsNo(t *testing.T) { + scheduler := newScheduler(0, 10) + require.True(t, scheduler.ShouldCheckpoint("sc", 10)) + + require.False(t, scheduler.ShouldCheckpoint("ss", 9)) +} + +func TestConcurrentAsksAtTheSameHeightAgree(t *testing.T) { + scheduler := newScheduler(0, 10) + + var answers [8]bool + var wg sync.WaitGroup + for i := range answers { + wg.Add(1) + go func(i int) { + defer wg.Done() + answers[i] = scheduler.ShouldCheckpoint("sc", 10) + }(i) + } + wg.Wait() + + for i, answer := range answers { + require.True(t, answer, "asker %d", i) + } +} + +// --------------------------------------------------------------------------- +// Stores at different heights +// --------------------------------------------------------------------------- + +// walkLeaderAndLaggard runs two stores up to a height, the second one lag blocks behind the first, +// and returns the heights each checkpointed at. A non-zero elapseEvery elapses the time interval +// once every that many heights, standing in for wall-clock passing as the chain advances. +func walkLeaderAndLaggard( + scheduler *CheckpointScheduler, to, lag, elapseEvery int64, +) (leaderTook, laggardTook []int64) { + for height := int64(1); height <= to; height++ { + if elapseEvery > 0 && height%elapseEvery == 0 { + scheduler.elapseTimeInterval() + } + if scheduler.ShouldCheckpoint("leader", height) { + leaderTook = append(leaderTook, height) + scheduler.MarkCheckpointComplete("leader", height) + } + behind := height - lag + if behind > 0 && scheduler.ShouldCheckpoint("laggard", behind) { + laggardTook = append(laggardTook, behind) + scheduler.MarkCheckpointComplete("laggard", behind) + } + } + return leaderTook, laggardTook +} + +// A store lagging by more than the block interval still takes every height the leader takes, since +// the height is held until it arrives. Waiting for it costs whole boundaries rather than shifting +// the grid: 300 passes while 200 is still held, so the next checkpoint is 400. +func TestEveryStoreTakesTheSameHeightsHoweverFarBehind(t *testing.T) { + scheduler := newScheduler(0, 100) + + leaderTook, laggardTook := walkLeaderAndLaggard(scheduler, 1000, 150, 0) + + require.Equal(t, []int64{100, 200, 400, 600, 800, 1000}, leaderTook) + require.Equal(t, []int64{200, 400, 600, 800}, laggardTook, + "100 predates the laggard's first ask, and it has yet to reach 1000") +} + +func TestEveryStoreTakesTheSameHeightsOnATimeInterval(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + + leaderTook, laggardTook := walkLeaderAndLaggard(scheduler, 1000, 150, 300) + + require.Equal(t, []int64{300, 600, 900}, leaderTook) + require.Equal(t, []int64{300, 600}, laggardTook, "the laggard has yet to reach 900") +} + +// The height is held even once the intervals have elapsed, so the leader cannot run ahead onto a +// height the store behind it would never be offered. +func TestAHeightIsHeldUntilEveryStoreHasReportedIt(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 100)) + require.True(t, scheduler.ShouldCheckpoint("ss", 100)) + scheduler.MarkCheckpointComplete("sc", 100) + + scheduler.elapseTimeInterval() + require.False(t, scheduler.ShouldCheckpoint("sc", 200), "ss has not reported 100") + + scheduler.MarkCheckpointComplete("ss", 100) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 300)) +} + +// A store whose first ask lands while a height is held is not one of the stores that height waits +// for: it may already be past that version, and holding for it would wedge the schedule. +func TestAStoreRegisteringWhileAHeightIsHeldJoinsTheNextOne(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 100)) + + require.False(t, scheduler.ShouldCheckpoint("ss", 150), "ss registers by asking") + scheduler.MarkCheckpointComplete("sc", 100) + + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 200)) + scheduler.MarkCheckpointComplete("sc", 200) + scheduler.elapseTimeInterval() + require.False(t, scheduler.ShouldCheckpoint("sc", 300), "200 is now held for ss as well") +} + +// A store whose version jumps over the held height would otherwise hold it forever, since it never +// asks at that height again. Asking above it is proof it has passed, so it is released and only +// that store misses the checkpoint. +func TestAStoreThatPassesTheHeldHeightIsReleased(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + require.False(t, scheduler.ShouldCheckpoint("ss", 98), "ss registers before the interval elapses") + + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 100)) + scheduler.MarkCheckpointComplete("sc", 100) + + require.False(t, scheduler.ShouldCheckpoint("ss", 101), "ss jumped from 99 to 101") + + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 200), "100 is no longer held for ss") +} + +// A no is recorded before the jumper asks, so skipPastHeight never runs if it sits behind +// alreadyRejected. The jumper must still be released: otherwise it stays in awaiting forever. +func TestAStoreThatJumpsIsReleasedAfterARefusedHeightAbove(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + require.False(t, scheduler.ShouldCheckpoint("ss", 98), "ss registers before the interval elapses") + + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 100)) + scheduler.MarkCheckpointComplete("sc", 100) + + require.False(t, scheduler.ShouldCheckpoint("sc", 101), "sc asks above 100 first") + require.False(t, scheduler.ShouldCheckpoint("ss", 101), "ss lands on a height already refused") + + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 200), "ss must have been released from 100") +} + +// A store commits later versions while its own checkpoint of the held height is still running, so +// asking above that height is not proof it skipped it. Releasing it there would start the intervals +// while it is mid-write. +func TestAStoreWritingTheHeldHeightIsNotReleased(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 100)) + + require.False(t, scheduler.ShouldCheckpoint("sc", 101), "sc is still writing 100") + + scheduler.elapseTimeInterval() + require.False(t, scheduler.ShouldCheckpoint("sc", 200), "100 is still held for sc") + + scheduler.MarkCheckpointComplete("sc", 100) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 300)) +} + +// --------------------------------------------------------------------------- +// Completion +// --------------------------------------------------------------------------- + +// A store that stops asking holds its height for good: no later height is picked however long the +// intervals have had to elapse, and nothing recovers short of a restart. A store still asking is +// released once it passes the height, so this is the stalled-store case rather than the skipped one. +func TestAStalledStoreStopsTheScheduler(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 100)) + scheduler.MarkCheckpointComplete("sc", 100) + + // ss registers, is held for 200, then stops asking entirely. + require.False(t, scheduler.ShouldCheckpoint("ss", 150)) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 200)) + scheduler.MarkCheckpointComplete("sc", 200) + + for _, height := range []int64{300, 400, 500} { + scheduler.elapseTimeInterval() + require.False(t, scheduler.ShouldCheckpoint("sc", height), "ss never reported 200") + } +} + +// Reporting a height whose checkpoint failed is what keeps the scheduler moving, which is why the +// call carries no success flag: a store defers it and the next height comes due as usual. +func TestReportingAFailedHeightKeepsTheSchedulerMoving(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 100)) + + scheduler.MarkCheckpointComplete("sc", 100) + + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 200)) +} + +// The interval runs from the last store finishing rather than the first, so the gap to the next +// checkpoint is not spent by a store that is still writing this one. +func TestTheIntervalRunsFromTheLastStoreCompleting(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 100)) + require.True(t, scheduler.ShouldCheckpoint("ss", 100)) + + scheduler.MarkCheckpointComplete("sc", 100) + afterFirst := scheduler.checkpointedAt + scheduler.MarkCheckpointComplete("ss", 100) + + require.True(t, scheduler.checkpointedAt.After(afterFirst)) +} + +func TestMarkCheckpointCompleteIgnoresAnotherVersion(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 100)) + + scheduler.MarkCheckpointComplete("sc", 99) + scheduler.MarkCheckpointComplete("sc", 101) + + require.False(t, scheduler.allStoresCheckpointed(), "neither version is the height being held") +} + +func TestMarkCheckpointCompleteIgnoresAnUnregisteredStore(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 100)) + require.True(t, scheduler.ShouldCheckpoint("ss", 100)) + + scheduler.MarkCheckpointComplete("receipt", 100) + + require.False(t, scheduler.allStoresCheckpointed(), "the height is still held for sc and ss") +} + +// A store reporting a height it already reported must not restart the intervals, which would push +// the next checkpoint out every time a straggler that took the height late reports. +func TestARepeatedReportDoesNotRestartTheIntervals(t *testing.T) { + scheduler := newScheduler(time.Hour, 0) + scheduler.elapseTimeInterval() + require.True(t, scheduler.ShouldCheckpoint("sc", 100)) + + scheduler.MarkCheckpointComplete("sc", 100) + completedAt := scheduler.checkpointedAt + scheduler.MarkCheckpointComplete("sc", 100) + + require.Equal(t, completedAt, scheduler.checkpointedAt) +} diff --git a/sei-db/db_engine/pebbledb/mvcc/db_test.go b/sei-db/db_engine/pebbledb/mvcc/db_test.go index 375d0202ba..9d3ca40fca 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db_test.go +++ b/sei-db/db_engine/pebbledb/mvcc/db_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/suite" "github.com/sei-protocol/sei-chain/sei-db/config" - "github.com/sei-protocol/sei-chain/sei-db/controller" sstest "github.com/sei-protocol/sei-chain/sei-db/db_engine/test" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + sssnapshot "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/snapshot" ) func TestStorageTestSuite(t *testing.T) { @@ -63,13 +63,13 @@ func TestVersionedCheckpointPreservesFutureLiveMarker(t *testing.T) { dest := filepath.Join(t.TempDir(), "snapshot") done := make(chan error, 1) - controller.ScheduleCheckpoint(store, dest, nil, func(err error) { + sssnapshot.ScheduleCheckpoint(store, dest, nil, func(err error) { done <- err }) require.NoError(t, <-done) // The caller stamps only the label. Earliest is inherited from the // checkpointed DB because prune advances it before deleting history. - require.NoError(t, controller.SetCheckpointVersion(store, dest, 5)) + require.NoError(t, sssnapshot.SetCheckpointVersion(store, dest, 5)) require.Equal(t, int64(10), store.GetLatestVersion()) require.Equal(t, int64(4), store.GetEarliestVersion()) @@ -97,10 +97,10 @@ func TestScheduledCheckpointCanBeCanceledAtBarrier(t *testing.T) { dest := filepath.Join(t.TempDir(), "snapshot") done := make(chan error, 1) - controller.ScheduleCheckpoint(store, dest, func() bool { return false }, func(err error) { + sssnapshot.ScheduleCheckpoint(store, dest, func() bool { return false }, func(err error) { done <- err }) - require.ErrorIs(t, <-done, controller.ErrCheckpointCanceled) + require.ErrorIs(t, <-done, sssnapshot.ErrCheckpointCanceled) require.NoDirExists(t, dest) } diff --git a/sei-db/db_engine/types/types.go b/sei-db/db_engine/types/types.go index e4462949ec..3a2d9d0a13 100644 --- a/sei-db/db_engine/types/types.go +++ b/sei-db/db_engine/types/types.go @@ -180,8 +180,9 @@ type RollbackValidator interface { // The interfaces above are engine capabilities. Deciding when a checkpoint runs, // and what version it is labeled with, is coordination rather than engine -// behavior and lives in sei-db/management: CheckpointScheduler, -// ScheduleCheckpoint, SetCheckpointVersion and ErrCheckpointCanceled. +// behavior: sei-db/controller picks the height every store checkpoints at, and +// sei-db/state_db/ss/snapshot composes these capabilities into the checkpoint an +// SS member store performs. // --------------------------------------------------------------------------- // SS DB layer diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index 3c404470e2..c116f091f8 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -7,7 +7,6 @@ import ( "sync/atomic" "time" - "github.com/sei-protocol/sei-chain/sei-db/controller" sssnapshot "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/snapshot" ) @@ -251,7 +250,7 @@ func (c *snapshotCoordinator) requestSnapshot(version int64, start time.Time) er return c.isRunning() && !canceled.Load() } - report := controller.FanIn(len(c.members), func(err error) { + report := sssnapshot.FanIn(len(c.members), func(err error) { c.startPublish(version, staged, err, start) }) for i, member := range c.members { @@ -295,7 +294,7 @@ func (c *snapshotCoordinator) startPublish( defer c.publishing.Done() defer c.finishSnapshot() if checkpointErr != nil { - if errors.Is(checkpointErr, controller.ErrCheckpointCanceled) { + if errors.Is(checkpointErr, sssnapshot.ErrCheckpointCanceled) { sssnapshot.RecordCompletion(start, "canceled") } else { sssnapshot.RecordCompletion(start, "failure") diff --git a/sei-db/state_db/ss/composite/snapshot_test.go b/sei-db/state_db/ss/composite/snapshot_test.go index 8ead3909ec..75fa6b02ef 100644 --- a/sei-db/state_db/ss/composite/snapshot_test.go +++ b/sei-db/state_db/ss/composite/snapshot_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/sei-protocol/sei-chain/sei-db/config" - "github.com/sei-protocol/sei-chain/sei-db/controller" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/evm" sssnapshot "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/snapshot" @@ -31,7 +30,7 @@ func (s *controlledSnapshotScheduler) ScheduleCheckpoint( ) { s.pending <- func() { if !shouldRun() { - done(controller.ErrCheckpointCanceled) + done(sssnapshot.ErrCheckpointCanceled) return } if s.fail { @@ -328,13 +327,13 @@ func openTestManagerWithRetention( t.Helper() source := t.TempDir() manager, err := sssnapshot.Open(sssnapshot.Config{ - Name: name, - Root: root, - SourceDirs: []string{source}, - Backend: config.PebbleDBBackend, - KeepRecent: keepRecent, - Scheduler: scheduler, - Floor: floor, + Name: name, + Root: root, + SourceDirs: []string{source}, + Backend: config.PebbleDBBackend, + KeepRecent: keepRecent, + Checkpointer: scheduler, + Floor: floor, }) require.NoError(t, err) return manager diff --git a/sei-db/state_db/ss/cosmos/store.go b/sei-db/state_db/ss/cosmos/store.go index d2d076deed..35b550d452 100644 --- a/sei-db/state_db/ss/cosmos/store.go +++ b/sei-db/state_db/ss/cosmos/store.go @@ -7,7 +7,6 @@ import ( dbm "github.com/tendermint/tm-db" "github.com/sei-protocol/sei-chain/sei-db/config" - "github.com/sei-protocol/sei-chain/sei-db/controller" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/proto" sssnapshot "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/snapshot" @@ -103,15 +102,15 @@ func (s *CosmosStateStore) Close() error { } func (s *CosmosStateStore) SupportsCheckpoint() bool { - return controller.SupportsCheckpoint(s.db) + return sssnapshot.SupportsCheckpoint(s.db) } func (s *CosmosStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) { - controller.ScheduleCheckpoint(s.db, destDir, shouldRun, done) + sssnapshot.ScheduleCheckpoint(s.db, destDir, shouldRun, done) } func (s *CosmosStateStore) SetCheckpointVersion(destDir string, version int64) error { - return controller.SetCheckpointVersion(s.db, destDir, version) + return sssnapshot.SetCheckpointVersion(s.db, destDir, version) } func (s *CosmosStateStore) StartSnapshots( @@ -127,7 +126,7 @@ func (s *CosmosStateStore) StartSnapshots( Backend: ssConfig.Backend, KeepRecent: ssConfig.SnapshotKeepRecent, ExternalPruning: ssConfig.ExternalPruning, - Scheduler: s, + Checkpointer: s, Floor: floor, }) if err != nil { diff --git a/sei-db/state_db/ss/evm/store.go b/sei-db/state_db/ss/evm/store.go index 1e98d1e9d2..02b86d299f 100644 --- a/sei-db/state_db/ss/evm/store.go +++ b/sei-db/state_db/ss/evm/store.go @@ -12,7 +12,6 @@ import ( commonevm "github.com/sei-protocol/sei-chain/sei-db/common/keys" "github.com/sei-protocol/sei-chain/sei-db/config" - "github.com/sei-protocol/sei-chain/sei-db/controller" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/backend" @@ -427,7 +426,7 @@ func (s *EVMStateStore) Close() error { func (s *EVMStateStore) SupportsCheckpoint() bool { for _, db := range s.managedDBs { - if !controller.SupportsCheckpoint(db) { + if !sssnapshot.SupportsCheckpoint(db) { return false } } @@ -455,7 +454,7 @@ func (s *EVMStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool done(errors.New("EVM state store has no managed DB to checkpoint")) return } - controller.ScheduleCheckpoint(db, destDir, shouldRun, done) + sssnapshot.ScheduleCheckpoint(db, destDir, shouldRun, done) return } @@ -465,10 +464,10 @@ func (s *EVMStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool } storeTypes := AllEVMStoreTypes() - report := controller.FanIn(len(storeTypes), done) + report := sssnapshot.FanIn(len(storeTypes), done) for _, storeType := range storeTypes { name := StoreTypeName(storeType) - controller.ScheduleCheckpoint(s.subDBs[storeType], subDBPath(destDir, storeType), shouldRun, func(err error) { + sssnapshot.ScheduleCheckpoint(s.subDBs[storeType], subDBPath(destDir, storeType), shouldRun, func(err error) { if err != nil { err = fmt.Errorf("checkpoint EVM sub-DB %s: %w", name, err) } @@ -483,11 +482,11 @@ func (s *EVMStateStore) SetCheckpointVersion(destDir string, version int64) erro if db == nil { return errors.New("EVM state store has no managed DB to stamp") } - return controller.SetCheckpointVersion(db, destDir, version) + return sssnapshot.SetCheckpointVersion(db, destDir, version) } for _, storeType := range AllEVMStoreTypes() { name := StoreTypeName(storeType) - if err := controller.SetCheckpointVersion(s.subDBs[storeType], subDBPath(destDir, storeType), version); err != nil { + if err := sssnapshot.SetCheckpointVersion(s.subDBs[storeType], subDBPath(destDir, storeType), version); err != nil { return fmt.Errorf("set EVM sub-DB %s checkpoint version: %w", name, err) } } @@ -506,7 +505,7 @@ func (s *EVMStateStore) StartSnapshots( Backend: ssConfig.Backend, KeepRecent: ssConfig.SnapshotKeepRecent, ExternalPruning: ssConfig.ExternalPruning, - Scheduler: s, + Checkpointer: s, Floor: floor, }) if err != nil { diff --git a/sei-db/state_db/ss/snapshot/checkpoint.go b/sei-db/state_db/ss/snapshot/checkpoint.go new file mode 100644 index 0000000000..c121e4e256 --- /dev/null +++ b/sei-db/state_db/ss/snapshot/checkpoint.go @@ -0,0 +1,92 @@ +package snapshot + +import ( + "errors" + "fmt" + "sync" + + "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" +) + +// Checkpointer is the backend side of an SS member snapshot: the Manager owns the +// directories a snapshot occupies, and a Checkpointer writes the database into one. +// +// The engine-side capabilities this builds on — types.Checkpointable, +// types.DrainBarrier and types.CheckpointVersionSetter — stay with the engines that +// implement them. An SS member store implements Checkpointer by composing them over +// the one or more engines it manages. +type Checkpointer interface { + SupportsCheckpoint() bool + ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) + SetCheckpointVersion(destDir string, version int64) error +} + +// ErrCheckpointCanceled reports that a queued checkpoint was canceled before +// it started. +var ErrCheckpointCanceled = errors.New("state store checkpoint canceled") + +// SupportsCheckpoint reports whether db carries every engine capability a scheduled checkpoint needs. +// A store built from several engines answers for the set: one engine short of the capabilities makes +// the whole snapshot unpublishable. +func SupportsCheckpoint(db types.StateStore) bool { + _, checkpointable := db.(types.Checkpointable) + _, barrier := db.(types.DrainBarrier) + _, versionSetter := db.(types.CheckpointVersionSetter) + return checkpointable && barrier && versionSetter +} + +// FanIn returns a report callback for n parallel branches. Each branch calls it once, and the last +// call passes done the first error any branch reported, or nil. +func FanIn(n int, done func(error)) func(error) { + var ( + mu sync.Mutex + remaining = n + firstErr error + ) + return func(err error) { + mu.Lock() + if err != nil && firstErr == nil { + firstErr = err + } + remaining-- + isLast := remaining == 0 + // Read under the lock: a peer branch may report between the unlock and the call to done. + outcome := firstErr + mu.Unlock() + if isLast { + done(outcome) + } + } +} + +// ScheduleCheckpoint checkpoints an engine after all writes already enqueued +// on it have been applied. +func ScheduleCheckpoint(db types.StateStore, destDir string, shouldRun func() bool, done func(error)) { + cp, ok := db.(types.Checkpointable) + if !ok { + done(fmt.Errorf("state store backend %T does not support checkpoints", db)) + return + } + barrier, ok := db.(types.DrainBarrier) + if !ok { + done(fmt.Errorf("state store backend %T does not support ordered checkpoint barriers", db)) + return + } + barrier.ScheduleAtDrain(func() { + if shouldRun != nil && !shouldRun() { + done(ErrCheckpointCanceled) + return + } + done(cp.Checkpoint(destDir)) + }) +} + +// SetCheckpointVersion makes a completed checkpoint self-describing without +// changing the live database. +func SetCheckpointVersion(db types.StateStore, destDir string, version int64) error { + setter, ok := db.(types.CheckpointVersionSetter) + if !ok { + return fmt.Errorf("state store backend %T cannot set checkpoint version", db) + } + return setter.SetCheckpointVersion(destDir, version) +} diff --git a/sei-db/state_db/ss/snapshot/manager.go b/sei-db/state_db/ss/snapshot/manager.go index f6a06aa560..aa99ca6816 100644 --- a/sei-db/state_db/ss/snapshot/manager.go +++ b/sei-db/state_db/ss/snapshot/manager.go @@ -14,7 +14,6 @@ import ( "time" "github.com/sei-protocol/sei-chain/sei-db/common/utils" - "github.com/sei-protocol/sei-chain/sei-db/controller" "github.com/sei-protocol/seilog" ) @@ -42,7 +41,7 @@ type Config struct { Backend string KeepRecent int ExternalPruning bool - Scheduler controller.CheckpointScheduler + Checkpointer Checkpointer // Floor names a height this member's retention must keep. Leave it nil when the member is the only // one that has to hold the height a restore starts from. Floor *Floor @@ -115,7 +114,7 @@ type Manager struct { backend string keepRecent int externalPruning bool - scheduler controller.CheckpointScheduler + checkpointer Checkpointer floor *Floor snapshotSizes map[int64]int64 @@ -187,10 +186,10 @@ func ListSnapshotVersions(root string) ([]int64, error) { // Open prepares a snapshot root and returns a Manager for one SS member. func Open(cfg Config) (*Manager, error) { - if cfg.Scheduler == nil { - return nil, fmt.Errorf("%s snapshot scheduler is nil", cfg.Name) + if cfg.Checkpointer == nil { + return nil, fmt.Errorf("%s snapshot checkpointer is nil", cfg.Name) } - if !cfg.Scheduler.SupportsCheckpoint() { + if !cfg.Checkpointer.SupportsCheckpoint() { return nil, fmt.Errorf("%s backend %q does not support checkpoints", cfg.Name, cfg.Backend) } if err := verifyHardlinks(cfg.Root, cfg.SourceDirs); err != nil { @@ -202,7 +201,7 @@ func Open(cfg Config) (*Manager, error) { backend: cfg.Backend, keepRecent: cfg.KeepRecent, externalPruning: cfg.ExternalPruning, - scheduler: cfg.Scheduler, + checkpointer: cfg.Checkpointer, floor: cfg.Floor, snapshotSizes: map[int64]int64{}, } @@ -276,7 +275,7 @@ func (m *Manager) Schedule(staged *Staged, shouldRun func() bool, done func(erro done(fmt.Errorf("%s staged snapshot belongs to a different manager", m.name)) return } - m.scheduler.ScheduleCheckpoint(staged.tmpDir, shouldRun, done) + m.checkpointer.ScheduleCheckpoint(staged.tmpDir, shouldRun, done) } func (m *Manager) Commit(staged *Staged) error { @@ -297,7 +296,7 @@ func (m *Manager) Commit(staged *Staged) error { defer m.publishMu.Unlock() defer m.prune() - if err := m.scheduler.SetCheckpointVersion(staged.tmpDir, staged.version); err != nil { + if err := m.checkpointer.SetCheckpointVersion(staged.tmpDir, staged.version); err != nil { _ = os.RemoveAll(staged.tmpDir) return fmt.Errorf("set %s snapshot version: %w", m.name, err) } @@ -484,7 +483,7 @@ func (m *Manager) prune() { } func (m *Manager) pruneWALToOldestSnapshot() { - pruner, ok := m.scheduler.(snapshotWALPruner) + pruner, ok := m.checkpointer.(snapshotWALPruner) if !ok { return } diff --git a/sei-db/state_db/ss/snapshot/manager_test.go b/sei-db/state_db/ss/snapshot/manager_test.go index a2f51160cb..8a4fdc12b8 100644 --- a/sei-db/state_db/ss/snapshot/manager_test.go +++ b/sei-db/state_db/ss/snapshot/manager_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/sei-protocol/sei-chain/sei-db/config" - "github.com/sei-protocol/sei-chain/sei-db/controller" "github.com/stretchr/testify/require" ) @@ -23,7 +22,7 @@ func (*controlledScheduler) SupportsCheckpoint() bool { func (s *controlledScheduler) ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) { s.pending <- func() { if !shouldRun() { - done(controller.ErrCheckpointCanceled) + done(ErrCheckpointCanceled) return } if s.fail { @@ -146,11 +145,11 @@ func TestOpenClearsLeftoverHardlinkProbes(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(root, linkProbeName), nil, 0o600)) _, err := Open(Config{ - Name: "test", - Root: root, - SourceDirs: []string{source}, - Backend: config.PebbleDBBackend, - Scheduler: &controlledScheduler{pending: make(chan func(), 1)}, + Name: "test", + Root: root, + SourceDirs: []string{source}, + Backend: config.PebbleDBBackend, + Checkpointer: &controlledScheduler{pending: make(chan func(), 1)}, }) require.NoError(t, err) @@ -207,7 +206,7 @@ func openManagerWithFloor( Backend: config.PebbleDBBackend, KeepRecent: keepRecent, ExternalPruning: external, - Scheduler: scheduler, + Checkpointer: scheduler, Floor: floor, }) require.NoError(t, err)