Skip to content

move pebble checkpoints off of the execution goroutine #4039

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cody-littley wants to merge 1 commit into
base: main
Choose a base branch
from
Open

move pebble checkpoints off of the execution goroutine #4039

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions app/testdata/state-commit.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ FlatKVConfig.Fsync = bool(false)
FlatKVConfig.AsyncWriteBuffer = int(0)
FlatKVConfig.SnapshotInterval = uint32(10000)
FlatKVConfig.SnapshotKeepRecent = uint32(1)
FlatKVConfig.MaxSnapshotLagBlocks = uint32(512)
FlatKVConfig.ExternalPruning = bool(false)
FlatKVConfig.EnablePebbleMetrics = bool(true)
FlatKVConfig.EnableReadWriteMetrics = bool(false)
Expand All @@ -28,7 +29,7 @@ FlatKVConfig.AccountStoreConfig.EstimatedOverheadPerEntry = uint64(256)
FlatKVConfig.AccountStoreConfig.Name = string("account")
FlatKVConfig.AccountStoreConfig.MetricsEnabled = bool(true)
FlatKVConfig.AccountStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
FlatKVConfig.AccountStoreConfig.MaxUnflushedVersions = uint64(4)
FlatKVConfig.AccountStoreConfig.MaxUnflushedVersions = uint64(1024)

@yzang2019 yzang2019 Aug 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems to be a pretty big jump? Why do we need that big of backlog?

FlatKVConfig.AccountStoreConfig.TargetBytesPerFlush = uint64(4194304)
FlatKVConfig.AccountStoreConfig.ReservedPrefix = string("_meta/")
FlatKVConfig.AccountStoreConfig.FlushSync = bool(false)
Expand All @@ -42,7 +43,7 @@ FlatKVConfig.CodeStoreConfig.EstimatedOverheadPerEntry = uint64(256)
FlatKVConfig.CodeStoreConfig.Name = string("code")
FlatKVConfig.CodeStoreConfig.MetricsEnabled = bool(true)
FlatKVConfig.CodeStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
FlatKVConfig.CodeStoreConfig.MaxUnflushedVersions = uint64(4)
FlatKVConfig.CodeStoreConfig.MaxUnflushedVersions = uint64(1024)
FlatKVConfig.CodeStoreConfig.TargetBytesPerFlush = uint64(4194304)
FlatKVConfig.CodeStoreConfig.ReservedPrefix = string("_meta/")
FlatKVConfig.CodeStoreConfig.FlushSync = bool(false)
Expand All @@ -56,7 +57,7 @@ FlatKVConfig.StorageStoreConfig.EstimatedOverheadPerEntry = uint64(256)
FlatKVConfig.StorageStoreConfig.Name = string("storage")
FlatKVConfig.StorageStoreConfig.MetricsEnabled = bool(true)
FlatKVConfig.StorageStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
FlatKVConfig.StorageStoreConfig.MaxUnflushedVersions = uint64(4)
FlatKVConfig.StorageStoreConfig.MaxUnflushedVersions = uint64(1024)
FlatKVConfig.StorageStoreConfig.TargetBytesPerFlush = uint64(4194304)
FlatKVConfig.StorageStoreConfig.ReservedPrefix = string("_meta/")
FlatKVConfig.StorageStoreConfig.FlushSync = bool(false)
Expand All @@ -70,7 +71,7 @@ FlatKVConfig.MiscStoreConfig.EstimatedOverheadPerEntry = uint64(256)
FlatKVConfig.MiscStoreConfig.Name = string("misc")
FlatKVConfig.MiscStoreConfig.MetricsEnabled = bool(true)
FlatKVConfig.MiscStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
FlatKVConfig.MiscStoreConfig.MaxUnflushedVersions = uint64(4)
FlatKVConfig.MiscStoreConfig.MaxUnflushedVersions = uint64(1024)
FlatKVConfig.MiscStoreConfig.TargetBytesPerFlush = uint64(4194304)
FlatKVConfig.MiscStoreConfig.ReservedPrefix = string("_meta/")
FlatKVConfig.MiscStoreConfig.FlushSync = bool(false)
Expand Down
9 changes: 5 additions & 4 deletions sei-cosmos/server/config/testdata/server_config.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ StateCommit.FlatKVConfig.Fsync = bool(false)
StateCommit.FlatKVConfig.AsyncWriteBuffer = int(0)
StateCommit.FlatKVConfig.SnapshotInterval = uint32(10000)
StateCommit.FlatKVConfig.SnapshotKeepRecent = uint32(1)
StateCommit.FlatKVConfig.MaxSnapshotLagBlocks = uint32(512)
StateCommit.FlatKVConfig.ExternalPruning = bool(false)
StateCommit.FlatKVConfig.EnablePebbleMetrics = bool(true)
StateCommit.FlatKVConfig.EnableReadWriteMetrics = bool(false)
Expand All @@ -86,7 +87,7 @@ StateCommit.FlatKVConfig.AccountStoreConfig.EstimatedOverheadPerEntry = uint64(2
StateCommit.FlatKVConfig.AccountStoreConfig.Name = string("account")
StateCommit.FlatKVConfig.AccountStoreConfig.MetricsEnabled = bool(true)
StateCommit.FlatKVConfig.AccountStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
StateCommit.FlatKVConfig.AccountStoreConfig.MaxUnflushedVersions = uint64(4)
StateCommit.FlatKVConfig.AccountStoreConfig.MaxUnflushedVersions = uint64(1024)
StateCommit.FlatKVConfig.AccountStoreConfig.TargetBytesPerFlush = uint64(4194304)
StateCommit.FlatKVConfig.AccountStoreConfig.ReservedPrefix = string("_meta/")
StateCommit.FlatKVConfig.AccountStoreConfig.FlushSync = bool(false)
Expand All @@ -100,7 +101,7 @@ StateCommit.FlatKVConfig.CodeStoreConfig.EstimatedOverheadPerEntry = uint64(256)
StateCommit.FlatKVConfig.CodeStoreConfig.Name = string("code")
StateCommit.FlatKVConfig.CodeStoreConfig.MetricsEnabled = bool(true)
StateCommit.FlatKVConfig.CodeStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
StateCommit.FlatKVConfig.CodeStoreConfig.MaxUnflushedVersions = uint64(4)
StateCommit.FlatKVConfig.CodeStoreConfig.MaxUnflushedVersions = uint64(1024)
StateCommit.FlatKVConfig.CodeStoreConfig.TargetBytesPerFlush = uint64(4194304)
StateCommit.FlatKVConfig.CodeStoreConfig.ReservedPrefix = string("_meta/")
StateCommit.FlatKVConfig.CodeStoreConfig.FlushSync = bool(false)
Expand All @@ -114,7 +115,7 @@ StateCommit.FlatKVConfig.StorageStoreConfig.EstimatedOverheadPerEntry = uint64(2
StateCommit.FlatKVConfig.StorageStoreConfig.Name = string("storage")
StateCommit.FlatKVConfig.StorageStoreConfig.MetricsEnabled = bool(true)
StateCommit.FlatKVConfig.StorageStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
StateCommit.FlatKVConfig.StorageStoreConfig.MaxUnflushedVersions = uint64(4)
StateCommit.FlatKVConfig.StorageStoreConfig.MaxUnflushedVersions = uint64(1024)
StateCommit.FlatKVConfig.StorageStoreConfig.TargetBytesPerFlush = uint64(4194304)
StateCommit.FlatKVConfig.StorageStoreConfig.ReservedPrefix = string("_meta/")
StateCommit.FlatKVConfig.StorageStoreConfig.FlushSync = bool(false)
Expand All @@ -128,7 +129,7 @@ StateCommit.FlatKVConfig.MiscStoreConfig.EstimatedOverheadPerEntry = uint64(256)
StateCommit.FlatKVConfig.MiscStoreConfig.Name = string("misc")
StateCommit.FlatKVConfig.MiscStoreConfig.MetricsEnabled = bool(true)
StateCommit.FlatKVConfig.MiscStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
StateCommit.FlatKVConfig.MiscStoreConfig.MaxUnflushedVersions = uint64(4)
StateCommit.FlatKVConfig.MiscStoreConfig.MaxUnflushedVersions = uint64(1024)
StateCommit.FlatKVConfig.MiscStoreConfig.TargetBytesPerFlush = uint64(4194304)
StateCommit.FlatKVConfig.MiscStoreConfig.ReservedPrefix = string("_meta/")
StateCommit.FlatKVConfig.MiscStoreConfig.FlushSync = bool(false)
Expand Down
41 changes: 41 additions & 0 deletions sei-cosmos/storev2/rootmulti/flatkv_helpers_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"encoding/binary"
"errors"
"testing"
"time"

"github.com/sei-protocol/sei-chain/sei-cosmos/store/types"
errorutils "github.com/sei-protocol/sei-chain/sei-db/common/errors"
Expand All @@ -19,9 +20,49 @@ import (
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype"
scmemiavl "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl"
sctypes "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types"
sscomposite "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/composite"
"github.com/stretchr/testify/require"
)

// requireStateStoreCaughtUp waits for the state store to reach version.
//
// The state store applies a block asynchronously: ApplyChangesetAsync hands it to a background writer
// whose queue is AsyncWriteBuffer deep, so a commit returning says nothing about the state store having
// applied that block. Any test that asserts a state store version, or that rolls back — rollback
// refuses a state store sitting below the target — has to establish this first.
func requireStateStoreCaughtUp(t *testing.T, store *Store, version int64) {
t.Helper()
require.NotNil(t, store.ssStore)
require.Eventually(t, func() bool {
return store.ssStore.GetLatestVersion() >= version
}, 10*time.Second, 5*time.Millisecond,
"state store never caught up to version %d", version)
}

// requireStateStoreSnapshotAtOrBelow waits for a state store snapshot at or below target to be on
// disk, which is what rolling the state store back to target requires as its base.
//
// State store snapshots are published asynchronously and their coordinator declines a boundary while
// another snapshot is still in flight, so committing past a boundary does not mean its snapshot exists.
// With a per-block interval only the first boundary is usually accepted, the rest reporting "in_flight".
func requireStateStoreSnapshotAtOrBelow(t *testing.T, homeDir string, target int64) {
t.Helper()
root := utils.GetStateStoreSnapshotsPath(homeDir)
require.Eventually(t, func() bool {
versions, err := sscomposite.ListSnapshotVersions(root)
if err != nil {
return false
}
for _, version := range versions {
if version > 0 && version <= target {
return true
}
}
return false
}, 10*time.Second, 5*time.Millisecond,
"no state store snapshot at or below version %d was ever published under %s", target, root)
}

// ---------------------------------------------------------------------------
// Config helpers
// ---------------------------------------------------------------------------
Expand Down
5 changes: 5 additions & 0 deletions sei-cosmos/storev2/rootmulti/flatkv_recovery_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ func TestRollbackToVersionRollsBackStateStore(t *testing.T) {
for block := 1; block <= 5; block++ {
simulateBlock(t, store, storeKeys, block, evmData)
}
requireStateStoreCaughtUp(t, store, 5)
requireStateStoreSnapshotAtOrBelow(t, dir, 3)

require.NoError(t, store.RollbackToVersion(3))
require.Equal(t, int64(3), store.LastCommitID().Version)
Expand Down Expand Up @@ -110,6 +112,7 @@ func TestRollbackToVersionWithoutStateStoreSnapshots(t *testing.T) {
for block := 1; block <= 5; block++ {
simulateBlock(t, store, storeKeys, block, evmData)
}
requireStateStoreCaughtUp(t, store, 5)

require.NoError(t, store.RollbackToVersion(3))
require.Equal(t, int64(3), store.LastCommitID().Version)
Expand Down Expand Up @@ -140,6 +143,8 @@ func TestRollbackToVersionProceedsWhenStateStoreCannotFollow(t *testing.T) {
for block := 1; block <= 5; block++ {
simulateBlock(t, store, storeKeys, block, evmData)
}
requireStateStoreCaughtUp(t, store, 5)

rollbackable := store.ssStore
store.ssStore = nonRollbackableStateStore{rollbackable}

Expand Down
2 changes: 1 addition & 1 deletion sei-db/db_engine/view/view_manager_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func DefaultViewManagerConfig(name string, reservedPrefix string) *ViewManagerCo
Name: name,
MetricsEnabled: true,
MetricsScrapeIntervalSeconds: 10,
MaxUnflushedVersions: 4,
MaxUnflushedVersions: 1024,

@seidroid seidroid Bot Aug 27, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[suggestion] Raising the default from 4 to 1024 applies to all four FlatKV view managers, and this is the only bound on that backlog: MaxSize above is documented as constraining "only the DB read-cache, not view data, since views cannot be freed without compromising consistency semantics". So the cap is a version count with no byte-based counterpart — 1024 block diffs of arbitrary size may now be resident per manager, where previously it was 4.

The increase is required by the design (with 4, Commit would stall inside view-manager backpressure for the whole checkpoint, defeating the point of the PR), and MaxSnapshotLagBlocks caps the snapshot-driven backlog at ~513. But the new value also governs the unrelated case this field was written for — Pebble being the bottleneck — where nothing bounds the accumulation at 512, and a slow-flush episode can now hold 256× as much in memory before backpressure engages.

Worth recording the reasoning here (why 1024 rather than something just above MaxSnapshotLagBlocks) and noting the relationship between the two knobs, since they are set in different packages and only stay consistent by hand.

@blindchaser blindchaser Aug 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

any OOM risk here to MaxUnflushedVersions to 1024?

TargetBytesPerFlush: unit.MB * 4,
ReservedPrefix: reservedPrefix,
FlushSync: false,
Expand Down
12 changes: 12 additions & 0 deletions sei-db/state_db/sc/composite/store_auto_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,18 @@ func TestComposite_Auto_ExportImportRoundTrip(t *testing.T) {
runBlocks(t, src, workload, 1)
h := src.Version()

// FlatKV writes snapshots off the execution thread, so a commit returning no longer means its
// snapshot churn has finished. Exporting reads a snapshot by copying its directory, and pruning the
// oldest snapshot is the last step of publishing a new one, so without this wait the writer can
// delete the directory the export is part way through copying.
//
// Reached through the concrete store because quiescing the writer is not part of the flatkv.Store
// abstraction: no production caller needs it, and this test only does because it drives commits and
// reads from one goroutine and so has a quiet period to establish.
flatKVStore, ok := src.flatKV.(*flatkv.CommitStore)
require.True(t, ok)
require.NoError(t, flatKVStore.FlushSnapshots())

exp, err := src.Exporter(h)
require.NoError(t, err)
items := drainCompositeExporter(t, exp)
Expand Down
1 change: 0 additions & 1 deletion sei-db/state_db/sc/composite/store_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ func (f *failingEVMStore) RootHash() ([]byte, int64) { retur
func (f *failingEVMStore) Version() int64 { return 0 }
func (f *failingEVMStore) PendingVersion() int64 { return 0 }
func (f *failingEVMStore) GetLatestVersion() (int64, error) { return 0, nil }
func (f *failingEVMStore) WriteSnapshot(string) error { return nil }
func (f *failingEVMStore) Rollback(int64) error { return nil }
func (f *failingEVMStore) Exporter(int64) (types.Exporter, error) { return nil, nil }
func (f *failingEVMStore) Importer(int64) (types.Importer, error) { return nil, nil }
Expand Down
3 changes: 0 additions & 3 deletions sei-db/state_db/sc/flatkv/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,9 +152,6 @@ type Store interface {
// inspect the store's height without taking ownership of it.
GetLatestVersion() (int64, error)

// WriteSnapshot writes a complete snapshot to dir.
WriteSnapshot(dir string) error

// Rollback rewinds a store opened with LoadLatest to targetVersion and prunes everything above it:
// snapshots, WAL blocks and committed state. It is the only way to move a committable store backwards,
// and the result keeps committing from targetVersion+1. An unreachable target is rejected before
Expand Down
27 changes: 12 additions & 15 deletions sei-db/state_db/sc/flatkv/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,6 @@ import (
"github.com/sei-protocol/sei-chain/sei-db/db_engine/view"
)

const (
DefaultSnapshotInterval uint32 = 10000
DefaultSnapshotKeepRecent uint32 = 1
)

// Config defines configuration for the FlatKV (EVM) commit store.
type Config struct {
// DataDir is the root directory for the FlatKV data files.
Expand All @@ -22,27 +17,32 @@ type Config struct {
// Fsync controls whether every view manager's flush is fsync'd. It overwrites each store
// config's FlushSync, so the four databases are always synced alike. The state WAL is
// unaffected and always writes NoSync.
// Default: false
Fsync bool `mapstructure:"fsync"`

// AsyncWriteBuffer defines the size of the async write buffer for data DBs.
// Set <= 0 for synchronous writes.
// Default: 0 (synchronous)
AsyncWriteBuffer int `mapstructure:"async-write-buffer"`

// SnapshotInterval defines how often (in blocks) a PebbleDB checkpoint
// snapshot is taken. 0 disables auto-snapshots.
// Without periodic snapshots the WAL grows unbounded and every restart
// replays the entire history from snapshot-0.
// Default: 10000
SnapshotInterval uint32 `mapstructure:"snapshot-interval"`

// SnapshotKeepRecent defines how many old snapshots to keep besides the
// latest one. 0 means keep only the current snapshot (no old snapshots).
// Ignored entirely when ExternalPruning is set.
// Default: 1
SnapshotKeepRecent uint32 `mapstructure:"snapshot-keep-recent"`

// MaxSnapshotLagBlocks is how many committed blocks may queue up behind a snapshot that is still
// being written before Commit blocks. A value below 1 is treated as 1.
//
// A snapshot being written holds every database pinned at its own height, so no later block can
// reach disk until it completes, and each one is retained in memory meanwhile. This bounds how far
// that can run, trading a pause in block production for the memory the backlog would otherwise
// consume. It bounds blocks rather than bytes, so it mitigates exhaustion rather than preventing it.
MaxSnapshotLagBlocks uint32 `mapstructure:"max-snapshot-lag-blocks"`

@seidroid seidroid Bot Aug 27, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[suggestion] The mapstructure:"max-snapshot-lag-blocks" tag is inert: neither sei-cosmos/server/config.GetConfig (which explicitly reads five other state-commit.flatkv.* keys — fsync, async-write-buffer, snapshot-interval, snapshot-keep-recent, enable-read-write-metrics) nor app/parseSCConfigs reads this key, and the Viper path ignores unknown TOML keys. An operator who sets state-commit.flatkv.max-snapshot-lag-blocks in app.toml gets silence, and the queue stays pinned at 512.

This knob is the whole of the writer's backpressure and the one lever an operator has when a checkpoint outruns block production, so it is the flatkv field most worth wiring rather than least. Suggest adding the guarded read alongside the other four in GetConfig, and a row in sei-cosmos/server/config/config_fuzz_test.go ({Key: "state-commit.flatkv.max-snapshot-lag-blocks", Path: "StateCommit.FlatKVConfig.MaxSnapshotLagBlocks", Set: ...}) per testutil/configtest/AGENTS.md. If it is deliberately not operator-tunable for now, dropping the tag (as ExternalPruning does with mapstructure:"-") records that decision instead of implying one that does not hold.

@blindchaser blindchaser Aug 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1


// ExternalPruning hands retention to the StorageGarbageCollector: the store stops pruning its
// own snapshots (SnapshotKeepRecent) and stops truncating the state WAL.
//
Expand All @@ -51,16 +51,12 @@ type Config struct {
//
// With it on, snapshots are retained by height rather than by count, so the number kept becomes
// RollbackWindow / SnapshotInterval instead of SnapshotKeepRecent + 1.
//
// Default: false
ExternalPruning bool `mapstructure:"-"`

// EnablePebbleMetrics defines if the Pebble metrics should be enabled.
// Default: true
EnablePebbleMetrics bool `mapstructure:"enable-pebble-metrics"`

// EnableReadWriteMetrics emits simple estimated read/write counters for FlatKV's Pebble DBs.
// Default: false
EnableReadWriteMetrics bool `mapstructure:"enable-read-write-metrics"`

// AccountDBConfig defines the PebbleDB configuration for the account database.
Expand Down Expand Up @@ -133,8 +129,9 @@ func DefaultConfig() *Config {
cfg := &Config{
Fsync: false,
AsyncWriteBuffer: 0,
SnapshotInterval: DefaultSnapshotInterval,
SnapshotKeepRecent: DefaultSnapshotKeepRecent,
SnapshotInterval: 10000,
SnapshotKeepRecent: 1,
MaxSnapshotLagBlocks: 512,
EnablePebbleMetrics: true,
AccountDBConfig: pebbledb.DefaultConfig(),
AccountStoreConfig: defaultStoreConfig("account"),
Expand Down
4 changes: 2 additions & 2 deletions sei-db/state_db/sc/flatkv/config/flatkv_test_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ func DefaultTestConfig(t *testing.T) *Config {
t.Helper()
return &Config{
DataDir: filepath.Join(t.TempDir(), "flatkv"),
SnapshotInterval: DefaultSnapshotInterval,
SnapshotKeepRecent: DefaultSnapshotKeepRecent,
SnapshotInterval: 10000,
SnapshotKeepRecent: 1,
AccountDBConfig: smallTestPebbleConfig(),
AccountStoreConfig: smallTestViewManagerConfig("account"),
CodeDBConfig: smallTestPebbleConfig(),
Expand Down
6 changes: 3 additions & 3 deletions sei-db/state_db/sc/flatkv/importer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@ func (imp *KVImporter) AddNode(node *types.SnapshotNode) {
// Abort tears down the worker pipeline without finalizing the import.
// It records reason as the first pipeline error (so any in-flight worker
// also bails fast) and then runs Close, which observes the non-nil error
// and skips FinalizeImport / WriteSnapshot. The on-disk FlatKV directory
// and skips FinalizeImport / outOfBandSnapshot. The on-disk FlatKV directory
// is left at its pre-import committed version, allowing the operator to
// retry without --force.
//
Expand All @@ -325,7 +325,7 @@ func (imp *KVImporter) Abort(reason error) error {
// Close on both the success and error paths.
//
// If the first pipeline error has already been recorded (either by a
// worker or by Abort), Close skips FinalizeImport / WriteSnapshot so the
// worker or by Abort), Close skips FinalizeImport / outOfBandSnapshot so the
// store stays at its pre-import version.
func (imp *KVImporter) Close() error {
imp.finishOnce.Do(func() {
Expand Down Expand Up @@ -375,7 +375,7 @@ func (imp *KVImporter) Close() error {
// Write a snapshot so the imported data survives store reopen / restart.
// Import bypasses the WAL, so without a snapshot the next LoadLatest
// would clone from the pre-import snapshot and lose all imported data.
if err = imp.store.WriteSnapshot(""); err != nil {
if err = imp.store.outOfBandSnapshot(); err != nil {
err = fmt.Errorf("failed to import when writing snapshot: %w", err)
return
}
Expand Down
6 changes: 3 additions & 3 deletions sei-db/state_db/sc/flatkv/lthash_correctness_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1292,7 +1292,7 @@ func TestLtHashSnapshotCatchupFullScan(t *testing.T) {
for i := byte(1); i <= 3; i++ {
commitMixedState(t, s1, i)
}
require.NoError(t, s1.WriteSnapshot(""))
require.NoError(t, s1.outOfBandSnapshot())

// Blocks 4-7: more state (will need WAL catchup on reopen)
for i := byte(4); i <= 7; i++ {
Expand Down Expand Up @@ -1339,7 +1339,7 @@ func TestLtHashRollbackFullScan(t *testing.T) {
for i := byte(1); i <= 5; i++ {
commitMixedState(t, s, i)
}
require.NoError(t, s.WriteSnapshot(""))
require.NoError(t, s.outOfBandSnapshot())
hashAtV5 := rootHash(s)

for i := byte(6); i <= 8; i++ {
Expand Down Expand Up @@ -1418,7 +1418,7 @@ func TestLtHashMultipleRollbacks(t *testing.T) {
for i := byte(1); i <= 5; i++ {
commitMixedState(t, s, i)
}
require.NoError(t, s.WriteSnapshot(""))
require.NoError(t, s.outOfBandSnapshot())

// Original timeline: blocks 6-8 with round byte as-is
for i := byte(6); i <= 8; i++ {
Expand Down
7 changes: 7 additions & 0 deletions sei-db/state_db/sc/flatkv/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ var (
CatchupLatency metric.Float64Histogram
CatchupReplayNumBlocks metric.Int64Counter
SnapshotWriteLatency metric.Float64Histogram
SnapshotQueueDepth metric.Int64Gauge
SnapshotPruneLatency metric.Float64Histogram
SnapshotPruneAttempts metric.Int64Counter
CurrentSnapshotHeight metric.Int64Gauge
Expand Down Expand Up @@ -98,6 +99,12 @@ var (
metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(commonmetrics.LongLatencyBuckets...),
)),
SnapshotQueueDepth: must(flatkvMeter.Int64Gauge(
"flatkv_snapshot_queue_depth",
metric.WithDescription(
"Committed blocks queued behind a FlatKV snapshot that is still being written"),
metric.WithUnit("{count}"),
)),
SnapshotPruneLatency: must(flatkvMeter.Float64Histogram(
"flatkv_snapshot_prune_latency",
metric.WithDescription("Time taken to prune FlatKV snapshots"),
Expand Down
4 changes: 2 additions & 2 deletions sei-db/state_db/sc/flatkv/perdb_lthash_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ func TestPerDBLtHashCatchupReplay(t *testing.T) {

commitMixedState(t, s1, 1)
commitMixedState(t, s1, 2)
require.NoError(t, s1.WriteSnapshot(""))
require.NoError(t, s1.outOfBandSnapshot())

commitMixedState(t, s1, 3)
commitMixedState(t, s1, 4)
Expand Down Expand Up @@ -385,7 +385,7 @@ func TestPerDBLtHashRollback(t *testing.T) {
commitMixedState(t, s, 1)
commitMixedState(t, s, 2)
commitMixedState(t, s, 3)
require.NoError(t, s.WriteSnapshot(""))
require.NoError(t, s.outOfBandSnapshot())

commitMixedState(t, s, 4)
commitMixedState(t, s, 5)
Expand Down
Loading
Loading