diff --git a/sei-db/state_db/sc/migration/README.md b/sei-db/state_db/sc/migration/README.md index 9b6b56eb95..56337e9598 100644 --- a/sei-db/state_db/sc/migration/README.md +++ b/sei-db/state_db/sc/migration/README.md @@ -71,7 +71,8 @@ The data path during an active migration is built from four pieces: - [router_builder.go](router_builder.go) holds the per-mode builders (`buildMigrateEVMRouter`, `buildEVMMigratedRouter`, `buildMigrateAllButBankRouter`, `buildAllMigratedButBankRouter`, `buildMigrateBankRouter`); the per-mode ASCII data-flow diagrams in that file are the operational spec for "what writes where on each block." - [thread_safe_router.go](thread_safe_router.go) wraps a built router so external `Read` calls and `ApplyChangeSets` are serialized. -The `MigrationManager` itself is *not* safe for concurrent use; callers must not share one across goroutines without external synchronization. [`BuildRouter`](router_builder.go) wraps every router it returns in [`NewThreadSafeRouter`](thread_safe_router.go), so callers that go through `BuildRouter` get a thread-safe handle for free. +The `MigrationManager` itself is *not* safe for concurrent use; callers must not share one across goroutines without external synchronization. [`BuildRouter`](router_builder.go) wraps every non-thread safe router it returns in [`NewThreadSafeRouter`](thread_safe_router.go), so callers that go through `BuildRouter` get a thread-safe handle for free. Note that some +routers are thread innately safe, and so do not need to be wrapped. ## Migration metadata diff --git a/sei-db/state_db/sc/migration/dual_write_router.go b/sei-db/state_db/sc/migration/dual_write_router.go new file mode 100644 index 0000000000..f247f0d1cc --- /dev/null +++ b/sei-db/state_db/sc/migration/dual_write_router.go @@ -0,0 +1,102 @@ +package migration + +import ( + "fmt" + + ics23 "github.com/confio/ics23/go" + "github.com/sei-protocol/sei-chain/sei-db/proto" + db "github.com/tendermint/tm-db" +) + +var _ Router = (*TestOnlyDualWriteRouter)(nil) + +// A router that dual-writes traffic, sending each batch of changesets to both backends. Read +// requests, requests for proofs, and requests for iteration are not dual-written, and are instead +// served exclusively by the primary backend. +// +// CRITICAL: this is a test-only router and should never be deployed to production machines. +type TestOnlyDualWriteRouter struct { + primary *Route + secondary DBWriter +} + +// Create a new test-only dual-write router. +// +// CRITICAL: this is a test-only router and should never be deployed to production machines. +func NewTestOnlyDualWriteRouter( + // Read, proof, and iteration traffic is served by this route, and writes are also sent here. + // Module names associated with this route are ignored; this route forwards all regardless of the module names. + primary *Route, + // Write traffic is dual-written and also sent here. Reads, proofs, and iteration are not sent here. + secondary DBWriter, +) (*TestOnlyDualWriteRouter, error) { + + if primary == nil { + return nil, fmt.Errorf("primary must not be nil") + } + if primary.proofBuilder == nil { + return nil, fmt.Errorf("primary proof builder must not be nil") + } + if primary.iteratorBuilder == nil { + return nil, fmt.Errorf("primary iterator builder must not be nil") + } + if secondary == nil { + return nil, fmt.Errorf("secondary must not be nil") + } + + return &TestOnlyDualWriteRouter{primary: primary, secondary: secondary}, nil +} + +func (t *TestOnlyDualWriteRouter) ApplyChangeSets(changesets []*proto.NamedChangeSet) error { + err := t.primary.writer(changesets) + if err != nil { + return fmt.Errorf("primary writer: %w", err) + } + + err = t.secondary(changesets) + if err != nil { + return fmt.Errorf("secondary writer: %w", err) + } + + return nil +} + +func (t *TestOnlyDualWriteRouter) GetProof(store string, key []byte) (*ics23.CommitmentProof, error) { + proof, err := t.primary.proofBuilder(store, key) + if err != nil { + return nil, fmt.Errorf("primary proof builder: %w", err) + } + return proof, nil +} + +func (t *TestOnlyDualWriteRouter) Iterator( + store string, + start []byte, + end []byte, + ascending bool, +) (db.Iterator, error) { + iterator, err := t.primary.iteratorBuilder(store, start, end, ascending) + if err != nil { + return nil, fmt.Errorf("primary iterator builder: %w", err) + } + return iterator, nil +} + +func (t *TestOnlyDualWriteRouter) Read(store string, key []byte) ([]byte, bool, error) { + value, found, err := t.primary.reader(store, key) + if err != nil { + return nil, false, fmt.Errorf("primary reader: %w", err) + } + return value, found, nil +} + +// BuildRoute returns a Route that dispatches the given module names to +// this DualWriteRouter. Reads, writes, iteration and proof requests +// for those modules will all flow through this dual-write router. +// +// Module names must be unique; NewRoute's validation rules apply. The +// returned Route may be passed to NewModuleRouter alongside other +// Routes to compose multi-database setups. +func (t *TestOnlyDualWriteRouter) BuildRoute(moduleNames ...string) (*Route, error) { + return NewRoute(t.Read, t.ApplyChangeSets, t.Iterator, t.GetProof, moduleNames...) +} diff --git a/sei-db/state_db/sc/migration/dual_write_router_test.go b/sei-db/state_db/sc/migration/dual_write_router_test.go new file mode 100644 index 0000000000..5a10b869a3 --- /dev/null +++ b/sei-db/state_db/sc/migration/dual_write_router_test.go @@ -0,0 +1,610 @@ +package migration + +import ( + "errors" + "testing" + + ics23 "github.com/confio/ics23/go" + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" +) + +// noopIter returns (nil, nil) for any input. Used to satisfy the +// constructor's strict non-nil iterator-builder requirement when a test +// does not exercise iteration. +func noopIter(_ string, _, _ []byte, _ bool) (dbm.Iterator, error) { + return nil, nil +} + +// noopProof returns (nil, nil) for any input. Used to satisfy the +// constructor's strict non-nil proof-builder requirement when a test +// does not exercise proofs. +func noopProof(_ string, _ []byte) (*ics23.CommitmentProof, error) { + return nil, nil +} + +// --- Constructor tests --- + +func TestNewTestOnlyDualWriteRouter_Success(t *testing.T) { + primary := newRouteWithBuilders(t, newMockDB(), noopIter, noopProof, "evm") + r, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + require.NotNil(t, r) +} + +func TestNewTestOnlyDualWriteRouter_NilPrimaryRejected(t *testing.T) { + r, err := NewTestOnlyDualWriteRouter(nil, newMockDB().writer()) + require.Error(t, err) + require.Nil(t, r) + require.Contains(t, err.Error(), "primary") +} + +func TestNewTestOnlyDualWriteRouter_NilSecondaryRejected(t *testing.T) { + primary := newRouteWithBuilders(t, newMockDB(), noopIter, noopProof, "evm") + r, err := NewTestOnlyDualWriteRouter(primary, nil) + require.Error(t, err) + require.Nil(t, r) + require.Contains(t, err.Error(), "secondary") +} + +// TestNewTestOnlyDualWriteRouter_NilProofBuilderRejected pins the +// strict-at-construction contract: a primary with a nil proofBuilder is +// rejected. This is intentionally stricter than ModuleRouter (which +// errors lazily at GetProof time) and means a route built by +// routeToFlatKV (which deliberately passes nil for both builders) cannot +// be the primary today. +func TestNewTestOnlyDualWriteRouter_NilProofBuilderRejected(t *testing.T) { + primary := newRouteWithBuilders(t, newMockDB(), noopIter, nil, "evm") + r, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.Error(t, err) + require.Nil(t, r) + require.Contains(t, err.Error(), "proof builder") +} + +// TestNewTestOnlyDualWriteRouter_NilIteratorBuilderRejected pins the +// strict-at-construction contract: a primary with a nil iteratorBuilder +// is rejected. Same rationale as the proofBuilder case above. +func TestNewTestOnlyDualWriteRouter_NilIteratorBuilderRejected(t *testing.T) { + primary := newRouteWithBuilders(t, newMockDB(), nil, noopProof, "evm") + r, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.Error(t, err) + require.Nil(t, r) + require.Contains(t, err.Error(), "iterator builder") +} + +// --- ApplyChangeSets tests --- +// +// Ordering and short-circuit behavior are NOT pinned by these tests: +// callers are expected to restore both backends to a safe snapshot on +// any error, so the dual-write router makes no guarantees about which +// writer runs first or whether one runs when the other fails. + +func TestApplyChangeSets_FansOutToBoth(t *testing.T) { + primaryDB := newMockDB() + secondaryDB := newMockDB() + primary := newRouteWithBuilders(t, primaryDB, noopIter, noopProof, "evm") + r, err := NewTestOnlyDualWriteRouter(primary, secondaryDB.writer()) + require.NoError(t, err) + + require.NoError(t, r.ApplyChangeSets([]*proto.NamedChangeSet{ + namedCS("evm", kv("k", "v")), + })) + + pv, ok := primaryDB.get("evm", "k") + require.True(t, ok, "primary must receive the write") + require.Equal(t, []byte("v"), pv) + + sv, ok := secondaryDB.get("evm", "k") + require.True(t, ok, "secondary must receive the write") + require.Equal(t, []byte("v"), sv) +} + +func TestApplyChangeSets_PrimaryErrorPropagated(t *testing.T) { + sentinel := errors.New("primary boom") + primary, err := NewRoute( + newMockDB().reader(), + failWriter(sentinel), + noopIter, noopProof, + "evm", + ) + require.NoError(t, err) + r, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + + applyErr := r.ApplyChangeSets([]*proto.NamedChangeSet{ + namedCS("evm", kv("k", "v")), + }) + require.Error(t, applyErr) + require.ErrorIs(t, applyErr, sentinel) + require.Contains(t, applyErr.Error(), "primary writer") +} + +func TestApplyChangeSets_SecondaryErrorPropagated(t *testing.T) { + sentinel := errors.New("secondary boom") + primary := newRouteWithBuilders(t, newMockDB(), noopIter, noopProof, "evm") + r, err := NewTestOnlyDualWriteRouter(primary, failWriter(sentinel)) + require.NoError(t, err) + + applyErr := r.ApplyChangeSets([]*proto.NamedChangeSet{ + namedCS("evm", kv("k", "v")), + }) + require.Error(t, applyErr) + require.ErrorIs(t, applyErr, sentinel) + require.Contains(t, applyErr.Error(), "secondary writer") +} + +// TestApplyChangeSets_ForwardsAllChangesetsRegardlessOfPrimaryModules +// pins the deliberate "no module-filtering" behavior: the dual-write +// router hands every changeset to both writers regardless of what +// modules the primary route claims to handle. The composition story +// relies on an outer ModuleRouter doing the gating. +func TestApplyChangeSets_ForwardsAllChangesetsRegardlessOfPrimaryModules(t *testing.T) { + primaryDB := newMockDB() + secondaryDB := newMockDB() + primary := newRouteWithBuilders(t, primaryDB, noopIter, noopProof, "evm") + r, err := NewTestOnlyDualWriteRouter(primary, secondaryDB.writer()) + require.NoError(t, err) + + require.NoError(t, r.ApplyChangeSets([]*proto.NamedChangeSet{ + namedCS("evm", kv("ek", "ev")), + namedCS("bank", kv("bk", "bv")), + })) + + for _, db := range []*mockDB{primaryDB, secondaryDB} { + v, ok := db.get("evm", "ek") + require.True(t, ok, "evm changeset must reach both writers") + require.Equal(t, []byte("ev"), v) + v, ok = db.get("bank", "bk") + require.True(t, ok, "bank changeset must reach both writers despite primary listing only evm") + require.Equal(t, []byte("bv"), v) + } +} + +func TestApplyChangeSets_NilAndEmptyChangesetsForwarded(t *testing.T) { + primaryCalls := 0 + primary, err := NewRoute( + newMockDB().reader(), + func(_ []*proto.NamedChangeSet) error { + primaryCalls++ + return nil + }, + noopIter, noopProof, + "evm", + ) + require.NoError(t, err) + + secondaryCalls := 0 + secondary := func(_ []*proto.NamedChangeSet) error { + secondaryCalls++ + return nil + } + + r, err := NewTestOnlyDualWriteRouter(primary, secondary) + require.NoError(t, err) + + require.NoError(t, r.ApplyChangeSets(nil)) + require.NoError(t, r.ApplyChangeSets([]*proto.NamedChangeSet{})) + + require.Equal(t, 2, primaryCalls) + require.Equal(t, 2, secondaryCalls) +} + +// --- Read tests --- + +func TestRead_DelegatesToPrimary(t *testing.T) { + primaryDB := newMockDB() + primaryDB.seed(map[string]map[string][]byte{"evm": {"k": []byte("v")}}) + primary := newRouteWithBuilders(t, primaryDB, noopIter, noopProof, "evm") + r, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + + val, found, err := r.Read("evm", []byte("k")) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, []byte("v"), val) +} + +func TestRead_NotFoundReturnsFalse(t *testing.T) { + primary := newRouteWithBuilders(t, newMockDB(), noopIter, noopProof, "evm") + r, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + + val, found, err := r.Read("evm", []byte("missing")) + require.NoError(t, err) + require.False(t, found) + require.Nil(t, val) +} + +func TestRead_PrimaryErrorWrapped(t *testing.T) { + sentinel := errors.New("disk on fire") + primary, err := NewRoute( + failReader(sentinel), + newMockDB().writer(), + noopIter, noopProof, + "evm", + ) + require.NoError(t, err) + r, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + + val, found, readErr := r.Read("evm", []byte("k")) + require.Error(t, readErr) + require.ErrorIs(t, readErr, sentinel) + require.Contains(t, readErr.Error(), "primary reader") + require.False(t, found) + require.Nil(t, val) +} + +func TestRead_DoesNotConsultSecondary(t *testing.T) { + primaryDB := newMockDB() + primaryDB.seed(map[string]map[string][]byte{"evm": {"k": []byte("v")}}) + primary := newRouteWithBuilders(t, primaryDB, noopIter, noopProof, "evm") + + secondaryCalls := 0 + secondary := func(_ []*proto.NamedChangeSet) error { + secondaryCalls++ + return nil + } + r, err := NewTestOnlyDualWriteRouter(primary, secondary) + require.NoError(t, err) + + _, _, err = r.Read("evm", []byte("k")) + require.NoError(t, err) + _, _, err = r.Read("evm", []byte("missing")) + require.NoError(t, err) + require.Equal(t, 0, secondaryCalls, "Read must never invoke the secondary writer") +} + +// --- Iterator tests --- + +func TestIterator_DelegatesToPrimary(t *testing.T) { + var ( + gotStore string + gotStart []byte + gotEnd []byte + gotAscending bool + ) + recordingIter := func(store string, start []byte, end []byte, ascending bool) (dbm.Iterator, error) { + gotStore = store + gotStart = start + gotEnd = end + gotAscending = ascending + return nil, nil + } + primary := newRouteWithBuilders(t, newMockDB(), recordingIter, noopProof, "evm") + r, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + + iter, err := r.Iterator("evm", []byte("a"), []byte("z"), true) + require.NoError(t, err) + require.Nil(t, iter) + require.Equal(t, "evm", gotStore) + require.Equal(t, []byte("a"), gotStart) + require.Equal(t, []byte("z"), gotEnd) + require.True(t, gotAscending) +} + +func TestIterator_PrimaryErrorWrapped(t *testing.T) { + sentinel := errors.New("iterator boom") + failingIter := func(_ string, _, _ []byte, _ bool) (dbm.Iterator, error) { + return nil, sentinel + } + primary := newRouteWithBuilders(t, newMockDB(), failingIter, noopProof, "evm") + r, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + + iter, iterErr := r.Iterator("evm", []byte("a"), []byte("z"), true) + require.Error(t, iterErr) + require.ErrorIs(t, iterErr, sentinel) + require.Contains(t, iterErr.Error(), "primary iterator builder") + require.Nil(t, iter) +} + +// --- GetProof tests --- + +func TestGetProof_DelegatesToPrimary(t *testing.T) { + var ( + gotStore string + gotKey []byte + ) + sentinelProof := &ics23.CommitmentProof{} + recordingProof := func(store string, key []byte) (*ics23.CommitmentProof, error) { + gotStore = store + gotKey = key + return sentinelProof, nil + } + primary := newRouteWithBuilders(t, newMockDB(), noopIter, recordingProof, "evm") + r, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + + proof, err := r.GetProof("evm", []byte("k")) + require.NoError(t, err) + require.Same(t, sentinelProof, proof) + require.Equal(t, "evm", gotStore) + require.Equal(t, []byte("k"), gotKey) +} + +func TestGetProof_PrimaryErrorWrapped(t *testing.T) { + sentinel := errors.New("proof boom") + failingProof := func(_ string, _ []byte) (*ics23.CommitmentProof, error) { + return nil, sentinel + } + primary := newRouteWithBuilders(t, newMockDB(), noopIter, failingProof, "evm") + r, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + + proof, proofErr := r.GetProof("evm", []byte("k")) + require.Error(t, proofErr) + require.ErrorIs(t, proofErr, sentinel) + require.Contains(t, proofErr.Error(), "primary proof builder") + require.Nil(t, proof) +} + +// --- BuildRoute tests --- +// +// BuildRoute returns a *Route whose function fields are method values +// bound to the dual-write router. These tests exercise the route in the +// same way ModuleRouter would: by invoking the route's function fields +// directly. Ordering and short-circuit behavior on writer errors are +// intentionally NOT pinned, matching the convention established above +// for ApplyChangeSets. + +func TestDualWriteBuildRoute_ReturnsValidRoute(t *testing.T) { + primary := newRouteWithBuilders(t, newMockDB(), noopIter, noopProof, "evm") + dwr, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + + route, err := dwr.BuildRoute("evm", "bank") + require.NoError(t, err) + require.NotNil(t, route) + require.Equal(t, []string{"evm", "bank"}, route.modules) + require.NotNil(t, route.reader) + require.NotNil(t, route.writer) + require.NotNil(t, route.iteratorBuilder) + require.NotNil(t, route.proofBuilder) +} + +func TestDualWriteBuildRoute_DuplicateModuleNamesRejected(t *testing.T) { + primary := newRouteWithBuilders(t, newMockDB(), noopIter, noopProof, "evm") + dwr, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + + route, err := dwr.BuildRoute("evm", "bank", "evm") + require.Error(t, err) + require.Nil(t, route) + require.Contains(t, err.Error(), "evm") + require.Contains(t, err.Error(), "more than once") +} + +func TestDualWriteBuildRoute_EmptyModulesAllowed(t *testing.T) { + primary := newRouteWithBuilders(t, newMockDB(), noopIter, noopProof, "evm") + dwr, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + + route, err := dwr.BuildRoute() + require.NoError(t, err) + require.NotNil(t, route) +} + +func TestDualWriteBuildRoute_ReaderDispatchesToPrimary(t *testing.T) { + primaryDB := newMockDB() + primaryDB.seed(map[string]map[string][]byte{"evm": {"k": []byte("v")}}) + primary := newRouteWithBuilders(t, primaryDB, noopIter, noopProof, "evm") + + secondaryCalls := 0 + secondary := func(_ []*proto.NamedChangeSet) error { + secondaryCalls++ + return nil + } + dwr, err := NewTestOnlyDualWriteRouter(primary, secondary) + require.NoError(t, err) + route, err := dwr.BuildRoute("evm") + require.NoError(t, err) + + val, found, err := route.reader("evm", []byte("k")) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, []byte("v"), val) + require.Equal(t, 0, secondaryCalls, + "secondary writer must not be consulted on reads through the route") +} + +func TestDualWriteBuildRoute_ReaderErrorsWrapped(t *testing.T) { + sentinel := errors.New("disk on fire") + primary, err := NewRoute(failReader(sentinel), + newMockDB().writer(), + noopIter, noopProof, + "evm") + require.NoError(t, err) + dwr, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + route, err := dwr.BuildRoute("evm") + require.NoError(t, err) + + val, found, readErr := route.reader("evm", []byte("k")) + require.Error(t, readErr) + require.ErrorIs(t, readErr, sentinel) + require.Contains(t, readErr.Error(), "primary reader") + require.False(t, found) + require.Nil(t, val) +} + +func TestDualWriteBuildRoute_WriterFansOutToBothBackends(t *testing.T) { + primaryDB := newMockDB() + secondaryDB := newMockDB() + primary := newRouteWithBuilders(t, primaryDB, noopIter, noopProof, "evm") + dwr, err := NewTestOnlyDualWriteRouter(primary, secondaryDB.writer()) + require.NoError(t, err) + route, err := dwr.BuildRoute("evm") + require.NoError(t, err) + + require.NoError(t, route.writer([]*proto.NamedChangeSet{ + namedCS("evm", kv("k", "v")), + })) + + pv, ok := primaryDB.get("evm", "k") + require.True(t, ok, "primary must receive the write through the route") + require.Equal(t, []byte("v"), pv) + sv, ok := secondaryDB.get("evm", "k") + require.True(t, ok, "secondary must receive the write through the route") + require.Equal(t, []byte("v"), sv) +} + +func TestDualWriteBuildRoute_WriterPrimaryErrorWrapped(t *testing.T) { + sentinel := errors.New("primary boom") + primary, err := NewRoute(newMockDB().reader(), + failWriter(sentinel), + noopIter, noopProof, + "evm") + require.NoError(t, err) + dwr, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + route, err := dwr.BuildRoute("evm") + require.NoError(t, err) + + writeErr := route.writer([]*proto.NamedChangeSet{namedCS("evm", kv("k", "v"))}) + require.Error(t, writeErr) + require.ErrorIs(t, writeErr, sentinel) + require.Contains(t, writeErr.Error(), "primary writer") +} + +func TestDualWriteBuildRoute_WriterSecondaryErrorWrapped(t *testing.T) { + sentinel := errors.New("secondary boom") + primary := newRouteWithBuilders(t, newMockDB(), noopIter, noopProof, "evm") + dwr, err := NewTestOnlyDualWriteRouter(primary, failWriter(sentinel)) + require.NoError(t, err) + route, err := dwr.BuildRoute("evm") + require.NoError(t, err) + + writeErr := route.writer([]*proto.NamedChangeSet{namedCS("evm", kv("k", "v"))}) + require.Error(t, writeErr) + require.ErrorIs(t, writeErr, sentinel) + require.Contains(t, writeErr.Error(), "secondary writer") +} + +func TestDualWriteBuildRoute_IteratorDelegatesToPrimary(t *testing.T) { + wantIter := &stubIterator{id: "primary"} + recordingIter, calls := recordingIteratorBuilder(wantIter, nil) + primary := newRouteWithBuilders(t, newMockDB(), recordingIter, noopProof, "evm") + dwr, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + route, err := dwr.BuildRoute("evm") + require.NoError(t, err) + + gotIter, err := route.iteratorBuilder("evm", []byte("a"), []byte("z"), true) + require.NoError(t, err) + require.Same(t, wantIter, gotIter, + "route iterator must return the exact iterator from the primary's builder") + require.Len(t, *calls, 1) + require.Equal(t, iteratorCall{ + store: "evm", start: []byte("a"), end: []byte("z"), ascending: true, + }, (*calls)[0]) +} + +func TestDualWriteBuildRoute_IteratorErrorsWrapped(t *testing.T) { + sentinel := errors.New("iterator boom") + failingIter, _ := recordingIteratorBuilder(nil, sentinel) + primary := newRouteWithBuilders(t, newMockDB(), failingIter, noopProof, "evm") + dwr, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + route, err := dwr.BuildRoute("evm") + require.NoError(t, err) + + iter, iterErr := route.iteratorBuilder("evm", nil, nil, true) + require.Error(t, iterErr) + require.ErrorIs(t, iterErr, sentinel) + require.Contains(t, iterErr.Error(), "primary iterator builder") + require.Nil(t, iter) +} + +func TestDualWriteBuildRoute_ProofDelegatesToPrimary(t *testing.T) { + wantProof := &ics23.CommitmentProof{} + recordingProof, calls := recordingProofBuilder(wantProof, nil) + primary := newRouteWithBuilders(t, newMockDB(), noopIter, recordingProof, "evm") + dwr, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + route, err := dwr.BuildRoute("evm") + require.NoError(t, err) + + gotProof, err := route.proofBuilder("evm", []byte("k")) + require.NoError(t, err) + require.Same(t, wantProof, gotProof, + "route proof builder must return the exact proof from the primary's builder") + require.Len(t, *calls, 1) + require.Equal(t, "evm", (*calls)[0].store) + require.Equal(t, []byte("k"), (*calls)[0].key) +} + +func TestDualWriteBuildRoute_ProofErrorsWrapped(t *testing.T) { + sentinel := errors.New("proof boom") + failingProof, _ := recordingProofBuilder(nil, sentinel) + primary := newRouteWithBuilders(t, newMockDB(), noopIter, failingProof, "evm") + dwr, err := NewTestOnlyDualWriteRouter(primary, newMockDB().writer()) + require.NoError(t, err) + route, err := dwr.BuildRoute("evm") + require.NoError(t, err) + + proof, proofErr := route.proofBuilder("evm", []byte("k")) + require.Error(t, proofErr) + require.ErrorIs(t, proofErr, sentinel) + require.Contains(t, proofErr.Error(), "primary proof builder") + require.Nil(t, proof) +} + +// TestDualWriteBuildRoute_IntegrationWithModuleRouter exercises the +// composition story BuildRoute is designed for: the dual-write router +// contributes a Route for one module ("evm"), and an unrelated route +// owns another ("bank"). Through the outer ModuleRouter, evm writes +// must fan out to both primary and secondary, while non-evm writes +// must never reach the secondary. +func TestDualWriteBuildRoute_IntegrationWithModuleRouter(t *testing.T) { + primaryDB := newMockDB() + secondaryDB := newMockDB() + primary := newRouteWithBuilders(t, primaryDB, noopIter, noopProof, "evm") + dwr, err := NewTestOnlyDualWriteRouter(primary, secondaryDB.writer()) + require.NoError(t, err) + + dualWriteRoute, err := dwr.BuildRoute("evm") + require.NoError(t, err) + + otherDB := newMockDB() + otherRoute := newRouteWithBuilders(t, otherDB, noopIter, noopProof, "bank") + + router, err := NewModuleRouter(dualWriteRoute, otherRoute) + require.NoError(t, err) + + require.NoError(t, router.ApplyChangeSets([]*proto.NamedChangeSet{ + namedCS("evm", kv("k1", "v1")), + namedCS("bank", kv("k2", "v2")), + })) + + v, ok := primaryDB.get("evm", "k1") + require.True(t, ok, "evm primary must receive evm writes") + require.Equal(t, []byte("v1"), v) + v, ok = secondaryDB.get("evm", "k1") + require.True(t, ok, "evm secondary must receive evm writes") + require.Equal(t, []byte("v1"), v) + + _, ok = primaryDB.get("bank", "k2") + require.False(t, ok, "bank writes must not leak into evm primary") + _, ok = secondaryDB.get("bank", "k2") + require.False(t, ok, "bank writes must not leak into evm secondary") + + v, ok = otherDB.get("bank", "k2") + require.True(t, ok, "bank backing must receive bank writes") + require.Equal(t, []byte("v2"), v) + _, ok = otherDB.get("evm", "k1") + require.False(t, ok, "evm writes must not leak into the bank backing") + + val, found, err := router.Read("evm", []byte("k1")) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, []byte("v1"), val, + "reads for evm through the outer router come from the primary only") + + val, found, err = router.Read("bank", []byte("k2")) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, []byte("v2"), val) +} diff --git a/sei-db/state_db/sc/migration/migration_steady_state_test.go b/sei-db/state_db/sc/migration/migration_steady_state_test.go new file mode 100644 index 0000000000..f0b7bdd554 --- /dev/null +++ b/sei-db/state_db/sc/migration/migration_steady_state_test.go @@ -0,0 +1,485 @@ +package migration + +import ( + "testing" + + "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/common/testutil" + "github.com/stretchr/testify/require" +) + +// TestBasisCase exercises the test framework itself end-to-end against raw +// memiavl + flatKV stores. No production router is involved: every +// changeset is fanned out to all three backends in lockstep via the +// multiRouter, then post-run state is verified for oracle equivalence and +// matching key counts. A regression here points at the framework or the +// raw stores, not at any migration logic. +func TestBasisCase(t *testing.T) { + + rng := testutil.NewTestRandom() + + // Real memiavl backend with a passthrough test router that forwards + // reads and writes verbatim to it, performing no routing of its own. + memiavlDB := NewTestMemIAVLCommitStore(t, t.TempDir(), keys.MemIAVLStoreKeys) + memiavlRouter := NewTestMemIAVLRouter(t, memiavlDB) + + // Real flatKV backend with a similarly passthrough test router. + flatKVDB := NewTestFlatKVCommitStore(t, t.TempDir()) + flatKVRouter := NewTestFlatKVRouter(t, flatKVDB) + + // Oracle: an in-memory map keyed by (store, key). It is the source of + // truth that the verification phase compares the real backends against. + inMemoryRouter := NewTestInMemoryRouter() + + keysInUse := newLiveKeySet() + + // Tees every ApplyChangeSets call to all three backends so they + // accumulate identical state. Reads go through every backend and the + // multiRouter errors if any disagree, providing a per-read consistency + // check on top of the post-run verification. + multiRouter := NewTestMultiRouter(t, inMemoryRouter, memiavlRouter, flatKVRouter) + + commitBoth := func() { + _, err := memiavlDB.Commit() + require.NoError(t, err, "memiavl commit") + _, err = flatKVDB.Commit() + require.NoError(t, err, "flatKV commit") + } + + // Drive a mixed insert / update / delete / read workload across every + // production module store, fanning every write to memiavl, flatKV, and + // the oracle simultaneously. + SimulateBlocks(t, + multiRouter, + commitBoth, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 20, // deletes per block + 200, // new keys per block + 100, // blocks to simulate + ) + + // Verify that both backends contain all the data the oracle knows about. + inMemoryRouter.VerifyContainsSameData(t, memiavlRouter) + inMemoryRouter.VerifyContainsSameData(t, flatKVRouter) + + // Key count check: the oracle knows the exact number of live logical keys. + // Both backends must contain exactly that many keys. This rules out any + // phantom keys (extra rows) that VerifyContainsSameData cannot detect. + expectedKeyCount := int64(keysInUse.Len()) + require.Equal(t, expectedKeyCount, GetMemIAVLKeyCount(t, memiavlDB), "memiavl key count") + require.Equal(t, expectedKeyCount, GetFlatKVKeyCount(t, flatKVDB), "flatkv key count") + + require.NoError(t, memiavlDB.Close(), "close memiavl") + require.NoError(t, flatKVDB.Close(), "close flatKV") +} + +// Test the MemiavlOnly steady-state router. This is the pre-migration version 0 +// schema: every module routes to memiavl and there is no migration manager (or +// flatKV) in the data path. Bootstrap will pass nil for flatKV in this mode in +// production, and this test does the same. MigrationStore is intentionally not +// mounted on memiavl because V0 nodes will be deployed before the migration +// store is introduced; a realistic V0 layout has no MigrationStore tree. +func TestMemiavlOnly(t *testing.T) { + + rng := testutil.NewTestRandom() + + memiavlDB := NewTestMemIAVLCommitStore(t, t.TempDir(), keys.MemIAVLStoreKeys) + + inMemoryRouter := NewTestInMemoryRouter() + keysInUse := newLiveKeySet() + + memiavlOnlyRouter, err := BuildRouter(t.Context(), MemiavlOnly, memiavlDB, nil, 0) + require.NoError(t, err) + + commit := func() { + _, err := memiavlDB.Commit() + require.NoError(t, err, "memiavl commit") + } + + SimulateBlocks(t, + NewTestMultiRouter(t, memiavlOnlyRouter, inMemoryRouter), + commit, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 20, // deletes per block + 200, // new keys per block + 100, // blocks to simulate + ) + + // Read path correctness: every oracle key is reachable through the router. + inMemoryRouter.VerifyContainsSameData(t, memiavlOnlyRouter) + + // Exact key count check. The oracle's logical key count must equal the + // total physical key count across every memiavl tree. This doubles as the + // "no migration bookkeeping written" check: any spurious write to an + // internal store would inflate the count and fail this assertion. + require.Equal(t, int64(keysInUse.Len()), GetMemIAVLKeyCount(t, memiavlDB), + "memiavl should contain exactly the simulated keys with no phantom rows") + + require.NoError(t, memiavlDB.Close(), "close memiavl") +} + +// Test the EVMMigrated steady-state router. This is the post-MigrateEVM +// migration version 1 schema: EVM data lives entirely in flatKV, every +// other module lives entirely in memIAVL, and there is no migration +// manager in the data path. Because the schema is stable, a single +// long simulation is sufficient — there is no in-flight migration to +// resume across a restart. +func TestEVMMigrated(t *testing.T) { + + rng := testutil.NewTestRandom() + + // Include MigrationStore in memiavl so ReadMigrationVersion/Boundary + // can probe it without hitting "store not found"; the EVMMigrated + // router itself never touches MigrationStore, so the tree stays empty. + memiavlStores := append(keys.MemIAVLStoreKeys, MigrationStore) //nolint:gocritic + memiavlDB := NewTestMemIAVLCommitStore(t, t.TempDir(), memiavlStores) + flatKVDB := NewTestFlatKVCommitStore(t, t.TempDir()) + + inMemoryRouter := NewTestInMemoryRouter() + keysInUse := newLiveKeySet() + + evmMigratedRouter, err := BuildRouter(t.Context(), EVMMigrated, memiavlDB, flatKVDB, 0) + require.NoError(t, err) + + commitBoth := func() { + _, err := memiavlDB.Commit() + require.NoError(t, err, "memiavl commit") + _, err = flatKVDB.Commit() + require.NoError(t, err, "flatKV commit") + } + + SimulateBlocks(t, + NewTestMultiRouter(t, evmMigratedRouter, inMemoryRouter), + commitBoth, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 20, // deletes per block + 200, // new keys per block + 100, // blocks to simulate + ) + + // Read path correctness: every oracle key is reachable through the router. + inMemoryRouter.VerifyContainsSameData(t, evmMigratedRouter) + + // Count EVM vs non-EVM keys in the oracle. + var evmKeyCount, nonEVMKeyCount int64 + for _, kp := range keysInUse.keys { + if kp.store == keys.EVMStoreKey { + evmKeyCount++ + } else { + nonEVMKeyCount++ + } + } + + // Key count check. Unlike TestMigrateEVM, there is no migration manager + // in this router, so flatKV holds exactly the EVM keys (no version / + // boundary metadata) and memiavl holds exactly the non-EVM keys. + require.Equal(t, evmKeyCount, GetFlatKVKeyCount(t, flatKVDB), + "flatKV should contain only EVM keys in steady state") + require.Equal(t, nonEVMKeyCount, GetMemIAVLKeyCount(t, memiavlDB), + "memiavl should contain only non-EVM keys in steady state") + + // Placement check: each oracle key must be in the correct backend and absent from the other. + inMemoryRouter.VerifyKeyPlacement(t, memiavlDB, flatKVDB, + map[string]bool{keys.EVMStoreKey: true}, + ) + + // The steady-state router must not write any migration metadata to + // either backend — that is the responsibility of the migration manager, + // which is not present in this data path. + _, found := ReadMigrationVersionFromFlatKV(t, flatKVDB) + require.False(t, found, "EVMMigrated router must not write a migration version to flatKV") + _, found = ReadMigrationVersionFromMemIAVL(t, memiavlDB) + require.False(t, found, "EVMMigrated router must not write a migration version to memiavl") + _, found = ReadMigrationBoundaryFromFlatKV(t, flatKVDB) + require.False(t, found, "EVMMigrated router must not write a migration boundary to flatKV") + _, found = ReadMigrationBoundaryFromMemIAVL(t, memiavlDB) + require.False(t, found, "EVMMigrated router must not write a migration boundary to memiavl") + + require.NoError(t, memiavlDB.Close(), "close memiavl") + require.NoError(t, flatKVDB.Close(), "close flatKV") +} + +// Test the AllMigratedButBank steady-state router. This is the +// post-MigrateAllButBank migration version 2 schema: every module except +// bank/ lives in flatKV, bank/ lives in memiavl, and there is no migration +// manager in the data path. Because the schema is stable, a single long +// simulation is sufficient — there is no in-flight migration to resume +// across a restart. +func TestAllMigratedButBank(t *testing.T) { + + rng := testutil.NewTestRandom() + + // Include MigrationStore in memiavl so ReadMigrationVersion/Boundary + // can probe it without hitting "store not found"; the AllMigratedButBank + // router itself never touches MigrationStore, so the tree stays empty. + memiavlStores := append(keys.MemIAVLStoreKeys, MigrationStore) //nolint:gocritic + memiavlDB := NewTestMemIAVLCommitStore(t, t.TempDir(), memiavlStores) + flatKVDB := NewTestFlatKVCommitStore(t, t.TempDir()) + + inMemoryRouter := NewTestInMemoryRouter() + keysInUse := newLiveKeySet() + + allMigratedButBankRouter, err := BuildRouter(t.Context(), AllMigratedButBank, memiavlDB, flatKVDB, 0) + require.NoError(t, err) + + commitBoth := func() { + _, err := memiavlDB.Commit() + require.NoError(t, err, "memiavl commit") + _, err = flatKVDB.Commit() + require.NoError(t, err, "flatKV commit") + } + + SimulateBlocks(t, + NewTestMultiRouter(t, allMigratedButBankRouter, inMemoryRouter), + commitBoth, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 20, // deletes per block + 200, // new keys per block + 100, // blocks to simulate + ) + + // Read path correctness: every oracle key is reachable through the router. + inMemoryRouter.VerifyContainsSameData(t, allMigratedButBankRouter) + + // Count bank vs non-bank keys in the oracle. + var bankKeyCount, nonBankKeyCount int64 + for _, kp := range keysInUse.keys { + if kp.store == keys.BankStoreKey { + bankKeyCount++ + } else { + nonBankKeyCount++ + } + } + + // Key count check. Unlike TestMigrateAllButBank, there is no migration + // manager in this router, so flatKV holds exactly the non-bank keys + // (no version / boundary metadata) and memiavl holds exactly the bank + // keys. + require.Equal(t, nonBankKeyCount, GetFlatKVKeyCount(t, flatKVDB), + "flatKV should contain only non-bank keys in steady state") + require.Equal(t, bankKeyCount, GetMemIAVLKeyCount(t, memiavlDB), + "memiavl should contain only bank keys in steady state") + + // Placement check. Build a flatKV-store map containing every module + // except bank — i.e. every store whose keys must end up in flatKV. + flatKVStores := make(map[string]bool, len(keys.MemIAVLStoreKeys)) + for _, s := range keys.MemIAVLStoreKeys { + if s != keys.BankStoreKey { + flatKVStores[s] = true + } + } + inMemoryRouter.VerifyKeyPlacement(t, memiavlDB, flatKVDB, flatKVStores) + + // The steady-state router must not write any migration metadata to + // either backend — that is the responsibility of the migration manager, + // which is not present in this data path. + _, found := ReadMigrationVersionFromFlatKV(t, flatKVDB) + require.False(t, found, "AllMigratedButBank router must not write a migration version to flatKV") + _, found = ReadMigrationVersionFromMemIAVL(t, memiavlDB) + require.False(t, found, "AllMigratedButBank router must not write a migration version to memiavl") + _, found = ReadMigrationBoundaryFromFlatKV(t, flatKVDB) + require.False(t, found, "AllMigratedButBank router must not write a migration boundary to flatKV") + _, found = ReadMigrationBoundaryFromMemIAVL(t, memiavlDB) + require.False(t, found, "AllMigratedButBank router must not write a migration boundary to memiavl") + + require.NoError(t, memiavlDB.Close(), "close memiavl") + require.NoError(t, flatKVDB.Close(), "close flatKV") +} + +// Test the FlatKVOnly steady-state router. This is the post-MigrateBank +// terminal version 3 schema: every module routes to flatKV and there is no +// migration manager (or memiavl) in the data path. Bootstrap will pass nil +// for memiavl in this mode in production, and this test does the same. +func TestFlatKVOnly(t *testing.T) { + + rng := testutil.NewTestRandom() + + flatKVDB := NewTestFlatKVCommitStore(t, t.TempDir()) + + inMemoryRouter := NewTestInMemoryRouter() + keysInUse := newLiveKeySet() + + flatKVOnlyRouter, err := BuildRouter(t.Context(), FlatKVOnly, nil, flatKVDB, 0) + require.NoError(t, err) + + commit := func() { + _, err := flatKVDB.Commit() + require.NoError(t, err, "flatKV commit") + } + + SimulateBlocks(t, + NewTestMultiRouter(t, flatKVOnlyRouter, inMemoryRouter), + commit, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 20, // deletes per block + 200, // new keys per block + 100, // blocks to simulate + ) + + // Read path correctness: every oracle key is reachable through the router. + inMemoryRouter.VerifyContainsSameData(t, flatKVOnlyRouter) + + // Exact key count check. The oracle's logical key count must equal the + // physical row count in flatKV. With random 20-byte EVM addresses, the + // nonce/codehash account-row merging in flatKV does not collapse rows + // (collisions are astronomically unlikely), so logical and physical + // counts agree — same assumption TestBasisCase relies on. + require.Equal(t, int64(keysInUse.Len()), GetFlatKVKeyCount(t, flatKVDB), + "flatKV should contain exactly the simulated keys with no phantom rows") + + // The terminal-state router must not write any migration metadata to + // flatKV — that is the responsibility of the migration manager, which + // is not present in this data path. + _, found := ReadMigrationVersionFromFlatKV(t, flatKVDB) + require.False(t, found, "FlatKVOnly router must not write a migration version to flatKV") + _, found = ReadMigrationBoundaryFromFlatKV(t, flatKVDB) + require.False(t, found, "FlatKVOnly router must not write a migration boundary to flatKV") + + require.NoError(t, flatKVDB.Close(), "close flatKV") +} + +// Test the test-only DualWrite router. Every module routes to memiavl; +// evm/ traffic is additionally fanned out to flatKV. There is no migration +// manager in the data path. This mode emulates the legacy +// CompositeCommitStore "dual write" mode that a teammate uses for +// testing — it must not be deployed to production but must remain +// supported for parity with the existing composite-store tests. +// +// Invariant pinned by this test: +// - memiavl holds every key (evm + non-evm) +// - flatKV holds exactly the evm keys +// - reads through the router come from memiavl (the primary) +// - no migration metadata is written to either backend +func TestDualWrite(t *testing.T) { + + rng := testutil.NewTestRandom() + + // Include MigrationStore in memiavl so ReadMigrationVersion/Boundary + // can probe it without hitting "store not found"; the dual-write + // router itself never touches MigrationStore, so the tree stays empty. + memiavlStores := append(keys.MemIAVLStoreKeys, MigrationStore) //nolint:gocritic + memiavlDB := NewTestMemIAVLCommitStore(t, t.TempDir(), memiavlStores) + flatKVDB := NewTestFlatKVCommitStore(t, t.TempDir()) + + inMemoryRouter := NewTestInMemoryRouter() + keysInUse := newLiveKeySet() + + dualWriteRouter, err := BuildRouter(t.Context(), TestOnlyDualWrite, memiavlDB, flatKVDB, 0) + require.NoError(t, err) + + commitBoth := func() { + _, err := memiavlDB.Commit() + require.NoError(t, err, "memiavl commit") + _, err = flatKVDB.Commit() + require.NoError(t, err, "flatKV commit") + } + + SimulateBlocks(t, + NewTestMultiRouter(t, dualWriteRouter, inMemoryRouter), + commitBoth, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 20, // deletes per block + 200, // new keys per block + 100, // blocks to simulate + ) + + // Read path correctness: every oracle key is reachable through the + // router. Reads come from memiavl (the primary), which under the + // dual-write invariant holds every key, so this passes for both + // evm and non-evm modules. + inMemoryRouter.VerifyContainsSameData(t, dualWriteRouter) + + // Count EVM vs non-EVM keys in the oracle. + var evmKeyCount, nonEVMKeyCount int64 + for _, kp := range keysInUse.keys { + if kp.store == keys.EVMStoreKey { + evmKeyCount++ + } else { + nonEVMKeyCount++ + } + } + + // Key count check. Unlike the steady-state routers, dual-write + // keeps every key in memiavl and additionally mirrors evm keys + // into flatKV. No migration metadata is written, so flatKV's + // physical key count equals exactly the evm logical key count + // (same physical-vs-logical caveat as TestBasisCase / TestFlatKVOnly: + // random 20-byte EVM addresses make collapsing-row collisions + // astronomically unlikely). + require.Equal(t, evmKeyCount+nonEVMKeyCount, GetMemIAVLKeyCount(t, memiavlDB), + "memiavl must hold every key (evm + non-evm) under dual-write") + require.Equal(t, evmKeyCount, GetFlatKVKeyCount(t, flatKVDB), + "flatKV must hold exactly the dual-written evm keys") + + // Per-key dual-write invariant: every oracle key is in memiavl, + // and present in flatKV iff its store is keys.EVMStoreKey. + // VerifyKeyPlacement assumes mutually-exclusive placement and so + // can't be used here; assert directly. + memIAVLGet := func(store string, key []byte) ([]byte, bool) { + childStore := memiavlDB.GetChildStoreByName(store) + if childStore == nil { + return nil, false + } + v := childStore.Get(key) + return v, v != nil + } + for _, kp := range keysInUse.keys { + key := []byte(kp.key) + expected, _, err := inMemoryRouter.Read(kp.store, key) + require.NoError(t, err) + + got, ok := memIAVLGet(kp.store, key) + require.True(t, ok, "store %q key %x must be in memiavl under dual-write", kp.store, key) + require.Equal(t, expected, got, "store %q key %x value mismatch in memiavl", kp.store, key) + + flatVal, flatFound := flatKVDB.Get(kp.store, key) + if kp.store == keys.EVMStoreKey { + require.True(t, flatFound, + "evm store key %x must be mirrored to flatKV under dual-write", key) + require.Equal(t, expected, flatVal, + "evm store key %x value mismatch in flatKV", key) + } else { + require.False(t, flatFound, + "non-evm store %q key %x must not appear in flatKV under dual-write", kp.store, key) + } + } + + // The dual-write router must not write any migration metadata to + // either backend — that is the responsibility of the migration + // manager, which is not present in this data path. + _, found := ReadMigrationVersionFromFlatKV(t, flatKVDB) + require.False(t, found, "TestOnlyDualWrite router must not write a migration version to flatKV") + _, found = ReadMigrationVersionFromMemIAVL(t, memiavlDB) + require.False(t, found, "TestOnlyDualWrite router must not write a migration version to memiavl") + _, found = ReadMigrationBoundaryFromFlatKV(t, flatKVDB) + require.False(t, found, "TestOnlyDualWrite router must not write a migration boundary to flatKV") + _, found = ReadMigrationBoundaryFromMemIAVL(t, memiavlDB) + require.False(t, found, "TestOnlyDualWrite router must not write a migration boundary to memiavl") + + require.NoError(t, memiavlDB.Close(), "close memiavl") + require.NoError(t, flatKVDB.Close(), "close flatKV") +} diff --git a/sei-db/state_db/sc/migration/migration_transitions_test.go b/sei-db/state_db/sc/migration/migration_transitions_test.go new file mode 100644 index 0000000000..16d46dc428 --- /dev/null +++ b/sei-db/state_db/sc/migration/migration_transitions_test.go @@ -0,0 +1,583 @@ +package migration + +import ( + "testing" + + "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/common/testutil" + "github.com/stretchr/testify/require" +) + +// Test the MigrateEVM data migration. At the start of this migration, all data lives in memIAVL. +// At the end of this migration, all evm/ data lives in flatkv, and all other data remains in memIAVL. +// +// This test evaluates the 0->1 migration path. +func TestMigrateEVM(t *testing.T) { + + rng := testutil.NewTestRandom() + + // Reserve stable directories so we can close and reopen the stores + // mid-migration to simulate a process restart. + memiavlDir := t.TempDir() + flatKVDir := t.TempDir() + + memiavlStores := append(keys.MemIAVLStoreKeys, MigrationStore) //nolint:gocritic + + // All data is initially in memiavl. MigrationStore is included so the + // migration manager can read/write its version and boundary metadata there, + // but it is intentionally excluded from SimulateBlocks so no user data + // lands in it. + memiavlDB := NewTestMemIAVLCommitStore(t, memiavlDir, memiavlStores) + memiavlRouter := NewTestMemIAVLRouter(t, memiavlDB) + + // Empty flatKV store; the migration will populate it with EVM keys + // and a single MigrationStore version-key entry. + flatKVDB := NewTestFlatKVCommitStore(t, flatKVDir) + + // Oracle: in-memory map of (store, key) -> value. Drives the post-run + // equivalence check. + inMemoryRouter := NewTestInMemoryRouter() + + keysInUse := newLiveKeySet() + + commitBoth := func() { + _, err := memiavlDB.Commit() + require.NoError(t, err, "memiavl commit") + _, err = flatKVDB.Commit() + require.NoError(t, err, "flatKV commit") + } + + // Phase 1 (v0 baseline): populate memiavl with data across all modules. + // The multiRouter only contains memiavl + oracle, so no changesets reach + // flatKV; commitBoth still calls flatKVDB.Commit() each block, advancing + // its version in lockstep against an empty changeset. + SimulateBlocks(t, + NewTestMultiRouter(t, memiavlRouter, inMemoryRouter), + commitBoth, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 20, // deletes per block + 100, // new keys per block + 100, // blocks to simulate + ) + + // Build a migration router that will migrate the evm/ data to flatkv. + migrationRouter, err := BuildRouter(t.Context(), MigrateEVM, memiavlDB, flatKVDB, 100) + require.NoError(t, err) + + // Phase 2: drive 2 blocks through the migration router. Phase 1 produced + // ~500 EVM keys (1 of 20 modules at 100 new keys/block * 100 blocks); + // with a batch size of 100 the migration drains those source keys in 5 + // blocks, so 2 blocks is deliberately short to leave the migration in + // flight at the restart point: ~200 of the ~500 source keys migrated to + // flatKV, ~300 still un-migrated in memiavl. AssertMigrationInFlight + // below verifies this split before we close the DBs. + SimulateBlocks(t, + NewTestMultiRouter(t, migrationRouter, inMemoryRouter), + commitBoth, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 10, // deletes per block + 10, // new keys per block + 2, // blocks to simulate + ) + + // Sanity check: the test must actually catch the migration in flight, + // otherwise the restart below is degenerate (no boundary to resume from). + inMemoryRouter.AssertMigrationInFlight(t, memiavlDB, flatKVDB, keys.EVMStoreKey) + + // Close and reopen both backends. SimulateBlocks committed after every + // block, so the on-disk state is consistent. The reopened router must + // recover the migration manager's state from disk metadata - the + // boundary key (migration cursor) and the source version stored in + // flatKV. + require.NoError(t, memiavlDB.Close(), "close memiavl before restart") + require.NoError(t, flatKVDB.Close(), "close flatKV before restart") + + memiavlDB = NewTestMemIAVLCommitStore(t, memiavlDir, memiavlStores) + memiavlRouter = NewTestMemIAVLRouter(t, memiavlDB) + flatKVDB = NewTestFlatKVCommitStore(t, flatKVDir) + + // Re-declare commitBoth for visual continuity. Strictly speaking the + // original closure already observes the rebound memiavlDB / flatKVDB + // (Go closures capture local variables by reference), but redeclaring + // keeps the post-restart setup symmetric with phase 1. + commitBoth = func() { + _, err := memiavlDB.Commit() + require.NoError(t, err, "memiavl commit") + _, err = flatKVDB.Commit() + require.NoError(t, err, "flatKV commit") + } + + // Rebuild the migration router on top of the freshly-reopened DBs. The + // manager recovers its state from disk - either resuming from the + // boundary, or coming up in passthrough if the version key already + // records the target version. + migrationRouter, err = BuildRouter(t.Context(), MigrateEVM, memiavlDB, flatKVDB, 100) + require.NoError(t, err, "rebuild migration router after restart") + + // Sanity check: all oracle data is still reachable through the rebuilt + // router. Exercises the post-restart hybrid read path: ~200 EVM keys + // already in flatKV, ~300 still in memiavl awaiting migration. + inMemoryRouter.VerifyContainsSameData(t, migrationRouter) + + // Phase 3: 100 more blocks after the restart. The first ~3 blocks finish + // draining the ~300 un-migrated source EVM keys (batch 100); the + // remaining ~97 blocks run in passthrough mode and exercise normal + // user-key churn against the post-completion write path. + SimulateBlocks(t, + NewTestMultiRouter(t, migrationRouter, inMemoryRouter), + commitBoth, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 10, // deletes per block + 10, // new keys per block + 100, // blocks to simulate + ) + + // All oracle data must be reachable through the migration router. + inMemoryRouter.VerifyContainsSameData(t, migrationRouter) + + // Count EVM vs non-EVM keys in the oracle. + var evmKeyCount, nonEVMKeyCount int64 + for _, kp := range keysInUse.keys { + if kp.store == keys.EVMStoreKey { + evmKeyCount++ + } else { + nonEVMKeyCount++ + } + } + + // Key count check. + // flatKV holds EVM data + exactly 1 migration metadata key (MigrationVersionKey). + // MigrationBoundaryKey is deleted on the final migration block, leaving only the version key. + // memiavl holds only non-EVM keys; its MigrationStore tree is empty (version written to flatKV). + require.Equal(t, evmKeyCount+1, GetFlatKVKeyCount(t, flatKVDB), + "flatKV should contain EVM keys plus one migration version metadata key") + require.Equal(t, nonEVMKeyCount, GetMemIAVLKeyCount(t, memiavlDB), + "memiavl should contain only non-EVM keys") + + // Migration version check. + flatKVVersion, found := ReadMigrationVersionFromFlatKV(t, flatKVDB) + require.True(t, found, "migration version key must be present in flatKV after migration") + require.Equal(t, uint64(Version1_MigrateEVM), flatKVVersion, + "flatKV migration version should be Version1_MigrateEVM") + _, found = ReadMigrationVersionFromMemIAVL(t, memiavlDB) + require.False(t, found, + "migration version key must not be present in memiavl (it is written exclusively to flatKV)") + + // Migration boundary check. The boundary key tracks the in-progress + // migration cursor. On the final migration block it is deleted in the same + // atomic write that records the new version, so post-completion it must be + // absent from both backends. + _, found = ReadMigrationBoundaryFromFlatKV(t, flatKVDB) + require.False(t, found, + "migration boundary key must be cleared from flatKV after migration completes") + _, found = ReadMigrationBoundaryFromMemIAVL(t, memiavlDB) + require.False(t, found, + "migration boundary key must not be present in memiavl") + + // Placement check: each oracle key must be in the correct backend and absent from the other. + inMemoryRouter.VerifyKeyPlacement(t, memiavlDB, flatKVDB, + map[string]bool{keys.EVMStoreKey: true}, + ) + + require.NoError(t, memiavlDB.Close(), "close memiavl") + require.NoError(t, flatKVDB.Close(), "close flatKV") +} + +// Test the MigrateAllButBank data migration. At the start of this migration, +// evm/ data lives in flatkv and everything else lives in memiavl (i.e. the +// EVMMigrated steady state). At the end, every module except bank/ lives in +// flatkv; bank/ remains in memiavl. +// +// This test evaluates the 1->2 migration path. The setup phase relies on the +// EVMMigrated router (verified by TestEVMMigrated) to lay down a realistic +// v1 schema, then explicitly seeds flatKV's MigrationVersionKey since the +// EVMMigrated router does not itself write that bookkeeping. +func TestMigrateAllButBank(t *testing.T) { + + rng := testutil.NewTestRandom() + + // Reserve stable directories so we can close and reopen the stores + // mid-migration to simulate a process restart. + memiavlDir := t.TempDir() + flatKVDir := t.TempDir() + + // MigrationStore is included so the migration manager can read/write its + // version and boundary metadata in memiavl during phase 2; SimulateBlocks + // is restricted to MemIAVLStoreKeys so no user data lands there. + memiavlStores := append(keys.MemIAVLStoreKeys, MigrationStore) //nolint:gocritic + + memiavlDB := NewTestMemIAVLCommitStore(t, memiavlDir, memiavlStores) + flatKVDB := NewTestFlatKVCommitStore(t, flatKVDir) + + inMemoryRouter := NewTestInMemoryRouter() + keysInUse := newLiveKeySet() + + commitBoth := func() { + _, err := memiavlDB.Commit() + require.NoError(t, err, "memiavl commit") + _, err = flatKVDB.Commit() + require.NoError(t, err, "flatKV commit") + } + + // --- Phase 1: EVMMigrated setup --- + // Lay down v1 state: evm/ in flatKV, everything else in memiavl. Drives + // roughly equal load across all real modules so the non-evm-non-bank + // stores accumulate enough keys to make the v1->v2 migration meaningful. + evmMigratedRouter, err := BuildRouter(t.Context(), EVMMigrated, memiavlDB, flatKVDB, 0) + require.NoError(t, err, "build EVMMigrated router") + SimulateBlocks(t, + NewTestMultiRouter(t, evmMigratedRouter, inMemoryRouter), + commitBoth, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 20, // deletes per block + 100, // new keys per block + 100, // blocks to simulate + ) + + // The EVMMigrated router has no route for MigrationStore, so it never + // writes the migration version key. A real chain at v1 would have that + // key already (left behind by the prior MigrateEVM run); seed it + // directly so the upcoming MigrateAllButBank constructor can read v1 + // from the new DB instead of erroring out. + SeedMigrationVersionInFlatKV(t, flatKVDB, Version1_MigrateEVM) + + // --- Phase 2: partial MigrateAllButBank --- + migrationRouter, err := BuildRouter(t.Context(), MigrateAllButBank, memiavlDB, flatKVDB, 100) + require.NoError(t, err, "build MigrateAllButBank router") + + // 50 blocks * 100 batch ≈ 5,000 keys migrated, well short of the ~9,000 + // non-evm-non-bank keys produced in setup; this guarantees we end this + // phase with a partially-migrated state and a persisted boundary. + SimulateBlocks(t, + NewTestMultiRouter(t, migrationRouter, inMemoryRouter), + commitBoth, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 10, // deletes per block + 10, // new keys per block + 50, // blocks to simulate + ) + + // Sanity check: the test must actually catch the migration in flight, + // otherwise the restart below is degenerate (no boundary to resume + // from). Spans every store currently being migrated - i.e. every + // production module except bank. + migratingStores := make([]string, 0, len(keys.MemIAVLStoreKeys)-1) + for _, s := range keys.MemIAVLStoreKeys { + if s != keys.BankStoreKey { + migratingStores = append(migratingStores, s) + } + } + inMemoryRouter.AssertMigrationInFlight(t, memiavlDB, flatKVDB, migratingStores...) + + // --- Restart --- + // Close and reopen both backends to verify the in-progress migration is + // not corrupted by a restart and resumes from the persisted boundary. + // SimulateBlocks already committed after each block, so closing here is + // safe. + require.NoError(t, memiavlDB.Close(), "close memiavl before restart") + require.NoError(t, flatKVDB.Close(), "close flatKV before restart") + + memiavlDB = NewTestMemIAVLCommitStore(t, memiavlDir, memiavlStores) + flatKVDB = NewTestFlatKVCommitStore(t, flatKVDir) + + commitBoth = func() { + _, err := memiavlDB.Commit() + require.NoError(t, err, "memiavl commit") + _, err = flatKVDB.Commit() + require.NoError(t, err, "flatKV commit") + } + + migrationRouter, err = BuildRouter(t.Context(), MigrateAllButBank, memiavlDB, flatKVDB, 100) + require.NoError(t, err, "rebuild MigrateAllButBank router after restart") + + // Sanity check: all oracle data is still reachable through the rebuilt + // router. This exercises the post-restart hybrid read path (some keys + // in memiavl, some already migrated to flatKV). + inMemoryRouter.VerifyContainsSameData(t, migrationRouter) + + // --- Phase 3: finish migration --- + // 100 more blocks * 100 batch = 10,000 capacity vs. ~5,400 keys still to + // drain (4,000 left over from phase 2 + ~1,400 new keys added during + // phases 2+3). Comfortable margin to converge. + SimulateBlocks(t, + NewTestMultiRouter(t, migrationRouter, inMemoryRouter), + commitBoth, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 10, // deletes per block + 10, // new keys per block + 100, // blocks to simulate + ) + + // --- Verification --- + + // All oracle data must be reachable through the migration router. + inMemoryRouter.VerifyContainsSameData(t, migrationRouter) + + // Count bank vs non-bank keys in the oracle. + var bankKeyCount, nonBankKeyCount int64 + for _, kp := range keysInUse.keys { + if kp.store == keys.BankStoreKey { + bankKeyCount++ + } else { + nonBankKeyCount++ + } + } + + // Key count check. + // flatKV holds every non-bank key + exactly 1 migration metadata key + // (MigrationVersionKey). MigrationBoundaryKey is deleted on the final + // migration block, leaving only the version key. + // memiavl holds only bank keys; its MigrationStore tree is empty + // (version written to flatKV, boundary deleted). + require.Equal(t, nonBankKeyCount+1, GetFlatKVKeyCount(t, flatKVDB), + "flatKV should hold every non-bank key plus the migration version key") + require.Equal(t, bankKeyCount, GetMemIAVLKeyCount(t, memiavlDB), + "memiavl should hold only bank keys") + + // Migration version check. + flatKVVersion, found := ReadMigrationVersionFromFlatKV(t, flatKVDB) + require.True(t, found, "migration version key must be present in flatKV after migration") + require.Equal(t, uint64(Version2_MigrateAllButBank), flatKVVersion, + "flatKV migration version should be Version2_MigrateAllButBank") + _, found = ReadMigrationVersionFromMemIAVL(t, memiavlDB) + require.False(t, found, + "migration version key must not be present in memiavl (it is written exclusively to flatKV)") + + // Migration boundary check. The boundary key tracks the in-progress + // migration cursor. On the final migration block it is deleted in the + // same atomic write that records the new version, so post-completion it + // must be absent from both backends. + _, found = ReadMigrationBoundaryFromFlatKV(t, flatKVDB) + require.False(t, found, + "migration boundary key must be cleared from flatKV after migration completes") + _, found = ReadMigrationBoundaryFromMemIAVL(t, memiavlDB) + require.False(t, found, + "migration boundary key must not be present in memiavl") + + // Placement check. Build a flatKV-store map containing every module + // except bank — i.e. every store whose keys must end up in flatKV. + flatKVStores := make(map[string]bool, len(keys.MemIAVLStoreKeys)) + for _, s := range keys.MemIAVLStoreKeys { + if s != keys.BankStoreKey { + flatKVStores[s] = true + } + } + inMemoryRouter.VerifyKeyPlacement(t, memiavlDB, flatKVDB, flatKVStores) + + require.NoError(t, memiavlDB.Close(), "close memiavl") + require.NoError(t, flatKVDB.Close(), "close flatKV") +} + +// Test the MigrateBank data migration. At the start of this migration, every +// module except bank/ already lives in flatKV, and bank/ lives in memiavl +// (i.e. the AllMigratedButBank steady state). At the end, every module +// lives in flatKV; memiavl is empty. +// +// This test evaluates the 2->3 migration path. The setup phase relies on the +// AllMigratedButBank router (verified by TestAllMigratedButBank) to lay down +// a realistic v2 schema, then explicitly seeds flatKV's MigrationVersionKey +// since the AllMigratedButBank router does not itself write that bookkeeping. +func TestMigrateBank(t *testing.T) { + + rng := testutil.NewTestRandom() + + // Reserve stable directories so we can close and reopen the stores + // mid-migration to simulate a process restart. + memiavlDir := t.TempDir() + flatKVDir := t.TempDir() + + // MigrationStore is included so the migration manager can read/write its + // version and boundary metadata in memiavl during phase 2; SimulateBlocks + // is restricted to MemIAVLStoreKeys so no user data lands there. + memiavlStores := append(keys.MemIAVLStoreKeys, MigrationStore) //nolint:gocritic + + memiavlDB := NewTestMemIAVLCommitStore(t, memiavlDir, memiavlStores) + flatKVDB := NewTestFlatKVCommitStore(t, flatKVDir) + + inMemoryRouter := NewTestInMemoryRouter() + keysInUse := newLiveKeySet() + + commitBoth := func() { + _, err := memiavlDB.Commit() + require.NoError(t, err, "memiavl commit") + _, err = flatKVDB.Commit() + require.NoError(t, err, "flatKV commit") + } + + // --- Phase 1: AllMigratedButBank setup --- + // Lay down v2 state: bank/ in memiavl, everything else in flatKV. Drives + // roughly equal load across all real modules so bank/ accumulates enough + // keys to make the v2->v3 migration meaningful. + allMigratedButBankRouter, err := BuildRouter(t.Context(), AllMigratedButBank, memiavlDB, flatKVDB, 0) + require.NoError(t, err, "build AllMigratedButBank router") + SimulateBlocks(t, + NewTestMultiRouter(t, allMigratedButBankRouter, inMemoryRouter), + commitBoth, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 20, // deletes per block + 100, // new keys per block + 100, // blocks to simulate + ) + + // The AllMigratedButBank router has no route for MigrationStore, so it + // never writes the migration version key. A real chain at v2 would have + // that key already (left behind by the prior MigrateAllButBank run); + // seed it directly so the upcoming MigrateBank constructor can read v2 + // from the new DB instead of erroring out. + SeedMigrationVersionInFlatKV(t, flatKVDB, Version2_MigrateAllButBank) + + // --- Phase 2: MigrateBank --- + migrationRouter, err := BuildRouter(t.Context(), MigrateBank, memiavlDB, flatKVDB, 100) + require.NoError(t, err, "build MigrateBank router") + + // Drive 2 blocks through the migration router. Phase 1 produced ~500 + // bank keys (1 of 20 modules at 100 new keys/block * 100 blocks); with + // a batch size of 100 the migration drains those source keys in 5 + // blocks, so 2 blocks is deliberately short to leave the migration in + // flight at the restart point: ~200 of the ~500 source bank keys + // migrated to flatKV, ~300 still un-migrated in memiavl. + // AssertMigrationInFlight below verifies this split before we close + // the DBs. + SimulateBlocks(t, + NewTestMultiRouter(t, migrationRouter, inMemoryRouter), + commitBoth, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 10, // deletes per block + 10, // new keys per block + 2, // blocks to simulate + ) + + // Sanity check: the test must actually catch the migration in flight, + // otherwise the restart below is degenerate (no boundary to resume from). + inMemoryRouter.AssertMigrationInFlight(t, memiavlDB, flatKVDB, keys.BankStoreKey) + + // --- Restart --- + // Close and reopen both backends. SimulateBlocks committed after every + // block, so the on-disk state is consistent. The reopened router must + // recover the migration manager's state from disk metadata - the + // boundary key (migration cursor) and the source version stored in + // flatKV. + require.NoError(t, memiavlDB.Close(), "close memiavl before restart") + require.NoError(t, flatKVDB.Close(), "close flatKV before restart") + + memiavlDB = NewTestMemIAVLCommitStore(t, memiavlDir, memiavlStores) + flatKVDB = NewTestFlatKVCommitStore(t, flatKVDir) + + commitBoth = func() { + _, err := memiavlDB.Commit() + require.NoError(t, err, "memiavl commit") + _, err = flatKVDB.Commit() + require.NoError(t, err, "flatKV commit") + } + + migrationRouter, err = BuildRouter(t.Context(), MigrateBank, memiavlDB, flatKVDB, 100) + require.NoError(t, err, "rebuild MigrateBank router after restart") + + // Sanity check: all oracle data is still reachable through the rebuilt + // router. Exercises the post-restart hybrid read path: ~200 bank keys + // already in flatKV, ~300 still in memiavl awaiting migration, and + // every other module (already in flatKV from the v2 setup) routed + // directly to flatKV. + inMemoryRouter.VerifyContainsSameData(t, migrationRouter) + + // --- Phase 3: finish migration --- + // 100 more blocks after the restart. The first ~3 blocks finish draining + // the ~300 un-migrated source bank keys (batch 100); the remaining ~97 + // blocks run in passthrough mode and exercise normal user-key churn + // against the post-completion write path. + SimulateBlocks(t, + NewTestMultiRouter(t, migrationRouter, inMemoryRouter), + commitBoth, + rng, + keysInUse, + keys.MemIAVLStoreKeys, + 100, // reads per block + 100, // updates per block + 10, // deletes per block + 10, // new keys per block + 100, // blocks to simulate + ) + + // --- Verification --- + + // All oracle data must be reachable through the migration router. + inMemoryRouter.VerifyContainsSameData(t, migrationRouter) + + // After v3 every key in the oracle lives in flatKV. + totalKeyCount := int64(keysInUse.Len()) + + // Key count check. + // flatKV holds every key + exactly 1 migration metadata key + // (MigrationVersionKey). MigrationBoundaryKey is deleted on the final + // migration block, leaving only the version key. + // memiavl is empty: the bank tree was drained by the migration, no + // other tree ever held user data, and the MigrationStore tree never + // received the version key (it is written exclusively to flatKV). + require.Equal(t, totalKeyCount+1, GetFlatKVKeyCount(t, flatKVDB), + "flatKV should hold every key plus the migration version key") + require.Equal(t, int64(0), GetMemIAVLKeyCount(t, memiavlDB), + "memiavl should be empty after migration") + + // Migration version check. + flatKVVersion, found := ReadMigrationVersionFromFlatKV(t, flatKVDB) + require.True(t, found, "migration version key must be present in flatKV after migration") + require.Equal(t, uint64(Version3_FlatKVOnly), flatKVVersion, + "flatKV migration version should be Version3_FlatKVOnly") + _, found = ReadMigrationVersionFromMemIAVL(t, memiavlDB) + require.False(t, found, + "migration version key must not be present in memiavl (it is written exclusively to flatKV)") + + // Migration boundary check. The boundary key tracks the in-progress + // migration cursor. On the final migration block it is deleted in the + // same atomic write that records the new version, so post-completion it + // must be absent from both backends. + _, found = ReadMigrationBoundaryFromFlatKV(t, flatKVDB) + require.False(t, found, + "migration boundary key must be cleared from flatKV after migration completes") + _, found = ReadMigrationBoundaryFromMemIAVL(t, memiavlDB) + require.False(t, found, + "migration boundary key must not be present in memiavl") + + // Placement check. After v3, every module's keys must be in flatKV and + // absent from memiavl, including bank/. + flatKVStores := make(map[string]bool, len(keys.MemIAVLStoreKeys)) + for _, s := range keys.MemIAVLStoreKeys { + flatKVStores[s] = true + } + inMemoryRouter.VerifyKeyPlacement(t, memiavlDB, flatKVDB, flatKVStores) + + require.NoError(t, memiavlDB.Close(), "close memiavl") + require.NoError(t, flatKVDB.Close(), "close flatKV") +} diff --git a/sei-db/state_db/sc/migration/router_builder.go b/sei-db/state_db/sc/migration/router_builder.go index f31a104e04..7c06c30bfd 100644 --- a/sei-db/state_db/sc/migration/router_builder.go +++ b/sei-db/state_db/sc/migration/router_builder.go @@ -1,15 +1,556 @@ package migration import ( + "context" "fmt" + "time" ics23 "github.com/confio/ics23/go" + "github.com/sei-protocol/sei-chain/sei-db/common/keys" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" dbm "github.com/tendermint/tm-db" ) +// Builds a router for the given migration write mode. A router is responsible for splitting +// reads/writes between the memiavl and flatkv backends. +func BuildRouter( + ctx context.Context, + writeMode WriteMode, + memIAVL *memiavl.CommitStore, + flatKV *flatkv.CommitStore, + // If this router will be doing data migration, this is the number of keys to migrate in each batch. + migrationBatchSize int, +) (Router, error) { + + switch writeMode { + case MemiavlOnly: + router, err := buildMemiavlOnlyRouter(memIAVL) + if err != nil { + return nil, fmt.Errorf("buildMemiavlOnlyRouter: %w", err) + } + return router, nil + case MigrateEVM: + router, err := buildMigrateEVMRouter(ctx, memIAVL, flatKV, migrationBatchSize) + if err != nil { + return nil, fmt.Errorf("buildMigrateEVMRouter: %w", err) + } + threadSafe, err := NewThreadSafeRouter(router) + if err != nil { + return nil, fmt.Errorf("NewThreadSafeRouter: %w", err) + } + return threadSafe, nil + case EVMMigrated: + router, err := buildEVMMigratedRouter(memIAVL, flatKV) + if err != nil { + return nil, fmt.Errorf("buildEVMMigratedRouter: %w", err) + } + threadSafe, err := NewThreadSafeRouter(router) + if err != nil { + return nil, fmt.Errorf("NewThreadSafeRouter: %w", err) + } + return threadSafe, nil + case MigrateAllButBank: + router, err := buildMigrateAllButBankRouter(ctx, memIAVL, flatKV, migrationBatchSize) + if err != nil { + return nil, fmt.Errorf("buildMigrateAllButBankRouter: %w", err) + } + threadSafe, err := NewThreadSafeRouter(router) + if err != nil { + return nil, fmt.Errorf("NewThreadSafeRouter: %w", err) + } + return threadSafe, nil + case AllMigratedButBank: + router, err := buildAllMigratedButBankRouter(memIAVL, flatKV) + if err != nil { + return nil, fmt.Errorf("buildAllMigratedButBankRouter: %w", err) + } + threadSafe, err := NewThreadSafeRouter(router) + if err != nil { + return nil, fmt.Errorf("NewThreadSafeRouter: %w", err) + } + return threadSafe, nil + case MigrateBank: + router, err := buildMigrateBankRouter(ctx, memIAVL, flatKV, migrationBatchSize) + if err != nil { + return nil, fmt.Errorf("buildMigrateBankRouter: %w", err) + } + threadSafe, err := NewThreadSafeRouter(router) + if err != nil { + return nil, fmt.Errorf("NewThreadSafeRouter: %w", err) + } + return threadSafe, nil + case FlatKVOnly: + router, err := buildFlatKVOnlyRouter(flatKV) + if err != nil { + return nil, fmt.Errorf("buildFlatKVOnlyRouter: %w", err) + } + return router, nil + case TestOnlyDualWrite: + router, err := buildTestOnlyDualWriteRouter(memIAVL, flatKV) + if err != nil { + return nil, fmt.Errorf("buildTestOnlyDualWriteRouter: %w", err) + } + threadSafe, err := NewThreadSafeRouter(router) + if err != nil { + return nil, fmt.Errorf("NewThreadSafeRouter: %w", err) + } + return threadSafe, nil + default: + return nil, fmt.Errorf("unsupported write mode: %s", writeMode) + } +} + +/* Data flow: MemiavlOnly (0) + + ┌──────────────┐ ┌─────────┐ +──all-modules────────▶ │ moduleRouter │ ──────────all-modules──────────▶ │ memIAVL │ + └──────────────┘ └─────────┘ +*/ + +// Build a router for handling write mode MemiavlOnly. Operates on a schema at migration version 0. +func buildMemiavlOnlyRouter( + memIAVL *memiavl.CommitStore, +) (Router, error) { + if memIAVL == nil { + return nil, fmt.Errorf("memIAVL is nil") + } + + route, err := routeToMemIAVL(memIAVL, keys.MemIAVLStoreKeys...) + if err != nil { + return nil, fmt.Errorf("routeToMemIAVL: %w", err) + } + + router, err := NewModuleRouter(route) + if err != nil { + return nil, fmt.Errorf("NewModuleRouter: %w", err) + } + + return router, nil +} + +/* Data flow: MigrateEVM (0 -> 1) + + ┌──────────────┐ ┌─────────┐ +──all-modules────────▶ │ moduleRouter │ ──everything-except-evm/───────▶ │ memIAVL │ + └──────────────┘ └─────────┘ + │ ▲ + evm/ │ + │ ┌──────un-migrated-keys─────────────────┘ + │ │ + ▼ │ + ┌──────────────────┐ ┌────────┐ + │ migrationManager │ ────────migrated-keys──────▶ │ flatKV │ + └──────────────────┘ └────────┘ +*/ + +// Build a router for handling write mode MigrateEVM. Migrates from version 0 to version 1. +func buildMigrateEVMRouter( + ctx context.Context, + memIAVL *memiavl.CommitStore, + flatKV *flatkv.CommitStore, + migrationBatchSize int, +) (Router, error) { + + if memIAVL == nil { + return nil, fmt.Errorf("memIAVL is nil") + } + if flatKV == nil { + return nil, fmt.Errorf("flatKV is nil") + } + if migrationBatchSize <= 0 { + return nil, fmt.Errorf("migrationBatchSize must be greater than 0") + } + + // Manages migration and routing for keys in the evm/ module. + migrationManager, err := NewMigrationManager( + migrationBatchSize, + Version0_MemiavlOnly, + Version1_MigrateEVM, + buildMemIAVLReader(memIAVL), + buildMemIAVLWriter(memIAVL), + buildFlatKVReader(flatKV), + buildFlatKVWriter(flatKV), + NewMemiavlMigrationIterator(memIAVL.GetDB(), []string{keys.EVMStoreKey}), + NewMigrationMetrics(ctx, Version1_MigrateEVM, 10*time.Second), + ) + if err != nil { + return nil, fmt.Errorf("NewMigrationManager: %w", err) + } + + nonEVMModules, err := keys.AllModulesExcept(keys.EVMStoreKey) + if err != nil { + return nil, fmt.Errorf("AllModulesExcept: %w", err) + } + nonEVMRoute, err := routeToMemIAVL(memIAVL, nonEVMModules...) + if err != nil { + return nil, fmt.Errorf("routeToMemIAVL: %w", err) + } + + evmRoute, err := migrationManager.BuildRoute(keys.EVMStoreKey) + if err != nil { + return nil, fmt.Errorf("BuildRoute: %w", err) + } + + moduleRouter, err := NewModuleRouter(nonEVMRoute, evmRoute) + if err != nil { + return nil, fmt.Errorf("NewModuleRouter: %w", err) + } + + return moduleRouter, nil +} + +/* Data flow: EVMMigrated (1) + + ┌──────────────┐ ┌─────────┐ +──all-modules────────▶ │ moduleRouter │ ──everything-except-evm/───────▶ │ memIAVL │ + └──────────────┘ └─────────┘ + │ + │ + │ + │ + │ + │ ┌────────┐ + └────────────evm/────────────────────────▶ │ flatKV │ + └────────┘ +*/ + +// Build a router for handling write mode EVMMigrated. Operates on a schema at migration version 1. +func buildEVMMigratedRouter( + memIAVL *memiavl.CommitStore, + flatKV *flatkv.CommitStore, +) (Router, error) { + + if memIAVL == nil { + return nil, fmt.Errorf("memIAVL is nil") + } + if flatKV == nil { + return nil, fmt.Errorf("flatKV is nil") + } + + nonEVMModules, err := keys.AllModulesExcept(keys.EVMStoreKey) + if err != nil { + return nil, fmt.Errorf("AllModulesExcept: %w", err) + } + nonEVMRoute, err := routeToMemIAVL(memIAVL, nonEVMModules...) + if err != nil { + return nil, fmt.Errorf("routeToMemIAVL: %w", err) + } + + evmRoute, err := routeToFlatKV(flatKV, keys.EVMStoreKey) + if err != nil { + return nil, fmt.Errorf("routeToFlatKV: %w", err) + } + + moduleRouter, err := NewModuleRouter(nonEVMRoute, evmRoute) + if err != nil { + return nil, fmt.Errorf("NewModuleRouter: %w", err) + } + + return moduleRouter, nil +} + +/* Data flow: MigrateAllButBank (1 -> 2) + + ┌──────────────┐ ┌─────────┐ +──all-modules────────▶ │ moduleRouter │ ──────────────bank/────────────▶ │ memIAVL │ + └──────────────┘ └─────────┘ + │ │ ▲ + │ all but │ + │ bank/ and evm/ ┌──────un-migrated-keys─────────┘ + │ │ │ + │ ▼ │ + │ ┌──────────────────┐ ┌────────┐ + │ │ migrationManager │ ───migrated-keys──────▶ │ flatKV │ + │ └──────────────────┘ └────────┘ + │ ▲ + │ │ + └────────────────────────────evm/────────────────────┘ +*/ + +// Build a router for handling write mode MigrateAllButBank. Migrates from version 1 to version 2. +func buildMigrateAllButBankRouter( + ctx context.Context, + memIAVL *memiavl.CommitStore, + flatKV *flatkv.CommitStore, + migrationBatchSize int, +) (Router, error) { + + if memIAVL == nil { + return nil, fmt.Errorf("memIAVL is nil") + } + if flatKV == nil { + return nil, fmt.Errorf("flatKV is nil") + } + if migrationBatchSize <= 0 { + return nil, fmt.Errorf("migrationBatchSize must be greater than 0") + } + + allModulesButEvmAndBank, err := keys.AllModulesExcept(keys.EVMStoreKey, keys.BankStoreKey) + if err != nil { + return nil, fmt.Errorf("AllModulesExcept: %w", err) + } + + // Manages migration and routing for all keys except evm/ (already migrated) and bank/ (not migrating yet) + migrationManager, err := NewMigrationManager( + migrationBatchSize, + Version1_MigrateEVM, + Version2_MigrateAllButBank, + buildMemIAVLReader(memIAVL), + buildMemIAVLWriter(memIAVL), + buildFlatKVReader(flatKV), + buildFlatKVWriter(flatKV), + NewMemiavlMigrationIterator(memIAVL.GetDB(), allModulesButEvmAndBank), + NewMigrationMetrics(ctx, Version2_MigrateAllButBank, 10*time.Second), + ) + if err != nil { + return nil, fmt.Errorf("NewMigrationManager: %w", err) + } + + bankRoute, err := routeToMemIAVL(memIAVL, keys.BankStoreKey) + if err != nil { + return nil, fmt.Errorf("routeToMemIAVL: %w", err) + } + + evmRoute, err := routeToFlatKV(flatKV, keys.EVMStoreKey) + if err != nil { + return nil, fmt.Errorf("routeToFlatKV: %w", err) + } + + allOtherModulesRoute, err := migrationManager.BuildRoute(allModulesButEvmAndBank...) + if err != nil { + return nil, fmt.Errorf("BuildRoute: %w", err) + } + + moduleRouter, err := NewModuleRouter(bankRoute, evmRoute, allOtherModulesRoute) + if err != nil { + return nil, fmt.Errorf("NewModuleRouter: %w", err) + } + + return moduleRouter, nil +} + +/* Data flow: AllMigratedButBank (2) + + ┌──────────────┐ ┌─────────┐ +──all-modules────────▶ │ moduleRouter │ ───bank/───────────────────────▶ │ memIAVL │ + └──────────────┘ └─────────┘ + │ + │ + │ + │ + │ + │ ┌────────┐ + └────────────all─but─bank/───────────────▶ │ flatKV │ + └────────┘ +*/ + +// Build a router for handling write mode AllMigratedButBank. Operates on a schema at migration version 2. +func buildAllMigratedButBankRouter( + memIAVL *memiavl.CommitStore, + flatKV *flatkv.CommitStore, +) (Router, error) { + + if memIAVL == nil { + return nil, fmt.Errorf("memIAVL is nil") + } + if flatKV == nil { + return nil, fmt.Errorf("flatKV is nil") + } + + allButBankModules, err := keys.AllModulesExcept(keys.BankStoreKey) + if err != nil { + return nil, fmt.Errorf("AllModulesExcept: %w", err) + } + nonBankRoute, err := routeToFlatKV(flatKV, allButBankModules...) + if err != nil { + return nil, fmt.Errorf("routeToFlatKV: %w", err) + } + + bankRoute, err := routeToMemIAVL(memIAVL, keys.BankStoreKey) + if err != nil { + return nil, fmt.Errorf("routeToMemIAVL: %w", err) + } + + moduleRouter, err := NewModuleRouter(nonBankRoute, bankRoute) + if err != nil { + return nil, fmt.Errorf("NewModuleRouter: %w", err) + } + + return moduleRouter, nil +} + +/* Data flow: MigrateBank (2 -> 3) + + ┌──────────────┐ ┌─────────┐ +──all-modules────────▶ │ moduleRouter │ │ memIAVL │ + └──────────────┘ └─────────┘ + │ │ ▲ + │ bank/ ┌──────un-migrated-keys─────────┘ + │ │ │ + │ ▼ │ + │ ┌──────────────────┐ ┌────────┐ + │ │ migrationManager │ ───migrated-keys──────▶ │ flatKV │ + │ └──────────────────┘ └────────┘ + │ ▲ + │ │ + └───────────────────all─but─bank/────────────────────┘ +*/ + +// Build a router for handling write mode MigrateBank. Migrates from version 2 to version 3. +func buildMigrateBankRouter( + ctx context.Context, + memIAVL *memiavl.CommitStore, + flatKV *flatkv.CommitStore, + migrationBatchSize int, +) (Router, error) { + + if memIAVL == nil { + return nil, fmt.Errorf("memIAVL is nil") + } + if flatKV == nil { + return nil, fmt.Errorf("flatKV is nil") + } + if migrationBatchSize <= 0 { + return nil, fmt.Errorf("migrationBatchSize must be greater than 0") + } + + allButBankModules, err := keys.AllModulesExcept(keys.BankStoreKey) + if err != nil { + return nil, fmt.Errorf("AllModulesExcept: %w", err) + } + + // Manages migration and routing for keys in the bank/ module (the + // final module remaining in memiavl; every other module already + // lives in flatkv from prior migrations). + migrationManager, err := NewMigrationManager( + migrationBatchSize, + Version2_MigrateAllButBank, + Version3_FlatKVOnly, + buildMemIAVLReader(memIAVL), + buildMemIAVLWriter(memIAVL), + buildFlatKVReader(flatKV), + buildFlatKVWriter(flatKV), + NewMemiavlMigrationIterator(memIAVL.GetDB(), []string{keys.BankStoreKey}), + NewMigrationMetrics(ctx, Version3_FlatKVOnly, 10*time.Second), + ) + if err != nil { + return nil, fmt.Errorf("NewMigrationManager: %w", err) + } + + bankRoute, err := migrationManager.BuildRoute(keys.BankStoreKey) + if err != nil { + return nil, fmt.Errorf("BuildRoute: %w", err) + } + + allOtherModulesRoute, err := routeToFlatKV(flatKV, allButBankModules...) + if err != nil { + return nil, fmt.Errorf("routeToFlatKV: %w", err) + } + + moduleRouter, err := NewModuleRouter(bankRoute, allOtherModulesRoute) + if err != nil { + return nil, fmt.Errorf("NewModuleRouter: %w", err) + } + + return moduleRouter, nil +} + +/* Data flow: FlatKVOnly (3) + + ┌──────────────┐ ┌────────┐ +──all-modules────────▶ │ moduleRouter │ ──────────all-modules──────────▶ │ flatKV │ + └──────────────┘ └────────┘ +*/ + +// Build a router for handling write mode FlatKVOnly. Operates on a schema at migration version 3. +func buildFlatKVOnlyRouter( + flatKV *flatkv.CommitStore, +) (Router, error) { + if flatKV == nil { + return nil, fmt.Errorf("flatKV is nil") + } + + route, err := routeToFlatKV(flatKV, keys.MemIAVLStoreKeys...) + if err != nil { + return nil, fmt.Errorf("routeToFlatKV: %w", err) + } + + router, err := NewModuleRouter(route) + if err != nil { + return nil, fmt.Errorf("NewModuleRouter: %w", err) + } + + return router, nil +} + +/* Data flow: dual write (test only) + + ┌──────────────┐ ┌─────────┐ +──all-modules────────▶ │ moduleRouter │ ──everything-except-evm/───────▶ │ memIAVL │ + └──────────────┘ └─────────┘ + │ ▲ + evm/ │ + │ ┌──────evm/─reads-and-writes────────────┘ + │ │ + ▼ │ + ┌───────────────────┐ ┌────────┐ + │ dual write router │ ───────evm/-writes────────▶ │ flatKV │ + └───────────────────┘ └────────┘ +*/ + +// Build a test-only dual-write router. +// +// CRITICAL: this is a test-only router and should never be deployed to production machines. +func buildTestOnlyDualWriteRouter( + memIAVL *memiavl.CommitStore, + flatKV *flatkv.CommitStore, +) (Router, error) { + if memIAVL == nil { + return nil, fmt.Errorf("memIAVL is nil") + } + if flatKV == nil { + return nil, fmt.Errorf("flatKV is nil") + } + + // Sends evm/ traffic to both memIAVL and flatKV. + // Note that a TestOnlyDualWriteRouter ignores module names; it's only job is to duplicate traffic. + // The routes given to the dual write router do not specify modules for this reason. + memiavlEvmRoute, err := routeToMemIAVL(memIAVL) + if err != nil { + return nil, fmt.Errorf("routeToMemIAVL: %w", err) + } + dualWriteRouter, err := NewTestOnlyDualWriteRouter( + memiavlEvmRoute, + buildFlatKVWriter(flatKV), + ) + if err != nil { + return nil, fmt.Errorf("NewTestOnlyDualWriteRouter: %w", err) + } + + nonEVMModules, err := keys.AllModulesExcept(keys.EVMStoreKey) + if err != nil { + return nil, fmt.Errorf("AllModulesExcept: %w", err) + } + nonEVMRoute, err := routeToMemIAVL(memIAVL, nonEVMModules...) + if err != nil { + return nil, fmt.Errorf("routeToMemIAVL: %w", err) + } + + evmRoute, err := dualWriteRouter.BuildRoute(keys.EVMStoreKey) + if err != nil { + return nil, fmt.Errorf("BuildRoute: %w", err) + } + + moduleRouter, err := NewModuleRouter(nonEVMRoute, evmRoute) + if err != nil { + return nil, fmt.Errorf("NewModuleRouter: %w", err) + } + + return moduleRouter, nil +} + // Build a function capable of reading data from memiavl. func buildMemIAVLReader(memIAVL *memiavl.CommitStore) DBReader { return func(store string, key []byte) ([]byte, bool, error) { diff --git a/sei-db/state_db/sc/migration/write_mode.go b/sei-db/state_db/sc/migration/write_mode.go index 8d5550e27a..d5a9775c9b 100644 --- a/sei-db/state_db/sc/migration/write_mode.go +++ b/sei-db/state_db/sc/migration/write_mode.go @@ -9,6 +9,12 @@ import "fmt" type WriteMode string const ( + + // MemiavlOnly writes all data to memiavl only. + // + // Migration version 0. + MemiavlOnly WriteMode = "memiavl_only" + // MigrateEVM migrates the evm/ module from memiavl to flatkv. // // Handles the transition from migration version 0 to 1, @@ -38,13 +44,24 @@ const ( // Handles the transition from migration version 2 to 3, // and continues to function once we reach migration version 3. MigrateBank WriteMode = "migrate_bank" + + // All data is written to FlatKV. + // + // Migration version 3. + FlatKVOnly WriteMode = "flatkv_only" + + // TestOnlyDualWrite is a test-only dual-write router. EVM traffic is written to both memiavl and flatkv, + // but all other traffic is written to memiavl only. + // + // CRITICAL: this is a test-only router and should never be deployed to production machines. + TestOnlyDualWrite WriteMode = "test_only_dual_write" ) // IsValid returns true if the migration write mode is a recognized value. func (m WriteMode) IsValid() bool { switch m { - case MigrateEVM, EVMMigrated, - MigrateAllButBank, AllMigratedButBank, MigrateBank: + case MemiavlOnly, MigrateEVM, EVMMigrated, + MigrateAllButBank, AllMigratedButBank, MigrateBank, FlatKVOnly, TestOnlyDualWrite: return true default: return false