From b7cea4f9c6759648c03f65c4c1580af97d43013e Mon Sep 17 00:00:00 2001 From: blindchaser Date: Mon, 6 Jul 2026 23:36:28 -0400 Subject: [PATCH 1/5] feat(seidb): composite + replay digest modes and migration-boundary omission for evm-logical-digest Port the evolved evm-logical-digest tooling to main: - `--backend composite` (with `--flatkv-dir` / `--memiavl-dir`): digests the full mid-migration EVM logical view as the union of flatkv rows plus the memiavl rows not yet migrated past the boundary, so an in-progress node can be compared against a memiavl-only truth node. - `--memiavl-open-mode snapshot|replay`: snapshot (default, fast: sequential scan of the completed snapshot kvs file) vs replay (opens a read-only DB, replays the changelog to --height, walks the mmap tree) for heights with no on-disk snapshot. Documented the speed trade-off and when to prefer each. - Omit the FlatKV-only `migration/migration-boundary` cursor from the legacy bucket comparison, mirroring the existing migration-version handling, so an in-progress (composite) node compares apples-to-apples against completed / memiavl-only nodes. Extracted legacyForCompare() and added a regression test. Co-authored-by: Cursor --- .../seidb/operations/evm_logical_digest.go | 513 ++++++++++++++++-- .../operations/evm_logical_digest_test.go | 80 ++- 2 files changed, 546 insertions(+), 47 deletions(-) diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index d700b76c98..1ff2b67358 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -18,6 +18,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/migration" "github.com/spf13/cobra" ) @@ -30,6 +31,15 @@ import ( // compared apples-to-apples against memiavl-only output. var migrationVersionPhysKey = []byte(migration.MigrationStore + "/" + migration.MigrationVersionKey) +// migrationBoundaryPhysKey is the FlatKV physical key of the in-progress +// migration cursor. Like migrationVersionPhysKey it is a FlatKV-only +// MigrationStore row that a memiavl-only node never owns, but it is present only +// while a migration is in flight (the MigrationManager deletes it on completion, +// atomically writing MigrationVersionKey instead). Excluding it lets an +// in-progress node's FlatKV/composite digest compare apples-to-apples against a +// completed or memiavl-only node, which carry no boundary row. +var migrationBoundaryPhysKey = []byte(migration.MigrationStore + "/" + migration.MigrationBoundaryKey) + // EvmLogicalDigestCmd computes a backend-independent digest of the EVM logical // state (account / code / storage canonical buckets) so a memIAVL node and a // FlatKV node can be compared at the same chain height. @@ -55,9 +65,31 @@ var migrationVersionPhysKey = []byte(migration.MigrationStore + "/" + migration. // global order while memIAVL is scanned by leaf index, nor that merged accounts // are flushed out of order at Finalize. // +// Performance / mode selection (memiavl side only; --memiavl-open-mode): +// - snapshot (default, FAST): sequentially scans the completed snapshot kvs +// file at snapshot-/evm. Requires an on-disk memiavl snapshot AT +// that exact height (or --height 0 for the current symlink). This is the +// preferred mode whenever the target height lines up with an existing +// snapshot boundary. +// - replay (SLOW): opens a read-only DB, replays the changelog up to +// --height, then walks the in-memory/mmap tree. Roughly an order of +// magnitude slower than snapshot (changelog replay + per-leaf tree walk +// instead of a sequential file read). Use it only when no snapshot exists +// at the target height — e.g. nodes whose snapshot rewrite lags the tip, so +// an arbitrary comparison height has no snapshot- on disk. +// +// The flatkv side is always a pebble WAL-replay-to-height and is fast +// regardless. So when comparing across nodes, pick a height that is an existing +// memiavl snapshot on every node and use snapshot mode; fall back to replay only +// when no such common height is reachable within each backend's retained window. +// // The primary comparison is account+code+storage. The legacy bucket is printed -// separately, plus a marker-adjusted comparison line, because FlatKV can contain -// a FlatKV-only migration-version row that a memiavl-only truth node never owns. +// separately, plus marker-adjusted comparison lines, because FlatKV can contain +// FlatKV-only MigrationStore rows that a memiavl-only truth node never owns: the +// migration-version marker (present once a migration completes) and the +// migration-boundary cursor (present only while a migration is in flight). Both +// are XORed out of the legacy bucket for the final comparison so that memiavl, +// mid-migration, and completed nodes all agree. // // Usage: // @@ -68,12 +100,19 @@ var migrationVersionPhysKey = []byte(migration.MigrationStore + "/" + migration. // --db-dir /.sei/data/state_commit/flatkv --height 213200000 // // # memIAVL digest at the same height (0 = current symlink), using the -// # default semantic mode. This independently decodes raw EVM keys without -// # flatkv.ImportTranslator. memiavl does not replay WAL in this tool; it -// # opens snapshot-/evm or current/evm. +// # default semantic + snapshot mode: independently decodes raw EVM keys +// # without flatkv.ImportTranslator, reading the completed snapshot kvs file +// # at snapshot-/evm (or current/evm). This is the fast path and +// # requires a snapshot at that exact height. // seidb evm-logical-digest --backend memiavl \ // --db-dir /.sei/data/state_commit/memiavl --height 213200000 // +// # Same, but for a height with no on-disk snapshot (e.g. snapshot rewrite +// # lags the tip): replay the changelog to the height first. Slower — prefer +// # snapshot mode whenever the height matches an existing snapshot boundary. +// seidb evm-logical-digest --backend memiavl --memiavl-open-mode replay \ +// --db-dir /.sei/data/state_commit/memiavl --height 213205000 +// // # Translator-based memIAVL digest. This proves FlatKV state matches the // # current migration mapping and is useful when debugging ImportTranslator. // seidb evm-logical-digest --backend memiavl \ @@ -101,9 +140,12 @@ func EvmLogicalDigestCmd() *cobra.Command { Short: "Backend-independent digest of EVM logical state (account/code/storage) for memiavl vs flatkv comparison", RunE: runEvmLogicalDigest, } - cmd.Flags().String("backend", "", "Backend to read: flatkv | memiavl") + cmd.Flags().String("backend", "", "Backend to read: flatkv | memiavl | composite") cmd.Flags().StringP("db-dir", "d", "", "For flatkv: the flatkv data dir. For memiavl: the memiavl root dir (contains current/ and snapshot-* )") + cmd.Flags().String("flatkv-dir", "", "Composite mode: flatkv data dir") + cmd.Flags().String("memiavl-dir", "", "Composite mode: memiavl root dir (contains current/ and snapshot-* )") cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-/evm (0 = current symlink)") + cmd.Flags().String("memiavl-open-mode", "snapshot", "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: replays changelog to --height then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary") cmd.Flags().String("memiavl-normalization", "semantic", "memiavl digest/inspect normalization: semantic/independent (raw EVM key/value decoder) | translator (current migration mapping)") cmd.Flags().String("inspect-bucket", "", "Inspect one normalized bucket (account|code|storage|legacy) instead of printing the global digest") cmd.Flags().Int("key-offset", 0, "Inspect mode: byte offset into physical key before applying --key-prefix / sharding") @@ -167,6 +209,15 @@ type evmDigest struct { // should match memiavl-only output exactly. migrationVersionFound bool migrationVersionHash [sha256.Size]byte + + // migrationBoundaryFound/Hash capture the FlatKV-only + // "migration/migration-boundary" cursor, present only while a migration is + // in flight. It is folded into the legacy bucket like any other row but + // tracked separately so print can XOR it back out — mirroring the + // migration-version handling — so an in-progress node matches a completed + // or memiavl-only node. + migrationBoundaryFound bool + migrationBoundaryHash [sha256.Size]byte } type digestPrintContext struct { @@ -211,6 +262,10 @@ func (d *evmDigest) addLogical(bucket string, physKey, logical, rawVal []byte) { d.migrationVersionFound = true d.migrationVersionHash = sum } + if bytes.Equal(physKey, migrationBoundaryPhysKey) { + d.migrationBoundaryFound = true + d.migrationBoundaryHash = sum + } } } @@ -251,21 +306,44 @@ func normalizeEVMFlatKVPair(physKey, val []byte) (string, []byte, error) { } } +// legacyForCompare returns the legacy bucket accumulator and count with the +// FlatKV-only MigrationStore marker rows XORed back out: the migration-version +// marker (present on a completed node) and the migration-boundary cursor +// (present on an in-progress node). A memiavl-only node owns neither, so after +// this adjustment memiavl, mid-migration, and completed nodes all produce the +// same legacy digest for identical EVM state. +func (d *evmDigest) legacyForCompare() (acc [sha256.Size]byte, count uint64) { + acc = d.legacy.acc + count = d.legacy.count + if d.migrationVersionFound { + for i := 0; i < sha256.Size; i++ { + acc[i] ^= d.migrationVersionHash[i] + } + count-- + } + if d.migrationBoundaryFound { + for i := 0; i < sha256.Size; i++ { + acc[i] ^= d.migrationBoundaryHash[i] + } + count-- + } + return acc, count +} + func (d *evmDigest) print(ctx digestPrintContext) { fmt.Println("EVM logical digest report") printDigestContext(ctx) fmt.Println() - legacyForCompare := d.legacy.acc - legacyCountForCompare := d.legacy.count + legacyForCompare, legacyCountForCompare := d.legacyForCompare() if d.migrationVersionFound { - for i := 0; i < sha256.Size; i++ { - legacyForCompare[i] ^= d.migrationVersionHash[i] - } - legacyCountForCompare-- fmt.Println("flatkv_marker_adjustment: omitted migration/migration-version from legacy bucket in final result") fmt.Println() } + if d.migrationBoundaryFound { + fmt.Println("flatkv_marker_adjustment: omitted migration/migration-boundary from legacy bucket in final result") + fmt.Println() + } fmt.Println("Bucket digests (final digest inputs)") fmt.Printf("account count=%d bucket_digest=%X\n", d.account.count, d.account.acc) @@ -304,13 +382,19 @@ func printDigestContext(ctx digestPrintContext) { func runEvmLogicalDigest(cmd *cobra.Command, _ []string) error { backend, _ := cmd.Flags().GetString("backend") dbDir, _ := cmd.Flags().GetString("db-dir") + flatKVDir, _ := cmd.Flags().GetString("flatkv-dir") + memIAVLDir, _ := cmd.Flags().GetString("memiavl-dir") height, _ := cmd.Flags().GetInt64("height") - if dbDir == "" { + if dbDir == "" && backend != "composite" { return errors.New("must provide --db-dir") } inspectBucket, _ := cmd.Flags().GetString("inspect-bucket") memiavlNormalization, _ := cmd.Flags().GetString("memiavl-normalization") + memiavlOpenMode, _ := cmd.Flags().GetString("memiavl-open-mode") if inspectBucket != "" { + if memiavlOpenMode != "snapshot" { + return fmt.Errorf("--inspect-bucket does not support --memiavl-open-mode=%q yet", memiavlOpenMode) + } return runEvmLogicalInspect(cmd, backend, dbDir, height, inspectBucket, memiavlNormalization) } @@ -331,12 +415,223 @@ func runEvmLogicalDigest(cmd *cobra.Command, _ []string) error { case "flatkv": return digestFlatKV(dbDir, height, findTarget) case "memiavl": - return digestMemIAVL(dbDir, height, findTarget, memiavlNormalization) + return digestMemIAVL(dbDir, height, findTarget, memiavlNormalization, memiavlOpenMode) + case "composite": + if flatKVDir == "" || memIAVLDir == "" { + return errors.New("--backend composite requires --flatkv-dir and --memiavl-dir") + } + return digestCompositeMigrateEVM(flatKVDir, memIAVLDir, height, findTarget, memiavlOpenMode) default: - return fmt.Errorf("unknown --backend %q (want flatkv|memiavl)", backend) + return fmt.Errorf("unknown --backend %q (want flatkv|memiavl|composite)", backend) } } +func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findTarget []byte, memiavlOpenMode string) error { + opened, err := openFlatKVReadOnly(flatKVDir, height) + if err != nil { + return fmt.Errorf("open flatkv read-only: %w", err) + } + defer func() { _ = opened.Close() }() + + boundary, versionKnown, migrationVersion, err := readFlatKVMigrationState(opened) + if err != nil { + return err + } + + ctx := digestPrintContext{ + backend: "composite", + mode: "migrate_evm", + dbDir: fmt.Sprintf("flatkv=%s memiavl=%s", flatKVDir, memIAVLDir), + requestedHeight: height, + version: opened.Version(), + } + var memReplayDB *memiavl.DB + var memEvmSnapshotDir string + var memVersion int64 + switch memiavlOpenMode { + case "", "snapshot": + memEvmSnapshotDir, err = resolveMemIAVLEvmSnapshotDir(memIAVLDir, height) + if err != nil { + return err + } + memVersion, err = readMemIAVLSnapshotVersion(memEvmSnapshotDir) + if err != nil { + return err + } + ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl snapshot=%s", opened.Version(), memEvmSnapshotDir) + ctx.normalization = fmt.Sprintf("flatkv rows plus memiavl rows not migrated by boundary=%s version_known=%t migration_version=%d memiavl_version=%d", boundary.String(), versionKnown, migrationVersion, memVersion) + case "replay": + memReplayDB, err = openMemiAVLReplayReadOnly(memIAVLDir, height) + if err != nil { + return err + } + defer func() { _ = memReplayDB.Close() }() + memVersion = memReplayDB.Version() + ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl read-only replay dir=%s", opened.Version(), memIAVLDir) + ctx.normalization = fmt.Sprintf("flatkv rows plus replayed memiavl rows not migrated by boundary=%s version_known=%t migration_version=%d memiavl_version=%d", boundary.String(), versionKnown, migrationVersion, memVersion) + default: + return fmt.Errorf("unknown --memiavl-open-mode %q (want snapshot|replay)", memiavlOpenMode) + } + printDigestStart(ctx) + fmt.Println("Scan progress: composite migrate_evm logical view -> flatkv rows plus memiavl rows to the right of boundary") + + d := evmDigest{findTarget: findTarget} + accounts := make(map[string]*semanticAccountDigestState) + + if err := consumeCompositeFlatKV(opened, &d, accounts); err != nil { + return err + } + if boundary.Status() != migration.MigrationComplete { + if memReplayDB != nil { + if err := consumeCompositeMemiAVLReplay(memReplayDB, boundary, &d, accounts); err != nil { + return err + } + } else { + if err := consumeCompositeMemiAVL(memEvmSnapshotDir, boundary, &d, accounts); err != nil { + return err + } + } + } + finalizeSemanticAccounts(accounts, d.addLogical) + d.print(ctx) + return nil +} + +func readFlatKVMigrationState(store *openedFlatKV) (migration.MigrationBoundary, bool, uint64, error) { + if data, ok := store.Get(migration.MigrationStore, []byte(migration.MigrationVersionKey)); ok { + if len(data) != 8 { + return migration.MigrationBoundary{}, false, 0, fmt.Errorf("flatkv migration version length=%d, want 8", len(data)) + } + return migration.MigrationBoundaryComplete, true, binary.BigEndian.Uint64(data), nil + } + if data, ok := store.Get(migration.MigrationStore, []byte(migration.MigrationBoundaryKey)); ok { + b, err := migration.DeserializeMigrationBoundary(data) + return b, false, 0, err + } + return migration.MigrationBoundaryNotStarted, false, 0, nil +} + +func consumeCompositeFlatKV(opened *openedFlatKV, d *evmDigest, accounts map[string]*semanticAccountDigestState) error { + iter, err := opened.RawGlobalIterator() + if err != nil { + return fmt.Errorf("raw global iterator: %w", err) + } + defer func() { _ = iter.Close() }() + var seen uint64 + for ; iter.Valid(); iter.Next() { + seen++ + k := iter.Key() + if classifyFlatKVPhysicalKey(k) == flatkvBucketAccount { + if err := mergeCompositeFlatKVAccount(accounts, k, iter.Value()); err != nil { + return err + } + } else if err := d.consume(k, iter.Value()); err != nil { + return err + } + if seen%20000000 == 0 { + fmt.Printf(" progress backend=composite source=flatkv input_physical_rows=%d account_buffered=%d code=%d storage=%d legacy=%d\n", seen, len(accounts), d.code.count, d.storage.count, d.legacy.count) + } + } + if err := iter.Error(); err != nil { + return fmt.Errorf("iterate flatkv: %w", err) + } + fmt.Printf(" composite flatkv rows=%d\n", seen) + return nil +} + +func mergeCompositeFlatKVAccount(accounts map[string]*semanticAccountDigestState, physKey, val []byte) error { + kind, addr, err := ktype.StripEVMPhysicalKey(physKey) + if err != nil { + return err + } + if kind != ktype.EVMKeyAccount { + return fmt.Errorf("flatkv account key %X kind=%d, want account", physKey, kind) + } + ad, err := vtype.DeserializeAccountData(val) + if err != nil { + return err + } + acct := getSemanticAccount(accounts, addr) + bal := ad.GetBalance() + copy(acct.balance[:], bal[:]) + acct.nonce = ad.GetNonce() + copy(acct.codeHash[:], ad.GetCodeHash()[:]) + return nil +} + +func consumeCompositeMemiAVL(evmSnapshotDir string, boundary migration.MigrationBoundary, d *evmDigest, accounts map[string]*semanticAccountDigestState) error { + kvsPath := filepath.Join(evmSnapshotDir, "kvs") + f, err := os.Open(filepath.Clean(kvsPath)) + if err != nil { + return fmt.Errorf("open kvs %s: %w", kvsPath, err) + } + defer func() { _ = f.Close() }() + r := bufio.NewReaderSize(f, 16*1024*1024) + var lenbuf [4]byte + var leaves, consumed uint64 + for { + if _, err := io.ReadFull(r, lenbuf[:]); err != nil { + if errors.Is(err, io.EOF) { + break + } + return fmt.Errorf("read key len: %w", err) + } + keyLen := binary.LittleEndian.Uint32(lenbuf[:]) + k := make([]byte, keyLen) + if _, err := io.ReadFull(r, k); err != nil { + return fmt.Errorf("read key: %w", err) + } + if _, err := io.ReadFull(r, lenbuf[:]); err != nil { + return fmt.Errorf("read val len: %w", err) + } + valLen := binary.LittleEndian.Uint32(lenbuf[:]) + v := make([]byte, valLen) + if _, err := io.ReadFull(r, v); err != nil { + return fmt.Errorf("read val: %w", err) + } + leaves++ + if !boundary.IsMigrated(keys.EVMStoreKey, k) { + if err := d.consumeSemanticMemiavlLeaf(accounts, k, v); err != nil { + return err + } + consumed++ + } + if leaves%20000000 == 0 { + fmt.Printf(" progress backend=composite source=memiavl input_leaves=%d consumed_unmigrated=%d account_buffered=%d code=%d storage=%d legacy=%d\n", leaves, consumed, len(accounts), d.code.count, d.storage.count, d.legacy.count) + } + } + fmt.Printf(" composite memiavl leaves=%d consumed_unmigrated=%d\n", leaves, consumed) + return nil +} + +func consumeCompositeMemiAVLReplay(db *memiavl.DB, boundary migration.MigrationBoundary, d *evmDigest, accounts map[string]*semanticAccountDigestState) error { + tree := db.TreeByName(keys.EVMStoreKey) + if tree == nil { + return fmt.Errorf("memiavl tree %q not found", keys.EVMStoreKey) + } + iter := tree.Iterator(nil, nil, true) + defer func() { _ = iter.Close() }() + var leaves, consumed uint64 + for ; iter.Valid(); iter.Next() { + leaves++ + k := iter.Key() + if !boundary.IsMigrated(keys.EVMStoreKey, k) { + if err := d.consumeSemanticMemiavlLeaf(accounts, k, iter.Value()); err != nil { + return err + } + consumed++ + } + if leaves%20000000 == 0 { + fmt.Printf(" progress backend=composite source=memiavl-replay input_leaves=%d consumed_unmigrated=%d account_buffered=%d code=%d storage=%d legacy=%d\n", leaves, consumed, len(accounts), d.code.count, d.storage.count, d.legacy.count) + } + } + if err := iter.Error(); err != nil { + return fmt.Errorf("iterate replayed memiavl: %w", err) + } + fmt.Printf(" composite memiavl-replay leaves=%d consumed_unmigrated=%d\n", leaves, consumed) + return nil +} + func digestFlatKV(dbDir string, height int64, findTarget []byte) error { opened, err := openFlatKVReadOnly(dbDir, height) if err != nil { @@ -367,9 +662,6 @@ func digestFlatKV(dbDir string, height int64, findTarget []byte) error { for ; iter.Valid(); iter.Next() { k := iter.Key() seen++ - if !shouldIncludeFlatKVEVMLogicalDigestKey(k) { - continue - } if err := d.consume(k, iter.Value()); err != nil { return err } @@ -385,14 +677,6 @@ func digestFlatKV(dbDir string, height int64, findTarget []byte) error { return nil } -func shouldIncludeFlatKVEVMLogicalDigestKey(physKey []byte) bool { - if bytes.Equal(physKey, migrationVersionPhysKey) { - return true - } - moduleName, _, err := ktype.StripModulePrefix(physKey) - return err == nil && moduleName == keys.EVMStoreKey -} - type inspectAccumulator struct { inspectBucket string keyOffset int @@ -403,7 +687,7 @@ type inspectAccumulator struct { details bool shards map[string]*digestBucket matched uint64 - listed int + listed uint64 } func runEvmLogicalInspect(cmd *cobra.Command, backend, dbDir string, height int64, inspectBucket string, memiavlNormalization string) error { @@ -473,7 +757,7 @@ func (a *inspectAccumulator) consumeLogical(bucket string, physKey, logical []by } a.matched++ if a.list { - if a.listLimit <= 0 || a.listed < a.listLimit { + if a.listLimit <= 0 || int(a.listed) < a.listLimit { if meta != "" { fmt.Printf("key=%X logical=%X %s\n", physKey, logical, meta) } else { @@ -531,9 +815,6 @@ func inspectFlatKV(dbDir string, height int64, acc *inspectAccumulator) error { var seen uint64 for ; iter.Valid(); iter.Next() { seen++ - if !shouldIncludeFlatKVEVMLogicalDigestKey(iter.Key()) { - continue - } meta := "" if acc.details && acc.list { var derr error @@ -858,7 +1139,13 @@ func flatKVValueMeta(physKey, val []byte) (string, error) { // kvs record layout (little-endian), repeated leafCount times until EOF: // // keyLen uint32 | key [keyLen] | valLen uint32 | value [valLen] -func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization string) error { +func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization string, openMode string) error { + if openMode == "replay" { + return digestMemIAVLReplay(dbDir, height, findTarget, normalization) + } + if openMode != "" && openMode != "snapshot" { + return fmt.Errorf("unknown --memiavl-open-mode %q (want snapshot|replay)", openMode) + } switch normalization { case "", "semantic", "independent": return digestMemIAVLSemantic(dbDir, height, findTarget) @@ -869,6 +1156,159 @@ func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization } } +func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) { + db, err := memiavl.OpenDB(height, memiavl.Options{ + Dir: dbDir, + ReadOnly: true, + ZeroCopy: true, + }) + if err != nil { + return nil, fmt.Errorf("open memiavl read-only replay: %w", err) + } + return db, nil +} + +func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normalization string) error { + db, err := openMemiAVLReplayReadOnly(dbDir, height) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + + switch normalization { + case "", "semantic", "independent": + return digestMemIAVLReplaySemantic(dbDir, height, db, findTarget) + case "translator": + return digestMemIAVLReplayTranslator(dbDir, height, db, findTarget) + default: + return fmt.Errorf("unknown --memiavl-normalization %q (want semantic|independent|translator)", normalization) + } +} + +func digestMemIAVLReplaySemantic(dbDir string, height int64, db *memiavl.DB, findTarget []byte) error { + tree := db.TreeByName(keys.EVMStoreKey) + if tree == nil { + return fmt.Errorf("memiavl tree %q not found", keys.EVMStoreKey) + } + iter := tree.Iterator(nil, nil, true) + defer func() { _ = iter.Close() }() + + d := evmDigest{findTarget: findTarget} + accounts := make(map[string]*semanticAccountDigestState) + ctx := digestPrintContext{ + backend: "memiavl", + mode: "semantic-replay", + dbDir: dbDir, + source: "read-only memiavl DB opened from snapshot + changelog replay", + normalization: "independent semantic decoder for replayed memiavl EVM keys; does not call flatkv.ImportTranslator", + requestedHeight: height, + version: db.Version(), + } + printDigestStart(ctx) + fmt.Println("Scan progress: replayed memiavl iterator -> independently decoded EVM logical bucket counts") + fmt.Println("Note: semantic replay mode walks the in-memory/mmap tree, not the snapshot kvs file.") + + var leaves uint64 + for ; iter.Valid(); iter.Next() { + leaves++ + if err := d.consumeSemanticMemiavlLeaf(accounts, iter.Key(), iter.Value()); err != nil { + return err + } + if leaves%20000000 == 0 { + fmt.Printf(" progress backend=memiavl mode=semantic-replay input_leaves=%d digested account=deferred_until_finalize account_buffered=%d code=%d storage=%d legacy=%d\n", + leaves, len(accounts), d.code.count, d.storage.count, d.legacy.count) + } + } + if err := iter.Error(); err != nil { + return fmt.Errorf("iterate replayed memiavl: %w", err) + } + + d.finalizeSemanticAccounts(accounts) + fmt.Printf(" finalize backend=memiavl mode=semantic-replay account=%d code=%d storage=%d legacy=%d\n", + d.account.count, d.code.count, d.storage.count, d.legacy.count) + fmt.Printf(" memiavl-replay total leaves=%d\n", leaves) + d.print(ctx) + return nil +} + +func digestMemIAVLReplayTranslator(dbDir string, height int64, db *memiavl.DB, findTarget []byte) error { + tree := db.TreeByName(keys.EVMStoreKey) + if tree == nil { + return fmt.Errorf("memiavl tree %q not found", keys.EVMStoreKey) + } + iter := tree.Iterator(nil, nil, true) + defer func() { _ = iter.Close() }() + + translator := flatkv.NewImportTranslator(0) + d := evmDigest{findTarget: findTarget} + ctx := digestPrintContext{ + backend: "memiavl", + mode: "translator-replay", + dbDir: dbDir, + source: "read-only memiavl DB opened from snapshot + changelog replay", + normalization: "replayed memiavl leaves translated with flatkv.ImportTranslator, then reduced to logical payload", + requestedHeight: height, + version: db.Version(), + } + printDigestStart(ctx) + fmt.Println("Scan progress: replayed memiavl iterator -> translated flatkv logical bucket counts") + fmt.Println("Note: account rows are merged by the translator at finalize, so progress shows account=deferred_until_finalize.") + + var leaves uint64 + const batchCap = 8192 + batch := make([]*proto.KVPair, 0, batchCap) + flush := func() error { + if len(batch) == 0 { + return nil + } + cs := &proto.NamedChangeSet{Name: keys.EVMStoreKey, Changeset: proto.ChangeSet{Pairs: batch}} + pairs, terr := translator.Translate(cs) + if terr != nil { + return fmt.Errorf("translate batch: %w", terr) + } + for _, p := range pairs { + if cerr := d.consume(p.Key, p.Value); cerr != nil { + return cerr + } + } + batch = batch[:0] + return nil + } + + for ; iter.Valid(); iter.Next() { + leaves++ + batch = append(batch, &proto.KVPair{Key: iter.Key(), Value: iter.Value()}) + if len(batch) >= batchCap { + if err := flush(); err != nil { + return err + } + } + if leaves%20000000 == 0 { + if err := flush(); err != nil { + return err + } + fmt.Printf(" progress backend=memiavl mode=translator-replay input_leaves=%d digested account=deferred_until_finalize code=%d storage=%d legacy=%d\n", + leaves, d.code.count, d.storage.count, d.legacy.count) + } + } + if err := iter.Error(); err != nil { + return fmt.Errorf("iterate replayed memiavl: %w", err) + } + if err := flush(); err != nil { + return err + } + for _, p := range translator.Finalize() { + if err := d.consume(p.Key, p.Value); err != nil { + return err + } + } + fmt.Printf(" finalize backend=memiavl mode=translator-replay account=%d code=%d storage=%d legacy=%d\n", + d.account.count, d.code.count, d.storage.count, d.legacy.count) + fmt.Printf(" memiavl-replay total leaves=%d\n", leaves) + d.print(ctx) + return nil +} + func digestMemIAVLTranslator(dbDir string, height int64, findTarget []byte) error { evmSnapshotDir, err := resolveMemIAVLEvmSnapshotDir(dbDir, height) if err != nil { @@ -1006,6 +1446,7 @@ func digestMemIAVLTranslator(dbDir string, height int64, findTarget []byte) erro } type semanticAccountDigestState struct { + balance [32]byte nonce uint64 codeHash [32]byte } @@ -1017,6 +1458,11 @@ func (s *semanticAccountDigestState) isZeroAccount() bool { if s.nonce != 0 { return false } + for _, b := range s.balance { + if b != 0 { + return false + } + } for _, b := range s.codeHash { if b != 0 { return false @@ -1027,8 +1473,7 @@ func (s *semanticAccountDigestState) isZeroAccount() bool { func (s *semanticAccountDigestState) logicalPayload() []byte { logical := make([]byte, 72) - // Balance is not represented in the current memiavl EVM keyspace, so the - // first 32 bytes intentionally remain zero. + copy(logical[:32], s.balance[:]) binary.BigEndian.PutUint64(logical[32:40], s.nonce) copy(logical[40:], s.codeHash[:]) return logical diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest_test.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest_test.go index 2777891043..f6d099e486 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest_test.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest_test.go @@ -8,6 +8,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype" "github.com/stretchr/testify/require" ) @@ -87,19 +88,6 @@ func TestInspectMemiavlRejectsUnknownNormalizationBeforeOpeningSnapshot(t *testi require.ErrorContains(t, err, `unknown --memiavl-normalization "bogus"`) } -func TestShouldIncludeFlatKVEVMLogicalDigestKey(t *testing.T) { - addr := bytesOfLen(keys.AddressLen, 0x42) - slot := bytesOfLen(32, 0x07) - storageKeyBytes := append(append([]byte{}, addr...), slot...) - - require.True(t, shouldIncludeFlatKVEVMLogicalDigestKey(ktype.EVMPhysicalKey(keys.EVMKeyStorage, storageKeyBytes))) - require.True(t, shouldIncludeFlatKVEVMLogicalDigestKey(ktype.ModulePhysicalKey(keys.EVMStoreKey, []byte{0xFF, 0xAA}))) - require.True(t, shouldIncludeFlatKVEVMLogicalDigestKey(migrationVersionPhysKey)) - - require.False(t, shouldIncludeFlatKVEVMLogicalDigestKey(ktype.ModulePhysicalKey("bank", []byte("balance")))) - require.False(t, shouldIncludeFlatKVEVMLogicalDigestKey([]byte("malformed-key-without-module-prefix"))) -} - func coreEVMRawPairs() []*proto.KVPair { addr := bytesOfLen(keys.AddressLen, 0x42) slot := bytesOfLen(32, 0x07) @@ -142,3 +130,69 @@ func nonceBytes(n uint64) []byte { binary.BigEndian.PutUint64(bz, n) return bz } + +// TestLegacyForCompareOmitsMigrationMarkerRows pins the marker adjustment that +// lets a memiavl-only node, a completed node (carrying the migration-version +// marker), and an in-progress node (carrying the migration-boundary cursor) all +// produce the same legacy digest for identical EVM state. Both FlatKV-only +// MigrationStore rows are folded into the legacy bucket during the scan but must +// be XORed back out for the final cross-backend comparison. +func TestLegacyForCompareOmitsMigrationMarkerRows(t *testing.T) { + legacyKey := append([]byte{0x09}, bytesOfLen(keys.AddressLen, 0x33)...) + legacyVal := vtype.NewLegacyData().SetBlockHeight(10).SetValue([]byte{0xDE, 0xAD}).Serialize() + versionVal := vtype.NewLegacyData().SetBlockHeight(20).SetValue([]byte{0x01}).Serialize() + boundaryVal := vtype.NewLegacyData().SetBlockHeight(30).SetValue([]byte{0x02, 0x03}).Serialize() + + // memiavl-only / clean node: only the plain legacy row. + clean := evmDigest{} + require.NoError(t, clean.consume(legacyKey, legacyVal)) + + // completed node: plain row + migration-version marker. + completed := evmDigest{} + require.NoError(t, completed.consume(legacyKey, legacyVal)) + require.NoError(t, completed.consume(migrationVersionPhysKey, versionVal)) + require.True(t, completed.migrationVersionFound) + + // in-progress node: plain row + migration-boundary cursor. + inProgress := evmDigest{} + require.NoError(t, inProgress.consume(legacyKey, legacyVal)) + require.NoError(t, inProgress.consume(migrationBoundaryPhysKey, boundaryVal)) + require.True(t, inProgress.migrationBoundaryFound) + + // Raw legacy buckets differ because each folds in its marker row. + require.NotEqual(t, clean.legacy, completed.legacy) + require.NotEqual(t, clean.legacy, inProgress.legacy) + + // After the marker adjustment all three agree (digest and count). + cleanAcc, cleanCount := clean.legacyForCompare() + compAcc, compCount := completed.legacyForCompare() + progAcc, progCount := inProgress.legacyForCompare() + require.Equal(t, cleanAcc, compAcc, "completed node must match clean after omitting migration-version") + require.Equal(t, cleanCount, compCount) + require.Equal(t, cleanAcc, progAcc, "in-progress node must match clean after omitting migration-boundary") + require.Equal(t, cleanCount, progCount) +} + +func TestCompositeAccountMergeCombinesFlatKVAndMemiavlFragments(t *testing.T) { + addr := bytesOfLen(keys.AddressLen, 0x55) + codeHash := bytesOfLen(32, 0xCC) + balance := bytesOfLen(32, 0x11) + bal, err := vtype.ParseBalance(balance) + require.NoError(t, err) + + flatKVAccount := vtype.NewAccountData().SetBlockHeight(123).SetBalance(bal).SetNonce(9) + accounts := make(map[string]*semanticAccountDigestState) + require.NoError(t, mergeCompositeFlatKVAccount(accounts, ktype.EVMPhysicalKey(keys.EVMKeyNonce, addr), flatKVAccount.Serialize())) + + composite := evmDigest{} + require.NoError(t, composite.consumeSemanticMemiavlLeaf(accounts, keys.BuildEVMKey(keys.EVMKeyCodeHash, addr), codeHash)) + composite.finalizeSemanticAccounts(accounts) + + expected := evmDigest{} + codeHashParsed, err := vtype.ParseCodeHash(codeHash) + require.NoError(t, err) + fullAccount := vtype.NewAccountData().SetBlockHeight(456).SetBalance(bal).SetNonce(9).SetCodeHash(codeHashParsed) + require.NoError(t, expected.consume(ktype.EVMPhysicalKey(keys.EVMKeyNonce, addr), fullAccount.Serialize())) + + require.Equal(t, expected.account, composite.account) +} From 0a6b2bb7eb9fd84dacfc195bc6a3f3b0c4a5ee2b Mon Sep 17 00:00:00 2001 From: blindchaser Date: Mon, 6 Jul 2026 23:57:14 -0400 Subject: [PATCH 2/5] refactor(seidb): dedup evm-logical-digest memiavl scan variants Collapse the snapshot vs replay and semantic vs translator memiavl digest duplication behind a shared evmLeafSource abstraction (one snapshot kvs-file scanner + one replay-tree scanner) and two shared cores (runMemiavlSemanticDigest / runMemiavlTranslatorDigest). The inspect memiavl paths and the composite unmigrated-tail consume are folded onto the same scanners. Behavior is unchanged except the snapshot translator progress log now carries `mode=translator` for consistency with the other modes; FINAL_DIGEST / bucket output is identical. File shrinks ~190 lines. Co-authored-by: Cursor --- .../seidb/operations/evm_logical_digest.go | 503 ++++++------------ 1 file changed, 156 insertions(+), 347 deletions(-) diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 1ff2b67358..4bc95a1b0d 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -483,11 +483,15 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT } if boundary.Status() != migration.MigrationComplete { if memReplayDB != nil { - if err := consumeCompositeMemiAVLReplay(memReplayDB, boundary, &d, accounts); err != nil { + if err := consumeCompositeMemiavl(func(fn func(rawKey, rawVal []byte) error) error { + return scanMemiavlReplayEVMLeaves(memReplayDB, fn) + }, "memiavl-replay", boundary, &d, accounts); err != nil { return err } } else { - if err := consumeCompositeMemiAVL(memEvmSnapshotDir, boundary, &d, accounts); err != nil { + if err := consumeCompositeMemiavl(func(fn func(rawKey, rawVal []byte) error) error { + return scanMemiavlSnapshotEVMLeaves(memEvmSnapshotDir, fn) + }, "memiavl", boundary, &d, accounts); err != nil { return err } } @@ -559,36 +563,13 @@ func mergeCompositeFlatKVAccount(accounts map[string]*semanticAccountDigestState return nil } -func consumeCompositeMemiAVL(evmSnapshotDir string, boundary migration.MigrationBoundary, d *evmDigest, accounts map[string]*semanticAccountDigestState) error { - kvsPath := filepath.Join(evmSnapshotDir, "kvs") - f, err := os.Open(filepath.Clean(kvsPath)) - if err != nil { - return fmt.Errorf("open kvs %s: %w", kvsPath, err) - } - defer func() { _ = f.Close() }() - r := bufio.NewReaderSize(f, 16*1024*1024) - var lenbuf [4]byte +// consumeCompositeMemiavl folds the memiavl EVM leaves NOT yet migrated past the +// boundary into d (the unmigrated tail of the composite mid-migration view). +// srcLabel selects the snapshot vs replay wording so both modes emit the same +// progress output as before. +func consumeCompositeMemiavl(scan evmLeafSource, srcLabel string, boundary migration.MigrationBoundary, d *evmDigest, accounts map[string]*semanticAccountDigestState) error { var leaves, consumed uint64 - for { - if _, err := io.ReadFull(r, lenbuf[:]); err != nil { - if errors.Is(err, io.EOF) { - break - } - return fmt.Errorf("read key len: %w", err) - } - keyLen := binary.LittleEndian.Uint32(lenbuf[:]) - k := make([]byte, keyLen) - if _, err := io.ReadFull(r, k); err != nil { - return fmt.Errorf("read key: %w", err) - } - if _, err := io.ReadFull(r, lenbuf[:]); err != nil { - return fmt.Errorf("read val len: %w", err) - } - valLen := binary.LittleEndian.Uint32(lenbuf[:]) - v := make([]byte, valLen) - if _, err := io.ReadFull(r, v); err != nil { - return fmt.Errorf("read val: %w", err) - } + if err := scan(func(k, v []byte) error { leaves++ if !boundary.IsMigrated(keys.EVMStoreKey, k) { if err := d.consumeSemanticMemiavlLeaf(accounts, k, v); err != nil { @@ -597,38 +578,14 @@ func consumeCompositeMemiAVL(evmSnapshotDir string, boundary migration.Migration consumed++ } if leaves%20000000 == 0 { - fmt.Printf(" progress backend=composite source=memiavl input_leaves=%d consumed_unmigrated=%d account_buffered=%d code=%d storage=%d legacy=%d\n", leaves, consumed, len(accounts), d.code.count, d.storage.count, d.legacy.count) - } - } - fmt.Printf(" composite memiavl leaves=%d consumed_unmigrated=%d\n", leaves, consumed) - return nil -} - -func consumeCompositeMemiAVLReplay(db *memiavl.DB, boundary migration.MigrationBoundary, d *evmDigest, accounts map[string]*semanticAccountDigestState) error { - tree := db.TreeByName(keys.EVMStoreKey) - if tree == nil { - return fmt.Errorf("memiavl tree %q not found", keys.EVMStoreKey) - } - iter := tree.Iterator(nil, nil, true) - defer func() { _ = iter.Close() }() - var leaves, consumed uint64 - for ; iter.Valid(); iter.Next() { - leaves++ - k := iter.Key() - if !boundary.IsMigrated(keys.EVMStoreKey, k) { - if err := d.consumeSemanticMemiavlLeaf(accounts, k, iter.Value()); err != nil { - return err - } - consumed++ - } - if leaves%20000000 == 0 { - fmt.Printf(" progress backend=composite source=memiavl-replay input_leaves=%d consumed_unmigrated=%d account_buffered=%d code=%d storage=%d legacy=%d\n", leaves, consumed, len(accounts), d.code.count, d.storage.count, d.legacy.count) + fmt.Printf(" progress backend=composite source=%s input_leaves=%d consumed_unmigrated=%d account_buffered=%d code=%d storage=%d legacy=%d\n", + srcLabel, leaves, consumed, len(accounts), d.code.count, d.storage.count, d.legacy.count) } + return nil + }); err != nil { + return err } - if err := iter.Error(); err != nil { - return fmt.Errorf("iterate replayed memiavl: %w", err) - } - fmt.Printf(" composite memiavl-replay leaves=%d consumed_unmigrated=%d\n", leaves, consumed) + fmt.Printf(" composite %s leaves=%d consumed_unmigrated=%d\n", srcLabel, leaves, consumed) return nil } @@ -861,13 +818,6 @@ func inspectMemIAVLTranslator(dbDir string, height int64, acc *inspectAccumulato if err != nil { return err } - kvsPath := filepath.Join(evmSnapshotDir, "kvs") - f, err := os.Open(filepath.Clean(kvsPath)) - if err != nil { - return fmt.Errorf("open kvs %s: %w", kvsPath, err) - } - defer func() { _ = f.Close() }() - r := bufio.NewReaderSize(f, 16*1024*1024) translator := flatkv.NewImportTranslator(0) var leaves uint64 @@ -891,37 +841,18 @@ func inspectMemIAVLTranslator(dbDir string, height int64, acc *inspectAccumulato return nil } - var lenbuf [4]byte - for { - if _, err := io.ReadFull(r, lenbuf[:]); err != nil { - if errors.Is(err, io.EOF) { - break - } - return fmt.Errorf("read key len: %w", err) - } - keyLen := binary.LittleEndian.Uint32(lenbuf[:]) - k := make([]byte, keyLen) - if _, err := io.ReadFull(r, k); err != nil { - return fmt.Errorf("read key: %w", err) - } - if _, err := io.ReadFull(r, lenbuf[:]); err != nil { - return fmt.Errorf("read val len: %w", err) - } - valLen := binary.LittleEndian.Uint32(lenbuf[:]) - v := make([]byte, valLen) - if _, err := io.ReadFull(r, v); err != nil { - return fmt.Errorf("read val: %w", err) - } + if err := scanMemiavlSnapshotEVMLeaves(evmSnapshotDir, func(k, v []byte) error { leaves++ if leaves%20000000 == 0 { fmt.Printf(" ...memiavl inspect leaves=%d matched=%d\n", leaves, acc.matched) } batch = append(batch, &proto.KVPair{Key: k, Value: v}) if len(batch) >= batchCap { - if ferr := flush(); ferr != nil { - return ferr - } + return flush() } + return nil + }); err != nil { + return err } if err := flush(); err != nil { return err @@ -949,13 +880,6 @@ func inspectMemIAVLSemantic(dbDir string, height int64, acc *inspectAccumulator) if err != nil { return err } - kvsPath := filepath.Join(evmSnapshotDir, "kvs") - f, err := os.Open(filepath.Clean(kvsPath)) - if err != nil { - return fmt.Errorf("open kvs %s: %w", kvsPath, err) - } - defer func() { _ = f.Close() }() - r := bufio.NewReaderSize(f, 16*1024*1024) var accounts map[string]*semanticAccountDigestState if acc.inspectBucket == flatkvBucketAccount { @@ -964,35 +888,15 @@ func inspectMemIAVLSemantic(dbDir string, height int64, acc *inspectAccumulator) consume := func(bucket string, physKey, logical, _ []byte) { acc.consumeLogical(bucket, physKey, logical, "") } - var lenbuf [4]byte var leaves uint64 - for { - if _, err := io.ReadFull(r, lenbuf[:]); err != nil { - if errors.Is(err, io.EOF) { - break - } - return fmt.Errorf("read key len: %w", err) - } - keyLen := binary.LittleEndian.Uint32(lenbuf[:]) - k := make([]byte, keyLen) - if _, err := io.ReadFull(r, k); err != nil { - return fmt.Errorf("read key: %w", err) - } - if _, err := io.ReadFull(r, lenbuf[:]); err != nil { - return fmt.Errorf("read val len: %w", err) - } - valLen := binary.LittleEndian.Uint32(lenbuf[:]) - v := make([]byte, valLen) - if _, err := io.ReadFull(r, v); err != nil { - return fmt.Errorf("read val: %w", err) - } + if err := scanMemiavlSnapshotEVMLeaves(evmSnapshotDir, func(k, v []byte) error { leaves++ if leaves%20000000 == 0 { fmt.Printf(" ...memiavl inspect mode=semantic leaves=%d matched=%d\n", leaves, acc.matched) } - if err := consumeSemanticMemiavlLeaf(accounts, k, v, consume, "inspect"); err != nil { - return err - } + return consumeSemanticMemiavlLeaf(accounts, k, v, consume, "inspect") + }); err != nil { + return err } finalizeSemanticAccounts(accounts, consume) fmt.Printf(" memiavl inspect total leaves=%d\n", leaves) @@ -1185,76 +1089,105 @@ func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normaliz } } -func digestMemIAVLReplaySemantic(dbDir string, height int64, db *memiavl.DB, findTarget []byte) error { +// evmLeafSource streams raw memiavl EVM (key,val) leaves to fn, stopping on the +// first error. It abstracts the two ways the tool reads memiavl EVM leaves — a +// snapshot kvs file scan and a replayed read-only tree walk — so the semantic +// and translator digest cores below are shared between snapshot and replay modes. +type evmLeafSource func(fn func(rawKey, rawVal []byte) error) error + +// scanMemiavlSnapshotEVMLeaves streams every leaf from a memiavl EVM snapshot +// kvs file (length-prefixed keyLen|key|valLen|val, little-endian). +func scanMemiavlSnapshotEVMLeaves(evmSnapshotDir string, fn func(rawKey, rawVal []byte) error) error { + kvsPath := filepath.Join(evmSnapshotDir, "kvs") + f, err := os.Open(filepath.Clean(kvsPath)) + if err != nil { + return fmt.Errorf("open kvs %s: %w", kvsPath, err) + } + defer func() { _ = f.Close() }() + r := bufio.NewReaderSize(f, 16*1024*1024) + var lenbuf [4]byte + for { + if _, err := io.ReadFull(r, lenbuf[:]); err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return fmt.Errorf("read key len: %w", err) + } + k := make([]byte, binary.LittleEndian.Uint32(lenbuf[:])) + if _, err := io.ReadFull(r, k); err != nil { + return fmt.Errorf("read key: %w", err) + } + if _, err := io.ReadFull(r, lenbuf[:]); err != nil { + return fmt.Errorf("read val len: %w", err) + } + v := make([]byte, binary.LittleEndian.Uint32(lenbuf[:])) + if _, err := io.ReadFull(r, v); err != nil { + return fmt.Errorf("read val: %w", err) + } + if err := fn(k, v); err != nil { + return err + } + } +} + +// scanMemiavlReplayEVMLeaves streams every leaf from the EVM tree of a read-only +// replayed memiavl DB. +func scanMemiavlReplayEVMLeaves(db *memiavl.DB, fn func(rawKey, rawVal []byte) error) error { tree := db.TreeByName(keys.EVMStoreKey) if tree == nil { return fmt.Errorf("memiavl tree %q not found", keys.EVMStoreKey) } iter := tree.Iterator(nil, nil, true) defer func() { _ = iter.Close() }() + for ; iter.Valid(); iter.Next() { + if err := fn(iter.Key(), iter.Value()); err != nil { + return err + } + } + if err := iter.Error(); err != nil { + return fmt.Errorf("iterate replayed memiavl: %w", err) + } + return nil +} +// runMemiavlSemanticDigest digests memiavl EVM leaves with the independent +// semantic decoder (no flatkv.ImportTranslator), buffering account fragments and +// merging them at finalize. modeLabel/totalLabel select the snapshot vs replay +// wording so each mode emits the same progress output as before. +func runMemiavlSemanticDigest(ctx digestPrintContext, modeLabel, totalLabel string, findTarget []byte, scan evmLeafSource) error { d := evmDigest{findTarget: findTarget} accounts := make(map[string]*semanticAccountDigestState) - ctx := digestPrintContext{ - backend: "memiavl", - mode: "semantic-replay", - dbDir: dbDir, - source: "read-only memiavl DB opened from snapshot + changelog replay", - normalization: "independent semantic decoder for replayed memiavl EVM keys; does not call flatkv.ImportTranslator", - requestedHeight: height, - version: db.Version(), - } - printDigestStart(ctx) - fmt.Println("Scan progress: replayed memiavl iterator -> independently decoded EVM logical bucket counts") - fmt.Println("Note: semantic replay mode walks the in-memory/mmap tree, not the snapshot kvs file.") - var leaves uint64 - for ; iter.Valid(); iter.Next() { + if err := scan(func(k, v []byte) error { leaves++ - if err := d.consumeSemanticMemiavlLeaf(accounts, iter.Key(), iter.Value()); err != nil { + if err := d.consumeSemanticMemiavlLeaf(accounts, k, v); err != nil { return err } if leaves%20000000 == 0 { - fmt.Printf(" progress backend=memiavl mode=semantic-replay input_leaves=%d digested account=deferred_until_finalize account_buffered=%d code=%d storage=%d legacy=%d\n", - leaves, len(accounts), d.code.count, d.storage.count, d.legacy.count) + fmt.Printf(" progress backend=memiavl mode=%s input_leaves=%d digested account=deferred_until_finalize account_buffered=%d code=%d storage=%d legacy=%d\n", + modeLabel, leaves, len(accounts), d.code.count, d.storage.count, d.legacy.count) } + return nil + }); err != nil { + return err } - if err := iter.Error(); err != nil { - return fmt.Errorf("iterate replayed memiavl: %w", err) - } - d.finalizeSemanticAccounts(accounts) - fmt.Printf(" finalize backend=memiavl mode=semantic-replay account=%d code=%d storage=%d legacy=%d\n", - d.account.count, d.code.count, d.storage.count, d.legacy.count) - fmt.Printf(" memiavl-replay total leaves=%d\n", leaves) + fmt.Printf(" finalize backend=memiavl mode=%s account=%d code=%d storage=%d legacy=%d\n", + modeLabel, d.account.count, d.code.count, d.storage.count, d.legacy.count) + fmt.Printf(" %s=%d\n", totalLabel, leaves) d.print(ctx) return nil } -func digestMemIAVLReplayTranslator(dbDir string, height int64, db *memiavl.DB, findTarget []byte) error { - tree := db.TreeByName(keys.EVMStoreKey) - if tree == nil { - return fmt.Errorf("memiavl tree %q not found", keys.EVMStoreKey) - } - iter := tree.Iterator(nil, nil, true) - defer func() { _ = iter.Close() }() - +// runMemiavlTranslatorDigest digests memiavl EVM leaves by routing every leaf +// through flatkv.ImportTranslator (the exact classifyAndPrefix + merge path +// CommitStore.ApplyChangeSets uses) and then through the shared consume, making +// the memiavl and flatkv digests byte-identical by construction. Translate +// streams storage/code/legacy out immediately and only buffers account fragments +// until Finalize, so RSS stays bounded. +func runMemiavlTranslatorDigest(ctx digestPrintContext, modeLabel, totalLabel string, findTarget []byte, scan evmLeafSource) error { translator := flatkv.NewImportTranslator(0) d := evmDigest{findTarget: findTarget} - ctx := digestPrintContext{ - backend: "memiavl", - mode: "translator-replay", - dbDir: dbDir, - source: "read-only memiavl DB opened from snapshot + changelog replay", - normalization: "replayed memiavl leaves translated with flatkv.ImportTranslator, then reduced to logical payload", - requestedHeight: height, - version: db.Version(), - } - printDigestStart(ctx) - fmt.Println("Scan progress: replayed memiavl iterator -> translated flatkv logical bucket counts") - fmt.Println("Note: account rows are merged by the translator at finalize, so progress shows account=deferred_until_finalize.") - - var leaves uint64 const batchCap = 8192 batch := make([]*proto.KVPair, 0, batchCap) flush := func() error { @@ -1274,10 +1207,10 @@ func digestMemIAVLReplayTranslator(dbDir string, height int64, db *memiavl.DB, f batch = batch[:0] return nil } - - for ; iter.Valid(); iter.Next() { + var leaves uint64 + if err := scan(func(k, v []byte) error { leaves++ - batch = append(batch, &proto.KVPair{Key: iter.Key(), Value: iter.Value()}) + batch = append(batch, &proto.KVPair{Key: k, Value: v}) if len(batch) >= batchCap { if err := flush(); err != nil { return err @@ -1287,12 +1220,12 @@ func digestMemIAVLReplayTranslator(dbDir string, height int64, db *memiavl.DB, f if err := flush(); err != nil { return err } - fmt.Printf(" progress backend=memiavl mode=translator-replay input_leaves=%d digested account=deferred_until_finalize code=%d storage=%d legacy=%d\n", - leaves, d.code.count, d.storage.count, d.legacy.count) + fmt.Printf(" progress backend=memiavl mode=%s input_leaves=%d digested account=deferred_until_finalize code=%d storage=%d legacy=%d\n", + modeLabel, leaves, d.code.count, d.storage.count, d.legacy.count) } - } - if err := iter.Error(); err != nil { - return fmt.Errorf("iterate replayed memiavl: %w", err) + return nil + }); err != nil { + return err } if err := flush(); err != nil { return err @@ -1302,57 +1235,56 @@ func digestMemIAVLReplayTranslator(dbDir string, height int64, db *memiavl.DB, f return err } } - fmt.Printf(" finalize backend=memiavl mode=translator-replay account=%d code=%d storage=%d legacy=%d\n", - d.account.count, d.code.count, d.storage.count, d.legacy.count) - fmt.Printf(" memiavl-replay total leaves=%d\n", leaves) + fmt.Printf(" finalize backend=memiavl mode=%s account=%d code=%d storage=%d legacy=%d\n", + modeLabel, d.account.count, d.code.count, d.storage.count, d.legacy.count) + fmt.Printf(" %s=%d\n", totalLabel, leaves) d.print(ctx) return nil } +func digestMemIAVLReplaySemantic(dbDir string, height int64, db *memiavl.DB, findTarget []byte) error { + ctx := digestPrintContext{ + backend: "memiavl", + mode: "semantic-replay", + dbDir: dbDir, + source: "read-only memiavl DB opened from snapshot + changelog replay", + normalization: "independent semantic decoder for replayed memiavl EVM keys; does not call flatkv.ImportTranslator", + requestedHeight: height, + version: db.Version(), + } + printDigestStart(ctx) + fmt.Println("Scan progress: replayed memiavl iterator -> independently decoded EVM logical bucket counts") + fmt.Println("Note: semantic replay mode walks the in-memory/mmap tree, not the snapshot kvs file.") + return runMemiavlSemanticDigest(ctx, "semantic-replay", "memiavl-replay total leaves", findTarget, + func(fn func(rawKey, rawVal []byte) error) error { return scanMemiavlReplayEVMLeaves(db, fn) }) +} + +func digestMemIAVLReplayTranslator(dbDir string, height int64, db *memiavl.DB, findTarget []byte) error { + ctx := digestPrintContext{ + backend: "memiavl", + mode: "translator-replay", + dbDir: dbDir, + source: "read-only memiavl DB opened from snapshot + changelog replay", + normalization: "replayed memiavl leaves translated with flatkv.ImportTranslator, then reduced to logical payload", + requestedHeight: height, + version: db.Version(), + } + printDigestStart(ctx) + fmt.Println("Scan progress: replayed memiavl iterator -> translated flatkv logical bucket counts") + fmt.Println("Note: account rows are merged by the translator at finalize, so progress shows account=deferred_until_finalize.") + return runMemiavlTranslatorDigest(ctx, "translator-replay", "memiavl-replay total leaves", findTarget, + func(fn func(rawKey, rawVal []byte) error) error { return scanMemiavlReplayEVMLeaves(db, fn) }) +} + func digestMemIAVLTranslator(dbDir string, height int64, findTarget []byte) error { evmSnapshotDir, err := resolveMemIAVLEvmSnapshotDir(dbDir, height) if err != nil { return err } - version, err := readMemIAVLSnapshotVersion(evmSnapshotDir) if err != nil { return err } - - kvsPath := filepath.Join(evmSnapshotDir, "kvs") - f, err := os.Open(filepath.Clean(kvsPath)) - if err != nil { - return fmt.Errorf("open kvs %s: %w", kvsPath, err) - } - defer func() { _ = f.Close() }() - r := bufio.NewReaderSize(f, 16*1024*1024) - - // Route EVERY leaf through ImportTranslator and then through the same - // `consume` used for the FlatKV side. This makes the two backends - // byte-identical by construction: - // - // - Translate runs the exact classifyAndPrefix + processStorageChanges / - // processCodeChanges / processLegacyChanges / mergeAccountUpdates that - // CommitStore.ApplyChangeSets uses, so the physical keys AND the - // FlatKV-serialized values it emits match what is physically stored in - // pebble on the FlatKV node. - // - consume then deserializes those values (DeserializeStorageData / - // DeserializeCodeData / DeserializeAccountData) and digests only the - // height-independent logical payload — the identical code path both - // backends take. - // - // We deliberately do NOT take the old hybrid shortcut (emitting node.Value() - // raw with a hand-built key for storage/code): a raw memIAVL leaf value is - // not guaranteed to be byte-identical to FlatKV's normalized StorageData / - // CodeData payload, which would make the digest diverge spuriously even when - // the underlying state is identical. - // - // Memory: Translate streams storage/code/legacy pairs out immediately and - // only buffers account fragments (nonce/codehash) until Finalize, so feeding - // all leaves through it does not blow up RSS beyond the account buffer. - translator := flatkv.NewImportTranslator(0) - d := evmDigest{findTarget: findTarget} ctx := digestPrintContext{ backend: "memiavl", mode: "translator", @@ -1365,84 +1297,10 @@ func digestMemIAVLTranslator(dbDir string, height int64, findTarget []byte) erro printDigestStart(ctx) fmt.Println("Scan progress: memiavl input_leaves -> translated flatkv logical bucket counts") fmt.Println("Note: account rows are merged by the translator at finalize, so progress shows account=deferred_until_finalize.") - var leaves uint64 - - const batchCap = 8192 - batch := make([]*proto.KVPair, 0, batchCap) - flush := func() error { - if len(batch) == 0 { - return nil - } - cs := &proto.NamedChangeSet{Name: keys.EVMStoreKey, Changeset: proto.ChangeSet{Pairs: batch}} - pairs, terr := translator.Translate(cs) - if terr != nil { - return fmt.Errorf("translate batch: %w", terr) - } - for _, p := range pairs { - if cerr := d.consume(p.Key, p.Value); cerr != nil { - return cerr - } - } - batch = batch[:0] - return nil - } - - var lenbuf [4]byte - for { - // key length - if _, err := io.ReadFull(r, lenbuf[:]); err != nil { - if errors.Is(err, io.EOF) { - break - } - return fmt.Errorf("read key len: %w", err) - } - keyLen := binary.LittleEndian.Uint32(lenbuf[:]) - k := make([]byte, keyLen) - if _, err := io.ReadFull(r, k); err != nil { - return fmt.Errorf("read key: %w", err) - } - // value length - if _, err := io.ReadFull(r, lenbuf[:]); err != nil { - return fmt.Errorf("read val len: %w", err) - } - valLen := binary.LittleEndian.Uint32(lenbuf[:]) - v := make([]byte, valLen) - if _, err := io.ReadFull(r, v); err != nil { - return fmt.Errorf("read val: %w", err) - } - - leaves++ - batch = append(batch, &proto.KVPair{Key: k, Value: v}) - if len(batch) >= batchCap { - if ferr := flush(); ferr != nil { - return ferr - } - } - if leaves%20000000 == 0 { - // Flush before reporting so progress counts include all leaves seen so far. - if ferr := flush(); ferr != nil { - return ferr - } - fmt.Printf(" progress backend=memiavl input_leaves=%d digested account=deferred_until_finalize code=%d storage=%d legacy=%d\n", - leaves, d.code.count, d.storage.count, d.legacy.count) - } - } - - if err := flush(); err != nil { - return err - } - // Flush merged accounts buffered across all batches. - for _, p := range translator.Finalize() { - if cerr := d.consume(p.Key, p.Value); cerr != nil { - return cerr - } - } - fmt.Printf(" finalize backend=memiavl account=%d code=%d storage=%d legacy=%d\n", - d.account.count, d.code.count, d.storage.count, d.legacy.count) - - fmt.Printf(" memiavl total leaves=%d\n", leaves) - d.print(ctx) - return nil + return runMemiavlTranslatorDigest(ctx, "translator", "memiavl total leaves", findTarget, + func(fn func(rawKey, rawVal []byte) error) error { + return scanMemiavlSnapshotEVMLeaves(evmSnapshotDir, fn) + }) } type semanticAccountDigestState struct { @@ -1484,22 +1342,10 @@ func digestMemIAVLSemantic(dbDir string, height int64, findTarget []byte) error if err != nil { return err } - version, err := readMemIAVLSnapshotVersion(evmSnapshotDir) if err != nil { return err } - - kvsPath := filepath.Join(evmSnapshotDir, "kvs") - f, err := os.Open(filepath.Clean(kvsPath)) - if err != nil { - return fmt.Errorf("open kvs %s: %w", kvsPath, err) - } - defer func() { _ = f.Close() }() - r := bufio.NewReaderSize(f, 16*1024*1024) - - d := evmDigest{findTarget: findTarget} - accounts := make(map[string]*semanticAccountDigestState) ctx := digestPrintContext{ backend: "memiavl", mode: "semantic", @@ -1512,47 +1358,10 @@ func digestMemIAVLSemantic(dbDir string, height int64, findTarget []byte) error printDigestStart(ctx) fmt.Println("Scan progress: memiavl input_leaves -> independently decoded EVM logical bucket counts") fmt.Println("Note: semantic mode does not call flatkv.ImportTranslator; account rows are merged locally at finalize.") - - var lenbuf [4]byte - var leaves uint64 - for { - if _, err := io.ReadFull(r, lenbuf[:]); err != nil { - if errors.Is(err, io.EOF) { - break - } - return fmt.Errorf("read key len: %w", err) - } - keyLen := binary.LittleEndian.Uint32(lenbuf[:]) - k := make([]byte, keyLen) - if _, err := io.ReadFull(r, k); err != nil { - return fmt.Errorf("read key: %w", err) - } - if _, err := io.ReadFull(r, lenbuf[:]); err != nil { - return fmt.Errorf("read val len: %w", err) - } - valLen := binary.LittleEndian.Uint32(lenbuf[:]) - v := make([]byte, valLen) - if _, err := io.ReadFull(r, v); err != nil { - return fmt.Errorf("read val: %w", err) - } - - leaves++ - if err := d.consumeSemanticMemiavlLeaf(accounts, k, v); err != nil { - return err - } - if leaves%20000000 == 0 { - fmt.Printf(" progress backend=memiavl mode=semantic input_leaves=%d digested account=deferred_until_finalize account_buffered=%d code=%d storage=%d legacy=%d\n", - leaves, len(accounts), d.code.count, d.storage.count, d.legacy.count) - } - } - - d.finalizeSemanticAccounts(accounts) - fmt.Printf(" finalize backend=memiavl mode=semantic account=%d code=%d storage=%d legacy=%d\n", - d.account.count, d.code.count, d.storage.count, d.legacy.count) - - fmt.Printf(" memiavl total leaves=%d\n", leaves) - d.print(ctx) - return nil + return runMemiavlSemanticDigest(ctx, "semantic", "memiavl total leaves", findTarget, + func(fn func(rawKey, rawVal []byte) error) error { + return scanMemiavlSnapshotEVMLeaves(evmSnapshotDir, fn) + }) } func (d *evmDigest) finalizeSemanticAccounts(accounts map[string]*semanticAccountDigestState) { From d9cb0cdb40ad284227caf06e3f9aa8b299ce5f0b Mon Sep 17 00:00:00 2001 From: blindchaser Date: Tue, 7 Jul 2026 00:00:39 -0400 Subject: [PATCH 3/5] fix(seidb): exclude non-EVM FlatKV rows from evm-logical-digest Restore the EVM-module ingestion filter on the FlatKV scan paths (digestFlatKV, consumeCompositeFlatKV, inspectFlatKV). FlatKV's RawGlobalIterator yields every module's rows and routes non-EVM modules into the legacy bucket, while a memiavl node walks the EVM tree alone; without the filter the legacy bucket and FINAL_DIGEST diverge from a memiavl-only node once FlatKV holds non-EVM Cosmos state. The FlatKV-only migration markers are still let through and XORed out in legacyForCompare. Addresses Cursor Bugbot review. Co-authored-by: Cursor --- .../seidb/operations/evm_logical_digest.go | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 4bc95a1b0d..281f16787c 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -515,6 +515,21 @@ func readFlatKVMigrationState(store *openedFlatKV) (migration.MigrationBoundary, return migration.MigrationBoundaryNotStarted, false, 0, nil } +// shouldIncludeFlatKVEVMLogicalDigestKey reports whether a raw FlatKV physical +// row belongs in the EVM logical digest. FlatKV's RawGlobalIterator yields every +// module's rows, but only the EVM module participates in the comparison against a +// memiavl node (which walks the EVM tree alone). The FlatKV-only migration +// markers are let through here and then XORed back out in legacyForCompare; all +// other non-EVM module rows (e.g. Cosmos state migrated into FlatKV) are excluded +// so the legacy bucket and FINAL_DIGEST stay comparable across backends. +func shouldIncludeFlatKVEVMLogicalDigestKey(physKey []byte) bool { + if bytes.Equal(physKey, migrationVersionPhysKey) || bytes.Equal(physKey, migrationBoundaryPhysKey) { + return true + } + moduleName, _, err := ktype.StripModulePrefix(physKey) + return err == nil && moduleName == keys.EVMStoreKey +} + func consumeCompositeFlatKV(opened *openedFlatKV, d *evmDigest, accounts map[string]*semanticAccountDigestState) error { iter, err := opened.RawGlobalIterator() if err != nil { @@ -525,6 +540,9 @@ func consumeCompositeFlatKV(opened *openedFlatKV, d *evmDigest, accounts map[str for ; iter.Valid(); iter.Next() { seen++ k := iter.Key() + if !shouldIncludeFlatKVEVMLogicalDigestKey(k) { + continue + } if classifyFlatKVPhysicalKey(k) == flatkvBucketAccount { if err := mergeCompositeFlatKVAccount(accounts, k, iter.Value()); err != nil { return err @@ -619,6 +637,9 @@ func digestFlatKV(dbDir string, height int64, findTarget []byte) error { for ; iter.Valid(); iter.Next() { k := iter.Key() seen++ + if !shouldIncludeFlatKVEVMLogicalDigestKey(k) { + continue + } if err := d.consume(k, iter.Value()); err != nil { return err } @@ -772,6 +793,9 @@ func inspectFlatKV(dbDir string, height int64, acc *inspectAccumulator) error { var seen uint64 for ; iter.Valid(); iter.Next() { seen++ + if !shouldIncludeFlatKVEVMLogicalDigestKey(iter.Key()) { + continue + } meta := "" if acc.details && acc.list { var derr error From b49543caed8f578edec676fab9e395315488b161 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Tue, 7 Jul 2026 00:34:31 -0400 Subject: [PATCH 4/5] fix(seidb): satisfy goconst/gosec lint in evm-logical-digest - Extract memiavl-open-mode / normalization string literals (snapshot, replay, semantic, independent, translator) into named constants (goconst). - Change inspectAccumulator.listed to int and drop the uint64->int conversion in the list-limit check (gosec G115). Co-authored-by: Cursor --- .../seidb/operations/evm_logical_digest.go | 48 +++++++++++-------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 281f16787c..2282056e92 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -40,6 +40,16 @@ var migrationVersionPhysKey = []byte(migration.MigrationStore + "/" + migration. // completed or memiavl-only node, which carry no boundary row. var migrationBoundaryPhysKey = []byte(migration.MigrationStore + "/" + migration.MigrationBoundaryKey) +// memiavl-open-mode and memiavl-normalization flag values, named so they are not +// repeated as bare string literals (goconst). +const ( + memiavlOpenModeSnapshot = "snapshot" + memiavlOpenModeReplay = "replay" + memiavlNormSemantic = "semantic" + memiavlNormIndependent = "independent" + memiavlNormTranslator = "translator" +) + // EvmLogicalDigestCmd computes a backend-independent digest of the EVM logical // state (account / code / storage canonical buckets) so a memIAVL node and a // FlatKV node can be compared at the same chain height. @@ -145,8 +155,8 @@ func EvmLogicalDigestCmd() *cobra.Command { cmd.Flags().String("flatkv-dir", "", "Composite mode: flatkv data dir") cmd.Flags().String("memiavl-dir", "", "Composite mode: memiavl root dir (contains current/ and snapshot-* )") cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-/evm (0 = current symlink)") - cmd.Flags().String("memiavl-open-mode", "snapshot", "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: replays changelog to --height then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary") - cmd.Flags().String("memiavl-normalization", "semantic", "memiavl digest/inspect normalization: semantic/independent (raw EVM key/value decoder) | translator (current migration mapping)") + cmd.Flags().String("memiavl-open-mode", memiavlOpenModeSnapshot, "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: replays changelog to --height then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary") + cmd.Flags().String("memiavl-normalization", memiavlNormSemantic, "memiavl digest/inspect normalization: semantic/independent (raw EVM key/value decoder) | translator (current migration mapping)") cmd.Flags().String("inspect-bucket", "", "Inspect one normalized bucket (account|code|storage|legacy) instead of printing the global digest") cmd.Flags().Int("key-offset", 0, "Inspect mode: byte offset into physical key before applying --key-prefix / sharding") cmd.Flags().String("key-prefix", "", "Inspect mode: hex prefix, relative to --key-offset, used to filter physical keys") @@ -392,7 +402,7 @@ func runEvmLogicalDigest(cmd *cobra.Command, _ []string) error { memiavlNormalization, _ := cmd.Flags().GetString("memiavl-normalization") memiavlOpenMode, _ := cmd.Flags().GetString("memiavl-open-mode") if inspectBucket != "" { - if memiavlOpenMode != "snapshot" { + if memiavlOpenMode != memiavlOpenModeSnapshot { return fmt.Errorf("--inspect-bucket does not support --memiavl-open-mode=%q yet", memiavlOpenMode) } return runEvmLogicalInspect(cmd, backend, dbDir, height, inspectBucket, memiavlNormalization) @@ -449,7 +459,7 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT var memEvmSnapshotDir string var memVersion int64 switch memiavlOpenMode { - case "", "snapshot": + case "", memiavlOpenModeSnapshot: memEvmSnapshotDir, err = resolveMemIAVLEvmSnapshotDir(memIAVLDir, height) if err != nil { return err @@ -460,7 +470,7 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT } ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl snapshot=%s", opened.Version(), memEvmSnapshotDir) ctx.normalization = fmt.Sprintf("flatkv rows plus memiavl rows not migrated by boundary=%s version_known=%t migration_version=%d memiavl_version=%d", boundary.String(), versionKnown, migrationVersion, memVersion) - case "replay": + case memiavlOpenModeReplay: memReplayDB, err = openMemiAVLReplayReadOnly(memIAVLDir, height) if err != nil { return err @@ -665,7 +675,7 @@ type inspectAccumulator struct { details bool shards map[string]*digestBucket matched uint64 - listed uint64 + listed int } func runEvmLogicalInspect(cmd *cobra.Command, backend, dbDir string, height int64, inspectBucket string, memiavlNormalization string) error { @@ -735,7 +745,7 @@ func (a *inspectAccumulator) consumeLogical(bucket string, physKey, logical []by } a.matched++ if a.list { - if a.listLimit <= 0 || int(a.listed) < a.listLimit { + if a.listLimit <= 0 || a.listed < a.listLimit { if meta != "" { fmt.Printf("key=%X logical=%X %s\n", physKey, logical, meta) } else { @@ -820,9 +830,9 @@ func inspectFlatKV(dbDir string, height int64, acc *inspectAccumulator) error { func inspectMemIAVL(dbDir string, height int64, acc *inspectAccumulator, normalization string) error { switch normalization { - case "", "semantic", "independent": + case "", memiavlNormSemantic, memiavlNormIndependent: return inspectMemIAVLSemantic(dbDir, height, acc) - case "translator": + case memiavlNormTranslator: return inspectMemIAVLTranslator(dbDir, height, acc) default: return fmt.Errorf("unknown --memiavl-normalization %q (want semantic|independent|translator)", normalization) @@ -1068,16 +1078,16 @@ func flatKVValueMeta(physKey, val []byte) (string, error) { // // keyLen uint32 | key [keyLen] | valLen uint32 | value [valLen] func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization string, openMode string) error { - if openMode == "replay" { + if openMode == memiavlOpenModeReplay { return digestMemIAVLReplay(dbDir, height, findTarget, normalization) } - if openMode != "" && openMode != "snapshot" { + if openMode != "" && openMode != memiavlOpenModeSnapshot { return fmt.Errorf("unknown --memiavl-open-mode %q (want snapshot|replay)", openMode) } switch normalization { - case "", "semantic", "independent": + case "", memiavlNormSemantic, memiavlNormIndependent: return digestMemIAVLSemantic(dbDir, height, findTarget) - case "translator": + case memiavlNormTranslator: return digestMemIAVLTranslator(dbDir, height, findTarget) default: return fmt.Errorf("unknown --memiavl-normalization %q (want semantic|independent|translator)", normalization) @@ -1104,9 +1114,9 @@ func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normaliz defer func() { _ = db.Close() }() switch normalization { - case "", "semantic", "independent": + case "", memiavlNormSemantic, memiavlNormIndependent: return digestMemIAVLReplaySemantic(dbDir, height, db, findTarget) - case "translator": + case memiavlNormTranslator: return digestMemIAVLReplayTranslator(dbDir, height, db, findTarget) default: return fmt.Errorf("unknown --memiavl-normalization %q (want semantic|independent|translator)", normalization) @@ -1311,7 +1321,7 @@ func digestMemIAVLTranslator(dbDir string, height int64, findTarget []byte) erro } ctx := digestPrintContext{ backend: "memiavl", - mode: "translator", + mode: memiavlNormTranslator, dbDir: dbDir, source: evmSnapshotDir + " (snapshot/current only; no memiavl WAL replay)", normalization: "memiavl leaves translated with flatkv.ImportTranslator, then reduced to logical payload", @@ -1321,7 +1331,7 @@ func digestMemIAVLTranslator(dbDir string, height int64, findTarget []byte) erro printDigestStart(ctx) fmt.Println("Scan progress: memiavl input_leaves -> translated flatkv logical bucket counts") fmt.Println("Note: account rows are merged by the translator at finalize, so progress shows account=deferred_until_finalize.") - return runMemiavlTranslatorDigest(ctx, "translator", "memiavl total leaves", findTarget, + return runMemiavlTranslatorDigest(ctx, memiavlNormTranslator, "memiavl total leaves", findTarget, func(fn func(rawKey, rawVal []byte) error) error { return scanMemiavlSnapshotEVMLeaves(evmSnapshotDir, fn) }) @@ -1372,7 +1382,7 @@ func digestMemIAVLSemantic(dbDir string, height int64, findTarget []byte) error } ctx := digestPrintContext{ backend: "memiavl", - mode: "semantic", + mode: memiavlNormSemantic, dbDir: dbDir, source: evmSnapshotDir + " (snapshot/current only; no memiavl WAL replay)", normalization: "independent semantic decoder for raw memiavl EVM keys; does not call flatkv.ImportTranslator", @@ -1382,7 +1392,7 @@ func digestMemIAVLSemantic(dbDir string, height int64, findTarget []byte) error printDigestStart(ctx) fmt.Println("Scan progress: memiavl input_leaves -> independently decoded EVM logical bucket counts") fmt.Println("Note: semantic mode does not call flatkv.ImportTranslator; account rows are merged locally at finalize.") - return runMemiavlSemanticDigest(ctx, "semantic", "memiavl total leaves", findTarget, + return runMemiavlSemanticDigest(ctx, memiavlNormSemantic, "memiavl total leaves", findTarget, func(fn func(rawKey, rawVal []byte) error) error { return scanMemiavlSnapshotEVMLeaves(evmSnapshotDir, fn) }) From 178f60615bcc843b8e88662f048c8c4efe9b0311 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Tue, 7 Jul 2026 16:26:58 -0400 Subject: [PATCH 5/5] docs(seidb): add --backend composite usage example to evm-logical-digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document the mid-migration workflow (flatkv migrated rows ∪ memiavl not-yet-migrated rows) and when to use --memiavl-open-mode replay, which was missing from the Usage block despite composite being the tool's main mode for migrating nodes. Co-authored-by: Cursor --- .../tools/cmd/seidb/operations/evm_logical_digest.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 2282056e92..45b15b8e9d 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -123,6 +123,16 @@ const ( // seidb evm-logical-digest --backend memiavl --memiavl-open-mode replay \ // --db-dir /.sei/data/state_commit/memiavl --height 213205000 // +// # Mid-migration node: digest the full EVM logical view as the union of +// # flatkv (migrated rows) and memiavl (rows not yet past the boundary), to +// # compare a migrating node against a memiavl-only node at the same height. +// # Use --memiavl-open-mode replay when memiavl's retained snapshot height is +// # outside flatkv's retained snapshot window (the common case on a live +// # migrating node, where the two backends keep snapshots at different heights). +// seidb evm-logical-digest --backend composite --memiavl-open-mode replay \ +// --flatkv-dir /.sei/data/state_commit/flatkv \ +// --memiavl-dir /.sei/data/state_commit/memiavl --height 213200000 +// // # Translator-based memIAVL digest. This proves FlatKV state matches the // # current migration mapping and is useful when debugging ImportTranslator. // seidb evm-logical-digest --backend memiavl \