From 412f38a2d07defc4fd7e2d6b8b0d7f5aed754bba Mon Sep 17 00:00:00 2001 From: YimingZang Date: Wed, 26 Aug 2026 15:08:44 -0700 Subject: [PATCH 1/2] Fix SS migration doc --- docs/migration/giga_store_migration.md | 100 ++++++++++++++++++------- 1 file changed, 74 insertions(+), 26 deletions(-) diff --git a/docs/migration/giga_store_migration.md b/docs/migration/giga_store_migration.md index b83d7fda80..40264c716f 100644 --- a/docs/migration/giga_store_migration.md +++ b/docs/migration/giga_store_migration.md @@ -9,23 +9,25 @@ stores: | Layer | Cosmos backend | EVM backend | |-------|----------------|-------------| | **SC** (State Commit, app hash) | memiavl | FlatKV | -| **SS** (State Store, historical queries) | single MVCC DB (Pebble/Rocks) | dedicated EVM SS MVCC DB(s) under `data/evm_ss/` | +| **SS** (State Store, historical queries) | single MVCC DB (Pebble/Rocks) under `data/state_store/cosmos/{backend}` | dedicated EVM SS MVCC DB under `data/state_store/evm/{backend}` | Only the **SS** layer changes for this migration. SC layer config is unaffected. ## Prerequisite - This migration guide is for **RPC nodes only**. Validator nodes and archive nodes are not supported by this migration flow yet. -- Migrating to Giga SS Store **requires a full state sync**. There is no in-place - migration path and no live "dual-write then split" workflow. A state sync wipes the - local data directory and imports a fresh snapshot into the new layout. +- Migrating to Giga SS Store **requires a full state sync**, or restoring a + data-directory snapshot taken from a node that already has Giga SS Store enabled. + There is no in-place migration path and no live "dual-write then split" workflow. + A state sync wipes the local data directory and imports a fresh snapshot into the + new layout. - `sc-enable = true` and `ss-enable = true`. Both must be enabled for this migration. ## Benefits - EVM reads served exclusively from a dedicated EVM SS database. - Non-EVM modules no longer pay write amplification for EVM state. - Backend change (PebbleDB ↔ RocksDB) can be combined with the same state sync since - `ss-backend` drives both the Cosmos SS MVCC DB and the EVM SS sub-DBs. + `ss-backend` drives both the Cosmos SS MVCC DB and the EVM SS DB. ## What's different about EVM SS EVM SS is **point-query only by design** (`Get` / `Has`). Iteration is explicitly @@ -46,7 +48,7 @@ sc-enable = true [state-store] ss-enable = true -# DBBackend for the Cosmos SS MVCC DB and for every EVM SS sub-DB. +# DBBackend for the Cosmos SS MVCC DB and for the EVM SS DB. # Supported: pebbledb, rocksdb. Default pebbledb. ss-backend = "pebbledb" @@ -55,6 +57,12 @@ ss-backend = "pebbledb" # else. When true, EVM data is routed exclusively to the EVM SS backend; non-EVM data # stays in Cosmos SS. No fallback between backends. evm-ss-split = true + +# Split EVM key families across multiple DBs inside the EVM SS directory. +# Default false: all EVM state lives in a single DB (the recommended, safer layout). +# Setting true can improve performance but is experimental and not fully tested. +# Leave this at false unless you are deliberately evaluating that path. +evm-ss-separate-dbs = false ``` If you are switching backend in the same step: @@ -64,9 +72,9 @@ If you are switching backend in the same step: layout. ### Step 2: State Sync -Giga SS Store is fully compatible with the existing state snapshot format. On import, -the composite state store routes each snapshot node based on the importing node's -`evm-ss-split`: +Giga SS Store is fully compatible with the existing **P2P state-sync** snapshot format. +On import, the composite state store routes each snapshot node based on the importing +node's `evm-ss-split`: - With `evm-ss-split = true`, EVM snapshot nodes go only into EVM SS and non-EVM nodes go only into Cosmos SS. @@ -76,6 +84,21 @@ the composite state store routes each snapshot node based on the importing node' Both stores end up fully populated at the snapshot height, so the node can start serving reads immediately. +P2P state-sync snapshots (the chunks peers serve over the network) are **not** +layout-sensitive. Giga SS Store only changes how the importing node writes those +nodes onto disk. + +**Data-directory snapshots** (a tar of `~/.sei/data`) *are* layout-sensitive. A +tarball taken from a Giga SS node contains `data/state_store/evm/` (and +`data/state_store/cosmos/`) instead of a single mixed Cosmos SS directory. You can +enable Giga SS Store by restoring such a snapshot and setting `evm-ss-split = true`; +you do not need to P2P state sync again. A tarball from a non-Giga node cannot be +used this way — convert those nodes with the state sync flow below. + +Snapshot hosts that publish `data/` tarballs may offer both Giga SS and non-Giga +copies while the fleet migrates. Use a Giga SS snapshot only when you intend to run +with `evm-ss-split = true`. + Use the state sync flow documented in the [SeiDB Migration Guide](./seidb_migration.md#step-3-state-sync). Minimal shape: @@ -115,12 +138,19 @@ systemctl restart seid ``` ## Verification -To confirm Giga SS Store is active, check the startup logs for both: +To confirm Giga SS Store is active, check the startup logs. With the default +(recommended) `evm-ss-separate-dbs = false`, they look like: + +``` +msg="SeiDB SS is enabled" backend=pebbledb +msg="SeiDB EVM StateStore optimization is enabled" separateDBs=false +msg="EVM state store enabled" logger=db/state-db/ss/composite dir=/home//.sei/data/state_store/evm/pebbledb separateDBs=false +``` -- `"SeiDB SS is enabled"` with the configured `backend`. -- `"SeiDB EVM StateStore optimization is enabled"` with the `separateDBs` label. -- `"EVM state store enabled"` from the composite store constructor with the `dir` - and `separateDBs` labels. +`separateDBs=false` is expected. It means EVM state is stored in a **single** DB +inside `data/state_store/evm/{backend}`, not split across per-type sub-DBs. That +is the default, for safety. `separateDBs=true` appears only if you opted into the +experimental `evm-ss-separate-dbs = true` setting. On an RPC node, confirm `debug_traceBlockByNumber` succeeds after state sync completes: @@ -139,9 +169,10 @@ Cosmos SS DBs are inconsistent. They specifically catch the footgun of flipping 1. **EVM SS directory missing or empty** (before the EVM SS is opened). When `evm-ss-split = true`, `NewCompositeStateStore` refuses to proceed if Cosmos SS - already has committed history but the EVM SS directory (`data/evm_ss/` by default) - does not exist or is empty. Running before the sub-DBs are opened means a rejected - config does not leave a confusing empty `data/evm_ss/` behind. + already has committed history but the EVM SS directory + (`data/state_store/evm/{backend}` by default) does not exist or is empty. Running + before the DB is opened means a rejected config does not leave a confusing empty + `data/state_store/evm/` behind. 2. **EVM SS DB empty post-open, pre-recovery.** Belt-and-suspenders for (1) when the directory exists but its DBs are empty. The WAL only covers the last `KeepRecent` @@ -152,24 +183,30 @@ Cosmos SS DBs are inconsistent. They specifically catch the footgun of flipping inconsistent. A non-zero earliest-version divergence aborts startup. If any check fires, the correct fix is either (a) complete the state sync described -above, or (b) set `evm-ss-split = false`. If `data/evm_ss/` is stale from a failed -attempt, remove it before state syncing. +above, or (b) set `evm-ss-split = false`. If `data/state_store/evm/` is stale from a +failed attempt, remove it before state syncing. ## Rollback Steps To roll back: - Set `evm-ss-split = false` in `app.toml`. -- Restart the node. The EVM SS DB under `data/evm_ss/` will not be opened but will - remain on disk until manually removed. +- Restart the node. The EVM SS DB under `data/state_store/evm/` will not be opened + but will remain on disk until manually removed. -To fully reclaim EVM SS disk usage, stop the node and delete `data/evm_ss/` after -reverting the setting. +To fully reclaim EVM SS disk usage, stop the node and delete `data/state_store/evm/` +after reverting the setting. Nodes that still have the legacy `data/evm_ss/` +directory should delete that instead — the node keeps using `data/evm_ss/` if it +already exists. ## FAQ ### Where can I find the data files after migrating? -- Cosmos SS data lives under the same directory as before (typically `data/pebbledb/` - for the default `pebbledb` backend). -- EVM SS data lives under `data/evm_ss/`. +- Cosmos SS data lives under `data/state_store/cosmos/{backend}` after this + migration (e.g. `data/state_store/cosmos/pebbledb`). Nodes that already had + `data/pebbledb/` keep using that legacy path; a wipe + state sync uses the new + layout. +- EVM SS data lives under `data/state_store/evm/{backend}` (e.g. + `data/state_store/evm/pebbledb`). The legacy path `data/evm_ss/` is used only if + that directory already exists on disk. - SC data (memiavl + FlatKV) is untouched by this migration. ### Does Giga SS Store change the app hash or consensus? @@ -195,5 +232,16 @@ Cosmos SS has. A live flip would leave the EVM SS DB empty while the composite s refuses to fall back to Cosmos SS, which would translate into missing EVM state at query time. The safety checks above block this scenario at startup. +### Does `separateDBs=false` in the startup log mean Giga SS Store is off? +No. `separateDBs` is the `evm-ss-separate-dbs` flag, not `evm-ss-split`. With Giga +SS Store enabled (`evm-ss-split = true`) and the default `evm-ss-separate-dbs = +false`, EVM state lives in a dedicated EVM SS directory as a single DB. Splitting +EVM into per-type sub-DBs (`evm-ss-separate-dbs = true`) is experimental. + +### Can I enable Giga SS Store from a data-directory snapshot instead of P2P state sync? +Yes, if the snapshot is a `data/` tarball taken from a node that already has Giga SS +Store enabled. Restore it, set `evm-ss-split = true`, and start. P2P state-sync +snapshots are unaffected either way — they do not embed the on-disk SS layout. + ### Does Giga SS Store support historical proofs? No, same as SeiDB. SS stores raw KVs and does not reconstruct IAVL-style proofs. From 0986ed2ac07b71e3e9abcff9758cf4ff0db9afa6 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 27 Aug 2026 23:23:46 -0700 Subject: [PATCH 2/2] More fix to the doc --- docs/migration/giga_store_migration.md | 18 ++++++++++++++---- sei-db/config/toml.go | 3 ++- sei-db/config/toml_test.go | 4 ++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/docs/migration/giga_store_migration.md b/docs/migration/giga_store_migration.md index 40264c716f..486cba4a14 100644 --- a/docs/migration/giga_store_migration.md +++ b/docs/migration/giga_store_migration.md @@ -92,8 +92,13 @@ nodes onto disk. tarball taken from a Giga SS node contains `data/state_store/evm/` (and `data/state_store/cosmos/`) instead of a single mixed Cosmos SS directory. You can enable Giga SS Store by restoring such a snapshot and setting `evm-ss-split = true`; -you do not need to P2P state sync again. A tarball from a non-Giga node cannot be -used this way — convert those nodes with the state sync flow below. +you do not need to P2P state sync again. The restoring node's `ss-backend`, +`evm-ss-separate-dbs`, and any `ss-db-directory` / `evm-ss-db-directory` overrides +must match the node the snapshot was taken from: both SS paths are backend-qualified +(`data/state_store/{cosmos,evm}/{backend}`), so a backend mismatch opens empty +directories and the node starts against restored SC with empty SS instead of +refusing to launch. A tarball from a non-Giga node cannot be used this way — +convert those nodes with the state sync flow below. Snapshot hosts that publish `data/` tarballs may offer both Giga SS and non-Giga copies while the fleet migrates. Use a Giga SS snapshot only when you intend to run @@ -240,8 +245,13 @@ EVM into per-type sub-DBs (`evm-ss-separate-dbs = true`) is experimental. ### Can I enable Giga SS Store from a data-directory snapshot instead of P2P state sync? Yes, if the snapshot is a `data/` tarball taken from a node that already has Giga SS -Store enabled. Restore it, set `evm-ss-split = true`, and start. P2P state-sync -snapshots are unaffected either way — they do not embed the on-disk SS layout. +Store enabled. Restore it, set `evm-ss-split = true`, and match the source node's +`ss-backend`, `evm-ss-separate-dbs`, and any `ss-db-directory` / `evm-ss-db-directory` +overrides. Both SS paths are backend-qualified +(`data/state_store/{cosmos,evm}/{backend}`), so a backend mismatch opens empty +directories and the node starts against restored SC with empty SS instead of +refusing to launch. P2P state-sync snapshots are unaffected either way — they do +not embed the on-disk SS layout. ### Does Giga SS Store support historical proofs? No, same as SeiDB. SS stores raw KVs and does not reconstruct IAVL-style proofs. diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index 34e0d8b42b..34cf8cf8e8 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -154,7 +154,8 @@ ss-enable-read-write-metrics = {{ .StateStore.EnableReadWriteMetrics }} ss-snapshot-enable = {{ .StateStore.SnapshotEnable }} # EVMDBDirectory defines the directory for the optional EVM state-store DB(s). -# If unset, defaults to /data/evm_ss when EVM SS is enabled. +# If unset, defaults to /data/state_store/evm/{backend}. Nodes that +# already have /data/evm_ss keep using that path. evm-ss-db-directory = "{{ .StateStore.EVMDBDirectory }}" # EVMSplit controls whether EVM data is routed to a dedicated SS backend. diff --git a/sei-db/config/toml_test.go b/sei-db/config/toml_test.go index a07fbb05fd..0ae65d6eb9 100644 --- a/sei-db/config/toml_test.go +++ b/sei-db/config/toml_test.go @@ -99,6 +99,10 @@ func TestStateStoreConfigTemplate(t *testing.T) { require.Contains(t, output, "ss-enable-read-write-metrics = false", "Missing state-store read/write metrics flag") require.Contains(t, output, "ss-snapshot-enable = false", "Missing or incorrect ss-snapshot-enable") require.Contains(t, output, `evm-ss-db-directory = ""`, "Missing evm-ss-db-directory") + require.Contains(t, output, "/data/state_store/evm/{backend}", + "evm-ss-db-directory comment must name the current default path") + require.NotContains(t, output, "/data/evm_ss when EVM SS is enabled", + "evm-ss-db-directory comment must not name data/evm_ss as the default") require.Contains(t, output, `evm-ss-split = false`, "Missing or incorrect evm-ss-split") require.Contains(t, output, "evm-ss-separate-dbs = false", "Missing or incorrect evm-ss-separate-dbs") }