From bbea8b8238928c09105ec6394f6ff811a41d1ec7 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Tue, 25 Aug 2026 17:19:06 +0200 Subject: [PATCH 1/5] Bound /subspace ABCI queries to prevent memory-exhaustion DoS (PLT-799). Reject empty prefixes, cap pair/byte accumulation during iteration, and limit concurrent SS fast-path scans so unauthenticated callers cannot OOM a node with wide subspace queries. Co-authored-by: Cursor --- app/config_fuzz_test.go | 12 ++- app/seidb.go | 12 +++ app/testdata/state-commit.golden | 3 + app/testdata/state-commit.keys.golden | 3 + sei-cosmos/store/types/errors.go | 3 + sei-cosmos/storev2/commitment/store.go | 35 +++--- sei-cosmos/storev2/commitment/store_test.go | 44 +++++++- sei-cosmos/storev2/query/subspace.go | 80 ++++++++++++++ sei-cosmos/storev2/query/subspace_test.go | 87 +++++++++++++++ sei-cosmos/storev2/rootmulti/metrics.go | 6 ++ sei-cosmos/storev2/rootmulti/store.go | 82 +++++++++++--- sei-cosmos/storev2/rootmulti/store_test.go | 113 ++++++++++++++++++++ sei-cosmos/storev2/state/store.go | 33 +++--- sei-db/config/sc_config.go | 16 +++ sei-db/config/toml.go | 9 ++ sei-db/config/toml_test.go | 3 + 16 files changed, 477 insertions(+), 64 deletions(-) create mode 100644 sei-cosmos/storev2/query/subspace.go create mode 100644 sei-cosmos/storev2/query/subspace_test.go diff --git a/app/config_fuzz_test.go b/app/config_fuzz_test.go index 9d42328eee..70d6bb6583 100644 --- a/app/config_fuzz_test.go +++ b/app/config_fuzz_test.go @@ -70,6 +70,9 @@ var scKeys = []configtest.KeySpec{ {Key: FlagSCHistoricalProofMaxInFlight, Path: "HistoricalProofMaxInFlight", Cast: configtest.CastInt}, {Key: FlagSCHistoricalProofRateLimit, Path: "HistoricalProofRateLimit", Cast: configtest.CastFloat64}, {Key: FlagSCHistoricalProofBurst, Path: "HistoricalProofBurst", Cast: configtest.CastInt}, + {Key: FlagSCSubspaceQueryMaxInFlight, Path: "SubspaceQueryMaxInFlight", Cast: configtest.CastInt}, + {Key: FlagSCSubspaceMaxPairs, Path: "SubspaceMaxPairs", Cast: configtest.CastInt}, + {Key: FlagSCSubspaceMaxBytes, Path: "SubspaceMaxBytes", Cast: configtest.CastInt}, { Key: FlagSCHashLoggerEnable, Path: "HashLogger.Enable", Cast: configtest.CastBool, Why: "default true; guarded so an absent key does not silently turn hash logging off", @@ -238,15 +241,18 @@ func FuzzParseSCConfigs(f *testing.F) { seeds.AddRow(uint(4), fuzzing.KindInt64, "", int64(-1), false) // negative into an unchecked unsigned cast: resolves 0 seeds.AddRow(uint(7), fuzzing.KindFloat64, "", int64(2), false) // prefetch threshold as a float seeds.AddRow(uint(1), fuzzing.KindString, "/var/lib/sei/sc", int64(0), false) - seeds.AddRow(uint(13), fuzzing.KindString, "not-a-bool", int64(0), false) // unchecked: resolves false, no error - seeds.AddRow(uint(15), fuzzing.KindInt64, "", int64(0), false) // explicit 0 taken verbatim + seeds.AddRow(uint(16), fuzzing.KindString, "not-a-bool", int64(0), false) // unchecked: resolves false, no error + seeds.AddRow(uint(18), fuzzing.KindInt64, "", int64(0), false) // explicit 0 taken verbatim // Two rows default to their cast's zero, which is also what the malformed seed resolves // to on an unchecked read, so neither of the per-row seeds above moves the field off the // value an absent key produces. Each gets one value that converts to something else, // which is what holds the reader to the key name rather than only to the cast. seeds.AddRow(uint(9), fuzzing.KindBool, "", int64(0), true) // flatkv read/write metrics on; the default is off - seeds.AddRow(uint(15), fuzzing.KindInt64, "", int64(100000), false) // block-count retention on; the default is 0, meaning disabled + seeds.AddRow(uint(18), fuzzing.KindInt64, "", int64(100000), false) // block-count retention on; the default is 0, meaning disabled + seeds.AddRow(uint(13), fuzzing.KindInt64, "", int64(5), false) // subspace max inflight above default 2 + seeds.AddRow(uint(14), fuzzing.KindInt64, "", int64(500), false) // subspace max pairs below default 1000 + seeds.AddRow(uint(15), fuzzing.KindInt64, "", int64(1024), false) // subspace max bytes below default 4 MiB configtest.CheckEveryRowHasADiscriminatingSeed(f, "state-commit", readSC, scKeys, seeds, scKeysWithTargetsOfTheirOwn...) diff --git a/app/seidb.go b/app/seidb.go index 716583de37..c010be532a 100644 --- a/app/seidb.go +++ b/app/seidb.go @@ -28,6 +28,9 @@ const ( FlagSCHistoricalProofMaxInFlight = "state-commit.sc-historical-proof-max-inflight" FlagSCHistoricalProofRateLimit = "state-commit.sc-historical-proof-rate-limit" FlagSCHistoricalProofBurst = "state-commit.sc-historical-proof-burst" + FlagSCSubspaceQueryMaxInFlight = "state-commit.sc-subspace-query-max-inflight" + FlagSCSubspaceMaxPairs = "state-commit.sc-subspace-max-pairs" + FlagSCSubspaceMaxBytes = "state-commit.sc-subspace-max-bytes" FlagSCWriteMode = "state-commit.sc-write-mode" FlagSCWriteModeEnableAuto = "state-commit.sc-write-mode-enable-auto" FlagSCFlatKVReadWriteMetrics = "state-commit.flatkv.enable-read-write-metrics" @@ -165,6 +168,15 @@ func parseSCConfigs(appOpts servertypes.AppOptions) config.StateCommitConfig { if v := appOpts.Get(FlagSCHistoricalProofBurst); v != nil { scConfig.HistoricalProofBurst = cast.ToInt(v) } + if v := appOpts.Get(FlagSCSubspaceQueryMaxInFlight); v != nil { + scConfig.SubspaceQueryMaxInFlight = cast.ToInt(v) + } + if v := appOpts.Get(FlagSCSubspaceMaxPairs); v != nil { + scConfig.SubspaceMaxPairs = cast.ToInt(v) + } + if v := appOpts.Get(FlagSCSubspaceMaxBytes); v != nil { + scConfig.SubspaceMaxBytes = cast.ToInt(v) + } // Hash logger. Guard each read with v != nil so an absent app.toml entry preserves the default // (notably Enable, which defaults to true) instead of clobbering it to the zero value. diff --git a/app/testdata/state-commit.golden b/app/testdata/state-commit.golden index 22881423c1..09019130c4 100644 --- a/app/testdata/state-commit.golden +++ b/app/testdata/state-commit.golden @@ -63,6 +63,9 @@ FlatKVConfig.LtHashThreadsPerCore = float64(1) HistoricalProofMaxInFlight = int(1) HistoricalProofRateLimit = float64(1) HistoricalProofBurst = int(1) +SubspaceQueryMaxInFlight = int(2) +SubspaceMaxPairs = int(1000) +SubspaceMaxBytes = int(4194304) HashLogger.Enable = bool(true) HashLogger.Directory = string("") HashLogger.BlocksToRetain = uint(0) diff --git a/app/testdata/state-commit.keys.golden b/app/testdata/state-commit.keys.golden index e77e8ba067..41a0052b15 100644 --- a/app/testdata/state-commit.keys.golden +++ b/app/testdata/state-commit.keys.golden @@ -11,6 +11,9 @@ "state-commit.sc-historical-proof-max-inflight" "state-commit.sc-historical-proof-rate-limit" "state-commit.sc-historical-proof-burst" +"state-commit.sc-subspace-query-max-inflight" +"state-commit.sc-subspace-max-pairs" +"state-commit.sc-subspace-max-bytes" "state-commit.sc-hash-logger-enable" "state-commit.sc-hash-logger-directory" "state-commit.sc-hash-logger-blocks-to-retain" diff --git a/sei-cosmos/store/types/errors.go b/sei-cosmos/store/types/errors.go index c85cfcc9f2..188a2b3856 100644 --- a/sei-cosmos/store/types/errors.go +++ b/sei-cosmos/store/types/errors.go @@ -8,4 +8,7 @@ const StoreCodespace = "store" var ( ErrInvalidProof = sdkerrors.Register(StoreCodespace, 2, "invalid proof") + + // ErrSubspaceCapExceeded is returned when a /subspace scan would exceed configured limits. + ErrSubspaceCapExceeded = sdkerrors.Register(StoreCodespace, 3, "subspace result exceeds limit") ) diff --git a/sei-cosmos/storev2/commitment/store.go b/sei-cosmos/storev2/commitment/store.go index 9272425cc9..af92c3f5de 100644 --- a/sei-cosmos/storev2/commitment/store.go +++ b/sei-cosmos/storev2/commitment/store.go @@ -3,15 +3,14 @@ package commitment import ( "bytes" "context" - "fmt" "io" "cosmossdk.io/errors" "github.com/sei-protocol/sei-chain/sei-cosmos/store/cachekv" "github.com/sei-protocol/sei-chain/sei-cosmos/store/tracekv" "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/storev2/query" sdkerrors "github.com/sei-protocol/sei-chain/sei-cosmos/types/errors" - "github.com/sei-protocol/sei-chain/sei-cosmos/types/kv" seidbproto "github.com/sei-protocol/sei-chain/sei-db/proto" sctypes "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" @@ -25,13 +24,15 @@ var ( // Store Implements types.KVStore and CommitKVStore. type Store struct { - tree sctypes.CommitKVStore - changeSet seidbproto.ChangeSet + tree sctypes.CommitKVStore + changeSet seidbproto.ChangeSet + subspaceLimits query.Limits } -func NewStore(tree sctypes.CommitKVStore) *Store { +func NewStore(tree sctypes.CommitKVStore, subspaceLimits query.Limits) *Store { return &Store{ - tree: tree, + tree: tree, + subspaceLimits: subspaceLimits, } } @@ -133,7 +134,7 @@ func (st *Store) HasPendingChanges() bool { return len(st.changeSet.Pairs) > 0 } -func (st *Store) Query(_ context.Context, req abci.RequestQuery) (res abci.ResponseQuery) { +func (st *Store) Query(ctx context.Context, req abci.RequestQuery) (res abci.ResponseQuery) { if req.Height > 0 && req.Height != st.tree.Version() { return sdkerrors.QueryResult(errors.Wrap(sdkerrors.ErrInvalidHeight, "invalid height")) } @@ -152,24 +153,14 @@ func (st *Store) Query(_ context.Context, req abci.RequestQuery) (res abci.Respo op := types.NewIavlCommitmentOp(res.Key, commitmentProof) res.ProofOps = &crypto.ProofOps{Ops: []crypto.ProofOp{op.ProofOp()}} case "/subspace": - pairs := kv.Pairs{ - Pairs: make([]kv.Pair, 0), + if len(req.Data) == 0 { + return sdkerrors.QueryResult(errors.Wrap(sdkerrors.ErrInvalidRequest, "subspace prefix must not be empty")) } - - subspace := req.Data - res.Key = subspace - - iterator := types.KVStorePrefixIterator(st, subspace) - for ; iterator.Valid(); iterator.Next() { - pairs.Pairs = append(pairs.Pairs, kv.Pair{Key: iterator.Key(), Value: iterator.Value()}) - } - _ = iterator.Close() - - bz, err := pairs.Marshal() + res.Key = req.Data + bz, err := query.ScanSubspace(ctx, st, req.Data, st.subspaceLimits) if err != nil { - panic(fmt.Errorf("failed to marshal KV pairs: %w", err)) + return sdkerrors.QueryResult(err) } - res.Value = bz default: return sdkerrors.QueryResult(errors.Wrapf(sdkerrors.ErrUnknownRequest, "unexpected query path: %v", req.Path)) diff --git a/sei-cosmos/storev2/commitment/store_test.go b/sei-cosmos/storev2/commitment/store_test.go index 8ba93e1766..12a3f84ef2 100644 --- a/sei-cosmos/storev2/commitment/store_test.go +++ b/sei-cosmos/storev2/commitment/store_test.go @@ -4,12 +4,52 @@ import ( "testing" "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/storev2/query" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" + abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/stretchr/testify/require" ) func TestLastCommitID(t *testing.T) { - tree := memiavl.New(100) - store := NewStore(tree) + tree := memiavl.New(0) + store := NewStore(tree, query.Limits{}) require.Equal(t, types.CommitID{Hash: tree.RootHash()}, store.LastCommitID()) } + +func TestQuerySubspace_EmptyPrefixRejected(t *testing.T) { + tree := memiavl.New(0) + store := NewStore(tree, query.Limits{}) + + resp := store.Query(t.Context(), abci.RequestQuery{Path: "/subspace"}) + require.NotEqualValues(t, 0, resp.Code) + require.Contains(t, resp.Log, "subspace prefix must not be empty") +} + +func TestQuerySubspace_NarrowPrefixSucceeds(t *testing.T) { + tree := memiavl.New(0) + tree.Set([]byte("ab1"), []byte("v1")) + tree.Set([]byte("ab2"), []byte("v2")) + tree.Set([]byte("xy1"), []byte("v3")) + store := NewStore(tree, query.Limits{MaxPairs: 10, MaxBytes: query.DefaultMaxSubspaceBytes}) + + resp := store.Query(t.Context(), abci.RequestQuery{ + Path: "/subspace", + Data: []byte("ab"), + }) + require.EqualValues(t, 0, resp.Code) + require.NotEmpty(t, resp.Value) +} + +func TestQueryKey_UnaffectedBySubspaceLimits(t *testing.T) { + tree := memiavl.New(0) + key := []byte("k") + tree.Set(key, []byte("v")) + store := NewStore(tree, query.Limits{MaxPairs: 1, MaxBytes: 1}) + + resp := store.Query(t.Context(), abci.RequestQuery{ + Path: "/key", + Data: key, + }) + require.EqualValues(t, 0, resp.Code) + require.Equal(t, []byte("v"), resp.Value) +} diff --git a/sei-cosmos/storev2/query/subspace.go b/sei-cosmos/storev2/query/subspace.go new file mode 100644 index 0000000000..dfc8963773 --- /dev/null +++ b/sei-cosmos/storev2/query/subspace.go @@ -0,0 +1,80 @@ +package query + +import ( + "context" + "fmt" + + storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/types/kv" + abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" +) + +const ( + DefaultMaxSubspacePairs = 1_000 + DefaultMaxSubspaceBytes = 4 * 1024 * 1024 // 4 MiB +) + +// Limits bounds how many pairs and bytes a /subspace scan may materialize. +type Limits struct { + MaxPairs int + MaxBytes int +} + +// effective returns limits with non-positive fields replaced by package defaults. +func (l Limits) effective() Limits { + if l.MaxPairs <= 0 { + l.MaxPairs = DefaultMaxSubspacePairs + } + if l.MaxBytes <= 0 { + l.MaxBytes = DefaultMaxSubspaceBytes + } + return l +} + +// ScanSubspace walks prefix in st and returns marshaled kv.Pairs. +// It stops with ErrSubspaceCapExceeded when either limit would be exceeded. +func ScanSubspace(ctx context.Context, st storetypes.KVStore, prefix []byte, limits Limits) ([]byte, error) { + limits = limits.effective() + + pairs := kv.Pairs{ + Pairs: make([]kv.Pair, 0), + } + totalBytes := 0 + + iterator := storetypes.KVStorePrefixIterator(st, prefix) + defer func() { _ = iterator.Close() }() + + for ; iterator.Valid(); iterator.Next() { + if err := ctx.Err(); err != nil { + return nil, err + } + + key := iterator.Key() + value := iterator.Value() + pairBytes := len(key) + len(value) + + if len(pairs.Pairs)+1 > limits.MaxPairs || totalBytes+pairBytes > limits.MaxBytes { + return nil, storetypes.ErrSubspaceCapExceeded.Wrap("use a narrower prefix") + } + + pairs.Pairs = append(pairs.Pairs, kv.Pair{Key: key, Value: value}) + totalBytes += pairBytes + } + + bz, err := pairs.Marshal() + if err != nil { + panic(fmt.Errorf("failed to marshal KV pairs: %w", err)) + } + return bz, nil +} + +// IsCapExceeded reports whether err is a subspace scan cap rejection. +func IsCapExceeded(err error) bool { + return storetypes.ErrSubspaceCapExceeded.Is(err) +} + +// IsCapExceededResponse reports whether res is a subspace scan cap rejection. +func IsCapExceededResponse(res abci.ResponseQuery) bool { + return res.Codespace == storetypes.StoreCodespace && + res.Code == storetypes.ErrSubspaceCapExceeded.ABCICode() +} diff --git a/sei-cosmos/storev2/query/subspace_test.go b/sei-cosmos/storev2/query/subspace_test.go new file mode 100644 index 0000000000..2edf4c6397 --- /dev/null +++ b/sei-cosmos/storev2/query/subspace_test.go @@ -0,0 +1,87 @@ +package query + +import ( + "context" + "testing" + "time" + + "github.com/sei-protocol/sei-chain/sei-cosmos/store/mem" + storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" + sdkerrors "github.com/sei-protocol/sei-chain/sei-cosmos/types/errors" + "github.com/sei-protocol/sei-chain/sei-cosmos/types/kv" + "github.com/stretchr/testify/require" +) + +func seedStore(t *testing.T, n int, valueSize int) *mem.Store { + t.Helper() + store := mem.NewStore() + val := make([]byte, valueSize) + prefix := []byte("p") + for i := range n { + key := append(append([]byte{}, prefix...), byte(i)) + store.Set(key, val) + } + return store +} + +func TestScanSubspace_CapAtPairs(t *testing.T) { + store := seedStore(t, 5, 1) + + _, err := ScanSubspace(t.Context(), store, []byte("p"), Limits{MaxPairs: 3, MaxBytes: DefaultMaxSubspaceBytes}) + require.Error(t, err) + require.True(t, storetypes.ErrSubspaceCapExceeded.Is(err)) +} + +func TestScanSubspace_CapAtBytes(t *testing.T) { + store := seedStore(t, 3, 10) + + _, err := ScanSubspace(t.Context(), store, []byte("p"), Limits{MaxPairs: 100, MaxBytes: 15}) + require.Error(t, err) + require.True(t, storetypes.ErrSubspaceCapExceeded.Is(err)) +} + +func TestScanSubspace_NarrowPrefixSucceeds(t *testing.T) { + store := mem.NewStore() + store.Set([]byte("abc"), []byte("v1")) + store.Set([]byte("abd"), []byte("v2")) + store.Set([]byte("xyz"), []byte("v3")) + + bz, err := ScanSubspace(t.Context(), store, []byte("ab"), Limits{MaxPairs: 10, MaxBytes: DefaultMaxSubspaceBytes}) + require.NoError(t, err) + + var pairs kv.Pairs + require.NoError(t, pairs.Unmarshal(bz)) + require.Len(t, pairs.Pairs, 2) +} + +func TestScanSubspace_ContextCancelStopsAllocation(t *testing.T) { + store := seedStore(t, 100, 64) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + _, err := ScanSubspace(ctx, store, []byte("p"), Limits{MaxPairs: 1000, MaxBytes: DefaultMaxSubspaceBytes}) + require.Error(t, err) + require.ErrorIs(t, err, context.Canceled) + require.False(t, storetypes.ErrSubspaceCapExceeded.Is(err)) +} + +func TestScanSubspace_ContextDeadline(t *testing.T) { + store := seedStore(t, 100, 64) + + ctx, cancel := context.WithTimeout(t.Context(), time.Nanosecond) + defer cancel() + time.Sleep(time.Millisecond) + + _, err := ScanSubspace(ctx, store, []byte("p"), Limits{MaxPairs: 1000, MaxBytes: DefaultMaxSubspaceBytes}) + require.Error(t, err) + require.ErrorIs(t, err, context.DeadlineExceeded) +} + +func TestIsCapExceededResponse(t *testing.T) { + _, err := ScanSubspace(t.Context(), seedStore(t, 5, 1), []byte("p"), Limits{MaxPairs: 1, MaxBytes: DefaultMaxSubspaceBytes}) + require.True(t, IsCapExceeded(err)) + + res := sdkerrors.QueryResult(err) + require.True(t, IsCapExceededResponse(res)) +} diff --git a/sei-cosmos/storev2/rootmulti/metrics.go b/sei-cosmos/storev2/rootmulti/metrics.go index 68d429351c..0bb73a8512 100644 --- a/sei-cosmos/storev2/rootmulti/metrics.go +++ b/sei-cosmos/storev2/rootmulti/metrics.go @@ -21,6 +21,7 @@ var ( iavlTotalValueBytes metric.Int64Gauge iavlTotalNumKeys metric.Int64Gauge stateSyncKeysExported metric.Int64Counter + subspaceQueryRejected metric.Int64Counter }{ scCommitLatency: must(meter.Float64Histogram( "sc_commit_latency", @@ -58,6 +59,11 @@ var ( metric.WithDescription("Number of keys exported during state sync"), metric.WithUnit("{count}"), )), + subspaceQueryRejected: must(meter.Int64Counter( + "subspace_query_rejected", + metric.WithDescription("Number of rejected /subspace ABCI queries by reason"), + metric.WithUnit("{count}"), + )), } ) diff --git a/sei-cosmos/storev2/rootmulti/store.go b/sei-cosmos/storev2/rootmulti/store.go index fdda5b7afc..3694667f50 100644 --- a/sei-cosmos/storev2/rootmulti/store.go +++ b/sei-cosmos/storev2/rootmulti/store.go @@ -26,6 +26,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-cosmos/store/transient" "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" "github.com/sei-protocol/sei-chain/sei-cosmos/storev2/commitment" + "github.com/sei-protocol/sei-chain/sei-cosmos/storev2/query" "github.com/sei-protocol/sei-chain/sei-cosmos/storev2/state" "github.com/sei-protocol/sei-chain/sei-cosmos/telemetry" sdkerrors "github.com/sei-protocol/sei-chain/sei-cosmos/types/errors" @@ -76,6 +77,9 @@ type Store struct { histProofSem chan struct{} histProofLimiter *rate.Limiter + subspaceQuerySem chan struct{} + subspaceLimits query.Limits + snapshotSCStoreWarnOnce sync.Once // Hash logger state (per-block hash logging; a debugging/forensics tool). See hashlog.go. @@ -117,6 +121,11 @@ func NewStore( maxInFlight = 1 } + subspaceMaxInFlight := scConfig.SubspaceQueryMaxInFlight + if subspaceMaxInFlight <= 0 { + subspaceMaxInFlight = config.DefaultSCSubspaceQueryMaxInFlight + } + burst := scConfig.HistoricalProofBurst if burst <= 0 { burst = 1 @@ -139,13 +148,18 @@ func NewStore( } } store := &Store{ - scStore: scStore, - storesParams: make(map[types.StoreKey]storeParams), - storeKeys: make(map[string]types.StoreKey), - ckvStores: make(map[types.StoreKey]types.CommitKVStore), - gigaKeys: gigaKeys, - histProofSem: make(chan struct{}, maxInFlight), - histProofLimiter: limiter, + scStore: scStore, + storesParams: make(map[types.StoreKey]storeParams), + storeKeys: make(map[string]types.StoreKey), + ckvStores: make(map[types.StoreKey]types.CommitKVStore), + gigaKeys: gigaKeys, + histProofSem: make(chan struct{}, maxInFlight), + histProofLimiter: limiter, + subspaceQuerySem: make(chan struct{}, subspaceMaxInFlight), + subspaceLimits: query.Limits{ + MaxPairs: scConfig.SubspaceMaxPairs, + MaxBytes: scConfig.SubspaceMaxBytes, + }, hashLoggerConfig: scConfig.HashLogger, hashLoggerDisabled: !scConfig.HashLogger.Enable, scDir: scDir, @@ -388,7 +402,7 @@ func (rs *Store) CacheMultiStoreWithVersion(version int64) (types.CacheMultiStor if store.GetStoreType() != types.StoreTypeIAVL { stores[k] = store } else { - stores[k] = state.NewStore(rs.ssStore, k, version) + stores[k] = state.NewStore(rs.ssStore, k, version, rs.subspaceLimits) } } } else if version <= 0 || (rs.lastCommitInfo != nil && version == rs.lastCommitInfo.Version) { @@ -431,7 +445,7 @@ func (rs *Store) CacheMultiStoreForExport(version int64) (types.CacheMultiStore, for k, store := range rs.ckvStores { if store.GetStoreType() == types.StoreTypeIAVL { tree := scStore.GetChildStoreByName(k.Name()) - stores[k] = commitment.NewStore(tree) + stores[k] = commitment.NewStore(tree, rs.subspaceLimits) } } rs.mtx.RUnlock() @@ -478,7 +492,7 @@ func (rs *Store) CacheMultiStoreFromCommitter(snap sctypes.Committer) (types.Cac if tree == nil { return nil, fmt.Errorf("snapshot missing child store %q", k.Name()) } - stores[k] = commitment.NewStore(tree) + stores[k] = commitment.NewStore(tree, rs.subspaceLimits) } return cachemulti.NewStore(nil, stores, rs.storeKeys, nil, nil, nil), nil } @@ -698,7 +712,7 @@ func (rs *Store) loadCommitStoreFromParams(key types.StoreKey, params storeParam if tree == nil { return nil, fmt.Errorf("new store is not added in upgrades: %s", key.Name()) } - return types.CommitKVStore(commitment.NewStore(tree)), nil + return types.CommitKVStore(commitment.NewStore(tree, rs.subspaceLimits)), nil case types.StoreTypeDB: panic("recursive MultiStores not yet supported") case types.StoreTypeTransient: @@ -774,7 +788,7 @@ func (rs *Store) SetMigrationBatchSize(batchSize int) error { if !ok || mode != sctypes.MemiavlOnly { return nil } - // Effective mode is memiavl_only. Only an auto store may be advanced to + // effective mode is memiavl_only. Only an auto store may be advanced to // migrate_evm at runtime; a node pinned to fixed memiavl_only must not. configured, hasConfigured := rs.ConfiguredWriteMode() if !hasConfigured { @@ -947,8 +961,17 @@ func (rs *Store) Query(ctx context.Context, req abci.RequestQuery) abci.Response if err := rs.validateSSReadVersion(version); err != nil { return sdkerrors.QueryResult(errors.Wrap(sdkerrors.ErrInvalidHeight, err.Error())) } - store := types.Queryable(state.NewStore(rs.ssStore, types.NewKVStoreKey(storeName), version)) - return store.Query(ctx, req) + if req.Path == "/subspace" { + if err := rs.tryAcquireSubspaceQueryPermit(); err != nil { + storev2Metrics.subspaceQueryRejected.Add(ctx, 1, otelmetric.WithAttributes( + attribute.String("reason", "semaphore"), + )) + return sdkerrors.QueryResult(err) + } + defer rs.releaseSubspaceQueryPermit() + } + store := types.Queryable(state.NewStore(rs.ssStore, types.NewKVStoreKey(storeName), version, rs.subspaceLimits)) + return rs.finishSubspaceQuery(ctx, req, store.Query(ctx, req)) } var ( @@ -957,7 +980,7 @@ func (rs *Store) Query(ctx context.Context, req abci.RequestQuery) abci.Response ) if latest { // latest never needs historical LoadVersion clone - store = types.Queryable(commitment.NewStore(rs.scStore.GetChildStoreByName(storeName))) + store = types.Queryable(commitment.NewStore(rs.scStore.GetChildStoreByName(storeName), rs.subspaceLimits)) commitInfo = convertCommitInfo(rs.scStore.LastCommitInfo()) commitInfo = amendCommitInfo(commitInfo, rs.storesParams) } else { @@ -997,12 +1020,12 @@ func (rs *Store) Query(ctx context.Context, req abci.RequestQuery) abci.Response } defer func() { _ = scStore.Close() }() - store = types.Queryable(commitment.NewStore(scStore.GetChildStoreByName(storeName))) + store = types.Queryable(commitment.NewStore(scStore.GetChildStoreByName(storeName), rs.subspaceLimits)) commitInfo = convertCommitInfo(scStore.LastCommitInfo()) commitInfo = amendCommitInfo(commitInfo, rs.storesParams) } - res := store.Query(ctx, req) + res := rs.finishSubspaceQuery(ctx, req, store.Query(ctx, req)) // If underlying query failed (e.g. invalid height/path) or doesn' need proof, return as-is. if res.Code != 0 || !needProof { @@ -1024,6 +1047,31 @@ func (rs *Store) Query(ctx context.Context, req abci.RequestQuery) abci.Response return res } +func (rs *Store) tryAcquireSubspaceQueryPermit() error { + select { + case rs.subspaceQuerySem <- struct{}{}: + return nil + default: + return errors.Wrap(sdkerrors.ErrConflict, "subspace query busy") + } +} + +func (rs *Store) releaseSubspaceQueryPermit() { + select { + case <-rs.subspaceQuerySem: + default: + } +} + +func (rs *Store) finishSubspaceQuery(ctx context.Context, req abci.RequestQuery, res abci.ResponseQuery) abci.ResponseQuery { + if req.Path == "/subspace" && res.Code != 0 && query.IsCapExceededResponse(res) { + storev2Metrics.subspaceQueryRejected.Add(ctx, 1, otelmetric.WithAttributes( + attribute.String("reason", "cap_exceeded"), + )) + } + return res +} + func (rs *Store) tryAcquireHistProofPermit() error { if rs.histProofLimiter != nil && !rs.histProofLimiter.Allow() { return errors.Wrap(sdkerrors.ErrConflict, "historical proof rate limited") diff --git a/sei-cosmos/storev2/rootmulti/store_test.go b/sei-cosmos/storev2/rootmulti/store_test.go index e026add773..5256c99cb7 100644 --- a/sei-cosmos/storev2/rootmulti/store_test.go +++ b/sei-cosmos/storev2/rootmulti/store_test.go @@ -497,3 +497,116 @@ func TestQuery_LatestProofBypassesHistoricalPermit(t *testing.T) { require.EqualValues(t, 0, resp.Code) require.Equal(t, valV1, resp.Value) } + +func TestTryAcquireSubspaceQueryPermit(t *testing.T) { + store := &Store{ + subspaceQuerySem: make(chan struct{}, 2), + } + + require.NoError(t, store.tryAcquireSubspaceQueryPermit()) + require.NoError(t, store.tryAcquireSubspaceQueryPermit()) + + err := store.tryAcquireSubspaceQueryPermit() + require.Error(t, err) + require.Contains(t, err.Error(), "subspace query busy") + + store.releaseSubspaceQueryPermit() + store.releaseSubspaceQueryPermit() + store.releaseSubspaceQueryPermit() // no-op when empty + require.NoError(t, store.tryAcquireSubspaceQueryPermit()) +} + +func TestQuery_SubspaceSemaphoreRejectsWhenSaturated(t *testing.T) { + home := t.TempDir() + scCfg := config.DefaultStateCommitConfig() + scCfg.Enable = true + scCfg.MemIAVLConfig.AsyncCommitBuffer = 0 + scCfg.SubspaceQueryMaxInFlight = 2 + + ssCfg := config.DefaultStateStoreConfig() + ssCfg.Enable = true + + store := NewStore(home, scCfg, ssCfg, []string{}) + defer func() { _ = store.Close() }() + + key := types.NewKVStoreKey("bank") + store.MountStoreWithDB(key, types.StoreTypeIAVL, nil) + require.NoError(t, store.LoadLatestVersion()) + + kv := store.GetStoreByName("bank").(types.KVStore) + kv.Set([]byte("ab1"), []byte("v1")) + require.Equal(t, int64(1), store.Commit(true).Version) + waitUntilSSVersion(t, store, 1) + + store.subspaceQuerySem <- struct{}{} + store.subspaceQuerySem <- struct{}{} + defer func() { + <-store.subspaceQuerySem + <-store.subspaceQuerySem + }() + + resp := store.Query(t.Context(), abci.RequestQuery{ + Path: "/bank/subspace", + Data: []byte("ab"), + }) + require.NotEqualValues(t, 0, resp.Code) + require.Contains(t, resp.Log, "subspace query busy") +} + +func TestQuery_SubspaceNarrowPrefixAndKeyUnaffected(t *testing.T) { + home := t.TempDir() + scCfg := config.DefaultStateCommitConfig() + scCfg.Enable = true + scCfg.MemIAVLConfig.AsyncCommitBuffer = 0 + + ssCfg := config.DefaultStateStoreConfig() + ssCfg.Enable = true + + store := NewStore(home, scCfg, ssCfg, []string{}) + defer func() { _ = store.Close() }() + + key := types.NewKVStoreKey("bank") + store.MountStoreWithDB(key, types.StoreTypeIAVL, nil) + require.NoError(t, store.LoadLatestVersion()) + + keyBytes := []byte("ab1") + kv := store.GetStoreByName("bank").(types.KVStore) + kv.Set(keyBytes, []byte("v1")) + kv.Set([]byte("ab2"), []byte("v2")) + require.Equal(t, int64(1), store.Commit(true).Version) + waitUntilSSVersion(t, store, 1) + + subspaceResp := store.Query(t.Context(), abci.RequestQuery{ + Path: "/bank/subspace", + Data: []byte("ab"), + }) + require.EqualValues(t, 0, subspaceResp.Code) + require.NotEmpty(t, subspaceResp.Value) + + keyResp := store.Query(t.Context(), abci.RequestQuery{ + Path: "/bank/key", + Data: keyBytes, + }) + require.EqualValues(t, 0, keyResp.Code) + require.Equal(t, []byte("v1"), keyResp.Value) +} + +func TestQuery_SubspaceEmptyPrefixRejected(t *testing.T) { + home := t.TempDir() + scCfg := config.DefaultStateCommitConfig() + ssCfg := config.DefaultStateStoreConfig() + ssCfg.Enable = true + + store := NewStore(home, scCfg, ssCfg, []string{}) + defer func() { _ = store.Close() }() + + key := types.NewKVStoreKey("bank") + store.MountStoreWithDB(key, types.StoreTypeIAVL, nil) + require.NoError(t, store.LoadLatestVersion()) + + resp := store.Query(t.Context(), abci.RequestQuery{ + Path: "/bank/subspace", + }) + require.NotEqualValues(t, 0, resp.Code) + require.Contains(t, resp.Log, "subspace prefix must not be empty") +} diff --git a/sei-cosmos/storev2/state/store.go b/sei-cosmos/storev2/state/store.go index 0a73d94b34..9292106c41 100644 --- a/sei-cosmos/storev2/state/store.go +++ b/sei-cosmos/storev2/state/store.go @@ -2,7 +2,6 @@ package state import ( "context" - "fmt" "io" "cosmossdk.io/errors" @@ -10,8 +9,8 @@ import ( "github.com/sei-protocol/sei-chain/sei-cosmos/store/cachekv" "github.com/sei-protocol/sei-chain/sei-cosmos/store/tracekv" "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/storev2/query" sdkerrors "github.com/sei-protocol/sei-chain/sei-cosmos/types/errors" - "github.com/sei-protocol/sei-chain/sei-cosmos/types/kv" seidbtypes "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" ) @@ -26,13 +25,14 @@ var ( // Store wraps a SS store and implements a cosmos KVStore type Store struct { - store seidbtypes.StateStore - storeKey types.StoreKey - version int64 + store seidbtypes.StateStore + storeKey types.StoreKey + version int64 + subspaceLimits query.Limits } -func NewStore(store seidbtypes.StateStore, storeKey types.StoreKey, version int64) *Store { - return &Store{store, storeKey, version} +func NewStore(store seidbtypes.StateStore, storeKey types.StoreKey, version int64, subspaceLimits query.Limits) *Store { + return &Store{store, storeKey, version, subspaceLimits} } func (st *Store) GetStoreType() types.StoreType { @@ -99,7 +99,7 @@ func (st *Store) GetWorkingHash() ([]byte, error) { panic("get working hash operation is not supported") } -func (st *Store) Query(_ context.Context, req abci.RequestQuery) (res abci.ResponseQuery) { +func (st *Store) Query(ctx context.Context, req abci.RequestQuery) (res abci.ResponseQuery) { if req.Height > 0 && req.Height > st.version { return sdkerrors.QueryResult(errors.Wrap(sdkerrors.ErrInvalidHeight, "invalid height")) } @@ -109,20 +109,13 @@ func (st *Store) Query(_ context.Context, req abci.RequestQuery) (res abci.Respo res.Key = req.Data // data holds the key bytes res.Value = st.Get(res.Key) case "/subspace": - pairs := kv.Pairs{ - Pairs: make([]kv.Pair, 0), + if len(req.Data) == 0 { + return sdkerrors.QueryResult(errors.Wrap(sdkerrors.ErrInvalidRequest, "subspace prefix must not be empty")) } - subspace := req.Data - res.Key = subspace - iterator := types.KVStorePrefixIterator(st, subspace) - for ; iterator.Valid(); iterator.Next() { - pairs.Pairs = append(pairs.Pairs, kv.Pair{Key: iterator.Key(), Value: iterator.Value()}) - } - _ = iterator.Close() - - bz, err := pairs.Marshal() + res.Key = req.Data + bz, err := query.ScanSubspace(ctx, st, req.Data, st.subspaceLimits) if err != nil { - panic(fmt.Errorf("failed to marshal KV pairs: %w", err)) + return sdkerrors.QueryResult(err) } res.Value = bz default: diff --git a/sei-db/config/sc_config.go b/sei-db/config/sc_config.go index 6b3bda0a54..48f81bf2f8 100644 --- a/sei-db/config/sc_config.go +++ b/sei-db/config/sc_config.go @@ -14,6 +14,10 @@ const ( DefaultSCHistoricalProofRateLimit = 1.0 // req/s, <=0 disables rate limit DefaultSCHistoricalProofBurst = 1 + DefaultSCSubspaceQueryMaxInFlight = 2 + DefaultSCSubspaceMaxPairs = 1_000 + DefaultSCSubspaceMaxBytes = 4 * 1024 * 1024 // 4 MiB + legacySCWriteModeCosmosOnly = "cosmos_only" ) @@ -102,6 +106,15 @@ type StateCommitConfig struct { // Token bucket burst for historical proof queries. HistoricalProofBurst int `mapstructure:"historical-proof-burst"` + // Max concurrent unproven /subspace queries on the SS fast path. + SubspaceQueryMaxInFlight int `mapstructure:"subspace-query-max-inflight"` + + // Max key/value pairs a /subspace scan may return. + SubspaceMaxPairs int `mapstructure:"subspace-max-pairs"` + + // Max key+value bytes a /subspace scan may accumulate. + SubspaceMaxBytes int `mapstructure:"subspace-max-bytes"` + // HashLogger configures the per-block hash logger (a debugging/forensics tool). Enabled by default. // Loaded via explicit sc-hash-logger-* flag reads in app.parseSCConfigs, not mapstructure. HashLogger HashLoggerConfig @@ -118,6 +131,9 @@ func DefaultStateCommitConfig() StateCommitConfig { HistoricalProofMaxInFlight: DefaultSCHistoricalProofMaxInFlight, HistoricalProofRateLimit: DefaultSCHistoricalProofRateLimit, HistoricalProofBurst: DefaultSCHistoricalProofBurst, + SubspaceQueryMaxInFlight: DefaultSCSubspaceQueryMaxInFlight, + SubspaceMaxPairs: DefaultSCSubspaceMaxPairs, + SubspaceMaxBytes: DefaultSCSubspaceMaxBytes, HashLogger: DefaultHashLoggerConfig(), } } diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index 34e0d8b42b..9911fec5d6 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -22,6 +22,15 @@ sc-historical-proof-rate-limit = {{ .StateCommit.HistoricalProofRateLimit }} # Historical proof query burst size sc-historical-proof-burst = {{ .StateCommit.HistoricalProofBurst }} +# Max concurrent unproven /subspace queries (SS fast path) +sc-subspace-query-max-inflight = {{ .StateCommit.SubspaceQueryMaxInFlight }} + +# Max pairs a /subspace scan may return +sc-subspace-max-pairs = {{ .StateCommit.SubspaceMaxPairs }} + +# Max key+value bytes a /subspace scan may accumulate (4 MiB default) +sc-subspace-max-bytes = {{ .StateCommit.SubspaceMaxBytes }} + # AsyncCommitBuffer defines the size of asynchronous commit queue, this greatly improve block catching-up # performance, setting to 0 means synchronous commit. sc-async-commit-buffer = {{ .StateCommit.MemIAVLConfig.AsyncCommitBuffer }} diff --git a/sei-db/config/toml_test.go b/sei-db/config/toml_test.go index a07fbb05fd..7c2231f84d 100644 --- a/sei-db/config/toml_test.go +++ b/sei-db/config/toml_test.go @@ -49,6 +49,9 @@ func TestStateCommitConfigTemplate(t *testing.T) { require.Contains(t, output, "sc-historical-proof-max-inflight = 1", "Missing or incorrect sc-historical-proof-max-inflight") require.Contains(t, output, "sc-historical-proof-rate-limit = 1", "Missing or incorrect sc-historical-proof-rate-limit") require.Contains(t, output, "sc-historical-proof-burst = 1", "Missing or incorrect sc-historical-proof-burst") + require.Contains(t, output, "sc-subspace-query-max-inflight = 2", "Missing or incorrect sc-subspace-query-max-inflight") + require.Contains(t, output, "sc-subspace-max-pairs = 1000", "Missing or incorrect sc-subspace-max-pairs") + require.Contains(t, output, "sc-subspace-max-bytes = 4194304", "Missing or incorrect sc-subspace-max-bytes") // The FlatKV section header is kept, but no FlatKV configs are exposed yet. require.Contains(t, output, "[state-commit.flatkv]", "Missing FlatKV section") From 7e000bee5b0d43c6cb1a837492200fc597389c6f Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Wed, 26 Aug 2026 14:48:15 +0200 Subject: [PATCH 2/5] Address AI review feedback on /subspace query hardening. Thread request context through IteratorOn on the SS path, centralize the empty-prefix guard in ScanSubspace, and document <=0 default semantics for subspace limits in app.toml. Co-authored-by: Cursor --- sei-cosmos/storev2/commitment/store.go | 3 --- sei-cosmos/storev2/query/subspace.go | 8 +++++++- sei-cosmos/storev2/state/store.go | 3 --- sei-db/config/toml.go | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sei-cosmos/storev2/commitment/store.go b/sei-cosmos/storev2/commitment/store.go index af92c3f5de..a4b43bcfa1 100644 --- a/sei-cosmos/storev2/commitment/store.go +++ b/sei-cosmos/storev2/commitment/store.go @@ -153,9 +153,6 @@ func (st *Store) Query(ctx context.Context, req abci.RequestQuery) (res abci.Res op := types.NewIavlCommitmentOp(res.Key, commitmentProof) res.ProofOps = &crypto.ProofOps{Ops: []crypto.ProofOp{op.ProofOp()}} case "/subspace": - if len(req.Data) == 0 { - return sdkerrors.QueryResult(errors.Wrap(sdkerrors.ErrInvalidRequest, "subspace prefix must not be empty")) - } res.Key = req.Data bz, err := query.ScanSubspace(ctx, st, req.Data, st.subspaceLimits) if err != nil { diff --git a/sei-cosmos/storev2/query/subspace.go b/sei-cosmos/storev2/query/subspace.go index dfc8963773..437f1ae6cb 100644 --- a/sei-cosmos/storev2/query/subspace.go +++ b/sei-cosmos/storev2/query/subspace.go @@ -4,7 +4,9 @@ import ( "context" "fmt" + "cosmossdk.io/errors" storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" + sdkerrors "github.com/sei-protocol/sei-chain/sei-cosmos/types/errors" "github.com/sei-protocol/sei-chain/sei-cosmos/types/kv" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" ) @@ -34,6 +36,10 @@ func (l Limits) effective() Limits { // ScanSubspace walks prefix in st and returns marshaled kv.Pairs. // It stops with ErrSubspaceCapExceeded when either limit would be exceeded. func ScanSubspace(ctx context.Context, st storetypes.KVStore, prefix []byte, limits Limits) ([]byte, error) { + if len(prefix) == 0 { + return nil, errors.Wrap(sdkerrors.ErrInvalidRequest, "subspace prefix must not be empty") + } + limits = limits.effective() pairs := kv.Pairs{ @@ -41,7 +47,7 @@ func ScanSubspace(ctx context.Context, st storetypes.KVStore, prefix []byte, lim } totalBytes := 0 - iterator := storetypes.KVStorePrefixIterator(st, prefix) + iterator := storetypes.IteratorOn(st, ctx, prefix, storetypes.PrefixEndBytes(prefix), true) defer func() { _ = iterator.Close() }() for ; iterator.Valid(); iterator.Next() { diff --git a/sei-cosmos/storev2/state/store.go b/sei-cosmos/storev2/state/store.go index 9292106c41..7bf4c9e40a 100644 --- a/sei-cosmos/storev2/state/store.go +++ b/sei-cosmos/storev2/state/store.go @@ -109,9 +109,6 @@ func (st *Store) Query(ctx context.Context, req abci.RequestQuery) (res abci.Res res.Key = req.Data // data holds the key bytes res.Value = st.Get(res.Key) case "/subspace": - if len(req.Data) == 0 { - return sdkerrors.QueryResult(errors.Wrap(sdkerrors.ErrInvalidRequest, "subspace prefix must not be empty")) - } res.Key = req.Data bz, err := query.ScanSubspace(ctx, st, req.Data, st.subspaceLimits) if err != nil { diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index 9911fec5d6..e3894f44ea 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -22,13 +22,13 @@ sc-historical-proof-rate-limit = {{ .StateCommit.HistoricalProofRateLimit }} # Historical proof query burst size sc-historical-proof-burst = {{ .StateCommit.HistoricalProofBurst }} -# Max concurrent unproven /subspace queries (SS fast path) +# Max concurrent unproven /subspace queries (SS fast path; <=0 resolves to default 2) sc-subspace-query-max-inflight = {{ .StateCommit.SubspaceQueryMaxInFlight }} -# Max pairs a /subspace scan may return +# Max pairs a /subspace scan may return (<=0 resolves to default 1000; no unlimited setting) sc-subspace-max-pairs = {{ .StateCommit.SubspaceMaxPairs }} -# Max key+value bytes a /subspace scan may accumulate (4 MiB default) +# Max key+value bytes a /subspace scan may accumulate (<=0 resolves to default 4 MiB; no unlimited setting) sc-subspace-max-bytes = {{ .StateCommit.SubspaceMaxBytes }} # AsyncCommitBuffer defines the size of asynchronous commit queue, this greatly improve block catching-up From ec316c873455582fe141619430ef844476887d57 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Wed, 26 Aug 2026 17:51:57 +0200 Subject: [PATCH 3/5] Return context errors before opening SS subspace iterators. Cancelled or timed-out /subspace queries were panicking on the SS path because state.Store fatally handles iterator-construction failures. Co-authored-by: Cursor --- sei-cosmos/storev2/query/subspace.go | 3 +++ sei-cosmos/storev2/query/subspace_test.go | 31 +++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/sei-cosmos/storev2/query/subspace.go b/sei-cosmos/storev2/query/subspace.go index 437f1ae6cb..d194846e7a 100644 --- a/sei-cosmos/storev2/query/subspace.go +++ b/sei-cosmos/storev2/query/subspace.go @@ -39,6 +39,9 @@ func ScanSubspace(ctx context.Context, st storetypes.KVStore, prefix []byte, lim if len(prefix) == 0 { return nil, errors.Wrap(sdkerrors.ErrInvalidRequest, "subspace prefix must not be empty") } + if err := ctx.Err(); err != nil { + return nil, err + } limits = limits.effective() diff --git a/sei-cosmos/storev2/query/subspace_test.go b/sei-cosmos/storev2/query/subspace_test.go index 2edf4c6397..7176cdfd02 100644 --- a/sei-cosmos/storev2/query/subspace_test.go +++ b/sei-cosmos/storev2/query/subspace_test.go @@ -54,6 +54,37 @@ func TestScanSubspace_NarrowPrefixSucceeds(t *testing.T) { require.Len(t, pairs.Pairs, 2) } +// panicOnCancelledCtxStore implements ContextIterator the way state.Store does: +// iterator construction on an already-cancelled context is fatal. +type panicOnCancelledCtxStore struct { + *mem.Store +} + +func (s *panicOnCancelledCtxStore) IteratorWithContext(ctx context.Context, start, end []byte) storetypes.Iterator { + if err := ctx.Err(); err != nil { + panic(err) + } + return s.Store.Iterator(start, end) +} + +func (s *panicOnCancelledCtxStore) ReverseIteratorWithContext(ctx context.Context, start, end []byte) storetypes.Iterator { + if err := ctx.Err(); err != nil { + panic(err) + } + return s.Store.ReverseIterator(start, end) +} + +func TestScanSubspace_ContextCancelBeforeIteratorOpen(t *testing.T) { + store := &panicOnCancelledCtxStore{Store: seedStore(t, 1, 1)} + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + _, err := ScanSubspace(ctx, store, []byte("p"), Limits{MaxPairs: 1000, MaxBytes: DefaultMaxSubspaceBytes}) + require.Error(t, err) + require.ErrorIs(t, err, context.Canceled) +} + func TestScanSubspace_ContextCancelStopsAllocation(t *testing.T) { store := seedStore(t, 100, 64) From 16f8d4e7fbf972b94f5f9d44fdeb3accc4a912dc Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Wed, 26 Aug 2026 17:56:02 +0200 Subject: [PATCH 4/5] Updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9db8140f7..653fc8a38f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [#3818](https://github.com/sei-protocol/sei-chain/pull/3818) feat(evmrpc): extend HTTP admission control (`max_request_body_bytes`, `max_concurrent_request_bytes`, `ws_admission_timeout`) to the WebSocket plane (:8546). WS oversize frames close with WebSocket close code 1009; budget-wait timeouts return JSON-RPC error `-32005` before the connection closes. `evmrpc_requests_rejected_total` gains a `protocol` label (`http` / `ws`). * [#3984](https://github.com/sei-protocol/sei-chain/pull/3984) feat(query): origin-aware pagination limits for ABCI queries. Untrusted callers on the ABCI/gRPC query path get configurable `max-limit`, `max-offset`, and flat `max-iterations` (defaults: 1000 / 10000 / 11000); requests above the caps are rejected upfront, and an exhausted iteration budget returns a partial page with `next_key` instead of failing. Trusted origins (new `[query] trusted-cidrs`) and the `[query] disable-limits` kill switch bypass the caps; the consensus/EVM precompile path is unaffected. * [#3990](https://github.com/sei-protocol/sei-chain/pull/3990) Freeze mode is limited to full nodes and disables transaction and evidence submission, mempool gossip, and state sync from startup while preserving query RPC and mempool-backed reads. Frozen and Autobahn nodes no longer advertise the unused mempool P2P channel. +* [#4009](https://github.com/sei-protocol/sei-chain/pull/4009) Bound `/store/*/subspace` ABCI queries with pair/byte caps, empty-prefix rejection, SS-path concurrency limits, and context-aware iteration to prevent memory-exhaustion DoS. ### Upgrade guide * **IBC core removal.** Removes the retired IBC core source, protobufs, light clients, CLI, and simulation support. Retired IBC stores remain mounted but are omitted from `export-genesis`; preserve the state database or use v6.6 freeze nodes for historical IBC data. @@ -43,6 +44,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [#3958](https://github.com/sei-protocol/sei-chain/pull/3958) **Feegrant removal.** Removes feegrant execution, module APIs, and the unreleased feegrant EVM precompile. The feegrant store remains mounted for historical state access. Transactions with a fee granter different from the payer are rejected. * **WebSocket frame size default drops from 10 MiB to 5 MiB.** Before this release, :8546 used a hardcoded 10 MiB frame cap. Both HTTP and WebSocket now share `[evm].max_request_body_bytes`, whose default is 5 MiB (`5242880`). WS clients that send frames in the 5-10 MiB range (large `eth_sendRawTransaction` batches, wide filter payloads, etc.) will be disconnected after upgrade unless the limit is raised. **Operators who relied on the old 10 MiB WS cap should set `max_request_body_bytes = 10485760` in `app.toml` before upgrading.** This also raises the HTTP body limit to 10 MiB. The exported `DefaultWebsocketMaxMessageSize` constant was removed; use the config knob instead. * [#3984](https://github.com/sei-protocol/sei-chain/pull/3984) **ABCI/gRPC pagination is now capped by default.** Untrusted callers requesting `limit` above 1000, `offset` above 10000, or a scan that exceeds 11000 total iterations now get `InvalidArgument` (over-cap) or a partial page with `next_key` (budget exhausted) instead of the previously unbounded scan. Clients that page with large limits/offsets, or trusted internal indexers, should either follow `next_key` for resumption or be added to the new `[query] trusted-cidrs` allowlist (or set `[query] disable-limits = true`) before upgrading. +* [#4009](https://github.com/sei-protocol/sei-chain/pull/4009) **`/store/*/subspace` scans are now capped.** Wide prefix scans that previously returned unbounded KV pairs now fail with `subspace result exceeds limit` once they would exceed the default caps of 1,000 pairs or 4 MiB of accumulated key+value bytes. Empty prefixes are rejected. Indexers and tooling that issue wide `/subspace` queries must narrow their prefixes, shard by sub-prefix, or raise `[state-commit] sc-subspace-max-pairs` and `sc-subspace-max-bytes` before upgrading. Values `<= 0` resolve to these defaults; there is no unlimited setting. * [#3927](https://github.com/sei-protocol/sei-chain/pull/3927) **Legacy Sei JSON-RPC and CLI removal.** Removes `sei_associate`, `sei_getBlockByHash`, `sei_getBlockByHashExcludeTraceFail`, `sei_getBlockTransactionCountByHash`, `sei_getBlockTransactionCountByNumber`, `sei_getEvmTx`, `sei_getFilterChanges`, `sei_getFilterLogs`, `sei_getLogs`, `sei_getTransactionByBlockHashAndIndex`, `sei_getTransactionByBlockNumberAndIndex`, `sei_getTransactionByHash`, `sei_getTransactionCount`, `sei_getTransactionErrorByHash`, `sei_getTransactionReceiptExcludeTraceFail`, `sei_getVMError`, `sei_newBlockFilter`, `sei_newFilter`, `sei_sign`, and `sei_uninstallFilter`. Use standard `eth_*` methods for EVM-originated data and `seid tx evm native-associate -y` for address association. There is no block- or filter-level replacement for discovering Cosmos-originated synthetic logs; clients that know the synthetic transaction hash can enable `sei_getTransactionReceipt`. ## v6.6 From 4c4f7b6e698e10a0a2f8cc88e50ce1864c82441d Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Fri, 28 Aug 2026 12:55:30 +0200 Subject: [PATCH 5/5] Reject aborted subspace scans and complete subspace config wiring. An SS iterator that stops mid-prefix must surface its error instead of marshaling a truncated result as success. Register the three subspace keys in config characterization and document that the in-flight semaphore applies only on the SS fast path. Co-authored-by: Cursor --- app/config_register.go | 6 +++ app/config_register_agreement_test.go | 3 ++ app/config_register_test.go | 7 ++- .../config/testdata/server_config.golden | 3 ++ sei-cosmos/storev2/query/subspace.go | 15 +++++++ sei-cosmos/storev2/query/subspace_test.go | 43 +++++++++++++++++++ sei-cosmos/storev2/rootmulti/store.go | 3 ++ 7 files changed, 78 insertions(+), 2 deletions(-) diff --git a/app/config_register.go b/app/config_register.go index dcfab7a493..ed4bebc347 100644 --- a/app/config_register.go +++ b/app/config_register.go @@ -162,6 +162,9 @@ type stateCommitSchema struct { HistoricalProofMaxInFlight int `mapstructure:"sc-historical-proof-max-inflight"` HistoricalProofRateLimit float64 `mapstructure:"sc-historical-proof-rate-limit"` HistoricalProofBurst int `mapstructure:"sc-historical-proof-burst"` + SubspaceQueryMaxInFlight int `mapstructure:"sc-subspace-query-max-inflight"` + SubspaceMaxPairs int `mapstructure:"sc-subspace-max-pairs"` + SubspaceMaxBytes int `mapstructure:"sc-subspace-max-bytes"` WriteMode string `mapstructure:"sc-write-mode"` WriteModeEnableAuto bool `mapstructure:"sc-write-mode-enable-auto"` HashLoggerEnable bool `mapstructure:"sc-hash-logger-enable"` @@ -195,6 +198,9 @@ func stateCommitDefaults(registry.Mode) any { HistoricalProofMaxInFlight: live.HistoricalProofMaxInFlight, HistoricalProofRateLimit: live.HistoricalProofRateLimit, HistoricalProofBurst: live.HistoricalProofBurst, + SubspaceQueryMaxInFlight: live.SubspaceQueryMaxInFlight, + SubspaceMaxPairs: live.SubspaceMaxPairs, + SubspaceMaxBytes: live.SubspaceMaxBytes, WriteMode: string(live.WriteMode), WriteModeEnableAuto: live.WriteModeEnableAuto, HashLoggerEnable: live.HashLogger.Enable, diff --git a/app/config_register_agreement_test.go b/app/config_register_agreement_test.go index ca84b68182..ddedfb4fcf 100644 --- a/app/config_register_agreement_test.go +++ b/app/config_register_agreement_test.go @@ -87,6 +87,9 @@ func readerValues(t *testing.T) map[string]string { FlagSCHistoricalProofMaxInFlight: fmt.Sprint(sc.HistoricalProofMaxInFlight), FlagSCHistoricalProofRateLimit: fmt.Sprint(sc.HistoricalProofRateLimit), FlagSCHistoricalProofBurst: fmt.Sprint(sc.HistoricalProofBurst), + FlagSCSubspaceQueryMaxInFlight: fmt.Sprint(sc.SubspaceQueryMaxInFlight), + FlagSCSubspaceMaxPairs: fmt.Sprint(sc.SubspaceMaxPairs), + FlagSCSubspaceMaxBytes: fmt.Sprint(sc.SubspaceMaxBytes), FlagSCWriteMode: fmt.Sprint(sc.WriteMode), FlagSCWriteModeEnableAuto: fmt.Sprint(sc.WriteModeEnableAuto), FlagSCHashLoggerEnable: fmt.Sprint(sc.HashLogger.Enable), diff --git a/app/config_register_test.go b/app/config_register_test.go index fab0305984..7e9a1ba1da 100644 --- a/app/config_register_test.go +++ b/app/config_register_test.go @@ -160,8 +160,8 @@ func TestStateStoreResolvesItsOtherValuesTheSameForEveryMode(t *testing.T) { // TestStateCommitDeclaresEveryKeyItsReaderResolves holds the schema against the read-site record. // -// Twenty keys: the seventeen the record holds as rows, and three it names beside them because each has a -// target of its own. The four keys under this section's flat key-value name that only the Cosmos server's +// Twenty-three keys: the twenty the record holds as rows, and three it names beside them because each has +// a target of its own. The four keys under this section's flat key-value name that only the Cosmos server's // reader resolves are not among them, and are not this section's to declare. func TestStateCommitDeclaresEveryKeyItsReaderResolves(t *testing.T) { requireDeclares(t, StateCommitSectionName, manifestKeys(scKeys, @@ -189,6 +189,9 @@ func TestStateCommitResolvesTheModuleDeclaredValues(t *testing.T) { FlagSCHistoricalProofMaxInFlight: live.HistoricalProofMaxInFlight, FlagSCHistoricalProofRateLimit: live.HistoricalProofRateLimit, FlagSCHistoricalProofBurst: live.HistoricalProofBurst, + FlagSCSubspaceQueryMaxInFlight: live.SubspaceQueryMaxInFlight, + FlagSCSubspaceMaxPairs: live.SubspaceMaxPairs, + FlagSCSubspaceMaxBytes: live.SubspaceMaxBytes, FlagSCWriteMode: string(live.WriteMode), FlagSCWriteModeEnableAuto: live.WriteModeEnableAuto, FlagSCHashLoggerEnable: live.HashLogger.Enable, diff --git a/sei-cosmos/server/config/testdata/server_config.golden b/sei-cosmos/server/config/testdata/server_config.golden index e0ec8c8138..f8937a7812 100644 --- a/sei-cosmos/server/config/testdata/server_config.golden +++ b/sei-cosmos/server/config/testdata/server_config.golden @@ -121,6 +121,9 @@ StateCommit.FlatKVConfig.LtHashThreadsPerCore = float64(1) StateCommit.HistoricalProofMaxInFlight = int(1) StateCommit.HistoricalProofRateLimit = float64(1) StateCommit.HistoricalProofBurst = int(1) +StateCommit.SubspaceQueryMaxInFlight = int(2) +StateCommit.SubspaceMaxPairs = int(1000) +StateCommit.SubspaceMaxBytes = int(4194304) StateCommit.HashLogger.Enable = bool(true) StateCommit.HashLogger.Directory = string("") StateCommit.HashLogger.BlocksToRetain = uint(0) diff --git a/sei-cosmos/storev2/query/subspace.go b/sei-cosmos/storev2/query/subspace.go index d194846e7a..57e87f9ada 100644 --- a/sei-cosmos/storev2/query/subspace.go +++ b/sei-cosmos/storev2/query/subspace.go @@ -70,6 +70,10 @@ func ScanSubspace(ctx context.Context, st storetypes.KVStore, prefix []byte, lim totalBytes += pairBytes } + if err := abortedScan(ctx, iterator); err != nil { + return nil, err + } + bz, err := pairs.Marshal() if err != nil { panic(fmt.Errorf("failed to marshal KV pairs: %w", err)) @@ -77,6 +81,17 @@ func ScanSubspace(ctx context.Context, st storetypes.KVStore, prefix []byte, lim return bz, nil } +// abortedScan returns why iteration stopped before the prefix was exhausted, or +// nil when it was. An SS iterator that gives up inside an MVCC skip loop goes +// invalid and reports the reason on Error(), which is otherwise indistinguishable +// from a complete scan and would be marshaled as a successful truncated result. +func abortedScan(ctx context.Context, iterator storetypes.Iterator) error { + if err := iterator.Error(); err != nil { + return err + } + return ctx.Err() +} + // IsCapExceeded reports whether err is a subspace scan cap rejection. func IsCapExceeded(err error) bool { return storetypes.ErrSubspaceCapExceeded.Is(err) diff --git a/sei-cosmos/storev2/query/subspace_test.go b/sei-cosmos/storev2/query/subspace_test.go index 7176cdfd02..fa77875860 100644 --- a/sei-cosmos/storev2/query/subspace_test.go +++ b/sei-cosmos/storev2/query/subspace_test.go @@ -116,3 +116,46 @@ func TestIsCapExceededResponse(t *testing.T) { res := sdkerrors.QueryResult(err) require.True(t, IsCapExceededResponse(res)) } + +// abortingIterator yields a fixed number of pairs and then goes invalid with an +// error set, the way an SS iterator reports a scan it gave up on mid-prefix. +type abortingIterator struct { + storetypes.Iterator + remaining int + err error +} + +func (it *abortingIterator) Valid() bool { + return it.err == nil && it.Iterator.Valid() +} + +func (it *abortingIterator) Next() { + it.remaining-- + if it.remaining <= 0 { + it.err = context.Canceled + return + } + it.Iterator.Next() +} + +func (it *abortingIterator) Error() error { + return it.err +} + +type abortingStore struct { + *mem.Store + pairsBeforeAbort int +} + +func (s *abortingStore) Iterator(start, end []byte) storetypes.Iterator { + return &abortingIterator{Iterator: s.Store.Iterator(start, end), remaining: s.pairsBeforeAbort} +} + +func TestScanSubspace_AbortedIterationIsNotSuccess(t *testing.T) { + store := &abortingStore{Store: seedStore(t, 10, 8), pairsBeforeAbort: 3} + + _, err := ScanSubspace(t.Context(), store, []byte("p"), Limits{MaxPairs: 1000, MaxBytes: DefaultMaxSubspaceBytes}) + require.Error(t, err) + require.ErrorIs(t, err, context.Canceled) + require.False(t, storetypes.ErrSubspaceCapExceeded.Is(err)) +} diff --git a/sei-cosmos/storev2/rootmulti/store.go b/sei-cosmos/storev2/rootmulti/store.go index 3694667f50..c6a2b175e9 100644 --- a/sei-cosmos/storev2/rootmulti/store.go +++ b/sei-cosmos/storev2/rootmulti/store.go @@ -77,6 +77,9 @@ type Store struct { histProofSem chan struct{} histProofLimiter *rate.Limiter + // subspaceQuerySem bounds concurrent /subspace scans on the SS fast path only. + // The commitment path is reached with SS disabled or a proof requested, where + // the pair/byte caps are the only bound. subspaceQuerySem chan struct{} subspaceLimits query.Limits