From f3cf900f0613d8523fb6052ff221bf9c2378607b Mon Sep 17 00:00:00 2001 From: Wen Date: Sun, 3 May 2026 21:08:47 -0700 Subject: [PATCH 1/2] autobahn: add data_prune_after to bound data.State memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit data.State.runPruning is a background goroutine that drops in-memory blocks/QCs/AppProposals older than a configurable duration, but the config knob (data.Config.PruneAfter) was never wired up — giga_router constructed data.NewState with only Committee set, so the pruner never spawned. data.State.PruneBefore (the giga_router-driven path based on cosmos-sdk RetainHeight) is also a no-op when the chain is configured with pruning="nothing" (Sei's localnode default, common in test setups), so in-memory data.State grew with the chain under sustained load and eventually OOM-killed nodes. Plumb DataPruneAfter through: AutobahnFileConfig.data_prune_after (json) → GigaRouterConfig.DataPruneAfter → data.Config.PruneAfter → data.State.runPruning Production default (gen-autobahn-config): 30m, gives operators plenty of recent history for /block, /tx, /trace_*, etc. while bounding memory under load. Localnode/test override (step4_config_override.sh): 1m, keeps data.State small under sustained-throughput tests where cosmos pruning is "nothing". --- docker/localnode/scripts/step4_config_override.sh | 10 ++++++++++ .../cmd/tendermint/commands/gen_autobahn_config.go | 6 ++++++ sei-tendermint/config/autobahn.go | 3 +++ sei-tendermint/internal/p2p/giga_router.go | 11 ++++++++++- sei-tendermint/node/setup.go | 8 ++++++-- 5 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docker/localnode/scripts/step4_config_override.sh b/docker/localnode/scripts/step4_config_override.sh index 9f7fd8e024..5febd1b204 100755 --- a/docker/localnode/scripts/step4_config_override.sh +++ b/docker/localnode/scripts/step4_config_override.sh @@ -109,6 +109,16 @@ if [ "$AUTOBAHN" = "true" ]; then # Generate autobahn config using seid seid tendermint gen-autobahn-config $NODE_DIRS --output "$AUTOBAHN_CONFIG" + # For localnode/throughput-test runs we use a much shorter data_prune_after + # than the production default to keep data.State memory bounded during + # sustained load (cosmos-sdk's pruning is "nothing" in this setup). + if command -v jq >/dev/null 2>&1; then + tmp=$(mktemp) + jq '.data_prune_after = "1m0s"' "$AUTOBAHN_CONFIG" > "$tmp" && mv "$tmp" "$AUTOBAHN_CONFIG" + else + sed -i 's|"data_prune_after": *"[^"]*"|"data_prune_after": "1m0s"|' "$AUTOBAHN_CONFIG" + fi + # Inject autobahn config file path into config.toml # Must be placed before any [section] header so TOML parser reads it as a top-level key. if grep -q "autobahn-config-file" ~/.sei/config/config.toml; then diff --git a/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go b/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go index 9297fa7a19..865abccf5d 100644 --- a/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go +++ b/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go @@ -77,6 +77,12 @@ Output is written to the file specified by --output.`, AllowEmptyBlocks: true, ViewTimeout: utils.Duration(1500 * time.Millisecond), DialInterval: utils.Duration(10 * time.Second), + // data_prune_after caps the age of in-memory blocks/QCs/AppProposals + // in data.State. 30m gives operators plenty of recent history for + // /block, /tx, etc. while bounding memory under sustained load. + // Test setups (e.g. docker/localnode) can override this to a + // smaller value. + DataPruneAfter: utils.Some(utils.Duration(30 * time.Minute)), } data, err := json.MarshalIndent(cfg, "", " ") diff --git a/sei-tendermint/config/autobahn.go b/sei-tendermint/config/autobahn.go index d27866828b..3aeb493f62 100644 --- a/sei-tendermint/config/autobahn.go +++ b/sei-tendermint/config/autobahn.go @@ -27,6 +27,9 @@ type AutobahnFileConfig struct { ViewTimeout utils.Duration `json:"view_timeout"` PersistentStateDir utils.Option[string] `json:"persistent_state_dir"` DialInterval utils.Duration `json:"dial_interval"` + // DataPruneAfter is the age at which data.State drops in-memory blocks, + // QCs, and AppProposals; nil disables time-based pruning. + DataPruneAfter utils.Option[utils.Duration] `json:"data_prune_after"` } // Validate performs basic validation of the autobahn file config. diff --git a/sei-tendermint/internal/p2p/giga_router.go b/sei-tendermint/internal/p2p/giga_router.go index 8c81500cc5..d39e44a1af 100644 --- a/sei-tendermint/internal/p2p/giga_router.go +++ b/sei-tendermint/internal/p2p/giga_router.go @@ -42,6 +42,12 @@ type GigaRouterConfig struct { Producer *producer.Config TxMempool *mempool.TxMempool GenDoc *types.GenesisDoc + // DataPruneAfter, when set, enables time-based pruning of in-memory + // data.State entries (blocks, AppProposals, QCs). Without it the + // in-memory window is bounded only by cosmos-sdk's RetainHeight via + // PruneBefore — which returns 0 for `pruning="nothing"` setups, leaving + // data.State to grow with the chain. + DataPruneAfter utils.Option[time.Duration] } type GigaRouter struct { @@ -84,7 +90,10 @@ func NewGigaRouter(cfg *GigaRouterConfig, key NodeSecretKey) (*GigaRouter, error if err != nil { return nil, fmt.Errorf("data.NewDataWAL(): %w", err) } - dataState, err := data.NewState(&data.Config{Committee: committee}, dataWAL) + dataState, err := data.NewState(&data.Config{ + Committee: committee, + PruneAfter: cfg.DataPruneAfter, + }, dataWAL) if err != nil { return nil, fmt.Errorf("data.NewState(): %w", err) } diff --git a/sei-tendermint/node/setup.go b/sei-tendermint/node/setup.go index 142b9d2961..630e004a8f 100644 --- a/sei-tendermint/node/setup.go +++ b/sei-tendermint/node/setup.go @@ -240,7 +240,7 @@ func buildGigaConfig( } maxGasPerBlock := uint64(genDoc.ConsensusParams.Block.MaxGas) //nolint:gosec // validated > 0 above - return &p2p.GigaRouterConfig{ + gigaCfg := &p2p.GigaRouterConfig{ DialInterval: time.Duration(fc.DialInterval), ValidatorAddrs: validatorAddrs, Consensus: &autobahnConsensus.Config{ @@ -260,7 +260,11 @@ func buildGigaConfig( }, TxMempool: txMempool, GenDoc: genDoc, - }, nil + } + if d, ok := fc.DataPruneAfter.Get(); ok { + gigaCfg.DataPruneAfter = utils.Some(time.Duration(d)) + } + return gigaCfg, nil } func createRouter( From f1737bf53507b93c34c2b28f29349e74f634c090 Mon Sep 17 00:00:00 2001 From: Wen Date: Thu, 7 May 2026 14:21:41 -0700 Subject: [PATCH 2/2] autobahn: relax localnet data_prune_after from 1m to 5m MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 1m localnet override is too aggressive for E2E test suites that take longer than a minute between cluster start and querying earlier blocks (e.g. eth_getBlockByNumber for fork-version fixtures, or sei_getTransactionReceiptExcludeTraceFail's seed-deploy-block lookup). Raising to 5m still actively exercises the pruning code path under sustained load (production runs accumulate millions of blocks; 5m is well within the in-memory window), while letting the integration suites finish their setup-then-query flow without hitting "height not available" on early blocks. Production default (gen-autobahn-config) remains 30m — unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- docker/localnode/scripts/step4_config_override.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/localnode/scripts/step4_config_override.sh b/docker/localnode/scripts/step4_config_override.sh index 5febd1b204..b3d8f01f3c 100755 --- a/docker/localnode/scripts/step4_config_override.sh +++ b/docker/localnode/scripts/step4_config_override.sh @@ -114,9 +114,9 @@ if [ "$AUTOBAHN" = "true" ]; then # sustained load (cosmos-sdk's pruning is "nothing" in this setup). if command -v jq >/dev/null 2>&1; then tmp=$(mktemp) - jq '.data_prune_after = "1m0s"' "$AUTOBAHN_CONFIG" > "$tmp" && mv "$tmp" "$AUTOBAHN_CONFIG" + jq '.data_prune_after = "5m0s"' "$AUTOBAHN_CONFIG" > "$tmp" && mv "$tmp" "$AUTOBAHN_CONFIG" else - sed -i 's|"data_prune_after": *"[^"]*"|"data_prune_after": "1m0s"|' "$AUTOBAHN_CONFIG" + sed -i 's|"data_prune_after": *"[^"]*"|"data_prune_after": "5m0s"|' "$AUTOBAHN_CONFIG" fi # Inject autobahn config file path into config.toml