performance experiments

app/abci_test.go

@@ -85,41 +85,60 @@ func TestBeginBlockAppliesMigrationBatchSize(t *testing.T) {
 	require.Equal(t, 321, after, "BeginBlock should push the gov param into the SC store")
 }
 
-// TestMigrationBatchSizeTakesEffectNextBlock is the full end-to-end timing
-// check: a governance proposal committed in block N (written into the block's
-// deliver state, then Commit) only changes the SC store's migration rate when
-// block N+1's BeginBlock runs and reads it from committed state.
+// TestMigrationBatchSizeTakesEffectNextBlock pins when a param change reaches the SC store: BeginBlock
+// applies whatever the param says at the moment it runs, so a proposal that lands later in a block cannot
+// change the rate that block is already migrating at — only the next block's BeginBlock picks it up.
+//
+// The BeginBlock step is driven directly rather than through FinalizeBlock/Commit because the sequence
+// this pins has no legal block-level spelling: getting the param written after BeginBlock but committed at
+// the same height means writing between FinalizeBlock and Commit, which changes committed state for a
+// height whose app hash was already announced to Tendermint. See TestMigrationBatchSizeAppliedAtBlockStart
+// for the same path through a real block.
 func TestMigrationBatchSizeTakesEffectNextBlock(t *testing.T) {
 	a := Setup(t, false, false, false)
-	bg := context.Background()
+	ctx := a.GetContextForDeliverTx([]byte{})
 
-	// Block 1: BeginBlock runs first (param still unset), then the gov
-	// proposal lands by writing into this block's deliver state, then Commit
-	// persists it to the committed multistore.
-	_, err := a.FinalizeBlock(bg, &abci.RequestFinalizeBlock{
-		Header: &tmproto.Header{ChainID: "sei-test", Height: 1, Time: time.Now()},
-	})
-	require.NoError(t, err)
+	// A block begins with the param unset, so the lazily-persisted default leaves migration paused.
+	a.applyMigrationBatchSize(ctx)
+	got, ok := a.rootStore.GetMigrationBatchSize()
+	require.True(t, ok)
+	require.Equal(t, 0, got, "the default param must leave migration paused")
 
+	// A gov proposal raises the rate part-way through that same block.
 	subspace, ok := a.ParamsKeeper.GetSubspace(migration.SubspaceName)
 	require.True(t, ok)
-	subspace.Set(a.GetContextForDeliverTx([]byte{}), migration.KeyNumKeysToMigratePerBlock, uint64(640))
+	subspace.Set(ctx, migration.KeyNumKeysToMigratePerBlock, uint64(640))
 
-	_, err = a.Commit(bg)
-	require.NoError(t, err)
+	got, ok = a.rootStore.GetMigrationBatchSize()
+	require.True(t, ok)
+	require.Equal(t, 0, got, "a param write must not change the rate the current block is migrating at")
 
-	// The param was committed in block 1, but BeginBlock(1) ran before it
-	// existed, so the rate is still paused at this point.
-	got, ok := a.rootStore.GetMigrationBatchSize()
+	// The next block's BeginBlock reads the param and applies it.
+	a.applyMigrationBatchSize(ctx)
+	got, ok = a.rootStore.GetMigrationBatchSize()
 	require.True(t, ok)
-	require.Equal(t, 0, got, "param committed in block 1 must not take effect within block 1")
+	require.Equal(t, 640, got, "the next BeginBlock must apply the new rate")
+}
 
-	// Block 2: BeginBlock reads the now-committed param and applies it.
-	_, err = a.FinalizeBlock(bg, &abci.RequestFinalizeBlock{
-		Header: &tmproto.Header{ChainID: "sei-test", Height: 2, Time: time.Now().Add(time.Second)},
+// TestMigrationBatchSizeAppliedAtBlockStart covers the same path through a real block: a param already in
+// state when the block starts is applied by that block's BeginBlock and survives the commit.
+func TestMigrationBatchSizeAppliedAtBlockStart(t *testing.T) {
+	a := Setup(t, false, false, false)
+	bg := context.Background()
+
+	// Written into the genesis deliver state, before any block's app hash has been taken.
+	subspace, ok := a.ParamsKeeper.GetSubspace(migration.SubspaceName)
+	require.True(t, ok)
+	subspace.Set(a.GetContextForDeliverTx([]byte{}), migration.KeyNumKeysToMigratePerBlock, uint64(640))
+
+	_, err := a.FinalizeBlock(bg, &abci.RequestFinalizeBlock{
+		Header: &tmproto.Header{ChainID: "sei-test", Height: 1, Time: time.Now()},
 	})
 	require.NoError(t, err)
+	_, err = a.Commit(bg)
+	require.NoError(t, err)
 
-	got, _ = a.rootStore.GetMigrationBatchSize()
-	require.Equal(t, 640, got, "migration rate must take effect on the block after the param is committed")
+	got, ok := a.rootStore.GetMigrationBatchSize()
+	require.True(t, ok)
+	require.Equal(t, 640, got, "BeginBlock must apply a param already in state when the block starts")
 }

app/testdata/state-commit.golden

@@ -15,54 +15,82 @@ FlatKVConfig.Fsync = bool(false)
 FlatKVConfig.AsyncWriteBuffer = int(0)
 FlatKVConfig.SnapshotInterval = uint32(10000)
 FlatKVConfig.SnapshotKeepRecent = uint32(1)
+FlatKVConfig.MaxSnapshotLagBlocks = uint32(8192)
+FlatKVConfig.HashQueueSize = uint32(64)
+FlatKVConfig.HashChanSize = uint32(1024)
 FlatKVConfig.ExternalPruning = bool(false)
 FlatKVConfig.EnablePebbleMetrics = bool(true)
 FlatKVConfig.EnableReadWriteMetrics = bool(false)
 FlatKVConfig.AccountDBConfig.DataDir = string("")
 FlatKVConfig.AccountDBConfig.EnableMetrics = bool(true)
 FlatKVConfig.AccountDBConfig.EnableReadWriteMetrics = bool(false)
 FlatKVConfig.AccountDBConfig.MetricsScrapeInterval = time.Duration(10s)
-FlatKVConfig.AccountCacheConfig.ShardCount = uint64(8)
-FlatKVConfig.AccountCacheConfig.MaxSize = uint64(1073741824)
-FlatKVConfig.AccountCacheConfig.EstimatedOverheadPerEntry = uint64(250)
-FlatKVConfig.AccountCacheConfig.MetricsName = string("")
-FlatKVConfig.AccountCacheConfig.MetricsScrapeInterval = time.Duration(0s)
+FlatKVConfig.AccountStoreConfig.ShardCount = uint64(8)
+FlatKVConfig.AccountStoreConfig.MaxSize = uint64(1073741824)
+FlatKVConfig.AccountStoreConfig.EstimatedOverheadPerEntry = uint64(256)
+FlatKVConfig.AccountStoreConfig.Name = string("account")
+FlatKVConfig.AccountStoreConfig.MetricsEnabled = bool(true)
+FlatKVConfig.AccountStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
+FlatKVConfig.AccountStoreConfig.MaxUnflushedVersions = uint64(4096)
+FlatKVConfig.AccountStoreConfig.TargetBytesPerFlush = uint64(4194304)
+FlatKVConfig.AccountStoreConfig.ReservedPrefix = string("_meta/")
+FlatKVConfig.AccountStoreConfig.FlushSync = bool(false)
 FlatKVConfig.CodeDBConfig.DataDir = string("")
 FlatKVConfig.CodeDBConfig.EnableMetrics = bool(true)
 FlatKVConfig.CodeDBConfig.EnableReadWriteMetrics = bool(false)
 FlatKVConfig.CodeDBConfig.MetricsScrapeInterval = time.Duration(10s)
-FlatKVConfig.CodeCacheConfig.ShardCount = uint64(8)
-FlatKVConfig.CodeCacheConfig.MaxSize = uint64(536870912)
-FlatKVConfig.CodeCacheConfig.EstimatedOverheadPerEntry = uint64(250)
-FlatKVConfig.CodeCacheConfig.MetricsName = string("")
-FlatKVConfig.CodeCacheConfig.MetricsScrapeInterval = time.Duration(0s)
+FlatKVConfig.CodeStoreConfig.ShardCount = uint64(8)
+FlatKVConfig.CodeStoreConfig.MaxSize = uint64(536870912)
+FlatKVConfig.CodeStoreConfig.EstimatedOverheadPerEntry = uint64(256)
+FlatKVConfig.CodeStoreConfig.Name = string("code")
+FlatKVConfig.CodeStoreConfig.MetricsEnabled = bool(true)
+FlatKVConfig.CodeStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
+FlatKVConfig.CodeStoreConfig.MaxUnflushedVersions = uint64(4096)
+FlatKVConfig.CodeStoreConfig.TargetBytesPerFlush = uint64(4194304)
+FlatKVConfig.CodeStoreConfig.ReservedPrefix = string("_meta/")
+FlatKVConfig.CodeStoreConfig.FlushSync = bool(false)
 FlatKVConfig.StorageDBConfig.DataDir = string("")
 FlatKVConfig.StorageDBConfig.EnableMetrics = bool(true)
 FlatKVConfig.StorageDBConfig.EnableReadWriteMetrics = bool(false)
 FlatKVConfig.StorageDBConfig.MetricsScrapeInterval = time.Duration(10s)
-FlatKVConfig.StorageCacheConfig.ShardCount = uint64(8)
-FlatKVConfig.StorageCacheConfig.MaxSize = uint64(4294967296)
-FlatKVConfig.StorageCacheConfig.EstimatedOverheadPerEntry = uint64(250)
-FlatKVConfig.StorageCacheConfig.MetricsName = string("")
-FlatKVConfig.StorageCacheConfig.MetricsScrapeInterval = time.Duration(0s)
+FlatKVConfig.StorageStoreConfig.ShardCount = uint64(8)
+FlatKVConfig.StorageStoreConfig.MaxSize = uint64(4294967296)
+FlatKVConfig.StorageStoreConfig.EstimatedOverheadPerEntry = uint64(256)
+FlatKVConfig.StorageStoreConfig.Name = string("storage")
+FlatKVConfig.StorageStoreConfig.MetricsEnabled = bool(true)
+FlatKVConfig.StorageStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
+FlatKVConfig.StorageStoreConfig.MaxUnflushedVersions = uint64(4096)
+FlatKVConfig.StorageStoreConfig.TargetBytesPerFlush = uint64(4194304)
+FlatKVConfig.StorageStoreConfig.ReservedPrefix = string("_meta/")
+FlatKVConfig.StorageStoreConfig.FlushSync = bool(false)
 FlatKVConfig.MiscDBConfig.DataDir = string("")
 FlatKVConfig.MiscDBConfig.EnableMetrics = bool(true)
 FlatKVConfig.MiscDBConfig.EnableReadWriteMetrics = bool(false)
 FlatKVConfig.MiscDBConfig.MetricsScrapeInterval = time.Duration(10s)
-FlatKVConfig.MiscCacheConfig.ShardCount = uint64(8)
-FlatKVConfig.MiscCacheConfig.MaxSize = uint64(536870912)
-FlatKVConfig.MiscCacheConfig.EstimatedOverheadPerEntry = uint64(250)
-FlatKVConfig.MiscCacheConfig.MetricsName = string("")
-FlatKVConfig.MiscCacheConfig.MetricsScrapeInterval = time.Duration(0s)
+FlatKVConfig.MiscStoreConfig.ShardCount = uint64(8)
+FlatKVConfig.MiscStoreConfig.MaxSize = uint64(536870912)
+FlatKVConfig.MiscStoreConfig.EstimatedOverheadPerEntry = uint64(256)
+FlatKVConfig.MiscStoreConfig.Name = string("misc")
+FlatKVConfig.MiscStoreConfig.MetricsEnabled = bool(true)
+FlatKVConfig.MiscStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
+FlatKVConfig.MiscStoreConfig.MaxUnflushedVersions = uint64(4096)
+FlatKVConfig.MiscStoreConfig.TargetBytesPerFlush = uint64(4194304)
+FlatKVConfig.MiscStoreConfig.ReservedPrefix = string("_meta/")
+FlatKVConfig.MiscStoreConfig.FlushSync = bool(false)
 FlatKVConfig.MetadataDBConfig.DataDir = string("")
 FlatKVConfig.MetadataDBConfig.EnableMetrics = bool(true)
 FlatKVConfig.MetadataDBConfig.EnableReadWriteMetrics = bool(false)
 FlatKVConfig.MetadataDBConfig.MetricsScrapeInterval = time.Duration(10s)
-FlatKVConfig.MetadataCacheConfig.ShardCount = uint64(8)
-FlatKVConfig.MetadataCacheConfig.MaxSize = uint64(536870912)
-FlatKVConfig.MetadataCacheConfig.EstimatedOverheadPerEntry = uint64(250)
-FlatKVConfig.MetadataCacheConfig.MetricsName = string("")
-FlatKVConfig.MetadataCacheConfig.MetricsScrapeInterval = time.Duration(0s)
+FlatKVConfig.MetadataStoreConfig.ShardCount = uint64(8)
+FlatKVConfig.MetadataStoreConfig.MaxSize = uint64(536870912)
+FlatKVConfig.MetadataStoreConfig.EstimatedOverheadPerEntry = uint64(256)
+FlatKVConfig.MetadataStoreConfig.Name = string("metadata")
+FlatKVConfig.MetadataStoreConfig.MetricsEnabled = bool(true)
+FlatKVConfig.MetadataStoreConfig.MetricsScrapeIntervalSeconds = float64(10)
+FlatKVConfig.MetadataStoreConfig.MaxUnflushedVersions = uint64(4096)
+FlatKVConfig.MetadataStoreConfig.TargetBytesPerFlush = uint64(4194304)
+FlatKVConfig.MetadataStoreConfig.ReservedPrefix = string("_meta/")
+FlatKVConfig.MetadataStoreConfig.FlushSync = bool(false)
 FlatKVConfig.ReaderThreadsPerCore = float64(2)
 FlatKVConfig.ReaderConstantThreadCount = int(0)
 FlatKVConfig.ReaderPoolQueueSize = int(1024)