From 904e834cdf00a22b7839d42b48432468a64e9faf Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Mon, 6 Jul 2026 10:36:08 -0500 Subject: [PATCH 01/10] add block number to ReadBlockByHash --- sei-db/ledger_db/block/block_db_test.go | 10 +-- sei-db/ledger_db/block/littblock/codec.go | 70 ++++++++++++++++--- .../ledger_db/block/littblock/codec_test.go | 35 ++++++++-- .../block/littblock/litt_block_db.go | 19 ++--- .../block/littblock/litt_block_iterator.go | 2 +- .../ledger_db/block/memblock/mem_block_db.go | 22 ++++-- sei-tendermint/autobahn/types/block_db.go | 5 +- 7 files changed, 124 insertions(+), 39 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 3b905cd4f5..3bbb11f228 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -98,7 +98,7 @@ func testEmptyDB(t *testing.T, build builder) { require.NoError(t, err) require.False(t, blk.IsPresent()) - byHash, err := db.ReadBlockByHash(types.GenBlockHeaderHash(utils.TestRngFromSeed(1))) + byHash, _, err := db.ReadBlockByHash(types.GenBlockHeaderHash(utils.TestRngFromSeed(1))) require.NoError(t, err) require.False(t, byHash.IsPresent()) @@ -135,7 +135,7 @@ func testReadRoundTrip(t *testing.T, build builder) { require.NoError(t, err) require.False(t, missNum.IsPresent()) - missHash, err := db.ReadBlockByHash(types.GenBlockHeaderHash(utils.TestRngFromSeed(1))) + missHash, _, err := db.ReadBlockByHash(types.GenBlockHeaderHash(utils.TestRngFromSeed(1))) require.NoError(t, err) require.False(t, missHash.IsPresent()) } @@ -630,11 +630,12 @@ func testWriteBlockGaps(t *testing.T, build builder) { require.True(t, ok, "block %d should exist", n) require.Equal(t, blocks[n].Header().Hash(), got.Header().Hash()) - byHash, err := db.ReadBlockByHash(blocks[n].Header().Hash()) + byHash, hashNum, err := db.ReadBlockByHash(blocks[n].Header().Hash()) require.NoError(t, err) got, ok = byHash.Get() require.True(t, ok, "block %d should be found by hash", n) require.Equal(t, blocks[n].Header().Hash(), got.Header().Hash()) + require.Equal(t, n, hashNum, "block %d hash lookup should return its number", n) } // Numbers in the gaps were never written and must miss. @@ -831,11 +832,12 @@ func assertBlocksReadable(t *testing.T, db types.BlockDB, batches []batch) { require.True(t, ok, "block %d should exist", n) require.Equal(t, blk.Header().Hash(), got.Header().Hash()) - byHash, err := db.ReadBlockByHash(blk.Header().Hash()) + byHash, hashNum, err := db.ReadBlockByHash(blk.Header().Hash()) require.NoError(t, err) got, ok = byHash.Get() require.True(t, ok, "block by hash should exist") require.Equal(t, blk.Header().Hash(), got.Header().Hash()) + require.Equal(t, n, hashNum, "block %d hash lookup should return its number", n) } } } diff --git a/sei-db/ledger_db/block/littblock/codec.go b/sei-db/ledger_db/block/littblock/codec.go index 7a75715f04..2e5c6d8183 100644 --- a/sei-db/ledger_db/block/littblock/codec.go +++ b/sei-db/ledger_db/block/littblock/codec.go @@ -2,6 +2,7 @@ package littblock import ( "encoding/binary" + "fmt" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" ) @@ -63,22 +64,71 @@ func decodeNumberKey(key []byte) types.GlobalBlockNumber { return decodeKey(key[1:]) } -// encodeBlock marshals a block to the bytes stored as its table value. -func encodeBlock(blk *types.Block) []byte { - return types.BlockConv.Marshal(blk) +// blockSerializationVersion and qcSerializationVersion are the current versions +// of the block and QC value framing written to LittDB. Every stored value is +// prefixed with its kind's version byte so the on-disk format can evolve +// independently per kind. The two version spaces are deliberately separate — a +// block and a QC must never share a version number — so they are free to +// diverge. Decode rejects any other version outright: this store is not yet in +// production, so no prior format is supported. +const blockSerializationVersion byte = 1 +const qcSerializationVersion byte = 1 + +// blockValuePrefixLen is the fixed header preceding a block's proto bytes: one +// version byte followed by the 8-byte big-endian GlobalBlockNumber. +const blockValuePrefixLen = 1 + 8 + +// encodeBlock marshals a block to the bytes stored as its table value. The value +// is framed as [version:1][GlobalBlockNumber:8 big-endian][proto(Block)]. The +// number is embedded so a by-hash lookup — which reaches this same shared value +// through a secondary key that carries only the hash — can still recover it. +func encodeBlock(n types.GlobalBlockNumber, blk *types.Block) []byte { + proto := types.BlockConv.Marshal(blk) + value := make([]byte, 0, blockValuePrefixLen+len(proto)) + value = append(value, blockSerializationVersion) + value = binary.BigEndian.AppendUint64(value, uint64(n)) + value = append(value, proto...) + return value } -// decodeBlock unmarshals a block from its stored table value. -func decodeBlock(value []byte) (*types.Block, error) { - return types.BlockConv.Unmarshal(value) +// decodeBlock unmarshals a block and its embedded GlobalBlockNumber from the +// value produced by encodeBlock. +func decodeBlock(value []byte) (types.GlobalBlockNumber, *types.Block, error) { + if len(value) < blockValuePrefixLen { + return 0, nil, fmt.Errorf("block value too short: %d bytes", len(value)) + } + if value[0] != blockSerializationVersion { + return 0, nil, fmt.Errorf("unsupported block serialization version %d", value[0]) + } + n := types.GlobalBlockNumber(binary.BigEndian.Uint64(value[1:blockValuePrefixLen])) + blk, err := types.BlockConv.Unmarshal(value[blockValuePrefixLen:]) + if err != nil { + return 0, nil, fmt.Errorf("failed to unmarshal block: %w", err) + } + return n, blk, nil } -// encodeQC marshals a FullCommitQC to the bytes stored as its table value. +// encodeQC marshals a FullCommitQC to the bytes stored as its table value, +// framed as [version:1][proto(FullCommitQC)]. func encodeQC(qc *types.FullCommitQC) []byte { - return types.FullCommitQCConv.Marshal(qc) + proto := types.FullCommitQCConv.Marshal(qc) + value := make([]byte, 0, 1+len(proto)) + value = append(value, qcSerializationVersion) + value = append(value, proto...) + return value } -// decodeQC unmarshals a FullCommitQC from its stored table value. +// decodeQC unmarshals a FullCommitQC from the value produced by encodeQC. func decodeQC(value []byte) (*types.FullCommitQC, error) { - return types.FullCommitQCConv.Unmarshal(value) + if len(value) < 1 { + return nil, fmt.Errorf("qc value too short: %d bytes", len(value)) + } + if value[0] != qcSerializationVersion { + return nil, fmt.Errorf("unsupported qc serialization version %d", value[0]) + } + qc, err := types.FullCommitQCConv.Unmarshal(value[1:]) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal qc: %w", err) + } + return qc, nil } diff --git a/sei-db/ledger_db/block/littblock/codec_test.go b/sei-db/ledger_db/block/littblock/codec_test.go index 4213425283..e9143f058d 100644 --- a/sei-db/ledger_db/block/littblock/codec_test.go +++ b/sei-db/ledger_db/block/littblock/codec_test.go @@ -66,15 +66,19 @@ func TestPrefixedKeys(t *testing.T) { func TestBlockRoundTrip(t *testing.T) { rng := utils.TestRngFromSeed(1) - for range 16 { + for i := range 16 { + n := types.GlobalBlockNumber(i * 7) blk := types.GenBlock(rng) - value := encodeBlock(blk) - decoded, err := decodeBlock(value) + value := encodeBlock(n, blk) + require.Equal(t, blockSerializationVersion, value[0], "value must be version-prefixed") + gotN, decoded, err := decodeBlock(value) require.NoError(t, err) + // The embedded block number must round-trip. + require.Equal(t, n, gotN) // Header hash uniquely identifies a block; equal hash => same block. require.Equal(t, blk.Header().Hash(), decoded.Header().Hash()) - // Re-encoding the decoded block must reproduce the same bytes. - require.Equal(t, value, encodeBlock(decoded)) + // Re-encoding the decoded block (with the same number) must reproduce the same bytes. + require.Equal(t, value, encodeBlock(gotN, decoded)) } } @@ -83,6 +87,7 @@ func TestQCRoundTrip(t *testing.T) { for range 16 { qc := types.GenFullCommitQC(rng) value := encodeQC(qc) + require.Equal(t, qcSerializationVersion, value[0], "value must be version-prefixed") decoded, err := decodeQC(value) require.NoError(t, err) // Re-encoding the decoded QC must reproduce the same bytes. @@ -91,10 +96,26 @@ func TestQCRoundTrip(t *testing.T) { } func TestDecodeRejectsGarbage(t *testing.T) { - // Invalid protobuf wire bytes must surface an error rather than a partial value. + // Invalid bytes must surface an error rather than a partial value. garbage := []byte{0xff, 0xff, 0xff, 0xff} - _, blockErr := decodeBlock(garbage) + _, _, blockErr := decodeBlock(garbage) require.Error(t, blockErr) _, qcErr := decodeQC(garbage) require.Error(t, qcErr) } + +func TestDecodeRejectsUnknownVersion(t *testing.T) { + rng := utils.TestRngFromSeed(3) + + // A block value whose version byte is not the current one must be rejected, + // even though the rest of the value is well-formed. + blockValue := encodeBlock(1, types.GenBlock(rng)) + blockValue[0] = blockSerializationVersion + 1 + _, _, blockErr := decodeBlock(blockValue) + require.Error(t, blockErr) + + qcValue := encodeQC(types.GenFullCommitQC(rng)) + qcValue[0] = qcSerializationVersion + 1 + _, qcErr := decodeQC(qcValue) + require.Error(t, qcErr) +} diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 97e0585e09..f67fe2cfba 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -144,7 +144,7 @@ func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error n, s.lastQCNext, types.ErrBlockMissingQC) } - value := encodeBlock(blk) + value := encodeBlock(n, blk) hash := blk.Header().Hash() hashAlias := &litttypes.SecondaryKey{ Key: blockHashKey(hash), @@ -255,28 +255,29 @@ func (s *blockDB) QCs(reverse bool) (types.QCIterator, error) { func (s *blockDB) ReadBlockByNumber( n types.GlobalBlockNumber, ) (utils.Option[*types.Block], error) { - return getBlock(s.table, blockKey(n)) + blk, _, err := getBlock(s.table, blockKey(n)) + return blk, err } func (s *blockDB) ReadBlockByHash( hash types.BlockHeaderHash, -) (utils.Option[*types.Block], error) { +) (utils.Option[*types.Block], types.GlobalBlockNumber, error) { return getBlock(s.table, blockHashKey(hash)) } -func getBlock(table littdb.Table, key []byte) (utils.Option[*types.Block], error) { +func getBlock(table littdb.Table, key []byte) (utils.Option[*types.Block], types.GlobalBlockNumber, error) { value, exists, err := table.Get(key) if err != nil { - return utils.None[*types.Block](), fmt.Errorf("failed to read block: %w", err) + return utils.None[*types.Block](), 0, fmt.Errorf("failed to read block: %w", err) } if !exists { - return utils.None[*types.Block](), nil + return utils.None[*types.Block](), 0, nil } - blk, err := decodeBlock(value) + n, blk, err := decodeBlock(value) if err != nil { - return utils.None[*types.Block](), fmt.Errorf("failed to unmarshal block: %w", err) + return utils.None[*types.Block](), 0, fmt.Errorf("failed to unmarshal block: %w", err) } - return utils.Some(blk), nil + return utils.Some(blk), n, nil } func (s *blockDB) ReadQCByBlockNumber( diff --git a/sei-db/ledger_db/block/littblock/litt_block_iterator.go b/sei-db/ledger_db/block/littblock/litt_block_iterator.go index 72460ee8b3..62bf6d5b94 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_iterator.go +++ b/sei-db/ledger_db/block/littblock/litt_block_iterator.go @@ -44,7 +44,7 @@ func (b *blockIterator) Block() (*types.Block, error) { if err != nil { return nil, fmt.Errorf("failed to read block value: %w", err) } - blk, err := decodeBlock(value) + _, blk, err := decodeBlock(value) if err != nil { return nil, fmt.Errorf("failed to unmarshal block: %w", err) } diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index ff631ef7a7..0e129c2508 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -19,13 +19,21 @@ type qcEntry struct { upper types.GlobalBlockNumber } +// hashEntry pairs a block with its GlobalBlockNumber so ReadBlockByHash can +// return the number, mirroring the littblock implementation which embeds it in +// the stored value. +type hashEntry struct { + blk *types.Block + n types.GlobalBlockNumber +} + // blockDB is an in-memory types.BlockDB. It holds blocks and QCs by pointer (no // marshaling) and is intended as a test/benchmark fixture, not a durable // implementation. type blockDB struct { mu sync.RWMutex byNumber map[types.GlobalBlockNumber]*types.Block - byHash map[types.BlockHeaderHash]*types.Block + byHash map[types.BlockHeaderHash]hashEntry qcsByLower map[types.GlobalBlockNumber]qcEntry // Write-order cursors (see types.BlockDB contract). @@ -39,7 +47,7 @@ type blockDB struct { func NewBlockDB() types.BlockDB { return &blockDB{ byNumber: make(map[types.GlobalBlockNumber]*types.Block), - byHash: make(map[types.BlockHeaderHash]*types.Block), + byHash: make(map[types.BlockHeaderHash]hashEntry), qcsByLower: make(map[types.GlobalBlockNumber]qcEntry), } } @@ -58,7 +66,7 @@ func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error n, s.lastQCNext, types.ErrBlockMissingQC) } s.byNumber[n] = blk - s.byHash[blk.Header().Hash()] = blk + s.byHash[blk.Header().Hash()] = hashEntry{blk: blk, n: n} s.lastBlockNumber = n s.hasBlocks = true return nil @@ -193,13 +201,13 @@ func (s *blockDB) ReadBlockByNumber( func (s *blockDB) ReadBlockByHash( hash types.BlockHeaderHash, -) (utils.Option[*types.Block], error) { +) (utils.Option[*types.Block], types.GlobalBlockNumber, error) { s.mu.RLock() defer s.mu.RUnlock() - if blk, ok := s.byHash[hash]; ok { - return utils.Some(blk), nil + if e, ok := s.byHash[hash]; ok { + return utils.Some(e.blk), e.n, nil } - return utils.None[*types.Block](), nil + return utils.None[*types.Block](), 0, nil } func (s *blockDB) ReadQCByBlockNumber( diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index c5baafcd3d..c58fa41433 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -205,11 +205,14 @@ type BlockDB interface { // given value. The hash is the same value as block.Header().Hash() // for the block that was passed to WriteBlock. // + // This method returns the block number for the block if it is found. + // not found, the block number returned is undefined. + // // Returns utils.None if no such block has been written. A pruned // block MAY also return None, but — as with ReadBlockByNumber — // reclamation is asynchronous, so a pruned block may remain readable // until it is actually reclaimed. Non-blocking. - ReadBlockByHash(hash BlockHeaderHash) (utils.Option[*Block], error) + ReadBlockByHash(hash BlockHeaderHash) (utils.Option[*Block], GlobalBlockNumber, error) // ReadQCByBlockNumber returns the FullCommitQC whose // GlobalRange().First ≤ n < GlobalRange().Next — i.e. the QC that From 175dccbb4f04ccf491633797cbaaf8793472a554 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 7 Jul 2026 10:41:14 -0500 Subject: [PATCH 02/10] made suggested changes --- sei-db/ledger_db/block/littblock/codec.go | 10 +++------- sei-tendermint/autobahn/types/block_db.go | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/sei-db/ledger_db/block/littblock/codec.go b/sei-db/ledger_db/block/littblock/codec.go index 2e5c6d8183..c39bf69276 100644 --- a/sei-db/ledger_db/block/littblock/codec.go +++ b/sei-db/ledger_db/block/littblock/codec.go @@ -64,14 +64,10 @@ func decodeNumberKey(key []byte) types.GlobalBlockNumber { return decodeKey(key[1:]) } -// blockSerializationVersion and qcSerializationVersion are the current versions -// of the block and QC value framing written to LittDB. Every stored value is -// prefixed with its kind's version byte so the on-disk format can evolve -// independently per kind. The two version spaces are deliberately separate — a -// block and a QC must never share a version number — so they are free to -// diverge. Decode rejects any other version outright: this store is not yet in -// production, so no prior format is supported. +// Serialization version for blocks. const blockSerializationVersion byte = 1 + +// Serialization version for QCs. const qcSerializationVersion byte = 1 // blockValuePrefixLen is the fixed header preceding a block's proto bytes: one diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index c58fa41433..81e4aebd5f 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -206,7 +206,7 @@ type BlockDB interface { // for the block that was passed to WriteBlock. // // This method returns the block number for the block if it is found. - // not found, the block number returned is undefined. + // If not found, the block number returned is undefined. // // Returns utils.None if no such block has been written. A pruned // block MAY also return None, but — as with ReadBlockByNumber — From f146305ab96b598d7384f252df1554906b47eb4f Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 9 Jul 2026 08:23:00 -0500 Subject: [PATCH 03/10] made suggested changes --- sei-db/ledger_db/block/block_db_test.go | 20 ++++++------- .../ledger_db/block/littblock/codec_test.go | 2 +- .../block/littblock/litt_block_db.go | 28 ++++++++++--------- .../ledger_db/block/memblock/mem_block_db.go | 6 ++-- sei-tendermint/autobahn/types/block.go | 8 ++++++ sei-tendermint/autobahn/types/block_db.go | 10 +++---- 6 files changed, 41 insertions(+), 33 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 3bbb11f228..266befeb4e 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -98,7 +98,7 @@ func testEmptyDB(t *testing.T, build builder) { require.NoError(t, err) require.False(t, blk.IsPresent()) - byHash, _, err := db.ReadBlockByHash(types.GenBlockHeaderHash(utils.TestRngFromSeed(1))) + byHash, err := db.ReadBlockByHash(types.GenBlockHeaderHash(utils.TestRngFromSeed(1))) require.NoError(t, err) require.False(t, byHash.IsPresent()) @@ -135,7 +135,7 @@ func testReadRoundTrip(t *testing.T, build builder) { require.NoError(t, err) require.False(t, missNum.IsPresent()) - missHash, _, err := db.ReadBlockByHash(types.GenBlockHeaderHash(utils.TestRngFromSeed(1))) + missHash, err := db.ReadBlockByHash(types.GenBlockHeaderHash(utils.TestRngFromSeed(1))) require.NoError(t, err) require.False(t, missHash.IsPresent()) } @@ -630,12 +630,12 @@ func testWriteBlockGaps(t *testing.T, build builder) { require.True(t, ok, "block %d should exist", n) require.Equal(t, blocks[n].Header().Hash(), got.Header().Hash()) - byHash, hashNum, err := db.ReadBlockByHash(blocks[n].Header().Hash()) + byHash, err := db.ReadBlockByHash(blocks[n].Header().Hash()) require.NoError(t, err) - got, ok = byHash.Get() + bwn, ok := byHash.Get() require.True(t, ok, "block %d should be found by hash", n) - require.Equal(t, blocks[n].Header().Hash(), got.Header().Hash()) - require.Equal(t, n, hashNum, "block %d hash lookup should return its number", n) + require.Equal(t, blocks[n].Header().Hash(), bwn.Block.Header().Hash()) + require.Equal(t, n, bwn.Number, "block %d hash lookup should return its number", n) } // Numbers in the gaps were never written and must miss. @@ -832,12 +832,12 @@ func assertBlocksReadable(t *testing.T, db types.BlockDB, batches []batch) { require.True(t, ok, "block %d should exist", n) require.Equal(t, blk.Header().Hash(), got.Header().Hash()) - byHash, hashNum, err := db.ReadBlockByHash(blk.Header().Hash()) + byHash, err := db.ReadBlockByHash(blk.Header().Hash()) require.NoError(t, err) - got, ok = byHash.Get() + bwn, ok := byHash.Get() require.True(t, ok, "block by hash should exist") - require.Equal(t, blk.Header().Hash(), got.Header().Hash()) - require.Equal(t, n, hashNum, "block %d hash lookup should return its number", n) + require.Equal(t, blk.Header().Hash(), bwn.Block.Header().Hash()) + require.Equal(t, n, bwn.Number, "block %d hash lookup should return its number", n) } } } diff --git a/sei-db/ledger_db/block/littblock/codec_test.go b/sei-db/ledger_db/block/littblock/codec_test.go index e9143f058d..3c5aff9eca 100644 --- a/sei-db/ledger_db/block/littblock/codec_test.go +++ b/sei-db/ledger_db/block/littblock/codec_test.go @@ -67,7 +67,7 @@ func TestPrefixedKeys(t *testing.T) { func TestBlockRoundTrip(t *testing.T) { rng := utils.TestRngFromSeed(1) for i := range 16 { - n := types.GlobalBlockNumber(i * 7) + n := types.GlobalBlockNumber(i) blk := types.GenBlock(rng) value := encodeBlock(n, blk) require.Equal(t, blockSerializationVersion, value[0], "value must be version-prefixed") diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index f67fe2cfba..9396a981b9 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -252,32 +252,34 @@ func (s *blockDB) QCs(reverse bool) (types.QCIterator, error) { return &qcIterator{it: it}, nil } -func (s *blockDB) ReadBlockByNumber( - n types.GlobalBlockNumber, -) (utils.Option[*types.Block], error) { - blk, _, err := getBlock(s.table, blockKey(n)) - return blk, err +func (s *blockDB) ReadBlockByNumber(n types.GlobalBlockNumber) (utils.Option[*types.Block], error) { + result, err := getBlock(s.table, blockKey(n)) + if err != nil { + return utils.None[*types.Block](), err + } + if bwn, ok := result.Get(); ok { + return utils.Some(bwn.Block), nil + } + return utils.None[*types.Block](), nil } -func (s *blockDB) ReadBlockByHash( - hash types.BlockHeaderHash, -) (utils.Option[*types.Block], types.GlobalBlockNumber, error) { +func (s *blockDB) ReadBlockByHash(hash types.BlockHeaderHash) (utils.Option[types.BlockWithNumber], error) { return getBlock(s.table, blockHashKey(hash)) } -func getBlock(table littdb.Table, key []byte) (utils.Option[*types.Block], types.GlobalBlockNumber, error) { +func getBlock(table littdb.Table, key []byte) (utils.Option[types.BlockWithNumber], error) { value, exists, err := table.Get(key) if err != nil { - return utils.None[*types.Block](), 0, fmt.Errorf("failed to read block: %w", err) + return utils.None[types.BlockWithNumber](), fmt.Errorf("failed to read block: %w", err) } if !exists { - return utils.None[*types.Block](), 0, nil + return utils.None[types.BlockWithNumber](), nil } n, blk, err := decodeBlock(value) if err != nil { - return utils.None[*types.Block](), 0, fmt.Errorf("failed to unmarshal block: %w", err) + return utils.None[types.BlockWithNumber](), fmt.Errorf("failed to unmarshal block: %w", err) } - return utils.Some(blk), n, nil + return utils.Some(types.BlockWithNumber{Block: blk, Number: n}), nil } func (s *blockDB) ReadQCByBlockNumber( diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index 0e129c2508..d1481c2f89 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -201,13 +201,13 @@ func (s *blockDB) ReadBlockByNumber( func (s *blockDB) ReadBlockByHash( hash types.BlockHeaderHash, -) (utils.Option[*types.Block], types.GlobalBlockNumber, error) { +) (utils.Option[types.BlockWithNumber], error) { s.mu.RLock() defer s.mu.RUnlock() if e, ok := s.byHash[hash]; ok { - return utils.Some(e.blk), e.n, nil + return utils.Some(types.BlockWithNumber{Block: e.blk, Number: e.n}), nil } - return utils.None[*types.Block](), 0, nil + return utils.None[types.BlockWithNumber](), nil } func (s *blockDB) ReadQCByBlockNumber( diff --git a/sei-tendermint/autobahn/types/block.go b/sei-tendermint/autobahn/types/block.go index 7b3e220e30..b4fcbd23f0 100644 --- a/sei-tendermint/autobahn/types/block.go +++ b/sei-tendermint/autobahn/types/block.go @@ -28,6 +28,14 @@ type BlockNumber uint64 // GlobalBlockNumber is the number of a block in the global chain. type GlobalBlockNumber uint64 +// BlockWithNumber pairs a block with its GlobalBlockNumber. It is used as the +// payload of the utils.Option returned by ReadBlockByHash so that the block +// number is only present when the block itself is present. +type BlockWithNumber struct { + Block *Block + Number GlobalBlockNumber +} + // BlockHeaderHash is the hash of a BlockHeader. type BlockHeaderHash hashable.Hash[*pb.BlockHeader] diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index 81e4aebd5f..5210c3f8c5 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -202,17 +202,15 @@ type BlockDB interface { ReadBlockByNumber(n GlobalBlockNumber) (utils.Option[*Block], error) // ReadBlockByHash returns the block whose header hashes to the - // given value. The hash is the same value as block.Header().Hash() - // for the block that was passed to WriteBlock. - // - // This method returns the block number for the block if it is found. - // If not found, the block number returned is undefined. + // given value, paired with its GlobalBlockNumber. The hash is the + // same value as block.Header().Hash() for the block that was passed + // to WriteBlock. // // Returns utils.None if no such block has been written. A pruned // block MAY also return None, but — as with ReadBlockByNumber — // reclamation is asynchronous, so a pruned block may remain readable // until it is actually reclaimed. Non-blocking. - ReadBlockByHash(hash BlockHeaderHash) (utils.Option[*Block], GlobalBlockNumber, error) + ReadBlockByHash(hash BlockHeaderHash) (utils.Option[BlockWithNumber], error) // ReadQCByBlockNumber returns the FullCommitQC whose // GlobalRange().First ≤ n < GlobalRange().Next — i.e. the QC that From fd8d06ed4262678b1a083ad819adc8fee689a558 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 9 Jul 2026 09:31:17 -0500 Subject: [PATCH 04/10] Prevent blocks from being read without QCs being present. --- sei-db/ledger_db/block/block_db_test.go | 96 +++++----- .../block/littblock/litt_block_db.go | 73 +++++++- .../block/littblock/litt_block_iterator.go | 41 ++++- .../littblock/litt_block_stranding_test.go | 172 ++++++++++++++++++ sei-tendermint/autobahn/types/block_db.go | 5 + sei-tendermint/autobahn/types/testonly.go | 11 ++ 6 files changed, 337 insertions(+), 61 deletions(-) create mode 100644 sei-db/ledger_db/block/littblock/litt_block_stranding_test.go diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 266befeb4e..f983c7f12c 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -57,6 +57,7 @@ func TestBlockDB(t *testing.T) { t.Run("RestartPersistsData", func(t *testing.T) { testRestartPersistsData(t, impl.build) }) t.Run("PruneRetainsAtOrAbove", func(t *testing.T) { testPruneRetainsAtOrAbove(t, impl.build) }) t.Run("PruneStraddleRetainsQC", func(t *testing.T) { testPruneStraddleRetainsQC(t, impl.build) }) + t.Run("PruneRefusesBelowWatermark", func(t *testing.T) { testPruneRefusesBelowWatermark(t, impl.build) }) t.Run("PruneIdempotentMonotonic", func(t *testing.T) { testPruneIdempotentMonotonic(t, impl.build) }) t.Run("WriteOrderRejected", func(t *testing.T) { testWriteOrderRejected(t, impl.build) }) t.Run("WriteOrderRejectedAfterRestart", func(t *testing.T) { @@ -256,6 +257,49 @@ func testPruneStraddleRetainsQC(t *testing.T, build builder) { require.Equal(t, straddled.first, got.QC().GlobalRange(committee).First) } +// testPruneRefusesBelowWatermark asserts the refuse direction of PruneBefore: +// once the watermark advances past a block, that block is no longer served by +// ReadBlockByNumber, ReadBlockByHash, or the Blocks iterator — so a caller can +// never observe a block whose covering QC may have been pruned out from under it. +func testPruneRefusesBelowWatermark(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + db, _ := openFresh(t, build) + defer func() { _ = db.Close() }() + writeAll(t, db, batches) + + // Prune at the start of the second batch: all of the first batch is below it. + watermark := batches[1].first + require.NoError(t, db.PruneBefore(watermark)) + + below := batches[0] + for i, blk := range below.blocks { + n := below.first + gbn(i) + require.Less(t, n, watermark) + + byNum, err := db.ReadBlockByNumber(n) + require.NoError(t, err) + require.False(t, byNum.IsPresent(), "block %d below watermark %d must not be served", n, watermark) + + byHash, err := db.ReadBlockByHash(blk.Header().Hash()) + require.NoError(t, err) + require.False(t, byHash.IsPresent(), "block %d below watermark %d must not be served by hash", n, watermark) + } + + blockIt, err := db.Blocks(false) + require.NoError(t, err) + defer func() { _ = blockIt.Close() }() + for { + ok, err := blockIt.Next() + require.NoError(t, err) + if !ok { + break + } + require.GreaterOrEqual(t, blockIt.Number(), watermark, + "iterator must not yield block %d below watermark %d", blockIt.Number(), watermark) + } +} + // testPruneIdempotentMonotonic asserts PruneBefore is idempotent and the // watermark only advances: re-pruning at the same point, or at a lower point, // is a no-op that neither errors nor disturbs retained data. @@ -761,54 +805,10 @@ func TestMemblockPruneStraddlingQC(t *testing.T) { require.True(t, below.IsPresent(), "memblock retains the straddling QC for sub-watermark in-range lookups") } -// TestLittblockReclaimsAcrossRestart verifies the durable reclamation path: data -// written, then pruned past after a restart (which seals the segments it landed -// in), is collected by GC. The active segment of a running DB only holds the -// newest data, which is never below the watermark — hence the restart. -func TestLittblockReclaimsAcrossRestart(t *testing.T) { - dir := t.TempDir() - committee, keys := buildCommittee() - batches := generateBatches(committee, keys) - - db, err := littblock.NewBlockDB(littConfig(t, dir)) - require.NoError(t, err) - writeAll(t, db, batches) - require.NoError(t, db.Flush()) - require.NoError(t, db.Close()) - - // Reopen: the segments written above are now sealed and collectable. - db2, err := littblock.NewBlockDB(littConfig(t, dir)) - require.NoError(t, err) - defer func() { _ = db2.Close() }() - - beyond := batches[len(batches)-1].next - require.NoError(t, db2.PruneBefore(beyond)) - require.NoError(t, littblock.ForceGC(db2)) - - for _, b := range batches { - opt, err := db2.ReadBlockByNumber(b.first) - require.NoError(t, err) - require.False(t, opt.IsPresent(), "block %d should be reclaimed after restart", b.first) - qc, err := db2.ReadQCByBlockNumber(b.first) - require.NoError(t, err) - require.False(t, qc.IsPresent(), "QC at %d should be reclaimed after restart", b.first) - } - - // After reclamation the iterators must surface nothing. - blockIt, err := db2.Blocks(false) - require.NoError(t, err) - ok, err := blockIt.Next() - require.NoError(t, err) - require.False(t, ok, "all blocks reclaimed: block iterator must be empty") - require.NoError(t, blockIt.Close()) - - qcIt, err := db2.QCs(false) - require.NoError(t, err) - ok, err = qcIt.Next() - require.NoError(t, err) - require.False(t, ok, "all QCs reclaimed: QC iterator must be empty") - require.NoError(t, qcIt.Close()) -} +// The durable reclamation path (data pruned past after a restart is physically +// collected by GC) is covered by TestLittblockReclaimsAcrossRestart in package +// littblock, which inspects the raw table directly — public reads can no longer +// distinguish "reclaimed" from "refused by the read watermark". // littConfig builds a littblock config rooted at dir with a tiny retention so // the prune watermark is the sole observable reclamation gate in tests. diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 9396a981b9..885d69bec4 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -25,9 +25,14 @@ type blockDB struct { db littdb.DB table littdb.Table - // watermark is the highest n passed to PruneBefore; the GC filters treat - // keys strictly below it as eligible for reclamation. Read from the GC - // goroutine, so accessed atomically. + // watermark is the highest n passed to PruneBefore (a floor re-derived at + // startup, see recoverReadWatermark). It serves two purposes: the GC filters + // treat keys strictly below it as eligible for reclamation, and reads and + // iterators refuse to serve any block strictly below it. The read gate is what + // upholds the "a served block is always covered by a served QC" invariant: + // asynchronous GC can strand a below-watermark block on disk (its covering QC + // reclaimed while the block's own segment lingers), but such a block is never + // served. Read from the GC goroutine, so accessed atomically. watermark atomic.Uint64 // Write-order cursors (see types.BlockDB contract). Guarded by mu. @@ -77,6 +82,10 @@ func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { _ = db.Close() return nil, fmt.Errorf("failed to recover write cursors: %w", err) } + if err := s.recoverReadWatermark(); err != nil { + _ = db.Close() + return nil, fmt.Errorf("failed to recover read watermark: %w", err) + } return s, nil } @@ -128,6 +137,41 @@ func (s *blockDB) recoverCursors() error { return nil } +// recoverReadWatermark re-derives a safe read watermark on open. The watermark +// is in-memory only, so a restart forgets every PruneBefore. That is fine for +// reclamation (nothing new is deleted), but we must protect against showing un-pruned +// blocks with pruned QCs. +func (s *blockDB) recoverReadWatermark() error { + it, err := s.table.Iterator(false) + if err != nil { + return fmt.Errorf("failed to open watermark recovery iterator: %w", err) + } + defer func() { _ = it.Close() }() + + for { + ok, err := it.Next() + if err != nil { + return fmt.Errorf("failed to advance watermark recovery iterator: %w", err) + } + if !ok { + break + } + key, isPrimary := it.GetKey() + if !isPrimary || keyKind(key) != kindQC { + continue + } + s.watermark.Store(uint64(decodeNumberKey(key))) + return nil + } + + // No QC survives. Any block still present is therefore stranded, so refuse + // every one by setting the watermark just past the newest block. + if s.hasBlocks { + s.watermark.Store(uint64(s.lastBlockNumber) + 1) + } + return nil +} + func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error { s.mu.Lock() defer s.mu.Unlock() @@ -241,7 +285,7 @@ func (s *blockDB) Blocks(reverse bool) (types.BlockIterator, error) { if err != nil { return nil, fmt.Errorf("failed to open blocks iterator: %w", err) } - return &blockIterator{it: it}, nil + return &blockIterator{it: it, watermark: s.watermark.Load()}, nil } func (s *blockDB) QCs(reverse bool) (types.QCIterator, error) { @@ -249,10 +293,14 @@ func (s *blockDB) QCs(reverse bool) (types.QCIterator, error) { if err != nil { return nil, fmt.Errorf("failed to open qcs iterator: %w", err) } - return &qcIterator{it: it}, nil + return &qcIterator{it: it, watermark: s.watermark.Load()}, nil } func (s *blockDB) ReadBlockByNumber(n types.GlobalBlockNumber) (utils.Option[*types.Block], error) { + // Refuse below-watermark blocks: they may be stranded (covering QC reclaimed). + if uint64(n) < s.watermark.Load() { + return utils.None[*types.Block](), nil + } result, err := getBlock(s.table, blockKey(n)) if err != nil { return utils.None[*types.Block](), err @@ -264,7 +312,16 @@ func (s *blockDB) ReadBlockByNumber(n types.GlobalBlockNumber) (utils.Option[*ty } func (s *blockDB) ReadBlockByHash(hash types.BlockHeaderHash) (utils.Option[types.BlockWithNumber], error) { - return getBlock(s.table, blockHashKey(hash)) + result, err := getBlock(s.table, blockHashKey(hash)) + if err != nil { + return utils.None[types.BlockWithNumber](), err + } + // The number is not known until the block is resolved; refuse it if it turns + // out to be below the watermark (potentially stranded from its covering QC). + if bwn, ok := result.Get(); ok && uint64(bwn.Number) < s.watermark.Load() { + return utils.None[types.BlockWithNumber](), nil + } + return result, nil } func getBlock(table littdb.Table, key []byte) (utils.Option[types.BlockWithNumber], error) { @@ -285,6 +342,10 @@ func getBlock(table littdb.Table, key []byte) (utils.Option[types.BlockWithNumbe func (s *blockDB) ReadQCByBlockNumber( n types.GlobalBlockNumber, ) (utils.Option[*types.FullCommitQC], error) { + // Below-watermark blocks are not served, so neither is their covering QC. + if uint64(n) < s.watermark.Load() { + return utils.None[*types.FullCommitQC](), nil + } value, exists, err := s.table.Get(qcKey(n)) if err != nil { return utils.None[*types.FullCommitQC](), fmt.Errorf("failed to read QC: %w", err) diff --git a/sei-db/ledger_db/block/littblock/litt_block_iterator.go b/sei-db/ledger_db/block/littblock/litt_block_iterator.go index 62bf6d5b94..28ba7f2df8 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_iterator.go +++ b/sei-db/ledger_db/block/littblock/litt_block_iterator.go @@ -14,9 +14,12 @@ var ( // blockIterator wraps a litt iterator over the shared ledger table, yielding one // entry per block: it skips QC keys and the secondary (hash-alias) keys, keeping -// only primary block-number keys. +// only primary block-number keys. It also skips blocks strictly below watermark, +// which may be stranded from their covering QC (see blockDB.watermark); the +// watermark is captured when the iterator is created. type blockIterator struct { - it littdb.Iterator + it littdb.Iterator + watermark uint64 } func (b *blockIterator) Next() (bool, error) { @@ -28,9 +31,14 @@ func (b *blockIterator) Next() (bool, error) { if !ok { return false, nil } - if key, isPrimary := b.it.GetKey(); isPrimary && keyKind(key) == kindBlock { - return true, nil + key, isPrimary := b.it.GetKey() + if !isPrimary || keyKind(key) != kindBlock { + continue + } + if uint64(decodeNumberKey(key)) < b.watermark { + continue } + return true, nil } } @@ -60,9 +68,13 @@ func (b *blockIterator) Close() error { // qcIterator wraps a litt iterator over the shared ledger table, yielding one // entry per QC: it skips block keys and the secondary (covered-number) keys, -// keeping only primary QC keys. +// keeping only primary QC keys. It also skips any QC whose entire covered range +// is strictly below watermark (Next <= watermark), since none of its blocks are +// served; a QC straddling the watermark still covers served blocks and is kept. +// The watermark is captured when the iterator is created. type qcIterator struct { - it littdb.Iterator + it littdb.Iterator + watermark uint64 } func (q *qcIterator) Next() (bool, error) { @@ -74,7 +86,22 @@ func (q *qcIterator) Next() (bool, error) { if !ok { return false, nil } - if key, isPrimary := q.it.GetKey(); isPrimary && keyKind(key) == kindQC { + key, isPrimary := q.it.GetKey() + if !isPrimary || keyKind(key) != kindQC { + continue + } + // The First of this QC is its primary-key number. If First >= watermark + // the whole range is served; otherwise decode the value to learn Next and + // keep it only if it straddles the watermark (covers a served block). + if uint64(decodeNumberKey(key)) >= q.watermark { + return true, nil + } + qc, err := q.QC() + if err != nil { + return false, err + } + next := uint64(decodeNumberKey(key)) + uint64(len(qc.Headers())) + if next > q.watermark { return true, nil } } diff --git a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go new file mode 100644 index 0000000000..1d84226f76 --- /dev/null +++ b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go @@ -0,0 +1,172 @@ +package littblock + +import ( + "math" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" +) + +// strandingConfig builds a config whose only segment-rollover trigger is a small +// MaxSegmentKeyCount, so a test can place a segment boundary at a precise key. +// Retention is tiny (the prune watermark is the sole reclamation gate) and GC is +// effectively background-disabled so ForceGC is the only thing that reclaims. +func strandingConfig(t *testing.T, dir string, maxSegmentKeyCount uint32) *LittBlockConfig { + cfg, err := DefaultConfig(dir) + require.NoError(t, err) + cfg.Retention = time.Nanosecond + cfg.Litt.TargetSegmentFileSize = math.MaxUint32 + cfg.Litt.MaxSegmentKeyCount = maxSegmentKeyCount + cfg.Litt.GCPeriod = time.Hour + cfg.Litt.Fsync = false + return cfg +} + +// writeSyntheticBatches writes numBatches contiguous batches of perQC blocks each +// (global numbers 0.., QC ranges [0,perQC), [perQC,2*perQC), ...). The QCs carry +// perQC opaque headers so the store's range accounting (Next = First + len) lines +// up; littblock does not verify signatures, so no committee is needed. +func writeSyntheticBatches(t *testing.T, db types.BlockDB, rng utils.Rng, numBatches int, perQC int) { + for i := 0; i < numBatches; i++ { + first := types.GlobalBlockNumber(i * perQC) //nolint:gosec // small test indices + next := first + types.GlobalBlockNumber(perQC) + qc := types.GenFullCommitQCN(rng, perQC) + require.NoError(t, db.WriteQC(first, next, qc)) + for j := 0; j < perQC; j++ { + require.NoError(t, db.WriteBlock(first+types.GlobalBlockNumber(j), types.GenBlock(rng))) //nolint:gosec + } + } +} + +// physicallyPresent reports whether a key exists in the raw table, bypassing the +// read-watermark gate. Used to distinguish a record that has been physically +// reclaimed from one that is present on disk but refused by the watermark. +func physicallyPresent(t *testing.T, impl *blockDB, key []byte) bool { + _, exists, err := impl.table.Get(key) + require.NoError(t, err) + return exists +} + +// TestLittblockStrandedBlockNotServedAfterRestart is the cross-segment stranding +// regression. GC reclaims a contiguous prefix of segments in write order, and a +// QC is always written before the blocks it covers, so a QC's covered range can +// straddle a segment boundary and its segment can be reclaimed while a later +// segment still holds some covered blocks — leaving those blocks on disk with no +// covering QC. Because the prune watermark is in-memory only, a restart forgets +// it and a naive store would re-serve the stranded blocks. +// +// With MaxSegmentKeyCount = 8 the QC Put (5 keys) plus the first two block Puts +// (2 keys each) fill segment 0 = {QC[0,5), b0, b1}; the remaining covered blocks +// spill into segment 1 = {b2, b3, b4, QC[5,10)}. Pruning to 5 makes segment 0 +// collectable (every key < 5) while segment 1 is pinned by QC[5,10) (key 5), so +// QC[0,5) is reclaimed but blocks 2..4 survive, stranded. +func TestLittblockStrandedBlockNotServedAfterRestart(t *testing.T) { + dir := t.TempDir() + rng := utils.TestRngFromSeed(1) + + db, err := NewBlockDB(strandingConfig(t, dir, 8)) + require.NoError(t, err) + writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19; QCs [0,5),[5,10),[10,15),[15,20) + require.NoError(t, db.Flush()) + require.NoError(t, db.Close()) + + // Reopen so the segments seal, then prune past the first QC and collect. + db2, err := NewBlockDB(strandingConfig(t, dir, 8)) + require.NoError(t, err) + require.NoError(t, db2.PruneBefore(5)) + require.NoError(t, ForceGC(db2)) + require.NoError(t, db2.Flush()) + require.NoError(t, db2.Close()) + + // Reopen with the in-memory watermark forgotten: this is where a store that + // did not re-derive the watermark would re-serve the stranded blocks 2..4. + db3, err := NewBlockDB(strandingConfig(t, dir, 8)) + require.NoError(t, err) + defer func() { _ = db3.Close() }() + impl := db3.(*blockDB) + + // The stranding really materialized on disk: block 2 is physically present + // but its covering QC's key is gone (reclaimed with segment 0). Blocks 0 and + // 1 were in the reclaimed segment and are physically gone. + require.True(t, physicallyPresent(t, impl, blockKey(2)), "block 2 must be physically stranded on disk") + require.False(t, physicallyPresent(t, impl, qcKey(2)), "covering QC key for block 2 must be reclaimed") + require.False(t, physicallyPresent(t, impl, blockKey(0)), "block 0 must be physically reclaimed") + require.False(t, physicallyPresent(t, impl, blockKey(1)), "block 1 must be physically reclaimed") + + // The watermark is re-derived as the lowest surviving QC's First. + require.Equal(t, uint64(5), impl.watermark.Load(), "recovered watermark must be the lowest surviving QC's First") + + // The gate refuses the stranded blocks and their (absent) QCs. + for n := types.GlobalBlockNumber(0); n < 5; n++ { + blk, err := db3.ReadBlockByNumber(n) + require.NoError(t, err) + require.False(t, blk.IsPresent(), "stranded/pruned block %d must not be served", n) + qc, err := db3.ReadQCByBlockNumber(n) + require.NoError(t, err) + require.False(t, qc.IsPresent(), "QC for pruned block %d must not be served", n) + } + + // Every served block has a readable covering QC (the invariant). + for n := types.GlobalBlockNumber(5); n < 20; n++ { + blk, err := db3.ReadBlockByNumber(n) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "block %d must be served", n) + qc, err := db3.ReadQCByBlockNumber(n) + require.NoError(t, err) + require.True(t, qc.IsPresent(), "covering QC for served block %d must be readable", n) + } + + // The block iterator never yields a stranded block, and each yielded block + // has a covering QC. + it, err := db3.Blocks(false) + require.NoError(t, err) + defer func() { _ = it.Close() }() + for { + ok, err := it.Next() + require.NoError(t, err) + if !ok { + break + } + n := it.Number() + require.GreaterOrEqual(t, uint64(n), uint64(5), "iterator must not yield stranded block %d", n) + qc, err := db3.ReadQCByBlockNumber(n) + require.NoError(t, err) + require.True(t, qc.IsPresent(), "iterated block %d must have a covering QC", n) + } +} + +// TestLittblockReclaimsAcrossRestart verifies the durable reclamation path with a +// white-box check on the raw table: data written, then pruned past after a +// restart (which seals the segments it landed in), is physically collected by GC. +// A raw-table check is required because the read watermark now refuses +// below-watermark reads regardless of physical reclamation, so public reads alone +// can no longer distinguish "reclaimed" from "gated". +func TestLittblockReclaimsAcrossRestart(t *testing.T) { + dir := t.TempDir() + rng := utils.TestRngFromSeed(2) + + db, err := NewBlockDB(strandingConfig(t, dir, 8)) + require.NoError(t, err) + writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19 + require.NoError(t, db.Flush()) + require.NoError(t, db.Close()) + + // Reopen: the segments written above are now sealed and collectable. + db2, err := NewBlockDB(strandingConfig(t, dir, 8)) + require.NoError(t, err) + defer func() { _ = db2.Close() }() + impl := db2.(*blockDB) + + require.NoError(t, db2.PruneBefore(20)) // past every block + require.NoError(t, ForceGC(db2)) + + // Every block and QC is physically gone from the raw table. + for n := types.GlobalBlockNumber(0); n < 20; n++ { + require.False(t, physicallyPresent(t, impl, blockKey(n)), "block %d must be reclaimed", n) + require.False(t, physicallyPresent(t, impl, qcKey(n)), "QC key %d must be reclaimed", n) + } +} diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index 5210c3f8c5..4384d43cf3 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -70,6 +70,11 @@ var ErrBlockMissingQC = errors.New("block: WriteBlock without covering QC") // is written, then after a crash if B is persisted then A is also persisted. // - Since QCs must always be written before the blocks they cover, a persisted block is always covered // by a persisted QC, but a persisted QC may or may not have its covered blocks persisted. +// +// # A readable block always has a readable covering QC +// +// Pruning never leaves a block readable without its covering QC also being readable. And if a block becomes +// crash recoverable, its QC is gauranteed to also be crash recoverable. type BlockDB interface { // WriteBlock persists a finalized block at GlobalBlockNumber n. A // block for height n may only be written after a QC covering n has diff --git a/sei-tendermint/autobahn/types/testonly.go b/sei-tendermint/autobahn/types/testonly.go index 708d907957..53fe90fe8f 100644 --- a/sei-tendermint/autobahn/types/testonly.go +++ b/sei-tendermint/autobahn/types/testonly.go @@ -317,6 +317,17 @@ func GenFullCommitQC(rng utils.Rng) *FullCommitQC { } } +// GenFullCommitQCN generates a FullCommitQC carrying exactly n headers. Unlike a +// QC built via NewFullCommitQC, its header count is not reconciled against the +// embedded CommitQC's range — it is for tests that only exercise a QC's header +// count (e.g. a store's range accounting), not its signatures or verification. +func GenFullCommitQCN(rng utils.Rng, n int) *FullCommitQC { + return &FullCommitQC{ + qc: GenCommitQC(rng), + headers: utils.GenSliceN(rng, n, GenBlockHeader), + } +} + // GenTimeoutVote generates a random TimeoutVote. func GenTimeoutVote(rng utils.Rng) *TimeoutVote { return NewTimeoutVote(GenView(rng), utils.Some(GenViewNumber(rng))) From 69ce4c6435561ee52ddba48f1e5df758d77137e9 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 9 Jul 2026 10:32:33 -0500 Subject: [PATCH 05/10] make suggested changes --- sei-tendermint/autobahn/types/block_db.go | 34 ++++++++++------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index 4384d43cf3..11bdb466ca 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -73,8 +73,8 @@ var ErrBlockMissingQC = errors.New("block: WriteBlock without covering QC") // // # A readable block always has a readable covering QC // -// Pruning never leaves a block readable without its covering QC also being readable. And if a block becomes -// crash recoverable, its QC is gauranteed to also be crash recoverable. +// Pruning never leaves a block readable without its covering QC also being readable. And if a block becomes +// crash recoverable, its QC is guaranteed to also be crash recoverable. type BlockDB interface { // WriteBlock persists a finalized block at GlobalBlockNumber n. A // block for height n may only be written after a QC covering n has @@ -196,13 +196,12 @@ type BlockDB interface { // ReadBlockByNumber returns the block at GlobalBlockNumber n. // - // Returns utils.None if no block has been written at n. A pruned - // block MAY also return None, but this is not guaranteed: - // reclamation is asynchronous, so a below-watermark block may remain - // readable until it is actually reclaimed. Implementations must not - // block waiting for a future write — "not yet written" is reported as - // utils.None identical to "never written". Blocking semantics - // (wait for a write at n) live above this interface, in + // Returns utils.None if no block is readable at n — either because + // none was written at n or because it has been pruned. A pruned block + // may or may not still be readable; callers must not rely on either. + // Never blocks waiting for a future write: "not yet written" is + // reported as utils.None identical to "never written". Blocking + // semantics (wait for a write at n) live above this interface, in // data.State. ReadBlockByNumber(n GlobalBlockNumber) (utils.Option[*Block], error) @@ -211,10 +210,9 @@ type BlockDB interface { // same value as block.Header().Hash() for the block that was passed // to WriteBlock. // - // Returns utils.None if no such block has been written. A pruned - // block MAY also return None, but — as with ReadBlockByNumber — - // reclamation is asynchronous, so a pruned block may remain readable - // until it is actually reclaimed. Non-blocking. + // Returns utils.None if no such block is readable — either because + // none was written or because it has been pruned (see + // ReadBlockByNumber). Non-blocking. ReadBlockByHash(hash BlockHeaderHash) (utils.Option[BlockWithNumber], error) // ReadQCByBlockNumber returns the FullCommitQC whose @@ -223,12 +221,10 @@ type BlockDB interface { // blocks, the same *FullCommitQC is returned for every n in its // range. // - // Returns utils.None if no QC has been written that covers n. A QC - // whose range is below the retention watermark MAY also return None, - // but this is not guaranteed: reclamation is asynchronous and coarse, - // and a QC straddling the watermark is retained outright, so a - // below-watermark lookup may still resolve to a retained QC. Callers - // must not rely on below-watermark lookups missing. Non-blocking. + // Returns utils.None if no readable QC covers n — either because none + // was written that covers n or because it has been pruned. A pruned + // QC may or may not still be readable; callers must not rely on + // either. Non-blocking. ReadQCByBlockNumber(n GlobalBlockNumber) (utils.Option[*FullCommitQC], error) // Close releases resources held by the store. After Close returns, From fc8af7acf0a1fec971477140670958d7398456d0 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 9 Jul 2026 15:02:54 -0500 Subject: [PATCH 06/10] Don't delete last block --- sei-db/ledger_db/block/block_db_test.go | 142 ++++++++++++++++++ .../block/littblock/litt_block_db.go | 24 ++- .../littblock/litt_block_stranding_test.go | 125 ++++++++++++++- .../ledger_db/block/memblock/mem_block_db.go | 15 ++ sei-tendermint/autobahn/types/block_db.go | 5 + 5 files changed, 298 insertions(+), 13 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index f983c7f12c..9ba7976b4e 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -1,6 +1,7 @@ package block import ( + "fmt" "testing" "time" @@ -59,6 +60,8 @@ func TestBlockDB(t *testing.T) { t.Run("PruneStraddleRetainsQC", func(t *testing.T) { testPruneStraddleRetainsQC(t, impl.build) }) t.Run("PruneRefusesBelowWatermark", func(t *testing.T) { testPruneRefusesBelowWatermark(t, impl.build) }) t.Run("PruneIdempotentMonotonic", func(t *testing.T) { testPruneIdempotentMonotonic(t, impl.build) }) + t.Run("PruneNeverEmpties", func(t *testing.T) { testPruneNeverEmpties(t, impl.build) }) + t.Run("PruneQCAheadOfBlocks", func(t *testing.T) { testPruneQCAheadOfBlocks(t, impl.build) }) t.Run("WriteOrderRejected", func(t *testing.T) { testWriteOrderRejected(t, impl.build) }) t.Run("WriteOrderRejectedAfterRestart", func(t *testing.T) { testWriteOrderRejectedAfterRestart(t, impl.build) @@ -332,6 +335,145 @@ func testPruneIdempotentMonotonic(t *testing.T, build builder) { } } +// testPruneNeverEmpties asserts the store is never emptied by pruning and that +// pruning is monotonic around the newest cohort. Any request whose watermark +// would enter the newest block's cohort — from just past the cohort's first, +// through the newest block, to well beyond every block — is clamped to the +// cohort's first, so the whole newest cohort (and its shared QC) stays readable +// while everything below is gone. The clamp lands on the cohort's first, not +// merely the newest block: the covering QC is retained regardless and covers the +// entire cohort, so a larger n must never retain more. Holds across both +// implementations. +func testPruneNeverEmpties(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + require.GreaterOrEqual(t, len(batches), 2, "need a below-cohort batch plus the newest cohort") + last := batches[len(batches)-1] // the newest block's cohort + newest := last.next - 1 + require.Greater(t, len(last.blocks), 1, "need a multi-block cohort to exercise a within-cohort prune") + + // Every request lands the watermark inside (or past) the newest cohort: + // within the cohort, exactly at the newest block, and well past every block. + // All must clamp identically to the cohort's first. + for _, prune := range []types.GlobalBlockNumber{last.first + 1, newest, last.next + 1000} { + t.Run(fmt.Sprintf("prune=%d", prune), func(t *testing.T) { + db, _ := openFresh(t, build) + defer func() { _ = db.Close() }() + writeAll(t, db, batches) + + require.NoError(t, db.PruneBefore(prune)) + + // Every block in the newest cohort is still served on every read path. + for i, blk := range last.blocks { + n := last.first + gbn(i) + + byNum, err := db.ReadBlockByNumber(n) + require.NoError(t, err) + got, ok := byNum.Get() + require.True(t, ok, "block %d in the newest cohort must survive PruneBefore(%d)", n, prune) + require.Equal(t, blk.Header().Hash(), got.Header().Hash()) + + byHash, err := db.ReadBlockByHash(blk.Header().Hash()) + require.NoError(t, err) + bwn, ok := byHash.Get() + require.True(t, ok, "block %d must survive lookup by hash", n) + require.Equal(t, n, bwn.Number) + + qc, err := db.ReadQCByBlockNumber(n) + require.NoError(t, err) + require.True(t, qc.IsPresent(), "the QC covering the newest cohort must survive") + } + + // A block below the newest cohort is gone (clamped watermark refuses/removes it). + belowBatch := batches[len(batches)-2] + require.Less(t, belowBatch.first, last.first) + below, err := db.ReadBlockByNumber(belowBatch.first) + require.NoError(t, err) + require.False(t, below.IsPresent(), "blocks below the newest cohort must not be served") + + // The block iterator yields exactly the newest cohort, and the QC + // iterator exactly its covering QC. + var expected []types.GlobalBlockNumber + for i := range last.blocks { + expected = append(expected, last.first+gbn(i)) + } + blockIt, err := db.Blocks(false) + require.NoError(t, err) + defer func() { _ = blockIt.Close() }() + var blockNums []types.GlobalBlockNumber + for { + ok, err := blockIt.Next() + require.NoError(t, err) + if !ok { + break + } + blockNums = append(blockNums, blockIt.Number()) + } + require.Equal(t, expected, blockNums, + "exactly the newest cohort must remain after PruneBefore(%d)", prune) + + qcIt, err := db.QCs(false) + require.NoError(t, err) + defer func() { _ = qcIt.Close() }() + qcCount := 0 + for { + ok, err := qcIt.Next() + require.NoError(t, err) + if !ok { + break + } + fqc, err := qcIt.QC() + require.NoError(t, err) + require.Equal(t, last.first, fqc.QC().GlobalRange(committee).First, + "only the QC covering the newest cohort must remain") + qcCount++ + } + require.Equal(t, 1, qcCount, "exactly one QC (covering the newest cohort) must remain") + }) + } +} + +// testPruneQCAheadOfBlocks pins the min() guard in the prune clamp. QCs are +// written before the blocks they cover, so between writing a QC and its first +// block — and after a crash that persisted a QC but not its blocks — the newest +// QC starts above the newest block (latestQCStartBlock > lastBlockNumber). A +// prune-to-empty request must clamp to the newest actual block, not the newest +// QC's first: clamping to the latter would push the watermark past every written +// block and empty the store. This holds across both implementations. +func testPruneQCAheadOfBlocks(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + require.GreaterOrEqual(t, len(batches), 2, "need a filled cohort plus an unfilled newest QC") + db, _ := openFresh(t, build) + defer func() { _ = db.Close() }() + + // Fill the first cohort, then write only the QC of the second — no blocks in + // its range. Now latestQCStartBlock (b1.first) exceeds lastBlockNumber (the + // last block of b0), since QCs are contiguous (b1.first == b0.next). + b0 := batches[0] + require.NoError(t, db.WriteQC(b0.first, b0.next, b0.qc)) + for i, blk := range b0.blocks { + require.NoError(t, db.WriteBlock(b0.first+gbn(i), blk)) + } + b1 := batches[1] + require.NoError(t, db.WriteQC(b1.first, b1.next, b1.qc)) + require.Equal(t, b0.next, b1.first, "QCs must be contiguous for this setup") + + newest := b0.next - 1 // newest actual block; b1.first == b0.next > newest + + require.NoError(t, db.PruneBefore(b1.next+1000)) + + // The newest actual block and its covering QC are still served: the clamp + // used min(latestQCStartBlock, lastBlockNumber), not latestQCStartBlock — + // otherwise the watermark would sit above every written block. + blk, err := db.ReadBlockByNumber(newest) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "newest block %d must survive; the clamp must not pass it", newest) + qc, err := db.ReadQCByBlockNumber(newest) + require.NoError(t, err) + require.True(t, qc.IsPresent(), "covering QC of the newest block must survive") +} + // testIteratorSnapshot asserts that an iterator observes only the records present // when it was created — writes made afterward are invisible to it. func testIteratorSnapshot(t *testing.T, build builder) { diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 885d69bec4..43f1d21b39 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -41,6 +41,9 @@ type blockDB struct { lastBlockNumber types.GlobalBlockNumber hasQC bool lastQCNext types.GlobalBlockNumber + + // latestQCStartBlock is therecently written QC's starting block number. + latestQCStartBlock types.GlobalBlockNumber } // NewBlockDB opens (or creates) a LittDB-backed types.BlockDB from config. The @@ -129,6 +132,7 @@ func (s *blockDB) recoverCursors() error { if err != nil { return fmt.Errorf("failed to unmarshal newest qc: %w", err) } + s.latestQCStartBlock = lowerBound s.lastQCNext = lowerBound + types.GlobalBlockNumber(len(qc.Headers())) s.hasQC = true } @@ -233,21 +237,27 @@ func (s *blockDB) WriteQC( return fmt.Errorf("failed to put QC [%d,%d): %w", lowerBound, upperBound, err) } + s.latestQCStartBlock = lowerBound s.lastQCNext = upperBound s.hasQC = true return nil } func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { - for { - cur := s.watermark.Load() - if uint64(n) <= cur { - return nil - } - if s.watermark.CompareAndSwap(cur, uint64(n)) { - return nil + s.mu.Lock() + defer s.mu.Unlock() + if s.hasBlocks { + if ceiling := min(s.latestQCStartBlock, s.lastBlockNumber); n > ceiling { + n = ceiling } } + // Advance the watermark monotonically. mu serializes writers and PruneBefore + // is the only one, so a plain load/store suffices; the field stays atomic + // only because the GC filter and readers load it without holding mu. + if uint64(n) > s.watermark.Load() { + s.watermark.Store(uint64(n)) + } + return nil } // gcFilter marks a key in the shared ledger table as reclaimable, dispatching on diff --git a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go index 1d84226f76..c9e94add01 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go @@ -140,11 +140,17 @@ func TestLittblockStrandedBlockNotServedAfterRestart(t *testing.T) { } // TestLittblockReclaimsAcrossRestart verifies the durable reclamation path with a -// white-box check on the raw table: data written, then pruned past after a -// restart (which seals the segments it landed in), is physically collected by GC. -// A raw-table check is required because the read watermark now refuses +// white-box check on the raw table: data written, then pruned after a restart +// (which seals the segments it landed in), is physically collected by GC. A +// raw-table check is required because the read watermark now refuses // below-watermark reads regardless of physical reclamation, so public reads alone // can no longer distinguish "reclaimed" from "gated". +// +// PruneBefore(20) requests pruning past every block, but the never-empty +// invariant clamps it to the newest block's cohort — the QC[15,20) range, whose +// first is 15. Blocks 15..19 and QC[15,20) stay readable, pinning their segments +// (seg5={QC[15,20),b15,b16}, seg6={b17,b18,b19}) against GC. Every fully-below +// segment — blocks 0..14 and QCs [0,5),[5,10),[10,15) — is reclaimed. func TestLittblockReclaimsAcrossRestart(t *testing.T) { dir := t.TempDir() rng := utils.TestRngFromSeed(2) @@ -161,12 +167,119 @@ func TestLittblockReclaimsAcrossRestart(t *testing.T) { defer func() { _ = db2.Close() }() impl := db2.(*blockDB) - require.NoError(t, db2.PruneBefore(20)) // past every block + require.NoError(t, db2.PruneBefore(20)) // past every block; capped to 19 (never-empty) require.NoError(t, ForceGC(db2)) - // Every block and QC is physically gone from the raw table. - for n := types.GlobalBlockNumber(0); n < 20; n++ { + // Blocks 0..14 and their QC keys are physically reclaimed. + for n := types.GlobalBlockNumber(0); n < 15; n++ { require.False(t, physicallyPresent(t, impl, blockKey(n)), "block %d must be reclaimed", n) require.False(t, physicallyPresent(t, impl, qcKey(n)), "QC key %d must be reclaimed", n) } + + // The newest block and its covering QC survive the capped prune: the cap + // pins block 19's segment and the straddling QC[15,20) against GC. + for n := types.GlobalBlockNumber(15); n < 20; n++ { + require.True(t, physicallyPresent(t, impl, blockKey(n)), "block %d must survive the capped prune", n) + } + require.True(t, physicallyPresent(t, impl, qcKey(15)), "covering QC[15,20) must survive the capped prune") + + // The whole newest cohort is served through the public API: the clamp lands + // on the cohort's first (15), so blocks 15..19 and their covering QC are + // readable, while block 14 (just below the cohort) is refused. + for n := types.GlobalBlockNumber(15); n < 20; n++ { + blk, err := db2.ReadBlockByNumber(n) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "cohort block %d must remain readable after a prune-to-empty request", n) + qc, err := db2.ReadQCByBlockNumber(n) + require.NoError(t, err) + require.True(t, qc.IsPresent(), "QC covering cohort block %d must remain readable", n) + } + below, err := db2.ReadBlockByNumber(14) + require.NoError(t, err) + require.False(t, below.IsPresent(), "block 14 below the newest cohort must not be served") +} + +// TestLittblockPartiallyPrunedQCRangeRetained pins the partial-prune rule for a +// single QC's covered range: when the watermark falls strictly inside a QC's +// range, pruning some of the range's early blocks (but not all) retains the +// ENTIRE range on disk. The QC straddles the watermark so it can never be +// reclaimed, and because a QC is always written before the blocks it covers, its +// segment sits at or before every block in its range in write order. GC reclaims +// only a contiguous write-order prefix of segments, so pinning the QC's segment +// pins every later segment too — every block in the range survives, including the +// sub-watermark ones. The read gate, not physical reclamation, is what hides +// those sub-watermark blocks from callers. +// +// The test is written so it would FAIL if the straddle invariant were broken: +// - It first asserts the fully-below segment (seg0) IS reclaimed. LittDB never +// GCs the live mutable file, so this is what proves GC actually ran with +// teeth on this data — without it the retention assertions below would hold +// vacuously (nothing ever collected). The write→flush→close→reopen dance +// seals the mutable file the data landed in so GC can reach it. +// - If the QC were wrongly treated as reclaimable, GC would collect seg1 (it +// sits right after seg0 in the reclaimed prefix), deleting QC[5,10). Blocks +// 7..9 stay served (their own segments are pinned by their at/above-watermark +// keys), so ReadQCByBlockNumber(7..9) would then return None — a served block +// with no covering QC — and the read-semantics assertions would fail. +// +// Layout (MaxSegmentKeyCount = 8, as in TestLittblockStrandedBlockNotServedAfterRestart): +// seg0 = {QC[0,5), b0, b1}, seg1 = {b2, b3, b4, QC[5,10)}, seg2 = {b5, b6, b7, b8}, +// seg3 = {b9, ...}. Pruning to 7 makes seg0 fully collectable, but leaves +// QC[5,10)'s secondary keys 7,8,9 at or above the watermark, so seg1 is pinned +// and every segment after it with it — blocks 5..9 all survive. +func TestLittblockPartiallyPrunedQCRangeRetained(t *testing.T) { + dir := t.TempDir() + rng := utils.TestRngFromSeed(3) + + db, err := NewBlockDB(strandingConfig(t, dir, 8)) + require.NoError(t, err) + writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19; QC[5,10) covers blocks 5..9 + require.NoError(t, db.Flush()) + require.NoError(t, db.Close()) + + // Reopen to seal the mutable file the data landed in, then prune to 7 — + // strictly inside QC[5,10). + db2, err := NewBlockDB(strandingConfig(t, dir, 8)) + require.NoError(t, err) + defer func() { _ = db2.Close() }() + impl := db2.(*blockDB) + + require.NoError(t, db2.PruneBefore(7)) + require.NoError(t, ForceGC(db2)) + + // GC actually had teeth: the fully-below segment (seg0 = {QC[0,5), b0, b1}) is + // reclaimed. If this fails, the data never made it out of the mutable file and + // the retention assertions below would be vacuous. + require.False(t, physicallyPresent(t, impl, blockKey(0)), "block 0 (fully below watermark) must be reclaimed") + require.False(t, physicallyPresent(t, impl, blockKey(1)), "block 1 (fully below watermark) must be reclaimed") + require.False(t, physicallyPresent(t, impl, qcKey(0)), "QC[0,5) (entirely below watermark) must be reclaimed") + + // The straddling QC[5,10) is not reclaimed (primary key plus a secondary that + // sits above the watermark). + require.True(t, physicallyPresent(t, impl, qcKey(5)), "straddling QC[5,10) primary key must survive") + require.True(t, physicallyPresent(t, impl, qcKey(9)), "straddling QC[5,10) secondary key must survive") + + // Its entire covered range survives on disk — including blocks 5 and 6, which + // are below the watermark. Pinning the QC's segment pins the whole range. + for n := types.GlobalBlockNumber(5); n < 10; n++ { + require.True(t, physicallyPresent(t, impl, blockKey(n)), + "block %d in the partially pruned QC range must be retained on disk", n) + } + + // Read semantics over the straddled range: the sub-watermark blocks are + // refused (even though present on disk), the at-or-above blocks are served, + // and the covering QC is readable for every served block. + for n := types.GlobalBlockNumber(5); n < 7; n++ { + blk, err := db2.ReadBlockByNumber(n) + require.NoError(t, err) + require.False(t, blk.IsPresent(), "sub-watermark block %d must not be served despite surviving on disk", n) + } + for n := types.GlobalBlockNumber(7); n < 10; n++ { + blk, err := db2.ReadBlockByNumber(n) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "block %d at/above watermark must be served", n) + qc, err := db2.ReadQCByBlockNumber(n) + require.NoError(t, err) + require.True(t, qc.IsPresent(), "covering QC for served block %d must be readable", n) + } } diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index d1481c2f89..c4cf29178a 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -41,6 +41,11 @@ type blockDB struct { lastBlockNumber types.GlobalBlockNumber hasQC bool lastQCNext types.GlobalBlockNumber + + // latestQCStartBlock is the most recently written QC's starting block number — + // the lowest block number in the newest cohort. PruneBefore clamps to it (see + // littblock). + latestQCStartBlock types.GlobalBlockNumber } // NewBlockDB returns an in-memory types.BlockDB. @@ -87,6 +92,7 @@ func (s *blockDB) WriteQC( lowerBound, s.lastQCNext, types.ErrQCNonContiguous) } s.qcsByLower[lowerBound] = qcEntry{qc: qc, lower: lowerBound, upper: upperBound} + s.latestQCStartBlock = lowerBound s.lastQCNext = upperBound s.hasQC = true return nil @@ -95,6 +101,15 @@ func (s *blockDB) WriteQC( func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { s.mu.Lock() defer s.mu.Unlock() + // Never let the watermark enter the newest block's cohort: clamp its ceiling + // at the cohort's first block (latestQCStartBlock), guarded by lastBlockNumber + // for a QC written ahead of its blocks. Keeps the newest cohort whole and + // pruning monotonic. See littblock and the BlockDB PruneBefore contract. + if s.hasBlocks { + if ceiling := min(s.latestQCStartBlock, s.lastBlockNumber); n > ceiling { + n = ceiling + } + } for num, blk := range s.byNumber { if num < n { delete(s.byNumber, num) diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index 11bdb466ca..f0b79c09c8 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -127,6 +127,11 @@ type BlockDB interface { // Idempotent: calling with n ≤ the existing retention watermark is // a no-op; the watermark only advances. // + // Pruning never empties the store. Once a block has been written, at + // least one block (and a QC covering it) always remains readable — a + // request that would remove every block is capped to retain the most + // recently written block (and the QC covering it). + // // Pruning is asynchronous and MAY BE DELAYED. PruneBefore records the // watermark and returns; reclamation happens later, on the // implementation's own schedule and potentially at a coarse From dab4004a2513feb3cbae2577610f8fb175dff817 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 10 Jul 2026 08:20:41 -0500 Subject: [PATCH 07/10] better error if qcs are corrupted/lost --- .../block/littblock/litt_block_db.go | 11 ++++--- .../littblock/litt_block_stranding_test.go | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 43f1d21b39..32cb9249d5 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -42,7 +42,7 @@ type blockDB struct { hasQC bool lastQCNext types.GlobalBlockNumber - // latestQCStartBlock is therecently written QC's starting block number. + // latestQCStartBlock is the most recently written QC's starting block number. latestQCStartBlock types.GlobalBlockNumber } @@ -168,10 +168,13 @@ func (s *blockDB) recoverReadWatermark() error { return nil } - // No QC survives. Any block still present is therefore stranded, so refuse - // every one by setting the watermark just past the newest block. if s.hasBlocks { - s.watermark.Store(uint64(s.lastBlockNumber) + 1) + // No QC survives. The never-empty prune invariant guarantees at least one + // (block, QC) pair is always retained, so blocks-without-QC is unreachable + // through normal operation — it means the store is corrupt (e.g. a QC WAL + // file was removed out of band). Refuse to open rather than serve blocks we + // can no longer trust. + return fmt.Errorf("corrupt store: newest block %d has no surviving QC covering it", s.lastBlockNumber) } return nil } diff --git a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go index c9e94add01..c9151ab6e2 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go @@ -283,3 +283,32 @@ func TestLittblockPartiallyPrunedQCRangeRetained(t *testing.T) { require.True(t, qc.IsPresent(), "covering QC for served block %d must be readable", n) } } + +// TestLittblockRefusesToOpenWithStrandedBlocks verifies the corruption guard in +// recoverReadWatermark. The never-empty prune invariant guarantees at least one +// (block, QC) pair is always retained, so a store holding a block with no +// surviving QC is corrupt (e.g. a QC WAL file removed out of band). Rather than +// serve blocks it can no longer trust, the store refuses to open. +// +// The state is unreachable through the public API — WriteBlock rejects an +// uncovered block, and pruning never reclaims the newest cohort's QC — so the +// test injects it directly: a block primary key written straight to the raw +// table with no covering QC, exactly the on-disk shape corruption leaves behind. +func TestLittblockRefusesToOpenWithStrandedBlocks(t *testing.T) { + dir := t.TempDir() + rng := utils.TestRngFromSeed(3) + + db, err := NewBlockDB(strandingConfig(t, dir, 8)) + require.NoError(t, err) + impl := db.(*blockDB) + + // Bypass WriteBlock's QC guard and write a lone block to the raw table. + require.NoError(t, impl.table.Put(blockKey(5), encodeBlock(5, types.GenBlock(rng)))) + require.NoError(t, db.Flush()) + require.NoError(t, db.Close()) + + // Reopen: recovery finds the block but no QC, so it refuses to open. + _, err = NewBlockDB(strandingConfig(t, dir, 8)) + require.Error(t, err) + require.ErrorContains(t, err, "no surviving QC") +} From 4a42d669c309b4595a8733e72ef79411a1084b92 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 10 Jul 2026 08:45:32 -0500 Subject: [PATCH 08/10] make suggested changes --- sei-db/ledger_db/block/block_db_test.go | 23 +++++++++++++ .../block/littblock/litt_block_db.go | 14 +++++--- .../littblock/litt_block_stranding_test.go | 32 +++++++++++++++++++ 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 9ba7976b4e..41583a8c01 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -61,6 +61,9 @@ func TestBlockDB(t *testing.T) { t.Run("PruneRefusesBelowWatermark", func(t *testing.T) { testPruneRefusesBelowWatermark(t, impl.build) }) t.Run("PruneIdempotentMonotonic", func(t *testing.T) { testPruneIdempotentMonotonic(t, impl.build) }) t.Run("PruneNeverEmpties", func(t *testing.T) { testPruneNeverEmpties(t, impl.build) }) + t.Run("PruneEmptyStoreThenWriteBelow", func(t *testing.T) { + testPruneEmptyStoreThenWriteBelow(t, impl.build) + }) t.Run("PruneQCAheadOfBlocks", func(t *testing.T) { testPruneQCAheadOfBlocks(t, impl.build) }) t.Run("WriteOrderRejected", func(t *testing.T) { testWriteOrderRejected(t, impl.build) }) t.Run("WriteOrderRejectedAfterRestart", func(t *testing.T) { @@ -335,6 +338,26 @@ func testPruneIdempotentMonotonic(t *testing.T, build builder) { } } +// testPruneEmptyStoreThenWriteBelow asserts a prune on an empty store neither +// refuses nor reclaims data written afterward, even below the requested point. +// Regression for the empty-store watermark bug, and a memblock/littblock parity +// check: an empty-store prune must not advance a read/GC watermark past data that +// does not exist yet. +func testPruneEmptyStoreThenWriteBelow(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + db, _ := openFresh(t, build) + defer func() { _ = db.Close() }() + + // Prune above where we are about to write, while the store is still empty. + require.NoError(t, db.PruneBefore(batches[1].first)) + + // Blocks start at 0, below the prune point; all must remain readable. + writeAll(t, db, batches) + assertBlocksReadable(t, db, batches) + assertQCsReadable(t, db, committee, batches) +} + // testPruneNeverEmpties asserts the store is never emptied by pruning and that // pruning is monotonic around the newest cohort. Any request whose watermark // would enter the newest block's cohort — from just past the cohort's first, diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 32cb9249d5..34f3f455b3 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -249,11 +249,17 @@ func (s *blockDB) WriteQC( func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { s.mu.Lock() defer s.mu.Unlock() - if s.hasBlocks { - if ceiling := min(s.latestQCStartBlock, s.lastBlockNumber); n > ceiling { - n = ceiling - } + + if !s.hasBlocks { + // Ignore prune requests if we've not got any data yet. Simplifies several edge cases + // and is technically a legal implementation of the contract in the godocs. + return nil } + + if ceiling := min(s.latestQCStartBlock, s.lastBlockNumber); n > ceiling { + n = ceiling + } + // Advance the watermark monotonically. mu serializes writers and PruneBefore // is the only one, so a plain load/store suffices; the field stays atomic // only because the GC filter and readers load it without holding mu. diff --git a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go index c9151ab6e2..6acbd0d713 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go @@ -312,3 +312,35 @@ func TestLittblockRefusesToOpenWithStrandedBlocks(t *testing.T) { require.Error(t, err) require.ErrorContains(t, err, "no surviving QC") } + +// TestLittblockEmptyStorePruneDoesNotReclaimLaterWrites is the regression for the +// empty-store prune bug: a PruneBefore on a store with no blocks must not advance +// the watermark, or blocks later written below the requested point would be +// refused by the read gate and physically reclaimed by GC. The watermark must +// never outrun the data it protects. +func TestLittblockEmptyStorePruneDoesNotReclaimLaterWrites(t *testing.T) { + dir := t.TempDir() + rng := utils.TestRngFromSeed(4) + + db, err := NewBlockDB(strandingConfig(t, dir, 8)) + require.NoError(t, err) + defer func() { _ = db.Close() }() + impl := db.(*blockDB) + + // Prune well past where we are about to write, while the store is still empty. + require.NoError(t, db.PruneBefore(1000)) + require.Equal(t, uint64(0), impl.watermark.Load(), "empty-store prune must not advance the watermark") + + // Write blocks 0..9 (below the pruned point) and force a GC pass. + writeSyntheticBatches(t, db, rng, 2, 5) // blocks 0..9; QCs [0,5),[5,10) + require.NoError(t, ForceGC(db)) + require.NoError(t, db.Flush()) + + // Every written block survives GC on disk and is served by the read gate. + for n := types.GlobalBlockNumber(0); n < 10; n++ { + require.True(t, physicallyPresent(t, impl, blockKey(n)), "block %d must survive GC", n) + blk, err := db.ReadBlockByNumber(n) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "block %d must be served", n) + } +} From 1ff9204baa0593ba0da34ef4d48377755c75b812 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 10 Jul 2026 14:13:13 -0500 Subject: [PATCH 09/10] fixed bug --- sei-db/ledger_db/block/block_db_test.go | 34 +++++++++++++++++++ .../ledger_db/block/memblock/mem_block_db.go | 11 +++--- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 41583a8c01..269c521c3b 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -65,6 +65,9 @@ func TestBlockDB(t *testing.T) { testPruneEmptyStoreThenWriteBelow(t, impl.build) }) t.Run("PruneQCAheadOfBlocks", func(t *testing.T) { testPruneQCAheadOfBlocks(t, impl.build) }) + t.Run("PruneQCOnlyThenWriteBlock", func(t *testing.T) { + testPruneQCOnlyThenWriteBlock(t, impl.build) + }) t.Run("WriteOrderRejected", func(t *testing.T) { testWriteOrderRejected(t, impl.build) }) t.Run("WriteOrderRejectedAfterRestart", func(t *testing.T) { testWriteOrderRejectedAfterRestart(t, impl.build) @@ -497,6 +500,37 @@ func testPruneQCAheadOfBlocks(t *testing.T, build builder) { require.True(t, qc.IsPresent(), "covering QC of the newest block must survive") } +// testPruneQCOnlyThenWriteBlock asserts that pruning while QCs exist but no +// blocks have been written yet does not delete the covering QC. A subsequent +// WriteBlock still passes its coverage check, so deleting the QC here would +// strand a readable block with no readable covering QC. Regression for the +// memblock PruneBefore fall-through (the clamp was guarded by hasBlocks but the +// deletion loops ran regardless); littblock returns early on !hasBlocks. +func testPruneQCOnlyThenWriteBlock(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + db, _ := openFresh(t, build) + defer func() { _ = db.Close() }() + + // Write only the QC of the first cohort — no blocks yet (hasQC, !hasBlocks). + b0 := batches[0] + require.NoError(t, db.WriteQC(b0.first, b0.next, b0.qc)) + + // Prune far past the QC. With no blocks, this must be a no-op; the QC cannot + // be deleted or a later covered WriteBlock would be orphaned. + require.NoError(t, db.PruneBefore(b0.next+1000)) + + // The block is still within [b0.first, b0.next), so its coverage check passes. + require.NoError(t, db.WriteBlock(b0.first, b0.blocks[0])) + + blk, err := db.ReadBlockByNumber(b0.first) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "block %d must be readable after write", b0.first) + qc, err := db.ReadQCByBlockNumber(b0.first) + require.NoError(t, err) + require.True(t, qc.IsPresent(), "covering QC of block %d must survive the earlier prune", b0.first) +} + // testIteratorSnapshot asserts that an iterator observes only the records present // when it was created — writes made afterward are invisible to it. func testIteratorSnapshot(t *testing.T, build builder) { diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index c4cf29178a..91445758c7 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -101,14 +101,17 @@ func (s *blockDB) WriteQC( func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { s.mu.Lock() defer s.mu.Unlock() + if !s.hasBlocks { + // No blocks yet: nothing to prune, and deleting QCs here would strand a + // future block whose coverage check still passes. Mirrors littblock. + return nil + } // Never let the watermark enter the newest block's cohort: clamp its ceiling // at the cohort's first block (latestQCStartBlock), guarded by lastBlockNumber // for a QC written ahead of its blocks. Keeps the newest cohort whole and // pruning monotonic. See littblock and the BlockDB PruneBefore contract. - if s.hasBlocks { - if ceiling := min(s.latestQCStartBlock, s.lastBlockNumber); n > ceiling { - n = ceiling - } + if ceiling := min(s.latestQCStartBlock, s.lastBlockNumber); n > ceiling { + n = ceiling } for num, blk := range s.byNumber { if num < n { From 8e63c26e65630fae51195cde52747cde5c8602da Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 10 Jul 2026 14:52:04 -0500 Subject: [PATCH 10/10] fix merge bug --- sei-db/ledger_db/block/block_db_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 762117e45c..74e9e0b016 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -450,7 +450,7 @@ func testPruneNeverEmpties(t *testing.T, build builder) { } fqc, err := qcIt.QC() require.NoError(t, err) - require.Equal(t, last.first, fqc.QC().GlobalRange(committee).First, + require.Equal(t, last.first, fqc.QC().GlobalRange().First, "only the QC covering the newest cohort must remain") qcCount++ }