Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions sei-db/ledger_db/block/block_db_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -632,9 +632,10 @@ func testWriteBlockGaps(t *testing.T, build builder) {

byHash, err := db.ReadBlockByHash(blocks[n].Header().Hash())
require.NoError(t, err)
got, ok = byHash.Get()
bwn, ok := byHash.Get()
require.True(t, ok, "block %d should be found by hash", n)
require.Equal(t, blocks[n].Header().Hash(), got.Header().Hash())
require.Equal(t, blocks[n].Header().Hash(), bwn.Block.Header().Hash())
require.Equal(t, n, bwn.Number, "block %d hash lookup should return its number", n)
}

// Numbers in the gaps were never written and must miss.
Expand Down Expand Up @@ -833,9 +834,10 @@ func assertBlocksReadable(t *testing.T, db types.BlockDB, batches []batch) {

byHash, err := db.ReadBlockByHash(blk.Header().Hash())
require.NoError(t, err)
got, ok = byHash.Get()
bwn, ok := byHash.Get()
require.True(t, ok, "block by hash should exist")
require.Equal(t, blk.Header().Hash(), got.Header().Hash())
require.Equal(t, blk.Header().Hash(), bwn.Block.Header().Hash())
require.Equal(t, n, bwn.Number, "block %d hash lookup should return its number", n)
}
}
}
Expand Down
66 changes: 56 additions & 10 deletions sei-db/ledger_db/block/littblock/codec.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package littblock

import (
"encoding/binary"
"fmt"

"github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types"
)
Expand Down Expand Up @@ -63,22 +64,67 @@ func decodeNumberKey(key []byte) types.GlobalBlockNumber {
return decodeKey(key[1:])
}

// encodeBlock marshals a block to the bytes stored as its table value.
func encodeBlock(blk *types.Block) []byte {
return types.BlockConv.Marshal(blk)
// Serialization version for blocks.
const blockSerializationVersion byte = 1
Comment thread
claude[bot] marked this conversation as resolved.

// Serialization version for QCs.
const qcSerializationVersion byte = 1

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do we need qcSerializationVersion to return a block number?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't. I just added a 1 byte serialization version to the data schema since it's good practice to encode serialization versions (and it was originally missing). This way, if we ever decide to alter the serialization schema for a QC in the future, we will be able to do so with little headache.


// blockValuePrefixLen is the fixed header preceding a block's proto bytes: one
// version byte followed by the 8-byte big-endian GlobalBlockNumber.
const blockValuePrefixLen = 1 + 8
Comment thread
wen-coding marked this conversation as resolved.

// encodeBlock marshals a block to the bytes stored as its table value. The value
// is framed as [version:1][GlobalBlockNumber:8 big-endian][proto(Block)]. The
// number is embedded so a by-hash lookup — which reaches this same shared value
// through a secondary key that carries only the hash — can still recover it.
func encodeBlock(n types.GlobalBlockNumber, blk *types.Block) []byte {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought we didn't have to encode block number into the block because it was the key. Would this hurt the performance? I guess not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is in the primary key, but it's not in the secondary key. LittDB keeps no map from secondary keys to primary keys, and so if we want this data when we get by block hash, it needs to go into the value data.

Technically yes, this does add some overhead. But in practical terms, this overhead is negligible. We're adding 8 bytes onto a 2 megabyte block (this is the size of the blocks Greg asked me to simulate in my benchmark).

proto := types.BlockConv.Marshal(blk)
value := make([]byte, 0, blockValuePrefixLen+len(proto))
value = append(value, blockSerializationVersion)
value = binary.BigEndian.AppendUint64(value, uint64(n))
value = append(value, proto...)
return value
}

// decodeBlock unmarshals a block from its stored table value.
func decodeBlock(value []byte) (*types.Block, error) {
return types.BlockConv.Unmarshal(value)
// decodeBlock unmarshals a block and its embedded GlobalBlockNumber from the
// value produced by encodeBlock.
func decodeBlock(value []byte) (types.GlobalBlockNumber, *types.Block, error) {
if len(value) < blockValuePrefixLen {
return 0, nil, fmt.Errorf("block value too short: %d bytes", len(value))
}
if value[0] != blockSerializationVersion {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[suggestion] This introduces a versioned on-disk value format ([version][number][proto] for blocks, and [version][proto] for QCs at line 122). Any block/QC value persisted by a prior build was raw protobuf with no version byte, so this check will reject it on upgrade — and since recoverCursors in NewBlockDB decodes the newest persisted QC, an existing store would fail to open. This looks safe only because littblock.NewBlockDB doesn't appear to be wired into any production path yet. Please confirm there is no persisted littblock data in the field; otherwise a migration or a legacy (unversioned) decode fallback is needed.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no production use yet

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[suggestion] This rejects any value whose first byte isn't the current version, including the old unversioned raw-proto format written by the base code. Since littblock is currently only used by blocksim and tests (no live-node consumers of ReadBlockByHash/NewBlockDB), establishing v1 cleanly is reasonable. But please confirm no environment has already persisted littblock blocks/QCs in the old format — otherwise reads will fail post-upgrade and you'd need an unversioned-fallback decode path here (and in decodeQC).

return 0, nil, fmt.Errorf("unsupported block serialization version %d", value[0])
}
n := types.GlobalBlockNumber(binary.BigEndian.Uint64(value[1:blockValuePrefixLen]))
blk, err := types.BlockConv.Unmarshal(value[blockValuePrefixLen:])
if err != nil {
return 0, nil, fmt.Errorf("failed to unmarshal block: %w", err)
}
return n, blk, nil
}

// encodeQC marshals a FullCommitQC to the bytes stored as its table value.
// encodeQC marshals a FullCommitQC to the bytes stored as its table value,
// framed as [version:1][proto(FullCommitQC)].
func encodeQC(qc *types.FullCommitQC) []byte {
return types.FullCommitQCConv.Marshal(qc)
proto := types.FullCommitQCConv.Marshal(qc)
value := make([]byte, 0, 1+len(proto))
value = append(value, qcSerializationVersion)
value = append(value, proto...)
return value
}

// decodeQC unmarshals a FullCommitQC from its stored table value.
// decodeQC unmarshals a FullCommitQC from the value produced by encodeQC.
func decodeQC(value []byte) (*types.FullCommitQC, error) {
return types.FullCommitQCConv.Unmarshal(value)
if len(value) < 1 {
return nil, fmt.Errorf("qc value too short: %d bytes", len(value))
}
if value[0] != qcSerializationVersion {
return nil, fmt.Errorf("unsupported qc serialization version %d", value[0])
}
qc, err := types.FullCommitQCConv.Unmarshal(value[1:])
if err != nil {
return nil, fmt.Errorf("failed to unmarshal qc: %w", err)
}
return qc, nil
}
35 changes: 28 additions & 7 deletions sei-db/ledger_db/block/littblock/codec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,15 +66,19 @@ func TestPrefixedKeys(t *testing.T) {

func TestBlockRoundTrip(t *testing.T) {
rng := utils.TestRngFromSeed(1)
for range 16 {
for i := range 16 {
n := types.GlobalBlockNumber(i)
blk := types.GenBlock(rng)
value := encodeBlock(blk)
decoded, err := decodeBlock(value)
value := encodeBlock(n, blk)
require.Equal(t, blockSerializationVersion, value[0], "value must be version-prefixed")
gotN, decoded, err := decodeBlock(value)
require.NoError(t, err)
// The embedded block number must round-trip.
require.Equal(t, n, gotN)
// Header hash uniquely identifies a block; equal hash => same block.
require.Equal(t, blk.Header().Hash(), decoded.Header().Hash())
// Re-encoding the decoded block must reproduce the same bytes.
require.Equal(t, value, encodeBlock(decoded))
// Re-encoding the decoded block (with the same number) must reproduce the same bytes.
require.Equal(t, value, encodeBlock(gotN, decoded))
}
}

Expand All @@ -83,6 +87,7 @@ func TestQCRoundTrip(t *testing.T) {
for range 16 {
qc := types.GenFullCommitQC(rng)
value := encodeQC(qc)
require.Equal(t, qcSerializationVersion, value[0], "value must be version-prefixed")
decoded, err := decodeQC(value)
require.NoError(t, err)
// Re-encoding the decoded QC must reproduce the same bytes.
Expand All @@ -91,10 +96,26 @@ func TestQCRoundTrip(t *testing.T) {
}

func TestDecodeRejectsGarbage(t *testing.T) {
// Invalid protobuf wire bytes must surface an error rather than a partial value.
// Invalid bytes must surface an error rather than a partial value.
garbage := []byte{0xff, 0xff, 0xff, 0xff}
_, blockErr := decodeBlock(garbage)
_, _, blockErr := decodeBlock(garbage)
require.Error(t, blockErr)
_, qcErr := decodeQC(garbage)
require.Error(t, qcErr)
}

func TestDecodeRejectsUnknownVersion(t *testing.T) {
rng := utils.TestRngFromSeed(3)

// A block value whose version byte is not the current one must be rejected,
// even though the rest of the value is well-formed.
blockValue := encodeBlock(1, types.GenBlock(rng))
blockValue[0] = blockSerializationVersion + 1
_, _, blockErr := decodeBlock(blockValue)
require.Error(t, blockErr)

qcValue := encodeQC(types.GenFullCommitQC(rng))
qcValue[0] = qcSerializationVersion + 1
_, qcErr := decodeQC(qcValue)
require.Error(t, qcErr)
}
31 changes: 17 additions & 14 deletions sei-db/ledger_db/block/littblock/litt_block_db.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error
n, s.lastQCNext, types.ErrBlockMissingQC)
}

value := encodeBlock(blk)
value := encodeBlock(n, blk)
hash := blk.Header().Hash()
hashAlias := &litttypes.SecondaryKey{
Key: blockHashKey(hash),
Expand Down Expand Up @@ -252,31 +252,34 @@ func (s *blockDB) QCs(reverse bool) (types.QCIterator, error) {
return &qcIterator{it: it}, nil
}

func (s *blockDB) ReadBlockByNumber(
n types.GlobalBlockNumber,
) (utils.Option[*types.Block], error) {
return getBlock(s.table, blockKey(n))
func (s *blockDB) ReadBlockByNumber(n types.GlobalBlockNumber) (utils.Option[*types.Block], error) {
result, err := getBlock(s.table, blockKey(n))
if err != nil {
return utils.None[*types.Block](), err
}
if bwn, ok := result.Get(); ok {
return utils.Some(bwn.Block), nil
}
return utils.None[*types.Block](), nil
}

func (s *blockDB) ReadBlockByHash(
hash types.BlockHeaderHash,
) (utils.Option[*types.Block], error) {
func (s *blockDB) ReadBlockByHash(hash types.BlockHeaderHash) (utils.Option[types.BlockWithNumber], error) {
return getBlock(s.table, blockHashKey(hash))
}

func getBlock(table littdb.Table, key []byte) (utils.Option[*types.Block], error) {
func getBlock(table littdb.Table, key []byte) (utils.Option[types.BlockWithNumber], error) {
value, exists, err := table.Get(key)
if err != nil {
return utils.None[*types.Block](), fmt.Errorf("failed to read block: %w", err)
return utils.None[types.BlockWithNumber](), fmt.Errorf("failed to read block: %w", err)
}
if !exists {
return utils.None[*types.Block](), nil
return utils.None[types.BlockWithNumber](), nil
}
blk, err := decodeBlock(value)
n, blk, err := decodeBlock(value)
if err != nil {
return utils.None[*types.Block](), fmt.Errorf("failed to unmarshal block: %w", err)
return utils.None[types.BlockWithNumber](), fmt.Errorf("failed to unmarshal block: %w", err)
}
return utils.Some(blk), nil
return utils.Some(types.BlockWithNumber{Block: blk, Number: n}), nil
}

func (s *blockDB) ReadQCByBlockNumber(
Expand Down
2 changes: 1 addition & 1 deletion sei-db/ledger_db/block/littblock/litt_block_iterator.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func (b *blockIterator) Block() (*types.Block, error) {
if err != nil {
return nil, fmt.Errorf("failed to read block value: %w", err)
}
blk, err := decodeBlock(value)
_, blk, err := decodeBlock(value)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal block: %w", err)
}
Expand Down
22 changes: 15 additions & 7 deletions sei-db/ledger_db/block/memblock/mem_block_db.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,21 @@ type qcEntry struct {
upper types.GlobalBlockNumber
}

// hashEntry pairs a block with its GlobalBlockNumber so ReadBlockByHash can
// return the number, mirroring the littblock implementation which embeds it in
// the stored value.
type hashEntry struct {
blk *types.Block
n types.GlobalBlockNumber
}

// blockDB is an in-memory types.BlockDB. It holds blocks and QCs by pointer (no
// marshaling) and is intended as a test/benchmark fixture, not a durable
// implementation.
type blockDB struct {
mu sync.RWMutex
byNumber map[types.GlobalBlockNumber]*types.Block
byHash map[types.BlockHeaderHash]*types.Block
byHash map[types.BlockHeaderHash]hashEntry
qcsByLower map[types.GlobalBlockNumber]qcEntry

// Write-order cursors (see types.BlockDB contract).
Expand All @@ -39,7 +47,7 @@ type blockDB struct {
func NewBlockDB() types.BlockDB {
return &blockDB{
byNumber: make(map[types.GlobalBlockNumber]*types.Block),
byHash: make(map[types.BlockHeaderHash]*types.Block),
byHash: make(map[types.BlockHeaderHash]hashEntry),
qcsByLower: make(map[types.GlobalBlockNumber]qcEntry),
}
}
Expand All @@ -58,7 +66,7 @@ func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error
n, s.lastQCNext, types.ErrBlockMissingQC)
}
s.byNumber[n] = blk
s.byHash[blk.Header().Hash()] = blk
s.byHash[blk.Header().Hash()] = hashEntry{blk: blk, n: n}
s.lastBlockNumber = n
s.hasBlocks = true
return nil
Expand Down Expand Up @@ -193,13 +201,13 @@ func (s *blockDB) ReadBlockByNumber(

func (s *blockDB) ReadBlockByHash(
hash types.BlockHeaderHash,
) (utils.Option[*types.Block], error) {
) (utils.Option[types.BlockWithNumber], error) {
s.mu.RLock()
defer s.mu.RUnlock()
if blk, ok := s.byHash[hash]; ok {
return utils.Some(blk), nil
if e, ok := s.byHash[hash]; ok {
return utils.Some(types.BlockWithNumber{Block: e.blk, Number: e.n}), nil
}
return utils.None[*types.Block](), nil
return utils.None[types.BlockWithNumber](), nil
}

func (s *blockDB) ReadQCByBlockNumber(
Expand Down
8 changes: 8 additions & 0 deletions sei-tendermint/autobahn/types/block.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,14 @@ type BlockNumber uint64
// GlobalBlockNumber is the number of a block in the global chain.
type GlobalBlockNumber uint64

// BlockWithNumber pairs a block with its GlobalBlockNumber. It is used as the
// payload of the utils.Option returned by ReadBlockByHash so that the block
// number is only present when the block itself is present.
type BlockWithNumber struct {
Block *Block
Number GlobalBlockNumber
}

// BlockHeaderHash is the hash of a BlockHeader.
type BlockHeaderHash hashable.Hash[*pb.BlockHeader]

Expand Down
7 changes: 4 additions & 3 deletions sei-tendermint/autobahn/types/block_db.go
Original file line number Diff line number Diff line change
Expand Up @@ -202,14 +202,15 @@ type BlockDB interface {
ReadBlockByNumber(n GlobalBlockNumber) (utils.Option[*Block], error)

// ReadBlockByHash returns the block whose header hashes to the
// given value. The hash is the same value as block.Header().Hash()
// for the block that was passed to WriteBlock.
// given value, paired with its GlobalBlockNumber. The hash is the
// same value as block.Header().Hash() for the block that was passed
// to WriteBlock.
//
// Returns utils.None if no such block has been written. A pruned
// block MAY also return None, but — as with ReadBlockByNumber —
// reclamation is asynchronous, so a pruned block may remain readable
// until it is actually reclaimed. Non-blocking.
ReadBlockByHash(hash BlockHeaderHash) (utils.Option[*Block], error)
ReadBlockByHash(hash BlockHeaderHash) (utils.Option[BlockWithNumber], error)

// ReadQCByBlockNumber returns the FullCommitQC whose
// GlobalRange().First ≤ n < GlobalRange().Next — i.e. the QC that
Expand Down
Loading