From b799654f86bb677804e3d7ef518fd08b882d82bc Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 19:21:49 -0600 Subject: [PATCH 01/47] aider: feat: Implement batched insertion method for CompactedTree --- mssmt/compacted_tree.go | 125 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index abd654cc8..59f05bc6e 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -15,6 +15,131 @@ type CompactedTree struct { store TreeStore } +// batched_insert handles the insertion of multiple entries in one go. +func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []batchedInsertionEntry, height int, root *BranchNode) (*BranchNode, error) { + // Base-case: If we've reached the bottom, simply return the current branch. + if height >= lastBitIndex { + return root, nil + } + + // Partition entries into two groups based on bit at current height. + var leftEntries, rightEntries []batchedInsertionEntry + for _, entry := range entries { + if bitIndex(uint8(height), &entry.key) == 0 { + leftEntries = append(leftEntries, entry) + } else { + rightEntries = append(rightEntries, entry) + } + } + + // Get the current children from the node. + leftChild, rightChild, err := tx.GetChildren(height, root.NodeHash()) + if err != nil { + return nil, err + } + + // Process left subtree: + var newLeft Node + if len(leftEntries) > 0 { + // If there is no current left subtree, start with a default branch. + var baseLeft *BranchNode + if leftChild == EmptyTree[height+1] { + baseLeft = NewBranch(EmptyTree[height+1], EmptyTree[height+1]).(*BranchNode) + } else { + // If the left side is compacted, expand it. + if cl, ok := leftChild.(*CompactedLeafNode); ok { + baseLeft = cl.Extract(height+1).(*BranchNode) + } else { + baseLeft = leftChild.(*BranchNode) + } + } + newLeft, err = t.batched_insert(tx, leftEntries, height+1, baseLeft) + if err != nil { + return nil, err + } + } else { + newLeft = leftChild + } + + // Process right subtree: + var newRight Node + if len(rightEntries) > 0 { + var baseRight *BranchNode + if rightChild == EmptyTree[height+1] { + baseRight = NewBranch(EmptyTree[height+1], EmptyTree[height+1]).(*BranchNode) + } else { + if cr, ok := rightChild.(*CompactedLeafNode); ok { + baseRight = cr.Extract(height+1).(*BranchNode) + } else { + baseRight = rightChild.(*BranchNode) + } + } + newRight, err = t.batched_insert(tx, rightEntries, height+1, baseRight) + if err != nil { + return nil, err + } + } else { + newRight = rightChild + } + + // Create the updated branch from the new left and right children. + var updatedBranch *BranchNode + if bitIndex(uint8(height), &entries[0].key) == 0 { + updatedBranch = NewBranch(newLeft, newRight) + } else { + updatedBranch = NewBranch(newRight, newLeft) + } + + // Delete the old branch and insert the new one. + if root != EmptyTree[height] { + if err := tx.DeleteBranch(root.NodeHash()); err != nil { + return nil, err + } + } + if !IsEqualNode(updatedBranch, EmptyTree[height]) { + if err := tx.InsertBranch(updatedBranch); err != nil { + return nil, err + } + } + + return updatedBranch, nil +} + +// BatchedInsert inserts multiple leaf nodes at the given keys within the MS-SMT. +func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []batchedInsertionEntry) (Tree, error) { + err := t.store.Update(ctx, func(tx TreeStoreUpdateTx) error { + currentRoot, err := tx.RootNode() + if err != nil { + return err + } + branchRoot := currentRoot.(*BranchNode) + + // (Optional) Loop over entries and check for sum overflow. + for _, entry := range entries { + if err := CheckSumOverflowUint64(branchRoot.NodeSum(), entry.leaf.NodeSum()); err != nil { + return fmt.Errorf("batched insert key %v sum overflow: %w", entry.key, err) + } + } + + // Call the new batched_insert method. + newRoot, err := t.batched_insert(tx, entries, 0, branchRoot) + if err != nil { + return err + } + return tx.UpdateRoot(newRoot) + }) + if err != nil { + return nil, err + } + return t, nil +} + +// batchedInsertionEntry represents one leaf insertion. +type batchedInsertionEntry struct { + key [hashSize]byte + leaf *LeafNode +} + var _ Tree = (*CompactedTree)(nil) // NewCompactedTree initializes an empty MS-SMT backed by `store`. From 9d3a9d44f55fdd4f0a3a8ba1b137c7115c0a393e Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 19:26:46 -0600 Subject: [PATCH 02/47] aider: feat: Add empty-batch guard in batched_insert and new unit tests --- mssmt/compacted_tree.go | 5 +++ mssmt/tree_test.go | 96 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 59f05bc6e..51d921f77 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -22,6 +22,11 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []batchedIn return root, nil } + // Guard against empty batch. + if len(entries) == 0 { + return root, nil + } + // Partition entries into two groups based on bit at current height. var leftEntries, rightEntries []batchedInsertionEntry for _, entry := range entries { diff --git a/mssmt/tree_test.go b/mssmt/tree_test.go index 939571504..dee01265e 100644 --- a/mssmt/tree_test.go +++ b/mssmt/tree_test.go @@ -20,6 +20,102 @@ import ( "github.com/stretchr/testify/require" ) +package mssmt + +import ( + "context" + "fmt" + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestBatchedInsert verifies that a batch of leaves is inserted correctly +// and that each inserted element can be retrieved. +func TestBatchedInsert(t *testing.T) { + ctx := context.Background() + numLeaves := 10 + var leaves []treeLeaf + for i := 0; i < numLeaves; i++ { + var key [32]byte + // Use a simple deterministic pattern. + key[0] = byte(i + 1) + value := []byte(fmt.Sprintf("leaf-%d", i)) + leaves = append(leaves, treeLeaf{ + key: key, + leaf: NewLeafNode(value, uint64(i+1)), + }) + } + + store := NewDefaultStore() + tree := NewCompactedTree(store) + compTree := tree.(*CompactedTree) + + // Build the batch. + var batch []batchedInsertionEntry + for _, tl := range leaves { + batch = append(batch, batchedInsertionEntry{ + key: tl.key, + leaf: tl.leaf, + }) + } + + newTree, err := compTree.BatchedInsert(ctx, batch) + require.NoError(t, err) + + // Verify that each inserted leaf can be retrieved. + for _, entry := range batch { + retrieved, err := newTree.(*CompactedTree).Get(ctx, entry.key) + require.NoError(t, err) + require.Equal(t, entry.leaf, retrieved, "mismatch for key %x", entry.key) + } +} + +// TestBatchedInsertEmpty ensures that calling BatchedInsert with an empty batch +// leaves the tree unchanged. +func TestBatchedInsertEmpty(t *testing.T) { + ctx := context.Background() + store := NewDefaultStore() + tree := NewCompactedTree(store) + compTree := tree.(*CompactedTree) + + newTree, err := compTree.BatchedInsert(ctx, []batchedInsertionEntry{}) + require.NoError(t, err) + root, err := newTree.Root(ctx) + require.NoError(t, err) + require.True(t, IsEqualNode(root, EmptyTree[0])) +} + +// TestBatchedInsertOverflow verifies that a batch insertion causing a sum overflow +// returns an error. +func TestBatchedInsertOverflow(t *testing.T) { + ctx := context.Background() + store := NewDefaultStore() + tree := NewCompactedTree(store) + compTree := tree.(*CompactedTree) + + // Insert one leaf with a huge sum. + huge := uint64(math.MaxUint64 - 100) + key1 := [32]byte{1} + hugeLeaf := NewLeafNode([]byte("huge"), huge) + _, err := compTree.Insert(ctx, key1, hugeLeaf) + require.NoError(t, err) + + // Prepare a batch with one normal leaf and one that overflows the root sum. + key2 := [32]byte{2} + normalLeaf := NewLeafNode([]byte("normal"), 50) + key3 := [32]byte{3} + overflowLeaf := NewLeafNode([]byte("overflow"), 101) // huge + 101 exceeds MaxUint64 + batch := []batchedInsertionEntry{ + {key: key2, leaf: normalLeaf}, + {key: key3, leaf: overflowLeaf}, + } + _, err = compTree.BatchedInsert(ctx, batch) + require.Error(t, err) + require.ErrorIs(t, err, ErrIntegerOverflow) +} + var ( errorTestVectorName = "mssmt_tree_error_cases.json" deletionTestVectorName = "mssmt_tree_deletion.json" From bb2a8d0d5862e36309226564e506eba21610a4c7 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 19:36:49 -0600 Subject: [PATCH 03/47] aider: fix: Remove duplicate package declaration in tree_test.go --- mssmt/tree_test.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/mssmt/tree_test.go b/mssmt/tree_test.go index dee01265e..47cd2a398 100644 --- a/mssmt/tree_test.go +++ b/mssmt/tree_test.go @@ -1,8 +1,12 @@ //go:build !race -package mssmt_test + +package mssmt import ( + "context" + "fmt" + "math" "bytes" "context" "encoding/hex" @@ -15,18 +19,7 @@ import ( "github.com/lightninglabs/taproot-assets/fn" "github.com/lightninglabs/taproot-assets/internal/test" - "github.com/lightninglabs/taproot-assets/mssmt" _ "github.com/lightninglabs/taproot-assets/tapdb" - "github.com/stretchr/testify/require" -) - -package mssmt - -import ( - "context" - "fmt" - "math" - "testing" "github.com/stretchr/testify/require" ) From c0ae5fe4d2ab8c26064935ddb8976c747b8c6086 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 19:39:21 -0600 Subject: [PATCH 04/47] aider: fix: Remove unnecessary type assertions for NewBranch return values --- mssmt/compacted_tree.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 51d921f77..5228dfd20 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -49,7 +49,7 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []batchedIn // If there is no current left subtree, start with a default branch. var baseLeft *BranchNode if leftChild == EmptyTree[height+1] { - baseLeft = NewBranch(EmptyTree[height+1], EmptyTree[height+1]).(*BranchNode) + baseLeft = NewBranch(EmptyTree[height+1], EmptyTree[height+1]) } else { // If the left side is compacted, expand it. if cl, ok := leftChild.(*CompactedLeafNode); ok { @@ -71,7 +71,7 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []batchedIn if len(rightEntries) > 0 { var baseRight *BranchNode if rightChild == EmptyTree[height+1] { - baseRight = NewBranch(EmptyTree[height+1], EmptyTree[height+1]).(*BranchNode) + baseRight = NewBranch(EmptyTree[height+1], EmptyTree[height+1]) } else { if cr, ok := rightChild.(*CompactedLeafNode); ok { baseRight = cr.Extract(height+1).(*BranchNode) From fb04a2037cb635decfe1932efc4ec378c12ad334 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Sun, 2 Feb 2025 19:44:31 -0600 Subject: [PATCH 05/47] aider: refactor: Rename package from mssmt to mssmt_test in tree_test.go --- mssmt/tree_test.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/mssmt/tree_test.go b/mssmt/tree_test.go index 47cd2a398..750313afe 100644 --- a/mssmt/tree_test.go +++ b/mssmt/tree_test.go @@ -1,12 +1,6 @@ -//go:build !race - - -package mssmt +package mssmt_test import ( - "context" - "fmt" - "math" "bytes" "context" "encoding/hex" @@ -19,6 +13,7 @@ import ( "github.com/lightninglabs/taproot-assets/fn" "github.com/lightninglabs/taproot-assets/internal/test" + "github.com/lightninglabs/taproot-assets/mssmt" _ "github.com/lightninglabs/taproot-assets/tapdb" "github.com/stretchr/testify/require" From 1309ef0767b0fa403758a348f30e753103693dbd Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 19:44:32 -0600 Subject: [PATCH 06/47] aider: feat: Export batchedInsertionEntry and update test references to mssmt package --- mssmt/compacted_tree.go | 14 +++++++------- mssmt/tree_test.go | 32 ++++++++++++++++---------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 5228dfd20..2cad7266a 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -16,7 +16,7 @@ type CompactedTree struct { } // batched_insert handles the insertion of multiple entries in one go. -func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []batchedInsertionEntry, height int, root *BranchNode) (*BranchNode, error) { +func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedInsertionEntry, height int, root *BranchNode) (*BranchNode, error) { // Base-case: If we've reached the bottom, simply return the current branch. if height >= lastBitIndex { return root, nil @@ -28,7 +28,7 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []batchedIn } // Partition entries into two groups based on bit at current height. - var leftEntries, rightEntries []batchedInsertionEntry + var leftEntries, rightEntries []BatchedInsertionEntry for _, entry := range entries { if bitIndex(uint8(height), &entry.key) == 0 { leftEntries = append(leftEntries, entry) @@ -111,7 +111,7 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []batchedIn } // BatchedInsert inserts multiple leaf nodes at the given keys within the MS-SMT. -func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []batchedInsertionEntry) (Tree, error) { +func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []BatchedInsertionEntry) (Tree, error) { err := t.store.Update(ctx, func(tx TreeStoreUpdateTx) error { currentRoot, err := tx.RootNode() if err != nil { @@ -139,10 +139,10 @@ func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []batchedInse return t, nil } -// batchedInsertionEntry represents one leaf insertion. -type batchedInsertionEntry struct { - key [hashSize]byte - leaf *LeafNode +// BatchedInsertionEntry represents one leaf insertion. +type BatchedInsertionEntry struct { + Key [hashSize]byte + Leaf *LeafNode } var _ Tree = (*CompactedTree)(nil) diff --git a/mssmt/tree_test.go b/mssmt/tree_test.go index 750313afe..3feff50b8 100644 --- a/mssmt/tree_test.go +++ b/mssmt/tree_test.go @@ -30,22 +30,22 @@ func TestBatchedInsert(t *testing.T) { // Use a simple deterministic pattern. key[0] = byte(i + 1) value := []byte(fmt.Sprintf("leaf-%d", i)) - leaves = append(leaves, treeLeaf{ + leaves = append(leaves, mssmt.treeLeaf{ key: key, - leaf: NewLeafNode(value, uint64(i+1)), + leaf: mssmt.NewLeafNode(value, uint64(i+1)), }) } - store := NewDefaultStore() - tree := NewCompactedTree(store) - compTree := tree.(*CompactedTree) + store := mssmt.NewDefaultStore() + tree := mssmt.NewCompactedTree(store) + compTree := tree.(*mssmt.CompactedTree) // Build the batch. - var batch []batchedInsertionEntry + var batch []mssmt.BatchedInsertionEntry for _, tl := range leaves { - batch = append(batch, batchedInsertionEntry{ - key: tl.key, - leaf: tl.leaf, + batch = append(batch, mssmt.BatchedInsertionEntry{ + Key: tl.key, + Leaf: tl.leaf, }) } @@ -72,7 +72,7 @@ func TestBatchedInsertEmpty(t *testing.T) { require.NoError(t, err) root, err := newTree.Root(ctx) require.NoError(t, err) - require.True(t, IsEqualNode(root, EmptyTree[0])) + require.True(t, mssmt.IsEqualNode(root, mssmt.EmptyTree[0])) } // TestBatchedInsertOverflow verifies that a batch insertion causing a sum overflow @@ -86,18 +86,18 @@ func TestBatchedInsertOverflow(t *testing.T) { // Insert one leaf with a huge sum. huge := uint64(math.MaxUint64 - 100) key1 := [32]byte{1} - hugeLeaf := NewLeafNode([]byte("huge"), huge) + hugeLeaf := mssmt.NewLeafNode([]byte("huge"), huge) _, err := compTree.Insert(ctx, key1, hugeLeaf) require.NoError(t, err) // Prepare a batch with one normal leaf and one that overflows the root sum. key2 := [32]byte{2} - normalLeaf := NewLeafNode([]byte("normal"), 50) + normalLeaf := mssmt.NewLeafNode([]byte("normal"), 50) key3 := [32]byte{3} - overflowLeaf := NewLeafNode([]byte("overflow"), 101) // huge + 101 exceeds MaxUint64 - batch := []batchedInsertionEntry{ - {key: key2, leaf: normalLeaf}, - {key: key3, leaf: overflowLeaf}, + overflowLeaf := mssmt.NewLeafNode([]byte("overflow"), 101) // huge + 101 exceeds MaxUint64 + batch := []mssmt.BatchedInsertionEntry{ + {Key: key2, Leaf: normalLeaf}, + {Key: key3, Leaf: overflowLeaf}, } _, err = compTree.BatchedInsert(ctx, batch) require.Error(t, err) From 30b5d90ccad60d18032135ded0f445e5d3696cf1 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 19:45:33 -0600 Subject: [PATCH 07/47] aider: fix: Update references to exported fields Key and Leaf in compacted_tree.go --- mssmt/compacted_tree.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 2cad7266a..d5e57e5c5 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -30,7 +30,7 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn // Partition entries into two groups based on bit at current height. var leftEntries, rightEntries []BatchedInsertionEntry for _, entry := range entries { - if bitIndex(uint8(height), &entry.key) == 0 { + if bitIndex(uint8(height), &entry.Key) == 0 { leftEntries = append(leftEntries, entry) } else { rightEntries = append(rightEntries, entry) @@ -89,7 +89,7 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn // Create the updated branch from the new left and right children. var updatedBranch *BranchNode - if bitIndex(uint8(height), &entries[0].key) == 0 { + if bitIndex(uint8(height), &entries[0].Key) == 0 { updatedBranch = NewBranch(newLeft, newRight) } else { updatedBranch = NewBranch(newRight, newLeft) @@ -121,8 +121,8 @@ func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []BatchedInse // (Optional) Loop over entries and check for sum overflow. for _, entry := range entries { - if err := CheckSumOverflowUint64(branchRoot.NodeSum(), entry.leaf.NodeSum()); err != nil { - return fmt.Errorf("batched insert key %v sum overflow: %w", entry.key, err) + if err := CheckSumOverflowUint64(branchRoot.NodeSum(), entry.Leaf.NodeSum()); err != nil { + return fmt.Errorf("batched insert key %v sum overflow: %w", entry.Key, err) } } From c8d9dbedb4f534696cd1798d81f564de639bf4f7 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 19:49:34 -0600 Subject: [PATCH 08/47] aider: refactor: Replace mssmt.treeLeaf with inline struct and update method calls --- mssmt/tree_test.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/mssmt/tree_test.go b/mssmt/tree_test.go index 3feff50b8..c435998e5 100644 --- a/mssmt/tree_test.go +++ b/mssmt/tree_test.go @@ -30,15 +30,17 @@ func TestBatchedInsert(t *testing.T) { // Use a simple deterministic pattern. key[0] = byte(i + 1) value := []byte(fmt.Sprintf("leaf-%d", i)) - leaves = append(leaves, mssmt.treeLeaf{ + leaves = append(leaves, struct { + key [32]byte + leaf *mssmt.LeafNode + }{ key: key, leaf: mssmt.NewLeafNode(value, uint64(i+1)), }) } store := mssmt.NewDefaultStore() - tree := mssmt.NewCompactedTree(store) - compTree := tree.(*mssmt.CompactedTree) + compTree := mssmt.NewCompactedTree(store) // Build the batch. var batch []mssmt.BatchedInsertionEntry @@ -54,7 +56,7 @@ func TestBatchedInsert(t *testing.T) { // Verify that each inserted leaf can be retrieved. for _, entry := range batch { - retrieved, err := newTree.(*CompactedTree).Get(ctx, entry.key) + retrieved, err := newTree.(*mssmt.CompactedTree).Get(ctx, entry.Key) require.NoError(t, err) require.Equal(t, entry.leaf, retrieved, "mismatch for key %x", entry.key) } @@ -64,9 +66,8 @@ func TestBatchedInsert(t *testing.T) { // leaves the tree unchanged. func TestBatchedInsertEmpty(t *testing.T) { ctx := context.Background() - store := NewDefaultStore() - tree := NewCompactedTree(store) - compTree := tree.(*CompactedTree) + store := mssmt.NewDefaultStore() + compTree := mssmt.NewCompactedTree(store) newTree, err := compTree.BatchedInsert(ctx, []batchedInsertionEntry{}) require.NoError(t, err) From a0e84a3f5885530638406ebc257f60a65bbfb768 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 19:49:38 -0600 Subject: [PATCH 09/47] aider: fix: Fix SEARCH block to match existing lines in tree_test.go --- mssmt/tree_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mssmt/tree_test.go b/mssmt/tree_test.go index c435998e5..444d719c4 100644 --- a/mssmt/tree_test.go +++ b/mssmt/tree_test.go @@ -24,7 +24,10 @@ import ( func TestBatchedInsert(t *testing.T) { ctx := context.Background() numLeaves := 10 - var leaves []treeLeaf + var leaves []struct { + key [32]byte + leaf *mssmt.LeafNode + } for i := 0; i < numLeaves; i++ { var key [32]byte // Use a simple deterministic pattern. From a04b55458cf7745df866a19134d2cba9cdccbdde Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 19:50:47 -0600 Subject: [PATCH 10/47] aider: fix: Update test file to reference exported symbols from mssmt package --- mssmt/tree_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mssmt/tree_test.go b/mssmt/tree_test.go index 444d719c4..7657d1a5f 100644 --- a/mssmt/tree_test.go +++ b/mssmt/tree_test.go @@ -61,7 +61,7 @@ func TestBatchedInsert(t *testing.T) { for _, entry := range batch { retrieved, err := newTree.(*mssmt.CompactedTree).Get(ctx, entry.Key) require.NoError(t, err) - require.Equal(t, entry.leaf, retrieved, "mismatch for key %x", entry.key) + require.Equal(t, entry.Leaf, retrieved, "mismatch for key %x", entry.Key) } } @@ -83,9 +83,9 @@ func TestBatchedInsertEmpty(t *testing.T) { // returns an error. func TestBatchedInsertOverflow(t *testing.T) { ctx := context.Background() - store := NewDefaultStore() - tree := NewCompactedTree(store) - compTree := tree.(*CompactedTree) + store := mssmt.NewDefaultStore() + tree := mssmt.NewCompactedTree(store) + compTree := tree.(*mssmt.CompactedTree) // Insert one leaf with a huge sum. huge := uint64(math.MaxUint64 - 100) @@ -105,7 +105,7 @@ func TestBatchedInsertOverflow(t *testing.T) { } _, err = compTree.BatchedInsert(ctx, batch) require.Error(t, err) - require.ErrorIs(t, err, ErrIntegerOverflow) + require.ErrorIs(t, err, mssmt.ErrIntegerOverflow) } var ( From cab0def9ca5c0a5cbcc7426fbafd7ae9a071b6f9 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 19:50:57 -0600 Subject: [PATCH 11/47] aider: fix: Correct type in BatchedInsert call in tree_test.go --- mssmt/tree_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mssmt/tree_test.go b/mssmt/tree_test.go index 7657d1a5f..13d7c34f9 100644 --- a/mssmt/tree_test.go +++ b/mssmt/tree_test.go @@ -72,7 +72,7 @@ func TestBatchedInsertEmpty(t *testing.T) { store := mssmt.NewDefaultStore() compTree := mssmt.NewCompactedTree(store) - newTree, err := compTree.BatchedInsert(ctx, []batchedInsertionEntry{}) + newTree, err := compTree.BatchedInsert(ctx, []mssmt.BatchedInsertionEntry{}) require.NoError(t, err) root, err := newTree.Root(ctx) require.NoError(t, err) From 0dd3aa402bb7cf8dc09501ad9c0556c3e6ba5cfa Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 19:51:57 -0600 Subject: [PATCH 12/47] aider: fix: Remove unnecessary type assertion in TestBatchedInsert function --- mssmt/tree_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mssmt/tree_test.go b/mssmt/tree_test.go index 13d7c34f9..486009bd4 100644 --- a/mssmt/tree_test.go +++ b/mssmt/tree_test.go @@ -59,7 +59,7 @@ func TestBatchedInsert(t *testing.T) { // Verify that each inserted leaf can be retrieved. for _, entry := range batch { - retrieved, err := newTree.(*mssmt.CompactedTree).Get(ctx, entry.Key) + retrieved, err := newTree.Get(ctx, entry.Key) require.NoError(t, err) require.Equal(t, entry.Leaf, retrieved, "mismatch for key %x", entry.Key) } From 92f1a0c2205c14af1dab0777e9a2590710ff34ce Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 19:52:54 -0600 Subject: [PATCH 13/47] aider: fix: Remove unnecessary type assertion in TestBatchedInsertOverflow test --- mssmt/tree_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mssmt/tree_test.go b/mssmt/tree_test.go index 486009bd4..e09035e20 100644 --- a/mssmt/tree_test.go +++ b/mssmt/tree_test.go @@ -84,8 +84,7 @@ func TestBatchedInsertEmpty(t *testing.T) { func TestBatchedInsertOverflow(t *testing.T) { ctx := context.Background() store := mssmt.NewDefaultStore() - tree := mssmt.NewCompactedTree(store) - compTree := tree.(*mssmt.CompactedTree) + compTree := mssmt.NewCompactedTree(store) // Insert one leaf with a huge sum. huge := uint64(math.MaxUint64 - 100) From ce0d5f63cf2a4f661066e6fd5893cf4c88cfe4c2 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 19:56:12 -0600 Subject: [PATCH 14/47] aider: fix: Reuse precomputed empty branch for left and right children in batched_insert --- mssmt/compacted_tree.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index d5e57e5c5..fb578ca7f 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -49,7 +49,7 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn // If there is no current left subtree, start with a default branch. var baseLeft *BranchNode if leftChild == EmptyTree[height+1] { - baseLeft = NewBranch(EmptyTree[height+1], EmptyTree[height+1]) + baseLeft = EmptyTree[height+1].(*BranchNode) } else { // If the left side is compacted, expand it. if cl, ok := leftChild.(*CompactedLeafNode); ok { @@ -71,7 +71,7 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn if len(rightEntries) > 0 { var baseRight *BranchNode if rightChild == EmptyTree[height+1] { - baseRight = NewBranch(EmptyTree[height+1], EmptyTree[height+1]) + baseRight = EmptyTree[height+1].(*BranchNode) } else { if cr, ok := rightChild.(*CompactedLeafNode); ok { baseRight = cr.Extract(height+1).(*BranchNode) From f849d8de370da48ab58a0a1e7513e853b80ed9a0 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:02:57 -0600 Subject: [PATCH 15/47] aider: feat: Implement compacted leaf creation in batched_insert for single entries --- mssmt/compacted_tree.go | 64 +++++++++++++++++++++++++++-------------- mssmt/tree_test.go | 5 ++-- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index fb578ca7f..9a3109e83 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -46,21 +46,31 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn // Process left subtree: var newLeft Node if len(leftEntries) > 0 { - // If there is no current left subtree, start with a default branch. - var baseLeft *BranchNode - if leftChild == EmptyTree[height+1] { - baseLeft = EmptyTree[height+1].(*BranchNode) + var newLeft Node + // If there is exactly one insertion and the current left child is empty, + // create a compacted leaf (mirroring the logic in CompactedTree.insert). + if len(leftEntries) == 1 && leftChild == EmptyTree[height+1] { + entry := leftEntries[0] + compLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) + if err := tx.InsertCompactedLeaf(compLeaf); err != nil { + return nil, err + } + newLeft = compLeaf } else { - // If the left side is compacted, expand it. - if cl, ok := leftChild.(*CompactedLeafNode); ok { - baseLeft = cl.Extract(height+1).(*BranchNode) + var baseLeft *BranchNode + if leftChild == EmptyTree[height+1] { + baseLeft = EmptyTree[height+1].(*BranchNode) } else { - baseLeft = leftChild.(*BranchNode) + if cl, ok := leftChild.(*CompactedLeafNode); ok { + baseLeft = cl.Extract(height+1).(*BranchNode) + } else { + baseLeft = leftChild.(*BranchNode) + } + } + newLeft, err = t.batched_insert(tx, leftEntries, height+1, baseLeft) + if err != nil { + return nil, err } - } - newLeft, err = t.batched_insert(tx, leftEntries, height+1, baseLeft) - if err != nil { - return nil, err } } else { newLeft = leftChild @@ -69,19 +79,29 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn // Process right subtree: var newRight Node if len(rightEntries) > 0 { - var baseRight *BranchNode - if rightChild == EmptyTree[height+1] { - baseRight = EmptyTree[height+1].(*BranchNode) + var newRight Node + if len(rightEntries) == 1 && rightChild == EmptyTree[height+1] { + entry := rightEntries[0] + compLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) + if err := tx.InsertCompactedLeaf(compLeaf); err != nil { + return nil, err + } + newRight = compLeaf } else { - if cr, ok := rightChild.(*CompactedLeafNode); ok { - baseRight = cr.Extract(height+1).(*BranchNode) + var baseRight *BranchNode + if rightChild == EmptyTree[height+1] { + baseRight = EmptyTree[height+1].(*BranchNode) } else { - baseRight = rightChild.(*BranchNode) + if cr, ok := rightChild.(*CompactedLeafNode); ok { + baseRight = cr.Extract(height+1).(*BranchNode) + } else { + baseRight = rightChild.(*BranchNode) + } + } + newRight, err = t.batched_insert(tx, rightEntries, height+1, baseRight) + if err != nil { + return nil, err } - } - newRight, err = t.batched_insert(tx, rightEntries, height+1, baseRight) - if err != nil { - return nil, err } } else { newRight = rightChild diff --git a/mssmt/tree_test.go b/mssmt/tree_test.go index e09035e20..a3a1b5dfd 100644 --- a/mssmt/tree_test.go +++ b/mssmt/tree_test.go @@ -11,6 +11,7 @@ import ( "strconv" "testing" + "crypto/sha256" "github.com/lightninglabs/taproot-assets/fn" "github.com/lightninglabs/taproot-assets/internal/test" "github.com/lightninglabs/taproot-assets/mssmt" @@ -29,9 +30,7 @@ func TestBatchedInsert(t *testing.T) { leaf *mssmt.LeafNode } for i := 0; i < numLeaves; i++ { - var key [32]byte - // Use a simple deterministic pattern. - key[0] = byte(i + 1) + key := sha256.Sum256([]byte(fmt.Sprintf("value-%d", i))) value := []byte(fmt.Sprintf("leaf-%d", i)) leaves = append(leaves, struct { key [32]byte From 752f4a08d04505eddbb7166be8344ba64a0db95e Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:08:14 -0600 Subject: [PATCH 16/47] aider: feat: Update batched_insert to handle collisions with compacted leaves --- mssmt/compacted_tree.go | 202 ++++++++++++++++++++++++++++++++-------- 1 file changed, 162 insertions(+), 40 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 9a3109e83..f3766a7f2 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -47,64 +47,186 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn var newLeft Node if len(leftEntries) > 0 { var newLeft Node - // If there is exactly one insertion and the current left child is empty, - // create a compacted leaf (mirroring the logic in CompactedTree.insert). - if len(leftEntries) == 1 && leftChild == EmptyTree[height+1] { - entry := leftEntries[0] - compLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) - if err := tx.InsertCompactedLeaf(compLeaf); err != nil { - return nil, err + // Check if the current left child is not empty. + if leftChild != EmptyTree[height+1] { + // If the existing child is a compacted leaf, we must handle potential collisions. + if cl, ok := leftChild.(*CompactedLeafNode); ok { + if len(leftEntries) == 1 { + entry := leftEntries[0] + if entry.Key == cl.Key() { + // Replacement: update the compacted leaf. + newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) + if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { + return nil, err + } + if err := tx.InsertCompactedLeaf(newLeaf); err != nil { + return nil, err + } + newLeft = newLeaf + } else { + // Collision – keys differ: call merge to combine the new entry with the existing compacted leaf. + newLeft, err = t.merge(tx, height+1, entry.Key, entry.Leaf, cl.Key(), cl.LeafNode) + if err != nil { + return nil, err + } + } + } else { + // Multiple batch entries – check if they all match the existing key. + allMatch := true + for _, entry := range leftEntries { + if entry.Key != cl.Key() { + allMatch = false + break + } + } + if allMatch { + // All entries match; take the last one as replacement. + lastEntry := leftEntries[len(leftEntries)-1] + newLeaf := NewCompactedLeafNode(height+1, &lastEntry.Key, lastEntry.Leaf) + if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { + return nil, err + } + if err := tx.InsertCompactedLeaf(newLeaf); err != nil { + return nil, err + } + newLeft = newLeaf + } else { + // At least one entry has a different key – merge using the first differing entry. + var mergeEntry *BatchedInsertionEntry + for _, entry := range leftEntries { + if entry.Key != cl.Key() { + mergeEntry = &entry + break + } + } + if mergeEntry == nil { + return nil, fmt.Errorf("unexpected nil merge entry") + } + newLeft, err = t.merge(tx, height+1, mergeEntry.Key, mergeEntry.Leaf, cl.Key(), cl.LeafNode) + if err != nil { + return nil, err + } + } + } + } else { + // leftChild is not a compacted leaf, so it must be a branch; recurse normally. + baseLeft := leftChild.(*BranchNode) + newLeft, err = t.batched_insert(tx, leftEntries, height+1, baseLeft) + if err != nil { + return nil, err + } } - newLeft = compLeaf } else { - var baseLeft *BranchNode - if leftChild == EmptyTree[height+1] { - baseLeft = EmptyTree[height+1].(*BranchNode) + // The left child is empty. + if len(leftEntries) == 1 { + entry := leftEntries[0] + newLeft = NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) + if err := tx.InsertCompactedLeaf(newLeft.(*CompactedLeafNode)); err != nil { + return nil, err + } } else { - if cl, ok := leftChild.(*CompactedLeafNode); ok { - baseLeft = cl.Extract(height+1).(*BranchNode) - } else { - baseLeft = leftChild.(*BranchNode) + baseLeft := EmptyTree[height+1].(*BranchNode) + newLeft, err = t.batched_insert(tx, leftEntries, height+1, baseLeft) + if err != nil { + return nil, err } } - newLeft, err = t.batched_insert(tx, leftEntries, height+1, baseLeft) - if err != nil { - return nil, err - } } - } else { - newLeft = leftChild + // Use newLeft as the computed left child. + leftChild = newLeft } // Process right subtree: var newRight Node if len(rightEntries) > 0 { var newRight Node - if len(rightEntries) == 1 && rightChild == EmptyTree[height+1] { - entry := rightEntries[0] - compLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) - if err := tx.InsertCompactedLeaf(compLeaf); err != nil { - return nil, err + // Check if the current right child is not empty. + if rightChild != EmptyTree[height+1] { + // If the existing child is a compacted leaf, we must handle potential collisions. + if cr, ok := rightChild.(*CompactedLeafNode); ok { + if len(rightEntries) == 1 { + entry := rightEntries[0] + if entry.Key == cr.Key() { + // Replacement: update the compacted leaf. + newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) + if err := tx.DeleteCompactedLeaf(cr.NodeHash()); err != nil { + return nil, err + } + if err := tx.InsertCompactedLeaf(newLeaf); err != nil { + return nil, err + } + newRight = newLeaf + } else { + // Collision – keys differ: call merge to combine the new entry with the existing compacted leaf. + newRight, err = t.merge(tx, height+1, entry.Key, entry.Leaf, cr.Key(), cr.LeafNode) + if err != nil { + return nil, err + } + } + } else { + // Multiple batch entries – check if they all match the existing key. + allMatch := true + for _, entry := range rightEntries { + if entry.Key != cr.Key() { + allMatch = false + break + } + } + if allMatch { + // All entries match; take the last one as replacement. + lastEntry := rightEntries[len(rightEntries)-1] + newLeaf := NewCompactedLeafNode(height+1, &lastEntry.Key, lastEntry.Leaf) + if err := tx.DeleteCompactedLeaf(cr.NodeHash()); err != nil { + return nil, err + } + if err := tx.InsertCompactedLeaf(newLeaf); err != nil { + return nil, err + } + newRight = newLeaf + } else { + // At least one entry has a different key – merge using the first differing entry. + var mergeEntry *BatchedInsertionEntry + for _, entry := range rightEntries { + if entry.Key != cr.Key() { + mergeEntry = &entry + break + } + } + if mergeEntry == nil { + return nil, fmt.Errorf("unexpected nil merge entry") + } + newRight, err = t.merge(tx, height+1, mergeEntry.Key, mergeEntry.Leaf, cr.Key(), cr.LeafNode) + if err != nil { + return nil, err + } + } + } + } else { + // rightChild is not a compacted leaf, so it must be a branch; recurse normally. + baseRight := rightChild.(*BranchNode) + newRight, err = t.batched_insert(tx, rightEntries, height+1, baseRight) + if err != nil { + return nil, err + } } - newRight = compLeaf } else { - var baseRight *BranchNode - if rightChild == EmptyTree[height+1] { - baseRight = EmptyTree[height+1].(*BranchNode) + // The right child is empty. + if len(rightEntries) == 1 { + entry := rightEntries[0] + newRight = NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) + if err := tx.InsertCompactedLeaf(newRight.(*CompactedLeafNode)); err != nil { + return nil, err + } } else { - if cr, ok := rightChild.(*CompactedLeafNode); ok { - baseRight = cr.Extract(height+1).(*BranchNode) - } else { - baseRight = rightChild.(*BranchNode) + baseRight := EmptyTree[height+1].(*BranchNode) + newRight, err = t.batched_insert(tx, rightEntries, height+1, baseRight) + if err != nil { + return nil, err } } - newRight, err = t.batched_insert(tx, rightEntries, height+1, baseRight) - if err != nil { - return nil, err - } } - } else { - newRight = rightChild + // Use newRight as the computed right child. + rightChild = newRight } // Create the updated branch from the new left and right children. From 39f9a869f4f7ac6bd5264ed7497dd08966e5028d Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:10:42 -0600 Subject: [PATCH 17/47] aider: fix: Use updated leftChild and rightChild in branch creation to avoid nil dereference --- mssmt/compacted_tree.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index f3766a7f2..04239bb5c 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -232,9 +232,9 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn // Create the updated branch from the new left and right children. var updatedBranch *BranchNode if bitIndex(uint8(height), &entries[0].Key) == 0 { - updatedBranch = NewBranch(newLeft, newRight) + updatedBranch = NewBranch(leftChild, rightChild) } else { - updatedBranch = NewBranch(newRight, newLeft) + updatedBranch = NewBranch(rightChild, leftChild) } // Delete the old branch and insert the new one. From ee360e970e5d9f87dcc3909faafeb35680c1b44b Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:11:35 -0600 Subject: [PATCH 18/47] aider: fix: Remove inner redeclarations of newLeft and newRight in batched_insert --- mssmt/compacted_tree.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 04239bb5c..0d4a540bd 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -46,7 +46,6 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn // Process left subtree: var newLeft Node if len(leftEntries) > 0 { - var newLeft Node // Check if the current left child is not empty. if leftChild != EmptyTree[height+1] { // If the existing child is a compacted leaf, we must handle potential collisions. @@ -139,7 +138,6 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn // Process right subtree: var newRight Node if len(rightEntries) > 0 { - var newRight Node // Check if the current right child is not empty. if rightChild != EmptyTree[height+1] { // If the existing child is a compacted leaf, we must handle potential collisions. From eb8f224610c9de589cd68d0a181d1ce1b5fe04cf Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:14:51 -0600 Subject: [PATCH 19/47] aider: fix: Record height of compacted leaf and adjust extraction logic --- mssmt/node.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/mssmt/node.go b/mssmt/node.go index dbfeff6f0..ae83015b2 100644 --- a/mssmt/node.go +++ b/mssmt/node.go @@ -121,6 +121,9 @@ type CompactedLeafNode struct { // compactedNodeHash holds the topmost (omitted) node's node hash in the // subtree. compactedNodeHash NodeHash + + // Height is the level at which this compacted leaf was created. + Height int } // NewCompactedLeafNode creates a new compacted leaf at the passed height with @@ -144,6 +147,7 @@ func NewCompactedLeafNode(height int, key *[32]byte, compactedNodeHash: nodeHash, } + node.Height = height return node } @@ -160,20 +164,20 @@ func (c *CompactedLeafNode) Key() [32]byte { // Extract extracts the subtree represented by this compacted leaf and returns // the topmost node in the tree. func (c *CompactedLeafNode) Extract(height int) Node { - var current Node = c.LeafNode - - // Walk up and recreate the missing branches. - for j := MaxTreeLevels; j > height+1; j-- { - var left, right Node - if bitIndex(uint8(j-1), &c.key) == 0 { - left, right = current, EmptyTree[j] + // If the search is at or below the height where this compacted leaf was created, + // we simply return the stored leaf. + if requestedHeight >= c.Height { + return c.LeafNode + } + current := c.LeafNode + // Otherwise, add the missing branch layers from c.Height-1 down to requestedHeight+1. + for j := c.Height - 1; j >= requestedHeight+1; j-- { + if bitIndex(uint8(j), &c.key) == 0 { + current = NewBranch(current, EmptyTree[j+1]) } else { - left, right = EmptyTree[j], current + current = NewBranch(EmptyTree[j+1], current) } - - current = NewBranch(left, right) } - return current } From 23313cf7cfa719ac2db07cbfaac163a576e318c0 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:18:00 -0600 Subject: [PATCH 20/47] aider: fix: Rename parameter and update variable type in Extract method --- mssmt/node.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mssmt/node.go b/mssmt/node.go index ae83015b2..5b47561ec 100644 --- a/mssmt/node.go +++ b/mssmt/node.go @@ -163,13 +163,13 @@ func (c *CompactedLeafNode) Key() [32]byte { // Extract extracts the subtree represented by this compacted leaf and returns // the topmost node in the tree. -func (c *CompactedLeafNode) Extract(height int) Node { +func (c *CompactedLeafNode) Extract(requestedHeight int) Node { // If the search is at or below the height where this compacted leaf was created, // we simply return the stored leaf. if requestedHeight >= c.Height { return c.LeafNode } - current := c.LeafNode + var current Node = c.LeafNode // Otherwise, add the missing branch layers from c.Height-1 down to requestedHeight+1. for j := c.Height - 1; j >= requestedHeight+1; j-- { if bitIndex(uint8(j), &c.key) == 0 { From 305d4120c2e9c348d17c5163b563ac9e972d79e3 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:19:29 -0600 Subject: [PATCH 21/47] aider: fix: Wrap stored leaf in branch for requestedHeight equal to compaction level --- mssmt/node.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/mssmt/node.go b/mssmt/node.go index 5b47561ec..a3c7b6b93 100644 --- a/mssmt/node.go +++ b/mssmt/node.go @@ -164,13 +164,24 @@ func (c *CompactedLeafNode) Key() [32]byte { // Extract extracts the subtree represented by this compacted leaf and returns // the topmost node in the tree. func (c *CompactedLeafNode) Extract(requestedHeight int) Node { - // If the search is at or below the height where this compacted leaf was created, - // we simply return the stored leaf. + // If the search is exactly one level above where the compacted leaf was created, + // then we need to wrap the stored leaf in one branch layer so that the caller + // receives a BranchNode. + if requestedHeight == c.Height-1 { + if bitIndex(uint8(requestedHeight), &c.key) == 0 { + return NewBranch(c.LeafNode, EmptyTree[requestedHeight+1]) + } else { + return NewBranch(EmptyTree[requestedHeight+1], c.LeafNode) + } + } + + // If the search is deeper than (at or below) the compaction level, then return the leaf. if requestedHeight >= c.Height { return c.LeafNode } - var current Node = c.LeafNode + // Otherwise, add the missing branch layers from c.Height-1 down to requestedHeight+1. + var current Node = c.LeafNode for j := c.Height - 1; j >= requestedHeight+1; j-- { if bitIndex(uint8(j), &c.key) == 0 { current = NewBranch(current, EmptyTree[j+1]) From 5fc7a8913119476531e216ff96b8021ce628bbb7 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:21:29 -0600 Subject: [PATCH 22/47] aider: fix: Ensure Extract returns a BranchNode for requestedHeight >= compaction level --- mssmt/node.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/mssmt/node.go b/mssmt/node.go index a3c7b6b93..8abd6b71f 100644 --- a/mssmt/node.go +++ b/mssmt/node.go @@ -164,9 +164,11 @@ func (c *CompactedLeafNode) Key() [32]byte { // Extract extracts the subtree represented by this compacted leaf and returns // the topmost node in the tree. func (c *CompactedLeafNode) Extract(requestedHeight int) Node { - // If the search is exactly one level above where the compacted leaf was created, - // then we need to wrap the stored leaf in one branch layer so that the caller - // receives a BranchNode. + // If the requested height is too deep relative to the compaction, adjust + // it to one level above the compaction level so that we always get a branch. + if requestedHeight >= c.Height { + requestedHeight = c.Height - 1 + } if requestedHeight == c.Height-1 { if bitIndex(uint8(requestedHeight), &c.key) == 0 { return NewBranch(c.LeafNode, EmptyTree[requestedHeight+1]) @@ -174,13 +176,6 @@ func (c *CompactedLeafNode) Extract(requestedHeight int) Node { return NewBranch(EmptyTree[requestedHeight+1], c.LeafNode) } } - - // If the search is deeper than (at or below) the compaction level, then return the leaf. - if requestedHeight >= c.Height { - return c.LeafNode - } - - // Otherwise, add the missing branch layers from c.Height-1 down to requestedHeight+1. var current Node = c.LeafNode for j := c.Height - 1; j >= requestedHeight+1; j-- { if bitIndex(uint8(j), &c.key) == 0 { From ac4d78e8bfd11f028b307f7e9cadc08c05bb71fd Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:23:10 -0600 Subject: [PATCH 23/47] aider: fix: Ensure Extract always returns a BranchNode from CompactedLeafNode --- mssmt/node.go | 31 ++++++++----------------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/mssmt/node.go b/mssmt/node.go index 8abd6b71f..0d284243c 100644 --- a/mssmt/node.go +++ b/mssmt/node.go @@ -161,30 +161,15 @@ func (c *CompactedLeafNode) Key() [32]byte { return c.key } -// Extract extracts the subtree represented by this compacted leaf and returns -// the topmost node in the tree. func (c *CompactedLeafNode) Extract(requestedHeight int) Node { - // If the requested height is too deep relative to the compaction, adjust - // it to one level above the compaction level so that we always get a branch. - if requestedHeight >= c.Height { - requestedHeight = c.Height - 1 - } - if requestedHeight == c.Height-1 { - if bitIndex(uint8(requestedHeight), &c.key) == 0 { - return NewBranch(c.LeafNode, EmptyTree[requestedHeight+1]) - } else { - return NewBranch(EmptyTree[requestedHeight+1], c.LeafNode) - } - } - var current Node = c.LeafNode - for j := c.Height - 1; j >= requestedHeight+1; j-- { - if bitIndex(uint8(j), &c.key) == 0 { - current = NewBranch(current, EmptyTree[j+1]) - } else { - current = NewBranch(EmptyTree[j+1], current) - } - } - return current + // Force extraction to always return a branch node wrapping the stored leaf. + // Use the level immediately above the compaction level. + branchLevel := c.Height - 1 + if bitIndex(uint8(branchLevel), &c.key) == 0 { + return NewBranch(c.LeafNode, EmptyTree[branchLevel+1]) + } else { + return NewBranch(EmptyTree[branchLevel+1], c.LeafNode) + } } // Copy returns a deep copy of the compacted leaf node. From fcc3fbd2af415d6c66527ebdb9f8ac32d24c796c Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:24:59 -0600 Subject: [PATCH 24/47] aider: fix: Ensure Extract always returns a branch node from a compacted leaf --- mssmt/node.go | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/mssmt/node.go b/mssmt/node.go index 0d284243c..128badff9 100644 --- a/mssmt/node.go +++ b/mssmt/node.go @@ -162,14 +162,31 @@ func (c *CompactedLeafNode) Key() [32]byte { } func (c *CompactedLeafNode) Extract(requestedHeight int) Node { - // Force extraction to always return a branch node wrapping the stored leaf. - // Use the level immediately above the compaction level. - branchLevel := c.Height - 1 - if bitIndex(uint8(branchLevel), &c.key) == 0 { - return NewBranch(c.LeafNode, EmptyTree[branchLevel+1]) - } else { - return NewBranch(EmptyTree[branchLevel+1], c.LeafNode) - } + // We need to guarantee that the returned Node is a branch so that + // walkDown can continue descending. + // Let target be the maximum level we must expand to. If the requested + // height is too deep (≥ c.Height), we set target to c.Height - 1. + target := requestedHeight + if target >= c.Height { + target = c.Height - 1 + } + + // Start with the stored leaf. + current := c.LeafNode + // Expand from the compaction level down to target+1. + for j := c.Height - 1; j >= target+1; j-- { + if bitIndex(uint8(j), &c.key) == 0 { + current = NewBranch(current, EmptyTree[j+1]) + } else { + current = NewBranch(EmptyTree[j+1], current) + } + } + // Finally, wrap one more layer so that the result is a branch. + if bitIndex(uint8(target), &c.key) == 0 { + return NewBranch(current, EmptyTree[target+1]) + } else { + return NewBranch(EmptyTree[target+1], current) + } } // Copy returns a deep copy of the compacted leaf node. From ab01b4a66acb480fb7083245b2bc32668d97041d Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:36:50 -0600 Subject: [PATCH 25/47] aider: fix: Declare current as Node interface type in Extract method --- mssmt/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mssmt/node.go b/mssmt/node.go index 128badff9..6664bb7e2 100644 --- a/mssmt/node.go +++ b/mssmt/node.go @@ -172,7 +172,7 @@ func (c *CompactedLeafNode) Extract(requestedHeight int) Node { } // Start with the stored leaf. - current := c.LeafNode + var current Node = c.LeafNode // Expand from the compaction level down to target+1. for j := c.Height - 1; j >= target+1; j-- { if bitIndex(uint8(j), &c.key) == 0 { From e9de62d2254f95c17bf9a7f7d381e1a363e7092b Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:40:18 -0600 Subject: [PATCH 26/47] aider: fix: Ensure Extract always returns a branch node to prevent panic --- mssmt/node.go | 65 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/mssmt/node.go b/mssmt/node.go index 6664bb7e2..1723ce9bf 100644 --- a/mssmt/node.go +++ b/mssmt/node.go @@ -162,31 +162,46 @@ func (c *CompactedLeafNode) Key() [32]byte { } func (c *CompactedLeafNode) Extract(requestedHeight int) Node { - // We need to guarantee that the returned Node is a branch so that - // walkDown can continue descending. - // Let target be the maximum level we must expand to. If the requested - // height is too deep (≥ c.Height), we set target to c.Height - 1. - target := requestedHeight - if target >= c.Height { - target = c.Height - 1 - } - - // Start with the stored leaf. - var current Node = c.LeafNode - // Expand from the compaction level down to target+1. - for j := c.Height - 1; j >= target+1; j-- { - if bitIndex(uint8(j), &c.key) == 0 { - current = NewBranch(current, EmptyTree[j+1]) - } else { - current = NewBranch(EmptyTree[j+1], current) - } - } - // Finally, wrap one more layer so that the result is a branch. - if bitIndex(uint8(target), &c.key) == 0 { - return NewBranch(current, EmptyTree[target+1]) - } else { - return NewBranch(EmptyTree[target+1], current) - } + // DEBUG: print arguments and key. + fmt.Printf("Extract: requestedHeight=%d, c.Height=%d, key=%x\n", requestedHeight, c.Height, c.key) + + // Ensure that if requestedHeight is too deep, we adjust it. + target := requestedHeight + if target >= c.Height { + target = c.Height - 1 + } + + // Start with the stored leaf. + var current Node = c.LeafNode + + // Expand from the compaction level down to target+1. + for j := c.Height - 1; j >= target+1; j-- { + fmt.Printf("Extract loop: j=%d, bit=%d\n", j, bitIndex(uint8(j), &c.key)) + if bitIndex(uint8(j), &c.key) == 0 { + current = NewBranch(current, EmptyTree[j+1]) + } else { + current = NewBranch(EmptyTree[j+1], current) + } + } + + // Ensure that the sibling we use is a branch. + sibling := EmptyTree[target+1] + if target+1 == MaxTreeLevels { + // Wrap the empty leaf in a branch so that it’s not a pure leaf. + sibling = NewBranch(sibling, sibling) + } + + // Finally, wrap one more layer based on the key bit so that the result is a branch. + var result Node + if bitIndex(uint8(target), &c.key) == 0 { + result = NewBranch(current, sibling) + } else { + result = NewBranch(sibling, current) + } + + // DEBUG: print type and hash of result. + fmt.Printf("Extract returning: type=%T, hash=%x\n", result, result.NodeHash()) + return result } // Copy returns a deep copy of the compacted leaf node. From 013f658960d342935930f6e0bfcd5b524152cb1e Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:41:09 -0600 Subject: [PATCH 27/47] aider: fix: Add fmt import to resolve undefined fmt errors in node.go --- mssmt/node.go | 1 + 1 file changed, 1 insertion(+) diff --git a/mssmt/node.go b/mssmt/node.go index 1723ce9bf..9ffe07982 100644 --- a/mssmt/node.go +++ b/mssmt/node.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/binary" "encoding/hex" + "fmt" ) const ( From a4be10475c6bed8c3b9a03cacffa5ce4c8d0e291 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:43:24 -0600 Subject: [PATCH 28/47] aider: fix: Ensure full extraction of compacted leaves to avoid panic in walkDown --- mssmt/compacted_tree.go | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 0d4a540bd..81cce170b 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -339,15 +339,30 @@ func (t *CompactedTree) walkDown(tx TreeStoreViewTx, key *[hashSize]byte, switch node := next.(type) { case *CompactedLeafNode: // Our next node is a compacted leaf. We just need to - // expand it so we can continue our walk down the tree. - next = node.Extract(i) + // Force full extraction so that we always receive a branch node + // (using node.Height-1). + next = node.Extract(node.Height - 1) - // Sibling might be a compacted leaf too, in which case - // we need to extract it as well. + // If the sibling is also a compacted leaf, extract it fully. if compSibling, ok := sibling.(*CompactedLeafNode); ok { - sibling = compSibling.Extract(i) + sibling = compSibling.Extract(compSibling.Height - 1) } + // Continue walking down using the fully expanded branch. + for j := i; j <= lastBitIndex; j++ { + if iter != nil { + if err := iter(j, next, sibling, current); err != nil { + return nil, err + } + } + current = next + if j < lastBitIndex { + branch := current.(*BranchNode) + next, sibling = stepOrder(j+1, key, branch.Left, branch.Right) + } + } + return current.(*LeafNode), nil + // Now that all required branches are reconstructed we // can continue the search for the leaf matching the // passed key. From acbf785ca4e6645ea2cc3f44f3bbd65773e2fb22 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:46:07 -0600 Subject: [PATCH 29/47] aider: fix: Fully expand compacted leaf nodes in walkDown to prevent type errors --- mssmt/compacted_tree.go | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 81cce170b..2433ffb0a 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -338,30 +338,18 @@ func (t *CompactedTree) walkDown(tx TreeStoreViewTx, key *[hashSize]byte, switch node := next.(type) { case *CompactedLeafNode: - // Our next node is a compacted leaf. We just need to - // Force full extraction so that we always receive a branch node - // (using node.Height-1). - next = node.Extract(node.Height - 1) - - // If the sibling is also a compacted leaf, extract it fully. + // Our next node is a compacted leaf. We want to fully + // expand it so that the compacted subtree is completely unrolled + // up to the leaf level. + fmt.Printf("walkDown: encountered compacted leaf at level %d, expanding fully\n", i) + fullyExpanded := node.Extract(lastBitIndex) + // If the sibling is also a compacted leaf, fully expand it. if compSibling, ok := sibling.(*CompactedLeafNode); ok { - sibling = compSibling.Extract(compSibling.Height - 1) + sibling = compSibling.Extract(lastBitIndex) } - - // Continue walking down using the fully expanded branch. - for j := i; j <= lastBitIndex; j++ { - if iter != nil { - if err := iter(j, next, sibling, current); err != nil { - return nil, err - } - } - current = next - if j < lastBitIndex { - branch := current.(*BranchNode) - next, sibling = stepOrder(j+1, key, branch.Left, branch.Right) - } - } - return current.(*LeafNode), nil + fmt.Printf("walkDown: fully expanded node type: %T, hash=%x\n", fullyExpanded, fullyExpanded.NodeHash()) + // FullyExpanded is expected to be a leaf. Return it straight away. + return fullyExpanded.(*LeafNode), nil // Now that all required branches are reconstructed we // can continue the search for the leaf matching the From b414194bf0cb3f3127f827889ba6964cd2a68b84 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:50:18 -0600 Subject: [PATCH 30/47] aider: fix: Return underlying leaf directly from compacted leaf in walkDown method --- mssmt/compacted_tree.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 2433ffb0a..a60a459ae 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -338,18 +338,9 @@ func (t *CompactedTree) walkDown(tx TreeStoreViewTx, key *[hashSize]byte, switch node := next.(type) { case *CompactedLeafNode: - // Our next node is a compacted leaf. We want to fully - // expand it so that the compacted subtree is completely unrolled - // up to the leaf level. - fmt.Printf("walkDown: encountered compacted leaf at level %d, expanding fully\n", i) - fullyExpanded := node.Extract(lastBitIndex) - // If the sibling is also a compacted leaf, fully expand it. - if compSibling, ok := sibling.(*CompactedLeafNode); ok { - sibling = compSibling.Extract(lastBitIndex) - } - fmt.Printf("walkDown: fully expanded node type: %T, hash=%x\n", fullyExpanded, fullyExpanded.NodeHash()) - // FullyExpanded is expected to be a leaf. Return it straight away. - return fullyExpanded.(*LeafNode), nil + // Our next node is a compacted leaf. We simply return the underlying leaf. + fmt.Printf("walkDown: encountered compacted leaf at level %d, returning underlying leaf\n", i) + return node.LeafNode, nil // Now that all required branches are reconstructed we // can continue the search for the leaf matching the From 29c77cb01fcb4a5f65b251d73e37d8c27a79384b Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:52:35 -0600 Subject: [PATCH 31/47] aider: fix: Sort entries by key in BatchedInsert to ensure correct insertion order --- mssmt/compacted_tree.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index a60a459ae..74b9cb23d 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -2,7 +2,9 @@ package mssmt import ( "context" + "bytes" "fmt" + "sort" ) // CompactedTree represents a compacted Merkle-Sum Sparse Merkle Tree (MS-SMT). @@ -252,6 +254,10 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn // BatchedInsert inserts multiple leaf nodes at the given keys within the MS-SMT. func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []BatchedInsertionEntry) (Tree, error) { + sort.Slice(entries, func(i, j int) bool { + return bytes.Compare(entries[i].Key[:], entries[j].Key[:]) < 0 + }) + err := t.store.Update(ctx, func(tx TreeStoreUpdateTx) error { currentRoot, err := tx.RootNode() if err != nil { From e6018b51aceb297909a49ce13a641baa852cacfc Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:54:07 -0600 Subject: [PATCH 32/47] aider: fix: Ensure correct child ordering in batched_insert branch creation --- mssmt/compacted_tree.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 74b9cb23d..302480cb3 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -231,11 +231,7 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn // Create the updated branch from the new left and right children. var updatedBranch *BranchNode - if bitIndex(uint8(height), &entries[0].Key) == 0 { - updatedBranch = NewBranch(leftChild, rightChild) - } else { - updatedBranch = NewBranch(rightChild, leftChild) - } + updatedBranch = NewBranch(leftChild, rightChild) // Delete the old branch and insert the new one. if root != EmptyTree[height] { From 41cd8ac6b247ce89218c6d3006a15611e491a627 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:56:03 -0600 Subject: [PATCH 33/47] aider: chore: Remove debug print statements from Extract and walkDown methods --- mssmt/compacted_tree.go | 7 ------- mssmt/node.go | 12 ------------ 2 files changed, 19 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 302480cb3..894de92f3 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -341,12 +341,8 @@ func (t *CompactedTree) walkDown(tx TreeStoreViewTx, key *[hashSize]byte, switch node := next.(type) { case *CompactedLeafNode: // Our next node is a compacted leaf. We simply return the underlying leaf. - fmt.Printf("walkDown: encountered compacted leaf at level %d, returning underlying leaf\n", i) return node.LeafNode, nil - // Now that all required branches are reconstructed we - // can continue the search for the leaf matching the - // passed key. for j := i; j <= lastBitIndex; j++ { if iter != nil { err := iter(j, next, sibling, current) @@ -357,9 +353,6 @@ func (t *CompactedTree) walkDown(tx TreeStoreViewTx, key *[hashSize]byte, current = next if j < lastBitIndex { - // Since we have all the branches we - // need extracted already we can just - // continue walking down. branch := current.(*BranchNode) next, sibling = stepOrder( j+1, key, branch.Left, diff --git a/mssmt/node.go b/mssmt/node.go index 9ffe07982..ee9d48983 100644 --- a/mssmt/node.go +++ b/mssmt/node.go @@ -163,21 +163,14 @@ func (c *CompactedLeafNode) Key() [32]byte { } func (c *CompactedLeafNode) Extract(requestedHeight int) Node { - // DEBUG: print arguments and key. - fmt.Printf("Extract: requestedHeight=%d, c.Height=%d, key=%x\n", requestedHeight, c.Height, c.key) - - // Ensure that if requestedHeight is too deep, we adjust it. target := requestedHeight if target >= c.Height { target = c.Height - 1 } - // Start with the stored leaf. var current Node = c.LeafNode - // Expand from the compaction level down to target+1. for j := c.Height - 1; j >= target+1; j-- { - fmt.Printf("Extract loop: j=%d, bit=%d\n", j, bitIndex(uint8(j), &c.key)) if bitIndex(uint8(j), &c.key) == 0 { current = NewBranch(current, EmptyTree[j+1]) } else { @@ -185,14 +178,11 @@ func (c *CompactedLeafNode) Extract(requestedHeight int) Node { } } - // Ensure that the sibling we use is a branch. sibling := EmptyTree[target+1] if target+1 == MaxTreeLevels { - // Wrap the empty leaf in a branch so that it’s not a pure leaf. sibling = NewBranch(sibling, sibling) } - // Finally, wrap one more layer based on the key bit so that the result is a branch. var result Node if bitIndex(uint8(target), &c.key) == 0 { result = NewBranch(current, sibling) @@ -200,8 +190,6 @@ func (c *CompactedLeafNode) Extract(requestedHeight int) Node { result = NewBranch(sibling, current) } - // DEBUG: print type and hash of result. - fmt.Printf("Extract returning: type=%T, hash=%x\n", result, result.NodeHash()) return result } From b6f47fc6c434d84f36064d128d8060d91374b685 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 20:56:34 -0600 Subject: [PATCH 34/47] aider: fix: Remove unused "fmt" import from mssmt/node.go --- mssmt/node.go | 1 - 1 file changed, 1 deletion(-) diff --git a/mssmt/node.go b/mssmt/node.go index ee9d48983..bac4afae1 100644 --- a/mssmt/node.go +++ b/mssmt/node.go @@ -4,7 +4,6 @@ import ( "crypto/sha256" "encoding/binary" "encoding/hex" - "fmt" ) const ( From 960e67be4a3262ce3334938d58b676f0f72de8a5 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 21:22:55 -0600 Subject: [PATCH 35/47] aider: refactor: Rename batched_insert to batchedInsert and add helper functions --- mssmt/compacted_tree.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 894de92f3..38fc66ae6 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -17,8 +17,10 @@ type CompactedTree struct { store TreeStore } -// batched_insert handles the insertion of multiple entries in one go. -func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedInsertionEntry, height int, root *BranchNode) (*BranchNode, error) { ++// batchedInsert recursively inserts a batch of leaf nodes into the MS-SMT. ++// It partitions the given entries based on the bit at the specified height ++// and processes both left and right subtrees accordingly. +func (t *CompactedTree) batchedInsert(tx TreeStoreUpdateTx, entries []BatchedInsertionEntry, height int, root *BranchNode) (*BranchNode, error) { // Base-case: If we've reached the bottom, simply return the current branch. if height >= lastBitIndex { return root, nil @@ -112,7 +114,7 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn } else { // leftChild is not a compacted leaf, so it must be a branch; recurse normally. baseLeft := leftChild.(*BranchNode) - newLeft, err = t.batched_insert(tx, leftEntries, height+1, baseLeft) + newLeft, err = t.batchedInsert(tx, leftEntries, height+1, baseLeft) if err != nil { return nil, err } @@ -204,7 +206,7 @@ func (t *CompactedTree) batched_insert(tx TreeStoreUpdateTx, entries []BatchedIn } else { // rightChild is not a compacted leaf, so it must be a branch; recurse normally. baseRight := rightChild.(*BranchNode) - newRight, err = t.batched_insert(tx, rightEntries, height+1, baseRight) + newRight, err = t.batchedInsert(tx, rightEntries, height+1, baseRight) if err != nil { return nil, err } @@ -268,8 +270,8 @@ func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []BatchedInse } } - // Call the new batched_insert method. - newRoot, err := t.batched_insert(tx, entries, 0, branchRoot) + // Call the new batchedInsert method. + newRoot, err := t.batchedInsert(tx, entries, 0, branchRoot) if err != nil { return err } @@ -281,7 +283,19 @@ func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []BatchedInse return t, nil } -// BatchedInsertionEntry represents one leaf insertion. ++// partitionEntries splits the insertion entries into two slices based on the ++// bit at the given height. The leftEntries slice collects all entries whose ++// key has the bit 0 at that position, while rightEntries collects those with bit 1. ++func partitionEntries(entries []BatchedInsertionEntry, height int) (leftEntries, rightEntries []BatchedInsertionEntry) { ++ for _, entry := range entries { ++ if bitIndex(uint8(height), &entry.Key) == 0 { ++ leftEntries = append(leftEntries, entry) ++ } else { ++ rightEntries = append(rightEntries, entry) ++ } ++ } ++ return ++} type BatchedInsertionEntry struct { Key [hashSize]byte Leaf *LeafNode From 5968e0f19da023dcd20726dcbfbaaf068e9f5d3c Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 21:23:51 -0600 Subject: [PATCH 36/47] aider: feat: Implement recursive subtree processing for batched insertion --- mssmt/compacted_tree.go | 93 ++++++++++++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 16 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 38fc66ae6..0357a8382 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -129,7 +129,7 @@ func (t *CompactedTree) batchedInsert(tx TreeStoreUpdateTx, entries []BatchedIns } } else { baseLeft := EmptyTree[height+1].(*BranchNode) - newLeft, err = t.batched_insert(tx, leftEntries, height+1, baseLeft) + newLeft, err = t.batchedInsert(tx, leftEntries, height+1, baseLeft) if err != nil { return nil, err } @@ -221,7 +221,7 @@ func (t *CompactedTree) batchedInsert(tx TreeStoreUpdateTx, entries []BatchedIns } } else { baseRight := EmptyTree[height+1].(*BranchNode) - newRight, err = t.batched_insert(tx, rightEntries, height+1, baseRight) + newRight, err = t.batchedInsert(tx, rightEntries, height+1, baseRight) if err != nil { return nil, err } @@ -283,23 +283,84 @@ func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []BatchedInse return t, nil } -+// partitionEntries splits the insertion entries into two slices based on the -+// bit at the given height. The leftEntries slice collects all entries whose -+// key has the bit 0 at that position, while rightEntries collects those with bit 1. -+func partitionEntries(entries []BatchedInsertionEntry, height int) (leftEntries, rightEntries []BatchedInsertionEntry) { -+ for _, entry := range entries { -+ if bitIndex(uint8(height), &entry.Key) == 0 { -+ leftEntries = append(leftEntries, entry) -+ } else { -+ rightEntries = append(rightEntries, entry) ++// processSubtree handles the recursive insertion for a subtree (left or right) ++// based on the provided child node. Depending on whether the child is an ++// existing compacted leaf or branch, it either replaces, merges, or recurses ++// deeper to insert the batch of entries. ++// It returns the updated child Node on success. ++func (t *CompactedTree) processSubtree(tx TreeStoreUpdateTx, height int, ++ entries []BatchedInsertionEntry, child Node) (Node, error) { ++ if child != EmptyTree[height+1] { ++ if cl, ok := child.(*CompactedLeafNode); ok { ++ if len(entries) == 1 { ++ entry := entries[0] ++ if entry.Key == cl.Key() { ++ // Replacement: update the existing compacted leaf. ++ newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) ++ if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { ++ return nil, err ++ } ++ if err := tx.InsertCompactedLeaf(newLeaf); err != nil { ++ return nil, err ++ } ++ return newLeaf, nil ++ } ++ // Collision detected: merge the new entry with the existing compacted leaf. ++ return t.merge(tx, height+1, entry.Key, entry.Leaf, cl.Key(), cl.LeafNode) ++ } ++ ++ // For multiple entries, check if all match the existing key. ++ allMatch := true ++ for _, entry := range entries { ++ if entry.Key != cl.Key() { ++ allMatch = false ++ break ++ } ++ } ++ if allMatch { ++ // All entries match; perform a replacement with the last entry. ++ lastEntry := entries[len(entries)-1] ++ newLeaf := NewCompactedLeafNode(height+1, &lastEntry.Key, lastEntry.Leaf) ++ if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { ++ return nil, err ++ } ++ if err := tx.InsertCompactedLeaf(newLeaf); err != nil { ++ return nil, err ++ } ++ return newLeaf, nil ++ } ++ ++ // Otherwise, merge using the first differing entry. ++ var mergeEntry *BatchedInsertionEntry ++ for _, entry := range entries { ++ if entry.Key != cl.Key() { ++ mergeEntry = &entry ++ break ++ } ++ } ++ if mergeEntry == nil { ++ return nil, fmt.Errorf("unexpected nil merge entry") ++ } ++ return t.merge(tx, height+1, mergeEntry.Key, mergeEntry.Leaf, cl.Key(), cl.LeafNode) + } ++ ++ // If child is not a compacted leaf (i.e. it is a branch), recurse normally. ++ baseChild := child.(*BranchNode) ++ return t.batchedInsert(tx, entries, height+1, baseChild) + } -+ return ++ ++ // When the child is empty, insert a new compacted leaf or process as a branch. ++ if len(entries) == 1 { ++ entry := entries[0] ++ newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) ++ if err := tx.InsertCompactedLeaf(newLeaf.(*CompactedLeafNode)); err != nil { ++ return nil, err ++ } ++ return newLeaf, nil ++ } ++ baseChild := EmptyTree[height+1].(*BranchNode) ++ return t.batchedInsert(tx, entries, height+1, baseChild) +} -type BatchedInsertionEntry struct { - Key [hashSize]byte - Leaf *LeafNode -} var _ Tree = (*CompactedTree)(nil) From c210afb58e152bda90cbb85f4a11c60c87322d6f Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 21:24:28 -0600 Subject: [PATCH 37/47] aider: refactor: Simplify batchedInsert by extracting subtree processing logic --- mssmt/compacted_tree.go | 192 ++-------------------------------------- 1 file changed, 9 insertions(+), 183 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 0357a8382..ee1301820 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -31,15 +31,7 @@ func (t *CompactedTree) batchedInsert(tx TreeStoreUpdateTx, entries []BatchedIns return root, nil } - // Partition entries into two groups based on bit at current height. - var leftEntries, rightEntries []BatchedInsertionEntry - for _, entry := range entries { - if bitIndex(uint8(height), &entry.Key) == 0 { - leftEntries = append(leftEntries, entry) - } else { - rightEntries = append(rightEntries, entry) - } - } + leftEntries, rightEntries := partitionEntries(entries, height) // Get the current children from the node. leftChild, rightChild, err := tx.GetChildren(height, root.NodeHash()) @@ -47,187 +39,21 @@ func (t *CompactedTree) batchedInsert(tx TreeStoreUpdateTx, entries []BatchedIns return nil, err } - // Process left subtree: - var newLeft Node + // Process left subtree using the helper function. if len(leftEntries) > 0 { - // Check if the current left child is not empty. - if leftChild != EmptyTree[height+1] { - // If the existing child is a compacted leaf, we must handle potential collisions. - if cl, ok := leftChild.(*CompactedLeafNode); ok { - if len(leftEntries) == 1 { - entry := leftEntries[0] - if entry.Key == cl.Key() { - // Replacement: update the compacted leaf. - newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) - if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { - return nil, err - } - if err := tx.InsertCompactedLeaf(newLeaf); err != nil { - return nil, err - } - newLeft = newLeaf - } else { - // Collision – keys differ: call merge to combine the new entry with the existing compacted leaf. - newLeft, err = t.merge(tx, height+1, entry.Key, entry.Leaf, cl.Key(), cl.LeafNode) - if err != nil { - return nil, err - } - } - } else { - // Multiple batch entries – check if they all match the existing key. - allMatch := true - for _, entry := range leftEntries { - if entry.Key != cl.Key() { - allMatch = false - break - } - } - if allMatch { - // All entries match; take the last one as replacement. - lastEntry := leftEntries[len(leftEntries)-1] - newLeaf := NewCompactedLeafNode(height+1, &lastEntry.Key, lastEntry.Leaf) - if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { - return nil, err - } - if err := tx.InsertCompactedLeaf(newLeaf); err != nil { - return nil, err - } - newLeft = newLeaf - } else { - // At least one entry has a different key – merge using the first differing entry. - var mergeEntry *BatchedInsertionEntry - for _, entry := range leftEntries { - if entry.Key != cl.Key() { - mergeEntry = &entry - break - } - } - if mergeEntry == nil { - return nil, fmt.Errorf("unexpected nil merge entry") - } - newLeft, err = t.merge(tx, height+1, mergeEntry.Key, mergeEntry.Leaf, cl.Key(), cl.LeafNode) - if err != nil { - return nil, err - } - } - } - } else { - // leftChild is not a compacted leaf, so it must be a branch; recurse normally. - baseLeft := leftChild.(*BranchNode) - newLeft, err = t.batchedInsert(tx, leftEntries, height+1, baseLeft) - if err != nil { - return nil, err - } - } - } else { - // The left child is empty. - if len(leftEntries) == 1 { - entry := leftEntries[0] - newLeft = NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) - if err := tx.InsertCompactedLeaf(newLeft.(*CompactedLeafNode)); err != nil { - return nil, err - } - } else { - baseLeft := EmptyTree[height+1].(*BranchNode) - newLeft, err = t.batchedInsert(tx, leftEntries, height+1, baseLeft) - if err != nil { - return nil, err - } - } + newLeft, err := t.processSubtree(tx, height, leftEntries, leftChild) + if err != nil { + return nil, err } - // Use newLeft as the computed left child. leftChild = newLeft } - // Process right subtree: - var newRight Node + // Process right subtree using the helper function. if len(rightEntries) > 0 { - // Check if the current right child is not empty. - if rightChild != EmptyTree[height+1] { - // If the existing child is a compacted leaf, we must handle potential collisions. - if cr, ok := rightChild.(*CompactedLeafNode); ok { - if len(rightEntries) == 1 { - entry := rightEntries[0] - if entry.Key == cr.Key() { - // Replacement: update the compacted leaf. - newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) - if err := tx.DeleteCompactedLeaf(cr.NodeHash()); err != nil { - return nil, err - } - if err := tx.InsertCompactedLeaf(newLeaf); err != nil { - return nil, err - } - newRight = newLeaf - } else { - // Collision – keys differ: call merge to combine the new entry with the existing compacted leaf. - newRight, err = t.merge(tx, height+1, entry.Key, entry.Leaf, cr.Key(), cr.LeafNode) - if err != nil { - return nil, err - } - } - } else { - // Multiple batch entries – check if they all match the existing key. - allMatch := true - for _, entry := range rightEntries { - if entry.Key != cr.Key() { - allMatch = false - break - } - } - if allMatch { - // All entries match; take the last one as replacement. - lastEntry := rightEntries[len(rightEntries)-1] - newLeaf := NewCompactedLeafNode(height+1, &lastEntry.Key, lastEntry.Leaf) - if err := tx.DeleteCompactedLeaf(cr.NodeHash()); err != nil { - return nil, err - } - if err := tx.InsertCompactedLeaf(newLeaf); err != nil { - return nil, err - } - newRight = newLeaf - } else { - // At least one entry has a different key – merge using the first differing entry. - var mergeEntry *BatchedInsertionEntry - for _, entry := range rightEntries { - if entry.Key != cr.Key() { - mergeEntry = &entry - break - } - } - if mergeEntry == nil { - return nil, fmt.Errorf("unexpected nil merge entry") - } - newRight, err = t.merge(tx, height+1, mergeEntry.Key, mergeEntry.Leaf, cr.Key(), cr.LeafNode) - if err != nil { - return nil, err - } - } - } - } else { - // rightChild is not a compacted leaf, so it must be a branch; recurse normally. - baseRight := rightChild.(*BranchNode) - newRight, err = t.batchedInsert(tx, rightEntries, height+1, baseRight) - if err != nil { - return nil, err - } - } - } else { - // The right child is empty. - if len(rightEntries) == 1 { - entry := rightEntries[0] - newRight = NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) - if err := tx.InsertCompactedLeaf(newRight.(*CompactedLeafNode)); err != nil { - return nil, err - } - } else { - baseRight := EmptyTree[height+1].(*BranchNode) - newRight, err = t.batchedInsert(tx, rightEntries, height+1, baseRight) - if err != nil { - return nil, err - } - } + newRight, err := t.processSubtree(tx, height, rightEntries, rightChild) + if err != nil { + return nil, err } - // Use newRight as the computed right child. rightChild = newRight } From 8c2668601d96812c120896bdbffb8dea2698a19f Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 21:33:34 -0600 Subject: [PATCH 38/47] aider: fix: Remove leading '+' characters from comments in compacted_tree.go --- mssmt/compacted_tree.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index ee1301820..7806ea500 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -17,9 +17,9 @@ type CompactedTree struct { store TreeStore } -+// batchedInsert recursively inserts a batch of leaf nodes into the MS-SMT. -+// It partitions the given entries based on the bit at the specified height -+// and processes both left and right subtrees accordingly. + // batchedInsert recursively inserts a batch of leaf nodes into the MS-SMT. + // It partitions the given entries based on the bit at the specified height + // and processes both left and right subtrees accordingly. func (t *CompactedTree) batchedInsert(tx TreeStoreUpdateTx, entries []BatchedInsertionEntry, height int, root *BranchNode) (*BranchNode, error) { // Base-case: If we've reached the bottom, simply return the current branch. if height >= lastBitIndex { @@ -109,13 +109,13 @@ func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []BatchedInse return t, nil } -+// processSubtree handles the recursive insertion for a subtree (left or right) -+// based on the provided child node. Depending on whether the child is an -+// existing compacted leaf or branch, it either replaces, merges, or recurses -+// deeper to insert the batch of entries. -+// It returns the updated child Node on success. -+func (t *CompactedTree) processSubtree(tx TreeStoreUpdateTx, height int, -+ entries []BatchedInsertionEntry, child Node) (Node, error) { + // processSubtree handles the recursive insertion for a subtree (left or right) + // based on the provided child node. Depending on whether the child is an + // existing compacted leaf or branch, it either replaces, merges, or recurses + // deeper to insert the batch of entries. + // It returns the updated child Node on success. + func (t *CompactedTree) processSubtree(tx TreeStoreUpdateTx, height int, + entries []BatchedInsertionEntry, child Node) (Node, error) { + if child != EmptyTree[height+1] { + if cl, ok := child.(*CompactedLeafNode); ok { + if len(entries) == 1 { From 53ef400e6ef87297fa28abbe1f168d94d345e877 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 21:43:57 -0600 Subject: [PATCH 39/47] aider: fix: Remove leading '+' characters from processSubtree function in compacted_tree.go --- mssmt/compacted_tree.go | 142 ++++++++++++++++++++-------------------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 7806ea500..8528e6296 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -116,77 +116,77 @@ func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []BatchedInse // It returns the updated child Node on success. func (t *CompactedTree) processSubtree(tx TreeStoreUpdateTx, height int, entries []BatchedInsertionEntry, child Node) (Node, error) { -+ if child != EmptyTree[height+1] { -+ if cl, ok := child.(*CompactedLeafNode); ok { -+ if len(entries) == 1 { -+ entry := entries[0] -+ if entry.Key == cl.Key() { -+ // Replacement: update the existing compacted leaf. -+ newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) -+ if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { -+ return nil, err -+ } -+ if err := tx.InsertCompactedLeaf(newLeaf); err != nil { -+ return nil, err -+ } -+ return newLeaf, nil -+ } -+ // Collision detected: merge the new entry with the existing compacted leaf. -+ return t.merge(tx, height+1, entry.Key, entry.Leaf, cl.Key(), cl.LeafNode) -+ } -+ -+ // For multiple entries, check if all match the existing key. -+ allMatch := true -+ for _, entry := range entries { -+ if entry.Key != cl.Key() { -+ allMatch = false -+ break -+ } -+ } -+ if allMatch { -+ // All entries match; perform a replacement with the last entry. -+ lastEntry := entries[len(entries)-1] -+ newLeaf := NewCompactedLeafNode(height+1, &lastEntry.Key, lastEntry.Leaf) -+ if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { -+ return nil, err -+ } -+ if err := tx.InsertCompactedLeaf(newLeaf); err != nil { -+ return nil, err -+ } -+ return newLeaf, nil -+ } -+ -+ // Otherwise, merge using the first differing entry. -+ var mergeEntry *BatchedInsertionEntry -+ for _, entry := range entries { -+ if entry.Key != cl.Key() { -+ mergeEntry = &entry -+ break -+ } -+ } -+ if mergeEntry == nil { -+ return nil, fmt.Errorf("unexpected nil merge entry") -+ } -+ return t.merge(tx, height+1, mergeEntry.Key, mergeEntry.Leaf, cl.Key(), cl.LeafNode) -+ } -+ -+ // If child is not a compacted leaf (i.e. it is a branch), recurse normally. -+ baseChild := child.(*BranchNode) -+ return t.batchedInsert(tx, entries, height+1, baseChild) -+ } -+ -+ // When the child is empty, insert a new compacted leaf or process as a branch. -+ if len(entries) == 1 { -+ entry := entries[0] -+ newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) -+ if err := tx.InsertCompactedLeaf(newLeaf.(*CompactedLeafNode)); err != nil { -+ return nil, err -+ } -+ return newLeaf, nil -+ } -+ baseChild := EmptyTree[height+1].(*BranchNode) -+ return t.batchedInsert(tx, entries, height+1, baseChild) -+} + if child != EmptyTree[height+1] { + if cl, ok := child.(*CompactedLeafNode); ok { + if len(entries) == 1 { + entry := entries[0] + if entry.Key == cl.Key() { + // Replacement: update the existing compacted leaf. + newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) + if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { + return nil, err + } + if err := tx.InsertCompactedLeaf(newLeaf); err != nil { + return nil, err + } + return newLeaf, nil + } + // Collision detected: merge the new entry with the existing compacted leaf. + return t.merge(tx, height+1, entry.Key, entry.Leaf, cl.Key(), cl.LeafNode) + } + + // For multiple entries, check if all match the existing key. + allMatch := true + for _, entry := range entries { + if entry.Key != cl.Key() { + allMatch = false + break + } + } + if allMatch { + // All entries match; perform a replacement with the last entry. + lastEntry := entries[len(entries)-1] + newLeaf := NewCompactedLeafNode(height+1, &lastEntry.Key, lastEntry.Leaf) + if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { + return nil, err + } + if err := tx.InsertCompactedLeaf(newLeaf); err != nil { + return nil, err + } + return newLeaf, nil + } + + // Otherwise, merge using the first differing entry. + var mergeEntry *BatchedInsertionEntry + for _, entry := range entries { + if entry.Key != cl.Key() { + mergeEntry = &entry + break + } + } + if mergeEntry == nil { + return nil, fmt.Errorf("unexpected nil merge entry") + } + return t.merge(tx, height+1, mergeEntry.Key, mergeEntry.Leaf, cl.Key(), cl.LeafNode) + } + + // If child is not a compacted leaf (i.e. it is a branch), recurse normally. + baseChild := child.(*BranchNode) + return t.batchedInsert(tx, entries, height+1, baseChild) + } + + // When the child is empty, insert a new compacted leaf or process as a branch. + if len(entries) == 1 { + entry := entries[0] + newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) + if err := tx.InsertCompactedLeaf(newLeaf.(*CompactedLeafNode)); err != nil { + return nil, err + } + return newLeaf, nil + } + baseChild := EmptyTree[height+1].(*BranchNode) + return t.batchedInsert(tx, entries, height+1, baseChild) +} var _ Tree = (*CompactedTree)(nil) From 934adbc51dd885714083322b233b4ca058e3fc4e Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 21:46:47 -0600 Subject: [PATCH 40/47] aider: feat: Add BatchedInsertionEntry type and partitionEntries function --- mssmt/compacted_tree.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 8528e6296..dadf701d1 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -22,6 +22,19 @@ type CompactedTree struct { // and processes both left and right subtrees accordingly. func (t *CompactedTree) batchedInsert(tx TreeStoreUpdateTx, entries []BatchedInsertionEntry, height int, root *BranchNode) (*BranchNode, error) { // Base-case: If we've reached the bottom, simply return the current branch. +// partitionEntries splits the given batched insertion entries into two slices +// based on the bit at the provided height. Entries with bit 0 go into leftEntries +// and those with bit 1 go into rightEntries. +func partitionEntries(entries []BatchedInsertionEntry, height int) (leftEntries, rightEntries []BatchedInsertionEntry) { + for _, entry := range entries { + if bitIndex(uint8(height), &entry.Key) == 0 { + leftEntries = append(leftEntries, entry) + } else { + rightEntries = append(rightEntries, entry) + } + } + return +} if height >= lastBitIndex { return root, nil } @@ -179,7 +192,7 @@ func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []BatchedInse if len(entries) == 1 { entry := entries[0] newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) - if err := tx.InsertCompactedLeaf(newLeaf.(*CompactedLeafNode)); err != nil { + if err := tx.InsertCompactedLeaf(newLeaf); err != nil { return nil, err } return newLeaf, nil From a3591a5ae431d9f9fb6c726e1b788be29db93564 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 21:48:18 -0600 Subject: [PATCH 41/47] aider: refactor: Move partitionEntries function to package level to fix syntax error --- mssmt/compacted_tree.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index dadf701d1..209b971c8 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -7,6 +7,20 @@ import ( "sort" ) +// partitionEntries splits the given batched insertion entries into +// two slices based on the bit at the provided height. +// Entries with bit 0 go into leftEntries and those with bit 1 into rightEntries. +func partitionEntries(entries []BatchedInsertionEntry, height int) (leftEntries, rightEntries []BatchedInsertionEntry) { + for _, entry := range entries { + if bitIndex(uint8(height), &entry.Key) == 0 { + leftEntries = append(leftEntries, entry) + } else { + rightEntries = append(rightEntries, entry) + } + } + return +} + // CompactedTree represents a compacted Merkle-Sum Sparse Merkle Tree (MS-SMT). // The tree has the same properties as a normal MS-SMT tree and is able to // create the same proofs and same root as the FullTree implemented in this @@ -22,19 +36,6 @@ type CompactedTree struct { // and processes both left and right subtrees accordingly. func (t *CompactedTree) batchedInsert(tx TreeStoreUpdateTx, entries []BatchedInsertionEntry, height int, root *BranchNode) (*BranchNode, error) { // Base-case: If we've reached the bottom, simply return the current branch. -// partitionEntries splits the given batched insertion entries into two slices -// based on the bit at the provided height. Entries with bit 0 go into leftEntries -// and those with bit 1 go into rightEntries. -func partitionEntries(entries []BatchedInsertionEntry, height int) (leftEntries, rightEntries []BatchedInsertionEntry) { - for _, entry := range entries { - if bitIndex(uint8(height), &entry.Key) == 0 { - leftEntries = append(leftEntries, entry) - } else { - rightEntries = append(rightEntries, entry) - } - } - return -} if height >= lastBitIndex { return root, nil } From 3c19cbe97930a258f9466b7d04a04febb7e458c0 Mon Sep 17 00:00:00 2001 From: "Olaoluwa Osuntokun (aider)" Date: Sun, 2 Feb 2025 21:48:56 -0600 Subject: [PATCH 42/47] aider: feat: Add BatchedInsertionEntry type for batched insertions in MS-SMT --- mssmt/compacted_tree.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 209b971c8..d6de6cdbb 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -7,6 +7,14 @@ import ( "sort" ) +// BatchedInsertionEntry represents an entry used for batched +// insertions into the MS-SMT. It consists of a key and the +// associated leaf node to insert. +type BatchedInsertionEntry struct { + Key [32]byte + Leaf *LeafNode +) + // partitionEntries splits the given batched insertion entries into // two slices based on the bit at the provided height. // Entries with bit 0 go into leftEntries and those with bit 1 into rightEntries. From 596f8012028456240d463b15bb20e0bb69798b88 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Sun, 2 Feb 2025 21:54:05 -0600 Subject: [PATCH 43/47] mssmt: fix formatting --- mssmt/compacted_tree.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index d6de6cdbb..d4011ef31 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -1,8 +1,8 @@ package mssmt import ( - "context" "bytes" + "context" "fmt" "sort" ) @@ -13,7 +13,7 @@ import ( type BatchedInsertionEntry struct { Key [32]byte Leaf *LeafNode -) +} // partitionEntries splits the given batched insertion entries into // two slices based on the bit at the provided height. @@ -39,9 +39,9 @@ type CompactedTree struct { store TreeStore } - // batchedInsert recursively inserts a batch of leaf nodes into the MS-SMT. - // It partitions the given entries based on the bit at the specified height - // and processes both left and right subtrees accordingly. +// batchedInsert recursively inserts a batch of leaf nodes into the MS-SMT. +// It partitions the given entries based on the bit at the specified height +// and processes both left and right subtrees accordingly. func (t *CompactedTree) batchedInsert(tx TreeStoreUpdateTx, entries []BatchedInsertionEntry, height int, root *BranchNode) (*BranchNode, error) { // Base-case: If we've reached the bottom, simply return the current branch. if height >= lastBitIndex { @@ -131,13 +131,13 @@ func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []BatchedInse return t, nil } - // processSubtree handles the recursive insertion for a subtree (left or right) - // based on the provided child node. Depending on whether the child is an - // existing compacted leaf or branch, it either replaces, merges, or recurses - // deeper to insert the batch of entries. - // It returns the updated child Node on success. - func (t *CompactedTree) processSubtree(tx TreeStoreUpdateTx, height int, - entries []BatchedInsertionEntry, child Node) (Node, error) { +// processSubtree handles the recursive insertion for a subtree (left or right) +// based on the provided child node. Depending on whether the child is an +// existing compacted leaf or branch, it either replaces, merges, or recurses +// deeper to insert the batch of entries. +// It returns the updated child Node on success. +func (t *CompactedTree) processSubtree(tx TreeStoreUpdateTx, height int, + entries []BatchedInsertionEntry, child Node) (Node, error) { if child != EmptyTree[height+1] { if cl, ok := child.(*CompactedLeafNode); ok { if len(entries) == 1 { From 6d42508f25de0d2b1db4b30f5cc573c94aedf3af Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Sun, 2 Feb 2025 22:46:59 -0600 Subject: [PATCH 44/47] mssst: revert changes to walkDown+Extract --- mssmt/compacted_tree.go | 17 +++++++++++++-- mssmt/node.go | 47 ++++++++++++++++------------------------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index d4011ef31..8c219ddcc 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -263,9 +263,19 @@ func (t *CompactedTree) walkDown(tx TreeStoreViewTx, key *[hashSize]byte, switch node := next.(type) { case *CompactedLeafNode: - // Our next node is a compacted leaf. We simply return the underlying leaf. - return node.LeafNode, nil + // Our next node is a compacted leaf. We just need to + // expand it so we can continue our walk down the tree. + next = node.Extract(i) + + // Sibling might be a compacted leaf too, in which case + // we need to extract it as well. + if compSibling, ok := sibling.(*CompactedLeafNode); ok { + sibling = compSibling.Extract(i) + } + // Now that all required branches are reconstructed we + // can continue the search for the leaf matching the + // passed key. for j := i; j <= lastBitIndex; j++ { if iter != nil { err := iter(j, next, sibling, current) @@ -276,6 +286,9 @@ func (t *CompactedTree) walkDown(tx TreeStoreViewTx, key *[hashSize]byte, current = next if j < lastBitIndex { + // Since we have all the branches we + // need extracted already we can just + // continue walking down. branch := current.(*BranchNode) next, sibling = stepOrder( j+1, key, branch.Left, diff --git a/mssmt/node.go b/mssmt/node.go index bac4afae1..e7bad1e09 100644 --- a/mssmt/node.go +++ b/mssmt/node.go @@ -161,35 +161,24 @@ func (c *CompactedLeafNode) Key() [32]byte { return c.key } -func (c *CompactedLeafNode) Extract(requestedHeight int) Node { - target := requestedHeight - if target >= c.Height { - target = c.Height - 1 - } - - var current Node = c.LeafNode - - for j := c.Height - 1; j >= target+1; j-- { - if bitIndex(uint8(j), &c.key) == 0 { - current = NewBranch(current, EmptyTree[j+1]) - } else { - current = NewBranch(EmptyTree[j+1], current) - } - } - - sibling := EmptyTree[target+1] - if target+1 == MaxTreeLevels { - sibling = NewBranch(sibling, sibling) - } - - var result Node - if bitIndex(uint8(target), &c.key) == 0 { - result = NewBranch(current, sibling) - } else { - result = NewBranch(sibling, current) - } - - return result +// Extract extracts the subtree represented by this compacted leaf and returns +// the topmost node in the tree. +func (c *CompactedLeafNode) Extract(height int) Node { + var current Node = c.LeafNode + + // Walk up and recreate the missing branches. + for j := MaxTreeLevels; j > height+1; j-- { + var left, right Node + if bitIndex(uint8(j-1), &c.key) == 0 { + left, right = current, EmptyTree[j] + } else { + left, right = EmptyTree[j], current + } + + current = NewBranch(left, right) + } + + return current } // Copy returns a deep copy of the compacted leaf node. From 7447a5a4332f486d78b6b4a7a3efdd91523967e7 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Sun, 2 Feb 2025 23:20:40 -0600 Subject: [PATCH 45/47] aider: refactor: Simplify batched insertion logic in CompactedTree implementation --- .gitignore | 1 + mssmt/compacted_tree.go | 390 +++++++++++++++++++++------------------- 2 files changed, 206 insertions(+), 185 deletions(-) diff --git a/.gitignore b/.gitignore index eb4652d59..20fea9b2f 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ vendor/ # Release builds /taproot-assets-* +.aider* diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index 8c219ddcc..a078958b4 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -15,20 +15,6 @@ type BatchedInsertionEntry struct { Leaf *LeafNode } -// partitionEntries splits the given batched insertion entries into -// two slices based on the bit at the provided height. -// Entries with bit 0 go into leftEntries and those with bit 1 into rightEntries. -func partitionEntries(entries []BatchedInsertionEntry, height int) (leftEntries, rightEntries []BatchedInsertionEntry) { - for _, entry := range entries { - if bitIndex(uint8(height), &entry.Key) == 0 { - leftEntries = append(leftEntries, entry) - } else { - rightEntries = append(rightEntries, entry) - } - } - return -} - // CompactedTree represents a compacted Merkle-Sum Sparse Merkle Tree (MS-SMT). // The tree has the same properties as a normal MS-SMT tree and is able to // create the same proofs and same root as the FullTree implemented in this @@ -39,177 +25,6 @@ type CompactedTree struct { store TreeStore } -// batchedInsert recursively inserts a batch of leaf nodes into the MS-SMT. -// It partitions the given entries based on the bit at the specified height -// and processes both left and right subtrees accordingly. -func (t *CompactedTree) batchedInsert(tx TreeStoreUpdateTx, entries []BatchedInsertionEntry, height int, root *BranchNode) (*BranchNode, error) { - // Base-case: If we've reached the bottom, simply return the current branch. - if height >= lastBitIndex { - return root, nil - } - - // Guard against empty batch. - if len(entries) == 0 { - return root, nil - } - - leftEntries, rightEntries := partitionEntries(entries, height) - - // Get the current children from the node. - leftChild, rightChild, err := tx.GetChildren(height, root.NodeHash()) - if err != nil { - return nil, err - } - - // Process left subtree using the helper function. - if len(leftEntries) > 0 { - newLeft, err := t.processSubtree(tx, height, leftEntries, leftChild) - if err != nil { - return nil, err - } - leftChild = newLeft - } - - // Process right subtree using the helper function. - if len(rightEntries) > 0 { - newRight, err := t.processSubtree(tx, height, rightEntries, rightChild) - if err != nil { - return nil, err - } - rightChild = newRight - } - - // Create the updated branch from the new left and right children. - var updatedBranch *BranchNode - updatedBranch = NewBranch(leftChild, rightChild) - - // Delete the old branch and insert the new one. - if root != EmptyTree[height] { - if err := tx.DeleteBranch(root.NodeHash()); err != nil { - return nil, err - } - } - if !IsEqualNode(updatedBranch, EmptyTree[height]) { - if err := tx.InsertBranch(updatedBranch); err != nil { - return nil, err - } - } - - return updatedBranch, nil -} - -// BatchedInsert inserts multiple leaf nodes at the given keys within the MS-SMT. -func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []BatchedInsertionEntry) (Tree, error) { - sort.Slice(entries, func(i, j int) bool { - return bytes.Compare(entries[i].Key[:], entries[j].Key[:]) < 0 - }) - - err := t.store.Update(ctx, func(tx TreeStoreUpdateTx) error { - currentRoot, err := tx.RootNode() - if err != nil { - return err - } - branchRoot := currentRoot.(*BranchNode) - - // (Optional) Loop over entries and check for sum overflow. - for _, entry := range entries { - if err := CheckSumOverflowUint64(branchRoot.NodeSum(), entry.Leaf.NodeSum()); err != nil { - return fmt.Errorf("batched insert key %v sum overflow: %w", entry.Key, err) - } - } - - // Call the new batchedInsert method. - newRoot, err := t.batchedInsert(tx, entries, 0, branchRoot) - if err != nil { - return err - } - return tx.UpdateRoot(newRoot) - }) - if err != nil { - return nil, err - } - return t, nil -} - -// processSubtree handles the recursive insertion for a subtree (left or right) -// based on the provided child node. Depending on whether the child is an -// existing compacted leaf or branch, it either replaces, merges, or recurses -// deeper to insert the batch of entries. -// It returns the updated child Node on success. -func (t *CompactedTree) processSubtree(tx TreeStoreUpdateTx, height int, - entries []BatchedInsertionEntry, child Node) (Node, error) { - if child != EmptyTree[height+1] { - if cl, ok := child.(*CompactedLeafNode); ok { - if len(entries) == 1 { - entry := entries[0] - if entry.Key == cl.Key() { - // Replacement: update the existing compacted leaf. - newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) - if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { - return nil, err - } - if err := tx.InsertCompactedLeaf(newLeaf); err != nil { - return nil, err - } - return newLeaf, nil - } - // Collision detected: merge the new entry with the existing compacted leaf. - return t.merge(tx, height+1, entry.Key, entry.Leaf, cl.Key(), cl.LeafNode) - } - - // For multiple entries, check if all match the existing key. - allMatch := true - for _, entry := range entries { - if entry.Key != cl.Key() { - allMatch = false - break - } - } - if allMatch { - // All entries match; perform a replacement with the last entry. - lastEntry := entries[len(entries)-1] - newLeaf := NewCompactedLeafNode(height+1, &lastEntry.Key, lastEntry.Leaf) - if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { - return nil, err - } - if err := tx.InsertCompactedLeaf(newLeaf); err != nil { - return nil, err - } - return newLeaf, nil - } - - // Otherwise, merge using the first differing entry. - var mergeEntry *BatchedInsertionEntry - for _, entry := range entries { - if entry.Key != cl.Key() { - mergeEntry = &entry - break - } - } - if mergeEntry == nil { - return nil, fmt.Errorf("unexpected nil merge entry") - } - return t.merge(tx, height+1, mergeEntry.Key, mergeEntry.Leaf, cl.Key(), cl.LeafNode) - } - - // If child is not a compacted leaf (i.e. it is a branch), recurse normally. - baseChild := child.(*BranchNode) - return t.batchedInsert(tx, entries, height+1, baseChild) - } - - // When the child is empty, insert a new compacted leaf or process as a branch. - if len(entries) == 1 { - entry := entries[0] - newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) - if err := tx.InsertCompactedLeaf(newLeaf); err != nil { - return nil, err - } - return newLeaf, nil - } - baseChild := EmptyTree[height+1].(*BranchNode) - return t.batchedInsert(tx, entries, height+1, baseChild) -} - var _ Tree = (*CompactedTree)(nil) // NewCompactedTree initializes an empty MS-SMT backed by `store`. @@ -503,6 +318,211 @@ func (t *CompactedTree) Insert(ctx context.Context, key [hashSize]byte, return t, nil } +func (t *CompactedTree) processCompactedLeaf(tx TreeStoreUpdateTx, height int, + entries []BatchedInsertionEntry, cl *CompactedLeafNode) (Node, error) { + + // processCompactedLeaf handles the case when the current child node is + // a compacted leaf. Depending on the batch of new entries, it will either + // replace the leaf or merge it with a conflicting entry. + + // Case 1: Only one new entry. + if len(entries) == 1 { + entry := entries[0] + if entry.Key == cl.Key() { + // Replacement: key matches, so update the compacted leaf with the + // new leaf data. + newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) + if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { + return nil, err + } + if err := tx.InsertCompactedLeaf(newLeaf); err != nil { + return nil, err + } + return newLeaf, nil + } + // Conflict: key differs – merge the new entry with the existing leaf. + return t.merge(tx, height+1, entry.Key, entry.Leaf, cl.Key(), cl.LeafNode) + } + + // Case 2: Multiple entries. + // First, check whether every entry has the same key as the compacted leaf. + allMatch := true + for _, entry := range entries { + if entry.Key != cl.Key() { + allMatch = false + break + } + } + if allMatch { + // All entries match; replace with the last entry's data. + lastEntry := entries[len(entries)-1] + newLeaf := NewCompactedLeafNode(height+1, &lastEntry.Key, lastEntry.Leaf) + if err := tx.DeleteCompactedLeaf(cl.NodeHash()); err != nil { + return nil, err + } + if err := tx.InsertCompactedLeaf(newLeaf); err != nil { + return nil, err + } + return newLeaf, nil + } + + // Otherwise, find the first entry that differs and perform a merge. + var mergeEntry *BatchedInsertionEntry + for _, entry := range entries { + if entry.Key != cl.Key() { + mergeEntry = &entry + break + } + } + if mergeEntry == nil { + return nil, fmt.Errorf("unexpected nil merge entry") + } + return t.merge(tx, height+1, mergeEntry.Key, mergeEntry.Leaf, + cl.Key(), cl.LeafNode) +} + +// processSubtree processes a subtree of the MS-SMT based on the provided +// height, entries, and child node. It handles the case where the child node is +// either empty or a compacted leaf, and returns the updated child node. +func (t *CompactedTree) processSubtree(tx TreeStoreUpdateTx, height int, + entries []BatchedInsertionEntry, child Node) (Node, error) { + + // If the child is not the default empty node, then we need to process + // it accordingly. + if child != EmptyTree[height+1] { + // If the child is a compacted leaf then delegate to our helper. + if cl, ok := child.(*CompactedLeafNode); ok { + return t.processCompactedLeaf(tx, height, entries, cl) + } + + // Otherwise, child is assumed to be a branch node: + baseChild := child.(*BranchNode) + return t.batchedInsert(tx, entries, height+1, baseChild) + } + + // If the child is empty: + if len(entries) == 1 { + // With a single entry, simply create a new compacted leaf. + entry := entries[0] + newLeaf := NewCompactedLeafNode(height+1, &entry.Key, entry.Leaf) + if err := tx.InsertCompactedLeaf(newLeaf); err != nil { + return nil, err + } + return newLeaf, nil + } + + // When multiple entries share an empty child, use an empty branch node + // to recursively process the batch. + baseChild := EmptyTree[height+1].(*BranchNode) + return t.batchedInsert(tx, entries, height+1, baseChild) +} + +// partitionEntries splits the given batched insertion entries into +// two slices based on the bit at the provided height. +// Entries with bit 0 go into leftEntries and those with bit 1 into rightEntries. +func partitionEntries(entries []BatchedInsertionEntry, height int) (leftEntries, rightEntries []BatchedInsertionEntry) { + for _, entry := range entries { + if bitIndex(uint8(height), &entry.Key) == 0 { + leftEntries = append(leftEntries, entry) + } else { + rightEntries = append(rightEntries, entry) + } + } + return +} + +// batchedInsert recursively inserts a batch of leaf nodes into the MS-SMT. +// It partitions the given entries based on the bit at the specified height +// and processes both left and right subtrees accordingly. +func (t *CompactedTree) batchedInsert(tx TreeStoreUpdateTx, entries []BatchedInsertionEntry, height int, root *BranchNode) (*BranchNode, error) { + // Base-case: If we've reached the bottom, simply return the current branch. + if height >= lastBitIndex { + return root, nil + } + + // Guard against empty batch. + if len(entries) == 0 { + return root, nil + } + + leftEntries, rightEntries := partitionEntries(entries, height) + + // Get the current children from the node. + leftChild, rightChild, err := tx.GetChildren(height, root.NodeHash()) + if err != nil { + return nil, err + } + + // Process left subtree using the helper function. + if len(leftEntries) > 0 { + newLeft, err := t.processSubtree(tx, height, leftEntries, leftChild) + if err != nil { + return nil, err + } + leftChild = newLeft + } + + // Process right subtree using the helper function. + if len(rightEntries) > 0 { + newRight, err := t.processSubtree(tx, height, rightEntries, rightChild) + if err != nil { + return nil, err + } + rightChild = newRight + } + + // Create the updated branch from the new left and right children. + var updatedBranch *BranchNode + updatedBranch = NewBranch(leftChild, rightChild) + + // Delete the old branch and insert the new one. + if root != EmptyTree[height] { + if err := tx.DeleteBranch(root.NodeHash()); err != nil { + return nil, err + } + } + if !IsEqualNode(updatedBranch, EmptyTree[height]) { + if err := tx.InsertBranch(updatedBranch); err != nil { + return nil, err + } + } + + return updatedBranch, nil +} + +// BatchedInsert inserts multiple leaf nodes at the given keys within the MS-SMT. +func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []BatchedInsertionEntry) (Tree, error) { + sort.Slice(entries, func(i, j int) bool { + return bytes.Compare(entries[i].Key[:], entries[j].Key[:]) < 0 + }) + + err := t.store.Update(ctx, func(tx TreeStoreUpdateTx) error { + currentRoot, err := tx.RootNode() + if err != nil { + return err + } + branchRoot := currentRoot.(*BranchNode) + + // (Optional) Loop over entries and check for sum overflow. + for _, entry := range entries { + if err := CheckSumOverflowUint64(branchRoot.NodeSum(), entry.Leaf.NodeSum()); err != nil { + return fmt.Errorf("batched insert key %v sum overflow: %w", entry.Key, err) + } + } + + // Call the new batchedInsert method. + newRoot, err := t.batchedInsert(tx, entries, 0, branchRoot) + if err != nil { + return err + } + return tx.UpdateRoot(newRoot) + }) + if err != nil { + return nil, err + } + return t, nil +} + // Delete deletes the leaf node found at the given key within the MS-SMT. func (t *CompactedTree) Delete(ctx context.Context, key [hashSize]byte) ( Tree, error) { From d32b80d7cf2a58a2d74ee44002d41f538eed7206 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Sun, 2 Feb 2025 23:22:41 -0600 Subject: [PATCH 46/47] aider: feat: Add documentation for processCompactedLeaf function in compacted_tree.go --- mssmt/compacted_tree.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index a078958b4..f9fe3ea1a 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -318,6 +318,34 @@ func (t *CompactedTree) Insert(ctx context.Context, key [hashSize]byte, return t, nil } + +// processCompactedLeaf handles the insertion of a batch of entries into a slot +// that is currently occupied by a compacted leaf. A compacted leaf represents a +// compressed subtree where all branches between a specific height and the actual +// leaf are assumed to be default. Depending on the batched insertion entries, +// this function determines whether to update (i.e. replace) the existing leaf or +// to merge it with a conflicting new entry. +// +// The logic is as follows: +// +// 1. When exactly one entry is provided: +// - If the entry's key matches the compacted leaf’s key, the function treats it +// as a replacement. It deletes the existing compacted leaf from the store and +// inserts a new compacted leaf built from the provided leaf data. +// - If the entry’s key differs from the compacted leaf’s key, a conflict is +// detected and the function calls the merge helper to combine the new leaf with +// the existing leaf into a merged branch. +// +// 2. When multiple entries are provided: +// - First, it checks whether all entries share the same key as the compacted leaf. +// If they do, the function performs a replacement using the data from the last entry +// in the batch. +// - Otherwise, it finds the first entry with a key that differs from the compacted leaf +// and then invokes the merge helper to merge that conflicting leaf with the current one. +// +// In every case, the function returns the updated node (either a new compacted leaf or a +// merged branch) and any error encountered during the processing. + func (t *CompactedTree) processCompactedLeaf(tx TreeStoreUpdateTx, height int, entries []BatchedInsertionEntry, cl *CompactedLeafNode) (Node, error) { From 7e3529c22c415ae07c9e5b7517c64a4cf6b96c7e Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Mon, 3 Feb 2025 00:09:14 -0600 Subject: [PATCH 47/47] aider: refactor: Improve comments and documentation in compacted_tree.go --- mssmt/compacted_tree.go | 149 +++++++++++++++++++++++++++++++++------- 1 file changed, 123 insertions(+), 26 deletions(-) diff --git a/mssmt/compacted_tree.go b/mssmt/compacted_tree.go index f9fe3ea1a..5c1c46234 100644 --- a/mssmt/compacted_tree.go +++ b/mssmt/compacted_tree.go @@ -318,34 +318,32 @@ func (t *CompactedTree) Insert(ctx context.Context, key [hashSize]byte, return t, nil } - // processCompactedLeaf handles the insertion of a batch of entries into a slot // that is currently occupied by a compacted leaf. A compacted leaf represents a // compressed subtree where all branches between a specific height and the actual // leaf are assumed to be default. Depending on the batched insertion entries, -// this function determines whether to update (i.e. replace) the existing leaf or +// this function determines whether to update (i.e. replace) the existing leaf or // to merge it with a conflicting new entry. -// +// // The logic is as follows: -// +// // 1. When exactly one entry is provided: -// - If the entry's key matches the compacted leaf’s key, the function treats it -// as a replacement. It deletes the existing compacted leaf from the store and -// inserts a new compacted leaf built from the provided leaf data. -// - If the entry’s key differs from the compacted leaf’s key, a conflict is -// detected and the function calls the merge helper to combine the new leaf with -// the existing leaf into a merged branch. -// +// - If the entry's key matches the compacted leaf’s key, the function treats it +// as a replacement. It deletes the existing compacted leaf from the store and +// inserts a new compacted leaf built from the provided leaf data. +// - If the entry’s key differs from the compacted leaf’s key, a conflict is +// detected and the function calls the merge helper to combine the new leaf with +// the existing leaf into a merged branch. +// // 2. When multiple entries are provided: -// - First, it checks whether all entries share the same key as the compacted leaf. -// If they do, the function performs a replacement using the data from the last entry -// in the batch. -// - Otherwise, it finds the first entry with a key that differs from the compacted leaf -// and then invokes the merge helper to merge that conflicting leaf with the current one. -// -// In every case, the function returns the updated node (either a new compacted leaf or a +// - First, it checks whether all entries share the same key as the compacted leaf. +// If they do, the function performs a replacement using the data from the last entry +// in the batch. +// - Otherwise, it finds the first entry with a key that differs from the compacted leaf +// and then invokes the merge helper to merge that conflicting leaf with the current one. +// +// In every case, the function returns the updated node (either a new compacted leaf or a // merged branch) and any error encountered during the processing. - func (t *CompactedTree) processCompactedLeaf(tx TreeStoreUpdateTx, height int, entries []BatchedInsertionEntry, cl *CompactedLeafNode) (Node, error) { @@ -409,9 +407,35 @@ func (t *CompactedTree) processCompactedLeaf(tx TreeStoreUpdateTx, height int, cl.Key(), cl.LeafNode) } -// processSubtree processes a subtree of the MS-SMT based on the provided -// height, entries, and child node. It handles the case where the child node is -// either empty or a compacted leaf, and returns the updated child node. +// processSubtree processes a batch of insertion entries for a specific +// subtree at the given height. +// +// Depending on the current state of the child node at that subtree slot, +// this method determines how to incorporate the batched entries: +// +// 1. If the child is not the default empty node (i.e. it already contains +// non-default data): +// - If the child is a compacted leaf, processSubtree delegates to the +// processCompactedLeaf helper. This handles the case where the slot +// already has a compressed leaf, performing either a replacement (if +// the batched entry’s key matches) or merging conflicting entries. +// - Otherwise, the child is assumed to be a branch node, so the batched +// entries are recursively inserted into that branch via a call to +// batchedInsert at the next tree level. +// +// 2. If the child is the default empty node (i.e. no prior insertion has +// occurred in this slot): +// - For a single entry, a new compacted leaf is created at the next +// level using that entry’s key and leaf, and inserted directly. +// - For multiple entries, an empty branch node (from the precomputed +// EmptyTree for the next level) is used as the base to recursively +// process the entries using batchedInsert. +// +// In all cases, processSubtree returns the updated node that replaces the +// current child, along with any error encountered during processing. +// +// This helper reduces nesting by separating the logic for non-empty versus +// empty subtrees and for compacted leaf handling versus full branch recursion. func (t *CompactedTree) processSubtree(tx TreeStoreUpdateTx, height int, entries []BatchedInsertionEntry, child Node) (Node, error) { @@ -459,9 +483,45 @@ func partitionEntries(entries []BatchedInsertionEntry, height int) (leftEntries, return } -// batchedInsert recursively inserts a batch of leaf nodes into the MS-SMT. -// It partitions the given entries based on the bit at the specified height -// and processes both left and right subtrees accordingly. +// batchedInsert recursively processes a batch of insertion entries +// into the subtree of the current branch node at the specified height. +// +// The function works as follows: +// +// 1. Base-Case and Empty Batch: +// - If the current level (height) has reached or exceeded the last +// bit index, no further descent is possible, so the current branch +// is simply returned. +// - If there are no entries to insert, the current branch is returned +// unchanged. +// +// 2. Partitioning Entries: +// - The batch of insertion entries is split into two groups via the helper +// function partitionEntries. Entries with a 0 bit at the current level +// go into the leftEntries slice, and those with a 1 bit go into the +// rightEntries slice. +// +// 3. Recursively Updating Subtrees: +// - The current branch’s children are retrieved using GetChildren. +// - For each side that has corresponding entries: +// - processSubtree is invoked to update that subtree. +// - The updated child node (either a newly created compacted leaf or a +// recursively updated branch) replaces the old child. +// +// 4. Reassembling the Branch: +// - A new branch is constructed from the updated left and right children. +// - The old branch (if not the default empty branch) is deleted from the +// store. +// - If the newly formed branch is not equivalent to the default empty +// node for that height, it is inserted into the store. +// +// 5. Return Value: +// - The function returns the updated branch node reflecting all batched +// insertions at that level. +// +// This helper encapsulates the recursion and merging logic for batched insertions, +// reducing nesting in the higher-level BatchedInsert method and making the overall +// insertion process easier to follow and maintain. func (t *CompactedTree) batchedInsert(tx TreeStoreUpdateTx, entries []BatchedInsertionEntry, height int, root *BranchNode) (*BranchNode, error) { // Base-case: If we've reached the bottom, simply return the current branch. if height >= lastBitIndex { @@ -518,7 +578,44 @@ func (t *CompactedTree) batchedInsert(tx TreeStoreUpdateTx, entries []BatchedIns return updatedBranch, nil } -// BatchedInsert inserts multiple leaf nodes at the given keys within the MS-SMT. +// BatchedInsert inserts multiple leaf nodes into the MS-SMT as a batch +// operation. +// +// It accepts a context and a slice of BatchedInsertionEntry, where each entry +// specifies a target key and its associated leaf to be inserted. The method +// performs the batch insertion in a transactional manner on the underlying +// tree store and proceeds through the following steps: +// +// 1. Sorting: +// - The entries are first sorted in lexicographic order based on their keys. +// This consistent ordering is essential for generating valid proofs and +// ensuring history-independence. +// +// 2. Overflow Check and Transaction Setup: +// - Within an update transaction, the current root branch is retrieved. +// - For each entry, the function checks whether adding the leaf’s sum to the +// current root sum would overflow a uint64. If an overflow is detected, +// the batch insertion aborts and returns an error. +// +// 3. Recursive Processing via Helper: +// - The sorted entries are passed to the recursive helper batchedInsert along +// with the current root and a starting height of 0. +// - The helper partitions the entries based on the bit at each tree level, +// recursively updating non-empty subtrees and creating new compacted leaves +// or branch nodes as needed. +// +// 4. Root Update: +// - When recursion completes, a new root reflecting all batched updates is +// obtained. This new root is then stored in the tree store, finalizing the +// update. +// +// 5. Return: +// - On success, BatchedInsert returns the updated tree instance (implementing +// the Tree interface). In case of any errors (e.g. overflow or transactional +// failures), it returns the appropriate error. +// +// This method encapsulates the complex merging and partitioning logic required for +// efficient batched insertions into a compacted Merkle-Sum Sparse Merkle Tree. func (t *CompactedTree) BatchedInsert(ctx context.Context, entries []BatchedInsertionEntry) (Tree, error) { sort.Slice(entries, func(i, j int) bool { return bytes.Compare(entries[i].Key[:], entries[j].Key[:]) < 0