Skip to content

Commit ff43bc7

Browse files
authored
Merge pull request #1870 from lightninglabs/wip/limit-proof-cache-size
tapdb: switch proof cache to size-based limits and add deep size estimation
2 parents 7013ac3 + c6574a1 commit ff43bc7

17 files changed

+849
-117
lines changed

docs/release-notes/release-notes-0.8.0.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
- [Functional Updates](#functional-updates)
99
- [RPC Updates](#rpc-updates)
1010
- [tapcli Updates](#tapcli-updates)
11+
- [Config Changes](#config-changes)
1112
- [Breaking Changes](#breaking-changes)
1213
- [Performance Improvements](#performance-improvements)
1314
- [Deprecations](#deprecations)
@@ -53,6 +54,13 @@
5354

5455
## tapcli Updates
5556

57+
## Config Changes
58+
59+
- [PR#1870](https://github.com/lightninglabs/taproot-assets/pull/1870)
60+
The `proofs-per-universe` configuration option is removed. New option
61+
`max-proof-cache-size` sets the proof cache limit in bytes and accepts
62+
human-readable values such as `64MB`.
63+
5664
## Code Health
5765

5866
## Breaking Changes

docs/release-notes/release-notes-template.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
- [Functional Updates](#functional-updates)
99
- [RPC Updates](#rpc-updates)
1010
- [tapcli Updates](#tapcli-updates)
11+
- [Config Changes](#config-changes)
1112
- [Breaking Changes](#breaking-changes)
1213
- [Performance Improvements](#performance-improvements)
1314
- [Deprecations](#deprecations)
@@ -36,6 +37,8 @@
3637

3738
## tapcli Updates
3839

40+
## Config Changes
41+
3942
## Code Health
4043

4144
## Breaking Changes

fn/memory.go

Lines changed: 156 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,25 @@
11
package fn
22

3-
import "errors"
3+
import (
4+
"errors"
5+
"reflect"
6+
"unsafe"
7+
)
48

59
var (
610
// ErrNilPointerDeference is returned when a nil pointer is
711
// dereferenced.
812
ErrNilPointerDeference = errors.New("nil pointer dereference")
913
)
1014

15+
var (
16+
// sliceHeaderSize is the size of a slice header.
17+
sliceHeaderSize = uint64(unsafe.Sizeof([]byte(nil)))
18+
19+
// stringHeaderSize is the size of a string header.
20+
stringHeaderSize = uint64(unsafe.Sizeof(""))
21+
)
22+
1123
// Ptr returns the pointer of the given value. This is useful in instances
1224
// where a function returns the value, but a pointer is wanted. Without this,
1325
// then an intermediate variable is needed.
@@ -68,3 +80,146 @@ func DerefPanic[T any](ptr *T) T {
6880

6981
return *ptr
7082
}
83+
84+
// LowerBoundByteSize returns a conservative deep-size estimate in bytes.
85+
//
86+
// Notes:
87+
// - Pointer-recursive and cycle safe; each heap allocation is counted once
88+
// using its data pointer.
89+
// - Lower bound: ignores allocator overhead, GC metadata, unused slice
90+
// capacity, map buckets/overflow, evacuation, rounding, and runtime
91+
// internals (chan/func).
92+
func LowerBoundByteSize(x any) uint64 {
93+
// seen is a map of heap object identities which have already been
94+
// counted.
95+
seen := make(map[uintptr]struct{})
96+
return byteSizeVisit(reflect.ValueOf(x), true, seen)
97+
}
98+
99+
// byteSizeVisit returns a conservative lower-bound byte count for `subject`.
100+
//
101+
// Notes:
102+
// - addSelf: include subject’s inline bytes when true. Parents pass false.
103+
// - seen: set of heap data pointers to avoid double counting and break
104+
// cycles.
105+
//
106+
// Lower bound: ignores allocator overhead, GC metadata, unused capacity, and
107+
// runtime internals.
108+
func byteSizeVisit(subject reflect.Value, addSelf bool,
109+
seen map[uintptr]struct{}) uint64 {
110+
111+
if !subject.IsValid() {
112+
return 0
113+
}
114+
115+
subjectType := subject.Type()
116+
subjectTypeKind := subjectType.Kind()
117+
118+
if subjectTypeKind == reflect.Interface {
119+
n := uint64(unsafe.Sizeof(subject.Interface()))
120+
if !subject.IsNil() {
121+
n += byteSizeVisit(subject.Elem(), true, seen)
122+
}
123+
return n
124+
}
125+
126+
switch subjectTypeKind {
127+
case reflect.Ptr:
128+
if subject.IsNil() {
129+
return 0
130+
}
131+
132+
ptr := subject.Pointer()
133+
if markSeen(ptr, seen) {
134+
return 0
135+
}
136+
137+
return byteSizeVisit(subject.Elem(), true, seen)
138+
139+
case reflect.Struct:
140+
n := uint64(0)
141+
if addSelf {
142+
n += uint64(subjectType.Size())
143+
}
144+
145+
for i := 0; i < subject.NumField(); i++ {
146+
n += byteSizeVisit(subject.Field(i), false, seen)
147+
}
148+
149+
return n
150+
151+
case reflect.Array:
152+
n := uint64(0)
153+
if addSelf {
154+
n += uint64(subjectType.Size())
155+
}
156+
157+
for i := 0; i < subject.Len(); i++ {
158+
n += byteSizeVisit(subject.Index(i), false, seen)
159+
}
160+
161+
return n
162+
163+
case reflect.Slice:
164+
if subject.IsNil() {
165+
return 0
166+
}
167+
168+
n := sliceHeaderSize
169+
dataPtr := subject.Pointer()
170+
if dataPtr != 0 && !markSeen(dataPtr, seen) {
171+
elem := subjectType.Elem()
172+
n += uint64(subject.Len()) * uint64(elem.Size())
173+
}
174+
175+
for i := 0; i < subject.Len(); i++ {
176+
n += byteSizeVisit(subject.Index(i), false, seen)
177+
}
178+
179+
return n
180+
181+
case reflect.String:
182+
n := stringHeaderSize
183+
dataPtr := subject.Pointer()
184+
if dataPtr != 0 && markSeen(dataPtr, seen) {
185+
return n
186+
}
187+
188+
return n + uint64(subject.Len())
189+
190+
case reflect.Map:
191+
n := uint64(unsafe.Sizeof(subject.Interface()))
192+
if subject.IsNil() {
193+
return n
194+
}
195+
196+
it := subject.MapRange()
197+
for it.Next() {
198+
n += byteSizeVisit(it.Key(), false, seen)
199+
n += byteSizeVisit(it.Value(), false, seen)
200+
}
201+
202+
return n
203+
204+
case reflect.Chan, reflect.Func, reflect.UnsafePointer:
205+
return uint64(unsafe.Sizeof(subject.Interface()))
206+
207+
default:
208+
if addSelf {
209+
return uint64(subjectType.Size())
210+
}
211+
212+
return 0
213+
}
214+
}
215+
216+
// markSeen marks the given pointer as seen and returns true if it was already
217+
// seen.
218+
func markSeen(ptr uintptr, seen map[uintptr]struct{}) bool {
219+
if _, ok := seen[ptr]; ok {
220+
return true
221+
}
222+
223+
seen[ptr] = struct{}{}
224+
return false
225+
}

fn/memory_test.go

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
package fn
2+
3+
import (
4+
"testing"
5+
"unsafe"
6+
7+
"github.com/stretchr/testify/require"
8+
)
9+
10+
// TestLowerBoundByteSizeNilAndPrimitives ensures the byte estimator handles nil
11+
// interfaces and primitive concrete values using their inline sizes.
12+
func TestLowerBoundByteSizeNilAndPrimitives(t *testing.T) {
13+
actualNil := LowerBoundByteSize(nil)
14+
require.Zero(
15+
t, actualNil, "nil interface bytes: expected 0, actual %d",
16+
actualNil,
17+
)
18+
19+
var num int64 = 99
20+
expectedInt := uint64(unsafe.Sizeof(num))
21+
actualInt := LowerBoundByteSize(num)
22+
require.Equal(
23+
t, expectedInt, actualInt,
24+
"int64 bytes mismatch: expected %d, actual %d",
25+
expectedInt, actualInt,
26+
)
27+
28+
const str = "taproot-assets"
29+
expectedString := stringHeaderSize + uint64(len(str))
30+
actualString := LowerBoundByteSize(str)
31+
require.Equal(
32+
t, expectedString, actualString,
33+
"string bytes mismatch: expected %d, actual %d",
34+
expectedString, actualString,
35+
)
36+
}
37+
38+
// TestLowerBoundByteSizeStructsAndSlices covers structs that embed slices
39+
// and validates shared backing arrays are only counted once via the seen set.
40+
func TestLowerBoundByteSizeStructsAndSlices(t *testing.T) {
41+
type structWithSlice struct {
42+
Count uint16
43+
Data []byte
44+
}
45+
46+
t.Run("structWithSlice", func(t *testing.T) {
47+
payload := []byte{1, 2, 3, 4}
48+
value := structWithSlice{
49+
Count: 42,
50+
Data: payload,
51+
}
52+
53+
expected := uint64(unsafe.Sizeof(structWithSlice{}))
54+
expected += sliceHeaderSize
55+
expected += uint64(len(payload))
56+
actual := LowerBoundByteSize(value)
57+
58+
require.Equal(
59+
t, expected, actual,
60+
"struct with slice size mismatch: expected %d, "+
61+
"actual %d",
62+
expected, actual,
63+
)
64+
})
65+
66+
t.Run("sharedBackingArrayCountedOnce", func(t *testing.T) {
67+
payload := []byte{5, 6, 7}
68+
69+
type twoSlices struct {
70+
Left []byte
71+
Right []byte
72+
}
73+
74+
value := twoSlices{
75+
Left: payload,
76+
Right: payload,
77+
}
78+
79+
expected := uint64(unsafe.Sizeof(twoSlices{}))
80+
expected += 2 * sliceHeaderSize
81+
expected += uint64(len(payload))
82+
actual := LowerBoundByteSize(value)
83+
84+
require.Equal(
85+
t, expected, actual,
86+
"shared backing array size mismatch: expected %d, "+
87+
"actual %d",
88+
expected, actual,
89+
)
90+
})
91+
}
92+
93+
// TestLowerBoundByteSizePointerCycle confirms pointer cycles do not blow up the
94+
// traversal and only count the struct once.
95+
func TestLowerBoundByteSizePointerCycle(t *testing.T) {
96+
type node struct {
97+
Value uint32
98+
Next *node
99+
}
100+
101+
root := &node{Value: 1}
102+
root.Next = root
103+
104+
expected := uint64(unsafe.Sizeof(node{}))
105+
actual := LowerBoundByteSize(root)
106+
require.Equal(
107+
t, expected, actual,
108+
"pointer cycle size mismatch: expected %d, actual %d",
109+
expected, actual,
110+
)
111+
}
112+
113+
// TestLowerBoundByteSizeMap verifies map headers and key/value payloads are
114+
// included in the lower bound calculation.
115+
func TestLowerBoundByteSizeMap(t *testing.T) {
116+
payload := []byte{9, 8, 7}
117+
value := map[string][]byte{
118+
"alpha": payload,
119+
}
120+
121+
expected := uint64(unsafe.Sizeof(any(value)))
122+
expected += stringHeaderSize + uint64(len("alpha"))
123+
expected += sliceHeaderSize + uint64(len(payload))
124+
actual := LowerBoundByteSize(value)
125+
126+
require.Equal(
127+
t, expected, actual,
128+
"map size mismatch: expected %d, actual %d",
129+
expected, actual,
130+
)
131+
}

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ require (
1616
github.com/caddyserver/certmagic v0.17.2
1717
github.com/davecgh/go-spew v1.1.1
1818
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0
19+
github.com/dustin/go-humanize v1.0.1
1920
github.com/go-errors/errors v1.0.1
2021
github.com/golang-migrate/migrate/v4 v4.17.0
2122
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
@@ -28,7 +29,7 @@ require (
2829
github.com/lightninglabs/aperture v0.3.13-beta
2930
github.com/lightninglabs/lightning-node-connect/hashmailrpc v1.0.3
3031
github.com/lightninglabs/lndclient v0.20.0-5
31-
github.com/lightninglabs/neutrino/cache v1.1.2
32+
github.com/lightninglabs/neutrino/cache v1.1.3
3233
github.com/lightninglabs/taproot-assets/taprpc v1.0.9
3334
github.com/lightningnetwork/lnd v0.20.0-beta
3435
github.com/lightningnetwork/lnd/cert v1.2.2
@@ -85,7 +86,6 @@ require (
8586
github.com/docker/docker v28.1.1+incompatible // indirect
8687
github.com/docker/go-connections v0.5.0 // indirect
8788
github.com/docker/go-units v0.5.0 // indirect
88-
github.com/dustin/go-humanize v1.0.1 // indirect
8989
github.com/fergusstrange/embedded-postgres v1.25.0 // indirect
9090
github.com/go-logr/logr v1.4.3 // indirect
9191
github.com/go-logr/stdr v1.2.2 // indirect

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1146,8 +1146,8 @@ github.com/lightninglabs/migrate/v4 v4.18.2-9023d66a-fork-pr-2 h1:eFjp1dIB2BhhQp
11461146
github.com/lightninglabs/migrate/v4 v4.18.2-9023d66a-fork-pr-2/go.mod h1:99BKpIi6ruaaXRM1A77eqZ+FWPQ3cfRa+ZVy5bmWMaY=
11471147
github.com/lightninglabs/neutrino v0.16.1 h1:5Kz4ToxncEVkpKC6fwUjXKtFKJhuxlG3sBB3MdJTJjs=
11481148
github.com/lightninglabs/neutrino v0.16.1/go.mod h1:L+5UAccpUdyM7yDgmQySgixf7xmwBgJtOfs/IP26jCs=
1149-
github.com/lightninglabs/neutrino/cache v1.1.2 h1:C9DY/DAPaPxbFC+xNNEI/z1SJY9GS3shmlu5hIQ798g=
1150-
github.com/lightninglabs/neutrino/cache v1.1.2/go.mod h1:XJNcgdOw1LQnanGjw8Vj44CvguYA25IMKjWFZczwZuo=
1149+
github.com/lightninglabs/neutrino/cache v1.1.3 h1:rgnabC41W+XaPuBTQrdeFjFCCAVKh1yctAgmb3Se9zA=
1150+
github.com/lightninglabs/neutrino/cache v1.1.3/go.mod h1:qxkJb+pUxR5p84jl5uIGFCR4dGdFkhNUwMSxw3EUWls=
11511151
github.com/lightninglabs/protobuf-go-hex-display v1.34.2-hex-display h1:w7FM5LH9Z6CpKxl13mS48idsu6F+cEZf0lkyiV+Dq9g=
11521152
github.com/lightninglabs/protobuf-go-hex-display v1.34.2-hex-display/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
11531153
github.com/lightningnetwork/lightning-onion v1.2.1-0.20240815225420-8b40adf04ab9 h1:6D3LrdagJweLLdFm1JNodZsBk6iU4TTsBBFLQ4yiXfI=

sample-tapd.conf

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -391,8 +391,9 @@
391391

392392
[multiverse-caches]
393393

394-
; The number of proofs that are cached per universe. (default: 5)
395-
; universe.multiverse-caches.proofs-per-universe=5
394+
; The maximum total size of the cached proofs. Accepts human readable values
395+
; such as 32MB or 1GB. (default: 32MB)
396+
; universe.multiverse-caches.max-proof-cache-size=32MB
396397

397398
; The number of universes that can have a cache of leaf keys. (default: 2000)
398399
; universe.multiverse-caches.leaves-num-cached-universes=2000

0 commit comments

Comments
 (0)