From c260e36ba0a54a1f83cd07ac8e8b0ac743f36adb Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 27 Jan 2025 20:52:02 +0100 Subject: [PATCH 01/21] add basic eip7732 (epbs) support --- clients/consensus/chainspec.go | 7 +++++++ go.mod | 2 +- go.sum | 4 ++-- handlers/index.go | 8 ++++++++ indexer/beacon/block_helper.go | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/clients/consensus/chainspec.go b/clients/consensus/chainspec.go index 3bb2b347..786aa76f 100644 --- a/clients/consensus/chainspec.go +++ b/clients/consensus/chainspec.go @@ -36,6 +36,8 @@ type ChainSpec struct { ElectraForkEpoch *uint64 `yaml:"ELECTRA_FORK_EPOCH"` Eip7594ForkVersion phase0.Version `yaml:"EIP7594_FORK_VERSION" check-if-fork:"Eip7594ForkEpoch"` Eip7594ForkEpoch *uint64 `yaml:"EIP7594_FORK_EPOCH"` + Eip7732ForkVersion phase0.Version `yaml:"EIP7732_FORK_VERSION" check-if-fork:"Eip7732ForkEpoch"` + Eip7732ForkEpoch *uint64 `yaml:"EIP7732_FORK_EPOCH"` SecondsPerSlot time.Duration `yaml:"SECONDS_PER_SLOT"` SlotsPerEpoch uint64 `yaml:"SLOTS_PER_EPOCH"` EpochsPerHistoricalVector uint64 `yaml:"EPOCHS_PER_HISTORICAL_VECTOR"` @@ -64,6 +66,11 @@ type ChainSpec struct { DataColumnSidecarSubnetCount *uint64 `yaml:"DATA_COLUMN_SIDECAR_SUBNET_COUNT" check-if-fork:"Eip7594ForkEpoch"` CustodyRequirement *uint64 `yaml:"CUSTODY_REQUIREMENT" check-if-fork:"Eip7594ForkEpoch"` + // EIP7732: ePBS + PtcSize uint64 `yaml:"PTC_SIZE" check-if-fork:"Eip7732ForkEpoch"` + MaxPayloadAttestations uint64 `yaml:"MAX_PAYLOAD_ATTESTATIONS" check-if-fork:"Eip7732ForkEpoch"` + DomainPtcAttester phase0.DomainType `yaml:"DOMAIN_PTC_ATTESTER" check-if-fork:"Eip7732ForkEpoch"` + // additional dora specific specs WhiskForkEpoch *uint64 } diff --git a/go.mod b/go.mod index 52eedc4f..cc0004d3 100644 --- a/go.mod +++ b/go.mod @@ -152,6 +152,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/attestantio/go-eth2-client v0.0.0-20250106164842-07b6ce39bb43 +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250127193505-8cfe1ff986fd replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index b54ab8ba..1d161da4 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,6 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= -github.com/attestantio/go-eth2-client v0.0.0-20250106164842-07b6ce39bb43 h1:QVr4VkeZGzR7Mxtg1PKZ6n97aQU4VfFbTVD5hZauNUM= -github.com/attestantio/go-eth2-client v0.0.0-20250106164842-07b6ce39bb43/go.mod h1:vy5jU/uDZ2+RcVzq5BfnG+bQ3/6uu9DGwCrGsPtjJ1A= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.17.0 h1:1X2TS7aHz1ELcC0yU1y2stUs/0ig5oMU6STFZGrhvHI= @@ -348,6 +346,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= +github.com/pk910/go-eth2-client v0.0.0-20250127193505-8cfe1ff986fd h1:xty/DNk1Y9vwj2Enx/D/eFBoSL0zf4V6wkqCRHsmliY= +github.com/pk910/go-eth2-client v0.0.0-20250127193505-8cfe1ff986fd/go.mod h1:vy5jU/uDZ2+RcVzq5BfnG+bQ3/6uu9DGwCrGsPtjJ1A= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/handlers/index.go b/handlers/index.go index 1005ef35..24a52efa 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -216,6 +216,14 @@ func buildIndexPageData() (*models.IndexPageData, time.Duration) { Active: uint64(currentEpoch) >= *specs.Eip7594ForkEpoch, }) } + if specs.Eip7732ForkEpoch != nil && *specs.Eip7732ForkEpoch < uint64(18446744073709551615) { + pageData.NetworkForks = append(pageData.NetworkForks, &models.IndexPageDataForks{ + Name: "eip7732", + Epoch: *specs.Eip7732ForkEpoch, + Version: specs.Eip7732ForkVersion[:], + Active: uint64(currentEpoch) >= *specs.Eip7732ForkEpoch, + }) + } // load recent epochs buildIndexPageRecentEpochsData(pageData, currentEpoch, finalizedEpoch, justifiedEpoch, recentEpochCount) diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 843a1100..34f03b06 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -9,6 +9,7 @@ import ( "github.com/attestantio/go-eth2-client/spec/bellatrix" "github.com/attestantio/go-eth2-client/spec/capella" "github.com/attestantio/go-eth2-client/spec/deneb" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/electra" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/utils" @@ -44,6 +45,9 @@ func marshalVersionedSignedBeaconBlockSSZ(dynSsz *dynssz.DynSsz, block *spec.Ver case spec.DataVersionElectra: version = uint64(block.Version) ssz, err = dynSsz.MarshalSSZ(block.Electra) + case spec.DataVersionEIP7732: + version = uint64(block.Version) + ssz, err = dynSsz.MarshalSSZ(block.EIP7732) default: err = fmt.Errorf("unknown block version") } @@ -110,6 +114,11 @@ func unmarshalVersionedSignedBeaconBlockSSZ(dynSsz *dynssz.DynSsz, version uint6 if err := dynSsz.UnmarshalSSZ(block.Electra, ssz); err != nil { return nil, fmt.Errorf("failed to decode electra signed beacon block: %v", err) } + case spec.DataVersionEIP7732: + block.EIP7732 = &eip7732.SignedBeaconBlock{} + if err := dynSsz.UnmarshalSSZ(block.EIP7732, ssz); err != nil { + return nil, fmt.Errorf("failed to decode eip7732 signed beacon block: %v", err) + } default: return nil, fmt.Errorf("unknown block version") } @@ -137,6 +146,9 @@ func marshalVersionedSignedBeaconBlockJson(block *spec.VersionedSignedBeaconBloc case spec.DataVersionElectra: version = uint64(block.Version) jsonRes, err = block.Electra.MarshalJSON() + case spec.DataVersionEIP7732: + version = uint64(block.Version) + jsonRes, err = block.EIP7732.MarshalJSON() default: err = fmt.Errorf("unknown block version") } @@ -185,6 +197,11 @@ func unmarshalVersionedSignedBeaconBlockJson(version uint64, ssz []byte) (*spec. if err := block.Electra.UnmarshalJSON(ssz); err != nil { return nil, fmt.Errorf("failed to decode electra signed beacon block: %v", err) } + case spec.DataVersionEIP7732: + block.EIP7732 = &eip7732.SignedBeaconBlock{} + if err := block.EIP7732.UnmarshalJSON(ssz); err != nil { + return nil, fmt.Errorf("failed to decode eip7732 signed beacon block: %v", err) + } default: return nil, fmt.Errorf("unknown block version") } @@ -218,6 +235,8 @@ func getBlockExecutionExtraData(v *spec.VersionedSignedBeaconBlock) ([]byte, err } return v.Electra.Message.Body.ExecutionPayload.ExtraData, nil + case spec.DataVersionEIP7732: + return nil, nil default: return nil, errors.New("unknown version") } @@ -262,6 +281,12 @@ func getStateRandaoMixes(v *spec.VersionedBeaconState) ([]phase0.Root, error) { } return v.Electra.RANDAOMixes, nil + case spec.DataVersionEIP7732: + if v.EIP7732 == nil || v.EIP7732.RANDAOMixes == nil { + return nil, errors.New("no eip7732 block") + } + + return v.EIP7732.RANDAOMixes, nil default: return nil, errors.New("unknown version") } @@ -282,6 +307,8 @@ func getStateDepositIndex(state *spec.VersionedBeaconState) uint64 { return state.Deneb.ETH1DepositIndex case spec.DataVersionElectra: return state.Electra.ETH1DepositIndex + case spec.DataVersionEIP7732: + return state.EIP7732.ETH1DepositIndex } return 0 } @@ -321,6 +348,12 @@ func getStateCurrentSyncCommittee(v *spec.VersionedBeaconState) ([]phase0.BLSPub } return v.Electra.CurrentSyncCommittee.Pubkeys, nil + case spec.DataVersionEIP7732: + if v.EIP7732 == nil || v.EIP7732.CurrentSyncCommittee == nil { + return nil, errors.New("no eip7732 block") + } + + return v.EIP7732.CurrentSyncCommittee.Pubkeys, nil default: return nil, errors.New("unknown version") } From 32a348f87aebb67dd119cf9317214d27106eca61 Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 27 Jan 2025 21:07:06 +0100 Subject: [PATCH 02/21] trigger CI From 311e4af4e15e7c3339da9d98f91edc1a74042820 Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 5 Feb 2025 11:43:20 +0100 Subject: [PATCH 03/21] bump go-eth2-client --- go.mod | 13 +++++++------ go.sum | 27 ++++++++++++++------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index cc0004d3..ab4fc254 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( ) require ( + github.com/emicklei/dot v1.6.4 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/minio/highwayhash v1.0.2 // indirect @@ -83,8 +84,8 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/ethereum/c-kzg-4844 v1.0.2 // indirect github.com/ethereum/go-verkle v0.2.2 // indirect - github.com/fatih/color v1.17.0 // indirect - github.com/ferranbt/fastssz v0.1.3 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/ferranbt/fastssz v0.1.4 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect @@ -106,8 +107,8 @@ require ( github.com/jackc/puddle v1.3.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mfridman/interpolate v0.0.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect @@ -140,7 +141,7 @@ require ( golang.org/x/net v0.33.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.29.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/Knetic/govaluate.v3 v3.0.0 gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect @@ -152,6 +153,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250127193505-8cfe1ff986fd +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250205104042-361ad9379622 replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index 1d161da4..9b23c21e 100644 --- a/go.sum +++ b/go.sum @@ -74,16 +74,18 @@ github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeM github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emicklei/dot v1.6.4 h1:cG9ycT67d9Yw22G+mAb4XiuUz6E6H1S0zePp/5Cwe/c= +github.com/emicklei/dot v1.6.4/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/ethereum/c-kzg-4844 v1.0.2 h1:8tV84BCEiPeOkiVgW9mpYBeBUir2bkCNVqxPwwVeO+s= github.com/ethereum/c-kzg-4844 v1.0.2/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/ethpandaops/ethwallclock v0.3.0 h1:xF5fwtBf+bHFHZKBnwiPFEuelW3sMM7SD3ZNFq1lJY4= github.com/ethpandaops/ethwallclock v0.3.0/go.mod h1:y0Cu+mhGLlem19vnAV2x0hpFS5KZ7oOi2SWYayv9l24= -github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= -github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= -github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo= -github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/ferranbt/fastssz v0.1.4 h1:OCDB+dYDEQDvAgtAGnTSidK1Pe2tW3nFV40XyMkTeDY= +github.com/ferranbt/fastssz v0.1.4/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -245,8 +247,8 @@ github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhd github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -278,8 +280,9 @@ github.com/mashingan/smapping v0.1.19 h1:SsEtuPn2UcM1croIupPtGLgWgpYRuS0rSQMvKD9 github.com/mashingan/smapping v0.1.19/go.mod h1:FjfiwFxGOuNxL/OT1WcrNAwTPx0YJeg5JiXwBB1nyig= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -346,8 +349,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= -github.com/pk910/go-eth2-client v0.0.0-20250127193505-8cfe1ff986fd h1:xty/DNk1Y9vwj2Enx/D/eFBoSL0zf4V6wkqCRHsmliY= -github.com/pk910/go-eth2-client v0.0.0-20250127193505-8cfe1ff986fd/go.mod h1:vy5jU/uDZ2+RcVzq5BfnG+bQ3/6uu9DGwCrGsPtjJ1A= +github.com/pk910/go-eth2-client v0.0.0-20250205104042-361ad9379622 h1:cKV3IQs3pPhKbcxTGftrKvKjM+QWH+yIeJq7oXFCej8= +github.com/pk910/go-eth2-client v0.0.0-20250205104042-361ad9379622/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -450,8 +453,6 @@ github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZ github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= -github.com/umbracle/gohashtree v0.0.2-alpha.0.20230207094856-5b775a815c10 h1:CQh33pStIp/E30b7TxDlXfM0145bn2e8boI30IxAhTg= -github.com/umbracle/gohashtree v0.0.2-alpha.0.20230207094856-5b775a815c10/go.mod h1:x/Pa0FF5Te9kdrlZKJK82YmAkvL8+f989USgz6Jiw7M= github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI= github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= @@ -600,8 +601,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg= google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M= google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc= From 3c911bedb7b258708ac435ab78828d35d7777a45 Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 5 Feb 2025 13:24:24 +0100 Subject: [PATCH 04/21] bump go-eth2-client --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ab4fc254..3fe8724c 100644 --- a/go.mod +++ b/go.mod @@ -153,6 +153,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250205104042-361ad9379622 +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7 replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index 9b23c21e..b53bb516 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= -github.com/pk910/go-eth2-client v0.0.0-20250205104042-361ad9379622 h1:cKV3IQs3pPhKbcxTGftrKvKjM+QWH+yIeJq7oXFCej8= -github.com/pk910/go-eth2-client v0.0.0-20250205104042-361ad9379622/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= +github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7 h1:T2yePXJvKXb+YWZKSuCKtuY1WMPsUu9StDFYW5ZAQSU= +github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From 0167d8b0034415b9a27caf691e5f8a9eb632d234 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 6 Feb 2025 00:44:31 +0100 Subject: [PATCH 05/21] subscribe to execution_payload events & load execution payload envelopes --- clients/consensus/client.go | 65 ++++++------ clients/consensus/clientlogic.go | 18 +++- clients/consensus/rpc/beaconapi.go | 17 ++++ clients/consensus/rpc/beaconstream.go | 34 ++++++- go.mod | 2 +- go.sum | 4 +- indexer/beacon/block.go | 140 +++++++++++++++++++++----- indexer/beacon/client.go | 53 +++++++++- indexer/beacon/requests.go | 17 ++++ 9 files changed, 285 insertions(+), 65 deletions(-) diff --git a/clients/consensus/client.go b/clients/consensus/client.go index c91131de..ef4d5523 100644 --- a/clients/consensus/client.go +++ b/clients/consensus/client.go @@ -22,36 +22,37 @@ type ClientConfig struct { } type Client struct { - pool *Pool - clientIdx uint16 - endpointConfig *ClientConfig - clientCtx context.Context - clientCtxCancel context.CancelFunc - rpcClient *rpc.BeaconClient - logger *logrus.Entry - isOnline bool - isSyncing bool - isOptimistic bool - versionStr string - nodeIdentity *rpc.NodeIdentity - clientType ClientType - lastEvent time.Time - retryCounter uint64 - lastError error - headMutex sync.RWMutex - headRoot phase0.Root - headSlot phase0.Slot - justifiedRoot phase0.Root - justifiedEpoch phase0.Epoch - finalizedRoot phase0.Root - finalizedEpoch phase0.Epoch - lastFinalityUpdateEpoch phase0.Epoch - lastPeerUpdateEpoch phase0.Epoch - lastSyncUpdateEpoch phase0.Epoch - peers []*v1.Peer - blockDispatcher Dispatcher[*v1.BlockEvent] - headDispatcher Dispatcher[*v1.HeadEvent] - checkpointDispatcher Dispatcher[*v1.Finality] + pool *Pool + clientIdx uint16 + endpointConfig *ClientConfig + clientCtx context.Context + clientCtxCancel context.CancelFunc + rpcClient *rpc.BeaconClient + logger *logrus.Entry + isOnline bool + isSyncing bool + isOptimistic bool + versionStr string + nodeIdentity *rpc.NodeIdentity + clientType ClientType + lastEvent time.Time + retryCounter uint64 + lastError error + headMutex sync.RWMutex + headRoot phase0.Root + headSlot phase0.Slot + justifiedRoot phase0.Root + justifiedEpoch phase0.Epoch + finalizedRoot phase0.Root + finalizedEpoch phase0.Epoch + lastFinalityUpdateEpoch phase0.Epoch + lastPeerUpdateEpoch phase0.Epoch + lastSyncUpdateEpoch phase0.Epoch + peers []*v1.Peer + blockDispatcher Dispatcher[*v1.BlockEvent] + headDispatcher Dispatcher[*v1.HeadEvent] + checkpointDispatcher Dispatcher[*v1.Finality] + executionPayloadDispatcher Dispatcher[*v1.ExecutionPayloadEvent] } func (pool *Pool) newPoolClient(clientIdx uint16, endpoint *ClientConfig) (*Client, error) { @@ -96,6 +97,10 @@ func (client *Client) SubscribeFinalizedEvent(capacity int) *Subscription[*v1.Fi return client.checkpointDispatcher.Subscribe(capacity, false) } +func (client *Client) SubscribeExecutionPayloadEvent(capacity int, blocking bool) *Subscription[*v1.ExecutionPayloadEvent] { + return client.executionPayloadDispatcher.Subscribe(capacity, blocking) +} + func (client *Client) GetPool() *Pool { return client.pool } diff --git a/clients/consensus/clientlogic.go b/clients/consensus/clientlogic.go index 09a9a285..5a163be7 100644 --- a/clients/consensus/clientlogic.go +++ b/clients/consensus/clientlogic.go @@ -133,7 +133,11 @@ func (client *Client) runClientLogic() error { } // start event stream - blockStream := client.rpcClient.NewBlockStream(client.clientCtx, client.logger, rpc.StreamBlockEvent|rpc.StreamHeadEvent|rpc.StreamFinalizedEvent) + blockStream := client.rpcClient.NewBlockStream( + client.clientCtx, + client.logger, + rpc.StreamBlockEvent|rpc.StreamHeadEvent|rpc.StreamFinalizedEvent|rpc.StreamExecutionPayloadEvent, + ) defer blockStream.Close() // process events @@ -171,6 +175,12 @@ func (client *Client) runClientLogic() error { if err != nil { client.logger.Warnf("failed processing finalized event: %v", err) } + + case rpc.StreamExecutionPayloadEvent: + err := client.processExecutionPayloadEvent(evt.Data.(*v1.ExecutionPayloadEvent)) + if err != nil { + client.logger.Warnf("failed processing execution payload event: %v", err) + } } client.logger.Tracef("event (%v) processing time: %v ms", evt.Event, time.Since(now).Milliseconds()) @@ -392,3 +402,9 @@ func (client *Client) pollClientHead() error { return nil } + +func (client *Client) processExecutionPayloadEvent(evt *v1.ExecutionPayloadEvent) error { + client.executionPayloadDispatcher.Fire(evt) + + return nil +} diff --git a/clients/consensus/rpc/beaconapi.go b/clients/consensus/rpc/beaconapi.go index a4ef43d0..70bbce3c 100644 --- a/clients/consensus/rpc/beaconapi.go +++ b/clients/consensus/rpc/beaconapi.go @@ -19,6 +19,7 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/capella" "github.com/attestantio/go-eth2-client/spec/deneb" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/rs/zerolog" "github.com/sirupsen/logrus" @@ -406,6 +407,22 @@ func (bc *BeaconClient) GetBlockBodyByBlockroot(ctx context.Context, blockroot p return result.Data, nil } +func (bc *BeaconClient) GetExecutionPayloadByBlockroot(ctx context.Context, blockroot phase0.Root) (*eip7732.SignedExecutionPayloadEnvelope, error) { + provider, isProvider := bc.clientSvc.(eth2client.ExecutionPayloadProvider) + if !isProvider { + return nil, fmt.Errorf("get execution payload not supported") + } + + result, err := provider.SignedExecutionPayloadEnvelope(ctx, &api.SignedExecutionPayloadEnvelopeOpts{ + Block: fmt.Sprintf("0x%x", blockroot), + }) + if err != nil { + return nil, err + } + + return result.Data, nil +} + func (bc *BeaconClient) GetState(ctx context.Context, stateRef string) (*spec.VersionedBeaconState, error) { provider, isProvider := bc.clientSvc.(eth2client.BeaconStateProvider) if !isProvider { diff --git a/clients/consensus/rpc/beaconstream.go b/clients/consensus/rpc/beaconstream.go index be6fd92c..5020b5cc 100644 --- a/clients/consensus/rpc/beaconstream.go +++ b/clients/consensus/rpc/beaconstream.go @@ -17,9 +17,10 @@ import ( ) const ( - StreamBlockEvent uint16 = 0x01 - StreamHeadEvent uint16 = 0x02 - StreamFinalizedEvent uint16 = 0x04 + StreamBlockEvent uint16 = 0x01 + StreamHeadEvent uint16 = 0x02 + StreamFinalizedEvent uint16 = 0x04 + StreamExecutionPayloadEvent uint16 = 0x08 ) type BeaconStreamEvent struct { @@ -87,6 +88,8 @@ func (bs *BeaconStream) startStream() { bs.processHeadEvent(evt) case "finalized_checkpoint": bs.processFinalizedEvent(evt) + case "execution_payload": + bs.processExecutionPayloadEvent(evt) } case <-stream.Ready: bs.ReadyChan <- &BeaconStreamStatus{ @@ -148,6 +151,16 @@ func (bs *BeaconStream) subscribeStream(endpoint string, events uint16) *eventst topicsCount++ } + if events&StreamExecutionPayloadEvent > 0 { + if topicsCount > 0 { + fmt.Fprintf(&topics, ",") + } + + fmt.Fprintf(&topics, "execution_payload") + + topicsCount++ + } + if topicsCount == 0 { return nil } @@ -225,6 +238,21 @@ func (bs *BeaconStream) processFinalizedEvent(evt eventsource.Event) { } } +func (bs *BeaconStream) processExecutionPayloadEvent(evt eventsource.Event) { + var parsed v1.ExecutionPayloadEvent + + err := json.Unmarshal([]byte(evt.Data()), &parsed) + if err != nil { + bs.logger.Warnf("beacon block stream failed to decode execution_payload event: %v", err) + return + } + + bs.EventChan <- &BeaconStreamEvent{ + Event: StreamExecutionPayloadEvent, + Data: &parsed, + } +} + func getRedactedURL(requrl string) string { var logurl string diff --git a/go.mod b/go.mod index 3fe8724c..f7cf4515 100644 --- a/go.mod +++ b/go.mod @@ -153,6 +153,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7 +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250205233438-1bf0201f31c2 replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index b53bb516..80a84d2e 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= -github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7 h1:T2yePXJvKXb+YWZKSuCKtuY1WMPsUu9StDFYW5ZAQSU= -github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= +github.com/pk910/go-eth2-client v0.0.0-20250205233438-1bf0201f31c2 h1:oHm1KoDCyQH3/YhGRdwgCAAKIPjxAkhAvc88aoHCAIw= +github.com/pk910/go-eth2-client v0.0.0-20250205233438-1bf0201f31c2/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index 9b1562ed..ec38c125 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -8,6 +8,7 @@ import ( "time" "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" @@ -16,26 +17,30 @@ import ( // Block represents a beacon block. type Block struct { - Root phase0.Root - Slot phase0.Slot - dynSsz *dynssz.DynSsz - parentRoot *phase0.Root - dependentRoot *phase0.Root - forkId ForkKey - forkChecked bool - headerMutex sync.Mutex - headerChan chan bool - header *phase0.SignedBeaconBlockHeader - blockMutex sync.Mutex - blockChan chan bool - block *spec.VersionedSignedBeaconBlock - blockIndex *BlockBodyIndex - isInFinalizedDb bool // block is in finalized table (slots) - isInUnfinalizedDb bool // block is in unfinalized table (unfinalized_blocks) - processingStatus dbtypes.UnfinalizedBlockStatus - seenMutex sync.RWMutex - seenMap map[uint16]*Client - processedActivity uint8 + Root phase0.Root + Slot phase0.Slot + dynSsz *dynssz.DynSsz + parentRoot *phase0.Root + dependentRoot *phase0.Root + forkId ForkKey + forkChecked bool + headerMutex sync.Mutex + headerChan chan bool + header *phase0.SignedBeaconBlockHeader + blockMutex sync.Mutex + blockChan chan bool + block *spec.VersionedSignedBeaconBlock + executionPayloadMutex sync.Mutex + executionPayloadChan chan bool + executionPayload *eip7732.SignedExecutionPayloadEnvelope + blockIndex *BlockBodyIndex + isInFinalizedDb bool // block is in finalized table (slots) + isInUnfinalizedDb bool // block is in unfinalized table (unfinalized_blocks) + hasExecutionPayload bool // block has an execution payload (either in cache or db) + processingStatus dbtypes.UnfinalizedBlockStatus + seenMutex sync.RWMutex + seenMap map[uint16]*Client + processedActivity uint8 } // BlockBodyIndex holds important block propoerties that are used as index for cache lookups. @@ -50,12 +55,13 @@ type BlockBodyIndex struct { // newBlock creates a new Block instance. func newBlock(dynSsz *dynssz.DynSsz, root phase0.Root, slot phase0.Slot) *Block { return &Block{ - Root: root, - Slot: slot, - dynSsz: dynSsz, - seenMap: make(map[uint16]*Client), - headerChan: make(chan bool), - blockChan: make(chan bool), + Root: root, + Slot: slot, + dynSsz: dynSsz, + seenMap: make(map[uint16]*Client), + headerChan: make(chan bool), + blockChan: make(chan bool), + executionPayloadChan: make(chan bool), } } @@ -142,6 +148,42 @@ func (block *Block) AwaitBlock(ctx context.Context, timeout time.Duration) *spec return block.block } +// GetExecutionPayload returns the execution payload of this block. +func (block *Block) GetExecutionPayload() *eip7732.SignedExecutionPayloadEnvelope { + if block.executionPayload != nil { + return block.executionPayload + } + + if block.hasExecutionPayload && block.isInUnfinalizedDb { + /* TODO: add execution payload to unfinalized blocks table + dbBlock := db.GetUnfinalizedBlock(block.Root[:]) + if dbBlock != nil { + blockBody, err := unmarshalVersionedSignedBeaconBlockSSZ(block.dynSsz, dbBlock.BlockVer, dbBlock.BlockSSZ) + if err == nil { + return blockBody + } + } + */ + } + + return nil +} + +// AwaitExecutionPayload waits for the execution payload of this block to be available. +func (block *Block) AwaitExecutionPayload(ctx context.Context, timeout time.Duration) *eip7732.SignedExecutionPayloadEnvelope { + if ctx == nil { + ctx = context.Background() + } + + select { + case <-block.executionPayloadChan: + case <-time.After(timeout): + case <-ctx.Done(): + } + + return block.executionPayload +} + // GetParentRoot returns the parent root of this block. func (block *Block) GetParentRoot() *phase0.Root { if block.parentRoot != nil { @@ -234,6 +276,52 @@ func (block *Block) EnsureBlock(loadBlock func() (*spec.VersionedSignedBeaconBlo return true, nil } +// SetExecutionPayload sets the execution payload of this block. +func (block *Block) SetExecutionPayload(payload *eip7732.SignedExecutionPayloadEnvelope) { + block.executionPayload = payload + + if block.executionPayloadChan != nil { + close(block.executionPayloadChan) + block.executionPayloadChan = nil + } +} + +// EnsureExecutionPayload ensures that the execution payload of this block is available. +func (block *Block) EnsureExecutionPayload(loadExecutionPayload func() (*eip7732.SignedExecutionPayloadEnvelope, error)) (bool, error) { + if block.executionPayload != nil { + return false, nil + } + + if block.isInUnfinalizedDb || block.isInFinalizedDb { + return false, nil + } + + block.executionPayloadMutex.Lock() + defer block.executionPayloadMutex.Unlock() + + if block.executionPayload != nil { + return false, nil + } + + payload, err := loadExecutionPayload() + if err != nil { + return false, err + } + + if payload == nil { + return false, nil + } + + block.executionPayload = payload + block.hasExecutionPayload = true + if block.executionPayloadChan != nil { + close(block.executionPayloadChan) + block.executionPayloadChan = nil + } + + return true, nil +} + // setBlockIndex sets the block index of this block. func (block *Block) setBlockIndex(body *spec.VersionedSignedBeaconBlock) { blockIndex := &BlockBodyIndex{} diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index c3733051..5720c4f6 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -10,6 +10,7 @@ import ( v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/db" @@ -30,8 +31,9 @@ type Client struct { archive bool skipValidators bool - blockSubscription *consensus.Subscription[*v1.BlockEvent] - headSubscription *consensus.Subscription[*v1.HeadEvent] + blockSubscription *consensus.Subscription[*v1.BlockEvent] + headSubscription *consensus.Subscription[*v1.HeadEvent] + executionPayloadSubscription *consensus.Subscription[*v1.ExecutionPayloadEvent] headRoot phase0.Root } @@ -79,6 +81,7 @@ func (c *Client) startIndexing() { // blocking block subscription with a buffer to ensure no blocks are missed c.blockSubscription = c.client.SubscribeBlockEvent(100, true) c.headSubscription = c.client.SubscribeHeadEvent(100, true) + c.executionPayloadSubscription = c.client.SubscribeExecutionPayloadEvent(100, true) go c.startClientLoop() } @@ -177,6 +180,11 @@ func (c *Client) runClientLoop() error { if err != nil { c.logger.Errorf("failed processing head %v (%v): %v", headEvent.Slot, headEvent.Block.String(), err) } + case executionPayloadEvent := <-c.executionPayloadSubscription.Channel(): + err := c.processExecutionPayloadEvent(executionPayloadEvent) + if err != nil { + c.logger.Errorf("failed processing execution payload %v (%v): %v", executionPayloadEvent.Slot, executionPayloadEvent.BlockRoot.String(), err) + } } } @@ -513,3 +521,44 @@ func (c *Client) backfillParentBlocks(headBlock *Block) error { } return nil } + +// processExecutionPayloadEvent processes an execution payload event from the event stream. +func (c *Client) processExecutionPayloadEvent(executionPayloadEvent *v1.ExecutionPayloadEvent) error { + if c.client.GetStatus() != consensus.ClientStatusOnline && c.client.GetStatus() != consensus.ClientStatusOptimistic { + // client is not ready, skip + return nil + } + + chainState := c.client.GetPool().GetChainState() + finalizedSlot := chainState.GetFinalizedSlot() + + var block *Block + + if executionPayloadEvent.Slot < finalizedSlot { + // block is in finalized epoch + // known block or a new orphaned block + + // don't add to cache, process this block right after loading the details + block = newBlock(c.indexer.dynSsz, executionPayloadEvent.BlockRoot, executionPayloadEvent.Slot) + + dbBlockHead := db.GetBlockHeadByRoot(executionPayloadEvent.BlockRoot[:]) + if dbBlockHead != nil { + block.isInFinalizedDb = true + block.parentRoot = (*phase0.Root)(dbBlockHead.ParentRoot) + } + + } else { + block = c.indexer.blockCache.getBlockByRoot(executionPayloadEvent.BlockRoot) + } + + if block == nil { + c.logger.Warnf("execution payload event for unknown block %v:%v [0x%x]", chainState.EpochOfSlot(executionPayloadEvent.Slot), executionPayloadEvent.Slot, executionPayloadEvent.BlockRoot) + return nil + } + + _, err := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + return LoadExecutionPayload(c.getContext(), c, executionPayloadEvent.BlockRoot) + }) + + return err +} diff --git a/indexer/beacon/requests.go b/indexer/beacon/requests.go index df6ec6fb..585be13c 100644 --- a/indexer/beacon/requests.go +++ b/indexer/beacon/requests.go @@ -6,6 +6,7 @@ import ( "time" "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" ) @@ -18,6 +19,9 @@ const beaconBodyRequestTimeout time.Duration = 30 * time.Second // BeaconStateRequestTimeout is the timeout duration for beacon state requests. const beaconStateRequestTimeout time.Duration = 600 * time.Second +// ExecutionPayloadRequestTimeout is the timeout duration for execution payload requests. +const executionPayloadRequestTimeout time.Duration = 30 * time.Second + const beaconStateRetryCount = 10 // LoadBeaconHeader loads the block header from the client. @@ -75,3 +79,16 @@ func LoadBeaconState(ctx context.Context, client *Client, root phase0.Root) (*sp return resState, nil } + +// LoadExecutionPayload loads the execution payload from the client. +func LoadExecutionPayload(ctx context.Context, client *Client, root phase0.Root) (*eip7732.SignedExecutionPayloadEnvelope, error) { + ctx, cancel := context.WithTimeout(ctx, executionPayloadRequestTimeout) + defer cancel() + + payload, err := client.client.GetRPCClient().GetExecutionPayloadByBlockroot(ctx, root) + if err != nil { + return nil, err + } + + return payload, nil +} From edd68ac381be1773072d13a95346510ff9e452e8 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 6 Feb 2025 20:24:09 +0100 Subject: [PATCH 06/21] bump go-eth2-client --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3fe8724c..2ffed078 100644 --- a/go.mod +++ b/go.mod @@ -153,6 +153,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7 +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250206185032-c7d9302edebf replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index b53bb516..71157889 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= -github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7 h1:T2yePXJvKXb+YWZKSuCKtuY1WMPsUu9StDFYW5ZAQSU= -github.com/pk910/go-eth2-client v0.0.0-20250205121015-66eea505b4b7/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= +github.com/pk910/go-eth2-client v0.0.0-20250206185032-c7d9302edebf h1:HaW86VhMTHp3I2C21Z+mkEEkwtaDAwj3iXopobENlUA= +github.com/pk910/go-eth2-client v0.0.0-20250206185032-c7d9302edebf/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From f820feb2b7c37885dc102d0bdf275d83451189e4 Mon Sep 17 00:00:00 2001 From: pk910 Date: Sat, 8 Feb 2025 23:59:54 +0100 Subject: [PATCH 07/21] epbs payload handling & db persistence --- clients/consensus/chainstate.go | 8 ++ db/orphaned_blocks.go | 12 +- .../pgsql/20250208225212_epbs-payload.sql | 16 +++ .../sqlite/20250208225212_epbs-payload.sql | 16 +++ db/unfinalized_blocks.go | 47 +++++--- dbtypes/dbtypes.go | 30 ++--- handlers/slot.go | 34 +++++- indexer/beacon/block.go | 61 +++++++--- indexer/beacon/block_helper.go | 76 ++++++++++++ indexer/beacon/client.go | 27 ++++- indexer/beacon/finalization.go | 14 +++ indexer/beacon/indexer.go | 16 ++- indexer/beacon/indexer_getter.go | 8 ++ indexer/beacon/pruning.go | 3 +- indexer/beacon/synchronizer.go | 23 +++- indexer/beacon/writedb.go | 110 +++++++++++++----- services/chainservice_blocks.go | 48 ++++++++ 17 files changed, 460 insertions(+), 89 deletions(-) create mode 100644 db/schema/pgsql/20250208225212_epbs-payload.sql create mode 100644 db/schema/sqlite/20250208225212_epbs-payload.sql diff --git a/clients/consensus/chainstate.go b/clients/consensus/chainstate.go index abc240a4..c0b40311 100644 --- a/clients/consensus/chainstate.go +++ b/clients/consensus/chainstate.go @@ -265,3 +265,11 @@ func (cs *ChainState) GetValidatorChurnLimit(validatorCount uint64) uint64 { return adaptable } + +func (cs *ChainState) IsEip7732Enabled(epoch phase0.Epoch) bool { + if cs.specs == nil { + return false + } + + return cs.specs.Eip7732ForkEpoch != nil && phase0.Epoch(*cs.specs.Eip7732ForkEpoch) <= epoch +} diff --git a/db/orphaned_blocks.go b/db/orphaned_blocks.go index 798b476e..92f44a95 100644 --- a/db/orphaned_blocks.go +++ b/db/orphaned_blocks.go @@ -9,15 +9,15 @@ func InsertOrphanedBlock(block *dbtypes.OrphanedBlock, tx *sqlx.Tx) error { _, err := tx.Exec(EngineQuery(map[dbtypes.DBEngineType]string{ dbtypes.DBEnginePgsql: ` INSERT INTO orphaned_blocks ( - root, header_ver, header_ssz, block_ver, block_ssz - ) VALUES ($1, $2, $3, $4, $5) + root, header_ver, header_ssz, block_ver, block_ssz, payload_ver, payload_ssz + ) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (root) DO NOTHING`, dbtypes.DBEngineSqlite: ` INSERT OR IGNORE INTO orphaned_blocks ( - root, header_ver, header_ssz, block_ver, block_ssz - ) VALUES ($1, $2, $3, $4, $5)`, + root, header_ver, header_ssz, block_ver, block_ssz, payload_ver, payload_ssz + ) VALUES ($1, $2, $3, $4, $5, $6, $7)`, }), - block.Root, block.HeaderVer, block.HeaderSSZ, block.BlockVer, block.BlockSSZ) + block.Root, block.HeaderVer, block.HeaderSSZ, block.BlockVer, block.BlockSSZ, block.PayloadVer, block.PayloadSSZ) if err != nil { return err } @@ -27,7 +27,7 @@ func InsertOrphanedBlock(block *dbtypes.OrphanedBlock, tx *sqlx.Tx) error { func GetOrphanedBlock(root []byte) *dbtypes.OrphanedBlock { block := dbtypes.OrphanedBlock{} err := ReaderDb.Get(&block, ` - SELECT root, header_ver, header_ssz, block_ver, block_ssz + SELECT root, header_ver, header_ssz, block_ver, block_ssz, payload_ver, payload_ssz FROM orphaned_blocks WHERE root = $1 `, root) diff --git a/db/schema/pgsql/20250208225212_epbs-payload.sql b/db/schema/pgsql/20250208225212_epbs-payload.sql new file mode 100644 index 00000000..4db3ab6c --- /dev/null +++ b/db/schema/pgsql/20250208225212_epbs-payload.sql @@ -0,0 +1,16 @@ +-- +goose Up +-- +goose StatementBegin + +ALTER TABLE public."unfinalized_blocks" ADD + "payload_ver" int NOT NULL DEFAULT 0, + "payload_ssz" bytea NULL; + +ALTER TABLE public."orphaned_blocks" ADD + "payload_ver" int NOT NULL DEFAULT 0, + "payload_ssz" bytea NULL; + +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +SELECT 'NOT SUPPORTED'; +-- +goose StatementEnd \ No newline at end of file diff --git a/db/schema/sqlite/20250208225212_epbs-payload.sql b/db/schema/sqlite/20250208225212_epbs-payload.sql new file mode 100644 index 00000000..1b20a8be --- /dev/null +++ b/db/schema/sqlite/20250208225212_epbs-payload.sql @@ -0,0 +1,16 @@ +-- +goose Up +-- +goose StatementBegin + +ALTER TABLE "unfinalized_blocks" ADD + "payload_ver" int NOT NULL DEFAULT 0, + "payload_ssz" BLOB NULL; + +ALTER TABLE "orphaned_blocks" ADD + "payload_ver" int NOT NULL DEFAULT 0, + "payload_ssz" BLOB NULL; + +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +SELECT 'NOT SUPPORTED'; +-- +goose StatementEnd \ No newline at end of file diff --git a/db/unfinalized_blocks.go b/db/unfinalized_blocks.go index 329b5c47..dcdc23c7 100644 --- a/db/unfinalized_blocks.go +++ b/db/unfinalized_blocks.go @@ -12,15 +12,15 @@ func InsertUnfinalizedBlock(block *dbtypes.UnfinalizedBlock, tx *sqlx.Tx) error _, err := tx.Exec(EngineQuery(map[dbtypes.DBEngineType]string{ dbtypes.DBEnginePgsql: ` INSERT INTO unfinalized_blocks ( - root, slot, header_ver, header_ssz, block_ver, block_ssz, status, fork_id - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + root, slot, header_ver, header_ssz, block_ver, block_ssz, payload_ver, payload_ssz, status, fork_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (root) DO NOTHING`, dbtypes.DBEngineSqlite: ` INSERT OR IGNORE INTO unfinalized_blocks ( - root, slot, header_ver, header_ssz, block_ver, block_ssz, status, fork_id - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + root, slot, header_ver, header_ssz, block_ver, block_ssz, payload_ver, payload_ssz, status, fork_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, }), - block.Root, block.Slot, block.HeaderVer, block.HeaderSSZ, block.BlockVer, block.BlockSSZ, block.Status, block.ForkId) + block.Root, block.Slot, block.HeaderVer, block.HeaderSSZ, block.BlockVer, block.BlockSSZ, block.PayloadVer, block.PayloadSSZ, block.Status, block.ForkId) if err != nil { return err } @@ -77,6 +77,14 @@ func UpdateUnfinalizedBlockForkId(roots [][]byte, forkId uint64, tx *sqlx.Tx) er return nil } +func UpdateUnfinalizedBlockPayload(root []byte, payloadVer uint64, payloadSSZ []byte, tx *sqlx.Tx) error { + _, err := tx.Exec(`UPDATE unfinalized_blocks SET payload_ver = $1, payload_ssz = $2 WHERE root = $3`, payloadVer, payloadSSZ, root) + if err != nil { + return err + } + return nil +} + func GetUnfinalizedBlocks(filter *dbtypes.UnfinalizedBlockFilter) []*dbtypes.UnfinalizedBlock { blockRefs := []*dbtypes.UnfinalizedBlock{} @@ -120,7 +128,7 @@ func StreamUnfinalizedBlocks(slot uint64, cb func(block *dbtypes.UnfinalizedBloc var sql strings.Builder args := []any{slot} - fmt.Fprint(&sql, `SELECT root, slot, header_ver, header_ssz, block_ver, block_ssz, status, fork_id FROM unfinalized_blocks WHERE slot >= $1`) + fmt.Fprint(&sql, `SELECT root, slot, header_ver, header_ssz, block_ver, block_ssz, payload_ver, payload_ssz, status, fork_id FROM unfinalized_blocks WHERE slot >= $1`) rows, err := ReaderDb.Query(sql.String(), args...) if err != nil { @@ -130,7 +138,7 @@ func StreamUnfinalizedBlocks(slot uint64, cb func(block *dbtypes.UnfinalizedBloc for rows.Next() { block := dbtypes.UnfinalizedBlock{} - err := rows.Scan(&block.Root, &block.Slot, &block.HeaderVer, &block.HeaderSSZ, &block.BlockVer, &block.BlockSSZ, &block.Status, &block.ForkId) + err := rows.Scan(&block.Root, &block.Slot, &block.HeaderVer, &block.HeaderSSZ, &block.BlockVer, &block.BlockSSZ, &block.PayloadVer, &block.PayloadSSZ, &block.Status, &block.ForkId) if err != nil { logger.Errorf("Error while scanning unfinalized block: %v", err) return err @@ -141,13 +149,26 @@ func StreamUnfinalizedBlocks(slot uint64, cb func(block *dbtypes.UnfinalizedBloc return nil } -func GetUnfinalizedBlock(root []byte) *dbtypes.UnfinalizedBlock { +func GetUnfinalizedBlock(root []byte, withHeader bool, withBody bool, withPayload bool) *dbtypes.UnfinalizedBlock { + var sql strings.Builder + fmt.Fprint(&sql, `SELECT root, slot`) + + if withHeader { + fmt.Fprint(&sql, `, header_ver, header_ssz`) + } + + if withBody { + fmt.Fprint(&sql, `, block_ver, block_ssz`) + } + + if withPayload { + fmt.Fprint(&sql, `, payload_ver, payload_ssz`) + } + + fmt.Fprint(&sql, `FROM unfinalized_blocks WHERE root = $1`) + block := dbtypes.UnfinalizedBlock{} - err := ReaderDb.Get(&block, ` - SELECT root, slot, header_ver, header_ssz, block_ver, block_ssz, status, fork_id - FROM unfinalized_blocks - WHERE root = $1 - `, root) + err := ReaderDb.Get(&block, sql.String(), root) if err != nil { logger.Errorf("Error while fetching unfinalized block 0x%x: %v", root, err) return nil diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index 49c78504..b3870560 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -73,11 +73,13 @@ type Epoch struct { } type OrphanedBlock struct { - Root []byte `db:"root"` - HeaderVer uint64 `db:"header_ver"` - HeaderSSZ []byte `db:"header_ssz"` - BlockVer uint64 `db:"block_ver"` - BlockSSZ []byte `db:"block_ssz"` + Root []byte `db:"root"` + HeaderVer uint64 `db:"header_ver"` + HeaderSSZ []byte `db:"header_ssz"` + BlockVer uint64 `db:"block_ver"` + BlockSSZ []byte `db:"block_ssz"` + PayloadVer uint64 `db:"payload_ver"` + PayloadSSZ []byte `db:"payload_ssz"` } type SlotAssignment struct { @@ -100,14 +102,16 @@ const ( ) type UnfinalizedBlock struct { - Root []byte `db:"root"` - Slot uint64 `db:"slot"` - HeaderVer uint64 `db:"header_ver"` - HeaderSSZ []byte `db:"header_ssz"` - BlockVer uint64 `db:"block_ver"` - BlockSSZ []byte `db:"block_ssz"` - Status UnfinalizedBlockStatus `db:"status"` - ForkId uint64 `db:"fork_id"` + Root []byte `db:"root"` + Slot uint64 `db:"slot"` + HeaderVer uint64 `db:"header_ver"` + HeaderSSZ []byte `db:"header_ssz"` + BlockVer uint64 `db:"block_ver"` + BlockSSZ []byte `db:"block_ssz"` + PayloadVer uint64 `db:"payload_ver"` + PayloadSSZ []byte `db:"payload_ssz"` + Status UnfinalizedBlockStatus `db:"status"` + ForkId uint64 `db:"fork_id"` } type UnfinalizedEpoch struct { diff --git a/handlers/slot.go b/handlers/slot.go index db9482c0..26318e26 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -681,6 +681,28 @@ func getSlotPageBlockData(blockData *services.CombinedBlockResponse, epochStatsV BlockNumber: uint64(executionPayload.BlockNumber), } getSlotPageTransactions(pageData, executionPayload.Transactions) + case spec.DataVersionEIP7732: + if blockData.Payload == nil { + break + } + executionPayload := blockData.Payload.Message.Payload + pageData.ExecutionData = &models.SlotPageExecutionData{ + ParentHash: executionPayload.ParentHash[:], + FeeRecipient: executionPayload.FeeRecipient[:], + StateRoot: executionPayload.StateRoot[:], + ReceiptsRoot: executionPayload.ReceiptsRoot[:], + LogsBloom: executionPayload.LogsBloom[:], + Random: executionPayload.PrevRandao[:], + GasLimit: uint64(executionPayload.GasLimit), + GasUsed: uint64(executionPayload.GasUsed), + Timestamp: uint64(executionPayload.Timestamp), + Time: time.Unix(int64(executionPayload.Timestamp), 0), + ExtraData: executionPayload.ExtraData, + BaseFeePerGas: executionPayload.BaseFeePerGas.Uint64(), + BlockHash: executionPayload.BlockHash[:], + BlockNumber: uint64(executionPayload.BlockNumber), + } + getSlotPageTransactions(pageData, executionPayload.Transactions) } } @@ -723,8 +745,16 @@ func getSlotPageBlockData(blockData *services.CombinedBlockResponse, epochStatsV } if specs.ElectraForkEpoch != nil && uint64(epoch) >= *specs.ElectraForkEpoch { - requests, err := blockData.Block.ExecutionRequests() - if err == nil && requests != nil { + var requests *electra.ExecutionRequests + if blockData.Block.Version >= spec.DataVersionEIP7732 { + if blockData.Payload != nil { + requests = blockData.Payload.Message.ExecutionRequests + } + } else { + requests, _ = blockData.Block.ExecutionRequests() + } + + if requests != nil { getSlotPageDepositRequests(pageData, requests.Deposits) getSlotPageWithdrawalRequests(pageData, requests.Withdrawals) getSlotPageConsolidationRequests(pageData, requests.Consolidations) diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index ec38c125..cc8aca8e 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -121,7 +121,7 @@ func (block *Block) GetBlock() *spec.VersionedSignedBeaconBlock { } if block.isInUnfinalizedDb { - dbBlock := db.GetUnfinalizedBlock(block.Root[:]) + dbBlock := db.GetUnfinalizedBlock(block.Root[:], false, true, false) if dbBlock != nil { blockBody, err := unmarshalVersionedSignedBeaconBlockSSZ(block.dynSsz, dbBlock.BlockVer, dbBlock.BlockSSZ) if err == nil { @@ -155,15 +155,13 @@ func (block *Block) GetExecutionPayload() *eip7732.SignedExecutionPayloadEnvelop } if block.hasExecutionPayload && block.isInUnfinalizedDb { - /* TODO: add execution payload to unfinalized blocks table - dbBlock := db.GetUnfinalizedBlock(block.Root[:]) + dbBlock := db.GetUnfinalizedBlock(block.Root[:], false, false, true) if dbBlock != nil { - blockBody, err := unmarshalVersionedSignedBeaconBlockSSZ(block.dynSsz, dbBlock.BlockVer, dbBlock.BlockSSZ) + payload, err := unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) if err == nil { - return blockBody + return payload } } - */ } return nil @@ -235,7 +233,7 @@ func (block *Block) EnsureHeader(loadHeader func() (*phase0.SignedBeaconBlockHea // SetBlock sets the versioned signed beacon block of this block. func (block *Block) SetBlock(body *spec.VersionedSignedBeaconBlock) { - block.setBlockIndex(body) + block.setBlockIndex(body, nil) block.block = body if block.blockChan != nil { @@ -266,7 +264,7 @@ func (block *Block) EnsureBlock(loadBlock func() (*spec.VersionedSignedBeaconBlo return false, err } - block.setBlockIndex(blockBody) + block.setBlockIndex(blockBody, nil) block.block = blockBody if block.blockChan != nil { close(block.blockChan) @@ -278,6 +276,7 @@ func (block *Block) EnsureBlock(loadBlock func() (*spec.VersionedSignedBeaconBlo // SetExecutionPayload sets the execution payload of this block. func (block *Block) SetExecutionPayload(payload *eip7732.SignedExecutionPayloadEnvelope) { + block.setBlockIndex(block.block, payload) block.executionPayload = payload if block.executionPayloadChan != nil { @@ -312,6 +311,7 @@ func (block *Block) EnsureExecutionPayload(loadExecutionPayload func() (*eip7732 return false, nil } + block.setBlockIndex(block.block, payload) block.executionPayload = payload block.hasExecutionPayload = true if block.executionPayloadChan != nil { @@ -323,12 +323,23 @@ func (block *Block) EnsureExecutionPayload(loadExecutionPayload func() (*eip7732 } // setBlockIndex sets the block index of this block. -func (block *Block) setBlockIndex(body *spec.VersionedSignedBeaconBlock) { - blockIndex := &BlockBodyIndex{} - blockIndex.Graffiti, _ = body.Graffiti() - blockIndex.ExecutionExtraData, _ = getBlockExecutionExtraData(body) - blockIndex.ExecutionHash, _ = body.ExecutionBlockHash() - blockIndex.ExecutionNumber, _ = body.ExecutionBlockNumber() +func (block *Block) setBlockIndex(body *spec.VersionedSignedBeaconBlock, payload *eip7732.SignedExecutionPayloadEnvelope) { + blockIndex := block.blockIndex + if blockIndex == nil { + blockIndex = &BlockBodyIndex{} + } + + if body != nil { + blockIndex.Graffiti, _ = body.Graffiti() + blockIndex.ExecutionExtraData, _ = getBlockExecutionExtraData(body) + blockIndex.ExecutionHash, _ = body.ExecutionBlockHash() + if execNumber, err := body.ExecutionBlockNumber(); err == nil { + blockIndex.ExecutionNumber = uint64(execNumber) + } + } + if payload != nil { + blockIndex.ExecutionNumber = uint64(payload.Message.Payload.BlockNumber) + } block.blockIndex = blockIndex } @@ -341,7 +352,7 @@ func (block *Block) GetBlockIndex() *BlockBodyIndex { blockBody := block.GetBlock() if blockBody != nil { - block.setBlockIndex(blockBody) + block.setBlockIndex(blockBody, block.GetExecutionPayload()) } return block.blockIndex @@ -383,13 +394,24 @@ func (block *Block) buildOrphanedBlock(compress bool) (*dbtypes.OrphanedBlock, e return nil, fmt.Errorf("marshal block ssz failed: %v", err) } - return &dbtypes.OrphanedBlock{ + orphanedBlock := &dbtypes.OrphanedBlock{ Root: block.Root[:], HeaderVer: 1, HeaderSSZ: headerSSZ, BlockVer: blockVer, BlockSSZ: blockSSZ, - }, nil + } + + if block.executionPayload != nil { + payloadVer, payloadSSZ, err := marshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, compress) + if err != nil { + return nil, fmt.Errorf("marshal execution payload ssz failed: %v", err) + } + orphanedBlock.PayloadVer = payloadVer + orphanedBlock.PayloadSSZ = payloadSSZ + } + + return orphanedBlock, nil } // unpruneBlockBody retrieves the block body from the database if it is not already present. @@ -398,9 +420,12 @@ func (block *Block) unpruneBlockBody() { return } - dbBlock := db.GetUnfinalizedBlock(block.Root[:]) + dbBlock := db.GetUnfinalizedBlock(block.Root[:], false, true, true) if dbBlock != nil { block.block, _ = unmarshalVersionedSignedBeaconBlockSSZ(block.dynSsz, dbBlock.BlockVer, dbBlock.BlockSSZ) + if len(dbBlock.PayloadSSZ) > 0 { + block.executionPayload, _ = unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) + } } } diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 34f03b06..a6dcd307 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -208,6 +208,82 @@ func unmarshalVersionedSignedBeaconBlockJson(version uint64, ssz []byte) (*spec. return block, nil } +// marshalVersionedSignedExecutionPayloadEnvelopeSSZ marshals a signed execution payload envelope using SSZ encoding. +func marshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, payload *eip7732.SignedExecutionPayloadEnvelope, compress bool) (version uint64, ssz []byte, err error) { + if utils.Config.KillSwitch.DisableSSZEncoding { + // SSZ encoding disabled, use json instead + version, ssz, err = marshalVersionedSignedExecutionPayloadEnvelopeJson(payload) + } else { + // SSZ encoding + version = uint64(spec.DataVersionEIP7732) + ssz, err = dynSsz.MarshalSSZ(payload) + } + + if compress { + ssz = compressBytes(ssz) + version |= compressionFlag + } + + return +} + +// unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ unmarshals a versioned signed execution payload envelope using SSZ encoding. +func unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, version uint64, ssz []byte) (*eip7732.SignedExecutionPayloadEnvelope, error) { + if (version & compressionFlag) != 0 { + // decompress + if d, err := decompressBytes(ssz); err != nil { + return nil, fmt.Errorf("failed to decompress: %v", err) + } else { + ssz = d + version &= ^compressionFlag + } + } + + if (version & jsonVersionFlag) != 0 { + // JSON encoding + return unmarshalVersionedSignedExecutionPayloadEnvelopeJson(version, ssz) + } + + if version != uint64(spec.DataVersionEIP7732) { + return nil, fmt.Errorf("unknown version") + } + + // SSZ encoding + payload := &eip7732.SignedExecutionPayloadEnvelope{} + if err := dynSsz.UnmarshalSSZ(payload, ssz); err != nil { + return nil, fmt.Errorf("failed to decode eip7732 signed execution payload envelope: %v", err) + } + + return payload, nil +} + +// marshalVersionedSignedExecutionPayloadEnvelopeJson marshals a versioned signed execution payload envelope using JSON encoding. +func marshalVersionedSignedExecutionPayloadEnvelopeJson(payload *eip7732.SignedExecutionPayloadEnvelope) (version uint64, jsonRes []byte, err error) { + version = uint64(spec.DataVersionEIP7732) + jsonRes, err = payload.MarshalJSON() + + version |= jsonVersionFlag + + return +} + +// unmarshalVersionedSignedExecutionPayloadEnvelopeJson unmarshals a versioned signed execution payload envelope using JSON encoding. +func unmarshalVersionedSignedExecutionPayloadEnvelopeJson(version uint64, ssz []byte) (*eip7732.SignedExecutionPayloadEnvelope, error) { + if version&jsonVersionFlag == 0 { + return nil, fmt.Errorf("no json encoding") + } + + if version-jsonVersionFlag != uint64(spec.DataVersionEIP7732) { + return nil, fmt.Errorf("unknown version") + } + + payload := &eip7732.SignedExecutionPayloadEnvelope{} + if err := payload.UnmarshalJSON(ssz); err != nil { + return nil, fmt.Errorf("failed to decode eip7732 signed execution payload envelope: %v", err) + } + return payload, nil +} + // getBlockExecutionExtraData returns the extra data from the execution payload of a versioned signed beacon block. func getBlockExecutionExtraData(v *spec.VersionedSignedBeaconBlock) ([]byte, error) { switch v.Version { diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index 5720c4f6..ee768525 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -556,9 +556,32 @@ func (c *Client) processExecutionPayloadEvent(executionPayloadEvent *v1.Executio return nil } - _, err := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + newPayload, err := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { return LoadExecutionPayload(c.getContext(), c, executionPayloadEvent.BlockRoot) }) + if err != nil { + return err + } - return err + if newPayload { + // write payload to db + payloadVer, payloadSSZ, err := marshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, c.indexer.blockCompression) + if err != nil { + return fmt.Errorf("marshal execution payload ssz failed: %v", err) + } + + err = db.RunDBTransaction(func(tx *sqlx.Tx) error { + err := db.UpdateUnfinalizedBlockPayload(block.Root[:], payloadVer, payloadSSZ, tx) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + } + + return nil } diff --git a/indexer/beacon/finalization.go b/indexer/beacon/finalization.go index 9aae391c..7f56668b 100644 --- a/indexer/beacon/finalization.go +++ b/indexer/beacon/finalization.go @@ -8,6 +8,7 @@ import ( v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" "github.com/ethpandaops/dora/dbtypes" @@ -145,6 +146,19 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R if block.block == nil { return true, fmt.Errorf("missing block body for canonical block %v (%v)", block.Slot, block.Root.String()) } + + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + if _, err := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + return LoadExecutionPayload(client.getContext(), client, block.Root) + }); err != nil { + client.logger.Warnf("failed loading finalized execution payload %v (%v): %v", block.Slot, block.Root.String(), err) + } + + if block.executionPayload == nil && !lastTry { + return false, fmt.Errorf("missing execution payload for canonical block %v (%v)", block.Slot, block.Root.String()) + } + } + canonicalBlocks = append(canonicalBlocks, block) } else { if block.block == nil { diff --git a/indexer/beacon/indexer.go b/indexer/beacon/indexer.go index a66bf632..6b580c00 100644 --- a/indexer/beacon/indexer.go +++ b/indexer/beacon/indexer.go @@ -303,6 +303,7 @@ func (indexer *Indexer) StartIndexer() { // restore unfinalized blocks from db restoredBlockCount := 0 restoredBodyCount := 0 + restoredPayloadCount := 0 t1 = time.Now() err = db.StreamUnfinalizedBlocks(uint64(finalizedSlot), func(dbBlock *dbtypes.UnfinalizedBlock) { @@ -334,10 +335,23 @@ func (indexer *Indexer) StartIndexer() { block.SetBlock(blockBody) restoredBodyCount++ } else { - block.setBlockIndex(blockBody) + block.setBlockIndex(blockBody, nil) block.isInFinalizedDb = true } + if len(dbBlock.PayloadSSZ) > 0 { + blockPayload, err := unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(indexer.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) + if err != nil { + indexer.logger.Warnf("could not restore unfinalized block payload %v [%x] from db: %v", dbBlock.Slot, dbBlock.Root, err) + } else if block.processingStatus == 0 { + block.SetExecutionPayload(blockPayload) + restoredPayloadCount++ + } else { + block.setBlockIndex(blockBody, blockPayload) + block.isInFinalizedDb = true + } + } + indexer.blockCache.addBlockToExecBlockMap(block) blockFork := indexer.forkCache.getForkById(block.forkId) diff --git a/indexer/beacon/indexer_getter.go b/indexer/beacon/indexer_getter.go index c595e9d0..a0ce297e 100644 --- a/indexer/beacon/indexer_getter.go +++ b/indexer/beacon/indexer_getter.go @@ -192,6 +192,14 @@ func (indexer *Indexer) GetOrphanedBlockByRoot(blockRoot phase0.Root) (*Block, e block.SetHeader(header) block.SetBlock(blockBody) + if len(orphanedBlock.PayloadSSZ) > 0 { + payload, err := unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(indexer.dynSsz, orphanedBlock.PayloadVer, orphanedBlock.PayloadSSZ) + if err != nil { + return nil, fmt.Errorf("could not restore orphaned block payload %v [%x] from db: %v", header.Message.Slot, orphanedBlock.Root, err) + } + block.SetExecutionPayload(payload) + } + return block, nil } diff --git a/indexer/beacon/pruning.go b/indexer/beacon/pruning.go index 65033d7f..8209ba6a 100644 --- a/indexer/beacon/pruning.go +++ b/indexer/beacon/pruning.go @@ -257,8 +257,9 @@ func (indexer *Indexer) processEpochPruning(pruneEpoch phase0.Epoch) (uint64, ui for _, block := range pruningBlocks { block.isInFinalizedDb = true block.processingStatus = dbtypes.UnfinalizedBlockStatusPruned - block.setBlockIndex(block.block) + block.setBlockIndex(block.block, block.executionPayload) block.block = nil + block.executionPayload = nil } // clean up epoch stats cache diff --git a/indexer/beacon/synchronizer.go b/indexer/beacon/synchronizer.go index 6d6e7e5e..a62fdadc 100644 --- a/indexer/beacon/synchronizer.go +++ b/indexer/beacon/synchronizer.go @@ -9,6 +9,7 @@ import ( "time" "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" "github.com/ethpandaops/dora/db" @@ -262,11 +263,17 @@ func (sync *synchronizer) loadBlockHeader(client *Client, slot phase0.Slot) (*ph } func (sync *synchronizer) loadBlockBody(client *Client, root phase0.Root) (*spec.VersionedSignedBeaconBlock, error) { - ctx, cancel := context.WithTimeout(sync.syncCtx, beaconHeaderRequestTimeout) + ctx, cancel := context.WithTimeout(sync.syncCtx, beaconBodyRequestTimeout) defer cancel() return LoadBeaconBlock(ctx, client, root) } +func (sync *synchronizer) loadBlockPayload(client *Client, root phase0.Root) (*eip7732.SignedExecutionPayloadEnvelope, error) { + ctx, cancel := context.WithTimeout(sync.syncCtx, executionPayloadRequestTimeout) + defer cancel() + return LoadExecutionPayload(ctx, client, root) +} + func (sync *synchronizer) syncEpoch(syncEpoch phase0.Epoch, client *Client, lastTry bool) (bool, error) { if !utils.Config.Indexer.ResyncForceUpdate && db.IsEpochSynchronized(uint64(syncEpoch)) { return true, nil @@ -312,6 +319,20 @@ func (sync *synchronizer) syncEpoch(syncEpoch phase0.Epoch, client *Client, last block.SetBlock(blockBody) } + if slot > 0 && chainState.IsEip7732Enabled(chainState.EpochOfSlot(slot)) { + blockPayload, err := sync.loadBlockPayload(client, phase0.Root(blockRoot)) + if err != nil && !lastTry { + return false, fmt.Errorf("error fetching slot %v execution payload: %v", slot, err) + } + if blockPayload == nil && !lastTry { + return false, fmt.Errorf("error fetching slot %v execution payload: not found", slot) + } + + if blockPayload != nil { + block.SetExecutionPayload(blockPayload) + } + } + sync.cachedBlocks[slot] = block } diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 3742f4ae..04dd8fed 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -4,6 +4,8 @@ import ( "fmt" "math" + "github.com/attestantio/go-eth2-client/spec/bellatrix" + "github.com/attestantio/go-eth2-client/spec/capella" "github.com/attestantio/go-eth2-client/spec/electra" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" @@ -231,6 +233,8 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override epochStatsValues = epochStats.GetValues(true) } + chainState := dbw.indexer.consensusPool.GetChainState() + graffiti, _ := blockBody.Graffiti() attestations, _ := blockBody.Attestations() deposits, _ := blockBody.Deposits() @@ -239,17 +243,33 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override proposerSlashings, _ := blockBody.ProposerSlashings() blsToExecChanges, _ := blockBody.BLSToExecutionChanges() syncAggregate, _ := blockBody.SyncAggregate() - executionBlockNumber, _ := blockBody.ExecutionBlockNumber() executionBlockHash, _ := blockBody.ExecutionBlockHash() - executionExtraData, _ := getBlockExecutionExtraData(blockBody) - executionTransactions, _ := blockBody.ExecutionTransactions() - executionWithdrawals, _ := blockBody.Withdrawals() + var executionBlockNumber uint64 + var executionExtraData []byte + var executionTransactions []bellatrix.Transaction + var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest - executionRequests, _ := blockBody.ExecutionRequests() - if executionRequests != nil { - depositRequests = executionRequests.Deposits + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + blockPayload := block.GetExecutionPayload() + if blockPayload != nil { + executionBlockNumber = blockPayload.Message.Payload.BlockNumber + executionExtraData = blockPayload.Message.Payload.ExtraData + executionTransactions = blockPayload.Message.Payload.Transactions + executionWithdrawals = blockPayload.Message.Payload.Withdrawals + depositRequests = blockPayload.Message.ExecutionRequests.Deposits + } + } else { + executionBlockNumber, _ = blockBody.ExecutionBlockNumber() + + executionExtraData, _ = getBlockExecutionExtraData(blockBody) + executionTransactions, _ = blockBody.ExecutionTransactions() + executionWithdrawals, _ = blockBody.Withdrawals() + executionRequests, _ := blockBody.ExecutionRequests() + if executionRequests != nil { + depositRequests = executionRequests.Deposits + } } dbBlock := dbtypes.Slot{ @@ -495,14 +515,22 @@ func (dbw *dbWriter) persistBlockDepositRequests(tx *sqlx.Tx, block *Block, orph } func (dbw *dbWriter) buildDbDepositRequests(block *Block, orphaned bool, overrideForkId *ForkKey) []*dbtypes.Deposit { - blockBody := block.GetBlock() - if blockBody == nil { - return nil - } + chainState := dbw.indexer.consensusPool.GetChainState() - requests, err := blockBody.ExecutionRequests() - if err != nil { - return nil + var requests *electra.ExecutionRequests + + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + payload := block.GetExecutionPayload() + if payload != nil { + requests = payload.Message.ExecutionRequests + } + } else { + blockBody := block.GetBlock() + if blockBody == nil { + return nil + } + + requests, _ = blockBody.ExecutionRequests() } deposits := requests.Deposits @@ -685,14 +713,25 @@ func (dbw *dbWriter) persistBlockConsolidationRequests(tx *sqlx.Tx, block *Block } func (dbw *dbWriter) buildDbConsolidationRequests(block *Block, orphaned bool, overrideForkId *ForkKey) []*dbtypes.ConsolidationRequest { - blockBody := block.GetBlock() - if blockBody == nil { - return nil - } + chainState := dbw.indexer.consensusPool.GetChainState() - requests, err := blockBody.ExecutionRequests() - if err != nil { - return nil + var requests *electra.ExecutionRequests + var blockNumber uint64 + + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + payload := block.GetExecutionPayload() + if payload != nil { + requests = payload.Message.ExecutionRequests + blockNumber = payload.Message.Payload.BlockNumber + } + } else { + blockBody := block.GetBlock() + if blockBody == nil { + return nil + } + + requests, _ = blockBody.ExecutionRequests() + blockNumber, _ = blockBody.ExecutionBlockNumber() } consolidations := requests.Consolidations @@ -701,8 +740,6 @@ func (dbw *dbWriter) buildDbConsolidationRequests(block *Block, orphaned bool, o return []*dbtypes.ConsolidationRequest{} } - blockNumber, _ := blockBody.ExecutionBlockNumber() - dbConsolidations := make([]*dbtypes.ConsolidationRequest, len(consolidations)) for idx, consolidation := range consolidations { dbConsolidation := &dbtypes.ConsolidationRequest{ @@ -749,14 +786,25 @@ func (dbw *dbWriter) persistBlockWithdrawalRequests(tx *sqlx.Tx, block *Block, o } func (dbw *dbWriter) buildDbWithdrawalRequests(block *Block, orphaned bool, overrideForkId *ForkKey) []*dbtypes.WithdrawalRequest { - blockBody := block.GetBlock() - if blockBody == nil { - return nil - } + chainState := dbw.indexer.consensusPool.GetChainState() - requests, err := blockBody.ExecutionRequests() - if err != nil { - return nil + var requests *electra.ExecutionRequests + var blockNumber uint64 + + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + payload := block.GetExecutionPayload() + if payload != nil { + requests = payload.Message.ExecutionRequests + blockNumber = payload.Message.Payload.BlockNumber + } + } else { + blockBody := block.GetBlock() + if blockBody == nil { + return nil + } + + requests, _ = blockBody.ExecutionRequests() + blockNumber, _ = blockBody.ExecutionBlockNumber() } withdrawalRequests := requests.Withdrawals @@ -765,8 +813,6 @@ func (dbw *dbWriter) buildDbWithdrawalRequests(block *Block, orphaned bool, over return []*dbtypes.WithdrawalRequest{} } - blockNumber, _ := blockBody.ExecutionBlockNumber() - dbWithdrawalRequests := make([]*dbtypes.WithdrawalRequest, len(withdrawalRequests)) for idx, withdrawalRequest := range withdrawalRequests { dbWithdrawalRequest := &dbtypes.WithdrawalRequest{ diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index 3dc8197b..3c58b669 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -9,6 +9,7 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/deneb" + "github.com/attestantio/go-eth2-client/spec/eip7732" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/db" @@ -21,6 +22,7 @@ type CombinedBlockResponse struct { Root phase0.Root Header *phase0.SignedBeaconBlockHeader Block *spec.VersionedSignedBeaconBlock + Payload *eip7732.SignedExecutionPayloadEnvelope Orphaned bool } @@ -67,6 +69,7 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot Root: blockInfo.Root, Header: blockInfo.GetHeader(), Block: blockInfo.GetBlock(), + Payload: blockInfo.GetExecutionPayload(), Orphaned: !bs.beaconIndexer.IsCanonicalBlock(blockInfo, nil), } } else if blockInfo, err := bs.beaconIndexer.GetOrphanedBlockByRoot(blockroot); blockInfo != nil || err != nil { @@ -77,6 +80,7 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot Root: blockInfo.Root, Header: blockInfo.GetHeader(), Block: blockInfo.GetBlock(), + Payload: blockInfo.GetExecutionPayload(), Orphaned: true, } } else { @@ -125,10 +129,32 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot if err != nil || block == nil { return nil, err } + + var payload *eip7732.SignedExecutionPayloadEnvelope + if block.Version >= spec.DataVersionEIP7732 { + for retry := headRetry; retry < headRetry+3; retry++ { + client := clients[headRetry%len(clients)] + payload, err = beacon.LoadExecutionPayload(ctx, client, blockroot) + if payload != nil { + break + } else if err != nil { + log := logrus.WithError(err) + if client != nil { + log = log.WithField("client", client.GetClient().GetName()) + } + log.Warnf("Error loading block payload for root 0x%x", blockroot) + } + } + if err != nil || payload == nil { + return nil, err + } + } + result = &CombinedBlockResponse{ Root: blockroot, Header: header, Block: block, + Payload: payload, Orphaned: false, } } @@ -160,6 +186,7 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl Root: cachedBlock.Root, Header: cachedBlock.GetHeader(), Block: cachedBlock.GetBlock(), + Payload: cachedBlock.GetExecutionPayload(), Orphaned: isOrphaned, } } else { @@ -210,10 +237,31 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl return nil, err } + var payload *eip7732.SignedExecutionPayloadEnvelope + if block.Version >= spec.DataVersionEIP7732 { + for retry := headRetry; retry < headRetry+3; retry++ { + client := clients[headRetry%len(clients)] + payload, err = beacon.LoadExecutionPayload(ctx, client, blockRoot) + if payload != nil { + break + } else if err != nil { + log := logrus.WithError(err) + if client != nil { + log = log.WithField("client", client.GetClient().GetName()) + } + log.Warnf("Error loading block payload for root 0x%x", blockRoot) + } + } + if err != nil || payload == nil { + return nil, err + } + } + result = &CombinedBlockResponse{ Root: blockRoot, Header: header, Block: block, + Payload: payload, Orphaned: orphaned, } } From 30a4d6dc118f34da1e813ef08ad1888fd44d608d Mon Sep 17 00:00:00 2001 From: pk910 Date: Sun, 9 Feb 2025 00:15:04 +0100 Subject: [PATCH 08/21] fix schema upgrade sql for sqlite --- db/schema/sqlite/20250208225212_epbs-payload.sql | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/db/schema/sqlite/20250208225212_epbs-payload.sql b/db/schema/sqlite/20250208225212_epbs-payload.sql index 1b20a8be..5cb7841e 100644 --- a/db/schema/sqlite/20250208225212_epbs-payload.sql +++ b/db/schema/sqlite/20250208225212_epbs-payload.sql @@ -1,13 +1,11 @@ -- +goose Up -- +goose StatementBegin -ALTER TABLE "unfinalized_blocks" ADD - "payload_ver" int NOT NULL DEFAULT 0, - "payload_ssz" BLOB NULL; +ALTER TABLE "unfinalized_blocks" ADD "payload_ver" int NOT NULL DEFAULT 0; +ALTER TABLE "unfinalized_blocks" ADD "payload_ssz" BLOB NULL; -ALTER TABLE "orphaned_blocks" ADD - "payload_ver" int NOT NULL DEFAULT 0, - "payload_ssz" BLOB NULL; +ALTER TABLE "orphaned_blocks" ADD "payload_ver" int NOT NULL DEFAULT 0; +ALTER TABLE "orphaned_blocks" ADD "payload_ssz" BLOB NULL; -- +goose StatementEnd -- +goose Down From efb9454417bb472d7efd2374e98c81244c651633 Mon Sep 17 00:00:00 2001 From: pk910 Date: Sun, 9 Feb 2025 00:34:43 +0100 Subject: [PATCH 09/21] various small fixes for epbs implementation --- go.mod | 2 +- go.sum | 4 ++-- indexer/beacon/block.go | 3 ++- indexer/beacon/indexer.go | 2 +- indexer/beacon/writedb.go | 12 ++++++++++++ 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 2ffed078..bf9a07ec 100644 --- a/go.mod +++ b/go.mod @@ -153,6 +153,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250206185032-c7d9302edebf +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250208232320-3e9ff5f7e7ae replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index 71157889..7dda2cd2 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= -github.com/pk910/go-eth2-client v0.0.0-20250206185032-c7d9302edebf h1:HaW86VhMTHp3I2C21Z+mkEEkwtaDAwj3iXopobENlUA= -github.com/pk910/go-eth2-client v0.0.0-20250206185032-c7d9302edebf/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= +github.com/pk910/go-eth2-client v0.0.0-20250208232320-3e9ff5f7e7ae h1:yTGoTjLjdtmYdZ4lntYD3rAEKglSlqZEALMK5NcGfig= +github.com/pk910/go-eth2-client v0.0.0-20250208232320-3e9ff5f7e7ae/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index cc8aca8e..b113c19e 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -278,6 +278,7 @@ func (block *Block) EnsureBlock(loadBlock func() (*spec.VersionedSignedBeaconBlo func (block *Block) SetExecutionPayload(payload *eip7732.SignedExecutionPayloadEnvelope) { block.setBlockIndex(block.block, payload) block.executionPayload = payload + block.hasExecutionPayload = true if block.executionPayloadChan != nil { close(block.executionPayloadChan) @@ -291,7 +292,7 @@ func (block *Block) EnsureExecutionPayload(loadExecutionPayload func() (*eip7732 return false, nil } - if block.isInUnfinalizedDb || block.isInFinalizedDb { + if block.hasExecutionPayload { return false, nil } diff --git a/indexer/beacon/indexer.go b/indexer/beacon/indexer.go index 6b580c00..e82c7129 100644 --- a/indexer/beacon/indexer.go +++ b/indexer/beacon/indexer.go @@ -348,7 +348,7 @@ func (indexer *Indexer) StartIndexer() { restoredPayloadCount++ } else { block.setBlockIndex(blockBody, blockPayload) - block.isInFinalizedDb = true + block.hasExecutionPayload = true } } diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 04dd8fed..318b9949 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -533,6 +533,10 @@ func (dbw *dbWriter) buildDbDepositRequests(block *Block, orphaned bool, overrid requests, _ = blockBody.ExecutionRequests() } + if requests == nil { + return []*dbtypes.Deposit{} + } + deposits := requests.Deposits dbDeposits := make([]*dbtypes.Deposit, len(deposits)) @@ -734,6 +738,10 @@ func (dbw *dbWriter) buildDbConsolidationRequests(block *Block, orphaned bool, o blockNumber, _ = blockBody.ExecutionBlockNumber() } + if requests == nil { + return []*dbtypes.ConsolidationRequest{} + } + consolidations := requests.Consolidations if len(consolidations) == 0 { @@ -807,6 +815,10 @@ func (dbw *dbWriter) buildDbWithdrawalRequests(block *Block, orphaned bool, over blockNumber, _ = blockBody.ExecutionBlockNumber() } + if requests == nil { + return []*dbtypes.WithdrawalRequest{} + } + withdrawalRequests := requests.Withdrawals if len(withdrawalRequests) == 0 { From 4cc1d38e69e27ed33d1b64794f44526a3b7d6966 Mon Sep 17 00:00:00 2001 From: pk910 Date: Sun, 9 Feb 2025 01:35:14 +0100 Subject: [PATCH 10/21] fixes for epbs payload loading --- go.mod | 2 +- go.sum | 4 +-- indexer/beacon/client.go | 54 ++++++++++++++++++++++++++++------------ 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index bf9a07ec..7da64df9 100644 --- a/go.mod +++ b/go.mod @@ -153,6 +153,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250208232320-3e9ff5f7e7ae +replace github.com/attestantio/go-eth2-client => github.com/pk910/go-eth2-client v0.0.0-20250208234843-6103e2d9c8ff replace github.com/ethereum/go-ethereum => github.com/s1na/go-ethereum v0.0.0-20250103133732-7e1b0ba7e83f diff --git a/go.sum b/go.sum index 7dda2cd2..064b0ee7 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pk910/dynamic-ssz v0.0.5 h1:VP9heGYUwzlpyhk28P2nCAzhvGsePJOOOO5vQMDh2qQ= github.com/pk910/dynamic-ssz v0.0.5/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= -github.com/pk910/go-eth2-client v0.0.0-20250208232320-3e9ff5f7e7ae h1:yTGoTjLjdtmYdZ4lntYD3rAEKglSlqZEALMK5NcGfig= -github.com/pk910/go-eth2-client v0.0.0-20250208232320-3e9ff5f7e7ae/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= +github.com/pk910/go-eth2-client v0.0.0-20250208234843-6103e2d9c8ff h1:8JGDEupV2yBL9fMWfjY9p54W51bfIKC7u39BXHeDxhY= +github.com/pk910/go-eth2-client v0.0.0-20250208234843-6103e2d9c8ff/go.mod h1:/KTLN3WuH1xrJL7ZZrpBoWM1xCCihnFbzequD5L+83o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index ee768525..4f25f5ab 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -146,7 +146,7 @@ func (c *Client) runClientLoop() error { c.headRoot = headRoot - headBlock, isNew, processingTimes, err := c.processBlock(headSlot, headRoot, nil) + headBlock, isNew, processingTimes, err := c.processBlock(headSlot, headRoot, nil, true) if err != nil { return fmt.Errorf("failed processing head block: %v", err) } @@ -294,7 +294,7 @@ func (c *Client) processHeadEvent(headEvent *v1.HeadEvent) error { // processStreamBlock processes a block received from the stream (either via block or head events). func (c *Client) processStreamBlock(slot phase0.Slot, root phase0.Root) (*Block, error) { - block, isNew, processingTimes, err := c.processBlock(slot, root, nil) + block, isNew, processingTimes, err := c.processBlock(slot, root, nil, false) if err != nil { return nil, err } @@ -348,7 +348,7 @@ func (c *Client) processReorg(oldHead *Block, newHead *Block) error { } // processBlock processes a block (from stream & polling). -func (c *Client) processBlock(slot phase0.Slot, root phase0.Root, header *phase0.SignedBeaconBlockHeader) (block *Block, isNew bool, processingTimes []time.Duration, err error) { +func (c *Client) processBlock(slot phase0.Slot, root phase0.Root, header *phase0.SignedBeaconBlockHeader, loadPayload bool) (block *Block, isNew bool, processingTimes []time.Duration, err error) { chainState := c.client.GetPool().GetChainState() finalizedSlot := chainState.GetFinalizedSlot() processingTimes = make([]time.Duration, 3) @@ -387,7 +387,6 @@ func (c *Client) processBlock(slot phase0.Slot, root phase0.Root, header *phase0 } isNew, err = block.EnsureBlock(func() (*spec.VersionedSignedBeaconBlock, error) { - t1 := time.Now() defer func() { processingTimes[0] += time.Since(t1) @@ -399,6 +398,25 @@ func (c *Client) processBlock(slot phase0.Slot, root phase0.Root, header *phase0 return } + if loadPayload { + newPayload, _ := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + t1 := time.Now() + defer func() { + processingTimes[0] += time.Since(t1) + }() + + return LoadExecutionPayload(c.getContext(), c, root) + }) + + if !isNew && newPayload { + // write payload to db + err = c.persistExecutionPayload(block) + if err != nil { + return + } + } + } + if slot >= finalizedSlot && isNew { c.indexer.blockCache.addBlockToParentMap(block) c.indexer.blockCache.addBlockToExecBlockMap(block) @@ -494,7 +512,7 @@ func (c *Client) backfillParentBlocks(headBlock *Block) error { if parentBlock == nil { var err error - parentBlock, isNewBlock, processingTimes, err = c.processBlock(parentSlot, parentRoot, parentHead) + parentBlock, isNewBlock, processingTimes, err = c.processBlock(parentSlot, parentRoot, parentHead, true) if err != nil { return fmt.Errorf("could not process block [0x%x]: %v", parentRoot, err) } @@ -565,23 +583,27 @@ func (c *Client) processExecutionPayloadEvent(executionPayloadEvent *v1.Executio if newPayload { // write payload to db - payloadVer, payloadSSZ, err := marshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, c.indexer.blockCompression) + err = c.persistExecutionPayload(block) if err != nil { - return fmt.Errorf("marshal execution payload ssz failed: %v", err) + return err } + } - err = db.RunDBTransaction(func(tx *sqlx.Tx) error { - err := db.UpdateUnfinalizedBlockPayload(block.Root[:], payloadVer, payloadSSZ, tx) - if err != nil { - return err - } + return nil +} - return nil - }) +func (c *Client) persistExecutionPayload(block *Block) error { + payloadVer, payloadSSZ, err := marshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, c.indexer.blockCompression) + if err != nil { + return fmt.Errorf("marshal execution payload ssz failed: %v", err) + } + + return db.RunDBTransaction(func(tx *sqlx.Tx) error { + err := db.UpdateUnfinalizedBlockPayload(block.Root[:], payloadVer, payloadSSZ, tx) if err != nil { return err } - } - return nil + return nil + }) } From 804d4b8237f30f9ce181331617860e5ce89de7e8 Mon Sep 17 00:00:00 2001 From: pk910 Date: Sun, 9 Feb 2025 16:07:38 +0100 Subject: [PATCH 11/21] fetch execution payloads and show execution payload status on UI --- db/epochs.go | 15 +++++------ .../pgsql/20250208225212_epbs-payload.sql | 13 ++++++++++ .../sqlite/20250208225212_epbs-payload.sql | 8 ++++++ db/slots.go | 25 ++++++++++--------- db/unfinalized_epochs.go | 20 +++++++++------ dbtypes/dbtypes.go | 3 +++ handlers/epoch.go | 1 + handlers/index.go | 9 +++++-- handlers/slots.go | 4 ++- handlers/slots_filtered.go | 6 +++-- handlers/validator_slots.go | 6 +++-- indexer/beacon/finalization.go | 4 --- indexer/beacon/synchronizer.go | 3 --- indexer/beacon/writedb.go | 25 +++++++++++++++---- static/css/layout.css | 10 ++++++++ templates/epoch/epoch.html | 6 ++--- templates/index/recentBlocks.html | 12 ++++----- templates/index/recentSlots.html | 12 ++++----- templates/slots/slots.html | 6 ++--- templates/slots_filtered/slots_filtered.html | 6 ++--- templates/validator_slots/slots.html | 10 ++++---- types/models/epoch.go | 1 + types/models/indexPage.go | 2 ++ types/models/slots.go | 1 + types/models/slots_filtered.go | 1 + types/models/validator_slots.go | 1 + 26 files changed, 139 insertions(+), 71 deletions(-) diff --git a/db/epochs.go b/db/epochs.go index cf3a1d6f..23885192 100644 --- a/db/epochs.go +++ b/db/epochs.go @@ -11,8 +11,8 @@ func InsertEpoch(epoch *dbtypes.Epoch, tx *sqlx.Tx) error { INSERT INTO epochs ( epoch, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, - proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) + proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, payload_count + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) ON CONFLICT (epoch) DO UPDATE SET validator_count = excluded.validator_count, validator_balance = excluded.validator_balance, @@ -31,17 +31,18 @@ func InsertEpoch(epoch *dbtypes.Epoch, tx *sqlx.Tx) error { proposer_slashing_count = excluded.proposer_slashing_count, bls_change_count = excluded.bls_change_count, eth_transaction_count = excluded.eth_transaction_count, - sync_participation = excluded.sync_participation`, + sync_participation = excluded.sync_participation, + payload_count = excluded.payload_count`, dbtypes.DBEngineSqlite: ` INSERT OR REPLACE INTO epochs ( epoch, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, - proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)`, + proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, payload_count + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)`, }), epoch.Epoch, epoch.ValidatorCount, epoch.ValidatorBalance, epoch.Eligible, epoch.VotedTarget, epoch.VotedHead, epoch.VotedTotal, epoch.BlockCount, epoch.OrphanedCount, epoch.AttestationCount, epoch.DepositCount, epoch.ExitCount, epoch.WithdrawCount, epoch.WithdrawAmount, epoch.AttesterSlashingCount, epoch.ProposerSlashingCount, - epoch.BLSChangeCount, epoch.EthTransactionCount, epoch.SyncParticipation) + epoch.BLSChangeCount, epoch.EthTransactionCount, epoch.SyncParticipation, epoch.PayloadCount) if err != nil { return err } @@ -63,7 +64,7 @@ func GetEpochs(firstEpoch uint64, limit uint32) []*dbtypes.Epoch { SELECT epoch, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, - proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation + proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, payload_count FROM epochs WHERE epoch <= $1 ORDER BY epoch DESC diff --git a/db/schema/pgsql/20250208225212_epbs-payload.sql b/db/schema/pgsql/20250208225212_epbs-payload.sql index 4db3ab6c..527e5645 100644 --- a/db/schema/pgsql/20250208225212_epbs-payload.sql +++ b/db/schema/pgsql/20250208225212_epbs-payload.sql @@ -9,6 +9,19 @@ ALTER TABLE public."orphaned_blocks" ADD "payload_ver" int NOT NULL DEFAULT 0, "payload_ssz" bytea NULL; +ALTER TABLE public."slots" ADD + "has_payload" boolean NOT NULL DEFAULT false; + +CREATE INDEX IF NOT EXISTS "slots_has_payload_idx" + ON public."slots" + ("has_payload" ASC NULLS LAST); + +ALTER TABLE public."epochs" ADD + "payload_count" int NOT NULL DEFAULT 0; + +ALTER TABLE public."unfinalized_epochs" ADD + "payload_count" int NOT NULL DEFAULT 0; + -- +goose StatementEnd -- +goose Down -- +goose StatementBegin diff --git a/db/schema/sqlite/20250208225212_epbs-payload.sql b/db/schema/sqlite/20250208225212_epbs-payload.sql index 5cb7841e..9bc45c06 100644 --- a/db/schema/sqlite/20250208225212_epbs-payload.sql +++ b/db/schema/sqlite/20250208225212_epbs-payload.sql @@ -7,6 +7,14 @@ ALTER TABLE "unfinalized_blocks" ADD "payload_ssz" BLOB NULL; ALTER TABLE "orphaned_blocks" ADD "payload_ver" int NOT NULL DEFAULT 0; ALTER TABLE "orphaned_blocks" ADD "payload_ssz" BLOB NULL; +ALTER TABLE "slots" ADD "has_payload" boolean NOT NULL DEFAULT false; + +CREATE INDEX IF NOT EXISTS "slots_has_payload_idx" ON "slots" ("has_payload" ASC); + +ALTER TABLE "epochs" ADD "payload_count" int NOT NULL DEFAULT 0; + +ALTER TABLE "unfinalized_epochs" ADD "payload_count" int NOT NULL DEFAULT 0; + -- +goose StatementEnd -- +goose Down -- +goose StatementBegin diff --git a/db/slots.go b/db/slots.go index c96b1688..8786cd5a 100644 --- a/db/slots.go +++ b/db/slots.go @@ -19,25 +19,26 @@ func InsertSlot(slot *dbtypes.Slot, tx *sqlx.Tx) error { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23) + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) ON CONFLICT (slot, root) DO UPDATE SET status = excluded.status, eth_block_extra = excluded.eth_block_extra, eth_block_extra_text = excluded.eth_block_extra_text, - fork_id = excluded.fork_id`, + fork_id = excluded.fork_id, + has_payload = excluded.has_payload`, dbtypes.DBEngineSqlite: ` INSERT OR REPLACE INTO slots ( slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23)`, + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24)`, }), slot.Slot, slot.Proposer, slot.Status, slot.Root, slot.ParentRoot, slot.StateRoot, slot.Graffiti, slot.GraffitiText, slot.AttestationCount, slot.DepositCount, slot.ExitCount, slot.WithdrawCount, slot.WithdrawAmount, slot.AttesterSlashingCount, slot.ProposerSlashingCount, slot.BLSChangeCount, slot.EthTransactionCount, slot.EthBlockNumber, slot.EthBlockHash, - slot.EthBlockExtra, slot.EthBlockExtraText, slot.SyncParticipation, slot.ForkId) + slot.EthBlockExtra, slot.EthBlockExtraText, slot.SyncParticipation, slot.ForkId, slot.HasPayload) if err != nil { return err } @@ -92,7 +93,7 @@ func GetSlotsRange(firstSlot uint64, lastSlot uint64, withMissing bool, withOrph "state_root", "root", "slot", "proposer", "status", "parent_root", "graffiti", "graffiti_text", "attestation_count", "deposit_count", "exit_count", "withdraw_count", "withdraw_amount", "attester_slashing_count", "proposer_slashing_count", "bls_change_count", "eth_transaction_count", "eth_block_number", "eth_block_hash", - "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", + "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "has_payload", } for _, blockField := range blockFields { fmt.Fprintf(&sql, ", slots.%v AS \"block.%v\"", blockField, blockField) @@ -124,7 +125,7 @@ func GetSlotsByParentRoot(parentRoot []byte) []*dbtypes.Slot { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload FROM slots WHERE parent_root = $1 ORDER BY slot DESC @@ -143,7 +144,7 @@ func GetSlotByRoot(root []byte) *dbtypes.Slot { root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload FROM slots WHERE root = $1 `, root) @@ -169,7 +170,7 @@ func GetSlotsByRoots(roots [][]byte) map[phase0.Root]*dbtypes.Slot { root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload FROM slots WHERE root IN (%v) ORDER BY slot DESC`, @@ -212,7 +213,7 @@ func GetSlotsByBlockHash(blockHash []byte) []*dbtypes.Slot { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload FROM slots WHERE eth_block_hash = $1 ORDER BY slot DESC @@ -272,7 +273,7 @@ func GetFilteredSlots(filter *dbtypes.BlockFilter, firstSlot uint64, offset uint "state_root", "root", "slot", "proposer", "status", "parent_root", "graffiti", "graffiti_text", "attestation_count", "deposit_count", "exit_count", "withdraw_count", "withdraw_amount", "attester_slashing_count", "proposer_slashing_count", "bls_change_count", "eth_transaction_count", "eth_block_number", "eth_block_hash", - "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", + "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "has_payload", } for _, blockField := range blockFields { fmt.Fprintf(&sql, ", slots.%v AS \"block.%v\"", blockField, blockField) diff --git a/db/unfinalized_epochs.go b/db/unfinalized_epochs.go index cc4230c8..50be7551 100644 --- a/db/unfinalized_epochs.go +++ b/db/unfinalized_epochs.go @@ -11,8 +11,9 @@ func InsertUnfinalizedEpoch(epoch *dbtypes.UnfinalizedEpoch, tx *sqlx.Tx) error INSERT INTO unfinalized_epochs ( epoch, dependent_root, epoch_head_root, epoch_head_fork_id, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, - withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22) + withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, + payload_count + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23) ON CONFLICT (epoch, dependent_root, epoch_head_root) DO UPDATE SET epoch_head_fork_id = excluded.epoch_head_fork_id, validator_count = excluded.validator_count, @@ -32,17 +33,20 @@ func InsertUnfinalizedEpoch(epoch *dbtypes.UnfinalizedEpoch, tx *sqlx.Tx) error proposer_slashing_count = excluded.proposer_slashing_count, bls_change_count = excluded.bls_change_count, eth_transaction_count = excluded.eth_transaction_count, - sync_participation = excluded.sync_participation`, + sync_participation = excluded.sync_participation, + payload_count = excluded.payload_count`, dbtypes.DBEngineSqlite: ` INSERT OR REPLACE INTO unfinalized_epochs ( epoch, dependent_root, epoch_head_root, epoch_head_fork_id, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, - withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22)`, + withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, + payload_count + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23)`, }), epoch.Epoch, epoch.DependentRoot, epoch.EpochHeadRoot, epoch.EpochHeadForkId, epoch.ValidatorCount, epoch.ValidatorBalance, epoch.Eligible, epoch.VotedTarget, epoch.VotedHead, epoch.VotedTotal, epoch.BlockCount, epoch.OrphanedCount, epoch.AttestationCount, epoch.DepositCount, epoch.ExitCount, epoch.WithdrawCount, epoch.WithdrawAmount, epoch.AttesterSlashingCount, epoch.ProposerSlashingCount, epoch.BLSChangeCount, epoch.EthTransactionCount, epoch.SyncParticipation, + epoch.PayloadCount, ) if err != nil { return err @@ -55,7 +59,8 @@ func StreamUnfinalizedEpochs(epoch uint64, cb func(duty *dbtypes.UnfinalizedEpoc SELECT epoch, dependent_root, epoch_head_root, epoch_head_fork_id, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, - withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation + withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, + payload_count FROM unfinalized_epochs WHERE epoch >= $1`, epoch) if err != nil { @@ -69,6 +74,7 @@ func StreamUnfinalizedEpochs(epoch uint64, cb func(duty *dbtypes.UnfinalizedEpoc &e.Epoch, &e.DependentRoot, &e.EpochHeadRoot, &e.EpochHeadForkId, &e.ValidatorCount, &e.ValidatorBalance, &e.Eligible, &e.VotedTarget, &e.VotedHead, &e.VotedTotal, &e.BlockCount, &e.OrphanedCount, &e.AttestationCount, &e.DepositCount, &e.ExitCount, &e.WithdrawCount, &e.WithdrawAmount, &e.AttesterSlashingCount, &e.ProposerSlashingCount, &e.BLSChangeCount, &e.EthTransactionCount, &e.SyncParticipation, + &e.PayloadCount, ) if err != nil { logger.Errorf("Error while scanning unfinalized epoch: %v", err) @@ -86,7 +92,7 @@ func GetUnfinalizedEpoch(epoch uint64, headRoot []byte) *dbtypes.UnfinalizedEpoc SELECT epoch, dependent_root, epoch_head_root, epoch_head_fork_id, validator_count, validator_balance, eligible, voted_target, voted_head, voted_total, block_count, orphaned_count, attestation_count, deposit_count, exit_count, withdraw_count, - withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation + withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, sync_participation, payload_count FROM unfinalized_epochs WHERE epoch = $1 AND epoch_head_root = $2 `, epoch, headRoot) diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index b3870560..a92231e7 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -48,6 +48,7 @@ type Slot struct { EthBlockExtraText string `db:"eth_block_extra_text"` SyncParticipation float32 `db:"sync_participation"` ForkId uint64 `db:"fork_id"` + HasPayload bool `db:"has_payload"` } type Epoch struct { @@ -70,6 +71,7 @@ type Epoch struct { BLSChangeCount uint64 `db:"bls_change_count"` EthTransactionCount uint64 `db:"eth_transaction_count"` SyncParticipation float32 `db:"sync_participation"` + PayloadCount uint64 `db:"payload_count"` } type OrphanedBlock struct { @@ -137,6 +139,7 @@ type UnfinalizedEpoch struct { BLSChangeCount uint64 `db:"bls_change_count"` EthTransactionCount uint64 `db:"eth_transaction_count"` SyncParticipation float32 `db:"sync_participation"` + PayloadCount uint64 `db:"payload_count"` } type Fork struct { diff --git a/handlers/epoch.go b/handlers/epoch.go index a7bb3269..b1c6b28c 100644 --- a/handlers/epoch.go +++ b/handlers/epoch.go @@ -173,6 +173,7 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) { Ts: chainState.SlotToTime(phase0.Slot(slot)), Scheduled: slot >= uint64(currentSlot) && dbSlot.Status == dbtypes.Missing, Status: uint8(dbSlot.Status), + NoPayload: !dbSlot.HasPayload && chainState.IsEip7732Enabled(phase0.Epoch(epoch)), Proposer: dbSlot.Proposer, ProposerName: services.GlobalBeaconService.GetValidatorName(dbSlot.Proposer), AttestationCount: dbSlot.AttestationCount, diff --git a/handlers/index.go b/handlers/index.go index 24a52efa..27c07e9d 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -284,13 +284,16 @@ func buildIndexPageRecentBlocksData(pageData *models.IndexPageData, recentBlockC if blockData == nil { continue } + + epoch := chainState.EpochOfSlot(phase0.Slot(blockData.Slot)) blockModel := &models.IndexPageDataBlocks{ - Epoch: uint64(chainState.EpochOfSlot(phase0.Slot(blockData.Slot))), + Epoch: uint64(epoch), Slot: blockData.Slot, Ts: chainState.SlotToTime(phase0.Slot(blockData.Slot)), Proposer: blockData.Proposer, ProposerName: services.GlobalBeaconService.GetValidatorName(blockData.Proposer), Status: uint64(blockData.Status), + NoPayload: !blockData.HasPayload && chainState.IsEip7732Enabled(epoch), BlockRoot: blockData.Root, } if blockData.EthBlockNumber != nil { @@ -329,11 +332,13 @@ func buildIndexPageRecentSlotsData(pageData *models.IndexPageData, firstSlot pha dbSlot := dbSlots[dbIdx] dbIdx++ + epoch := chainState.EpochOfSlot(phase0.Slot(dbSlot.Slot)) slotData := &models.IndexPageDataSlots{ Slot: slot, - Epoch: uint64(chainState.EpochOfSlot(phase0.Slot(dbSlot.Slot))), + Epoch: uint64(epoch), Ts: chainState.SlotToTime(phase0.Slot(slot)), Status: uint64(dbSlot.Status), + NoPayload: !dbSlot.HasPayload && chainState.IsEip7732Enabled(epoch), Proposer: dbSlot.Proposer, ProposerName: services.GlobalBeaconService.GetValidatorName(dbSlot.Proposer), BlockRoot: dbSlot.Root, diff --git a/handlers/slots.go b/handlers/slots.go index 1de7f1b7..67bb151d 100644 --- a/handlers/slots.go +++ b/handlers/slots.go @@ -142,12 +142,14 @@ func buildSlotsPageData(firstSlot uint64, pageSize uint64) (*models.SlotsPageDat dbSlot := dbSlots[dbIdx] dbIdx++ + epoch := chainState.EpochOfSlot(phase0.Slot(slot)) slotData := &models.SlotsPageDataSlot{ Slot: slot, - Epoch: uint64(chainState.EpochOfSlot(phase0.Slot(slot))), + Epoch: uint64(epoch), Ts: chainState.SlotToTime(phase0.Slot(slot)), Finalized: finalized, Status: uint8(dbSlot.Status), + NoPayload: !dbSlot.HasPayload && chainState.IsEip7732Enabled(epoch), Scheduled: slot >= uint64(currentSlot) && dbSlot.Status == dbtypes.Missing, Synchronized: dbSlot.SyncParticipation != -1, Proposer: dbSlot.Proposer, diff --git a/handlers/slots_filtered.go b/handlers/slots_filtered.go index dcb8ce79..fd17ba99 100644 --- a/handlers/slots_filtered.go +++ b/handlers/slots_filtered.go @@ -234,12 +234,13 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string break } slot := phase0.Slot(dbBlock.Slot) + epoch := chainState.EpochOfSlot(slot) slotData := &models.SlotsFilteredPageDataSlot{ Slot: uint64(slot), - Epoch: uint64(chainState.EpochOfSlot(slot)), + Epoch: uint64(epoch), Ts: chainState.SlotToTime(slot), - Finalized: finalizedEpoch >= chainState.EpochOfSlot(slot), + Finalized: finalizedEpoch >= epoch, Synchronized: true, Scheduled: slot >= currentSlot, Proposer: dbBlock.Proposer, @@ -265,6 +266,7 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string slotData.WithEthBlock = true slotData.EthBlockNumber = *dbBlock.Block.EthBlockNumber } + slotData.NoPayload = !dbBlock.Block.HasPayload && chainState.IsEip7732Enabled(epoch) } pageData.Slots = append(pageData.Slots, slotData) } diff --git a/handlers/validator_slots.go b/handlers/validator_slots.go index c80a7abd..dad03dc4 100644 --- a/handlers/validator_slots.go +++ b/handlers/validator_slots.go @@ -112,12 +112,13 @@ func buildValidatorSlotsPageData(validator uint64, pageIdx uint64, pageSize uint break } slot := blockAssignment.Slot + epoch := chainState.EpochOfSlot(phase0.Slot(slot)) slotData := &models.ValidatorSlotsPageDataSlot{ Slot: slot, - Epoch: uint64(chainState.EpochOfSlot(phase0.Slot(slot))), + Epoch: uint64(epoch), Ts: chainState.SlotToTime(phase0.Slot(slot)), - Finalized: finalizedEpoch >= chainState.EpochOfSlot(phase0.Slot(slot)), + Finalized: finalizedEpoch >= epoch, Status: uint8(0), Proposer: validator, ProposerName: pageData.Name, @@ -139,6 +140,7 @@ func buildValidatorSlotsPageData(validator uint64, pageIdx uint64, pageSize uint slotData.WithEthBlock = true slotData.EthBlockNumber = *dbBlock.EthBlockNumber } + slotData.NoPayload = !dbBlock.HasPayload && chainState.IsEip7732Enabled(epoch) } pageData.Slots = append(pageData.Slots, slotData) } diff --git a/indexer/beacon/finalization.go b/indexer/beacon/finalization.go index 7f56668b..9428db6c 100644 --- a/indexer/beacon/finalization.go +++ b/indexer/beacon/finalization.go @@ -153,10 +153,6 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R }); err != nil { client.logger.Warnf("failed loading finalized execution payload %v (%v): %v", block.Slot, block.Root.String(), err) } - - if block.executionPayload == nil && !lastTry { - return false, fmt.Errorf("missing execution payload for canonical block %v (%v)", block.Slot, block.Root.String()) - } } canonicalBlocks = append(canonicalBlocks, block) diff --git a/indexer/beacon/synchronizer.go b/indexer/beacon/synchronizer.go index a62fdadc..1af16be1 100644 --- a/indexer/beacon/synchronizer.go +++ b/indexer/beacon/synchronizer.go @@ -324,9 +324,6 @@ func (sync *synchronizer) syncEpoch(syncEpoch phase0.Epoch, client *Client, last if err != nil && !lastTry { return false, fmt.Errorf("error fetching slot %v execution payload: %v", slot, err) } - if blockPayload == nil && !lastTry { - return false, fmt.Errorf("error fetching slot %v execution payload: not found", slot) - } if blockPayload != nil { block.SetExecutionPayload(blockPayload) diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 318b9949..79614be4 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -250,6 +250,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override var executionTransactions []bellatrix.Transaction var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest + var hasPayload bool if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { blockPayload := block.GetExecutionPayload() @@ -259,6 +260,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override executionTransactions = blockPayload.Message.Payload.Transactions executionWithdrawals = blockPayload.Message.Payload.Withdrawals depositRequests = blockPayload.Message.ExecutionRequests.Deposits + hasPayload = true } } else { executionBlockNumber, _ = blockBody.ExecutionBlockNumber() @@ -288,6 +290,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override AttesterSlashingCount: uint64(len(attesterSlashings)), ProposerSlashingCount: uint64(len(proposerSlashings)), BLSChangeCount: uint64(len(blsToExecChanges)), + HasPayload: hasPayload, } if overrideForkId != nil { @@ -394,14 +397,26 @@ func (dbw *dbWriter) buildDbEpoch(epoch phase0.Epoch, blocks []*Block, epochStat proposerSlashings, _ := blockBody.ProposerSlashings() blsToExecChanges, _ := blockBody.BLSToExecutionChanges() syncAggregate, _ := blockBody.SyncAggregate() - executionTransactions, _ := blockBody.ExecutionTransactions() - executionWithdrawals, _ := blockBody.Withdrawals() + var executionTransactions []bellatrix.Transaction + var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest - executionRequests, _ := blockBody.ExecutionRequests() - if executionRequests != nil { - depositRequests = executionRequests.Deposits + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { + blockPayload := block.GetExecutionPayload() + if blockPayload != nil { + dbEpoch.PayloadCount++ + executionTransactions = blockPayload.Message.Payload.Transactions + executionWithdrawals = blockPayload.Message.Payload.Withdrawals + depositRequests = blockPayload.Message.ExecutionRequests.Deposits + } + } else { + executionTransactions, _ = blockBody.ExecutionTransactions() + executionWithdrawals, _ = blockBody.Withdrawals() + executionRequests, _ := blockBody.ExecutionRequests() + if executionRequests != nil { + depositRequests = executionRequests.Deposits + } } dbEpoch.AttestationCount += uint64(len(attestations)) diff --git a/static/css/layout.css b/static/css/layout.css index 8947b1d9..c10a4331 100644 --- a/static/css/layout.css +++ b/static/css/layout.css @@ -322,3 +322,13 @@ span.validator-label { .table-sm>:not(caption)>*>* { padding: 1px .25rem; } + +.badge.split-warning { + background: linear-gradient( + 90deg, + rgba(255,255,255,0) 0%, + rgba(255,255,255,0) 50%, + rgba(255,193,7,1) 50%, + rgba(255,193,7,1) 100% + ); +} diff --git a/templates/epoch/epoch.html b/templates/epoch/epoch.html index 3957ba1f..c400c2bf 100644 --- a/templates/epoch/epoch.html +++ b/templates/epoch/epoch.html @@ -176,15 +176,15 @@

{{ else if $slot.Scheduled }} Scheduled {{ else if eq $slot.Status 1 }} - Proposed + Proposed {{ else if eq $slot.Status 2 }} - Orphaned + Orphaned {{ else if not $epoch.Synchronized }} ? {{ else if eq $slot.Status 0 }} Missed {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/templates/index/recentBlocks.html b/templates/index/recentBlocks.html index ab59debc..9e78baeb 100644 --- a/templates/index/recentBlocks.html +++ b/templates/index/recentBlocks.html @@ -41,9 +41,9 @@

Genesis Missed - Proposed - Missed (Orphaned) - Unknown + Proposed + Missed (Orphaned) + Unknown @@ -74,11 +74,11 @@
Missed {{ else if eq .Status 1 }} - Proposed + Proposed {{ else if eq .Status 2 }} - Missed (Orphaned) + Missed (Orphaned) {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $block.Ts }} diff --git a/templates/index/recentSlots.html b/templates/index/recentSlots.html index 11861923..3be0f7ff 100644 --- a/templates/index/recentSlots.html +++ b/templates/index/recentSlots.html @@ -42,9 +42,9 @@
Genesis Missed - Proposed - Missed (Orphaned) - Unknown + Proposed + Missed (Orphaned) + Unknown @@ -97,11 +97,11 @@
Missed {{ else if eq .Status 1 }} - Proposed + Proposed {{ else if eq .Status 2 }} - Orphaned + Missed (Orphaned) {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/templates/slots/slots.html b/templates/slots/slots.html index b847031f..2a4a6d14 100644 --- a/templates/slots/slots.html +++ b/templates/slots/slots.html @@ -92,9 +92,9 @@

Slots

{{ if eq $slot.Slot 0 }} Genesis {{ else if eq $slot.Status 1 }} - Proposed + Proposed {{ else if eq $slot.Status 2 }} - Orphaned + Missed (Orphaned) {{ else if $slot.Scheduled }} Scheduled {{ else if not $slot.Synchronized }} @@ -102,7 +102,7 @@

Slots

{{ else if eq $slot.Status 0 }} Missed {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/templates/slots_filtered/slots_filtered.html b/templates/slots_filtered/slots_filtered.html index 7d4ed6b4..8400972e 100644 --- a/templates/slots_filtered/slots_filtered.html +++ b/templates/slots_filtered/slots_filtered.html @@ -183,9 +183,9 @@

Filtered Slots

{{- if eq $slot.Slot 0 }} Genesis {{- else if eq $slot.Status 1 }} - Proposed + Proposed {{- else if eq $slot.Status 2 }} - Orphaned + Missed (Orphaned) {{- else if $slot.Scheduled }} Scheduled {{- else if not $slot.Synchronized }} @@ -193,7 +193,7 @@

Filtered Slots

{{- else if eq $slot.Status 0 }} Missed {{- else }} - Unknown + Unknown {{- end }} {{- end }} diff --git a/templates/validator_slots/slots.html b/templates/validator_slots/slots.html index f8a02731..a89f09d8 100644 --- a/templates/validator_slots/slots.html +++ b/templates/validator_slots/slots.html @@ -71,16 +71,16 @@

Validator {{ format {{ if eq $slot.Slot 0 }} Genesis + {{ else if eq $slot.Status 1 }} + Proposed + {{ else if eq $slot.Status 2 }} + Missed (Orphaned) {{ else if $slot.Scheduled }} Scheduled {{ else if eq $slot.Status 0 }} Missed - {{ else if eq $slot.Status 1 }} - Proposed - {{ else if eq $slot.Status 2 }} - Orphaned {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/types/models/epoch.go b/types/models/epoch.go index e7a8d263..c15be304 100644 --- a/types/models/epoch.go +++ b/types/models/epoch.go @@ -44,6 +44,7 @@ type EpochPageDataSlot struct { Ts time.Time `json:"ts"` Scheduled bool `json:"scheduled"` Status uint8 `json:"status"` + NoPayload bool `json:"no_payload"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` AttestationCount uint64 `json:"attestation_count"` diff --git a/types/models/indexPage.go b/types/models/indexPage.go index c0a64898..af5d5c4d 100644 --- a/types/models/indexPage.go +++ b/types/models/indexPage.go @@ -65,6 +65,7 @@ type IndexPageDataBlocks struct { Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` Status uint64 `json:"status"` + NoPayload bool `json:"no_payload"` BlockRoot []byte `json:"block_root"` } @@ -76,6 +77,7 @@ type IndexPageDataSlots struct { Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` Status uint64 `json:"status"` + NoPayload bool `json:"no_payload"` BlockRoot []byte `json:"block_root"` ParentRoot []byte `json:"-"` ForkGraph []*IndexPageDataForkGraph `json:"fork_graph"` diff --git a/types/models/slots.go b/types/models/slots.go index 075277e3..a3fcf9d4 100644 --- a/types/models/slots.go +++ b/types/models/slots.go @@ -31,6 +31,7 @@ type SlotsPageDataSlot struct { Finalized bool `json:"scheduled"` Scheduled bool `json:"finalized"` Status uint8 `json:"status"` + NoPayload bool `json:"no_payload"` Synchronized bool `json:"synchronized"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` diff --git a/types/models/slots_filtered.go b/types/models/slots_filtered.go index b084733d..38fd10d4 100644 --- a/types/models/slots_filtered.go +++ b/types/models/slots_filtered.go @@ -56,6 +56,7 @@ type SlotsFilteredPageDataSlot struct { Finalized bool `json:"scheduled"` Scheduled bool `json:"finalized"` Status uint8 `json:"status"` + NoPayload bool `json:"no_payload"` Synchronized bool `json:"synchronized"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` diff --git a/types/models/validator_slots.go b/types/models/validator_slots.go index 22b33e1e..0f4ade49 100644 --- a/types/models/validator_slots.go +++ b/types/models/validator_slots.go @@ -34,6 +34,7 @@ type ValidatorSlotsPageDataSlot struct { Finalized bool `json:"scheduled"` Scheduled bool `json:"finalized"` Status uint8 `json:"status"` + NoPayload bool `json:"no_payload"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` AttestationCount uint64 `json:"attestation_count"` From 3c637f9f597a0b2c04e14ec98a154aaabfdc5764 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 11 Feb 2025 14:15:48 +0100 Subject: [PATCH 12/21] fix payload badges on knockout rendered startpage --- templates/index/recentBlocks.html | 6 +++--- templates/index/recentSlots.html | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/index/recentBlocks.html b/templates/index/recentBlocks.html index 9e78baeb..51b639fc 100644 --- a/templates/index/recentBlocks.html +++ b/templates/index/recentBlocks.html @@ -41,9 +41,9 @@
Genesis Missed - Proposed - Missed (Orphaned) - Unknown + Proposed + Missed (Orphaned) + Unknown diff --git a/templates/index/recentSlots.html b/templates/index/recentSlots.html index 3be0f7ff..bc348763 100644 --- a/templates/index/recentSlots.html +++ b/templates/index/recentSlots.html @@ -42,9 +42,9 @@
Genesis Missed - Proposed - Missed (Orphaned) - Unknown + Proposed + Missed (Orphaned) + Unknown From 6da622da310c9e52e9a786239c4ae8bb11c2a688 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 11 Feb 2025 14:56:45 +0100 Subject: [PATCH 13/21] show payload header data on slot details page --- handlers/slot.go | 16 ++++++ templates/slot/overview.html | 95 ++++++++++++++++++++++++++++++++---- types/models/slot.go | 18 ++++++- 3 files changed, 118 insertions(+), 11 deletions(-) diff --git a/handlers/slot.go b/handlers/slot.go index 26318e26..9defa093 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -682,10 +682,26 @@ func getSlotPageBlockData(blockData *services.CombinedBlockResponse, epochStatsV } getSlotPageTransactions(pageData, executionPayload.Transactions) case spec.DataVersionEIP7732: + payloadHeader := blockData.Block.EIP7732.Message.Body.SignedExecutionPayloadHeader + pageData.PayloadHeader = &models.SlotPagePayloadHeader{ + PayloadStatus: 0, + ParentBlockHash: payloadHeader.Message.ParentBlockHash[:], + ParentBlockRoot: payloadHeader.Message.ParentBlockRoot[:], + BlockHash: payloadHeader.Message.BlockHash[:], + GasLimit: uint64(payloadHeader.Message.GasLimit), + BuilderIndex: uint64(payloadHeader.Message.BuilderIndex), + BuilderName: services.GlobalBeaconService.GetValidatorName(uint64(payloadHeader.Message.BuilderIndex)), + Slot: uint64(payloadHeader.Message.Slot), + Value: uint64(payloadHeader.Message.Value), + BlobKzgCommitmentsRoot: payloadHeader.Message.BlobKZGCommitmentsRoot[:], + Signature: payloadHeader.Signature[:], + } + if blockData.Payload == nil { break } executionPayload := blockData.Payload.Message.Payload + pageData.PayloadHeader.PayloadStatus = 1 pageData.ExecutionData = &models.SlotPageExecutionData{ ParentHash: executionPayload.ParentHash[:], FeeRecipient: executionPayload.FeeRecipient[:], diff --git a/templates/slot/overview.html b/templates/slot/overview.html index ce631bcb..38a004df 100644 --- a/templates/slot/overview.html +++ b/templates/slot/overview.html @@ -152,30 +152,103 @@ {{ end }} - {{ if .Block.ExecutionData }} + + {{ if .Block.PayloadHeader }} {{ $block := .Block }} - {{ with .Block.ExecutionData }} + {{ with .Block.PayloadHeader }}
-
Execution Payload:
+
Payload Header:
+
-
Block Number:
-
{{ ethBlockLink .BlockNumber }}
+
Payload Status:
+
+ {{ if eq .PayloadStatus 0 }} + Missing + {{ else if eq .PayloadStatus 1 }} + Proposed + {{ else if eq .PayloadStatus 2 }} + Orphaned + {{ else }} + Unknown + {{ end }} +
Block Hash:
{{ ethBlockHashLink .BlockHash }} +
Parent Hash:
- 0x{{ printf "%x" .ParentHash }} + {{ ethBlockHashLink .ParentBlockHash }} + +
+
+ +
+
Builder Index:
+
+ {{ formatValidator .BuilderIndex .BuilderName }} +
+
+ +
+
Block Value:
+
+ {{ formatEthFromGwei .Value }} +
+
+ +
+
Gas Limit:
+
+ {{ .GasLimit }} +
+
+ +
+
Blob KZG Root:
+
+ 0x{{ printf "%x" .BlobKzgCommitmentsRoot }} +
+
+
+ {{ end }} + {{ end }} + {{ if .Block.ExecutionData }} + {{ $block := .Block }} + {{ with .Block.ExecutionData }} +
+
Execution Payload:
+
+
+
Block Number:
+
{{ ethBlockLink .BlockNumber }}
+
+ + {{ if not $block.PayloadHeader }} +
+
Block Hash:
+
+ {{ ethBlockHashLink .BlockHash }} +
+
+ +
+
Parent Hash:
+
+ 0x{{ printf "%x" .ParentHash }} +
+
+ {{ end }}
Fee Recipient:
@@ -189,10 +262,12 @@
{{ .GasUsed }}
-
-
Gas Limit:
-
{{ .GasLimit }}
-
+ {{ if not $block.PayloadHeader }} +
+
Gas Limit:
+
{{ .GasLimit }}
+
+ {{ end }}
Base fee per gas:
diff --git a/types/models/slot.go b/types/models/slot.go index 7ea14080..34901472 100644 --- a/types/models/slot.go +++ b/types/models/slot.go @@ -66,7 +66,9 @@ type SlotPageBlockData struct { WithdrawalRequestsCount uint64 `json:"withdrawal_requests_count"` ConsolidationRequestsCount uint64 `json:"consolidation_requests_count"` - ExecutionData *SlotPageExecutionData `json:"execution_data"` + PayloadHeader *SlotPagePayloadHeader `json:"payload_header"` + ExecutionData *SlotPageExecutionData `json:"execution_data"` + Attestations []*SlotPageAttestation `json:"attestations"` // Attestations included in this block Deposits []*SlotPageDeposit `json:"deposits"` // Deposits included in this block VoluntaryExits []*SlotPageVoluntaryExit `json:"voluntary_exits"` // Voluntary Exits included in this block @@ -98,6 +100,20 @@ type SlotPageExecutionData struct { BlockNumber uint64 `json:"block_number"` } +type SlotPagePayloadHeader struct { + PayloadStatus uint16 `json:"payload_status"` + ParentBlockHash []byte `json:"parent_block_hash"` + ParentBlockRoot []byte `json:"parent_block_root"` + BlockHash []byte `json:"block_hash"` + GasLimit uint64 `json:"gas_limit"` + BuilderIndex uint64 `json:"builder_index"` + BuilderName string `json:"builder_name"` + Slot uint64 `json:"slot"` + Value uint64 `json:"value"` + BlobKzgCommitmentsRoot []byte `json:"blob_kzg_commitments_root"` + Signature []byte `json:"signature"` +} + type SlotPageAttestation struct { Slot uint64 `json:"slot"` CommitteeIndex []uint64 `json:"committeeindex"` From 4fe1f72c207d043e31940a5f28029136d1e6e393 Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 12 Feb 2025 11:18:46 +0100 Subject: [PATCH 14/21] add payload_status & remove has_payload to allow additional payload status (orphaned) --- .../pgsql/20250208225212_epbs-payload.sql | 6 +- .../sqlite/20250208225212_epbs-payload.sql | 4 +- db/slots.go | 20 +++---- dbtypes/dbtypes.go | 56 +++++++++++-------- handlers/epoch.go | 7 ++- handlers/index.go | 48 ++++++++++------ handlers/slots.go | 7 ++- handlers/slots_filtered.go | 7 ++- handlers/validator_slots.go | 7 ++- indexer/beacon/writedb.go | 9 ++- static/css/layout.css | 10 ++++ templates/epoch/epoch.html | 6 +- templates/index/recentBlocks.html | 12 ++-- templates/index/recentSlots.html | 12 ++-- templates/slots/slots.html | 6 +- templates/slots_filtered/slots_filtered.html | 6 +- templates/validator_slots/slots.html | 6 +- types/models/epoch.go | 2 +- types/models/indexPage.go | 44 +++++++-------- types/models/slots.go | 2 +- types/models/slots_filtered.go | 2 +- types/models/validator_slots.go | 2 +- 22 files changed, 167 insertions(+), 114 deletions(-) diff --git a/db/schema/pgsql/20250208225212_epbs-payload.sql b/db/schema/pgsql/20250208225212_epbs-payload.sql index 527e5645..bccb6ba9 100644 --- a/db/schema/pgsql/20250208225212_epbs-payload.sql +++ b/db/schema/pgsql/20250208225212_epbs-payload.sql @@ -10,11 +10,11 @@ ALTER TABLE public."orphaned_blocks" ADD "payload_ssz" bytea NULL; ALTER TABLE public."slots" ADD - "has_payload" boolean NOT NULL DEFAULT false; + "payload_status" smallint NOT NULL DEFAULT 0; -CREATE INDEX IF NOT EXISTS "slots_has_payload_idx" +CREATE INDEX IF NOT EXISTS "slots_payload_status_idx" ON public."slots" - ("has_payload" ASC NULLS LAST); + ("payload_status" ASC NULLS LAST); ALTER TABLE public."epochs" ADD "payload_count" int NOT NULL DEFAULT 0; diff --git a/db/schema/sqlite/20250208225212_epbs-payload.sql b/db/schema/sqlite/20250208225212_epbs-payload.sql index 9bc45c06..8dc13a74 100644 --- a/db/schema/sqlite/20250208225212_epbs-payload.sql +++ b/db/schema/sqlite/20250208225212_epbs-payload.sql @@ -7,9 +7,9 @@ ALTER TABLE "unfinalized_blocks" ADD "payload_ssz" BLOB NULL; ALTER TABLE "orphaned_blocks" ADD "payload_ver" int NOT NULL DEFAULT 0; ALTER TABLE "orphaned_blocks" ADD "payload_ssz" BLOB NULL; -ALTER TABLE "slots" ADD "has_payload" boolean NOT NULL DEFAULT false; +ALTER TABLE "slots" ADD "payload_status" smallint NOT NULL DEFAULT 0; -CREATE INDEX IF NOT EXISTS "slots_has_payload_idx" ON "slots" ("has_payload" ASC); +CREATE INDEX IF NOT EXISTS "slots_payload_status_idx" ON "slots" ("payload_status" ASC); ALTER TABLE "epochs" ADD "payload_count" int NOT NULL DEFAULT 0; diff --git a/db/slots.go b/db/slots.go index 8786cd5a..c842f448 100644 --- a/db/slots.go +++ b/db/slots.go @@ -19,26 +19,26 @@ func InsertSlot(slot *dbtypes.Slot, tx *sqlx.Tx) error { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, payload_status ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) ON CONFLICT (slot, root) DO UPDATE SET status = excluded.status, eth_block_extra = excluded.eth_block_extra, eth_block_extra_text = excluded.eth_block_extra_text, fork_id = excluded.fork_id, - has_payload = excluded.has_payload`, + payload_status = excluded.payload_status`, dbtypes.DBEngineSqlite: ` INSERT OR REPLACE INTO slots ( slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, payload_status ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24)`, }), slot.Slot, slot.Proposer, slot.Status, slot.Root, slot.ParentRoot, slot.StateRoot, slot.Graffiti, slot.GraffitiText, slot.AttestationCount, slot.DepositCount, slot.ExitCount, slot.WithdrawCount, slot.WithdrawAmount, slot.AttesterSlashingCount, slot.ProposerSlashingCount, slot.BLSChangeCount, slot.EthTransactionCount, slot.EthBlockNumber, slot.EthBlockHash, - slot.EthBlockExtra, slot.EthBlockExtraText, slot.SyncParticipation, slot.ForkId, slot.HasPayload) + slot.EthBlockExtra, slot.EthBlockExtraText, slot.SyncParticipation, slot.ForkId, slot.PayloadStatus) if err != nil { return err } @@ -93,7 +93,7 @@ func GetSlotsRange(firstSlot uint64, lastSlot uint64, withMissing bool, withOrph "state_root", "root", "slot", "proposer", "status", "parent_root", "graffiti", "graffiti_text", "attestation_count", "deposit_count", "exit_count", "withdraw_count", "withdraw_amount", "attester_slashing_count", "proposer_slashing_count", "bls_change_count", "eth_transaction_count", "eth_block_number", "eth_block_hash", - "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "has_payload", + "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "payload_status", } for _, blockField := range blockFields { fmt.Fprintf(&sql, ", slots.%v AS \"block.%v\"", blockField, blockField) @@ -125,7 +125,7 @@ func GetSlotsByParentRoot(parentRoot []byte) []*dbtypes.Slot { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, payload_status FROM slots WHERE parent_root = $1 ORDER BY slot DESC @@ -144,7 +144,7 @@ func GetSlotByRoot(root []byte) *dbtypes.Slot { root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, payload_status FROM slots WHERE root = $1 `, root) @@ -170,7 +170,7 @@ func GetSlotsByRoots(roots [][]byte) map[phase0.Root]*dbtypes.Slot { root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, payload_status FROM slots WHERE root IN (%v) ORDER BY slot DESC`, @@ -213,7 +213,7 @@ func GetSlotsByBlockHash(blockHash []byte) []*dbtypes.Slot { slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id, has_payload + eth_block_extra, eth_block_extra_text, sync_participation, fork_id, payload_status FROM slots WHERE eth_block_hash = $1 ORDER BY slot DESC @@ -273,7 +273,7 @@ func GetFilteredSlots(filter *dbtypes.BlockFilter, firstSlot uint64, offset uint "state_root", "root", "slot", "proposer", "status", "parent_root", "graffiti", "graffiti_text", "attestation_count", "deposit_count", "exit_count", "withdraw_count", "withdraw_amount", "attester_slashing_count", "proposer_slashing_count", "bls_change_count", "eth_transaction_count", "eth_block_number", "eth_block_hash", - "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "has_payload", + "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", "payload_status", } for _, blockField := range blockFields { fmt.Fprintf(&sql, ", slots.%v AS \"block.%v\"", blockField, blockField) diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index a92231e7..8a6127cf 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -18,6 +18,14 @@ const ( Orphaned ) +type PayloadStatus uint8 + +const ( + PayloadStatusMissing PayloadStatus = iota + PayloadStatusCanonical + PayloadStatusOrphaned +) + type SlotHeader struct { Slot uint64 `db:"slot"` Proposer uint64 `db:"proposer"` @@ -25,30 +33,30 @@ type SlotHeader struct { } type Slot struct { - Slot uint64 `db:"slot"` - Proposer uint64 `db:"proposer"` - Status SlotStatus `db:"status"` - Root []byte `db:"root"` - ParentRoot []byte `db:"parent_root"` - StateRoot []byte `db:"state_root"` - Graffiti []byte `db:"graffiti"` - GraffitiText string `db:"graffiti_text"` - AttestationCount uint64 `db:"attestation_count"` - DepositCount uint64 `db:"deposit_count"` - ExitCount uint64 `db:"exit_count"` - WithdrawCount uint64 `db:"withdraw_count"` - WithdrawAmount uint64 `db:"withdraw_amount"` - AttesterSlashingCount uint64 `db:"attester_slashing_count"` - ProposerSlashingCount uint64 `db:"proposer_slashing_count"` - BLSChangeCount uint64 `db:"bls_change_count"` - EthTransactionCount uint64 `db:"eth_transaction_count"` - EthBlockNumber *uint64 `db:"eth_block_number"` - EthBlockHash []byte `db:"eth_block_hash"` - EthBlockExtra []byte `db:"eth_block_extra"` - EthBlockExtraText string `db:"eth_block_extra_text"` - SyncParticipation float32 `db:"sync_participation"` - ForkId uint64 `db:"fork_id"` - HasPayload bool `db:"has_payload"` + Slot uint64 `db:"slot"` + Proposer uint64 `db:"proposer"` + Status SlotStatus `db:"status"` + Root []byte `db:"root"` + ParentRoot []byte `db:"parent_root"` + StateRoot []byte `db:"state_root"` + Graffiti []byte `db:"graffiti"` + GraffitiText string `db:"graffiti_text"` + AttestationCount uint64 `db:"attestation_count"` + DepositCount uint64 `db:"deposit_count"` + ExitCount uint64 `db:"exit_count"` + WithdrawCount uint64 `db:"withdraw_count"` + WithdrawAmount uint64 `db:"withdraw_amount"` + AttesterSlashingCount uint64 `db:"attester_slashing_count"` + ProposerSlashingCount uint64 `db:"proposer_slashing_count"` + BLSChangeCount uint64 `db:"bls_change_count"` + EthTransactionCount uint64 `db:"eth_transaction_count"` + EthBlockNumber *uint64 `db:"eth_block_number"` + EthBlockHash []byte `db:"eth_block_hash"` + EthBlockExtra []byte `db:"eth_block_extra"` + EthBlockExtraText string `db:"eth_block_extra_text"` + SyncParticipation float32 `db:"sync_participation"` + ForkId uint64 `db:"fork_id"` + PayloadStatus PayloadStatus `db:"payload_status"` } type Epoch struct { diff --git a/handlers/epoch.go b/handlers/epoch.go index b1c6b28c..0df58eff 100644 --- a/handlers/epoch.go +++ b/handlers/epoch.go @@ -167,13 +167,18 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) { pageData.MissedCount++ } + payloadStatus := dbSlot.PayloadStatus + if !chainState.IsEip7732Enabled(phase0.Epoch(epoch)) { + payloadStatus = dbtypes.PayloadStatusCanonical + } + slotData := &models.EpochPageDataSlot{ Slot: slot, Epoch: uint64(chainState.EpochOfSlot(phase0.Slot(slot))), Ts: chainState.SlotToTime(phase0.Slot(slot)), Scheduled: slot >= uint64(currentSlot) && dbSlot.Status == dbtypes.Missing, Status: uint8(dbSlot.Status), - NoPayload: !dbSlot.HasPayload && chainState.IsEip7732Enabled(phase0.Epoch(epoch)), + PayloadStatus: uint8(payloadStatus), Proposer: dbSlot.Proposer, ProposerName: services.GlobalBeaconService.GetValidatorName(dbSlot.Proposer), AttestationCount: dbSlot.AttestationCount, diff --git a/handlers/index.go b/handlers/index.go index 27c07e9d..8c02e4e0 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -286,15 +286,21 @@ func buildIndexPageRecentBlocksData(pageData *models.IndexPageData, recentBlockC } epoch := chainState.EpochOfSlot(phase0.Slot(blockData.Slot)) + + payloadStatus := blockData.PayloadStatus + if !chainState.IsEip7732Enabled(epoch) { + payloadStatus = dbtypes.PayloadStatusCanonical + } + blockModel := &models.IndexPageDataBlocks{ - Epoch: uint64(epoch), - Slot: blockData.Slot, - Ts: chainState.SlotToTime(phase0.Slot(blockData.Slot)), - Proposer: blockData.Proposer, - ProposerName: services.GlobalBeaconService.GetValidatorName(blockData.Proposer), - Status: uint64(blockData.Status), - NoPayload: !blockData.HasPayload && chainState.IsEip7732Enabled(epoch), - BlockRoot: blockData.Root, + Epoch: uint64(epoch), + Slot: blockData.Slot, + Ts: chainState.SlotToTime(phase0.Slot(blockData.Slot)), + Proposer: blockData.Proposer, + ProposerName: services.GlobalBeaconService.GetValidatorName(blockData.Proposer), + Status: uint64(blockData.Status), + PayloadStatus: uint8(payloadStatus), + BlockRoot: blockData.Root, } if blockData.EthBlockNumber != nil { blockModel.WithEthBlock = true @@ -333,17 +339,23 @@ func buildIndexPageRecentSlotsData(pageData *models.IndexPageData, firstSlot pha dbIdx++ epoch := chainState.EpochOfSlot(phase0.Slot(dbSlot.Slot)) + + payloadStatus := dbSlot.PayloadStatus + if !chainState.IsEip7732Enabled(phase0.Epoch(epoch)) { + payloadStatus = dbtypes.PayloadStatusCanonical + } + slotData := &models.IndexPageDataSlots{ - Slot: slot, - Epoch: uint64(epoch), - Ts: chainState.SlotToTime(phase0.Slot(slot)), - Status: uint64(dbSlot.Status), - NoPayload: !dbSlot.HasPayload && chainState.IsEip7732Enabled(epoch), - Proposer: dbSlot.Proposer, - ProposerName: services.GlobalBeaconService.GetValidatorName(dbSlot.Proposer), - BlockRoot: dbSlot.Root, - ParentRoot: dbSlot.ParentRoot, - ForkGraph: make([]*models.IndexPageDataForkGraph, 0), + Slot: slot, + Epoch: uint64(epoch), + Ts: chainState.SlotToTime(phase0.Slot(slot)), + Status: uint64(dbSlot.Status), + PayloadStatus: uint8(payloadStatus), + Proposer: dbSlot.Proposer, + ProposerName: services.GlobalBeaconService.GetValidatorName(dbSlot.Proposer), + BlockRoot: dbSlot.Root, + ParentRoot: dbSlot.ParentRoot, + ForkGraph: make([]*models.IndexPageDataForkGraph, 0), } pageData.RecentSlots = append(pageData.RecentSlots, slotData) blockCount++ diff --git a/handlers/slots.go b/handlers/slots.go index 67bb151d..330d2bff 100644 --- a/handlers/slots.go +++ b/handlers/slots.go @@ -143,13 +143,18 @@ func buildSlotsPageData(firstSlot uint64, pageSize uint64) (*models.SlotsPageDat dbIdx++ epoch := chainState.EpochOfSlot(phase0.Slot(slot)) + payloadStatus := dbSlot.PayloadStatus + if !chainState.IsEip7732Enabled(phase0.Epoch(epoch)) { + payloadStatus = dbtypes.PayloadStatusCanonical + } + slotData := &models.SlotsPageDataSlot{ Slot: slot, Epoch: uint64(epoch), Ts: chainState.SlotToTime(phase0.Slot(slot)), Finalized: finalized, Status: uint8(dbSlot.Status), - NoPayload: !dbSlot.HasPayload && chainState.IsEip7732Enabled(epoch), + PayloadStatus: uint8(payloadStatus), Scheduled: slot >= uint64(currentSlot) && dbSlot.Status == dbtypes.Missing, Synchronized: dbSlot.SyncParticipation != -1, Proposer: dbSlot.Proposer, diff --git a/handlers/slots_filtered.go b/handlers/slots_filtered.go index fd17ba99..3acdaa76 100644 --- a/handlers/slots_filtered.go +++ b/handlers/slots_filtered.go @@ -266,7 +266,12 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string slotData.WithEthBlock = true slotData.EthBlockNumber = *dbBlock.Block.EthBlockNumber } - slotData.NoPayload = !dbBlock.Block.HasPayload && chainState.IsEip7732Enabled(epoch) + + payloadStatus := dbBlock.Block.PayloadStatus + if !chainState.IsEip7732Enabled(epoch) { + payloadStatus = dbtypes.PayloadStatusCanonical + } + slotData.PayloadStatus = uint8(payloadStatus) } pageData.Slots = append(pageData.Slots, slotData) } diff --git a/handlers/validator_slots.go b/handlers/validator_slots.go index dad03dc4..c5f70b60 100644 --- a/handlers/validator_slots.go +++ b/handlers/validator_slots.go @@ -140,7 +140,12 @@ func buildValidatorSlotsPageData(validator uint64, pageIdx uint64, pageSize uint slotData.WithEthBlock = true slotData.EthBlockNumber = *dbBlock.EthBlockNumber } - slotData.NoPayload = !dbBlock.HasPayload && chainState.IsEip7732Enabled(epoch) + + payloadStatus := dbBlock.PayloadStatus + if !chainState.IsEip7732Enabled(epoch) { + payloadStatus = dbtypes.PayloadStatusCanonical + } + slotData.PayloadStatus = uint8(payloadStatus) } pageData.Slots = append(pageData.Slots, slotData) } diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 79614be4..d1c7905c 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -250,7 +250,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override var executionTransactions []bellatrix.Transaction var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest - var hasPayload bool + var payloadStatus dbtypes.PayloadStatus if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { blockPayload := block.GetExecutionPayload() @@ -260,9 +260,12 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override executionTransactions = blockPayload.Message.Payload.Transactions executionWithdrawals = blockPayload.Message.Payload.Withdrawals depositRequests = blockPayload.Message.ExecutionRequests.Deposits - hasPayload = true + payloadStatus = dbtypes.PayloadStatusCanonical + } else { + payloadStatus = dbtypes.PayloadStatusMissing } } else { + payloadStatus = dbtypes.PayloadStatusCanonical executionBlockNumber, _ = blockBody.ExecutionBlockNumber() executionExtraData, _ = getBlockExecutionExtraData(blockBody) @@ -290,7 +293,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override AttesterSlashingCount: uint64(len(attesterSlashings)), ProposerSlashingCount: uint64(len(proposerSlashings)), BLSChangeCount: uint64(len(blsToExecChanges)), - HasPayload: hasPayload, + PayloadStatus: payloadStatus, } if overrideForkId != nil { diff --git a/static/css/layout.css b/static/css/layout.css index c10a4331..ddb8442a 100644 --- a/static/css/layout.css +++ b/static/css/layout.css @@ -332,3 +332,13 @@ span.validator-label { rgba(255,193,7,1) 100% ); } + +.badge.split-info { + background: linear-gradient( + 90deg, + rgba(255,255,255,0) 0%, + rgba(255,255,255,0) 50%, + rgba(13,202,240,1) 50%, + rgba(13,202,240,1) 100% + ); +} diff --git a/templates/epoch/epoch.html b/templates/epoch/epoch.html index c400c2bf..d625d170 100644 --- a/templates/epoch/epoch.html +++ b/templates/epoch/epoch.html @@ -176,15 +176,15 @@

{{ else if $slot.Scheduled }} Scheduled {{ else if eq $slot.Status 1 }} - Proposed + Proposed {{ else if eq $slot.Status 2 }} - Orphaned + Orphaned {{ else if not $epoch.Synchronized }} ? {{ else if eq $slot.Status 0 }} Missed {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/templates/index/recentBlocks.html b/templates/index/recentBlocks.html index 51b639fc..27a2950d 100644 --- a/templates/index/recentBlocks.html +++ b/templates/index/recentBlocks.html @@ -41,9 +41,9 @@

Genesis Missed - Proposed - Missed (Orphaned) - Unknown + Proposed + Missed (Orphaned) + Unknown @@ -74,11 +74,11 @@
Missed {{ else if eq .Status 1 }} - Proposed + Proposed {{ else if eq .Status 2 }} - Missed (Orphaned) + Missed (Orphaned) {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $block.Ts }} diff --git a/templates/index/recentSlots.html b/templates/index/recentSlots.html index bc348763..7a47f7cd 100644 --- a/templates/index/recentSlots.html +++ b/templates/index/recentSlots.html @@ -42,9 +42,9 @@
Genesis Missed - Proposed - Missed (Orphaned) - Unknown + Proposed + Missed (Orphaned) + Unknown @@ -97,11 +97,11 @@
Missed {{ else if eq .Status 1 }} - Proposed + Proposed {{ else if eq .Status 2 }} - Missed (Orphaned) + Missed (Orphaned) {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/templates/slots/slots.html b/templates/slots/slots.html index 2a4a6d14..b075653f 100644 --- a/templates/slots/slots.html +++ b/templates/slots/slots.html @@ -92,9 +92,9 @@

Slots

{{ if eq $slot.Slot 0 }} Genesis {{ else if eq $slot.Status 1 }} - Proposed + Proposed {{ else if eq $slot.Status 2 }} - Missed (Orphaned) + Missed (Orphaned) {{ else if $slot.Scheduled }} Scheduled {{ else if not $slot.Synchronized }} @@ -102,7 +102,7 @@

Slots

{{ else if eq $slot.Status 0 }} Missed {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/templates/slots_filtered/slots_filtered.html b/templates/slots_filtered/slots_filtered.html index 8400972e..87574174 100644 --- a/templates/slots_filtered/slots_filtered.html +++ b/templates/slots_filtered/slots_filtered.html @@ -183,9 +183,9 @@

Filtered Slots

{{- if eq $slot.Slot 0 }} Genesis {{- else if eq $slot.Status 1 }} - Proposed + Proposed {{- else if eq $slot.Status 2 }} - Missed (Orphaned) + Missed (Orphaned) {{- else if $slot.Scheduled }} Scheduled {{- else if not $slot.Synchronized }} @@ -193,7 +193,7 @@

Filtered Slots

{{- else if eq $slot.Status 0 }} Missed {{- else }} - Unknown + Unknown {{- end }} {{- end }} diff --git a/templates/validator_slots/slots.html b/templates/validator_slots/slots.html index a89f09d8..26aa83f0 100644 --- a/templates/validator_slots/slots.html +++ b/templates/validator_slots/slots.html @@ -72,15 +72,15 @@

Validator {{ format {{ if eq $slot.Slot 0 }} Genesis {{ else if eq $slot.Status 1 }} - Proposed + Proposed {{ else if eq $slot.Status 2 }} - Missed (Orphaned) + Missed (Orphaned) {{ else if $slot.Scheduled }} Scheduled {{ else if eq $slot.Status 0 }} Missed {{ else }} - Unknown + Unknown {{ end }} {{ formatRecentTimeShort $slot.Ts }} diff --git a/types/models/epoch.go b/types/models/epoch.go index c15be304..5c97fa21 100644 --- a/types/models/epoch.go +++ b/types/models/epoch.go @@ -44,7 +44,7 @@ type EpochPageDataSlot struct { Ts time.Time `json:"ts"` Scheduled bool `json:"scheduled"` Status uint8 `json:"status"` - NoPayload bool `json:"no_payload"` + PayloadStatus uint8 `json:"payload_status"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` AttestationCount uint64 `json:"attestation_count"` diff --git a/types/models/indexPage.go b/types/models/indexPage.go index af5d5c4d..d262f223 100644 --- a/types/models/indexPage.go +++ b/types/models/indexPage.go @@ -56,31 +56,31 @@ type IndexPageDataEpochs struct { } type IndexPageDataBlocks struct { - Epoch uint64 `json:"epoch"` - Slot uint64 `json:"slot"` - WithEthBlock bool `json:"has_block"` - EthBlock uint64 `json:"eth_block"` - EthBlockLink string `json:"eth_link"` - Ts time.Time `json:"ts"` - Proposer uint64 `json:"proposer"` - ProposerName string `json:"proposer_name"` - Status uint64 `json:"status"` - NoPayload bool `json:"no_payload"` - BlockRoot []byte `json:"block_root"` + Epoch uint64 `json:"epoch"` + Slot uint64 `json:"slot"` + WithEthBlock bool `json:"has_block"` + EthBlock uint64 `json:"eth_block"` + EthBlockLink string `json:"eth_link"` + Ts time.Time `json:"ts"` + Proposer uint64 `json:"proposer"` + ProposerName string `json:"proposer_name"` + Status uint64 `json:"status"` + PayloadStatus uint8 `json:"payload_status"` + BlockRoot []byte `json:"block_root"` } type IndexPageDataSlots struct { - Epoch uint64 `json:"epoch"` - Slot uint64 `json:"slot"` - EthBlock uint64 `json:"eth_block"` - Ts time.Time `json:"ts"` - Proposer uint64 `json:"proposer"` - ProposerName string `json:"proposer_name"` - Status uint64 `json:"status"` - NoPayload bool `json:"no_payload"` - BlockRoot []byte `json:"block_root"` - ParentRoot []byte `json:"-"` - ForkGraph []*IndexPageDataForkGraph `json:"fork_graph"` + Epoch uint64 `json:"epoch"` + Slot uint64 `json:"slot"` + EthBlock uint64 `json:"eth_block"` + Ts time.Time `json:"ts"` + Proposer uint64 `json:"proposer"` + ProposerName string `json:"proposer_name"` + Status uint64 `json:"status"` + PayloadStatus uint8 `json:"payload_status"` + BlockRoot []byte `json:"block_root"` + ParentRoot []byte `json:"-"` + ForkGraph []*IndexPageDataForkGraph `json:"fork_graph"` } type IndexPageDataForkGraph struct { diff --git a/types/models/slots.go b/types/models/slots.go index a3fcf9d4..651e91f2 100644 --- a/types/models/slots.go +++ b/types/models/slots.go @@ -31,7 +31,7 @@ type SlotsPageDataSlot struct { Finalized bool `json:"scheduled"` Scheduled bool `json:"finalized"` Status uint8 `json:"status"` - NoPayload bool `json:"no_payload"` + PayloadStatus uint8 `json:"payload_status"` Synchronized bool `json:"synchronized"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` diff --git a/types/models/slots_filtered.go b/types/models/slots_filtered.go index 38fd10d4..483d3211 100644 --- a/types/models/slots_filtered.go +++ b/types/models/slots_filtered.go @@ -56,7 +56,7 @@ type SlotsFilteredPageDataSlot struct { Finalized bool `json:"scheduled"` Scheduled bool `json:"finalized"` Status uint8 `json:"status"` - NoPayload bool `json:"no_payload"` + PayloadStatus uint8 `json:"payload_status"` Synchronized bool `json:"synchronized"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` diff --git a/types/models/validator_slots.go b/types/models/validator_slots.go index 0f4ade49..6d61408d 100644 --- a/types/models/validator_slots.go +++ b/types/models/validator_slots.go @@ -34,7 +34,7 @@ type ValidatorSlotsPageDataSlot struct { Finalized bool `json:"scheduled"` Scheduled bool `json:"finalized"` Status uint8 `json:"status"` - NoPayload bool `json:"no_payload"` + PayloadStatus uint8 `json:"payload_status"` Proposer uint64 `json:"proposer"` ProposerName string `json:"proposer_name"` AttestationCount uint64 `json:"attestation_count"` From 241771c0ba5594cf560a6e7916e48b92d156887e Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 17 Feb 2025 23:06:00 +0100 Subject: [PATCH 15/21] fix missing getters for eip7732 states --- indexer/beacon/block_helper.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 65649441..211d4309 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -454,6 +454,12 @@ func getStatePendingWithdrawals(v *spec.VersionedBeaconState) ([]*electra.Pendin } return v.Electra.PendingPartialWithdrawals, nil + case spec.DataVersionEIP7732: + if v.EIP7732 == nil || v.EIP7732.PendingPartialWithdrawals == nil { + return nil, errors.New("no eip7732 block") + } + + return v.EIP7732.PendingPartialWithdrawals, nil default: return nil, errors.New("unknown version") } @@ -478,6 +484,12 @@ func getStatePendingConsolidations(v *spec.VersionedBeaconState) ([]*electra.Pen } return v.Electra.PendingConsolidations, nil + case spec.DataVersionEIP7732: + if v.EIP7732 == nil || v.EIP7732.PendingConsolidations == nil { + return nil, errors.New("no eip7732 block") + } + + return v.EIP7732.PendingConsolidations, nil default: return nil, errors.New("unknown version") } From 0380f1366782dfce938d919ead8a6b2950f80f24 Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 7 Apr 2025 22:30:36 +0200 Subject: [PATCH 16/21] save execution payload to block store --- blockdb/blockdb.go | 16 +++++---- blockdb/pebble/pebble.go | 62 ++++++++++++++++++++++++++++++-- blockdb/s3/s3store.go | 40 ++++++++++++++++----- blockdb/types/engine.go | 15 ++++---- cmd/dora-utils/blockdb_sync.go | 26 +++++++++++--- indexer/beacon/block.go | 6 ++-- indexer/beacon/block_helper.go | 8 ++--- indexer/beacon/client.go | 2 +- indexer/beacon/indexer.go | 2 +- indexer/beacon/indexer_getter.go | 2 +- services/chainservice_blocks.go | 4 +++ 11 files changed, 145 insertions(+), 38 deletions(-) diff --git a/blockdb/blockdb.go b/blockdb/blockdb.go index 6ef50d23..2e13cb6c 100644 --- a/blockdb/blockdb.go +++ b/blockdb/blockdb.go @@ -45,17 +45,19 @@ func (db *BlockDb) Close() error { return db.engine.Close() } -func (db *BlockDb) GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { - return db.engine.GetBlock(ctx, slot, root, parseBlock) +func (db *BlockDb) GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error), parsePayload func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { + return db.engine.GetBlock(ctx, slot, root, parseBlock, parsePayload) } -func (db *BlockDb) AddBlock(ctx context.Context, slot uint64, root []byte, header_ver uint64, header_data []byte, body_ver uint64, body_data []byte) (bool, error) { +func (db *BlockDb) AddBlock(ctx context.Context, slot uint64, root []byte, header_ver uint64, header_data []byte, body_ver uint64, body_data []byte, payload_ver uint64, payload_data []byte) (bool, error) { return db.engine.AddBlock(ctx, slot, root, func() (*types.BlockData, error) { return &types.BlockData{ - HeaderVersion: header_ver, - HeaderData: header_data, - BodyVersion: body_ver, - BodyData: body_data, + HeaderVersion: header_ver, + HeaderData: header_data, + BodyVersion: body_ver, + BodyData: body_data, + PayloadVersion: payload_ver, + PayloadData: payload_data, }, nil }) } diff --git a/blockdb/pebble/pebble.go b/blockdb/pebble/pebble.go index d8619201..7bea79c2 100644 --- a/blockdb/pebble/pebble.go +++ b/blockdb/pebble/pebble.go @@ -14,8 +14,9 @@ const ( ) const ( - BlockTypeHeader uint16 = 1 - BlockTypeBody uint16 = 2 + BlockTypeHeader uint16 = 1 + BlockTypeBody uint16 = 2 + BlockTypePayload uint16 = 3 ) type PebbleEngine struct { @@ -97,7 +98,34 @@ func (e *PebbleEngine) getBlockBody(root []byte, parser func(uint64, []byte) (in return body, version, nil } -func (e *PebbleEngine) GetBlock(_ context.Context, _ uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { +func (e *PebbleEngine) getBlockPayload(root []byte, parser func(uint64, []byte) (interface{}, error)) (interface{}, uint64, error) { + key := make([]byte, 2+len(root)+2) + binary.BigEndian.PutUint16(key[:2], KeyNamespaceBlock) + copy(key[2:], root) + binary.BigEndian.PutUint16(key[2+len(root):], BlockTypePayload) + + res, closer, err := e.db.Get(key) + if err != nil && err != pebble.ErrNotFound { + return nil, 0, err + } + defer closer.Close() + + if err == pebble.ErrNotFound || len(res) == 0 { + return nil, 0, nil + } + + version := binary.BigEndian.Uint64(res[:8]) + block := res[8:] + + body, err := parser(version, block) + if err != nil { + return nil, 0, err + } + + return body, version, nil +} + +func (e *PebbleEngine) GetBlock(_ context.Context, _ uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error), parsePayload func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { header, header_ver, err := e.getBlockHeader(root) if err != nil { return nil, err @@ -124,6 +152,14 @@ func (e *PebbleEngine) GetBlock(_ context.Context, _ uint64, root []byte, parseB blockData.Body = body blockData.BodyVersion = body_ver + payload, payload_ver, err := e.getBlockPayload(root, parsePayload) + if err != nil { + return nil, err + } + + blockData.Payload = payload + blockData.PayloadVersion = payload_ver + return blockData, nil } @@ -157,6 +193,19 @@ func (e *PebbleEngine) addBlockBody(root []byte, version uint64, block []byte) e return e.db.Set(key, data, nil) } +func (e *PebbleEngine) addBlockPayload(root []byte, version uint64, payload []byte) error { + key := make([]byte, 2+len(root)+2) + binary.BigEndian.PutUint16(key[:2], KeyNamespaceBlock) + copy(key[2:], root) + binary.BigEndian.PutUint16(key[2+len(root):], BlockTypePayload) + + data := make([]byte, 8+len(payload)) + binary.BigEndian.PutUint64(data[:8], version) + copy(data[8:], payload) + + return e.db.Set(key, data, nil) +} + func (e *PebbleEngine) AddBlock(_ context.Context, _ uint64, root []byte, dataCb func() (*types.BlockData, error)) (bool, error) { key := make([]byte, 2+len(root)+2) binary.BigEndian.PutUint16(key[:2], KeyNamespaceBlock) @@ -182,5 +231,12 @@ func (e *PebbleEngine) AddBlock(_ context.Context, _ uint64, root []byte, dataCb return false, err } + if blockData.PayloadVersion != 0 { + err = e.addBlockPayload(root, blockData.PayloadVersion, blockData.PayloadData) + if err != nil { + return false, err + } + } + return true, nil } diff --git a/blockdb/s3/s3store.go b/blockdb/s3/s3store.go index 1110b4a6..e3aec098 100644 --- a/blockdb/s3/s3store.go +++ b/blockdb/s3/s3store.go @@ -60,10 +60,12 @@ func (e *S3Engine) getObjectKey(root []byte, slot uint64) string { } type objectMetadata struct { - objVersion uint32 - headerLength uint32 - bodyVersion uint32 - bodyLength uint32 + objVersion uint32 + headerLength uint32 + bodyVersion uint32 + bodyLength uint32 + payloadVersion uint32 + payloadLength uint32 } func (e *S3Engine) readObjectMetadata(data []byte) (*objectMetadata, int, error) { @@ -78,6 +80,13 @@ func (e *S3Engine) readObjectMetadata(data []byte) (*objectMetadata, int, error) metadata.bodyVersion = binary.BigEndian.Uint32(data[8:12]) metadata.bodyLength = binary.BigEndian.Uint32(data[12:16]) metadataLength += 12 + case 2: + metadata.headerLength = binary.BigEndian.Uint32(data[4:8]) + metadata.bodyVersion = binary.BigEndian.Uint32(data[8:12]) + metadata.bodyLength = binary.BigEndian.Uint32(data[12:16]) + metadata.payloadVersion = binary.BigEndian.Uint32(data[16:20]) + metadata.payloadLength = binary.BigEndian.Uint32(data[20:24]) + metadataLength += 20 } return metadata, metadataLength, nil @@ -92,12 +101,18 @@ func (e *S3Engine) writeObjectMetadata(metadata *objectMetadata) []byte { data = binary.BigEndian.AppendUint32(data, metadata.headerLength) data = binary.BigEndian.AppendUint32(data, metadata.bodyVersion) data = binary.BigEndian.AppendUint32(data, metadata.bodyLength) + case 2: + data = binary.BigEndian.AppendUint32(data, metadata.headerLength) + data = binary.BigEndian.AppendUint32(data, metadata.bodyVersion) + data = binary.BigEndian.AppendUint32(data, metadata.bodyLength) + data = binary.BigEndian.AppendUint32(data, metadata.payloadVersion) + data = binary.BigEndian.AppendUint32(data, metadata.payloadLength) } return data } -func (e *S3Engine) GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { +func (e *S3Engine) GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error), parsePayload func(uint64, []byte) (interface{}, error)) (*types.BlockData, error) { key := e.getObjectKey(root, slot) obj, err := e.client.GetObject(ctx, e.bucket, key, minio.GetObjectOptions{}) @@ -184,20 +199,29 @@ func (e *S3Engine) AddBlock(ctx context.Context, slot uint64, root []byte, dataC } metadata := &objectMetadata{ - objVersion: uint32(blockData.HeaderVersion), + objVersion: 1, headerLength: uint32(len(blockData.HeaderData)), bodyVersion: uint32(blockData.BodyVersion), bodyLength: uint32(len(blockData.BodyData)), } + if blockData.PayloadVersion != 0 { + metadata.objVersion = 2 + metadata.payloadVersion = uint32(blockData.PayloadVersion) + metadata.payloadLength = uint32(len(blockData.PayloadData)) + } + metadataBytes := e.writeObjectMetadata(metadata) metadataLength := len(metadataBytes) // Prepare data with header and body versions and lengths - data := make([]byte, metadataLength+int(metadata.headerLength)+int(metadata.bodyLength)) + data := make([]byte, metadataLength+int(metadata.headerLength)+int(metadata.bodyLength)+int(metadata.payloadLength)) copy(data[:metadataLength], metadataBytes) copy(data[metadataLength:metadataLength+int(metadata.headerLength)], blockData.HeaderData) - copy(data[metadataLength+int(metadata.headerLength):], blockData.BodyData) + copy(data[metadataLength+int(metadata.headerLength):metadataLength+int(metadata.headerLength)+int(metadata.bodyLength)], blockData.BodyData) + if metadata.objVersion == 2 { + copy(data[metadataLength+int(metadata.headerLength)+int(metadata.bodyLength):metadataLength+int(metadata.headerLength)+int(metadata.bodyLength)+int(metadata.payloadLength)], blockData.PayloadData) + } // Upload object _, err = e.client.PutObject( diff --git a/blockdb/types/engine.go b/blockdb/types/engine.go index 8152b501..33d3c7fc 100644 --- a/blockdb/types/engine.go +++ b/blockdb/types/engine.go @@ -3,14 +3,17 @@ package types import "context" type BlockData struct { - HeaderVersion uint64 - HeaderData []byte - BodyVersion uint64 - BodyData []byte - Body interface{} + HeaderVersion uint64 + HeaderData []byte + BodyVersion uint64 + BodyData []byte + Body interface{} + PayloadVersion uint64 + PayloadData []byte + Payload interface{} } type BlockDbEngine interface { Close() error - GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error)) (*BlockData, error) + GetBlock(ctx context.Context, slot uint64, root []byte, parseBlock func(uint64, []byte) (interface{}, error), parsePayload func(uint64, []byte) (interface{}, error)) (*BlockData, error) AddBlock(ctx context.Context, slot uint64, root []byte, dataCb func() (*BlockData, error)) (bool, error) } diff --git a/cmd/dora-utils/blockdb_sync.go b/cmd/dora-utils/blockdb_sync.go index 6dbd9c42..93664702 100644 --- a/cmd/dora-utils/blockdb_sync.go +++ b/cmd/dora-utils/blockdb_sync.go @@ -272,11 +272,29 @@ func processSlot(ctx context.Context, pool *consensus.Pool, dynSsz *dynssz.DynSs return nil, fmt.Errorf("failed to marshal block body for slot %d: %v", slot, err) } + var payloadVersion uint64 + var payloadBytes []byte + + chainState := pool.GetChainState() + if chainState.IsEip7732Enabled(chainState.EpochOfSlot(phase0.Slot(slot))) { + blockPayload, err := client.GetRPCClient().GetExecutionPayloadByBlockroot(ctx, blockHeader.Root) + if err != nil { + return nil, fmt.Errorf("failed to get block execution payload for slot %d: %v", slot, err) + } + + payloadVersion, payloadBytes, err = beacon.MarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz, blockPayload, true) + if err != nil { + return nil, fmt.Errorf("failed to marshal block execution payload for slot %d: %v", slot, err) + } + } + return &btypes.BlockData{ - HeaderVersion: 1, - HeaderData: headerBytes, - BodyVersion: version, - BodyData: bodyBytes, + HeaderVersion: 1, + HeaderData: headerBytes, + BodyVersion: version, + BodyData: bodyBytes, + PayloadVersion: payloadVersion, + PayloadData: payloadBytes, }, nil }) if err != nil { diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index 4c5b3682..d28a1579 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -190,7 +190,7 @@ func (block *Block) GetExecutionPayload() *eip7732.SignedExecutionPayloadEnvelop if block.hasExecutionPayload && block.isInUnfinalizedDb { dbBlock := db.GetUnfinalizedBlock(block.Root[:], false, false, true) if dbBlock != nil { - payload, err := unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) + payload, err := UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) if err == nil { return payload } @@ -461,7 +461,7 @@ func (block *Block) buildOrphanedBlock(compress bool) (*dbtypes.OrphanedBlock, e } if block.executionPayload != nil { - payloadVer, payloadSSZ, err := marshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, compress) + payloadVer, payloadSSZ, err := MarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, compress) if err != nil { return nil, fmt.Errorf("marshal execution payload ssz failed: %v", err) } @@ -512,7 +512,7 @@ func (block *Block) unpruneBlockBody() { if dbBlock != nil { block.block, _ = UnmarshalVersionedSignedBeaconBlockSSZ(block.dynSsz, dbBlock.BlockVer, dbBlock.BlockSSZ) if len(dbBlock.PayloadSSZ) > 0 { - block.executionPayload, _ = unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) + block.executionPayload, _ = UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) } } } diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index da4b6c74..9679256b 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -208,8 +208,8 @@ func unmarshalVersionedSignedBeaconBlockJson(version uint64, ssz []byte) (*spec. return block, nil } -// marshalVersionedSignedExecutionPayloadEnvelopeSSZ marshals a signed execution payload envelope using SSZ encoding. -func marshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, payload *eip7732.SignedExecutionPayloadEnvelope, compress bool) (version uint64, ssz []byte, err error) { +// MarshalVersionedSignedExecutionPayloadEnvelopeSSZ marshals a signed execution payload envelope using SSZ encoding. +func MarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, payload *eip7732.SignedExecutionPayloadEnvelope, compress bool) (version uint64, ssz []byte, err error) { if utils.Config.KillSwitch.DisableSSZEncoding { // SSZ encoding disabled, use json instead version, ssz, err = marshalVersionedSignedExecutionPayloadEnvelopeJson(payload) @@ -227,8 +227,8 @@ func marshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, pa return } -// unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ unmarshals a versioned signed execution payload envelope using SSZ encoding. -func unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, version uint64, ssz []byte) (*eip7732.SignedExecutionPayloadEnvelope, error) { +// UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ unmarshals a versioned signed execution payload envelope using SSZ encoding. +func UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, version uint64, ssz []byte) (*eip7732.SignedExecutionPayloadEnvelope, error) { if (version & compressionFlag) != 0 { // decompress if d, err := decompressBytes(ssz); err != nil { diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index 4f25f5ab..b41dd446 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -593,7 +593,7 @@ func (c *Client) processExecutionPayloadEvent(executionPayloadEvent *v1.Executio } func (c *Client) persistExecutionPayload(block *Block) error { - payloadVer, payloadSSZ, err := marshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, c.indexer.blockCompression) + payloadVer, payloadSSZ, err := MarshalVersionedSignedExecutionPayloadEnvelopeSSZ(block.dynSsz, block.executionPayload, c.indexer.blockCompression) if err != nil { return fmt.Errorf("marshal execution payload ssz failed: %v", err) } diff --git a/indexer/beacon/indexer.go b/indexer/beacon/indexer.go index 5d551379..623a93bf 100644 --- a/indexer/beacon/indexer.go +++ b/indexer/beacon/indexer.go @@ -361,7 +361,7 @@ func (indexer *Indexer) StartIndexer() { } if len(dbBlock.PayloadSSZ) > 0 { - blockPayload, err := unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(indexer.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) + blockPayload, err := UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(indexer.dynSsz, dbBlock.PayloadVer, dbBlock.PayloadSSZ) if err != nil { indexer.logger.Warnf("could not restore unfinalized block payload %v [%x] from db: %v", dbBlock.Slot, dbBlock.Root, err) } else if block.processingStatus == 0 { diff --git a/indexer/beacon/indexer_getter.go b/indexer/beacon/indexer_getter.go index 80b5f12e..7c80dd84 100644 --- a/indexer/beacon/indexer_getter.go +++ b/indexer/beacon/indexer_getter.go @@ -201,7 +201,7 @@ func (indexer *Indexer) GetOrphanedBlockByRoot(blockRoot phase0.Root) (*Block, e block.SetBlock(blockBody) if len(orphanedBlock.PayloadSSZ) > 0 { - payload, err := unmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(indexer.dynSsz, orphanedBlock.PayloadVer, orphanedBlock.PayloadSSZ) + payload, err := UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(indexer.dynSsz, orphanedBlock.PayloadVer, orphanedBlock.PayloadSSZ) if err != nil { return nil, fmt.Errorf("could not restore orphaned block payload %v [%x] from db: %v", header.Message.Slot, orphanedBlock.Root, err) } diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index b49deec5..a3f8391e 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -177,6 +177,8 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot if result == nil && header != nil && blockdb.GlobalBlockDb != nil { blockData, err := blockdb.GlobalBlockDb.GetBlock(ctx, uint64(header.Message.Slot), blockroot[:], func(version uint64, block []byte) (interface{}, error) { return beacon.UnmarshalVersionedSignedBeaconBlockSSZ(bs.beaconIndexer.GetDynSSZ(), version, block) + }, func(version uint64, payload []byte) (interface{}, error) { + return beacon.UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(bs.beaconIndexer.GetDynSSZ(), version, payload) }) if err == nil && blockData != nil { result = &CombinedBlockResponse{ @@ -309,6 +311,8 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl if result == nil && header != nil && blockdb.GlobalBlockDb != nil { blockData, err := blockdb.GlobalBlockDb.GetBlock(ctx, uint64(slot), blockRoot[:], func(version uint64, block []byte) (interface{}, error) { return beacon.UnmarshalVersionedSignedBeaconBlockSSZ(bs.beaconIndexer.GetDynSSZ(), version, block) + }, func(version uint64, payload []byte) (interface{}, error) { + return beacon.UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(bs.beaconIndexer.GetDynSSZ(), version, payload) }) if err == nil && blockData != nil { header := &phase0.SignedBeaconBlockHeader{} From 499ef5aca49863608c6437992798880a72bfe90f Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 15 Jul 2025 16:33:44 +0200 Subject: [PATCH 17/21] fix merge issue with blob kzg blob kzg commitments --- indexer/beacon/writedb.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 90170e55..fccbaba4 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -7,6 +7,7 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/bellatrix" "github.com/attestantio/go-eth2-client/spec/capella" + "github.com/attestantio/go-eth2-client/spec/deneb" "github.com/attestantio/go-eth2-client/spec/electra" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/clients/consensus" @@ -248,13 +249,13 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override blsToExecChanges, _ := blockBody.BLSToExecutionChanges() syncAggregate, _ := blockBody.SyncAggregate() executionBlockHash, _ := blockBody.ExecutionBlockHash() - blobKzgCommitments, _ := blockBody.BlobKZGCommitments() var executionBlockNumber uint64 var executionExtraData []byte var executionTransactions []bellatrix.Transaction var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest + var blobKzgCommitments []deneb.KZGCommitment var payloadStatus dbtypes.PayloadStatus if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { @@ -265,6 +266,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override executionTransactions = blockPayload.Message.Payload.Transactions executionWithdrawals = blockPayload.Message.Payload.Withdrawals depositRequests = blockPayload.Message.ExecutionRequests.Deposits + blobKzgCommitments = blockPayload.Message.BlobKZGCommitments payloadStatus = dbtypes.PayloadStatusCanonical } else { payloadStatus = dbtypes.PayloadStatusMissing @@ -276,6 +278,7 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override executionExtraData, _ = getBlockExecutionExtraData(blockBody) executionTransactions, _ = blockBody.ExecutionTransactions() executionWithdrawals, _ = blockBody.Withdrawals() + blobKzgCommitments, _ = blockBody.BlobKZGCommitments() executionRequests, _ := blockBody.ExecutionRequests() if executionRequests != nil { depositRequests = executionRequests.Deposits @@ -478,11 +481,11 @@ func (dbw *dbWriter) buildDbEpoch(epoch phase0.Epoch, blocks []*Block, epochStat proposerSlashings, _ := blockBody.ProposerSlashings() blsToExecChanges, _ := blockBody.BLSToExecutionChanges() syncAggregate, _ := blockBody.SyncAggregate() - blobKzgCommitments, _ := blockBody.BlobKZGCommitments() var executionTransactions []bellatrix.Transaction var executionWithdrawals []*capella.Withdrawal var depositRequests []*electra.DepositRequest + var blobKzgCommitments []deneb.KZGCommitment if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { blockPayload := block.GetExecutionPayload() @@ -491,10 +494,12 @@ func (dbw *dbWriter) buildDbEpoch(epoch phase0.Epoch, blocks []*Block, epochStat executionTransactions = blockPayload.Message.Payload.Transactions executionWithdrawals = blockPayload.Message.Payload.Withdrawals depositRequests = blockPayload.Message.ExecutionRequests.Deposits + blobKzgCommitments = blockPayload.Message.BlobKZGCommitments } } else { executionTransactions, _ = blockBody.ExecutionTransactions() executionWithdrawals, _ = blockBody.Withdrawals() + blobKzgCommitments, _ = blockBody.BlobKZGCommitments() executionRequests, _ := blockBody.ExecutionRequests() if executionRequests != nil { depositRequests = executionRequests.Deposits From eef1a3dafb44cba8a9637dddf681dabf2e307357 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 15 Jul 2025 16:57:04 +0200 Subject: [PATCH 18/21] add missing block helpers for eip7732 --- indexer/beacon/block_helper.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 234ec54e..763a67a9 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -454,6 +454,12 @@ func getStateDepositBalanceToConsume(v *spec.VersionedBeaconState) (phase0.Gwei, } return v.Electra.DepositBalanceToConsume, nil + case spec.DataVersionEIP7732: + if v.EIP7732 == nil { + return 0, errors.New("no eip7732 block") + } + + return v.EIP7732.DepositBalanceToConsume, nil default: return 0, errors.New("unknown version") } @@ -478,6 +484,12 @@ func getStatePendingDeposits(v *spec.VersionedBeaconState) ([]*electra.PendingDe } return v.Electra.PendingDeposits, nil + case spec.DataVersionEIP7732: + if v.EIP7732 == nil || v.EIP7732.PendingDeposits == nil { + return nil, errors.New("no eip7732 block") + } + + return v.EIP7732.PendingDeposits, nil default: return nil, errors.New("unknown version") } @@ -558,6 +570,8 @@ func getBlockSize(dynSsz *dynssz.DynSsz, block *spec.VersionedSignedBeaconBlock) return dynSsz.SizeSSZ(block.Deneb) case spec.DataVersionElectra: return dynSsz.SizeSSZ(block.Electra) + case spec.DataVersionEIP7732: + return dynSsz.SizeSSZ(block.EIP7732) default: return 0, errors.New("unknown version") } From 60104583b81f7e048785adcacc6f8a67f2d2bfba Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 22 Sep 2025 16:32:26 +0200 Subject: [PATCH 19/21] add gloas fork badge and internal getters --- clients/consensus/chainspec.go | 7 +++- clients/consensus/chainstate.go | 28 ++++++++++++++ handlers/api/network_forks_v1.go | 4 +- handlers/index.go | 13 +++++++ indexer/beacon/block_helper.go | 63 ++++++++++++++++++++++++++++++++ indexer/beacon/writedb.go | 16 ++++++++ 6 files changed, 128 insertions(+), 3 deletions(-) diff --git a/clients/consensus/chainspec.go b/clients/consensus/chainspec.go index 22fc057b..04f9b7c9 100644 --- a/clients/consensus/chainspec.go +++ b/clients/consensus/chainspec.go @@ -40,8 +40,10 @@ type ChainSpec struct { DenebForkEpoch *uint64 `yaml:"DENEB_FORK_EPOCH"` ElectraForkVersion phase0.Version `yaml:"ELECTRA_FORK_VERSION" check-if-fork:"ElectraForkEpoch"` ElectraForkEpoch *uint64 `yaml:"ELECTRA_FORK_EPOCH" check-if-fork:"ElectraForkEpoch"` - FuluForkVersion phase0.Version `yaml:"FULU_FORK_VERSION" check-if-fork:"FuluForkEpoch"` - FuluForkEpoch *uint64 `yaml:"FULU_FORK_EPOCH"` + FuluForkVersion phase0.Version `yaml:"FULU_FORK_VERSION" check-if-fork:"FuluForkEpoch"` + FuluForkEpoch *uint64 `yaml:"FULU_FORK_EPOCH" check-if-fork:"FuluForkEpoch"` + GloasForkVersion phase0.Version `yaml:"GLOAS_FORK_VERSION" check-if-fork:"GloasForkEpoch"` + GloasForkEpoch *uint64 `yaml:"GLOAS_FORK_EPOCH" check-if-fork:"GloasForkEpoch"` SecondsPerSlot time.Duration `yaml:"SECONDS_PER_SLOT"` SlotsPerEpoch uint64 `yaml:"SLOTS_PER_EPOCH"` EpochsPerHistoricalVector uint64 `yaml:"EPOCHS_PER_HISTORICAL_VECTOR"` @@ -65,6 +67,7 @@ type ChainSpec struct { MinEpochsForDataColumnSidecars uint64 `yaml:"MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS"` DepositChainId uint64 `yaml:"DEPOSIT_CHAIN_ID"` MinActivationBalance uint64 `yaml:"MIN_ACTIVATION_BALANCE"` + MaxBlobsPerBlock uint64 `yaml:"MAX_BLOBS_PER_BLOCK" check-if-fork:"DenebForkEpoch"` MaxPendingPartialsPerWithdrawalsSweep uint64 `yaml:"MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP" check-if-fork:"ElectraForkEpoch"` MaxPendingDepositsPerEpoch uint64 `yaml:"MAX_PENDING_DEPOSITS_PER_EPOCH" check-if-fork:"ElectraForkEpoch"` PendingPartialWithdrawalsLimit uint64 `yaml:"PENDING_PARTIAL_WITHDRAWALS_LIMIT" check-if-fork:"ElectraForkEpoch"` diff --git a/clients/consensus/chainstate.go b/clients/consensus/chainstate.go index f4945e16..a9d1457f 100644 --- a/clients/consensus/chainstate.go +++ b/clients/consensus/chainstate.go @@ -282,6 +282,34 @@ func (cs *ChainState) GetForkDigestForEpoch(epoch phase0.Epoch) phase0.ForkDiges return cs.GetForkDigest(currentForkVersion, currentBlobParams) } +func (cs *ChainState) GetBlobScheduleForEpoch(epoch phase0.Epoch) *BlobScheduleEntry { + if cs.specs == nil { + return nil + } + + var blobSchedule *BlobScheduleEntry + + if cs.specs.ElectraForkEpoch != nil && epoch >= phase0.Epoch(*cs.specs.ElectraForkEpoch) { + blobSchedule = &BlobScheduleEntry{ + Epoch: *cs.specs.ElectraForkEpoch, + MaxBlobsPerBlock: cs.specs.MaxBlobsPerBlockElectra, + } + } else if cs.specs.DenebForkEpoch != nil && epoch >= phase0.Epoch(*cs.specs.DenebForkEpoch) { + blobSchedule = &BlobScheduleEntry{ + Epoch: *cs.specs.DenebForkEpoch, + MaxBlobsPerBlock: cs.specs.MaxBlobsPerBlock, + } + } + + for i, blobScheduleEntry := range cs.specs.BlobSchedule { + if blobScheduleEntry.Epoch <= uint64(epoch) { + blobSchedule = &cs.specs.BlobSchedule[i] + } + } + + return blobSchedule +} + func (cs *ChainState) GetForkDigest(forkVersion phase0.Version, blobParams *BlobScheduleEntry) phase0.ForkDigest { if cs.specs == nil || cs.genesis == nil { return phase0.ForkDigest{} diff --git a/handlers/api/network_forks_v1.go b/handlers/api/network_forks_v1.go index a3391be8..21493d59 100644 --- a/handlers/api/network_forks_v1.go +++ b/handlers/api/network_forks_v1.go @@ -111,7 +111,8 @@ func buildNetworkForks(chainState *consensus.ChainState) []*APINetworkForkInfo { // Helper function to add consensus fork addConsensusFork := func(name string, forkEpoch *uint64, forkVersion phase0.Version) { if forkEpoch != nil && *forkEpoch < uint64(18446744073709551615) { - forkDigest := chainState.GetForkDigest(forkVersion, nil) + blobParams := chainState.GetBlobScheduleForEpoch(phase0.Epoch(*forkEpoch)) + forkDigest := chainState.GetForkDigest(forkVersion, blobParams) version := fmt.Sprintf("0x%x", forkVersion) epoch := *forkEpoch forks = append(forks, &APINetworkForkInfo{ @@ -134,6 +135,7 @@ func buildNetworkForks(chainState *consensus.ChainState) []*APINetworkForkInfo { addConsensusFork("Deneb", specs.DenebForkEpoch, specs.DenebForkVersion) addConsensusFork("Electra", specs.ElectraForkEpoch, specs.ElectraForkVersion) addConsensusFork("Fulu", specs.FuluForkEpoch, specs.FuluForkVersion) + addConsensusFork("Gloas", specs.GloasForkEpoch, specs.GloasForkVersion) // Add BPO forks from BLOB_SCHEDULE for i, blobSchedule := range specs.BlobSchedule { diff --git a/handlers/index.go b/handlers/index.go index 9c39294e..055d5f44 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -289,6 +289,19 @@ func buildIndexPageData() (*models.IndexPageData, time.Duration) { ForkDigest: forkDigest[:], }) } + if specs.GloasForkEpoch != nil && *specs.GloasForkEpoch < uint64(18446744073709551615) { + blobParams := chainState.GetBlobScheduleForEpoch(phase0.Epoch(*specs.GloasForkEpoch)) + forkDigest := chainState.GetForkDigest(specs.GloasForkVersion, blobParams) + pageData.NetworkForks = append(pageData.NetworkForks, &models.IndexPageDataForks{ + Name: "Gloas", + Epoch: *specs.GloasForkEpoch, + Version: specs.GloasForkVersion[:], + Time: uint64(chainState.EpochToTime(phase0.Epoch(*specs.GloasForkEpoch)).Unix()), + Active: uint64(currentEpoch) >= *specs.GloasForkEpoch, + Type: "consensus", + ForkDigest: forkDigest[:], + }) + } // Add BPO forks from BLOB_SCHEDULE for i, blobSchedule := range specs.BlobSchedule { diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index c943ede1..8fa7069a 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -10,6 +10,7 @@ import ( "github.com/attestantio/go-eth2-client/spec/capella" "github.com/attestantio/go-eth2-client/spec/deneb" "github.com/attestantio/go-eth2-client/spec/electra" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/utils" dynssz "github.com/pk910/dynamic-ssz" @@ -47,6 +48,9 @@ func MarshalVersionedSignedBeaconBlockSSZ(dynSsz *dynssz.DynSsz, block *spec.Ver case spec.DataVersionFulu: version = uint64(block.Version) ssz, err = dynSsz.MarshalSSZ(block.Fulu) + case spec.DataVersionGloas: + version = uint64(block.Version) + ssz, err = dynSsz.MarshalSSZ(block.Gloas) default: err = fmt.Errorf("unknown block version") } @@ -118,6 +122,11 @@ func UnmarshalVersionedSignedBeaconBlockSSZ(dynSsz *dynssz.DynSsz, version uint6 if err := dynSsz.UnmarshalSSZ(block.Fulu, ssz); err != nil { return nil, fmt.Errorf("failed to decode fulu signed beacon block: %v", err) } + case spec.DataVersionGloas: + block.Gloas = &gloas.SignedBeaconBlock{} + if err := dynSsz.UnmarshalSSZ(block.Gloas, ssz); err != nil { + return nil, fmt.Errorf("failed to decode gloas signed beacon block: %v", err) + } default: return nil, fmt.Errorf("unknown block version") } @@ -148,6 +157,9 @@ func MarshalVersionedSignedBeaconBlockJson(block *spec.VersionedSignedBeaconBloc case spec.DataVersionFulu: version = uint64(block.Version) jsonRes, err = block.Fulu.MarshalJSON() + case spec.DataVersionGloas: + version = uint64(block.Version) + jsonRes, err = block.Gloas.MarshalJSON() default: err = fmt.Errorf("unknown block version") } @@ -201,6 +213,11 @@ func unmarshalVersionedSignedBeaconBlockJson(version uint64, ssz []byte) (*spec. if err := block.Fulu.UnmarshalJSON(ssz); err != nil { return nil, fmt.Errorf("failed to decode fulu signed beacon block: %v", err) } + case spec.DataVersionGloas: + block.Gloas = &gloas.SignedBeaconBlock{} + if err := block.Gloas.UnmarshalJSON(ssz); err != nil { + return nil, fmt.Errorf("failed to decode gloas signed beacon block: %v", err) + } default: return nil, fmt.Errorf("unknown block version") } @@ -252,6 +269,12 @@ func getStateRandaoMixes(v *spec.VersionedBeaconState) ([]phase0.Root, error) { } return v.Fulu.RANDAOMixes, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.RANDAOMixes == nil { + return nil, errors.New("no gloas block") + } + + return v.Gloas.RANDAOMixes, nil default: return nil, errors.New("unknown version") } @@ -274,6 +297,8 @@ func getStateDepositIndex(state *spec.VersionedBeaconState) uint64 { return state.Electra.ETH1DepositIndex case spec.DataVersionFulu: return state.Fulu.ETH1DepositIndex + case spec.DataVersionGloas: + return state.Gloas.ETH1DepositIndex } return 0 } @@ -319,6 +344,12 @@ func getStateCurrentSyncCommittee(v *spec.VersionedBeaconState) ([]phase0.BLSPub } return v.Fulu.CurrentSyncCommittee.Pubkeys, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.CurrentSyncCommittee == nil { + return nil, errors.New("no gloas block") + } + + return v.Gloas.CurrentSyncCommittee.Pubkeys, nil default: return nil, errors.New("unknown version") } @@ -349,6 +380,12 @@ func getStateDepositBalanceToConsume(v *spec.VersionedBeaconState) (phase0.Gwei, } return v.Fulu.DepositBalanceToConsume, nil + case spec.DataVersionGloas: + if v.Gloas == nil { + return 0, errors.New("no gloas block") + } + + return v.Gloas.DepositBalanceToConsume, nil default: return 0, errors.New("unknown version") } @@ -379,6 +416,12 @@ func getStatePendingDeposits(v *spec.VersionedBeaconState) ([]*electra.PendingDe } return v.Fulu.PendingDeposits, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.PendingDeposits == nil { + return nil, errors.New("no gloas block") + } + + return v.Gloas.PendingDeposits, nil default: return nil, errors.New("unknown version") } @@ -409,6 +452,12 @@ func getStatePendingWithdrawals(v *spec.VersionedBeaconState) ([]*electra.Pendin } return v.Fulu.PendingPartialWithdrawals, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.PendingPartialWithdrawals == nil { + return nil, errors.New("no gloas block") + } + + return v.Gloas.PendingPartialWithdrawals, nil default: return nil, errors.New("unknown version") } @@ -439,6 +488,12 @@ func getStatePendingConsolidations(v *spec.VersionedBeaconState) ([]*electra.Pen } return v.Fulu.PendingConsolidations, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.PendingConsolidations == nil { + return nil, errors.New("no gloas block") + } + + return v.Gloas.PendingConsolidations, nil default: return nil, errors.New("unknown version") } @@ -465,6 +520,12 @@ func getStateProposerLookahead(v *spec.VersionedBeaconState) ([]phase0.Validator } return v.Fulu.ProposerLookahead, nil + case spec.DataVersionGloas: + if v.Gloas == nil || v.Gloas.ProposerLookahead == nil { + return nil, errors.New("no gloas block") + } + + return v.Gloas.ProposerLookahead, nil default: return nil, errors.New("unknown version") } @@ -487,6 +548,8 @@ func getBlockSize(dynSsz *dynssz.DynSsz, block *spec.VersionedSignedBeaconBlock) return dynSsz.SizeSSZ(block.Electra) case spec.DataVersionFulu: return dynSsz.SizeSSZ(block.Fulu) + case spec.DataVersionGloas: + return dynSsz.SizeSSZ(block.Gloas) default: return 0, errors.New("unknown version") } diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index cfc9f257..ff87a142 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -400,6 +400,15 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override dbBlock.EthBaseFee = utils.GetBaseFeeAsUint64(payload.BaseFeePerGas) dbBlock.EthFeeRecipient = payload.FeeRecipient[:] } + case spec.DataVersionGloas: + if blockBody.Gloas != nil && blockBody.Gloas.Message != nil && + blockBody.Gloas.Message.Body != nil && blockBody.Gloas.Message.Body.ExecutionPayload != nil { + payload := blockBody.Gloas.Message.Body.ExecutionPayload + dbBlock.EthGasUsed = payload.GasUsed + dbBlock.EthGasLimit = payload.GasLimit + dbBlock.EthBaseFee = utils.GetBaseFeeAsUint64(payload.BaseFeePerGas) + dbBlock.EthFeeRecipient = payload.FeeRecipient[:] + } } } @@ -555,6 +564,13 @@ func (dbw *dbWriter) buildDbEpoch(epoch phase0.Epoch, blocks []*Block, epochStat dbEpoch.EthGasUsed += payload.GasUsed dbEpoch.EthGasLimit += payload.GasLimit } + case spec.DataVersionGloas: + if blockBody.Gloas != nil && blockBody.Gloas.Message != nil && + blockBody.Gloas.Message.Body != nil && blockBody.Gloas.Message.Body.ExecutionPayload != nil { + payload := blockBody.Gloas.Message.Body.ExecutionPayload + dbEpoch.EthGasUsed += payload.GasUsed + dbEpoch.EthGasLimit += payload.GasLimit + } } } } From e35dc1a11c5d58ff72b49de2c6ab811ea84412eb Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 22 Sep 2025 23:08:01 +0200 Subject: [PATCH 20/21] fix merge conflicts --- clients/consensus/chainstate.go | 2 +- clients/consensus/rpc/beaconapi.go | 4 ++-- go.sum | 10 ++++++++++ handlers/slot.go | 2 +- indexer/beacon/block.go | 10 ++++++++-- indexer/beacon/block_helper.go | 10 +++++----- indexer/beacon/client.go | 6 +++--- indexer/beacon/finalization.go | 4 ++-- indexer/beacon/requests.go | 4 ++-- indexer/beacon/writedb.go | 22 ++++++++++++---------- services/chainservice_blocks.go | 12 ++++++------ 11 files changed, 52 insertions(+), 34 deletions(-) diff --git a/clients/consensus/chainstate.go b/clients/consensus/chainstate.go index e0ef4ca6..68476bd8 100644 --- a/clients/consensus/chainstate.go +++ b/clients/consensus/chainstate.go @@ -398,7 +398,7 @@ func (cs *ChainState) IsEip7732Enabled(epoch phase0.Epoch) bool { return false } - return cs.specs.Eip7732ForkEpoch != nil && phase0.Epoch(*cs.specs.Eip7732ForkEpoch) <= epoch + return cs.specs.GloasForkEpoch != nil && phase0.Epoch(*cs.specs.GloasForkEpoch) <= epoch } func (cs *ChainState) GetBalanceChurnLimit(totalActiveBalance uint64) uint64 { diff --git a/clients/consensus/rpc/beaconapi.go b/clients/consensus/rpc/beaconapi.go index 70bbce3c..6d4f05ae 100644 --- a/clients/consensus/rpc/beaconapi.go +++ b/clients/consensus/rpc/beaconapi.go @@ -19,7 +19,7 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/capella" "github.com/attestantio/go-eth2-client/spec/deneb" - "github.com/attestantio/go-eth2-client/spec/eip7732" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/rs/zerolog" "github.com/sirupsen/logrus" @@ -407,7 +407,7 @@ func (bc *BeaconClient) GetBlockBodyByBlockroot(ctx context.Context, blockroot p return result.Data, nil } -func (bc *BeaconClient) GetExecutionPayloadByBlockroot(ctx context.Context, blockroot phase0.Root) (*eip7732.SignedExecutionPayloadEnvelope, error) { +func (bc *BeaconClient) GetExecutionPayloadByBlockroot(ctx context.Context, blockroot phase0.Root) (*gloas.SignedExecutionPayloadEnvelope, error) { provider, isProvider := bc.clientSvc.(eth2client.ExecutionPayloadProvider) if !isProvider { return nil, fmt.Errorf("get execution payload not supported") diff --git a/go.sum b/go.sum index f15da993..0cdc5b6a 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,14 @@ github.com/OffchainLabs/hashtree v0.2.1-0.20250530191054-577f0b75c7f7 h1:0r1HjEx github.com/OffchainLabs/hashtree v0.2.1-0.20250530191054-577f0b75c7f7/go.mod h1:b07+cRZs+eAR8TR57CB9TQlt5Gnl/06Xs76xt/1wq0M= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= @@ -589,6 +597,8 @@ github.com/pion/webrtc/v4 v4.0.10 h1:Hq/JLjhqLxi+NmCtE8lnRPDr8H4LcNvwg8OxVcdv56Q github.com/pion/webrtc/v4 v4.0.10/go.mod h1:ViHLVaNpiuvaH8pdiuQxuA9awuE6KVzAXx3vVWilOck= github.com/pk910/dynamic-ssz v1.0.2 h1:X3CKAz0Q9wgj+zZYoatkGOsd123F65ShNqPiArFpV7Q= github.com/pk910/dynamic-ssz v1.0.2/go.mod h1:J7iiJ98fRx1KBceJK/K4fGWV2l5ITVbrnjUgGZM4ths= +github.com/pk910/go-eth2-client v0.0.0-20250922203000-f1d41f4d014c h1:XwF7W6QFd/a2sOoMuw7VTnLh8Ts6cb2UmmuHnNh5ewM= +github.com/pk910/go-eth2-client v0.0.0-20250922203000-f1d41f4d014c/go.mod h1:fvULSL9WtNskkOB4i+Yyr6BKpNHXvmpGZj9969fCrfY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/handlers/slot.go b/handlers/slot.go index b5eb12a3..34e89bc7 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -753,7 +753,7 @@ func getSlotPageBlockData(blockData *services.CombinedBlockResponse, epochStatsV if specs.ElectraForkEpoch != nil && uint64(epoch) >= *specs.ElectraForkEpoch { var requests *electra.ExecutionRequests - if blockData.Block.Version >= spec.DataVersionEIP7732 { + if blockData.Block.Version >= spec.DataVersionGloas { if blockData.Payload != nil { requests = blockData.Payload.Message.ExecutionRequests } diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index d5818b18..e5acef25 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -397,16 +397,22 @@ func (block *Block) setBlockIndex(body *spec.VersionedSignedBeaconBlock, payload if execNumber, err := body.ExecutionBlockNumber(); err == nil { blockIndex.ExecutionNumber = uint64(execNumber) } + if transactions, err := body.ExecutionTransactions(); err == nil { + blockIndex.EthTransactionCount = uint64(len(transactions)) + } + if blobKzgCommitments, err := body.BlobKZGCommitments(); err == nil { + blockIndex.BlobCount = uint64(len(blobKzgCommitments)) + } } if payload != nil { blockIndex.ExecutionNumber = uint64(payload.Message.Payload.BlockNumber) // Calculate transaction count - executionTransactions, _ := payload.Transactions() + executionTransactions := payload.Message.Payload.Transactions blockIndex.EthTransactionCount = uint64(len(executionTransactions)) // Calculate blob count - blobKzgCommitments, _ := body.BlobKZGCommitments() + blobKzgCommitments := payload.Message.BlobKZGCommitments blockIndex.BlobCount = uint64(len(blobKzgCommitments)) } diff --git a/indexer/beacon/block_helper.go b/indexer/beacon/block_helper.go index 1fcd5363..bafec412 100644 --- a/indexer/beacon/block_helper.go +++ b/indexer/beacon/block_helper.go @@ -231,7 +231,7 @@ func MarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, pa version, ssz, err = marshalVersionedSignedExecutionPayloadEnvelopeJson(payload) } else { // SSZ encoding - version = uint64(spec.DataVersionEIP7732) + version = uint64(spec.DataVersionGloas) ssz, err = dynSsz.MarshalSSZ(payload) } @@ -260,7 +260,7 @@ func UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, return unmarshalVersionedSignedExecutionPayloadEnvelopeJson(version, ssz) } - if version != uint64(spec.DataVersionEIP7732) { + if version != uint64(spec.DataVersionGloas) { return nil, fmt.Errorf("unknown version") } @@ -275,7 +275,7 @@ func UnmarshalVersionedSignedExecutionPayloadEnvelopeSSZ(dynSsz *dynssz.DynSsz, // marshalVersionedSignedExecutionPayloadEnvelopeJson marshals a versioned signed execution payload envelope using JSON encoding. func marshalVersionedSignedExecutionPayloadEnvelopeJson(payload *gloas.SignedExecutionPayloadEnvelope) (version uint64, jsonRes []byte, err error) { - version = uint64(spec.DataVersionEIP7732) + version = uint64(spec.DataVersionGloas) jsonRes, err = payload.MarshalJSON() version |= jsonVersionFlag @@ -289,7 +289,7 @@ func unmarshalVersionedSignedExecutionPayloadEnvelopeJson(version uint64, ssz [] return nil, fmt.Errorf("no json encoding") } - if version-jsonVersionFlag != uint64(spec.DataVersionEIP7732) { + if version-jsonVersionFlag != uint64(spec.DataVersionGloas) { return nil, fmt.Errorf("unknown version") } @@ -327,7 +327,7 @@ func getBlockExecutionExtraData(v *spec.VersionedSignedBeaconBlock) ([]byte, err } return v.Electra.Message.Body.ExecutionPayload.ExtraData, nil - case spec.DataVersionEIP7732: + case spec.DataVersionGloas: return nil, nil default: return nil, errors.New("unknown version") diff --git a/indexer/beacon/client.go b/indexer/beacon/client.go index 57f4793a..a96d7732 100644 --- a/indexer/beacon/client.go +++ b/indexer/beacon/client.go @@ -10,7 +10,7 @@ import ( v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/eip7732" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethereum/go-ethereum/common" "github.com/ethpandaops/dora/clients/consensus" @@ -412,7 +412,7 @@ func (c *Client) processBlock(slot phase0.Slot, root phase0.Root, header *phase0 } if loadPayload { - newPayload, _ := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + newPayload, _ := block.EnsureExecutionPayload(func() (*gloas.SignedExecutionPayloadEnvelope, error) { t1 := time.Now() defer func() { processingTimes[0] += time.Since(t1) @@ -614,7 +614,7 @@ func (c *Client) processExecutionPayloadEvent(executionPayloadEvent *v1.Executio return nil } - newPayload, err := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + newPayload, err := block.EnsureExecutionPayload(func() (*gloas.SignedExecutionPayloadEnvelope, error) { return LoadExecutionPayload(c.getContext(), c, executionPayloadEvent.BlockRoot) }) if err != nil { diff --git a/indexer/beacon/finalization.go b/indexer/beacon/finalization.go index ce67ebed..3bcc3486 100644 --- a/indexer/beacon/finalization.go +++ b/indexer/beacon/finalization.go @@ -10,7 +10,7 @@ import ( v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/eip7732" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/blockdb" "github.com/ethpandaops/dora/db" @@ -153,7 +153,7 @@ func (indexer *Indexer) finalizeEpoch(epoch phase0.Epoch, justifiedRoot phase0.R } if chainState.IsEip7732Enabled(chainState.EpochOfSlot(block.Slot)) { - if _, err := block.EnsureExecutionPayload(func() (*eip7732.SignedExecutionPayloadEnvelope, error) { + if _, err := block.EnsureExecutionPayload(func() (*gloas.SignedExecutionPayloadEnvelope, error) { return LoadExecutionPayload(client.getContext(), client, block.Root) }); err != nil { client.logger.Warnf("failed loading finalized execution payload %v (%v): %v", block.Slot, block.Root.String(), err) diff --git a/indexer/beacon/requests.go b/indexer/beacon/requests.go index 585be13c..f8182a79 100644 --- a/indexer/beacon/requests.go +++ b/indexer/beacon/requests.go @@ -6,7 +6,7 @@ import ( "time" "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/eip7732" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" ) @@ -81,7 +81,7 @@ func LoadBeaconState(ctx context.Context, client *Client, root phase0.Root) (*sp } // LoadExecutionPayload loads the execution payload from the client. -func LoadExecutionPayload(ctx context.Context, client *Client, root phase0.Root) (*eip7732.SignedExecutionPayloadEnvelope, error) { +func LoadExecutionPayload(ctx context.Context, client *Client, root phase0.Root) (*gloas.SignedExecutionPayloadEnvelope, error) { ctx, cancel := context.WithTimeout(ctx, executionPayloadRequestTimeout) defer cancel() diff --git a/indexer/beacon/writedb.go b/indexer/beacon/writedb.go index 8c6f0e50..6da1e991 100644 --- a/indexer/beacon/writedb.go +++ b/indexer/beacon/writedb.go @@ -274,10 +274,12 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override } else { payloadStatus = dbtypes.PayloadStatusCanonical executionBlockNumber, _ = blockBody.ExecutionBlockNumber() - - executionExtraData, _ = getBlockExecutionExtraData(blockBody) - executionTransactions, _ = blockBody.ExecutionTransactions() - executionWithdrawals, _ = blockBody.Withdrawals() + executionPayload, _ := blockBody.ExecutionPayload() + if executionPayload != nil { + executionExtraData, _ = executionPayload.ExtraData() + executionTransactions, _ = executionPayload.Transactions() + executionWithdrawals, _ = executionPayload.Withdrawals() + } blobKzgCommitments, _ = blockBody.BlobKZGCommitments() executionRequests, _ := blockBody.ExecutionRequests() if executionRequests != nil { @@ -418,9 +420,9 @@ func (dbw *dbWriter) buildDbBlock(block *Block, epochStats *EpochStats, override dbBlock.EthFeeRecipient = payload.FeeRecipient[:] } case spec.DataVersionGloas: - if blockBody.Gloas != nil && blockBody.Gloas.Message != nil && - blockBody.Gloas.Message.Body != nil && blockBody.Gloas.Message.Body.ExecutionPayload != nil { - payload := blockBody.Gloas.Message.Body.ExecutionPayload + blockPayload := block.GetExecutionPayload() + if blockPayload != nil { + payload := blockPayload.Message.Payload dbBlock.EthGasUsed = payload.GasUsed dbBlock.EthGasLimit = payload.GasLimit dbBlock.EthBaseFee = utils.GetBaseFeeAsUint64(payload.BaseFeePerGas) @@ -596,9 +598,9 @@ func (dbw *dbWriter) buildDbEpoch(epoch phase0.Epoch, blocks []*Block, epochStat dbEpoch.EthGasLimit += payload.GasLimit } case spec.DataVersionGloas: - if blockBody.Gloas != nil && blockBody.Gloas.Message != nil && - blockBody.Gloas.Message.Body != nil && blockBody.Gloas.Message.Body.ExecutionPayload != nil { - payload := blockBody.Gloas.Message.Body.ExecutionPayload + blockPayload := block.GetExecutionPayload() + if blockPayload != nil { + payload := blockPayload.Message.Payload dbEpoch.EthGasUsed += payload.GasUsed dbEpoch.EthGasLimit += payload.GasLimit } diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index 6379f7ea..db5a3617 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -9,7 +9,7 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/deneb" - "github.com/attestantio/go-eth2-client/spec/eip7732" + "github.com/attestantio/go-eth2-client/spec/gloas" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/blockdb" @@ -23,7 +23,7 @@ type CombinedBlockResponse struct { Root phase0.Root Header *phase0.SignedBeaconBlockHeader Block *spec.VersionedSignedBeaconBlock - Payload *eip7732.SignedExecutionPayloadEnvelope + Payload *gloas.SignedExecutionPayloadEnvelope Orphaned bool } @@ -136,7 +136,7 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot } var block *spec.VersionedSignedBeaconBlock - var payload *eip7732.SignedExecutionPayloadEnvelope + var payload *gloas.SignedExecutionPayloadEnvelope bodyRetry := 0 for ; bodyRetry < 3; bodyRetry++ { client := clients[bodyRetry%len(clients)] @@ -151,7 +151,7 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot } } - if block.Version >= spec.DataVersionEIP7732 { + if block.Version >= spec.DataVersionGloas { payload, err = beacon.LoadExecutionPayload(ctx, client, blockroot) if payload != nil { break @@ -277,7 +277,7 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl var err error var block *spec.VersionedSignedBeaconBlock - var payload *eip7732.SignedExecutionPayloadEnvelope + var payload *gloas.SignedExecutionPayloadEnvelope bodyRetry := 0 for ; bodyRetry < 3; bodyRetry++ { client := clients[bodyRetry%len(clients)] @@ -290,7 +290,7 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl log.Warnf("Error loading block body for slot %v", slot) } - if block.Version >= spec.DataVersionEIP7732 { + if block.Version >= spec.DataVersionGloas { payload, err = beacon.LoadExecutionPayload(ctx, client, blockRoot) if payload != nil { break From b4982aef71effedbbf42d755462dff68332b3219 Mon Sep 17 00:00:00 2001 From: pk910 Date: Mon, 22 Sep 2025 23:09:32 +0200 Subject: [PATCH 21/21] trigger CI