From e8dc19cbdd106483c5d3dfbb7cffd6f98fa328f0 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Thu, 27 Aug 2026 11:30:04 +0200 Subject: [PATCH 1/2] fix(sei-tendermint): bound P2P and consensus Prometheus label cardinality (PLT-1070, PLT-1071) Remove peer_id and validator_address labels from metrics fed by untrusted P2P input so time series count stays bounded. Co-authored-by: Cursor --- sei-tendermint/internal/consensus/metrics.gen.go | 14 +++++++------- sei-tendermint/internal/consensus/metrics.go | 11 +++++------ sei-tendermint/internal/consensus/reactor.go | 2 +- sei-tendermint/internal/consensus/state.go | 2 +- sei-tendermint/internal/p2p/metrics.gen.go | 8 ++++---- sei-tendermint/internal/p2p/metrics.go | 4 ++-- sei-tendermint/internal/p2p/transport.go | 1 - 7 files changed, 20 insertions(+), 22 deletions(-) diff --git a/sei-tendermint/internal/consensus/metrics.gen.go b/sei-tendermint/internal/consensus/metrics.gen.go index 69f41c6d5b..7c32879d64 100644 --- a/sei-tendermint/internal/consensus/metrics.gen.go +++ b/sei-tendermint/internal/consensus/metrics.gen.go @@ -179,8 +179,8 @@ func NewMetrics() *Metrics { Namespace: MetricsNamespace, Subsystem: MetricsSubsystem, Name: "block_parts", - Help: "Number of block parts transmitted by each peer.", - }, []string{"peer_id"}), + Help: "Number of block parts received from peers.", + }, nil), StepDuration: tmprometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: MetricsNamespace, Subsystem: MetricsSubsystem, @@ -267,7 +267,7 @@ func NewMetrics() *Metrics { Subsystem: MetricsSubsystem, Name: "late_votes", Help: "Number of votes received by the node since process start that correspond to earlier heights and rounds than this node is currently in.", - }, []string{"validator_address"}), + }, nil), FinalRound: tmprometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: MetricsNamespace, Subsystem: MetricsSubsystem, @@ -401,8 +401,8 @@ func (m *Metrics) StateSyncingAt() *tmprometheus.GaugeInt { return m.StateSyncing.WithLabelValues() } -func (m *Metrics) BlockPartsAt(peer_id string) *tmprometheus.CounterInt { - return m.BlockParts.WithLabelValues(peer_id) +func (m *Metrics) BlockPartsAt() *tmprometheus.CounterInt { + return m.BlockParts.WithLabelValues() } func (m *Metrics) StepDurationAt(step string) *tmprometheus.Histogram { @@ -457,8 +457,8 @@ func (m *Metrics) RoundVotingPowerPercentAt(vote_type string) prometheus.Gauge { return m.RoundVotingPowerPercent.WithLabelValues(vote_type) } -func (m *Metrics) LateVotesAt(validator_address string) *tmprometheus.CounterInt { - return m.LateVotes.WithLabelValues(validator_address) +func (m *Metrics) LateVotesAt() *tmprometheus.CounterInt { + return m.LateVotes.WithLabelValues() } func (m *Metrics) FinalRoundAt(proposer_address string) *tmprometheus.Histogram { diff --git a/sei-tendermint/internal/consensus/metrics.go b/sei-tendermint/internal/consensus/metrics.go index bfce5cc9c8..9f05f2039e 100644 --- a/sei-tendermint/internal/consensus/metrics.go +++ b/sei-tendermint/internal/consensus/metrics.go @@ -75,8 +75,8 @@ type Metrics struct { // Whether or not a node is state syncing. 1 if yes, 0 if no. StateSyncing tmprometheus.GaugeIntVec - // Number of block parts transmitted by each peer. - BlockParts tmprometheus.CounterIntVec `metrics_labels:"peer_id"` + // Number of block parts received from peers. + BlockParts tmprometheus.CounterIntVec // Histogram of durations for each step in the consensus protocol. StepDuration tmprometheus.HistogramVec `metrics_labels:"step" metrics_buckets:"exprange(0.1, 100, 8)"` @@ -147,7 +147,7 @@ type Metrics struct { // correspond to earlier heights and rounds than this node is currently // in. //metrics:Number of votes received by the node since process start that correspond to earlier heights and rounds than this node is currently in. - LateVotes tmprometheus.CounterIntVec `metrics_labels:"validator_address"` + LateVotes tmprometheus.CounterIntVec // FinalRound stores the final round id the proposal block reach consensus in. //metrics:The final round number for where the proposal block reach consensus in, starting at 0. @@ -227,9 +227,8 @@ func (m *Metrics) MarkRound(r int32, st time.Time) { m.RoundVotingPowerPercentAt(pcn).Set(0) } -func (m *Metrics) MarkLateVote(vote *types.Vote) { - validator := vote.ValidatorAddress.String() - m.LateVotesAt(validator).Add(1) +func (m *Metrics) MarkLateVote() { + m.LateVotesAt().Add(1) } func (m *Metrics) MarkFinalRound(round int32, proposer string) { diff --git a/sei-tendermint/internal/consensus/reactor.go b/sei-tendermint/internal/consensus/reactor.go index ed6dfc52e9..d53aa04b83 100644 --- a/sei-tendermint/internal/consensus/reactor.go +++ b/sei-tendermint/internal/consensus/reactor.go @@ -761,7 +761,7 @@ func (r *Reactor) handleDataMessage(ctx context.Context, m p2p.RecvMsg[*tmcons.M return nil case *BlockPartMessage: ps.SetHasProposalBlockPart(msg.Height, msg.Round, int(msg.Part.Index)) - Global.BlockPartsAt(string(m.From)).Add(1) + Global.BlockPartsAt().Add(1) return utils.Send(ctx, r.state.peerMsgQueue, msgInfo{msg, m.From, tmtime.Now()}) default: return fmt.Errorf("received unknown message on DataChannel: %T", msg) diff --git a/sei-tendermint/internal/consensus/state.go b/sei-tendermint/internal/consensus/state.go index f8183401ea..51a99cacf1 100644 --- a/sei-tendermint/internal/consensus/state.go +++ b/sei-tendermint/internal/consensus/state.go @@ -2456,7 +2456,7 @@ func (cs *State) addVote( "cs_height", cs.roundState.Height(), ) if vote.Height < cs.roundState.Height() || (vote.Height == cs.roundState.Height() && vote.Round < cs.roundState.Round()) { - Global.MarkLateVote(vote) + Global.MarkLateVote() } // A precommit for the previous height? diff --git a/sei-tendermint/internal/p2p/metrics.gen.go b/sei-tendermint/internal/p2p/metrics.gen.go index 0b4fe4b175..2a179f9b44 100644 --- a/sei-tendermint/internal/p2p/metrics.gen.go +++ b/sei-tendermint/internal/p2p/metrics.gen.go @@ -34,8 +34,8 @@ func NewMetrics() *Metrics { Namespace: MetricsNamespace, Subsystem: MetricsSubsystem, Name: "peer_receive_bytes_total", - Help: "Number of bytes per channel received from a given peer.", - }, []string{"peer_id", "chID", "message_type"}), + Help: "Number of bytes per channel received.", + }, []string{"chID", "message_type"}), newConnections: tmprometheus.NewCounterIntVec(prometheus.CounterOpts{ Namespace: MetricsNamespace, Subsystem: MetricsSubsystem, @@ -80,8 +80,8 @@ func (m *Metrics) peersAt() *tmprometheus.GaugeInt { return m.peers.WithLabelValues() } -func (m *Metrics) peerReceiveBytesTotalAt(peer_id string, chID string, message_type string) *tmprometheus.CounterInt { - return m.peerReceiveBytesTotal.WithLabelValues(peer_id, chID, message_type) +func (m *Metrics) peerReceiveBytesTotalAt(chID string, message_type string) *tmprometheus.CounterInt { + return m.peerReceiveBytesTotal.WithLabelValues(chID, message_type) } func (m *Metrics) newConnectionsAt(direction string, success string) *tmprometheus.CounterInt { diff --git a/sei-tendermint/internal/p2p/metrics.go b/sei-tendermint/internal/p2p/metrics.go index 6dab13c1de..c91678e611 100644 --- a/sei-tendermint/internal/p2p/metrics.go +++ b/sei-tendermint/internal/p2p/metrics.go @@ -30,8 +30,8 @@ var ( type Metrics struct { // Number of peers. peers prometheus.GaugeIntVec - // Number of bytes per channel received from a given peer. - peerReceiveBytesTotal prometheus.CounterIntVec `metrics_labels:"peer_id, chID, message_type"` + // Number of bytes per channel received. + peerReceiveBytesTotal prometheus.CounterIntVec `metrics_labels:"chID, message_type"` // Number of newly established connections. newConnections prometheus.CounterIntVec `metrics_labels:"direction, success"` diff --git a/sei-tendermint/internal/p2p/transport.go b/sei-tendermint/internal/p2p/transport.go index b1b59c9896..b5b9983b83 100644 --- a/sei-tendermint/internal/p2p/transport.go +++ b/sei-tendermint/internal/p2p/transport.go @@ -95,7 +95,6 @@ func (r *Router) connRecvRoutine(ctx context.Context, conn *ConnV2) error { Global.queueDroppedMsgsAt(fmt.Sprint(chID), "in").Add(1) } Global.peerReceiveBytesTotalAt( - string(conn.ID), fmt.Sprint(chID), r.lc.ValueToMetricLabel(msg), ).Add(int64(gogoproto.Size(msg))) From 4c9cf93dc3f196027337a8230c9c50de43dc042f Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Thu, 27 Aug 2026 12:05:30 +0200 Subject: [PATCH 2/2] fix(sei-tendermint): gate late_votes labels to the active validator set Keep per-validator SLO dashboard attribution while bounding cardinality by labeling current-set senders by address and rolling everyone else into other. Co-authored-by: Cursor --- sei-tendermint/internal/consensus/metrics.gen.go | 8 ++++---- sei-tendermint/internal/consensus/metrics.go | 16 +++++++++++----- sei-tendermint/internal/consensus/state.go | 2 +- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/sei-tendermint/internal/consensus/metrics.gen.go b/sei-tendermint/internal/consensus/metrics.gen.go index 7c32879d64..3a049ebcad 100644 --- a/sei-tendermint/internal/consensus/metrics.gen.go +++ b/sei-tendermint/internal/consensus/metrics.gen.go @@ -266,8 +266,8 @@ func NewMetrics() *Metrics { Namespace: MetricsNamespace, Subsystem: MetricsSubsystem, Name: "late_votes", - Help: "Number of votes received by the node since process start that correspond to earlier heights and rounds than this node is currently in.", - }, nil), + Help: "Number of late votes received by the node, labeled by validator address for the current validator set or other.", + }, []string{"validator_address"}), FinalRound: tmprometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: MetricsNamespace, Subsystem: MetricsSubsystem, @@ -457,8 +457,8 @@ func (m *Metrics) RoundVotingPowerPercentAt(vote_type string) prometheus.Gauge { return m.RoundVotingPowerPercent.WithLabelValues(vote_type) } -func (m *Metrics) LateVotesAt() *tmprometheus.CounterInt { - return m.LateVotes.WithLabelValues() +func (m *Metrics) LateVotesAt(validator_address string) *tmprometheus.CounterInt { + return m.LateVotes.WithLabelValues(validator_address) } func (m *Metrics) FinalRoundAt(proposer_address string) *tmprometheus.Histogram { diff --git a/sei-tendermint/internal/consensus/metrics.go b/sei-tendermint/internal/consensus/metrics.go index 9f05f2039e..966bf2df48 100644 --- a/sei-tendermint/internal/consensus/metrics.go +++ b/sei-tendermint/internal/consensus/metrics.go @@ -145,9 +145,9 @@ type Metrics struct { // LateVotes stores the number of votes that were received by this node that // correspond to earlier heights and rounds than this node is currently - // in. - //metrics:Number of votes received by the node since process start that correspond to earlier heights and rounds than this node is currently in. - LateVotes tmprometheus.CounterIntVec + // in, labeled by validator address for the current validator set. + //metrics:Number of late votes received by the node, labeled by validator address for the current validator set or other. + LateVotes tmprometheus.CounterIntVec `metrics_labels:"validator_address"` // FinalRound stores the final round id the proposal block reach consensus in. //metrics:The final round number for where the proposal block reach consensus in, starting at 0. @@ -227,8 +227,14 @@ func (m *Metrics) MarkRound(r int32, st time.Time) { m.RoundVotingPowerPercentAt(pcn).Set(0) } -func (m *Metrics) MarkLateVote() { - m.LateVotesAt().Add(1) +const lateVoteOtherLabel = "other" + +func (m *Metrics) MarkLateVote(addr types.Address, validators *types.ValidatorSet) { + label := lateVoteOtherLabel + if validators != nil && validators.HasAddress(addr) { + label = addr.String() + } + m.LateVotesAt(label).Add(1) } func (m *Metrics) MarkFinalRound(round int32, proposer string) { diff --git a/sei-tendermint/internal/consensus/state.go b/sei-tendermint/internal/consensus/state.go index 51a99cacf1..a5c13179d1 100644 --- a/sei-tendermint/internal/consensus/state.go +++ b/sei-tendermint/internal/consensus/state.go @@ -2456,7 +2456,7 @@ func (cs *State) addVote( "cs_height", cs.roundState.Height(), ) if vote.Height < cs.roundState.Height() || (vote.Height == cs.roundState.Height() && vote.Round < cs.roundState.Round()) { - Global.MarkLateVote() + Global.MarkLateVote(vote.ValidatorAddress, cs.roundState.Validators()) } // A precommit for the previous height?