From 3f8139a3055af21f833b0097ef9f737171f38c4a Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Thu, 6 Aug 2026 12:38:23 +0200 Subject: [PATCH 1/3] Fix multinode check lease misleading log --- multinode/multi_node.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/multinode/multi_node.go b/multinode/multi_node.go index e7dc0da..b688e3b 100644 --- a/multinode/multi_node.go +++ b/multinode/multi_node.go @@ -24,8 +24,8 @@ type multiNodeMetrics interface { // MultiNode is a generalized multi node client interface that includes methods to interact with different chains. // It also handles multiple node RPC connections simultaneously. type MultiNode[ - CHAIN_ID ID, - RPC any, +CHAIN_ID ID, +RPC any, ] struct { services.Service eng *services.Engine @@ -48,16 +48,16 @@ type MultiNode[ } func NewMultiNode[ - CHAIN_ID ID, - RPC any, +CHAIN_ID ID, +RPC any, ]( lggr logger.Logger, metrics multiNodeMetrics, - selectionMode string, // type of the "best" RPC selector (e.g HighestHead, RoundRobin, etc.) + selectionMode string, // type of the "best" RPC selector (e.g HighestHead, RoundRobin, etc.) leaseDuration time.Duration, // defines interval on which new "best" RPC should be selected primaryNodes []Node[CHAIN_ID, RPC], sendOnlyNodes []SendOnlyNode[CHAIN_ID, RPC], - chainID CHAIN_ID, // configured chain ID (used to verify that passed primaryNodes belong to the same chain) + chainID CHAIN_ID, // configured chain ID (used to verify that passed primaryNodes belong to the same chain) chainFamily string, // name of the chain family - used in the metrics deathDeclarationDelay time.Duration, ) *MultiNode[CHAIN_ID, RPC] { @@ -295,7 +295,6 @@ func (c *MultiNode[CHAIN_ID, RPC]) checkLease() { // Terminate client subscriptions. Services are responsible for reconnecting, which will be routed to the new // best node. Only terminate connections with more than 1 subscription to account for the aliveLoop subscription if n.State() == nodeStateAlive && n != bestNode { - c.lggr.Infof("Switching to best node from %q to %q", n.String(), bestNode.String()) n.UnsubscribeAllExceptAliveLoop() } } @@ -304,6 +303,7 @@ func (c *MultiNode[CHAIN_ID, RPC]) checkLease() { defer c.activeMu.Unlock() if bestNode != c.activeNode { if c.activeNode != nil { + c.lggr.Infof("Switching to best node from %q to %q", c.activeNode.String(), bestNode.String()) c.activeNode.UnsubscribeAllExceptAliveLoop() } c.activeNode = bestNode From 4b01879fac7e5105dac66a6259924740c543bfe6 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Thu, 6 Aug 2026 13:07:57 +0200 Subject: [PATCH 2/3] Add dedicated test --- multinode/multi_node_test.go | 66 +++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/multinode/multi_node_test.go b/multinode/multi_node_test.go index 7993079..adb1158 100644 --- a/multinode/multi_node_test.go +++ b/multinode/multi_node_test.go @@ -4,6 +4,7 @@ import ( "fmt" "math/big" "math/rand" + "sync/atomic" "testing" "time" @@ -58,6 +59,12 @@ func newHealthyNode(t *testing.T, chainID ID) *mockNode[ID, multiNodeRPCClient] } func newNodeWithState(t *testing.T, chainID ID, state nodeState) *mockNode[ID, multiNodeRPCClient] { + node := newStatelessNode(t, chainID) + node.On("State").Return(state).Maybe() + return node +} + +func newStatelessNode(t *testing.T, chainID ID) *mockNode[ID, multiNodeRPCClient] { node := newMockNode[ID, multiNodeRPCClient](t) node.On("ConfiguredChainID").Return(chainID).Once() node.On("Start", mock.Anything).Return(nil).Once() @@ -65,10 +72,26 @@ func newNodeWithState(t *testing.T, chainID ID, state nodeState) *mockNode[ID, m // #nosec G404 node.On("String").Return(fmt.Sprintf("healthy_node_%d", rand.Int())).Maybe() node.On("SetPoolChainInfoProvider", mock.Anything).Once() - node.On("State").Return(state).Maybe() return node } +// newNodeWithOrder returns a mock node with the given priority selector order whose health +// can be toggled between alive and unreachable via the returned setAlive func. +func newNodeWithOrder(t *testing.T, chainID ID, order int32) (*mockNode[ID, multiNodeRPCClient], func(alive bool)) { + node := newStatelessNode(t, chainID) + node.On("Order").Return(order).Maybe() + node.On("UnsubscribeAllExceptAliveLoop").Maybe() + var alive atomic.Bool + alive.Store(true) + node.On("State").Return(func() nodeState { + if alive.Load() { + return nodeStateAlive + } + return nodeStateUnreachable + }).Maybe() + return node, func(v bool) { alive.Store(v) } +} + func TestMultiNode_Dial(t *testing.T) { t.Parallel() @@ -272,27 +295,38 @@ func TestMultiNode_CheckLease(t *testing.T) { t.Run("Lease check updates active node", func(t *testing.T) { t.Parallel() chainID := RandomID() - node := newHealthyNode(t, chainID) - node.On("UnsubscribeAllExceptAliveLoop") - bestNode := newHealthyNode(t, chainID) - nodeSelector := newMockNodeSelector[ID, multiNodeRPCClient](t) - nodeSelector.On("Select").Return(bestNode) - lggr, observedLogs := logger.TestObserved(t, zap.InfoLevel) + // order and priority have inverted relationship. Lower order -> higher priority + highPriorityNode, setHighPriorityNodeAlive := newNodeWithOrder(t, chainID, 1) + lowPriorityNode, _ := newNodeWithOrder(t, chainID, 2) mn := newTestMultiNode(t, multiNodeOpts{ - selectionMode: NodeSelectionModeHighestHead, + selectionMode: NodeSelectionModePriorityLevel, chainID: chainID, - logger: lggr, - nodes: []Node[ID, multiNodeRPCClient]{node, bestNode}, + nodes: []Node[ID, multiNodeRPCClient]{highPriorityNode, lowPriorityNode}, leaseDuration: tests.TestInterval, }) - mn.nodeSelector = nodeSelector servicetest.Run(t, mn) - tests.AssertLogEventually(t, observedLogs, fmt.Sprintf("Switching to best node from %q to %q", node.String(), bestNode.String())) - tests.AssertEventually(t, func() bool { + + activeNode := func() Node[ID, multiNodeRPCClient] { mn.activeMu.RLock() - active := mn.activeNode - mn.activeMu.RUnlock() - return bestNode == active + defer mn.activeMu.RUnlock() + return mn.activeNode + } + + // lower order node has higher priority, so it should become active first + tests.AssertEventually(t, func() bool { + return activeNode() == highPriorityNode + }) + + // once the lower order node becomes unhealthy, the higher order node should take over + setHighPriorityNodeAlive(false) + tests.AssertEventually(t, func() bool { + return activeNode() == lowPriorityNode + }) + + // once the lower order node is healthy again, it should become active again + setHighPriorityNodeAlive(true) + tests.AssertEventually(t, func() bool { + return activeNode() == highPriorityNode }) }) t.Run("NodeStates returns proper states", func(t *testing.T) { From e8710e18bddca3e3df3057b71be3dd51679caa97 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Thu, 6 Aug 2026 14:45:29 +0200 Subject: [PATCH 3/3] fmt --- multinode/multi_node.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/multinode/multi_node.go b/multinode/multi_node.go index b688e3b..4c02b85 100644 --- a/multinode/multi_node.go +++ b/multinode/multi_node.go @@ -24,8 +24,8 @@ type multiNodeMetrics interface { // MultiNode is a generalized multi node client interface that includes methods to interact with different chains. // It also handles multiple node RPC connections simultaneously. type MultiNode[ -CHAIN_ID ID, -RPC any, + CHAIN_ID ID, + RPC any, ] struct { services.Service eng *services.Engine @@ -48,16 +48,16 @@ RPC any, } func NewMultiNode[ -CHAIN_ID ID, -RPC any, + CHAIN_ID ID, + RPC any, ]( lggr logger.Logger, metrics multiNodeMetrics, - selectionMode string, // type of the "best" RPC selector (e.g HighestHead, RoundRobin, etc.) + selectionMode string, // type of the "best" RPC selector (e.g HighestHead, RoundRobin, etc.) leaseDuration time.Duration, // defines interval on which new "best" RPC should be selected primaryNodes []Node[CHAIN_ID, RPC], sendOnlyNodes []SendOnlyNode[CHAIN_ID, RPC], - chainID CHAIN_ID, // configured chain ID (used to verify that passed primaryNodes belong to the same chain) + chainID CHAIN_ID, // configured chain ID (used to verify that passed primaryNodes belong to the same chain) chainFamily string, // name of the chain family - used in the metrics deathDeclarationDelay time.Duration, ) *MultiNode[CHAIN_ID, RPC] {