restart the service if the height stays the same for one minute (#4623)

Author: CoderZhi

chainservice/chainservice.go

@@ -7,6 +7,8 @@ package chainservice
 
 import (
 	"context"
+	"sync/atomic"
+	"time"
 
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/pkg/errors"
@@ -79,11 +81,37 @@ type ChainService struct {
 	apiStats                 *nodestats.APILocalStats
 	actionsync               *actsync.ActionSync
 	minter                   *factory.Minter
+
+	lastReceivedBlockHeight uint64
 }
 
 // Start starts the server
 func (cs *ChainService) Start(ctx context.Context) error {
-	return cs.lifecycle.OnStartSequentially(ctx)
+	if err := cs.lifecycle.OnStartSequentially(ctx); err != nil {
+		return errors.Wrap(err, "failed to start chain service")
+	}
+	go func() {
+		ticker := time.NewTicker(time.Minute)
+		defer ticker.Stop()
+
+		var lastHeight uint64
+		var lastReceivedBlockHeight uint64
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				currentHeight := cs.chain.TipHeight()
+				lrbh := atomic.LoadUint64(&cs.lastReceivedBlockHeight)
+				if currentHeight == lastHeight && lastReceivedBlockHeight != lrbh {
+					log.L().Panic("Blockchain height has not changed, restarting service")
+				}
+				lastHeight = currentHeight
+				lastReceivedBlockHeight = lrbh
+			}
+		}
+	}()
+	return nil
 }
 
 // Stop stops the server
@@ -150,6 +178,9 @@ func (cs *ChainService) HandleBlock(ctx context.Context, peer string, pbBlock *i
 	if err != nil {
 		return err
 	}
+	if atomic.LoadUint64(&cs.lastReceivedBlockHeight) < blk.Height() {
+		atomic.StoreUint64(&cs.lastReceivedBlockHeight, blk.Height())
+	}
 	return cs.blocksync.ProcessBlock(ctx, peer, blk)
 }