fix(gov): bound EndBlock vote tally work

sei-cosmos/x/gov/abci.go

@@ -13,7 +13,13 @@ import (
 
 var logger = seilog.NewLogger("cosmos", "x", "gov")
 
-// EndBlocker called every block, process inflation, update validator set.
+// MaxVotesProcessedPerBlock is the governance vote-record budget shared by tallying and cleanup.
+const MaxVotesProcessedPerBlock = 1000
+
+// minTallyCleanupVotesPerBlock reserves part of the budget for completed tally archives.
+const minTallyCleanupVotesPerBlock = 100
+
+// EndBlocker expires governance proposals and advances bounded vote tally work.
 func EndBlocker(ctx sdk.Context, keeper keeper.Keeper) {
 	endBlockerStart := time.Now()
 	defer func() {
@@ -50,11 +56,18 @@ func EndBlocker(ctx sdk.Context, keeper keeper.Keeper) {
 		return false
 	})
 
+	remainingVotes := MaxVotesProcessedPerBlock
+	remainingVotes -= keeper.CleanupTallyVotes(ctx, minTallyCleanupVotesPerBlock)
+
 	// fetch active proposals whose voting periods have ended (are passed the block time)
 	keeper.IterateActiveProposalsQueue(ctx, ctx.BlockHeader().Time, func(proposal types.Proposal) bool {
 		var tagValue, logMsg string
 
-		passes, burnDeposits, tallyResults := keeper.Tally(ctx, proposal)
+		complete, processed, passes, burnDeposits, tallyResults := keeper.TallyIncremental(ctx, proposal, remainingVotes)
+		remainingVotes -= processed
+		if !complete {
+			return true
+		}
 
 		// If an expedited proposal fails, we do not want to update
 		// the deposit at this point since the proposal is converted to regular.
@@ -141,6 +154,8 @@ func EndBlocker(ctx sdk.Context, keeper keeper.Keeper) {
 				sdk.NewAttribute(types.AttributeKeyProposalResult, tagValue),
 			),
 		)
-		return false
+		return remainingVotes == 0
 	})
+
+	keeper.CleanupTallyVotes(ctx, remainingVotes)
 }

sei-cosmos/x/gov/abci_test.go

@@ -2,6 +2,7 @@ package gov_test
 
 import (
 	"context"
+	"encoding/binary"
 	"testing"
 	"time"
 
@@ -606,6 +607,129 @@ func TestEndBlockerProposalHandlerFailed(t *testing.T) {
 	gov.EndBlocker(ctx, app.GovKeeper)
 }
 
+func TestEndBlockerBoundsVoteTallyAndCleanupWork(t *testing.T) {
+	app := seiapp.Setup(t, false, false, false)
+	ctx := app.BaseApp.NewContext(false, tmproto.Header{})
+
+	cleanupProposal, err := app.GovKeeper.SubmitProposal(ctx, TestProposal)
+	require.NoError(t, err)
+	app.GovKeeper.ActivateVotingPeriod(ctx, cleanupProposal)
+	cleanupProposal, found := app.GovKeeper.GetProposal(ctx, cleanupProposal.ProposalId)
+	require.True(t, found)
+	for i := 0; i < 101; i++ {
+		addr := make(sdk.AccAddress, 20)
+		binary.BigEndian.PutUint64(addr[12:], uint64(i+1))
+		require.NoError(t, app.GovKeeper.AddVote(
+			ctx,
+			cleanupProposal.ProposalId,
+			addr,
+			types.NewNonSplitVoteOption(types.OptionYes),
+		))
+	}
+	complete, processed, _, _, _ := app.GovKeeper.TallyIncremental(ctx, cleanupProposal, 101)
+	require.True(t, complete)
+	require.Equal(t, 101, processed)
+	app.GovKeeper.RemoveFromActiveProposalQueue(ctx, cleanupProposal.ProposalId, cleanupProposal.VotingEndTime)
+	cleanupProposal.Status = types.StatusRejected
+	app.GovKeeper.SetProposal(ctx, cleanupProposal)
+
+	proposal, err := app.GovKeeper.SubmitProposal(ctx, TestProposal)
+	require.NoError(t, err)
+	app.GovKeeper.ActivateVotingPeriod(ctx, proposal)
+	proposal, found = app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+
+	for i := 0; i < gov.MaxVotesProcessedPerBlock+1; i++ {
+		addr := make(sdk.AccAddress, 20)
+		binary.BigEndian.PutUint64(addr[12:], uint64(i+1))
+		require.NoError(t, app.GovKeeper.AddVote(
+			ctx,
+			proposal.ProposalId,
+			addr,
+			types.NewNonSplitVoteOption(types.OptionYes),
+		))
+	}
+
+	ctx = ctx.WithBlockTime(proposal.VotingEndTime)
+	gov.EndBlocker(ctx, app.GovKeeper)
+
+	proposal, found = app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+	require.Equal(t, types.StatusVotingPeriod, proposal.Status)
+	require.True(t, app.GovKeeper.IsTallying(ctx, proposal.ProposalId))
+	require.Len(t, app.GovKeeper.GetVotes(ctx, proposal.ProposalId), gov.MaxVotesProcessedPerBlock+1)
+	require.Len(t, app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, false), 900)
+	require.Len(t, app.GovKeeper.GetArchivedTallyVotes(ctx, cleanupProposal.ProposalId, false), 1)
+
+	newVoter := make(sdk.AccAddress, 20)
+	binary.BigEndian.PutUint64(newVoter[12:], uint64(gov.MaxVotesProcessedPerBlock+2))
+	err = app.GovKeeper.AddVote(ctx, proposal.ProposalId, newVoter, types.NewNonSplitVoteOption(types.OptionNo))
+	require.ErrorIs(t, err, types.ErrInactiveProposal)
+
+	gov.EndBlocker(ctx, app.GovKeeper)
+
+	proposal, found = app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+	require.Equal(t, types.StatusRejected, proposal.Status)
+	require.False(t, app.GovKeeper.IsTallying(ctx, proposal.ProposalId))
+	require.Empty(t, app.GovKeeper.GetVotes(ctx, proposal.ProposalId))
+	require.Len(t, app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, false), 103)
+
+	gov.EndBlocker(ctx, app.GovKeeper)
+	require.Empty(t, app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, false))
+}
+
+func TestEndBlockerKeepsExpeditedAndRegularTallyArchivesSeparate(t *testing.T) {
+	app := seiapp.Setup(t, false, false, false)
+	ctx := app.BaseApp.NewContext(false, tmproto.Header{})
+
+	proposal, err := app.GovKeeper.SubmitProposalWithExpedite(ctx, TestExpeditedProposal, true)
+	require.NoError(t, err)
+	app.GovKeeper.ActivateVotingPeriod(ctx, proposal)
+	proposal, found := app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+
+	for i := 0; i < 2*gov.MaxVotesProcessedPerBlock+1; i++ {
+		addr := make(sdk.AccAddress, 20)
+		binary.BigEndian.PutUint64(addr[12:], uint64(i+1))
+		require.NoError(t, app.GovKeeper.AddVote(
+			ctx,
+			proposal.ProposalId,
+			addr,
+			types.NewNonSplitVoteOption(types.OptionYes),
+		))
+	}
+
+	ctx = ctx.WithBlockTime(proposal.VotingEndTime)
+	gov.EndBlocker(ctx, app.GovKeeper)
+	gov.EndBlocker(ctx, app.GovKeeper)
+	gov.EndBlocker(ctx, app.GovKeeper)
+
+	proposal, found = app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+	require.Equal(t, types.StatusVotingPeriod, proposal.Status)
+	require.False(t, proposal.IsExpedited)
+	require.Len(t, app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, true), 1002)
+
+	regularVoter := make(sdk.AccAddress, 20)
+	binary.BigEndian.PutUint64(regularVoter[12:], uint64(2*gov.MaxVotesProcessedPerBlock+2))
+	require.NoError(t, app.GovKeeper.AddVote(
+		ctx,
+		proposal.ProposalId,
+		regularVoter,
+		types.NewNonSplitVoteOption(types.OptionNo),
+	))
+
+	ctx = ctx.WithBlockTime(proposal.VotingEndTime)
+	gov.EndBlocker(ctx, app.GovKeeper)
+
+	proposal, found = app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+	require.Equal(t, types.StatusRejected, proposal.Status)
+	require.Len(t, app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, true), 3)
+	require.Len(t, app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, false), 1)
+}
+
 // With expedited proposal's minimum deposit set higher than the default deposit, we must
 // initialize and deposit an amount depositMultiplier times larger
 // than the regular min deposit amount.

sei-cosmos/x/gov/genesis.go

@@ -37,6 +37,9 @@ func InitGenesis(ctx sdk.Context, ak types.AccountKeeper, bk types.BankKeeper, k
 			k.InsertInactiveProposalQueue(ctx, proposal.ProposalId, proposal.DepositEndTime)
 		case types.StatusVotingPeriod:
 			k.InsertActiveProposalQueue(ctx, proposal.ProposalId, proposal.VotingEndTime)
+			if !proposal.VotingEndTime.After(ctx.BlockTime()) {
+				k.InitializeTally(ctx, proposal)
+			}
 		}
 		k.SetProposal(ctx, proposal)
 	}

sei-cosmos/x/gov/genesis_test.go

@@ -2,6 +2,7 @@ package gov_test
 
 import (
 	"context"
+	"encoding/binary"
 	"encoding/json"
 	"testing"
 
@@ -168,3 +169,54 @@ func TestEqualProposals(t *testing.T) {
 	require.Equal(t, state1, state2)
 	require.True(t, state1.Equal(state2))
 }
+
+func TestExportGenesisIncludesVotesFromUnfinishedTally(t *testing.T) {
+	app := seiapp.Setup(t, false, false, false)
+	ctx := app.BaseApp.NewContext(false, tmproto.Header{})
+
+	proposal, err := app.GovKeeper.SubmitProposal(ctx, TestProposal)
+	require.NoError(t, err)
+	app.GovKeeper.ActivateVotingPeriod(ctx, proposal)
+	proposal, found := app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+
+	for i := 0; i < 3; i++ {
+		addr := make(sdk.AccAddress, 20)
+		binary.BigEndian.PutUint64(addr[12:], uint64(i+1))
+		require.NoError(t, app.GovKeeper.AddVote(
+			ctx,
+			proposal.ProposalId,
+			addr,
+			types.NewNonSplitVoteOption(types.OptionYes),
+		))
+	}
+
+	complete, processed, _, _, _ := app.GovKeeper.TallyIncremental(ctx, proposal, 1)
+	require.False(t, complete)
+	require.Equal(t, 1, processed)
+
+	genesis := gov.ExportGenesis(ctx, app.GovKeeper)
+	require.Len(t, genesis.Votes, 3)
+
+	importedApp := seiapp.Setup(t, false, false, false)
+	importedCtx := importedApp.BaseApp.NewContext(false, tmproto.Header{}).WithBlockTime(proposal.VotingEndTime)
+	gov.InitGenesis(
+		importedCtx,
+		importedApp.AccountKeeper,
+		importedApp.BankKeeper,
+		importedApp.GovKeeper,
+		genesis,
+	)
+
+	require.True(t, importedApp.GovKeeper.IsTallying(importedCtx, proposal.ProposalId))
+	require.Len(t, importedApp.GovKeeper.GetVotes(importedCtx, proposal.ProposalId), 3)
+	newVoter := make(sdk.AccAddress, 20)
+	binary.BigEndian.PutUint64(newVoter[12:], 4)
+	err = importedApp.GovKeeper.AddVote(
+		importedCtx,
+		proposal.ProposalId,
+		newVoter,
+		types.NewNonSplitVoteOption(types.OptionNo),
+	)
+	require.ErrorIs(t, err, types.ErrInactiveProposal)
+}

sei-cosmos/x/gov/keeper/grpc_query.go

@@ -134,8 +134,7 @@ func (q Keeper) Votes(c context.Context, req *types.QueryVotesRequest) (*types.Q
 	var votes types.Votes
 	ctx := sdk.UnwrapSDKContext(c)
 
-	store := ctx.KVStore(q.storeKey)
-	votesStore := prefix.NewStore(store, types.VotesKey(req.ProposalId))
+	votesStore := q.visibleVotesStore(ctx, req.ProposalId)
 
 	pageRes, err := query.Paginate(ctx, votesStore, req.Pagination, func(key []byte, value []byte) error {
 		var vote types.Vote

sei-cosmos/x/gov/keeper/grpc_query_test.go

@@ -434,6 +434,72 @@ func (suite *KeeperTestSuite) TestGRPCQueryVotes() {
 	}
 }
 
+func (suite *KeeperTestSuite) TestGRPCQueryVotesDuringIncrementalTally() {
+	app, ctx, queryClient, addrs := suite.app, suite.ctx, suite.queryClient, suite.addrs
+
+	proposal, err := app.GovKeeper.SubmitProposal(ctx, TestProposal)
+	suite.Require().NoError(err)
+	proposal.Status = types.StatusVotingPeriod
+	app.GovKeeper.SetProposal(ctx, proposal)
+
+	for i, addr := range addrs {
+		option := types.OptionYes
+		if i == 1 {
+			option = types.OptionNo
+		}
+		suite.Require().NoError(app.GovKeeper.AddVote(
+			ctx,
+			proposal.ProposalId,
+			addr,
+			types.NewNonSplitVoteOption(option),
+		))
+	}
+
+	complete, processed, _, _, _ := app.GovKeeper.TallyIncremental(ctx, proposal, 1)
+	suite.Require().False(complete)
+	suite.Require().Equal(1, processed)
+	archivedVotes := app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, false)
+	suite.Require().Len(archivedVotes, 1)
+
+	voteResponse, err := queryClient.Vote(gocontext.Background(), &types.QueryVoteRequest{
+		ProposalId: proposal.ProposalId,
+		Voter:      archivedVotes[0].Voter,
+	})
+	suite.Require().NoError(err)
+	suite.Require().Equal(archivedVotes[0], voteResponse.Vote)
+
+	firstPage, err := queryClient.Votes(gocontext.Background(), &types.QueryVotesRequest{
+		ProposalId: proposal.ProposalId,
+		Pagination: &query.PageRequest{Limit: 1, CountTotal: true},
+	})
+	suite.Require().NoError(err)
+	suite.Require().Len(firstPage.Votes, 1)
+	suite.Require().Equal(uint64(2), firstPage.Pagination.Total)
+	suite.Require().NotEmpty(firstPage.Pagination.NextKey)
+
+	secondPage, err := queryClient.Votes(gocontext.Background(), &types.QueryVotesRequest{
+		ProposalId: proposal.ProposalId,
+		Pagination: &query.PageRequest{Key: firstPage.Pagination.NextKey, Limit: 1},
+	})
+	suite.Require().NoError(err)
+	suite.Require().Len(secondPage.Votes, 1)
+	suite.Require().ElementsMatch(app.GovKeeper.GetVotes(ctx, proposal.ProposalId), append(firstPage.Votes, secondPage.Votes...))
+	suite.Require().Len(app.GovKeeper.GetAllVotes(ctx), 2)
+
+	reversePage, err := queryClient.Votes(gocontext.Background(), &types.QueryVotesRequest{
+		ProposalId: proposal.ProposalId,
+		Pagination: &query.PageRequest{Limit: 2, Reverse: true},
+	})
+	suite.Require().NoError(err)
+	suite.Require().ElementsMatch(app.GovKeeper.GetVotes(ctx, proposal.ProposalId), reversePage.Votes)
+
+	_, err = queryClient.TallyResult(gocontext.Background(), &types.QueryTallyResultRequest{ProposalId: proposal.ProposalId})
+	suite.Require().NoError(err)
+	suite.Require().True(app.GovKeeper.IsTallying(ctx, proposal.ProposalId))
+	suite.Require().Len(app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, false), 1)
+	suite.Require().Len(app.GovKeeper.GetVotes(ctx, proposal.ProposalId), 2)
+}
+
 func (suite *KeeperTestSuite) TestGRPCQueryParams() {
 	queryClient := suite.queryClient