Merge pull request #255 from canopy-network/eviction-optimization

Eviction/LoadRootChainInfo optimization

Author: rem1niscence

cmd/rpc/query.go

@@ -127,8 +127,14 @@ func (s *Server) RootChainInfo(w http.ResponseWriter, r *http.Request, _ httprou
 	if ok := unmarshal(w, r, req); !ok {
 		return
 	}
+	// if height is 0; set to the latest height
+	if req.Height == 0 {
+		req.Height = s.controller.FSM.Height()
+	}
+	// retrieve the saved last validator set if available
+	lastVS := s.controller.LastValidatorSet[req.Height][req.ID]
 	// load the root chain info directly
-	got, err := s.controller.FSM.LoadRootChainInfo(req.ID, req.Height)
+	got, err := s.controller.FSM.LoadRootChainInfo(req.ID, req.Height, lastVS)
 	if err != nil {
 		write(w, err, http.StatusBadRequest)
 		return

cmd/rpc/server.go

@@ -31,7 +31,7 @@ import (
 const (
 	colon = ":"
 
-	SoftwareVersion = "beta-0.1.11"
+	SoftwareVersion = "beta-0.1.12"
 	ContentType     = "Content-MessageType"
 	ApplicationJSON = "application/json; charset=utf-8"
 

controller/block.go

@@ -203,7 +203,7 @@ func (c *Controller) ValidateProposal(rcBuildHeight uint64, qc *lib.QuorumCertif
 		return
 	}
 	// play the block against the state machine to generate a block result
-	blockResult, err = c.ApplyAndValidateBlock(block, false)
+	blockResult, err = c.ApplyAndValidateBlock(block, c.LastValidatorSet[c.ChainHeight()][c.Config.ChainId], false)
 	if err != nil {
 		// exit with error
 		return
@@ -244,7 +244,7 @@ func (c *Controller) CommitCertificate(qc *lib.QuorumCertificate, block *lib.Blo
 		// reset the FSM to ensure stale proposal validations don't come into play
 		c.FSM.Reset()
 		// apply the block against the state machine
-		blockResult, err = c.ApplyAndValidateBlock(block, true)
+		blockResult, err = c.ApplyAndValidateBlock(block, c.LastValidatorSet[c.ChainHeight()][c.Config.ChainId], true)
 		if err != nil {
 			// exit with error
 			return
@@ -284,6 +284,8 @@ func (c *Controller) CommitCertificate(qc *lib.QuorumCertificate, block *lib.Blo
 	c.log.Infof("Committed block %s at H:%d 🔒", lib.BytesToTruncatedString(qc.BlockHash), block.BlockHeader.Height)
 	// set up the finite state machine for the next height
 	c.FSM, err = fsm.New(c.Config, storeI, c.Metrics, c.log)
+	// set the reference to lastCertificate on the new FSM
+	c.FSM.LastValidatorSet = c.LastValidatorSet
 	if err != nil {
 		// exit with error
 		return err
@@ -300,10 +302,15 @@ func (c *Controller) CommitCertificate(qc *lib.QuorumCertificate, block *lib.Blo
 	c.Mempool.FSM.Reset()
 	// update telemetry (using proper defer to ensure time.Since is evaluated at defer execution)
 	defer c.UpdateTelemetry(qc, block, time.Since(start))
-	// publish the root chain info to the nested chain subscribers
-	for _, id := range c.RCManager.ChainIds() {
+	// publish root chain information to all nested chain subscribers.
+	// Currently using hardcoded chain IDs (1, 2) instead of dynamically fetching IDs
+	// from RCManager since only these two chains exist. This will be updated to use
+	// dynamic chain discovery when additional chains are added.
+	// TODO: Optimize rcManager publishing for subchains (it should publish to themselves too by default)
+	// for _, id := range c.RCManager.ChainIds() {
+	for _, id := range []uint64{1, 2} {
 		// get the root chain info
-		info, e := c.FSM.LoadRootChainInfo(id, 0)
+		info, e := c.FSM.LoadRootChainInfo(id, 0, c.LastValidatorSet[c.ChainHeight()][id])
 		if e != nil {
 			// don't log 'no-validators' error as this is possible
 			if e.Error() != lib.ErrNoValidators().Error() {
@@ -313,9 +320,17 @@ func (c *Controller) CommitCertificate(qc *lib.QuorumCertificate, block *lib.Blo
 		}
 		// set the timestamp
 		info.Timestamp = ts
+		// save current validator set for the next height
+		valSet, _ := c.FSM.GetCommitteeMembers(id)
+		if _, found := c.LastValidatorSet[c.ChainHeight()+1]; !found {
+			c.LastValidatorSet[c.ChainHeight()+1] = make(map[uint64]*lib.ValidatorSet)
+		}
+		c.LastValidatorSet[c.ChainHeight()+1][id] = &valSet
 		// publish root chain information
 		go c.RCManager.Publish(id, info)
 	}
+	// remove older validator set heights
+	delete(c.LastValidatorSet, c.ChainHeight()-2)
 	// exit
 	return
 }
@@ -339,7 +354,7 @@ func (c *Controller) CommitCertificateParallel(qc *lib.QuorumCertificate, block
 		// reset the FSM to ensure stale proposal validations don't come into play
 		c.FSM.Reset()
 		// apply the block against the state machine
-		blockResult, err = c.ApplyAndValidateBlock(block, true)
+		blockResult, err = c.ApplyAndValidateBlock(block, c.LastValidatorSet[c.ChainHeight()][c.Config.ChainId], true)
 		if err != nil {
 			// exit with error
 			return
@@ -395,8 +410,18 @@ func (c *Controller) CommitCertificateParallel(qc *lib.QuorumCertificate, block
 		}
 		// publish the root chain info to the nested chain subscribers
 		for _, id := range c.RCManager.ChainIds() {
+			// get latest validator set
+			valSet, err := c.FSM.GetCommitteeMembers(id)
+			if err != nil {
+				return err
+			}
+			// TODO handle err
+			if _, found := c.LastValidatorSet[c.ChainHeight()+1]; !found {
+				c.LastValidatorSet[c.ChainHeight()+1] = make(map[uint64]*lib.ValidatorSet)
+			}
+			c.LastValidatorSet[c.ChainHeight()+1][id] = &valSet
 			// get the root chain info
-			info, e := c.FSM.LoadRootChainInfo(id, 0)
+			info, e := c.FSM.LoadRootChainInfo(id, 0, c.LastValidatorSet[c.ChainHeight()][id])
 			if e != nil {
 				// don't log 'no-validators' error as this is possible
 				if e.Error() != lib.ErrNoValidators().Error() {
@@ -445,7 +470,7 @@ func (c *Controller) CommitCertificateParallel(qc *lib.QuorumCertificate, block
 // INTERNAL HELPERS BELOW
 
 // ApplyAndValidateBlock() plays the block against the state machine which returns a result that is compared against the candidate block header
-func (c *Controller) ApplyAndValidateBlock(block *lib.Block, commit bool) (b *lib.BlockResult, err lib.ErrorI) {
+func (c *Controller) ApplyAndValidateBlock(block *lib.Block, lastValidatorSet *lib.ValidatorSet, commit bool) (b *lib.BlockResult, err lib.ErrorI) {
 	// define convenience variables for the block header, hash, and height
 	candidate, candidateHash, candidateHeight := block.BlockHeader, lib.BytesToString(block.BlockHeader.Hash), block.BlockHeader.Height
 	// check the last qc in the candidate and set it in the ephemeral indexer to prepare for block application
@@ -456,7 +481,7 @@ func (c *Controller) ApplyAndValidateBlock(block *lib.Block, commit bool) (b *li
 	// log the start of 'apply block'
 	c.log.Debugf("Applying block %s for height %d", candidateHash[:20], candidateHeight)
 	// apply the block against the state machine
-	compare, txResults, _, failed, err := c.FSM.ApplyBlock(context.Background(), block, false)
+	compare, txResults, _, failed, err := c.FSM.ApplyBlock(context.Background(), block, lastValidatorSet, false)
 	if err != nil {
 		// exit with error
 		return

controller/controller.go

@@ -18,11 +18,12 @@ var _ bft.Controller = new(Controller)
 
 // Controller acts as the 'manager' of the modules of the application
 type Controller struct {
-	Address    []byte             // self address
-	PublicKey  []byte             // self public key
-	PrivateKey crypto.PrivateKeyI // self private key
-	Config     lib.Config         // node configuration
-	Metrics    *lib.Metrics       // telemetry
+	Address          []byte                                  // self address
+	PublicKey        []byte                                  // self public key
+	PrivateKey       crypto.PrivateKeyI                      // self private key
+	Config           lib.Config                              // node configuration
+	Metrics          *lib.Metrics                            // telemetry
+	LastValidatorSet map[uint64]map[uint64]*lib.ValidatorSet // cache [height][chainID] -> set
 
 	FSM       *fsm.StateMachine // the core protocol component responsible for maintaining and updating the state of the blockchain
 	Mempool   *Mempool          // the in memory list of pending transactions
@@ -54,23 +55,26 @@ func New(fsm *fsm.StateMachine, c lib.Config, valKey crypto.PrivateKeyI, metrics
 	}
 	// create the controller
 	controller = &Controller{
-		Address:    address.Bytes(),
-		PublicKey:  valKey.PublicKey().Bytes(),
-		PrivateKey: valKey,
-		Config:     c,
-		Metrics:    metrics,
-		FSM:        fsm,
-		Mempool:    mempool,
-		Consensus:  nil,
-		P2P:        p2p.New(valKey, maxMembersPerCommittee, metrics, c, l),
-		isSyncing:  &atomic.Bool{},
-		log:        l,
-		Mutex:      &sync.Mutex{},
+		Address:          address.Bytes(),
+		PublicKey:        valKey.PublicKey().Bytes(),
+		PrivateKey:       valKey,
+		Config:           c,
+		Metrics:          metrics,
+		FSM:              fsm,
+		Mempool:          mempool,
+		Consensus:        nil,
+		P2P:              p2p.New(valKey, maxMembersPerCommittee, metrics, c, l),
+		isSyncing:        &atomic.Bool{},
+		LastValidatorSet: make(map[uint64]map[uint64]*lib.ValidatorSet),
+		log:              l,
+		Mutex:            &sync.Mutex{},
 	}
 	// initialize the consensus in the controller, passing a reference to itself
 	controller.Consensus, err = bft.New(c, valKey, fsm.Height(), fsm.Height()-1, controller, c.RunVDF, metrics, l)
 	// initialize the mempool controller
 	mempool.controller = controller
+	// add the last validator set reference to the FSM
+	fsm.LastValidatorSet = controller.LastValidatorSet
 	// if an error occurred initializing the bft module
 	if err != nil {
 		// exit with error
@@ -94,6 +98,21 @@ func (c *Controller) Start() {
 		c.log.Warnf("Attempting to connect to the root-chain")
 		// set a timer to go off once per second
 		t := time.NewTicker(time.Second)
+		// pre save the validator sets from previous and current heights
+		lastValSet, err := c.FSM.LoadCommittee(c.Config.ChainId, c.ChainHeight()-1)
+		if err != nil {
+			c.log.Fatal(err.Error())
+		}
+		currValSet, err := c.FSM.LoadCommittee(c.Config.ChainId, c.ChainHeight())
+		if err != nil {
+			c.log.Fatal(err.Error())
+		}
+		c.LastValidatorSet[c.ChainHeight()] = map[uint64]*lib.ValidatorSet{
+			c.Config.ChainId: &lastValSet,
+		}
+		c.LastValidatorSet[c.ChainHeight()+1] = map[uint64]*lib.ValidatorSet{
+			c.Config.ChainId: &currValSet,
+		}
 		// once function completes, stop the timer
 		defer t.Stop()
 		// each time the timer fires

controller/tx.go

@@ -3,12 +3,13 @@ package controller
 import (
 	"context"
 	"fmt"
-	"github.com/canopy-network/canopy/bft"
 	"math"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	"github.com/canopy-network/canopy/bft"
+
 	"github.com/canopy-network/canopy/fsm"
 	"github.com/canopy-network/canopy/lib"
 	"github.com/canopy-network/canopy/lib/crypto"
@@ -226,7 +227,8 @@ func (m *Mempool) CheckMempool() {
 	// set the cancel function
 	m.stop = stop
 	// apply the block against the state machine and populate the resulting merkle `roots` in the block header
-	block.BlockHeader, blockResult.Transactions, oversized, failed, err = m.FSM.ApplyBlock(ctx, block, true)
+	lastVs := m.controller.LastValidatorSet[m.FSM.Height()][m.controller.Config.ChainId]
+	block.BlockHeader, blockResult.Transactions, oversized, failed, err = m.FSM.ApplyBlock(ctx, block, lastVs, true)
 	if err != nil {
 		m.log.Warnf("Check Mempool error: %s", err.Error())
 		return

fsm/automatic.go

@@ -7,7 +7,7 @@ import (
 /* This file handles 'automatic' (non-transaction-induced) state changes that occur ath the beginning and ending of a block */
 
 // BeginBlock() is code that is executed at the start of `applying` the block
-func (s *StateMachine) BeginBlock() lib.ErrorI {
+func (s *StateMachine) BeginBlock(lastValidatorSet *lib.ValidatorSet) lib.ErrorI {
 	// prevent attempting to load the certificate for height 0
 	if s.Height() <= 1 {
 		return nil
@@ -39,11 +39,7 @@ func (s *StateMachine) BeginBlock() lib.ErrorI {
 	}
 	// if is root-chain: load the committee from state as the certificate result
 	// will match the evidence and there's no Transaction to HandleMessageCertificateResults
-	committee, err := s.LoadCommittee(s.Config.ChainId, s.Height()-1)
-	if err != nil {
-		return err
-	}
-	return s.HandleCertificateResults(lastCertificate, &committee)
+	return s.HandleCertificateResults(lastCertificate, lastValidatorSet)
 }
 
 // EndBlock() is code that is executed at the end of `applying` the block

fsm/automatic_test.go

@@ -2,11 +2,12 @@ package fsm
 
 import (
 	"fmt"
+	"math"
+	"testing"
+
 	"github.com/canopy-network/canopy/lib"
 	"github.com/canopy-network/canopy/lib/crypto"
 	"github.com/stretchr/testify/require"
-	"math"
-	"testing"
 )
 
 func TestBeginBlock(t *testing.T) {
@@ -122,8 +123,10 @@ func TestBeginBlock(t *testing.T) {
 			if test.isGenesis {
 				sm.height = 1
 			}
+			// get last validator set for begin block
 			// ensure expected error on function call
-			require.Equal(t, test.error, sm.BeginBlock())
+			valSet, _ := sm.LoadCommittee(lib.CanopyChainId, sm.Height()-1)
+			require.Equal(t, test.error, sm.BeginBlock(&valSet))
 			if test.error != nil {
 				return
 			}

fsm/message.go

@@ -2,6 +2,7 @@ package fsm
 
 import (
 	"bytes"
+
 	"github.com/canopy-network/canopy/lib"
 	"github.com/canopy-network/canopy/lib/crypto"
 )
@@ -318,14 +319,22 @@ func (s *StateMachine) HandleMessageCertificateResults(msg *MessageCertificateRe
 	if poolBalance == 0 {
 		return ErrNonSubsidizedCommittee()
 	}
-	// get committee for the QC
-	committee, err := s.LoadCommittee(chainId, msg.Qc.Header.RootHeight)
-	if err != nil {
-		return err
+	// get committee for the QC from the cache
+	var committee *lib.ValidatorSet
+	if s.LastValidatorSet != nil {
+		committee = s.LastValidatorSet[msg.Qc.Header.RootHeight+1][chainId]
+	}
+	if committee == nil {
+		// otherwise, retrieve it from the store
+		valSet, err := s.LoadCommittee(chainId, msg.Qc.Header.RootHeight)
+		if err != nil {
+			return err
+		}
+		committee = &valSet
 	}
 	// ensure it's a valid QC
 	// max block size is 0 here because there should not be a block attached to this QC
-	isPartialQC, err := msg.Qc.Check(committee, 0, &lib.View{NetworkId: uint64(s.NetworkID), ChainId: chainId}, false)
+	isPartialQC, err := msg.Qc.Check(*committee, 0, &lib.View{NetworkId: uint64(s.NetworkID), ChainId: chainId}, false)
 	if err != nil {
 		return err
 	}
@@ -334,7 +343,7 @@ func (s *StateMachine) HandleMessageCertificateResults(msg *MessageCertificateRe
 		return lib.ErrNoMaj23()
 	}
 	// handle the certificate results
-	return s.HandleCertificateResults(msg.Qc, &committee)
+	return s.HandleCertificateResults(msg.Qc, committee)
 }
 
 // HandleMessageSubsidy() is the proper handler for a `Subsidy` message