├── .codecov.yml
├── .github
    └── workflows
    │   ├── golangci-lint.yml
    │   ├── pr.yml
    │   └── property_tests.yml
├── .gitignore
├── .golangci.yml
├── Makefile
├── README.md
├── SECURITY.md
├── consensus_cfg.go
├── consensus_pbft.go
├── consensus_test.go
├── e2e
    ├── README.md
    ├── backend.go
    ├── cluster.go
    ├── cmd
    │   ├── fuzz
    │   │   ├── action.go
    │   │   ├── command.go
    │   │   └── runner.go
    │   ├── main.go
    │   └── replay
    │   │   ├── backend.go
    │   │   └── command.go
    ├── e2e_node_drop_test.go
    ├── e2e_noissue_test.go
    ├── e2e_partition_test.go
    ├── framework_test.go
    ├── fuzz_network_churn_test.go
    ├── fuzz_unreliable_network_test.go
    ├── go.mod
    ├── go.sum
    ├── helper
    │   ├── bool_slice.go
    │   └── helper.go
    ├── node.go
    ├── notifier
    │   ├── default.go
    │   └── notifier.go
    ├── otel-jaeger-config.yaml
    ├── rapid_test.go
    ├── replay
    │   ├── msg.go
    │   ├── msg_middleware.go
    │   ├── msg_persister.go
    │   └── msg_reader.go
    ├── transport
    │   ├── gossip.go
    │   ├── helper.go
    │   ├── partition.go
    │   ├── random.go
    │   └── transport.go
    └── validator_set.go
├── go.mod
├── go.sum
├── interfaces.go
├── message.go
├── msg_queue.go
├── msg_queue_test.go
├── proposal.go
├── state.go
├── state_test.go
├── stats
    ├── stats.go
    └── stats_test.go
├── test_helpers.go
└── view.go


/.codecov.yml:
--------------------------------------------------------------------------------
 1 | coverage:
 2 |   status:
 3 |     project:
 4 |       default:
 5 |         target: auto
 6 |         threshold: 0%
 7 |         base: auto
 8 |         flags:
 9 |           - unit
10 |         if_ci_failed: error
11 |         informational: false
12 |         only_pulls: false
13 | 


--------------------------------------------------------------------------------
/.github/workflows/golangci-lint.yml:
--------------------------------------------------------------------------------
 1 | name: golangci-lint
 2 | on:
 3 |   push:
 4 |     tags:
 5 |       - v*
 6 |     branches:
 7 |       - main
 8 |   pull_request:
 9 | jobs:
10 |   golangci:
11 |     name: lint
12 |     runs-on: ubuntu-latest
13 |     steps:
14 |       - name: Setup go
15 |         uses: actions/setup-go@v3
16 |         with:
17 |           go-version: '1.18'
18 |       - uses: actions/checkout@v3
19 |       - name: Go mod
20 |         run: go mod tidy
21 |       - name: golangci-lint
22 |         uses: golangci/golangci-lint-action@v3
23 |         with:
24 |           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
25 |           version: latest
26 | 
27 |           # Optional: working directory, useful for monorepos
28 |           # working-directory: somedir
29 | 
30 |           # Optional: golangci-lint command line arguments.
31 |           # args: --issues-exit-code=0
32 | 
33 |           # Optional: show only new issues if it's a pull request. The default value is `false`.
34 |           # only-new-issues: true
35 | 
36 |           # Optional: if set to true then the all caching functionality will be complete disabled,
37 |           #           takes precedence over all other caching options.
38 |           # skip-cache: true
39 | 
40 |           # Optional: if set to true then the action don't cache or restore ~/go/pkg.
41 |           # skip-pkg-cache: true
42 | 
43 |           # Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
44 |           # skip-build-cache: true


--------------------------------------------------------------------------------
/.github/workflows/pr.yml:
--------------------------------------------------------------------------------
 1 | name: CI
 2 | on: [ pull_request ]
 3 | 
 4 | concurrency:
 5 |   group: build-${{ github.event.pull_request.number || github.ref }}
 6 |   cancel-in-progress: true
 7 | 
 8 | jobs:
 9 |   build:
10 |     runs-on: ubuntu-latest
11 |     name: Go test
12 |     env:
13 |       SILENT: true
14 |     steps:
15 |       - uses: actions/checkout@v2
16 |       - name: Setup go
17 |         uses: actions/setup-go@v1
18 |         with:
19 |           go-version: '1.18'
20 |       - name: Go test
21 |         run: make test
22 |       - name: Upload coverage to Codecov
23 |         uses: codecov/codecov-action@v2
24 |         with:
25 |           file: ./coverage.out
26 |       - name: Go e2e test
27 |         run: make unit-e2e
28 |   fuzz:
29 |     runs-on: ubuntu-latest
30 |     name: Go fuzz test
31 |     env:
32 |       SILENT: true
33 |       FUZZ: true
34 |     steps:
35 |       - uses: actions/checkout@v2
36 |       - name: Setup go
37 |         uses: actions/setup-go@v1
38 |         with:
39 |           go-version: '1.17'
40 |       - name: Go fuzz test
41 |         timeout-minutes: 20
42 |         run: make unit-fuzz
43 | 


--------------------------------------------------------------------------------
/.github/workflows/property_tests.yml:
--------------------------------------------------------------------------------
 1 | name: Property tests
 2 | on: [pull_request]
 3 | jobs:
 4 |   build:
 5 |     runs-on: ubuntu-latest
 6 |     steps:
 7 |       - uses: actions/checkout@v2
 8 |       - name: Install Go
 9 |         uses: actions/setup-go@v2
10 |         with:
11 |           go-version: 1.18
12 |       - name: Run tests
13 |         run: make property-tests
14 |       - name: Archive test logs
15 |         if: always()
16 |         uses: actions/upload-artifact@v3
17 |         with:
18 |           name: property-logs
19 |           path: property-logs-*/
20 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | .DS_Store
 2 | */**/.DS_Store
 3 | SavedState/
 4 | **/launch.json
 5 | **/*.flow
 6 | **/*.log
 7 | **/*.txt
 8 | log*/
 9 | e2e/*.fail
10 | *.fail
11 | coverage.out
12 | 


--------------------------------------------------------------------------------
/.golangci.yml:
--------------------------------------------------------------------------------
 1 | run:
 2 |   timeout: 10m
 3 | 
 4 | linters-settings:
 5 |   dupl:
 6 |     threshold: 100
 7 |   goconst:
 8 |     min-len: 3
 9 |     min-occurrences: 3
10 |   govet:
11 |     check-shadowing: true
12 |   lll:
13 |     line-length: 300
14 | 
15 |   misspell:
16 |     locale: US
17 | 
18 | service:
19 |   golangci-lint-version: 1.45.0
20 | 
21 | linters:
22 |   disable-all: true
23 |   enable:
24 |     - govet # Suspicious constructs
25 |     - errcheck # Unchecked errors
26 |     - staticcheck # Static analysis checks
27 |     - gosimple # Simplify a code
28 |     - structcheck # Unused struct fields
29 |     - varcheck # Unused global variables and constants
30 |     - ineffassign # Unused assignments to existing variables
31 |     - deadcode # Unused code
32 |     - typecheck # Parses and type-checks Go code
33 |     - rowserrcheck # database/sql.Rows.Err() checked
34 |     - unconvert # Unnecessary type conversions
35 |     - goconst # Repeated strings that could be replaced by a constant
36 |     - gofmt # Whether the code was gofmt-ed
37 |     - goimports # Unused imports
38 |     - misspell # Misspelled English words in comments
39 |     - lll # Long lines
40 |     - unparam # Unused function parameters
41 |     - nakedret # Naked returns in functions greater than a specified function length (?)
42 |     - exportloopref # Unpinned variables in go programs
43 |     - nolintlint # Ill-formed or insufficient nolint directives
44 |     - depguard # Package imports are in a list of acceptable packages
45 |     - gosec # Security problems
46 |     - unused # Unused constants, variables, functions
47 |     - goprintffuncname # Printf-like functions are named with f at the end
48 |     - exportloopref # Exporting pointers for loop variables
49 |     - dupl # Code clone detection
50 | 
51 | issues:
52 |   exclude-rules:
53 |     - path: _test\.go
54 |       linters:
55 |         - gosec
56 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | nodes=5
 2 | duration=25m
 3 | 
 4 | test:
 5 | 	go test -v --race -shuffle=on -coverprofile=coverage.out -covermode=atomic ./...
 6 | 
 7 | 
 8 | property-tests:
 9 | 	cd ./e2e && go test -v -run TestProperty -rapid.steps 10000
10 | 
11 | fuzz-e2e:
12 | 	cd ./e2e && go run ./cmd/main.go fuzz-run -nodes=$(nodes) -duration=$(duration)
13 | 
14 | unit-e2e:
15 | 	cd ./e2e && go test -v -run TestE2E
16 | 
17 | unit-fuzz:
18 | 	cd ./e2e && go test -timeout=20m -run TestFuzz
19 | 
20 | lintci:
21 | 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.46.1
22 | 
23 | lint:
24 | 	@"$(GOPATH)/bin/golangci-lint" run --config ./.golangci.yml ./...
25 | 
26 | 
27 | .PHONY: test e2e property-tests
28 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | 
 2 | # Practical-BFT consensus
 3 | 
 4 | ## Tracing
 5 | 
 6 | You can use OpenTracing to trace the execution of the protocol. Each trace span represents a height/sequence.
 7 | 
 8 | ## E2E
 9 | 
10 | This repo includes integration tests under [/e2e](./e2e)
11 | 


--------------------------------------------------------------------------------
/SECURITY.md:
--------------------------------------------------------------------------------
 1 | # Polygon Technology Security Information
 2 | 
 3 | ## Link to vulnerability disclosure details (Bug Bounty).
 4 | 
 5 | - Websites and Applications: https://hackerone.com/polygon-technology
 6 | - Smart Contracts: https://immunefi.com/bounty/polygon
 7 | 
 8 | ## Languages that our team speaks and understands.
 9 | 
10 | Preferred-Languages: en
11 | 
12 | ## Security-related job openings at Polygon.
13 | 
14 | https://polygon.technology/careers
15 | 
16 | ## Polygon security contact details.
17 | 
18 | security@polygon.technology
19 | 
20 | ## The URL for accessing the security.txt file.
21 | 
22 | Canonical: https://polygon.technology/security.txt
23 | 


--------------------------------------------------------------------------------
/consensus_cfg.go:
--------------------------------------------------------------------------------
  1 | package pbft
  2 | 
  3 | import (
  4 | 	"log"
  5 | 	"os"
  6 | 	"time"
  7 | 
  8 | 	"go.opentelemetry.io/otel/trace"
  9 | 
 10 | 	"github.com/0xPolygon/pbft-consensus/stats"
 11 | )
 12 | 
 13 | const (
 14 | 	defaultTimeout     = 2 * time.Second
 15 | 	maxTimeout         = 300 * time.Second
 16 | 	maxTimeoutExponent = 8
 17 | )
 18 | 
 19 | type RoundTimeout func(round uint64) <-chan time.Time
 20 | 
 21 | type StatsCallback func(stats.Stats)
 22 | 
 23 | type ConfigOption func(*Config)
 24 | 
 25 | func WithLogger(l Logger) ConfigOption {
 26 | 	return func(c *Config) {
 27 | 		c.Logger = l
 28 | 	}
 29 | }
 30 | 
 31 | func WithTracer(t trace.Tracer) ConfigOption {
 32 | 	return func(c *Config) {
 33 | 		c.Tracer = t
 34 | 	}
 35 | }
 36 | 
 37 | func WithRoundTimeout(roundTimeout RoundTimeout) ConfigOption {
 38 | 	return func(c *Config) {
 39 | 		if roundTimeout != nil {
 40 | 			c.RoundTimeout = roundTimeout
 41 | 		}
 42 | 	}
 43 | }
 44 | 
 45 | func WithNotifier(notifier StateNotifier) ConfigOption {
 46 | 	return func(c *Config) {
 47 | 		if notifier != nil {
 48 | 			c.Notifier = notifier
 49 | 		}
 50 | 	}
 51 | }
 52 | 
 53 | type Config struct {
 54 | 	// ProposalTimeout is the time to wait for the proposal
 55 | 	// from the validator. It defaults to Timeout
 56 | 	ProposalTimeout time.Duration
 57 | 
 58 | 	// Timeout is the time to wait for validation and
 59 | 	// round change messages
 60 | 	Timeout time.Duration
 61 | 
 62 | 	// Logger is the logger to output info
 63 | 	Logger Logger
 64 | 
 65 | 	// Tracer is the OpenTelemetry tracer to log traces
 66 | 	Tracer trace.Tracer
 67 | 
 68 | 	// RoundTimeout is a function that calculates timeout based on a round number
 69 | 	RoundTimeout RoundTimeout
 70 | 
 71 | 	// Notifier is a reference to the struct which encapsulates handling messages and timeouts
 72 | 	Notifier StateNotifier
 73 | 
 74 | 	StatsCallback StatsCallback
 75 | }
 76 | 
 77 | func DefaultConfig() *Config {
 78 | 	return &Config{
 79 | 		Timeout:         defaultTimeout,
 80 | 		ProposalTimeout: defaultTimeout,
 81 | 		Logger:          log.New(os.Stderr, "", log.LstdFlags),
 82 | 		Tracer:          trace.NewNoopTracerProvider().Tracer(""),
 83 | 		RoundTimeout:    exponentialTimeout,
 84 | 		Notifier:        &DefaultStateNotifier{},
 85 | 	}
 86 | }
 87 | 
 88 | func (c *Config) ApplyOps(opts ...ConfigOption) {
 89 | 	for _, opt := range opts {
 90 | 		opt(c)
 91 | 	}
 92 | }
 93 | 
 94 | // exponentialTimeout is the default RoundTimeout function
 95 | func exponentialTimeout(round uint64) <-chan time.Time {
 96 | 	return time.NewTimer(exponentialTimeoutDuration(round)).C
 97 | }
 98 | 
 99 | // --- package-level helper functions ---
100 | // exponentialTimeout calculates the timeout duration depending on the current round.
101 | // Round acts as an exponent when determining timeout (2^round).
102 | func exponentialTimeoutDuration(round uint64) time.Duration {
103 | 	timeout := defaultTimeout
104 | 	// limit exponent to be in range of maxTimeout (<=8) otherwise use maxTimeout
105 | 	// this prevents calculating timeout that is greater than maxTimeout and
106 | 	// possible overflow for calculating timeout for rounds >33 since duration is in nanoseconds stored in int64
107 | 	if round <= maxTimeoutExponent {
108 | 		timeout += time.Duration(1<<round) * time.Second
109 | 	} else {
110 | 		timeout = maxTimeout
111 | 	}
112 | 
113 | 	return timeout
114 | }
115 | 
116 | // DefaultStateNotifier is a null object implementation of StateNotifier interface
117 | type DefaultStateNotifier struct {
118 | }
119 | 
120 | // HandleTimeout implements StateNotifier interface
121 | func (d *DefaultStateNotifier) HandleTimeout(NodeID, MsgType, *View) {}
122 | 
123 | // ReadNextMessage is an implementation of StateNotifier interface
124 | func (d *DefaultStateNotifier) ReadNextMessage(p *Pbft) (*MessageReq, []*MessageReq) {
125 | 	return p.ReadMessageWithDiscards()
126 | }
127 | 


--------------------------------------------------------------------------------
/consensus_pbft.go:
--------------------------------------------------------------------------------
  1 | package pbft
  2 | 
  3 | import (
  4 | 	"bytes"
  5 | 	"context"
  6 | 	"fmt"
  7 | 	"sync/atomic"
  8 | 	"time"
  9 | 
 10 | 	"go.opentelemetry.io/otel/attribute"
 11 | 	"go.opentelemetry.io/otel/trace"
 12 | 
 13 | 	"github.com/0xPolygon/pbft-consensus/stats"
 14 | )
 15 | 
 16 | type NodeID string
 17 | 
 18 | type State uint32
 19 | 
 20 | // Define the states in PBFT
 21 | const (
 22 | 	AcceptState State = iota
 23 | 	RoundChangeState
 24 | 	ValidateState
 25 | 	CommitState
 26 | 	SyncState
 27 | 	DoneState
 28 | )
 29 | 
 30 | // String returns the string representation of the passed in state
 31 | func (i State) String() string {
 32 | 	switch i {
 33 | 	case AcceptState:
 34 | 		return "AcceptState"
 35 | 	case RoundChangeState:
 36 | 		return "RoundChangeState"
 37 | 	case ValidateState:
 38 | 		return "ValidateState"
 39 | 	case CommitState:
 40 | 		return "CommitState"
 41 | 	case SyncState:
 42 | 		return "SyncState"
 43 | 	case DoneState:
 44 | 		return "DoneState"
 45 | 	}
 46 | 	panic(fmt.Sprintf("BUG: Pbft state not found %d", i))
 47 | }
 48 | 
 49 | type CommittedSeal struct {
 50 | 	// Signature value
 51 | 	Signature []byte
 52 | 
 53 | 	// Node that signed
 54 | 	NodeID NodeID
 55 | }
 56 | 
 57 | // SealedProposal represents the sealed proposal model
 58 | type SealedProposal struct {
 59 | 	Proposal       *Proposal
 60 | 	CommittedSeals []CommittedSeal
 61 | 	Proposer       NodeID
 62 | 	Number         uint64
 63 | }
 64 | 
 65 | // RoundInfo is the information about the round
 66 | type RoundInfo struct {
 67 | 	IsProposer   bool
 68 | 	Proposer     NodeID
 69 | 	Locked       bool
 70 | 	CurrentRound uint64
 71 | }
 72 | 
 73 | // Pbft represents the PBFT consensus mechanism object
 74 | type Pbft struct {
 75 | 	// Output logger
 76 | 	logger Logger
 77 | 
 78 | 	// config is the configuration of the consensus
 79 | 	config *Config
 80 | 
 81 | 	// inter is the interface with the runtime
 82 | 	backend Backend
 83 | 
 84 | 	// state is the reference to the current state machine
 85 | 	state *state
 86 | 
 87 | 	// validator is the signing key for this instance
 88 | 	validator SignKey
 89 | 
 90 | 	// ctx is the current execution context for an pbft round
 91 | 	ctx context.Context
 92 | 
 93 | 	// msgQueue is a queue that stores all the incomming gossip messages
 94 | 	msgQueue *msgQueue
 95 | 
 96 | 	// updateCh is a channel used to notify when a new gossip message arrives
 97 | 	updateCh chan struct{}
 98 | 
 99 | 	// Transport is the interface for the gossip transport
100 | 	transport Transport
101 | 
102 | 	// tracer is a reference to the OpenTelemetry tracer
103 | 	tracer trace.Tracer
104 | 
105 | 	// roundTimeout calculates timeout for a specific round
106 | 	roundTimeout RoundTimeout
107 | 
108 | 	// notifier is a reference to the struct which encapsulates handling messages and timeouts
109 | 	notifier StateNotifier
110 | 
111 | 	// stats encapsulates logic for statistics reporting
112 | 	stats *stats.Stats
113 | }
114 | 
115 | // New creates a new instance of the PBFT state machine
116 | func New(validator SignKey, transport Transport, opts ...ConfigOption) *Pbft {
117 | 	config := DefaultConfig()
118 | 	config.ApplyOps(opts...)
119 | 
120 | 	p := &Pbft{
121 | 		validator:    validator,
122 | 		state:        newState(),
123 | 		transport:    transport,
124 | 		msgQueue:     newMsgQueue(),
125 | 		updateCh:     make(chan struct{}, 1), //hack. There is a bug when you have several messages pushed on the same time.
126 | 		config:       config,
127 | 		logger:       config.Logger,
128 | 		tracer:       config.Tracer,
129 | 		roundTimeout: config.RoundTimeout,
130 | 		notifier:     config.Notifier,
131 | 		stats:        stats.NewStats(),
132 | 	}
133 | 
134 | 	p.logger.Printf("[INFO] validator key: addr=%s\n", p.validator.NodeID())
135 | 	return p
136 | }
137 | 
138 | func (p *Pbft) SetBackend(backend Backend) error {
139 | 	p.backend = backend
140 | 
141 | 	// set the next current sequence for this iteration
142 | 	p.setSequence(p.backend.Height())
143 | 
144 | 	// set the current set of validators
145 | 	p.state.validators = p.backend.ValidatorSet()
146 | 
147 | 	// initialize voting info
148 | 	if err := p.state.initializeVotingInfo(); err != nil {
149 | 		return err
150 | 	}
151 | 
152 | 	return nil
153 | }
154 | 
155 | // Run starts the PBFT consensus state machine
156 | func (p *Pbft) Run(ctx context.Context) {
157 | 	p.SetInitialState(ctx)
158 | 
159 | 	// start the trace span
160 | 	spanCtx, span := p.tracer.Start(context.Background(), fmt.Sprintf("Sequence-%d", p.state.view.Sequence))
161 | 	defer span.End()
162 | 
163 | 	// loop until we reach the a finish state
164 | 	for p.getState() != DoneState && p.getState() != SyncState {
165 | 		select {
166 | 		case <-ctx.Done():
167 | 			return
168 | 		default:
169 | 		}
170 | 
171 | 		// Start the state machine loop
172 | 		p.runCycle(spanCtx)
173 | 
174 | 		// emit stats when the round is ended
175 | 		p.emitStats()
176 | 	}
177 | }
178 | 
179 | func (p *Pbft) SetInitialState(ctx context.Context) {
180 | 	p.ctx = ctx
181 | 
182 | 	// the iteration always starts with the AcceptState.
183 | 	// AcceptState stages will reset the rest of the message queues.
184 | 	p.setState(AcceptState)
185 | }
186 | 
187 | func (p *Pbft) emitStats() {
188 | 	if p.config.StatsCallback != nil {
189 | 		p.config.StatsCallback(p.stats.Snapshot())
190 | 		// once emitted, reset the Stats
191 | 		p.stats.Reset()
192 | 	}
193 | }
194 | 
195 | func (p *Pbft) RunCycle(ctx context.Context) {
196 | 	p.runCycle(ctx)
197 | }
198 | 
199 | // runCycle represents the PBFT state machine loop
200 | func (p *Pbft) runCycle(ctx context.Context) {
201 | 	startTime := time.Now()
202 | 	state := p.getState()
203 | 	defer p.stats.StateDuration(state.String(), startTime)
204 | 
205 | 	// Log to the console
206 | 	if p.state.view != nil {
207 | 		p.logger.Printf("[DEBUG] cycle: state=%s, sequence=%d, round=%d", p.getState(), p.state.view.Sequence, p.state.GetCurrentRound())
208 | 	}
209 | 	// Based on the current state, execute the corresponding section
210 | 	switch state {
211 | 	case AcceptState:
212 | 		p.runAcceptState(ctx)
213 | 
214 | 	case ValidateState:
215 | 		p.runValidateState(ctx)
216 | 
217 | 	case RoundChangeState:
218 | 		p.runRoundChangeState(ctx)
219 | 
220 | 	case CommitState:
221 | 		p.runCommitState(ctx)
222 | 
223 | 	case DoneState:
224 | 		panic("BUG: We cannot iterate on DoneState")
225 | 	}
226 | }
227 | 
228 | func (p *Pbft) setSequence(sequence uint64) {
229 | 	p.state.view = &View{
230 | 		Sequence: sequence,
231 | 	}
232 | 	p.setRound(0)
233 | 	p.state.unlock()
234 | }
235 | 
236 | func (p *Pbft) setRound(round uint64) {
237 | 	p.state.SetCurrentRound(round)
238 | 
239 | 	// reset current timeout and start a new one
240 | 	p.state.timeoutChan = p.roundTimeout(round)
241 | }
242 | 
243 | // runAcceptState runs the Accept state loop
244 | //
245 | // The Accept state always checks the snapshot, and the validator set. If the current node is not in the validators set,
246 | // it moves back to the Sync state. On the other hand, if the node is a validator, it calculates the proposer.
247 | // If it turns out that the current node is the proposer, it builds a proposal, and sends preprepare and then prepare messages.
248 | func (p *Pbft) runAcceptState(ctx context.Context) { // start new round
249 | 	_, span := p.tracer.Start(ctx, "AcceptState")
250 | 	defer span.End()
251 | 
252 | 	p.stats.SetView(p.state.view.Sequence, p.state.view.Round)
253 | 	p.logger.Printf("[INFO] accept state: sequence %d, round %d", p.state.view.Sequence, p.state.view.Round)
254 | 
255 | 	if !p.state.validators.Includes(p.validator.NodeID()) {
256 | 		// we are not a validator anymore, move back to sync state
257 | 		p.logger.Print("[INFO] we are not a validator anymore")
258 | 		p.setState(SyncState)
259 | 		return
260 | 	}
261 | 
262 | 	// reset round messages
263 | 	p.state.resetRoundMsgs()
264 | 	p.state.CalcProposer()
265 | 
266 | 	isProposer := p.state.proposer == p.validator.NodeID()
267 | 	p.backend.Init(&RoundInfo{
268 | 		Proposer:     p.state.proposer,
269 | 		IsProposer:   isProposer,
270 | 		Locked:       p.state.IsLocked(),
271 | 		CurrentRound: p.state.GetCurrentRound(),
272 | 	})
273 | 
274 | 	// log the current state of this span
275 | 	span.SetAttributes(
276 | 		attribute.Bool("isproposer", isProposer),
277 | 		attribute.Bool("locked", p.state.IsLocked()),
278 | 		attribute.String("proposer", string(p.state.proposer)),
279 | 	)
280 | 
281 | 	var err error
282 | 
283 | 	if isProposer {
284 | 		p.logger.Printf("[INFO] we are the proposer")
285 | 
286 | 		if !p.state.IsLocked() {
287 | 			// since the state is not locked, we need to build a new proposal
288 | 			p.state.proposal, err = p.backend.BuildProposal()
289 | 			if err != nil {
290 | 				p.logger.Printf("[ERROR] failed to build proposal: %v", err)
291 | 				p.setState(RoundChangeState)
292 | 				return
293 | 			}
294 | 
295 | 			// calculate how much time do we have to wait to gossip the proposal
296 | 			delay := time.Until(p.state.proposal.Time)
297 | 			select {
298 | 			case <-time.After(delay):
299 | 			case <-ctx.Done():
300 | 				return
301 | 			}
302 | 		}
303 | 
304 | 		// send the preprepare message
305 | 		p.sendPreprepareMsg()
306 | 
307 | 		// send the prepare message since we are ready to move the state
308 | 		p.sendPrepareMsg()
309 | 
310 | 		// move to validation state for new prepare messages
311 | 		p.setState(ValidateState)
312 | 		return
313 | 	}
314 | 
315 | 	p.logger.Printf("[INFO] proposer calculated: proposer=%s, sequence=%d, round=%d", p.state.proposer, p.state.view.Sequence, p.state.view.Round)
316 | 
317 | 	// we are NOT a proposer for this height/round. Then, we have to wait
318 | 	// for a pre-prepare message from the proposer
319 | 
320 | 	// We only need to wait here for one type of message, the pre-prepare message from the proposer.
321 | 	// However, since we can receive bad pre-prepare messages we have to wait (or timeout) until
322 | 	// we get the message from the correct proposer.
323 | 	for p.getState() == AcceptState {
324 | 		msg, ok := p.getNextMessage(span)
325 | 		if !ok {
326 | 			return
327 | 		}
328 | 		if msg == nil {
329 | 			p.setState(RoundChangeState)
330 | 			continue
331 | 		}
332 | 		// TODO: Validate that the fields required for Preprepare are set (Proposal and Hash)
333 | 		if msg.From != p.state.proposer {
334 | 			p.logger.Printf("[ERROR] msg received from wrong proposer: expected=%s, found=%s", p.state.proposer, msg.From)
335 | 			continue
336 | 		}
337 | 
338 | 		// retrieve the proposal, the backend MUST validate that the hash belongs to the proposal
339 | 		proposal := &Proposal{
340 | 			Data: msg.Proposal,
341 | 			Hash: msg.Hash,
342 | 		}
343 | 		if p.state.IsLocked() && !p.state.proposal.Equal(proposal) {
344 | 			p.handleStateErr(errIncorrectLockedProposal)
345 | 			return
346 | 		}
347 | 
348 | 		if err := p.backend.Validate(proposal); err != nil {
349 | 			p.logger.Printf("[ERROR] failed to validate proposal. Error message: %v", err)
350 | 			p.setState(RoundChangeState)
351 | 			return
352 | 		}
353 | 
354 | 		if p.state.IsLocked() {
355 | 			// fast-track and send a commit message and wait for validations
356 | 			p.sendCommitMsg()
357 | 			p.setState(ValidateState)
358 | 		} else {
359 | 			p.state.proposal = proposal
360 | 			p.sendPrepareMsg()
361 | 			p.setState(ValidateState)
362 | 		}
363 | 	}
364 | }
365 | 
366 | // runValidateState implements the Validate state loop.
367 | //
368 | // The Validate state is rather simple - all nodes do in this state is read messages and add them to their local snapshot state
369 | func (p *Pbft) runValidateState(ctx context.Context) { // start new round
370 | 	_, span := p.tracer.Start(ctx, "ValidateState")
371 | 	// set the attributes of this span once runValidateState is done
372 | 	defer func() {
373 | 		p.setStateSpanAttributes(span)
374 | 		span.End()
375 | 	}()
376 | 
377 | 	hasCommitted := false
378 | 	sendCommit := func(span trace.Span) {
379 | 		// at this point either we have enough prepare messages
380 | 		// or commit messages so we can lock the proposal
381 | 		p.state.lock()
382 | 
383 | 		if !hasCommitted {
384 | 			// send the commit message
385 | 			p.sendCommitMsg()
386 | 			hasCommitted = true
387 | 
388 | 			span.AddEvent("Commit")
389 | 		}
390 | 	}
391 | 
392 | 	quorum := p.state.getQuorumSize()
393 | 	for p.getState() == ValidateState {
394 | 		msg, ok := p.getNextMessage(span)
395 | 		if !ok {
396 | 			// closing
397 | 			return
398 | 		}
399 | 		if msg == nil {
400 | 			// timeout
401 | 			p.setState(RoundChangeState)
402 | 			return
403 | 		}
404 | 
405 | 		// the message must have our local hash
406 | 		if !bytes.Equal(msg.Hash, p.state.proposal.Hash) {
407 | 			p.logger.Printf(fmt.Sprintf("[WARN]: incorrect hash in %s message from node %s", msg.Type.String(), msg.From))
408 | 			continue
409 | 		}
410 | 
411 | 		switch msg.Type {
412 | 		case MessageReq_Prepare:
413 | 			p.state.addPrepareMsg(msg)
414 | 		case MessageReq_Commit:
415 | 			if err := p.backend.ValidateCommit(msg.From, msg.Seal); err != nil {
416 | 				p.logger.Printf("[ERROR]: failed to validate commit: %v", err)
417 | 				continue
418 | 			}
419 | 			p.state.addCommitMsg(msg)
420 | 		default:
421 | 			panic(fmt.Errorf("BUG: Unexpected message type: %s in %s from node %s", msg.Type, p.getState(), msg.From))
422 | 		}
423 | 
424 | 		if p.state.prepared.getAccumulatedVotingPower() >= quorum {
425 | 			// we have received enough prepare messages
426 | 			sendCommit(span)
427 | 		}
428 | 
429 | 		if p.state.committed.getAccumulatedVotingPower() >= quorum {
430 | 			// we have received enough commit messages
431 | 			sendCommit(span)
432 | 
433 | 			// change to commit state just to get out of the loop
434 | 			p.setState(CommitState)
435 | 		}
436 | 	}
437 | }
438 | 
439 | // spanAddEventMessage reports given message to both PBFT built-in statistics reporting mechanism and open telemetry
440 | func (p *Pbft) spanAddEventMessage(typ string, span trace.Span, msg *MessageReq) {
441 | 	p.stats.IncrMsgCount(msg.Type.String(), p.state.validators.VotingPower()[msg.From])
442 | 
443 | 	span.AddEvent("Message", trace.WithAttributes(
444 | 		// message type
445 | 		attribute.String("typ", typ),
446 | 
447 | 		// type of message
448 | 		attribute.String("msg", msg.Type.String()),
449 | 
450 | 		// address of the sender
451 | 		attribute.String("from", string(msg.From)),
452 | 
453 | 		// view sequence
454 | 		attribute.Int64("sequence", int64(msg.View.Sequence)),
455 | 
456 | 		// round sequence
457 | 		attribute.Int64("round", int64(msg.View.Round)),
458 | 	))
459 | }
460 | 
461 | func (p *Pbft) setStateSpanAttributes(span trace.Span) {
462 | 	attr := []attribute.KeyValue{}
463 | 
464 | 	// round
465 | 	attr = append(attr, attribute.Int64("round", int64(p.state.view.Round)))
466 | 
467 | 	// number of commit messages
468 | 	attr = append(attr, attribute.Int("committed", p.state.numCommitted()))
469 | 
470 | 	// commit messages voting power
471 | 	attr = append(attr, attribute.Int64("committed.votingPower", int64(p.state.committed.getAccumulatedVotingPower())))
472 | 
473 | 	// number of prepare messages
474 | 	attr = append(attr, attribute.Int("prepared", p.state.numPrepared()))
475 | 
476 | 	// prepare messages voting power
477 | 	attr = append(attr, attribute.Int64("prepared.votingPower", int64(p.state.prepared.getAccumulatedVotingPower())))
478 | 
479 | 	// number of round change messages per round
480 | 	for round, msgs := range p.state.roundMessages {
481 | 		attr = append(attr, attribute.Int(fmt.Sprintf("roundChange_%d", round), msgs.length()))
482 | 	}
483 | 	span.SetAttributes(attr...)
484 | }
485 | 
486 | // resetRoundChangeSpan terminates previous span (if any) and starts a new one
487 | func (p *Pbft) resetRoundChangeSpan(span trace.Span, ctx context.Context, iteration int64) trace.Span {
488 | 	if span != nil {
489 | 		span.End()
490 | 	}
491 | 	_, span = p.tracer.Start(ctx, "RoundChangeState")
492 | 	span.SetAttributes(attribute.Int64("iteration", iteration))
493 | 	return span
494 | }
495 | 
496 | func (p *Pbft) runCommitState(ctx context.Context) {
497 | 	_, span := p.tracer.Start(ctx, "CommitState")
498 | 	defer span.End()
499 | 
500 | 	committedSeals := p.state.getCommittedSeals()
501 | 	proposal := p.state.proposal.Copy()
502 | 
503 | 	pp := &SealedProposal{
504 | 		Proposal:       proposal,
505 | 		CommittedSeals: committedSeals,
506 | 		Proposer:       p.state.proposer,
507 | 		Number:         p.state.view.Sequence,
508 | 	}
509 | 	if err := p.backend.Insert(pp); err != nil {
510 | 		// start a new round with the state unlocked since we need to
511 | 		// be able to propose/validate a different proposal
512 | 		p.logger.Printf("[ERROR] failed to insert proposal. Error message: %v", err)
513 | 		p.handleStateErr(errFailedToInsertProposal)
514 | 	} else {
515 | 		// move to done state to finish the current iteration of the state machine
516 | 		p.setState(DoneState)
517 | 	}
518 | }
519 | 
520 | var (
521 | 	errIncorrectLockedProposal = fmt.Errorf("locked proposal is incorrect")
522 | 	errVerificationFailed      = fmt.Errorf("proposal verification failed")
523 | 	errFailedToInsertProposal  = fmt.Errorf("failed to insert proposal")
524 | 	errInvalidTotalVotingPower = fmt.Errorf("invalid voting power configuration provided: total voting power must be greater than 0")
525 | )
526 | 
527 | func (p *Pbft) handleStateErr(err error) {
528 | 	p.state.err = err
529 | 	p.setState(RoundChangeState)
530 | }
531 | 
532 | func (p *Pbft) runRoundChangeState(ctx context.Context) {
533 | 	iteration := int64(1)
534 | 	span := p.resetRoundChangeSpan(nil, ctx, iteration)
535 | 
536 | 	sendRoundChange := func(round uint64) {
537 | 		p.logger.Printf("[DEBUG] local round change: round=%d", round)
538 | 		// set the new round
539 | 		p.setRound(round)
540 | 		// set state attributes to the span
541 | 		p.setStateSpanAttributes(span)
542 | 		// clean the round
543 | 		p.state.cleanRound(round)
544 | 		// send the round change message
545 | 		p.sendRoundChange()
546 | 		// terminate a span and start a new one
547 | 		iteration++
548 | 		span = p.resetRoundChangeSpan(span, ctx, iteration)
549 | 	}
550 | 
551 | 	sendNextRoundChange := func() {
552 | 		sendRoundChange(p.state.GetCurrentRound() + 1)
553 | 	}
554 | 
555 | 	checkTimeout := func() {
556 | 		// At this point we might be stuck in the network if:
557 | 		// - We have advanced the round but everyone else passed.
558 | 		// - We are removing those messages since they are old now.
559 | 		if bestHeight, stucked := p.backend.IsStuck(p.state.view.Sequence); stucked {
560 | 			span.AddEvent("OutOfSync", trace.WithAttributes(
561 | 				// our local height
562 | 				attribute.Int64("local", int64(p.state.view.Sequence)),
563 | 				// the best remote height
564 | 				attribute.Int64("remote", int64(bestHeight)),
565 | 			))
566 | 			// set state span attributes and terminate it
567 | 			p.setStateSpanAttributes(span)
568 | 			span.End()
569 | 			p.setState(SyncState)
570 | 			return
571 | 		}
572 | 
573 | 		// otherwise, it seems that we are in sync
574 | 		// and we should start a new round
575 | 		sendNextRoundChange()
576 | 	}
577 | 
578 | 	// if the round was triggered due to an error, we send our own
579 | 	// next round change
580 | 	if err := p.state.getErr(); err != nil {
581 | 		p.logger.Printf("[DEBUG] round change handle error. Error message: %v", err)
582 | 		sendNextRoundChange()
583 | 	} else {
584 | 		// otherwise, it is due to a timeout in any stage
585 | 		// First, we try to sync up with any max round already available
586 | 		// F + 1 round change messages for given round, where F denotes MaxFaulty is expected, in order to fast-track to maxRound
587 | 		if maxRound, ok := p.state.maxRound(); ok {
588 | 			p.logger.Printf("[DEBUG] round change, max round=%d", maxRound)
589 | 			sendRoundChange(maxRound)
590 | 		} else {
591 | 			// otherwise, do your best to sync up
592 | 			checkTimeout()
593 | 		}
594 | 	}
595 | 
596 | 	// create a timer for the round change
597 | 	for p.getState() == RoundChangeState {
598 | 		msg, ok := p.getNextMessage(span)
599 | 		if !ok {
600 | 			// closing
601 | 			return
602 | 		}
603 | 		if msg == nil {
604 | 			p.logger.Print("[DEBUG] round change timeout")
605 | 
606 | 			// checkTimeout will either produce a sync event and exit
607 | 			// or restart the timeout
608 | 			checkTimeout()
609 | 			continue
610 | 		}
611 | 
612 | 		// we only expect RoundChange messages right now
613 | 		p.state.addRoundChangeMsg(msg)
614 | 
615 | 		currentVotingPower := p.state.roundMessages[msg.View.Round].getAccumulatedVotingPower()
616 | 		// Round change quorum is 2*F round change messages (F denotes max faulty voting power)
617 | 		if currentVotingPower >= 2*p.state.getMaxFaultyVotingPower() {
618 | 			// start a new round immediately
619 | 			p.state.SetCurrentRound(msg.View.Round)
620 | 			// set state span attributes and terminate it
621 | 			p.setStateSpanAttributes(span)
622 | 			span.End()
623 | 			p.setState(AcceptState)
624 | 		} else if currentVotingPower >= p.state.getMaxFaultyVotingPower()+1 {
625 | 			// weak certificate, try to catch up if our round number is smaller
626 | 			if p.state.GetCurrentRound() < msg.View.Round {
627 | 				// update timer
628 | 				sendRoundChange(msg.View.Round)
629 | 			}
630 | 		}
631 | 	}
632 | }
633 | 
634 | // --- communication wrappers ---
635 | 
636 | func (p *Pbft) sendRoundChange() {
637 | 	p.gossip(MessageReq_RoundChange)
638 | }
639 | 
640 | func (p *Pbft) sendPreprepareMsg() {
641 | 	p.gossip(MessageReq_Preprepare)
642 | }
643 | 
644 | func (p *Pbft) sendPrepareMsg() {
645 | 	p.gossip(MessageReq_Prepare)
646 | }
647 | 
648 | func (p *Pbft) sendCommitMsg() {
649 | 	p.gossip(MessageReq_Commit)
650 | }
651 | 
652 | func (p *Pbft) gossip(msgType MsgType) {
653 | 	msg := &MessageReq{
654 | 		Type: msgType,
655 | 		From: p.validator.NodeID(),
656 | 	}
657 | 	if msgType != MessageReq_RoundChange {
658 | 		// Except for round change message in which we are deciding on the proposer,
659 | 		// the rest of the consensus message require the hash:
660 | 		// 1. Preprepare: notify the validators of the proposal + hash
661 | 		// 2. Prepare + Commit: safe check to only include messages from our round.
662 | 		msg.Hash = p.state.proposal.Hash
663 | 	}
664 | 
665 | 	// add View
666 | 	msg.View = p.state.view.Copy()
667 | 
668 | 	// if we are sending a preprepare message we need to include the proposal
669 | 	if msg.Type == MessageReq_Preprepare {
670 | 		msg.SetProposal(p.state.proposal.Data)
671 | 	}
672 | 
673 | 	// if the message is commit, we need to add the committed seal
674 | 	if msg.Type == MessageReq_Commit {
675 | 		// seal the hash of the proposal
676 | 		seal, err := p.validator.Sign(p.state.proposal.Hash)
677 | 		if err != nil {
678 | 			p.logger.Printf("[ERROR] failed to commit seal. Error message: %v", err)
679 | 			return
680 | 		}
681 | 		msg.Seal = seal
682 | 	}
683 | 
684 | 	if msg.Type != MessageReq_Preprepare {
685 | 		// send a copy to ourselves so that we can process this message as well
686 | 		msg2 := msg.Copy()
687 | 		msg2.From = p.validator.NodeID()
688 | 		p.PushMessage(msg2)
689 | 	}
690 | 	if err := p.transport.Gossip(msg); err != nil {
691 | 		p.logger.Printf("[ERROR] failed to gossip. Error message: %v", err)
692 | 	}
693 | }
694 | 
695 | // GetValidatorId returns validator NodeID
696 | func (p *Pbft) GetValidatorId() NodeID {
697 | 	return p.validator.NodeID()
698 | }
699 | 
700 | // GetState returns the current PBFT state
701 | func (p *Pbft) GetState() State {
702 | 	return p.getState()
703 | }
704 | 
705 | // getState returns the current PBFT state
706 | func (p *Pbft) getState() State {
707 | 	return p.state.getState()
708 | }
709 | 
710 | // IsState checks if the node is in the passed in state
711 | func (p *Pbft) IsState(s State) bool {
712 | 	return p.state.getState() == s
713 | }
714 | 
715 | func (p *Pbft) SetState(s State) {
716 | 	p.setState(s)
717 | }
718 | 
719 | // setState sets the PBFT state
720 | func (p *Pbft) setState(s State) {
721 | 	p.logger.Printf("[DEBUG] state change: '%s'", s)
722 | 	p.state.setState(s)
723 | }
724 | 
725 | // IsLocked returns if the current proposal is locked
726 | func (p *Pbft) IsLocked() bool {
727 | 	return atomic.LoadUint64(&p.state.locked) == 1
728 | }
729 | 
730 | // GetProposal returns current proposal in the pbft
731 | func (p *Pbft) GetProposal() *Proposal {
732 | 	return p.state.proposal
733 | }
734 | 
735 | func (p *Pbft) Round() uint64 {
736 | 	return atomic.LoadUint64(&p.state.view.Round)
737 | }
738 | 
739 | // getNextMessage reads a new message from the message queue
740 | func (p *Pbft) getNextMessage(span trace.Span) (*MessageReq, bool) {
741 | 	for {
742 | 		msg, discards := p.notifier.ReadNextMessage(p)
743 | 		// send the discard messages
744 | 		p.logger.Printf("[TRACE] Current state %s, number of prepared messages: %d (voting power: %d), number of committed messages %d (voting power: %d)",
745 | 			p.getState(), p.state.numPrepared(), p.state.prepared.getAccumulatedVotingPower(), p.state.numCommitted(), p.state.committed.getAccumulatedVotingPower())
746 | 
747 | 		for _, msg := range discards {
748 | 			p.logger.Printf("[TRACE] Discarded %s ", msg)
749 | 			p.spanAddEventMessage("dropMessage", span, msg)
750 | 		}
751 | 		if msg != nil {
752 | 			// add the event to the span
753 | 			p.spanAddEventMessage("message", span, msg)
754 | 			p.logger.Printf("[TRACE] Received %s", msg)
755 | 			return msg, true
756 | 		}
757 | 
758 | 		// wait until there is a new message or
759 | 		// someone closes the stopCh (i.e. timeout for round change)
760 | 		select {
761 | 		case <-p.state.timeoutChan:
762 | 			span.AddEvent("Timeout")
763 | 			p.notifier.HandleTimeout(p.validator.NodeID(), stateToMsg(p.getState()), &View{
764 | 				Round:    p.state.GetCurrentRound(),
765 | 				Sequence: p.state.view.Sequence,
766 | 			})
767 | 			p.logger.Printf("[TRACE] Message read timeout occurred")
768 | 			return nil, true
769 | 		case <-p.ctx.Done():
770 | 			return nil, false
771 | 		case <-p.updateCh:
772 | 		}
773 | 	}
774 | }
775 | 
776 | func (p *Pbft) PushMessageInternal(msg *MessageReq) {
777 | 	p.msgQueue.pushMessage(msg)
778 | 
779 | 	select {
780 | 	case p.updateCh <- struct{}{}:
781 | 	default:
782 | 	}
783 | }
784 | 
785 | // PushMessage pushes a new message to the message queue
786 | func (p *Pbft) PushMessage(msg *MessageReq) {
787 | 	if err := msg.Validate(); err != nil {
788 | 		p.logger.Printf("[ERROR]: failed to validate msg: %v", err)
789 | 		return
790 | 	}
791 | 
792 | 	p.PushMessageInternal(msg)
793 | }
794 | 
795 | // ReadMessageWithDiscards reads next message with discards from message queue based on current state, sequence and round
796 | func (p *Pbft) ReadMessageWithDiscards() (*MessageReq, []*MessageReq) {
797 | 	return p.msgQueue.readMessageWithDiscards(p.getState(), p.state.view)
798 | }
799 | 
800 | // MaxFaultyVotingPower is a wrapper function around state.MaxFaultyVotingPower
801 | func (p *Pbft) MaxFaultyVotingPower() uint64 {
802 | 	return p.state.getMaxFaultyVotingPower()
803 | }
804 | 
805 | // QuorumSize is a wrapper function around state.QuorumSize
806 | func (p *Pbft) QuorumSize() uint64 {
807 | 	return p.state.getQuorumSize()
808 | }
809 | 
810 | // CalculateQuorum calculates max faulty voting power and quorum size for given voting power map
811 | func CalculateQuorum(votingPower map[NodeID]uint64) (maxFaultyVotingPower uint64, quorumSize uint64, err error) {
812 | 	totalVotingPower := uint64(0)
813 | 	for _, v := range votingPower {
814 | 		totalVotingPower += v
815 | 	}
816 | 	if totalVotingPower == 0 {
817 | 		err = errInvalidTotalVotingPower
818 | 		return
819 | 	}
820 | 	maxFaultyVotingPower = (totalVotingPower - 1) / 3
821 | 	quorumSize = 2*maxFaultyVotingPower + 1
822 | 	return
823 | }
824 | 


--------------------------------------------------------------------------------
/e2e/README.md:
--------------------------------------------------------------------------------
 1 | 
 2 | # E2E PBFT
 3 | 
 4 | # Jaeger
 5 | 
 6 | E2E tests uses OpenTracing to profile the execution of the PBFT state machine among the nodes in the cluster. It uses Jaeger to collect the tracing metrics, setup the Jaeger collector with:
 7 | 
 8 | ```
 9 | $ docker run --net=host jaegertracing/all-in-one:1.27
10 | ```
11 | 
12 | You also need to run the OpenCollector to move data to Jaeger from the OpenTelemetry client in PBFT.
13 | 
14 | ```
15 | $ docker run --net=host -v "${PWD}/otel-jaeger-config.yaml":/otel-local-config.yaml otel/opentelemetry-collector --config otel-local-config.yaml
16 | ```
17 | 
18 | ## Tests
19 | 
20 | To log output of nodes into files, set environment variable E2E_LOG_TO_FILES to true.
21 | 
22 | ### TestE2E_NoIssue
23 | 
24 | Simple cluster with 5 machines.
25 | 
26 | ### TestE2E_NodeDrop
27 | 
28 | Cluster starts and then one node fails.
29 | 
30 | ### TestE2E_Partition_OneMajority
31 | 
32 | Cluster of 5 is partitioned in two sets, one with the majority (3) and one without (2).
33 | 


--------------------------------------------------------------------------------
/e2e/backend.go:
--------------------------------------------------------------------------------
  1 | package e2e
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"strconv"
  6 | 	"time"
  7 | 
  8 | 	"github.com/0xPolygon/pbft-consensus"
  9 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
 10 | )
 11 | 
 12 | // IntegrationBackend extends the pbft.Backend interface with additional e2e-related method
 13 | type IntegrationBackend interface {
 14 | 	pbft.Backend
 15 | 	SetBackendData(n *node)
 16 | }
 17 | 
 18 | // BackendFake implements IntegrationBackend interface
 19 | type BackendFake struct {
 20 | 	nodes           []string
 21 | 	votingPowerMap  map[pbft.NodeID]uint64
 22 | 	height          uint64
 23 | 	lastProposer    pbft.NodeID
 24 | 	proposalAddTime time.Duration
 25 | 
 26 | 	insertFunc   func(*pbft.SealedProposal) error
 27 | 	isStuckFunc  func(uint64) (uint64, bool)
 28 | 	validateFunc func(*pbft.Proposal) error
 29 | }
 30 | 
 31 | func (bf *BackendFake) BuildProposal() (*pbft.Proposal, error) {
 32 | 	tm := time.Now()
 33 | 	if bf.proposalAddTime > 0 {
 34 | 		tm = tm.Add(bf.proposalAddTime)
 35 | 	}
 36 | 
 37 | 	proposal := &pbft.Proposal{
 38 | 		Data: helper.GenerateProposal(),
 39 | 		Time: tm,
 40 | 	}
 41 | 	proposal.Hash = helper.Hash(proposal.Data)
 42 | 	return proposal, nil
 43 | }
 44 | 
 45 | func (bf *BackendFake) Height() uint64 {
 46 | 	return bf.height
 47 | }
 48 | 
 49 | func (bf *BackendFake) Init(*pbft.RoundInfo) {
 50 | }
 51 | 
 52 | func (bf *BackendFake) Insert(p *pbft.SealedProposal) error {
 53 | 	if bf.insertFunc != nil {
 54 | 		return bf.insertFunc(p)
 55 | 	}
 56 | 	return nil
 57 | }
 58 | 
 59 | func (bf *BackendFake) IsStuck(num uint64) (uint64, bool) {
 60 | 	if bf.isStuckFunc != nil {
 61 | 		return bf.isStuckFunc(num)
 62 | 	}
 63 | 	panic("IsStuck " + strconv.Itoa(int(num)))
 64 | }
 65 | 
 66 | func (bf *BackendFake) Validate(proposal *pbft.Proposal) error {
 67 | 	if bf.validateFunc != nil {
 68 | 		return bf.validateFunc(proposal)
 69 | 	}
 70 | 
 71 | 	return nil
 72 | }
 73 | 
 74 | func (bf *BackendFake) ValidatorSet() pbft.ValidatorSet {
 75 | 	valsAsNode := []pbft.NodeID{}
 76 | 	for _, i := range bf.nodes {
 77 | 		valsAsNode = append(valsAsNode, pbft.NodeID(i))
 78 | 	}
 79 | 
 80 | 	return &ValidatorSet{
 81 | 		Nodes:        valsAsNode,
 82 | 		LastProposer: bf.lastProposer,
 83 | 	}
 84 | }
 85 | 
 86 | func (bf *BackendFake) ValidateCommit(from pbft.NodeID, seal []byte) error {
 87 | 	return nil
 88 | }
 89 | 
 90 | // SetBackendData implements IntegrationBackend interface and sets the data needed for backend
 91 | func (bf *BackendFake) SetBackendData(n *node) {
 92 | 	bf.nodes = n.nodes
 93 | 	bf.lastProposer = n.c.getProposer(n.getSyncIndex())
 94 | 	bf.height = n.GetNodeHeight() + 1
 95 | 	bf.proposalAddTime = 1 * time.Second
 96 | 	bf.isStuckFunc = n.isStuck
 97 | 	bf.insertFunc = n.insert
 98 | 	bf.validateFunc = func(proposal *pbft.Proposal) error {
 99 | 		if n.isFaulty() {
100 | 			return fmt.Errorf("validation error")
101 | 		}
102 | 
103 | 		return nil
104 | 	}
105 | }
106 | 


--------------------------------------------------------------------------------
/e2e/cluster.go:
--------------------------------------------------------------------------------
  1 | package e2e
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"errors"
  6 | 	"fmt"
  7 | 	"log"
  8 | 	"sync"
  9 | 	"testing"
 10 | 	"time"
 11 | 
 12 | 	"github.com/0xPolygon/pbft-consensus/e2e/notifier"
 13 | 
 14 | 	"go.opentelemetry.io/otel"
 15 | 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
 16 | 	"go.opentelemetry.io/otel/propagation"
 17 | 	"go.opentelemetry.io/otel/sdk/resource"
 18 | 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 19 | 	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
 20 | 
 21 | 	"github.com/0xPolygon/pbft-consensus"
 22 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
 23 | 	"github.com/0xPolygon/pbft-consensus/e2e/transport"
 24 | )
 25 | 
 26 | var (
 27 | 	errMaxFaultyEmptyCluster  = errors.New("unable to determine max faulty nodes: cluster is empty")
 28 | 	errQuorumSizeEmptyCluster = errors.New("unable to determine quorum size, because cluster is empty")
 29 | )
 30 | 
 31 | // CreateBackend is a delegate that creates a new instance of IntegrationBackend interface
 32 | type CreateBackend func() IntegrationBackend
 33 | 
 34 | type ClusterConfig struct {
 35 | 	Count                 int
 36 | 	Name                  string
 37 | 	Prefix                string
 38 | 	LogsDir               string
 39 | 	ReplayMessageNotifier notifier.Notifier
 40 | 	TransportHandler      transport.Handler
 41 | 	RoundTimeout          pbft.RoundTimeout
 42 | 	CreateBackend         CreateBackend
 43 | }
 44 | 
 45 | type Cluster struct {
 46 | 	t                     *testing.T
 47 | 	lock                  sync.Mutex
 48 | 	nodes                 map[string]*node
 49 | 	tracer                *sdktrace.TracerProvider
 50 | 	transport             *transport.Transport
 51 | 	sealedProposals       []*pbft.SealedProposal
 52 | 	replayMessageNotifier notifier.Notifier
 53 | 	createBackend         CreateBackend
 54 | }
 55 | 
 56 | // NewPBFTCluster is the constructor of Cluster
 57 | func NewPBFTCluster(t *testing.T, config *ClusterConfig, hook ...transport.Hook) *Cluster {
 58 | 	names := make([]string, config.Count)
 59 | 	for i := 0; i < config.Count; i++ {
 60 | 		names[i] = fmt.Sprintf("%s_%d", config.Prefix, i)
 61 | 	}
 62 | 
 63 | 	tt := &transport.Transport{}
 64 | 	if len(hook) == 1 {
 65 | 		tt.AddHook(hook[0])
 66 | 	}
 67 | 
 68 | 	var directoryName string
 69 | 	if t != nil {
 70 | 		directoryName = t.Name()
 71 | 	}
 72 | 
 73 | 	if config.ReplayMessageNotifier == nil {
 74 | 		config.ReplayMessageNotifier = &notifier.DefaultNotifier{}
 75 | 	}
 76 | 
 77 | 	if config.CreateBackend == nil {
 78 | 		config.CreateBackend = func() IntegrationBackend { return &BackendFake{} }
 79 | 	}
 80 | 
 81 | 	logsDir, err := helper.CreateLogsDir(directoryName)
 82 | 	if err != nil {
 83 | 		log.Printf("[WARNING] Could not create logs directory. Reason: %v. Logging will be defaulted to standard output.", err)
 84 | 	} else {
 85 | 		config.LogsDir = logsDir
 86 | 	}
 87 | 
 88 | 	tt.SetLogger(log.New(helper.GetLoggerOutput("transport", logsDir), "", log.LstdFlags))
 89 | 
 90 | 	c := &Cluster{
 91 | 		t:                     t,
 92 | 		nodes:                 map[string]*node{},
 93 | 		tracer:                initTracer("fuzzy_" + config.Name),
 94 | 		transport:             tt,
 95 | 		sealedProposals:       []*pbft.SealedProposal{},
 96 | 		replayMessageNotifier: config.ReplayMessageNotifier,
 97 | 		createBackend:         config.CreateBackend,
 98 | 	}
 99 | 
100 | 	err = c.replayMessageNotifier.SaveMetaData(&names)
101 | 	if err != nil {
102 | 		log.Printf("[WARNING] Could not write node meta data to replay messages file. Reason: %v", err)
103 | 	}
104 | 
105 | 	for _, name := range names {
106 | 		trace := c.tracer.Tracer(name)
107 | 		n := newPBFTNode(name, config, names, trace, tt)
108 | 		n.c = c
109 | 		c.nodes[name] = n
110 | 	}
111 | 	return c
112 | }
113 | 
114 | func (c *Cluster) Nodes() []*node {
115 | 	c.lock.Lock()
116 | 	defer c.lock.Unlock()
117 | 
118 | 	list := make([]*node, len(c.nodes))
119 | 	i := 0
120 | 	for _, n := range c.nodes {
121 | 		list[i] = n
122 | 		i++
123 | 	}
124 | 	return list
125 | }
126 | 
127 | func (c *Cluster) SetHook(hook transport.Hook) {
128 | 	c.transport.AddHook(hook)
129 | }
130 | 
131 | func (c *Cluster) GetMaxHeight(nodes ...[]string) uint64 {
132 | 	queryNodes := c.resolveNodes(nodes...)
133 | 	var max uint64
134 | 	for _, node := range queryNodes {
135 | 		h := c.nodes[node].GetNodeHeight()
136 | 		if h > max {
137 | 			max = h
138 | 		}
139 | 	}
140 | 	return max
141 | }
142 | 
143 | func (c *Cluster) WaitForHeight(num uint64, timeout time.Duration, nodes ...[]string) error {
144 | 	// we need to check every node in the ensemble?
145 | 	// yes, this should test if everyone can agree on the final set.
146 | 	// note, if we include drops, we need to do sync otherwise this will never work
147 | 	queryNodes := c.resolveNodes(nodes...)
148 | 
149 | 	enough := func() bool {
150 | 		c.lock.Lock()
151 | 		defer c.lock.Unlock()
152 | 
153 | 		for _, name := range queryNodes {
154 | 			if c.nodes[name].GetNodeHeight() < num {
155 | 				return false
156 | 			}
157 | 		}
158 | 		return true
159 | 	}
160 | 
161 | 	timer := time.NewTimer(timeout)
162 | 	defer timer.Stop()
163 | 	for {
164 | 		select {
165 | 		case <-time.After(300 * time.Millisecond):
166 | 			if enough() {
167 | 				return nil
168 | 			}
169 | 		case <-timer.C:
170 | 			return fmt.Errorf("timeout")
171 | 		}
172 | 	}
173 | }
174 | 
175 | func (c *Cluster) GetNodesMap() map[string]*node {
176 | 	return c.nodes
177 | }
178 | 
179 | func (c *Cluster) GetRunningNodes() []*node {
180 | 	return c.getFilteredNodes(func(n *node) bool {
181 | 		return n.IsRunning()
182 | 	})
183 | }
184 | 
185 | func (c *Cluster) Start() {
186 | 	for _, n := range c.nodes {
187 | 		n.Start()
188 | 	}
189 | }
190 | 
191 | func (c *Cluster) StopNode(name string) {
192 | 	c.nodes[name].Stop()
193 | }
194 | 
195 | func (c *Cluster) Stop() {
196 | 	for _, n := range c.nodes {
197 | 		if n.IsRunning() {
198 | 			n.Stop()
199 | 		}
200 | 	}
201 | 	if err := c.tracer.Shutdown(context.Background()); err != nil {
202 | 		panic("failed to shutdown TracerProvider")
203 | 	}
204 | }
205 | 
206 | func (c *Cluster) GetTransportHook() transport.Hook {
207 | 	return c.transport.GetHook()
208 | }
209 | 
210 | // insertFinalProposal inserts final proposal from the node to the cluster
211 | func (c *Cluster) insertFinalProposal(sealProp *pbft.SealedProposal) error {
212 | 	c.lock.Lock()
213 | 	defer c.lock.Unlock()
214 | 
215 | 	insertIndex := sealProp.Number - 1
216 | 	lastIndex := len(c.sealedProposals) - 1
217 | 
218 | 	if lastIndex >= 0 {
219 | 		if insertIndex <= uint64(lastIndex) {
220 | 			// already exists
221 | 			if !c.sealedProposals[insertIndex].Proposal.Equal(sealProp.Proposal) {
222 | 				return errors.New("existing proposal on a given position is not not equal to the one being inserted to the same position")
223 | 			}
224 | 
225 | 			return nil
226 | 		} else if insertIndex != uint64(lastIndex+1) {
227 | 			return fmt.Errorf("expected that final proposal number is %v, but was %v", len(c.sealedProposals)+1, sealProp.Number)
228 | 		}
229 | 	}
230 | 	c.sealedProposals = append(c.sealedProposals, sealProp)
231 | 	return nil
232 | }
233 | 
234 | func (c *Cluster) resolveNodes(nodes ...[]string) []string {
235 | 	queryNodes := []string{}
236 | 	if len(nodes) == 1 {
237 | 		for _, n := range nodes[0] {
238 | 			if _, ok := c.nodes[n]; !ok {
239 | 				panic("node not found in query")
240 | 			}
241 | 		}
242 | 		queryNodes = nodes[0]
243 | 	} else {
244 | 		for n := range c.nodes {
245 | 			queryNodes = append(queryNodes, n)
246 | 		}
247 | 	}
248 | 	return queryNodes
249 | }
250 | 
251 | func (c *Cluster) isStuck(timeout time.Duration, nodes ...[]string) {
252 | 	queryNodes := c.resolveNodes(nodes...)
253 | 
254 | 	nodeHeight := map[string]uint64{}
255 | 	isStuck := func() bool {
256 | 		for _, n := range queryNodes {
257 | 			height := c.nodes[n].GetNodeHeight()
258 | 			if lastHeight, ok := nodeHeight[n]; ok {
259 | 				if lastHeight != height {
260 | 					return false
261 | 				}
262 | 			} else {
263 | 				nodeHeight[n] = height
264 | 			}
265 | 		}
266 | 		return true
267 | 	}
268 | 
269 | 	timer := time.NewTimer(timeout)
270 | 	for {
271 | 		select {
272 | 		case <-time.After(200 * time.Millisecond):
273 | 			if !isStuck() {
274 | 				c.t.Fatal("it is not stuck")
275 | 			}
276 | 		case <-timer.C:
277 | 			return
278 | 		}
279 | 	}
280 | }
281 | 
282 | func (c *Cluster) getNodes() []*node {
283 | 	nodes := make([]*node, 0, len(c.nodes))
284 | 	for _, nd := range c.nodes {
285 | 		nodes = append(nodes, nd)
286 | 	}
287 | 	return nodes
288 | }
289 | 
290 | func (c *Cluster) syncWithNetwork(nodeID string) (uint64, int64) {
291 | 	c.lock.Lock()
292 | 	defer c.lock.Unlock()
293 | 
294 | 	var height uint64
295 | 	var syncIndex = int64(-1) // initial sync index is -1
296 | 	for _, n := range c.nodes {
297 | 		if n.name == nodeID {
298 | 			continue
299 | 		}
300 | 		if hook := c.transport.GetHook(); hook != nil {
301 | 			// we need to see if this transport does allow those two nodes to be connected
302 | 			// Otherwise, that node should not be eligible to sync
303 | 			if !hook.Connects(pbft.NodeID(nodeID), pbft.NodeID(n.name)) {
304 | 				continue
305 | 			}
306 | 		}
307 | 		localHeight := n.GetNodeHeight()
308 | 		if localHeight > height {
309 | 			height = localHeight
310 | 			syncIndex = int64(localHeight) - 1 // we know that syncIndex is less than height by 1
311 | 		}
312 | 	}
313 | 	return height, syncIndex
314 | }
315 | 
316 | func (c *Cluster) getProposer(index int64) pbft.NodeID {
317 | 	c.lock.Lock()
318 | 	defer c.lock.Unlock()
319 | 
320 | 	proposer := pbft.NodeID("")
321 | 	if index >= 0 && int64(len(c.sealedProposals)-1) >= index {
322 | 		proposer = c.sealedProposals[index].Proposer
323 | 	}
324 | 
325 | 	return proposer
326 | }
327 | 
328 | // getFilteredNodes returns nodes which satisfy provided filter delegate function.
329 | // If filter is not provided, all the nodes will be retreived.
330 | func (c *Cluster) getFilteredNodes(filter func(*node) bool) []*node {
331 | 	if filter != nil {
332 | 		var filteredNodes []*node
333 | 		for _, n := range c.nodes {
334 | 			if filter(n) {
335 | 				filteredNodes = append(filteredNodes, n)
336 | 			}
337 | 		}
338 | 		return filteredNodes
339 | 	}
340 | 	return c.getNodes()
341 | }
342 | 
343 | func (c *Cluster) startNode(name string) {
344 | 	c.nodes[name].Start()
345 | }
346 | 
347 | // MaxFaulty is a wrapper function which invokes MaxFaultyVotingPower on PBFT consensus instance of the first node in cluster
348 | func (c *Cluster) MaxFaulty() (uint64, error) {
349 | 	nodes := c.getNodes()
350 | 	if len(nodes) == 0 {
351 | 		return 0, errMaxFaultyEmptyCluster
352 | 	}
353 | 	return nodes[0].pbft.MaxFaultyVotingPower(), nil
354 | }
355 | 
356 | // QuorumSize is a wrapper function which invokes QuorumSize on PBFT consensus instance of the first node in cluster
357 | func (c *Cluster) QuorumSize() (uint64, error) {
358 | 	nodes := c.getNodes()
359 | 	if len(nodes) == 0 {
360 | 		return 0, errQuorumSizeEmptyCluster
361 | 	}
362 | 	return nodes[0].pbft.QuorumSize(), nil
363 | }
364 | 
365 | func initTracer(name string) *sdktrace.TracerProvider {
366 | 	ctx := context.Background()
367 | 
368 | 	res, err := resource.New(ctx,
369 | 		resource.WithAttributes(
370 | 			// the service name used to display traces in backends
371 | 			semconv.ServiceNameKey.String(name),
372 | 		),
373 | 	)
374 | 	if err != nil {
375 | 		panic("failed to create resource")
376 | 	}
377 | 
378 | 	// Set up a trace exporter
379 | 	traceExporter, err := otlptracegrpc.New(ctx,
380 | 		otlptracegrpc.WithInsecure(),
381 | 		otlptracegrpc.WithEndpoint("localhost:4317"),
382 | 	)
383 | 	if err != nil {
384 | 		panic("failed to trace exporter")
385 | 	}
386 | 
387 | 	// Register the trace exporter with a TracerProvider, using a batch
388 | 	// span processor to aggregate spans before export.
389 | 	bsp := sdktrace.NewBatchSpanProcessor(traceExporter)
390 | 	tracerProvider := sdktrace.NewTracerProvider(
391 | 		sdktrace.WithSampler(sdktrace.AlwaysSample()),
392 | 		sdktrace.WithResource(res),
393 | 		sdktrace.WithSpanProcessor(bsp),
394 | 	)
395 | 
396 | 	// set global propagator to tracecontext (the default is no-op).
397 | 	otel.SetTextMapPropagator(propagation.TraceContext{})
398 | 
399 | 	return tracerProvider
400 | }
401 | 


--------------------------------------------------------------------------------
/e2e/cmd/fuzz/action.go:
--------------------------------------------------------------------------------
 1 | package fuzz
 2 | 
 3 | import (
 4 | 	"log"
 5 | 	"math/rand"
 6 | 	"time"
 7 | 
 8 | 	"github.com/0xPolygon/pbft-consensus/e2e"
 9 | 	"github.com/0xPolygon/pbft-consensus/e2e/transport"
10 | )
11 | 
12 | type RevertFunc func()
13 | 
14 | // Action represents the action behavior
15 | type Action interface {
16 | 	CanApply(c *e2e.Cluster) bool
17 | 	Apply(c *e2e.Cluster) RevertFunc
18 | }
19 | 
20 | // DropNode encapsulates logic for dropping nodes action.
21 | type DropNode struct{}
22 | 
23 | func (dn *DropNode) CanApply(c *e2e.Cluster) bool {
24 | 	runningNodes := len(c.GetRunningNodes())
25 | 	if runningNodes <= 0 {
26 | 		return false
27 | 	}
28 | 	maxFaultyNodes, err := c.MaxFaulty()
29 | 	if err != nil {
30 | 		log.Printf("[ERROR] %v", err)
31 | 		return false
32 | 	}
33 | 	remainingNodes := runningNodes - 1
34 | 	return remainingNodes >= int(maxFaultyNodes)
35 | }
36 | 
37 | func (dn *DropNode) Apply(c *e2e.Cluster) RevertFunc {
38 | 	runningNodes := c.GetRunningNodes()
39 | 	nodeToStop := runningNodes[rand.Intn(len(runningNodes))] //nolint:golint,gosec
40 | 	log.Printf("Dropping node: '%s'.", nodeToStop)
41 | 
42 | 	c.StopNode(nodeToStop.GetName())
43 | 
44 | 	return func() {
45 | 		log.Printf("Reverting stopped node %v\n", nodeToStop.GetName())
46 | 		nodeToStop.Start()
47 | 	}
48 | }
49 | 
50 | type Partition struct{}
51 | 
52 | func (action *Partition) CanApply(*e2e.Cluster) bool {
53 | 	return true
54 | }
55 | 
56 | func (action *Partition) Apply(c *e2e.Cluster) RevertFunc {
57 | 	nodes := c.Nodes()
58 | 
59 | 	hook := transport.NewPartition(500 * time.Millisecond)
60 | 	// create 2 partition with random number of nodes
61 | 	// minority with less than quorum size nodes and majority with the rest of the nodes
62 | 	quorumSize, err := c.QuorumSize()
63 | 	if err != nil {
64 | 		log.Printf("[ERROR] %v", err)
65 | 		return nil
66 | 	}
67 | 	var minorityPartition []string
68 | 	var majorityPartition []string
69 | 	minorityPartitionSize := rand.Intn(int(quorumSize + 1)) //nolint:golint,gosec
70 | 	i := 0
71 | 	for _, n := range nodes {
72 | 		if i < minorityPartitionSize {
73 | 			minorityPartition = append(minorityPartition, n.GetName())
74 | 			i++
75 | 		} else {
76 | 			majorityPartition = append(majorityPartition, n.GetName())
77 | 		}
78 | 	}
79 | 	log.Printf("Partitions ratio %d/%d, [%v], [%v]\n", len(majorityPartition), len(minorityPartition), majorityPartition, minorityPartition)
80 | 	hook.Partition(minorityPartition, majorityPartition)
81 | 
82 | 	c.SetHook(hook)
83 | 
84 | 	return func() {
85 | 		log.Println("Reverting partitions.")
86 | 		if tHook := c.GetTransportHook(); tHook != nil {
87 | 			tHook.Reset()
88 | 		}
89 | 	}
90 | }
91 | 
92 | func getAvailableActions() []Action {
93 | 	return []Action{
94 | 		&DropNode{},
95 | 		&Partition{},
96 | 	}
97 | }
98 | 


--------------------------------------------------------------------------------
/e2e/cmd/fuzz/command.go:
--------------------------------------------------------------------------------
 1 | package fuzz
 2 | 
 3 | import (
 4 | 	"flag"
 5 | 	"fmt"
 6 | 	"math/rand"
 7 | 	"time"
 8 | 
 9 | 	"github.com/mitchellh/cli"
10 | 
11 | 	"github.com/0xPolygon/pbft-consensus/e2e/replay"
12 | )
13 | 
14 | // Command is a struct containing data for running fuzz-run command
15 | type Command struct {
16 | 	UI cli.Ui
17 | 
18 | 	numberOfNodes uint
19 | 	duration      time.Duration
20 | }
21 | 
22 | // New is the constructor of Command
23 | func New(ui cli.Ui) *Command {
24 | 	return &Command{
25 | 		UI: ui,
26 | 	}
27 | }
28 | 
29 | // Help implements the cli.Command interface
30 | func (*Command) Help() string {
31 | 	return `Command runs the fuzz runner in fuzz framework based on provided configuration (nodes count and duration).
32 | 	
33 | 	Usage: fuzz-run -nodes={numberOfNodes} -duration={duration}
34 | 	
35 | 	Options:
36 | 	
37 | 	-nodes - Count of initially started nodes
38 | 	-duration - Duration of fuzz daemon running, must be longer than 1 minute (e.g., 2m, 5m, 1h, 2h)`
39 | }
40 | 
41 | // Synopsis implements the cli.Command interface
42 | func (*Command) Synopsis() string {
43 | 	return "Starts the PolyBFT fuzz runner"
44 | }
45 | 
46 | // Run implements the cli.Command interface and runs the command
47 | func (c *Command) Run(args []string) int {
48 | 	flagSet := c.NewFlagSet()
49 | 	err := flagSet.Parse(args)
50 | 	if err != nil {
51 | 		c.UI.Error(err.Error())
52 | 		return 1
53 | 	}
54 | 
55 | 	c.UI.Info("Starting PolyBFT fuzz runner...")
56 | 	c.UI.Info(fmt.Sprintf("Node count: %v\n", c.numberOfNodes))
57 | 	c.UI.Info(fmt.Sprintf("Duration: %v\n", c.duration))
58 | 
59 | 	rand.Seed(time.Now().Unix())
60 | 
61 | 	replayMessageHandler := replay.NewMessagesMiddlewareWithPersister()
62 | 
63 | 	rnnr := newRunner(c.numberOfNodes, replayMessageHandler)
64 | 	rnnr.run(c.duration)
65 | 	c.UI.Info("PolyBFT fuzz runner is stopped.")
66 | 
67 | 	if err = replayMessageHandler.CloseFile(); err != nil {
68 | 		c.UI.Error(fmt.Sprintf("Error while closing .flow file: '%s'\n", err))
69 | 		return 1
70 | 	}
71 | 
72 | 	return 0
73 | }
74 | 
75 | // NewFlagSet implements the FuzzCLICommand interface and creates a new flag set for command arguments
76 | func (c *Command) NewFlagSet() *flag.FlagSet {
77 | 	flagSet := flag.NewFlagSet("fuzz-run", flag.ContinueOnError)
78 | 	flagSet.UintVar(&c.numberOfNodes, "nodes", 5, "Count of initially started nodes")
79 | 	flagSet.DurationVar(&c.duration, "duration", 25*time.Minute, "Duration of fuzz daemon running")
80 | 
81 | 	return flagSet
82 | }
83 | 


--------------------------------------------------------------------------------
/e2e/cmd/fuzz/runner.go:
--------------------------------------------------------------------------------
  1 | package fuzz
  2 | 
  3 | import (
  4 | 	"log"
  5 | 	"math/rand"
  6 | 	"sync"
  7 | 	"time"
  8 | 
  9 | 	"github.com/0xPolygon/pbft-consensus/e2e"
 10 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
 11 | 	"github.com/0xPolygon/pbft-consensus/e2e/notifier"
 12 | )
 13 | 
 14 | var (
 15 | 	revertProbabilityThreshold = 20
 16 | 	waitForHeightTimeInterval  = 10 * time.Minute
 17 | )
 18 | 
 19 | type runner struct {
 20 | 	wg               sync.WaitGroup
 21 | 	cluster          *e2e.Cluster
 22 | 	availableActions []Action
 23 | }
 24 | 
 25 | // newRunner is the constructor of runner
 26 | func newRunner(initialNodesCount uint, replayMessageNotifier notifier.Notifier) *runner {
 27 | 	config := &e2e.ClusterConfig{
 28 | 		Count:                 int(initialNodesCount),
 29 | 		Name:                  "fuzz_cluster",
 30 | 		Prefix:                "NODE",
 31 | 		ReplayMessageNotifier: replayMessageNotifier,
 32 | 	}
 33 | 
 34 | 	return &runner{
 35 | 		availableActions: getAvailableActions(),
 36 | 		cluster:          e2e.NewPBFTCluster(nil, config),
 37 | 	}
 38 | }
 39 | 
 40 | func (r *runner) run(d time.Duration) {
 41 | 	r.cluster.Start()
 42 | 	defer r.cluster.Stop()
 43 | 	done := time.After(d)
 44 | 
 45 | 	// TODO: Randomize time interval?
 46 | 	applyTicker := time.NewTicker(5 * time.Second)
 47 | 	revertTicker := time.NewTicker(3 * time.Second)
 48 | 	validationTicker := time.NewTicker(1 * time.Minute)
 49 | 	defer applyTicker.Stop()
 50 | 	defer revertTicker.Stop()
 51 | 	defer validationTicker.Stop()
 52 | 
 53 | 	var reverts []RevertFunc
 54 | 
 55 | 	r.wg.Add(1)
 56 | 	go func() {
 57 | 		defer r.wg.Done()
 58 | 		for {
 59 | 			select {
 60 | 			case <-done:
 61 | 				log.Println("Done with execution")
 62 | 				return
 63 | 
 64 | 			case <-applyTicker.C:
 65 | 				log.Printf("[RUNNER] Applying action.")
 66 | 				actionIndex := rand.Intn(len(r.availableActions)) //nolint:golint,gosec
 67 | 				action := r.availableActions[actionIndex]
 68 | 				if action.CanApply(r.cluster) {
 69 | 					revertFn := action.Apply(r.cluster)
 70 | 					reverts = append(reverts, revertFn)
 71 | 				}
 72 | 
 73 | 			case <-revertTicker.C:
 74 | 				log.Printf("[RUNNER] Reverting action. %d revert actions available", len(reverts))
 75 | 				if len(reverts) == 0 {
 76 | 					continue
 77 | 				}
 78 | 
 79 | 				if helper.ShouldApply(revertProbabilityThreshold) {
 80 | 					revertIndex := rand.Intn(len(reverts)) //nolint:golint,gosec
 81 | 					revertFn := reverts[revertIndex]
 82 | 					reverts = append(reverts[:revertIndex], reverts[revertIndex+1:]...)
 83 | 					revertFn()
 84 | 				}
 85 | 
 86 | 			case <-validationTicker.C:
 87 | 				log.Printf("[RUNNER] Validating nodes")
 88 | 				validateNodes(r.cluster)
 89 | 			}
 90 | 		}
 91 | 	}()
 92 | 
 93 | 	r.wg.Wait()
 94 | }
 95 | 
 96 | // validateNodes checks if there is progress on the node height after the scenario run
 97 | func validateNodes(c *e2e.Cluster) {
 98 | 	if runningNodes, ok := validateCluster(c); ok {
 99 | 		currentHeight := c.GetMaxHeight(runningNodes)
100 | 		expectedHeight := currentHeight + 10
101 | 		log.Printf("Running nodes: %v. Current height: %v and waiting expected: %v height.\n", runningNodes, currentHeight, expectedHeight)
102 | 		err := c.WaitForHeight(expectedHeight, waitForHeightTimeInterval, runningNodes)
103 | 		if err != nil {
104 | 			transportHook := c.GetTransportHook()
105 | 			if transportHook != nil {
106 | 				log.Printf("Cluster partitions: %v\n", transportHook.GetPartitions())
107 | 			}
108 | 			for _, n := range c.Nodes() {
109 | 				log.Printf("Node: %v, running: %v, locked: %v, height: %v, proposal: %v\n", n.GetName(), n.IsRunning(), n.IsLocked(), n.GetNodeHeight(), n.GetProposal())
110 | 			}
111 | 			panic("Desired height not reached.")
112 | 		}
113 | 		log.Println("Cluster validation done.")
114 | 	} else {
115 | 		log.Println("Skipping validation, not enough running nodes for consensus.")
116 | 	}
117 | }
118 | 
119 | // validateCluster checks if there is enough running nodes that can make consensus
120 | func validateCluster(c *e2e.Cluster) ([]string, bool) {
121 | 	var runningNodes []string
122 | 	var partitions map[string][]string
123 | 	// running nodes in majority partition
124 | 	hook := c.GetTransportHook()
125 | 	if hook != nil {
126 | 		partitions = hook.GetPartitions()
127 | 	}
128 | 
129 | 	var majorityPartition []string
130 | 	if len(partitions) == 0 {
131 | 		// there are no partitions
132 | 		for _, n := range c.GetRunningNodes() {
133 | 			majorityPartition = append(majorityPartition, n.GetName())
134 | 		}
135 | 	} else {
136 | 		// get partition with the majority of nodes
137 | 		// all subsets are the same
138 | 		for _, p := range partitions {
139 | 			if len(p) > len(majorityPartition) {
140 | 				majorityPartition = p
141 | 			}
142 | 		}
143 | 	}
144 | 
145 | 	// loop through running nodes and check if they are in majority partition
146 | 	for _, n := range c.GetRunningNodes() {
147 | 		if helper.Contains(majorityPartition, n.GetName()) {
148 | 			runningNodes = append(runningNodes, n.GetName())
149 | 		}
150 | 	}
151 | 	quorumSize, err := c.QuorumSize()
152 | 	var enoughRunningNodes bool
153 | 	if err != nil {
154 | 		enoughRunningNodes = false
155 | 		log.Printf("[ERROR] failed to validate cluster. Error: %v", err)
156 | 	} else {
157 | 		enoughRunningNodes = len(runningNodes) >= int(quorumSize)
158 | 	}
159 | 	return runningNodes, enoughRunningNodes
160 | }
161 | 


--------------------------------------------------------------------------------
/e2e/cmd/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"log"
 5 | 	"os"
 6 | 
 7 | 	"github.com/mitchellh/cli"
 8 | 
 9 | 	"github.com/0xPolygon/pbft-consensus/e2e/cmd/fuzz"
10 | 	"github.com/0xPolygon/pbft-consensus/e2e/cmd/replay"
11 | )
12 | 
13 | func main() {
14 | 	commands := getCommands()
15 | 
16 | 	cli := &cli.CLI{
17 | 		Name:     "fuzz",
18 | 		Args:     os.Args[1:],
19 | 		Commands: commands,
20 | 	}
21 | 
22 | 	_, err := cli.Run()
23 | 	if err != nil {
24 | 		log.Fatalf("Error executing CLI: %s\n", err.Error())
25 | 	}
26 | }
27 | 
28 | // getCommands returns all registered commands
29 | func getCommands() map[string]cli.CommandFactory {
30 | 	ui := &cli.BasicUi{
31 | 		Reader:      os.Stdin,
32 | 		Writer:      os.Stdout,
33 | 		ErrorWriter: os.Stderr,
34 | 	}
35 | 	return map[string]cli.CommandFactory{
36 | 		"fuzz-run": func() (cli.Command, error) {
37 | 			return fuzz.New(ui), nil
38 | 		},
39 | 		"replay-messages": func() (cli.Command, error) {
40 | 			return replay.New(ui), nil
41 | 		},
42 | 	}
43 | }
44 | 


--------------------------------------------------------------------------------
/e2e/cmd/replay/backend.go:
--------------------------------------------------------------------------------
 1 | package replay
 2 | 
 3 | import (
 4 | 	"log"
 5 | 	"time"
 6 | 
 7 | 	"github.com/0xPolygon/pbft-consensus"
 8 | 	"github.com/0xPolygon/pbft-consensus/e2e"
 9 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
10 | 	"github.com/0xPolygon/pbft-consensus/e2e/replay"
11 | )
12 | 
13 | // backend implements the e2e.IntegrationBackend interface and implements its own pbft.BuildProposal method for replay
14 | type backend struct {
15 | 	e2e.BackendFake
16 | 	messageReader *replay.MessageReader
17 | }
18 | 
19 | // newBackend is the constructor of Backend
20 | func newBackend(messageReader *replay.MessageReader) *backend {
21 | 	return &backend{
22 | 		messageReader: messageReader,
23 | 	}
24 | }
25 | 
26 | // BuildProposal builds the next proposal. If it has a preprepare message for given height in .flow file it will take the proposal from file, otherwise it will generate a new one
27 | func (f *backend) BuildProposal() (*pbft.Proposal, error) {
28 | 	var data []byte
29 | 	sequence := f.Height()
30 | 	if prePrepareMessage, exists := f.messageReader.GetPrePrepareMessages(sequence); exists && prePrepareMessage != nil {
31 | 		data = prePrepareMessage.Proposal
32 | 	} else {
33 | 		log.Printf("[WARNING] Could not find PRE-PREPARE message for sequence: %v", sequence)
34 | 		data = helper.GenerateProposal()
35 | 	}
36 | 
37 | 	return &pbft.Proposal{
38 | 		Data: data,
39 | 		Time: time.Now().Add(1 * time.Second),
40 | 		Hash: helper.Hash(data),
41 | 	}, nil
42 | }
43 | 


--------------------------------------------------------------------------------
/e2e/cmd/replay/command.go:
--------------------------------------------------------------------------------
  1 | package replay
  2 | 
  3 | import (
  4 | 	"errors"
  5 | 	"flag"
  6 | 	"fmt"
  7 | 	"strings"
  8 | 	"sync"
  9 | 	"time"
 10 | 
 11 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
 12 | 
 13 | 	"github.com/mitchellh/cli"
 14 | 
 15 | 	"github.com/0xPolygon/pbft-consensus"
 16 | 	"github.com/0xPolygon/pbft-consensus/e2e"
 17 | 	"github.com/0xPolygon/pbft-consensus/e2e/replay"
 18 | )
 19 | 
 20 | // Command is a struct containing data for running replay-message command
 21 | type Command struct {
 22 | 	UI cli.Ui
 23 | 
 24 | 	filePath string
 25 | }
 26 | 
 27 | // New is the constructor of Command
 28 | func New(ui cli.Ui) *Command {
 29 | 	return &Command{
 30 | 		UI: ui,
 31 | 	}
 32 | }
 33 | 
 34 | // Help implements the cli.Command interface
 35 | func (c *Command) Help() string {
 36 | 	return `Runs the message and timeouts replay for analysis and testing purposes based on provided .flow file.
 37 | 	
 38 | 	Usage: replay-messages -file={fullPathToFlowFile}
 39 | 	
 40 | 	Options:
 41 | 	
 42 | 	-file - Full path to .flow file containing messages and timeouts to be replayed by the fuzz framework`
 43 | }
 44 | 
 45 | // Synopsis implements the cli.Command interface
 46 | func (c *Command) Synopsis() string {
 47 | 	return "Starts the replay of messages and timeouts in fuzz runner"
 48 | }
 49 | 
 50 | // Run implements the cli.Command interface and runs the command
 51 | func (c *Command) Run(args []string) int {
 52 | 	err := c.validateInput(args)
 53 | 	if err != nil {
 54 | 		c.UI.Error(err.Error())
 55 | 		return 1
 56 | 	}
 57 | 
 58 | 	messageReader := replay.NewMessageReader()
 59 | 
 60 | 	err = messageReader.OpenFile(c.filePath)
 61 | 	if err != nil {
 62 | 		c.UI.Error(err.Error())
 63 | 		return 1
 64 | 	}
 65 | 
 66 | 	nodeNames, err := messageReader.ReadNodeMetaData()
 67 | 	if err != nil {
 68 | 		c.UI.Error(err.Error())
 69 | 		return 1
 70 | 	}
 71 | 
 72 | 	i := strings.Index(nodeNames[0], "_")
 73 | 	prefix := ""
 74 | 	if i > -1 {
 75 | 		prefix = (nodeNames[0])[:i]
 76 | 	}
 77 | 
 78 | 	replayMessagesNotifier := replay.NewMessagesMiddlewareWithReader(messageReader)
 79 | 
 80 | 	nodesCount := len(nodeNames)
 81 | 	config := &e2e.ClusterConfig{
 82 | 		Count:                 nodesCount,
 83 | 		Name:                  "fuzz_cluster",
 84 | 		Prefix:                prefix,
 85 | 		ReplayMessageNotifier: replayMessagesNotifier,
 86 | 		RoundTimeout:          helper.GetPredefinedTimeout(time.Millisecond),
 87 | 		TransportHandler:      func(to pbft.NodeID, msg *pbft.MessageReq) { replayMessagesNotifier.HandleMessage(to, msg) },
 88 | 		CreateBackend: func() e2e.IntegrationBackend {
 89 | 			return newBackend(messageReader)
 90 | 		},
 91 | 	}
 92 | 
 93 | 	cluster := e2e.NewPBFTCluster(nil, config)
 94 | 
 95 | 	nodes := make(map[string]replay.Node)
 96 | 	for name, node := range cluster.GetNodesMap() {
 97 | 		nodes[name] = node
 98 | 	}
 99 | 
100 | 	messageReader.ReadMessages(nodes)
101 | 	err = messageReader.CloseFile()
102 | 	if err != nil {
103 | 		c.UI.Error(err.Error())
104 | 		return 1
105 | 	}
106 | 
107 | 	var wg sync.WaitGroup
108 | 	wg.Add(1)
109 | 
110 | 	nodesDone := make(map[string]bool, nodesCount)
111 | 	go func() {
112 | 		doneChan := messageReader.ProcessingDone()
113 | 		for {
114 | 			nodeDone := <-doneChan
115 | 			nodesDone[nodeDone] = true
116 | 			cluster.StopNode(nodeDone)
117 | 			if len(nodesDone) == nodesCount {
118 | 				wg.Done()
119 | 				return
120 | 			}
121 | 		}
122 | 	}()
123 | 
124 | 	cluster.Start()
125 | 	wg.Wait()
126 | 	cluster.Stop()
127 | 
128 | 	c.UI.Info("Done with execution")
129 | 	if err = replayMessagesNotifier.CloseFile(); err != nil {
130 | 		c.UI.Error(fmt.Sprintf("Error while closing .flow file: '%s'\n", err))
131 | 		return 1
132 | 	}
133 | 
134 | 	return 0
135 | }
136 | 
137 | // NewFlagSet implements the FuzzCLICommand interface and creates a new flag set for command arguments
138 | func (c *Command) NewFlagSet() *flag.FlagSet {
139 | 	flagSet := flag.NewFlagSet("replay-messages", flag.ContinueOnError)
140 | 	flagSet.StringVar(&c.filePath, "file", "", "Full path to .flow file containing messages and timeouts to be replayed by the fuzz framework")
141 | 
142 | 	return flagSet
143 | }
144 | 
145 | // validateInput parses arguments from CLI and validates their correctness
146 | func (c *Command) validateInput(args []string) error {
147 | 	flagSet := c.NewFlagSet()
148 | 	err := flagSet.Parse(args)
149 | 	if err != nil {
150 | 		return err
151 | 	}
152 | 
153 | 	if c.filePath == "" {
154 | 		err = errors.New("provided file path is empty")
155 | 		return err
156 | 	}
157 | 	return nil
158 | }
159 | 


--------------------------------------------------------------------------------
/e2e/e2e_node_drop_test.go:
--------------------------------------------------------------------------------
 1 | package e2e
 2 | 
 3 | import (
 4 | 	"testing"
 5 | 	"time"
 6 | 
 7 | 	"github.com/stretchr/testify/assert"
 8 | 
 9 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
10 | )
11 | 
12 | func TestE2E_NodeDrop(t *testing.T) {
13 | 	t.Parallel()
14 | 	config := &ClusterConfig{
15 | 		Count:        5,
16 | 		Name:         "node_drop",
17 | 		Prefix:       "ptr",
18 | 		RoundTimeout: helper.GetPredefinedTimeout(2 * time.Second),
19 | 	}
20 | 
21 | 	c := NewPBFTCluster(t, config)
22 | 	c.Start()
23 | 	// wait for two heights and stop node 1
24 | 	err := c.WaitForHeight(2, 3*time.Second)
25 | 	assert.NoError(t, err)
26 | 
27 | 	c.StopNode("ptr_0")
28 | 	err = c.WaitForHeight(10, 15*time.Second, helper.GenerateNodeNames(1, 4, "ptr_"))
29 | 	assert.NoError(t, err)
30 | 
31 | 	// sync dropped node by starting it again
32 | 	c.startNode("ptr_0")
33 | 	err = c.WaitForHeight(10, 15*time.Second)
34 | 	assert.NoError(t, err)
35 | 
36 | 	c.Stop()
37 | }
38 | 
39 | func TestE2E_BulkNodeDrop(t *testing.T) {
40 | 	t.Parallel()
41 | 
42 | 	clusterCount := 5
43 | 	bulkToDrop := 3
44 | 
45 | 	conf := &ClusterConfig{
46 | 		Count:        clusterCount,
47 | 		Name:         "bulk_node_drop",
48 | 		Prefix:       "ptr",
49 | 		RoundTimeout: helper.GetPredefinedTimeout(2 * time.Second),
50 | 	}
51 | 	c := NewPBFTCluster(t, conf)
52 | 	c.Start()
53 | 	err := c.WaitForHeight(2, 3*time.Second)
54 | 	assert.NoError(t, err)
55 | 
56 | 	// drop bulk of nodes from cluster
57 | 	dropNodes := helper.GenerateNodeNames(bulkToDrop, clusterCount-1, "ptr_")
58 | 	for _, node := range dropNodes {
59 | 		c.StopNode(node)
60 | 	}
61 | 	c.isStuck(15*time.Second, dropNodes)
62 | 
63 | 	// restart dropped nodes
64 | 	for _, node := range dropNodes {
65 | 		c.startNode(node)
66 | 	}
67 | 	err = c.WaitForHeight(5, 15*time.Second)
68 | 	assert.NoError(t, err)
69 | 
70 | 	c.Stop()
71 | }
72 | 


--------------------------------------------------------------------------------
/e2e/e2e_noissue_test.go:
--------------------------------------------------------------------------------
 1 | package e2e
 2 | 
 3 | import (
 4 | 	"testing"
 5 | 	"time"
 6 | 
 7 | 	"github.com/stretchr/testify/assert"
 8 | 
 9 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
10 | 	"github.com/0xPolygon/pbft-consensus/e2e/transport"
11 | )
12 | 
13 | func TestE2E_NoIssue(t *testing.T) {
14 | 	t.Parallel()
15 | 	config := &ClusterConfig{
16 | 		Count:        5,
17 | 		Name:         "noissue",
18 | 		Prefix:       "noissue",
19 | 		RoundTimeout: helper.GetPredefinedTimeout(2 * time.Second),
20 | 	}
21 | 
22 | 	c := NewPBFTCluster(t, config, transport.NewRandom(50*time.Millisecond))
23 | 	c.Start()
24 | 	defer c.Stop()
25 | 
26 | 	err := c.WaitForHeight(10, 1*time.Minute)
27 | 	assert.NoError(t, err)
28 | }
29 | 


--------------------------------------------------------------------------------
/e2e/e2e_partition_test.go:
--------------------------------------------------------------------------------
  1 | package e2e
  2 | 
  3 | import (
  4 | 	"math"
  5 | 	"strconv"
  6 | 	"testing"
  7 | 	"time"
  8 | 
  9 | 	"github.com/stretchr/testify/assert"
 10 | 
 11 | 	"github.com/0xPolygon/pbft-consensus"
 12 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
 13 | 	"github.com/0xPolygon/pbft-consensus/e2e/transport"
 14 | )
 15 | 
 16 | // Test proves existence of liveness issues which is described in
 17 | // Correctness Analysis of Istanbul Byzantine Fault Tolerance (https://arxiv.org/pdf/1901.07160.pdf).
 18 | // Specific problem this test is emulating is described in Chapter 7.1, Case 1.
 19 | // Summary of the problem is that there are not enough nodes to lock on single proposal,
 20 | // due to some issues where nodes being unable to deliver messages to all of the peers.
 21 | // Therefore nodes are split into two subsets locked on different proposals.
 22 | // Simulating one node from larger subset as faulty one, results in being unable to reach a consensus on a single proposal and that's what liveness issue is about.
 23 | // This test creates a single cluster of 5 nodes, but instead of letting all the peers communicate with each other,
 24 | // it routes messages only to specific nodes and induces that nodes lock on different proposal.
 25 | // In the round=1, it marks one node as faulty (meaning that it doesn't takes part in gossiping).
 26 | func TestE2E_Partition_LivenessIssue_Case1_FiveNodes_OneFaulty(t *testing.T) {
 27 | 	t.Parallel()
 28 | 
 29 | 	round0 := transport.RoundMetadata{
 30 | 		Round: 0,
 31 | 		// induce locking A_3 and A_4 on one proposal
 32 | 		RoutingMap: map[transport.Sender]transport.Receivers{
 33 | 			"A_0": {"A_3", "A_4"},
 34 | 			"A_3": {"A_0", "A_3", "A_4"},
 35 | 			"A_4": {"A_3", "A_4"},
 36 | 		},
 37 | 	}
 38 | 	round1 := transport.RoundMetadata{
 39 | 		Round: 1,
 40 | 		// induce locking lock A_0 and A_2 on another proposal
 41 | 		RoutingMap: map[transport.Sender]transport.Receivers{
 42 | 			"A_0": {"A_0", "A_2", "A_3", "A_4"},
 43 | 			"A_1": {"A_0", "A_2", "A_3", "A_4"},
 44 | 			"A_2": {"A_0", "A_1", "A_2", "A_3", "A_4"},
 45 | 
 46 | 			"A_3": {"A_0", "A_1", "A_2", "A_3", "A_4"},
 47 | 			"A_4": {"A_0", "A_1", "A_2", "A_3", "A_4"},
 48 | 		},
 49 | 	}
 50 | 	flowMap := map[uint64]transport.RoundMetadata{0: round0, 1: round1}
 51 | 
 52 | 	faultyNodeId := pbft.NodeID("A_1")
 53 | 
 54 | 	transport := transport.NewGenericGossip()
 55 | 	// If livenessGossipHandler returns false, message should not be transported.
 56 | 	livenessGossipHandler := func(senderId, receiverId pbft.NodeID, msg *pbft.MessageReq) bool {
 57 | 		if msg.View.Round > 1 || msg.View.Sequence > 2 {
 58 | 			// Faulty node is unresponsive after round 1, and all the other nodes are gossiping all the messages.
 59 | 			return senderId != faultyNodeId && receiverId != faultyNodeId
 60 | 		} else {
 61 | 			if msg.View.Round <= 1 && msg.Type == pbft.MessageReq_Commit {
 62 | 				// Cut all the commit messages gossiping for round 0 and 1
 63 | 				return false
 64 | 			}
 65 | 			if msg.View.Round == 1 && senderId == faultyNodeId &&
 66 | 				(msg.Type == pbft.MessageReq_RoundChange || msg.Type == pbft.MessageReq_Commit) {
 67 | 				// Case where we are in round 1 and 2 different nodes will lock the proposal
 68 | 				// (consequence of faulty node doesn't gossip round change and commit messages).
 69 | 				return false
 70 | 			}
 71 | 		}
 72 | 
 73 | 		return transport.ShouldGossipBasedOnMsgFlowMap(msg, senderId, receiverId)
 74 | 	}
 75 | 
 76 | 	transport.WithFlowMap(flowMap).WithGossipHandler(livenessGossipHandler)
 77 | 
 78 | 	config := &ClusterConfig{
 79 | 		Count:  5,
 80 | 		Name:   "liveness_issue",
 81 | 		Prefix: "A",
 82 | 	}
 83 | 
 84 | 	c := NewPBFTCluster(t, config, transport)
 85 | 	c.Start()
 86 | 	defer c.Stop()
 87 | 
 88 | 	err := c.WaitForHeight(3, 1*time.Minute, []string{"A_0", "A_2", "A_3", "A_4"})
 89 | 
 90 | 	if err != nil {
 91 | 		// log to check what is the end state
 92 | 		for _, n := range c.nodes {
 93 | 			proposal := n.pbft.GetProposal()
 94 | 			if proposal != nil {
 95 | 				t.Logf("Node %v, isProposalLocked: %v, proposal data: %v\n", n.name, n.pbft.IsLocked(), proposal.Data)
 96 | 			} else {
 97 | 				t.Logf("Node %v, isProposalLocked: %v, no proposal set\n", n.name, n.pbft.IsLocked())
 98 | 			}
 99 | 		}
100 | 	}
101 | 
102 | 	// TODO: Temporary assertion until liveness issue is resolved (after fix is merged we need to revert back to assert.NoError(t, err))
103 | 	assert.Error(t, err)
104 | }
105 | 
106 | // Test proves existence of liveness issues which is described in
107 | // Correctness Analysis of Istanbul Byzantine Fault Tolerance(https://arxiv.org/pdf/1901.07160.pdf).
108 | // Specific problem this test is emulating is described in Chapter 7.1, Case 2.
109 | // Summary of the problem is that there are not enough nodes to lock on single proposal,
110 | // due to some issues where nodes being unable to deliver messages to all of the peers.
111 | // When there are (nh) = 6 nodes, and they are split into three subsets, all locked on different proposals, where one of the nodes becomes faulty,
112 | // nodes are unable to reach a consensus on a single proposal, resulting in continuous RoundChange.
113 | // This test creates a single cluster of 6 nodes, but instead of letting all the peers communicate with each other,
114 | // it routes messages only to specific nodes and induces that nodes lock on different proposal.
115 | // In the round=3, it marks one node as faulty (meaning that it doesn't takes part in gossiping).
116 | func TestE2E_Partition_LivenessIssue_Case2_SixNodes_OneFaulty(t *testing.T) {
117 | 	t.Parallel()
118 | 
119 | 	round0 := transport.RoundMetadata{
120 | 		Round: 0,
121 | 		// lock A_1, A_4
122 | 		RoutingMap: map[transport.Sender]transport.Receivers{
123 | 			"A_0": {"A_1", "A_3", "A_4"},
124 | 			"A_3": {"A_1", "A_3", "A_4"},
125 | 			"A_4": {"A_1", "A_4"},
126 | 		},
127 | 	}
128 | 
129 | 	round2 := transport.RoundMetadata{
130 | 		Round: 2,
131 | 		// lock A_5
132 | 		RoutingMap: map[transport.Sender]transport.Receivers{
133 | 			"A_0": {"A_5", "A_2", "A_4"},
134 | 			"A_1": {"A_5", "A_0"},
135 | 			"A_2": {"A_5", "A_3"},
136 | 			"A_3": {"A_5"},
137 | 			"A_4": {"A_5"},
138 | 		},
139 | 	}
140 | 
141 | 	round3 := transport.RoundMetadata{
142 | 		Round: 3,
143 | 		// lock A_3 and A_0 on one proposal and A_2 will be faulty
144 | 		RoutingMap: map[transport.Sender]transport.Receivers{
145 | 			"A_3": {"A_0", "A_2", "A_3", "A_4"},
146 | 			"A_0": {"A_0", "A_3", "A_4"},
147 | 			"A_2": {"A_0", "A_1", "A_3", "A_4"},
148 | 		},
149 | 	}
150 | 	flowMap := map[uint64]transport.RoundMetadata{0: round0, 2: round2, 3: round3}
151 | 	transport := transport.NewGenericGossip()
152 | 	faultyNodeId := pbft.NodeID("A_2")
153 | 
154 | 	// If livenessGossipHandler returns false, message should not be transported.
155 | 	livenessGossipHandler := func(senderId, receiverId pbft.NodeID, msg *pbft.MessageReq) (sent bool) {
156 | 		if msg.View.Round == 1 && msg.Type == pbft.MessageReq_RoundChange {
157 | 			return true
158 | 		}
159 | 
160 | 		if msg.View.Round > 3 || msg.View.Sequence > 2 {
161 | 			// Faulty node is unresponsive after round 3, and all the other nodes are gossiping all the messages.
162 | 			return senderId != faultyNodeId && receiverId != faultyNodeId
163 | 		} else {
164 | 			if msg.View.Round <= 1 && msg.Type == pbft.MessageReq_Commit {
165 | 				// Cut all the commit messages gossiping for round 0 and 1
166 | 				return false
167 | 			}
168 | 			if msg.View.Round == 3 && senderId == faultyNodeId &&
169 | 				(msg.Type == pbft.MessageReq_RoundChange || msg.Type == pbft.MessageReq_Commit) {
170 | 				// Case where we are in round 3 and 2 different nodes will lock the proposal
171 | 				// (consequence of faulty node doesn't gossip round change and commit messages).
172 | 				return false
173 | 			}
174 | 		}
175 | 
176 | 		return transport.ShouldGossipBasedOnMsgFlowMap(msg, senderId, receiverId)
177 | 	}
178 | 
179 | 	transport.WithFlowMap(flowMap).WithGossipHandler(livenessGossipHandler)
180 | 
181 | 	config := &ClusterConfig{
182 | 		Count:  6,
183 | 		Name:   "liveness_issue",
184 | 		Prefix: "A",
185 | 	}
186 | 
187 | 	c := NewPBFTCluster(t, config, transport)
188 | 	c.Start()
189 | 	defer c.Stop()
190 | 
191 | 	err := c.WaitForHeight(3, 1*time.Minute, []string{"A_0", "A_1", "A_3", "A_4", "A_5"})
192 | 
193 | 	if err != nil {
194 | 		// log to check what is the end state
195 | 		for _, n := range c.nodes {
196 | 			proposal := n.pbft.GetProposal()
197 | 			if proposal != nil {
198 | 				t.Logf("Node %v, isProposalLocked: %v, proposal data: %v\n", n.name, n.pbft.IsLocked(), proposal.Data)
199 | 			} else {
200 | 				t.Logf("Node %v, isProposalLocked: %v, no proposal set\n", n.name, n.pbft.IsLocked())
201 | 			}
202 | 		}
203 | 	}
204 | 
205 | 	// TODO: Temporary assertion until liveness issue is resolved (after fix is merged we need to revert back to assert.NoError(t, err))
206 | 	assert.Error(t, err)
207 | }
208 | 
209 | // TestE2E_Network_Stuck_Locked_Node_Dropped is a test that creates a situation with no consensus
210 | // one node gets dropped from a network once the proposal gets locked (A_3)
211 | // two nodes are locked on the same proposal (A_0 and A_1)
212 | // and one node is not locked (A_2).
213 | func TestE2E_Network_Stuck_Locked_Node_Dropped(t *testing.T) {
214 | 	t.Parallel()
215 | 
216 | 	round0 := transport.RoundMetadata{
217 | 		Round: 0,
218 | 		RoutingMap: map[transport.Sender]transport.Receivers{
219 | 			"A_0": {"A_0", "A_1", "A_3", "A_2"},
220 | 			"A_1": {"A_0", "A_1", "A_3"},
221 | 			"A_2": {"A_0", "A_1", "A_2", "A_3"},
222 | 		},
223 | 	}
224 | 	flowMap := map[uint64]transport.RoundMetadata{0: round0}
225 | 	transport := transport.NewGenericGossip()
226 | 
227 | 	config := &ClusterConfig{
228 | 		Count:        4,
229 | 		Name:         "liveness_issue",
230 | 		Prefix:       "A",
231 | 		RoundTimeout: helper.GetPredefinedTimeout(2 * time.Second),
232 | 	}
233 | 
234 | 	c := NewPBFTCluster(t, config, transport)
235 | 	node3 := c.nodes["A_3"]
236 | 	// If livenessGossipHandler returns false, message should not be transported.
237 | 	gossipHandler := func(senderId, receiverId pbft.NodeID, msg *pbft.MessageReq) (sent bool) {
238 | 
239 | 		// all nodes are connected if sequence is > 1 or round > 0 for sequence 1
240 | 		if msg.View.Sequence > 1 || (msg.View.Sequence == 1 && msg.View.Round > 0) {
241 | 			return true
242 | 		}
243 | 		// stop node A_3 once it is locked
244 | 		if node3.IsRunning() && node3.IsLocked() {
245 | 			node3.Stop()
246 | 		}
247 | 		return transport.ShouldGossipBasedOnMsgFlowMap(msg, senderId, receiverId)
248 | 	}
249 | 
250 | 	transport.WithFlowMap(flowMap).WithGossipHandler(gossipHandler)
251 | 
252 | 	c.Start()
253 | 	defer c.Stop()
254 | 
255 | 	err := c.WaitForHeight(3, 1*time.Minute, []string{"A_0", "A_1", "A_2"})
256 | 
257 | 	// log to check what is the end state
258 | 	for _, n := range c.nodes {
259 | 		proposal := n.pbft.GetProposal()
260 | 		if proposal != nil {
261 | 			t.Logf("Node %v, running: %v, isProposalLocked: %v, proposal data: %v\n", n.name, n.IsRunning(), n.pbft.IsLocked(), proposal)
262 | 		} else {
263 | 			t.Logf("Node %v, running: %v, isProposalLocked: %v, no proposal set\n", n.name, n.IsRunning(), n.pbft.IsLocked())
264 | 		}
265 | 	}
266 | 	// TODO: Temporary assertion until liveness issue is fixed
267 | 	assert.Error(t, err)
268 | }
269 | 
270 | func TestE2E_Partition_OneMajority(t *testing.T) {
271 | 	t.Parallel()
272 | 	const nodesCnt = 5
273 | 	hook := transport.NewPartition(50 * time.Millisecond)
274 | 
275 | 	config := &ClusterConfig{
276 | 		Count:        nodesCnt,
277 | 		Name:         "majority_partition",
278 | 		Prefix:       "prt",
279 | 		RoundTimeout: helper.GetPredefinedTimeout(2 * time.Second),
280 | 	}
281 | 
282 | 	c := NewPBFTCluster(t, config, hook)
283 | 	c.Start()
284 | 	defer c.Stop()
285 | 
286 | 	err := c.WaitForHeight(5, 1*time.Minute)
287 | 	assert.NoError(t, err)
288 | 
289 | 	// create two partitions.
290 | 	majorityPartition := []string{"prt_0", "prt_1", "prt_2"}
291 | 	minorityPartition := []string{"prt_3", "prt_4"}
292 | 	hook.Partition(majorityPartition, minorityPartition)
293 | 
294 | 	// only the majority partition will be able to sync
295 | 	err = c.WaitForHeight(10, 1*time.Minute, majorityPartition)
296 | 	assert.NoError(t, err)
297 | 
298 | 	// the partition with two nodes is stuck
299 | 	c.isStuck(10*time.Second, minorityPartition)
300 | 
301 | 	// reset all partitions
302 | 	hook.Reset()
303 | 
304 | 	allNodes := make([]string, len(c.nodes))
305 | 	for i, node := range c.Nodes() {
306 | 		allNodes[i] = node.name
307 | 	}
308 | 	// all nodes should be able to sync
309 | 	err = c.WaitForHeight(15, 1*time.Minute, allNodes)
310 | 	assert.NoError(t, err)
311 | }
312 | 
313 | func TestE2E_Partition_MajorityCanValidate(t *testing.T) {
314 | 	t.Parallel()
315 | 	const nodesCnt = 7 // N = 3 * F + 1, F = 2
316 | 	hook := transport.NewPartition(50 * time.Millisecond)
317 | 
318 | 	config := &ClusterConfig{
319 | 		Count:        nodesCnt,
320 | 		Name:         "majority_partition",
321 | 		Prefix:       "prt",
322 | 		RoundTimeout: helper.GetPredefinedTimeout(2 * time.Second),
323 | 	}
324 | 
325 | 	c := NewPBFTCluster(t, config, hook)
326 | 	limit := int(math.Floor(nodesCnt*2.0/3.0)) + 1 // 2F+1 nodes can Validate
327 | 	for _, node := range c.Nodes() {
328 | 		node.setFaultyNode(node.name >= "prt_"+strconv.Itoa(limit))
329 | 	}
330 | 	c.Start()
331 | 	defer c.Stop()
332 | 	names := helper.GenerateNodeNames(0, limit, "prt_")
333 | 	err := c.WaitForHeight(4, 1*time.Minute, names)
334 | 	assert.NoError(t, err)
335 | 
336 | 	// restart minority and wait to sync
337 | 	for _, nd := range c.Nodes() {
338 | 		if nd.name >= "prt_"+strconv.Itoa(limit) {
339 | 			nd.restart()
340 | 		}
341 | 	}
342 | 
343 | 	err = c.WaitForHeight(4, 1*time.Minute)
344 | 	assert.NoError(t, err)
345 | }
346 | 
347 | func TestE2E_Partition_MajorityCantValidate(t *testing.T) {
348 | 	t.Parallel()
349 | 	const nodesCnt = 7 // N = 3 * F + 1, F = 2
350 | 	hook := transport.NewPartition(50 * time.Millisecond)
351 | 
352 | 	config := &ClusterConfig{
353 | 		Count:        nodesCnt,
354 | 		Name:         "majority_partition",
355 | 		Prefix:       "prt",
356 | 		RoundTimeout: helper.GetPredefinedTimeout(2 * time.Second),
357 | 	}
358 | 
359 | 	c := NewPBFTCluster(t, config, hook)
360 | 	limit := int(math.Floor(nodesCnt * 2.0 / 3.0)) // + 1 removed because 2F+1 nodes is majority
361 | 	for _, node := range c.Nodes() {
362 | 		node.setFaultyNode(node.name < "prt_"+strconv.Itoa(limit))
363 | 	}
364 | 	c.Start()
365 | 	defer c.Stop()
366 | 	names := helper.GenerateNodeNames(limit, nodesCnt, "prt_")
367 | 	err := c.WaitForHeight(3, 1*time.Minute, names)
368 | 	assert.Errorf(t, err, "Height reached for minority of nodes")
369 | }
370 | 
371 | func TestE2E_Partition_BigMajorityCantValidate(t *testing.T) {
372 | 	t.Parallel()
373 | 	const nodesCnt = 100
374 | 	hook := transport.NewPartition(50 * time.Millisecond)
375 | 
376 | 	config := &ClusterConfig{
377 | 		Count:        nodesCnt,
378 | 		Name:         "majority_partition",
379 | 		Prefix:       "prt",
380 | 		RoundTimeout: helper.GetPredefinedTimeout(2 * time.Second),
381 | 	}
382 | 
383 | 	c := NewPBFTCluster(t, config, hook)
384 | 	limit := int(math.Floor(nodesCnt * 2.0 / 3.0)) // + 1 removed because 2F+1 nodes is majority
385 | 	for _, node := range c.Nodes() {
386 | 		node.setFaultyNode(node.name <= "prt_"+strconv.Itoa(limit))
387 | 	}
388 | 	c.Start()
389 | 	defer c.Stop()
390 | 	nodeNames := helper.GenerateNodeNames(limit, nodesCnt, "prt_")
391 | 	err := c.WaitForHeight(8, 1*time.Minute, nodeNames)
392 | 	assert.Errorf(t, err, "Height reached for minority of nodes")
393 | }
394 | 


--------------------------------------------------------------------------------
/e2e/framework_test.go:
--------------------------------------------------------------------------------
 1 | package e2e
 2 | 
 3 | import (
 4 | 	"testing"
 5 | 	"time"
 6 | 
 7 | 	"github.com/0xPolygon/pbft-consensus"
 8 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
 9 | 
10 | 	"github.com/stretchr/testify/assert"
11 | )
12 | 
13 | func TestE2E_ClusterInsertFinalProposal(t *testing.T) {
14 | 	clusterConfig := &ClusterConfig{
15 | 		Count:  3,
16 | 		Name:   "cluster",
17 | 		Prefix: "N",
18 | 	}
19 | 	c := NewPBFTCluster(t, clusterConfig)
20 | 
21 | 	// valid proposal => insert it
22 | 	seq1Proposal := newSealedProposal([]byte{0x1}, "N0", 1)
23 | 	err := c.insertFinalProposal(seq1Proposal)
24 | 	assert.Nil(t, err)
25 | 	assert.Len(t, c.sealedProposals, 1)
26 | 
27 | 	// invalid proposal (different proposal data on sequence previously inserted) => discard it and return an error
28 | 	seq1DiffProposal := newSealedProposal([]byte{0x3}, "N0", 1)
29 | 	err = c.insertFinalProposal(seq1DiffProposal)
30 | 	assert.NotNil(t, err)
31 | 	assert.Len(t, c.sealedProposals, 1)
32 | 
33 | 	// same proposal data on same sequence previously entered => discard it, but don't return an error
34 | 	err = c.insertFinalProposal(seq1Proposal)
35 | 	assert.Nil(t, err)
36 | 	assert.Len(t, c.sealedProposals, 1)
37 | 
38 | 	// sequence-gapped proposal => discard it and return an error
39 | 	seq5Proposal := newSealedProposal([]byte{0x5}, "N1", 5)
40 | 	err = c.insertFinalProposal(seq5Proposal)
41 | 	assert.NotNil(t, err)
42 | 	assert.Len(t, c.sealedProposals, 1)
43 | 
44 | 	// valid proposal => insert it
45 | 	seq2Proposal := newSealedProposal([]byte{0x2}, "N1", 2)
46 | 	err = c.insertFinalProposal(seq2Proposal)
47 | 	assert.NoError(t, err)
48 | 	assert.Len(t, c.sealedProposals, 2)
49 | }
50 | 
51 | func newSealedProposal(proposalData []byte, proposer pbft.NodeID, number uint64) *pbft.SealedProposal {
52 | 	proposal := &pbft.Proposal{
53 | 		Data: proposalData,
54 | 		Time: time.Now(),
55 | 	}
56 | 	proposal.Hash = helper.Hash(proposal.Data)
57 | 	return &pbft.SealedProposal{
58 | 		Proposal: proposal,
59 | 		Proposer: proposer,
60 | 		Number:   number,
61 | 	}
62 | }
63 | 


--------------------------------------------------------------------------------
/e2e/fuzz_network_churn_test.go:
--------------------------------------------------------------------------------
 1 | package e2e
 2 | 
 3 | import (
 4 | 	"math/rand"
 5 | 	"strconv"
 6 | 	"testing"
 7 | 	"time"
 8 | 
 9 | 	"github.com/stretchr/testify/assert"
10 | 
11 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
12 | )
13 | 
14 | func TestFuzz_NetworkChurn(t *testing.T) {
15 | 	helper.IsFuzzEnabled(t)
16 | 
17 | 	t.Parallel()
18 | 	rand.Seed(time.Now().Unix())
19 | 	nodeCount := 20
20 | 	maxFaulty := nodeCount/3 - 1
21 | 	const prefix = "ptr_"
22 | 	config := &ClusterConfig{
23 | 		Count:        nodeCount,
24 | 		Name:         "network_churn",
25 | 		Prefix:       "ptr",
26 | 		RoundTimeout: helper.GetPredefinedTimeout(2 * time.Second),
27 | 	}
28 | 
29 | 	c := NewPBFTCluster(t, config)
30 | 	c.Start()
31 | 	defer c.Stop()
32 | 	runningNodeCount := nodeCount
33 | 	// randomly stop nodes every 3 seconds
34 | 	helper.ExecuteInTimerAndWait(3*time.Second, 30*time.Second, func(_ time.Duration) {
35 | 		nodeNo := rand.Intn(nodeCount)
36 | 		nodeID := prefix + strconv.Itoa(nodeNo)
37 | 		node := c.nodes[nodeID]
38 | 		if node.IsRunning() && runningNodeCount > nodeCount-maxFaulty {
39 | 			// node is running
40 | 			c.StopNode(nodeID)
41 | 			runningNodeCount--
42 | 		} else if !node.IsRunning() {
43 | 			// node is not running
44 | 			c.startNode(nodeID)
45 | 			runningNodeCount++
46 | 		}
47 | 	})
48 | 
49 | 	// get all running nodes after random drops
50 | 	var runningNodes []string
51 | 	for _, v := range c.nodes {
52 | 		if v.IsRunning() {
53 | 			runningNodes = append(runningNodes, v.name)
54 | 		}
55 | 	}
56 | 	t.Log("Checking height after churn.")
57 | 	// all running nodes must have the same height
58 | 	err := c.WaitForHeight(35, 5*time.Minute, runningNodes)
59 | 	assert.NoError(t, err)
60 | 
61 | 	// start rest of the nodes
62 | 	for _, v := range c.nodes {
63 | 		if !v.IsRunning() {
64 | 			v.Start()
65 | 			runningNodes = append(runningNodes, v.name)
66 | 		}
67 | 	}
68 | 	// all nodes must sync and have same height
69 | 	t.Log("Checking height after all nodes start.")
70 | 	err = c.WaitForHeight(45, 5*time.Minute, runningNodes)
71 | 	assert.NoError(t, err)
72 | }
73 | 


--------------------------------------------------------------------------------
/e2e/fuzz_unreliable_network_test.go:
--------------------------------------------------------------------------------
  1 | package e2e
  2 | 
  3 | import (
  4 | 	"math/rand"
  5 | 	"strconv"
  6 | 	"testing"
  7 | 	"time"
  8 | 
  9 | 	"github.com/stretchr/testify/assert"
 10 | 
 11 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
 12 | 	"github.com/0xPolygon/pbft-consensus/e2e/transport"
 13 | )
 14 | 
 15 | func TestFuzz_Unreliable_Network(t *testing.T) {
 16 | 	helper.IsFuzzEnabled(t)
 17 | 
 18 | 	t.Parallel()
 19 | 	rand.Seed(time.Now().Unix())
 20 | 	nodesCount := 20 + rand.Intn(11) // vary nodes [20,30]
 21 | 	maxFaulty := nodesCount/3 - 1
 22 | 	maxHeight := uint64(40)
 23 | 	currentHeight := uint64(0)
 24 | 	jitterMax := 300 * time.Millisecond
 25 | 	hook := transport.NewPartition(jitterMax)
 26 | 
 27 | 	config := &ClusterConfig{
 28 | 		Count:        nodesCount,
 29 | 		Name:         "network_unreliable",
 30 | 		Prefix:       "prt",
 31 | 		RoundTimeout: helper.GetPredefinedTimeout(2 * time.Second),
 32 | 	}
 33 | 
 34 | 	c := NewPBFTCluster(t, config, hook)
 35 | 	t.Logf("Starting cluster with %d nodes, max faulty %d.\n", nodesCount, maxFaulty)
 36 | 	c.Start()
 37 | 	defer c.Stop()
 38 | 
 39 | 	for {
 40 | 		currentHeight += 5
 41 | 		var minorityPartition []string
 42 | 		var majorityPartition []string
 43 | 		// create 2 partition with random number of nodes
 44 | 		// minority with no more that maxFaulty and majority with rest of the nodes
 45 | 		pSize := 1 + rand.Intn(maxFaulty)
 46 | 		for i := 0; i < pSize; i++ {
 47 | 			minorityPartition = append(minorityPartition, "prt_"+strconv.Itoa(i))
 48 | 		}
 49 | 		for i := pSize; i < nodesCount; i++ {
 50 | 			majorityPartition = append(majorityPartition, "prt_"+strconv.Itoa(i))
 51 | 		}
 52 | 		t.Logf("Partitions ratio %d/%d\n", len(majorityPartition), len(minorityPartition))
 53 | 
 54 | 		hook.Partition(minorityPartition, majorityPartition)
 55 | 		t.Logf("Checking for height %v, started with nodes %d\n", currentHeight, nodesCount)
 56 | 		err := c.WaitForHeight(currentHeight, 10*time.Minute, majorityPartition)
 57 | 		if err != nil {
 58 | 			t.Fatal(err)
 59 | 		}
 60 | 
 61 | 		// randomly drop if possible nodes from the partition pick one number
 62 | 		dropN := rand.Intn(maxFaulty - pSize + 1)
 63 | 		t.Logf("Dropping: %v nodes.\n", dropN)
 64 | 
 65 | 		currentHeight += 5
 66 | 		// stop N nodes from majority partition
 67 | 		for i := 0; i < dropN; i++ {
 68 | 			c.nodes["prt_"+strconv.Itoa(pSize+i)].Stop()
 69 | 		}
 70 | 
 71 | 		var runningMajorityNodes []string
 72 | 		var stoppedNodes []string
 73 | 		for _, v := range c.nodes {
 74 | 			if v.IsRunning() {
 75 | 				for _, bp := range majorityPartition {
 76 | 					if bp == v.name { // is part of the bigPartition
 77 | 						runningMajorityNodes = append(runningMajorityNodes, v.name)
 78 | 					}
 79 | 				}
 80 | 			} else {
 81 | 				stoppedNodes = append(stoppedNodes, v.name)
 82 | 			}
 83 | 		}
 84 | 		// check all running nodes in majority partition for the block height
 85 | 		t.Logf("Checking for height %v, started with nodes %d\n", currentHeight, nodesCount)
 86 | 		err = c.WaitForHeight(currentHeight, 10*time.Minute, runningMajorityNodes)
 87 | 		assert.NoError(t, err)
 88 | 
 89 | 		// restart network for this iteration
 90 | 		hook.Reset()
 91 | 		for _, stopped := range stoppedNodes {
 92 | 			c.nodes[stopped].Start()
 93 | 		}
 94 | 
 95 | 		if currentHeight >= maxHeight {
 96 | 			break
 97 | 		}
 98 | 	}
 99 | 	hook.Reset()
100 | 	// all nodes in the network should be synced after starting all nodes and partition restart
101 | 	finalHeight := maxHeight + 10
102 | 	t.Logf("Checking final height %v, nodes: %d\n", finalHeight, nodesCount)
103 | 	err := c.WaitForHeight(finalHeight, 20*time.Minute)
104 | 	assert.NoError(t, err)
105 | }
106 | 


--------------------------------------------------------------------------------
/e2e/go.mod:
--------------------------------------------------------------------------------
 1 | module github.com/0xPolygon/pbft-consensus/e2e
 2 | 
 3 | go 1.17
 4 | 
 5 | replace github.com/0xPolygon/pbft-consensus => ../
 6 | 
 7 | require (
 8 | 	github.com/0xPolygon/pbft-consensus v0.0.0-20211104133347-f8d6b7df3746
 9 | 	github.com/mitchellh/cli v1.1.2
10 | 	github.com/stretchr/testify v1.7.0
11 | 	go.opentelemetry.io/otel v1.1.0
12 | 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.1.0
13 | 	go.opentelemetry.io/otel/sdk v1.1.0
14 | 	go.opentelemetry.io/otel/trace v1.1.0
15 | 	golang.org/x/sync v0.0.0-20220513210516-0976fa681c29
16 | 	pgregory.net/rapid v0.4.7
17 | )
18 | 
19 | require (
20 | 	github.com/Masterminds/goutils v1.1.0 // indirect
21 | 	github.com/Masterminds/semver v1.5.0 // indirect
22 | 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
23 | 	github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 // indirect
24 | 	github.com/bgentry/speakeasy v0.1.0 // indirect
25 | 	github.com/fatih/color v1.7.0 // indirect
26 | 	github.com/google/uuid v1.1.2 // indirect
27 | 	github.com/hashicorp/errwrap v1.0.0 // indirect
28 | 	github.com/hashicorp/go-multierror v1.0.0 // indirect
29 | 	github.com/huandu/xstrings v1.3.2 // indirect
30 | 	github.com/imdario/mergo v0.3.11 // indirect
31 | 	github.com/mattn/go-colorable v0.0.9 // indirect
32 | 	github.com/mattn/go-isatty v0.0.3 // indirect
33 | 	github.com/mitchellh/copystructure v1.0.0 // indirect
34 | 	github.com/mitchellh/reflectwalk v1.0.0 // indirect
35 | 	github.com/posener/complete v1.1.1 // indirect
36 | 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
37 | )
38 | 
39 | require (
40 | 	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
41 | 	github.com/davecgh/go-spew v1.1.0 // indirect
42 | 	github.com/golang/protobuf v1.5.2 // indirect
43 | 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
44 | 	github.com/kr/text v0.1.0 // indirect
45 | 	github.com/pmezard/go-difflib v1.0.0 // indirect
46 | 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.1.0 // indirect
47 | 	go.opentelemetry.io/proto/otlp v0.9.0 // indirect
48 | 	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
49 | 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
50 | 	golang.org/x/text v0.3.3 // indirect
51 | 	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
52 | 	google.golang.org/grpc v1.42.0 // indirect
53 | 	google.golang.org/protobuf v1.27.1 // indirect
54 | 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
55 | )
56 | 


--------------------------------------------------------------------------------
/e2e/go.sum:
--------------------------------------------------------------------------------
  1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
  2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
  3 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
  4 | github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg=
  5 | github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
  6 | github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
  7 | github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
  8 | github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60=
  9 | github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 10 | github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 11 | github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to=
 12 | github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 13 | github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
 14 | github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 15 | github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
 16 | github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 17 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 18 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 19 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 20 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 21 | github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 22 | github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
 23 | github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 24 | github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 25 | github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 26 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 27 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 28 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 29 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 30 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 31 | github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 32 | github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 33 | github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 34 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 35 | github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 36 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 37 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 38 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 39 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 40 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 41 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 42 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 43 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 44 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 45 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 46 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 47 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 48 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 49 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 50 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 51 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 52 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 53 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 54 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 55 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 56 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 57 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 58 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 59 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 60 | github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 61 | github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 62 | github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 63 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 64 | github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
 65 | github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 66 | github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 67 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 68 | github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 69 | github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 70 | github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
 71 | github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 72 | github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
 73 | github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 74 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 75 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 76 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 77 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 78 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 79 | github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 80 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 81 | github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
 82 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 83 | github.com/mitchellh/cli v1.1.2 h1:PvH+lL2B7IQ101xQL63Of8yFS2y+aDlsFcsqNc+u/Kw=
 84 | github.com/mitchellh/cli v1.1.2/go.mod h1:6iaV0fGdElS6dPBx0EApTxHrcWvmJphyh2n8YBLPPZ4=
 85 | github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
 86 | github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
 87 | github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY=
 88 | github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 89 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 90 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 91 | github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w=
 92 | github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 93 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 94 | github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 95 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 96 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 97 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 98 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 99 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
100 | go.opentelemetry.io/otel v1.1.0 h1:8p0uMLcyyIx0KHNTgO8o3CW8A1aA+dJZJW6PvnMz0Wc=
101 | go.opentelemetry.io/otel v1.1.0/go.mod h1:7cww0OW51jQ8IaZChIEdqLwgh+44+7uiTdWsAL0wQpA=
102 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.1.0 h1:PxBRMkrJnY4HRgToPzoLrTdQDHQf9MeFg5oGzTqtzco=
103 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.1.0/go.mod h1:/E4iniSqAEvqbq6KM5qThKZR2sd42kDvD+SrYt00vRw=
104 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.1.0 h1:4UC7muAl2UqSoTV0RqgmpTz/cRLH6R9cHt9BvVcq5Bo=
105 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.1.0/go.mod h1:Gyc0evUosTBVNRqTFGuu0xqebkEWLkLwv42qggTCwro=
106 | go.opentelemetry.io/otel/sdk v1.1.0 h1:j/1PngUJIDOddkCILQYTevrTIbWd494djgGkSsMit+U=
107 | go.opentelemetry.io/otel/sdk v1.1.0/go.mod h1:3aQvM6uLm6C4wJpHtT8Od3vNzeZ34Pqc6bps8MywWzo=
108 | go.opentelemetry.io/otel/trace v1.1.0 h1:N25T9qCL0+7IpOT8RrRy0WYlL7y6U0WiUJzXcVdXY/o=
109 | go.opentelemetry.io/otel/trace v1.1.0/go.mod h1:i47XtdcBQiktu5IsrPqOHe8w+sBmnLwwHt8wiUsWGTI=
110 | go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
111 | go.opentelemetry.io/proto/otlp v0.9.0 h1:C0g6TWmQYvjKRnljRULLWUVJGy8Uvu0NEL/5frY2/t4=
112 | go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg=
113 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
114 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
115 | golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
116 | golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
117 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
118 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
119 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
120 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
121 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
122 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
123 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
124 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
125 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
126 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
127 | golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
128 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
129 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
130 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
131 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
132 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
133 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
134 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
135 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
136 | golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 h1:w8s32wxx3sY+OjLlv9qltkLU5yvJzxjjgiHWLjdIcw4=
137 | golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
138 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
139 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
140 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
141 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
142 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
143 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
144 | golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
145 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
146 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
147 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
148 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
149 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
150 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
151 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
152 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
153 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
154 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
155 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
156 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
157 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
158 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
159 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
160 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
161 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
162 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
163 | google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
164 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
165 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
166 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
167 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
168 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
169 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
170 | google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
171 | google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
172 | google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
173 | google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
174 | google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A=
175 | google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
176 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
177 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
178 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
179 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
180 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
181 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
182 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
183 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
184 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
185 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
186 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
187 | google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
188 | google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
189 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
190 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
191 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
192 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
193 | gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
194 | gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
195 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
196 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
197 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
198 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
199 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
200 | pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g=
201 | pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
202 | 


--------------------------------------------------------------------------------
/e2e/helper/bool_slice.go:
--------------------------------------------------------------------------------
 1 | package helper
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"sync"
 6 | )
 7 | 
 8 | type BoolSlice struct {
 9 | 	slice []bool
10 | 	mtx   sync.RWMutex
11 | }
12 | 
13 | func NewBoolSlice(ln int) *BoolSlice {
14 | 	return &BoolSlice{
15 | 		slice: make([]bool, ln),
16 | 	}
17 | }
18 | 
19 | func (bs *BoolSlice) Set(i int, b bool) {
20 | 	bs.mtx.Lock()
21 | 	defer bs.mtx.Unlock()
22 | 	bs.slice[i] = b
23 | }
24 | 
25 | func (bs *BoolSlice) Get(i int) bool {
26 | 	bs.mtx.Lock()
27 | 	defer bs.mtx.Unlock()
28 | 	return bs.slice[i]
29 | }
30 | 
31 | func (bs *BoolSlice) Iterate(f func(k int, v bool)) {
32 | 	bs.mtx.RLock()
33 | 	defer bs.mtx.RUnlock()
34 | 	for k, v := range bs.slice {
35 | 		f(k, v)
36 | 	}
37 | }
38 | 
39 | func (bs *BoolSlice) CalculateNum(val bool) int {
40 | 	bs.mtx.RLock()
41 | 	defer bs.mtx.RUnlock()
42 | 	nm := 0
43 | 	for _, v := range bs.slice {
44 | 		if v == val {
45 | 			nm++
46 | 		}
47 | 	}
48 | 	return nm
49 | }
50 | 
51 | func (bs *BoolSlice) String() string {
52 | 	bs.mtx.RLock()
53 | 	defer bs.mtx.RUnlock()
54 | 	return fmt.Sprintf("%v", bs.slice)
55 | }
56 | 


--------------------------------------------------------------------------------
/e2e/helper/helper.go:
--------------------------------------------------------------------------------
  1 | package helper
  2 | 
  3 | import (
  4 | 	//nolint:golint,gosec
  5 | 	"crypto/sha1"
  6 | 	"fmt"
  7 | 	"io"
  8 | 	"io/ioutil"
  9 | 	"log"
 10 | 	"math/rand"
 11 | 	"os"
 12 | 	"path/filepath"
 13 | 	"strconv"
 14 | 	"testing"
 15 | 	"time"
 16 | 
 17 | 	"github.com/0xPolygon/pbft-consensus"
 18 | )
 19 | 
 20 | const trueString = "true"
 21 | 
 22 | func Hash(p []byte) []byte {
 23 | 	h := sha1.New() //nolint:golint,gosec
 24 | 	h.Write(p)
 25 | 	return h.Sum(nil)
 26 | }
 27 | 
 28 | func GenerateProposal() []byte {
 29 | 	prop := make([]byte, 4)
 30 | 	_, _ = rand.Read(prop) //nolint:golint,gosec
 31 | 	return prop
 32 | }
 33 | 
 34 | func GenerateNodeNames(from int, count int, prefix string) []string {
 35 | 	var names []string
 36 | 	for j := from; j < count; j++ {
 37 | 		names = append(names, prefix+strconv.Itoa(j))
 38 | 	}
 39 | 	return names
 40 | }
 41 | 
 42 | func ExecuteInTimerAndWait(tickTime time.Duration, duration time.Duration, fn func(time.Duration)) {
 43 | 	end := executeInTimer(tickTime, duration, fn)
 44 | 	<-end
 45 | }
 46 | 
 47 | func executeInTimer(tickTime time.Duration, duration time.Duration, fn func(time.Duration)) chan struct{} {
 48 | 	tick := time.NewTicker(tickTime)
 49 | 	tickerDone := make(chan struct{})
 50 | 	end := make(chan struct{})
 51 | 	startTime := time.Now()
 52 | 	go func() {
 53 | 		for {
 54 | 			select {
 55 | 			case v := <-tick.C:
 56 | 				elapsedTime := v.Sub(startTime)
 57 | 				fn(elapsedTime)
 58 | 			case <-tickerDone:
 59 | 				close(end)
 60 | 				return
 61 | 			}
 62 | 		}
 63 | 	}()
 64 | 
 65 | 	after := time.After(duration)
 66 | 	go func() {
 67 | 		<-after
 68 | 		tick.Stop()
 69 | 		close(tickerDone)
 70 | 	}()
 71 | 	return end
 72 | }
 73 | 
 74 | func Contains(nodes []string, node string) bool {
 75 | 	for _, n := range nodes {
 76 | 		if n == node {
 77 | 			return true
 78 | 		}
 79 | 	}
 80 | 
 81 | 	return false
 82 | }
 83 | 
 84 | // ShouldApply is used to check if random event meets the threshold
 85 | func ShouldApply(threshold int) bool {
 86 | 	r := rand.Intn(101) //nolint:golint,gosec
 87 | 	return r >= threshold
 88 | }
 89 | 
 90 | func IsFuzzEnabled(t *testing.T) {
 91 | 	if os.Getenv("FUZZ") != trueString {
 92 | 		t.Skip("Fuzz tests are disabled.")
 93 | 	}
 94 | }
 95 | 
 96 | func CreateLogsDir(directoryName string) (string, error) {
 97 | 	//logs directory will be generated at the root of the e2e project
 98 | 	var logsDir string
 99 | 	var err error
100 | 
101 | 	if directoryName == "" {
102 | 		directoryName = fmt.Sprintf("logs_%v", time.Now().Format(time.RFC3339))
103 | 	}
104 | 
105 | 	if os.Getenv("E2E_LOG_TO_FILES") == trueString {
106 | 		logsDir, err = os.MkdirTemp("../", directoryName+"-")
107 | 	}
108 | 
109 | 	return logsDir, err
110 | }
111 | 
112 | func GetLoggerOutput(name string, logsDir string) io.Writer {
113 | 	var loggerOutput io.Writer
114 | 	var err error
115 | 	if os.Getenv("SILENT") == trueString {
116 | 		loggerOutput = ioutil.Discard
117 | 	} else if logsDir != "" {
118 | 		loggerOutput, err = os.OpenFile(filepath.Join(logsDir, name+".log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
119 | 		if err != nil {
120 | 			log.Printf("[WARNING] Failed to open file for node: %v. Reason: %v. Fallbacked to standard output.", name, err)
121 | 			loggerOutput = os.Stdout
122 | 		}
123 | 	} else {
124 | 		loggerOutput = os.Stdout
125 | 	}
126 | 	return loggerOutput
127 | }
128 | 
129 | // GetPredefinedTimeout is a closure to the function which is returning given predefined timeout.
130 | func GetPredefinedTimeout(timeout time.Duration) pbft.RoundTimeout {
131 | 	return func(u uint64) <-chan time.Time {
132 | 		return time.NewTimer(timeout).C
133 | 	}
134 | }
135 | 
136 | // IsTimeoutMessage checks if message in .flow file represents a timeout
137 | func IsTimeoutMessage(message *pbft.MessageReq) bool {
138 | 	return message.Hash == nil && message.Proposal == nil && message.Seal == nil && message.From == ""
139 | }
140 | 


--------------------------------------------------------------------------------
/e2e/node.go:
--------------------------------------------------------------------------------
  1 | package e2e
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"fmt"
  6 | 	"log"
  7 | 	"sync/atomic"
  8 | 
  9 | 	"go.opentelemetry.io/otel/trace"
 10 | 
 11 | 	"github.com/0xPolygon/pbft-consensus"
 12 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
 13 | 	"github.com/0xPolygon/pbft-consensus/e2e/transport"
 14 | )
 15 | 
 16 | type node struct {
 17 | 	// index of node synchronization with the cluster
 18 | 	localSyncIndex int64
 19 | 
 20 | 	c *Cluster
 21 | 
 22 | 	name     string
 23 | 	pbft     *pbft.Pbft
 24 | 	cancelFn context.CancelFunc
 25 | 	running  uint64
 26 | 
 27 | 	// validator nodes
 28 | 	nodes []string
 29 | 
 30 | 	// indicate if the node is faulty
 31 | 	faulty uint64
 32 | }
 33 | 
 34 | func newPBFTNode(name string, clusterConfig *ClusterConfig, nodes []string, trace trace.Tracer, tt *transport.Transport) *node {
 35 | 	loggerOutput := helper.GetLoggerOutput(name, clusterConfig.LogsDir)
 36 | 
 37 | 	con := pbft.New(
 38 | 		pbft.ValidatorKeyMock(name),
 39 | 		tt,
 40 | 		pbft.WithTracer(trace),
 41 | 		pbft.WithLogger(log.New(loggerOutput, "", log.LstdFlags)),
 42 | 		pbft.WithNotifier(clusterConfig.ReplayMessageNotifier),
 43 | 		pbft.WithRoundTimeout(clusterConfig.RoundTimeout),
 44 | 	)
 45 | 
 46 | 	if clusterConfig.TransportHandler != nil {
 47 | 		//for replay messages when we do not want to gossip messages
 48 | 		tt.Register(pbft.NodeID(name), clusterConfig.TransportHandler)
 49 | 	} else {
 50 | 		tt.Register(pbft.NodeID(name), func(to pbft.NodeID, msg *pbft.MessageReq) {
 51 | 			// pipe messages from mock transport to pbft
 52 | 			con.PushMessage(msg)
 53 | 			clusterConfig.ReplayMessageNotifier.HandleMessage(to, msg)
 54 | 		})
 55 | 	}
 56 | 
 57 | 	return &node{
 58 | 		nodes:   nodes,
 59 | 		name:    name,
 60 | 		pbft:    con,
 61 | 		running: 0,
 62 | 		// set to init index -1 so that zero value is not the same as first index
 63 | 		localSyncIndex: -1,
 64 | 	}
 65 | }
 66 | 
 67 | func (n *node) GetName() string {
 68 | 	return n.name
 69 | }
 70 | 
 71 | func (n *node) String() string {
 72 | 	return n.name
 73 | }
 74 | 
 75 | // GetNodeHeight returns node height depending on node index
 76 | // difference between height and syncIndex is 1
 77 | // first inserted proposal is on index 0 with height 1
 78 | func (n *node) GetNodeHeight() uint64 {
 79 | 	return uint64(n.getSyncIndex()) + 1
 80 | }
 81 | 
 82 | func (n *node) IsLocked() bool {
 83 | 	return n.pbft.IsLocked()
 84 | }
 85 | 
 86 | func (n *node) GetProposal() *pbft.Proposal {
 87 | 	return n.pbft.GetProposal()
 88 | }
 89 | 
 90 | func (n *node) PushMessageInternal(message *pbft.MessageReq) {
 91 | 	n.pbft.PushMessageInternal(message)
 92 | }
 93 | 
 94 | func (n *node) Start() {
 95 | 	if n.IsRunning() {
 96 | 		panic(fmt.Errorf("node '%s' is already started", n))
 97 | 	}
 98 | 
 99 | 	// create the ctx and the cancelFn
100 | 	ctx, cancelFn := context.WithCancel(context.Background())
101 | 	n.cancelFn = cancelFn
102 | 	atomic.StoreUint64(&n.running, 1)
103 | 	go func() {
104 | 		defer func() {
105 | 			atomic.StoreUint64(&n.running, 0)
106 | 		}()
107 | 	SYNC:
108 | 		_, syncIndex := n.c.syncWithNetwork(n.name)
109 | 		n.setSyncIndex(syncIndex)
110 | 		for {
111 | 			fsm := n.c.createBackend()
112 | 			fsm.SetBackendData(n)
113 | 
114 | 			if err := n.pbft.SetBackend(fsm); err != nil {
115 | 				panic(err)
116 | 			}
117 | 
118 | 			// start the execution
119 | 			n.pbft.Run(ctx)
120 | 			err := n.c.replayMessageNotifier.SaveState()
121 | 			if err != nil {
122 | 				log.Printf("[WARNING] Could not write state to file. Reason: %v", err)
123 | 			}
124 | 
125 | 			switch n.pbft.GetState() {
126 | 			case pbft.SyncState:
127 | 				// we need to go back to sync
128 | 				goto SYNC
129 | 			case pbft.DoneState:
130 | 				// everything worked, move to the next iteration
131 | 				currentSyncIndex := n.getSyncIndex()
132 | 				n.setSyncIndex(currentSyncIndex + 1)
133 | 			default:
134 | 				// stopped
135 | 				return
136 | 			}
137 | 		}
138 | 	}()
139 | }
140 | 
141 | func (n *node) Stop() {
142 | 	if !n.IsRunning() {
143 | 		panic(fmt.Errorf("node %s is already stopped", n.name))
144 | 	}
145 | 	n.cancelFn()
146 | 	// block until node is running
147 | 	for n.IsRunning() {
148 | 	}
149 | }
150 | 
151 | func (n *node) IsRunning() bool {
152 | 	return atomic.LoadUint64(&n.running) != 0
153 | }
154 | 
155 | func (n *node) getSyncIndex() int64 {
156 | 	return atomic.LoadInt64(&n.localSyncIndex)
157 | }
158 | 
159 | func (n *node) setSyncIndex(idx int64) {
160 | 	atomic.StoreInt64(&n.localSyncIndex, idx)
161 | }
162 | 
163 | func (n *node) isStuck(num uint64) (uint64, bool) {
164 | 	// get max height in the network
165 | 	height, _ := n.c.syncWithNetwork(n.name)
166 | 	if height > num {
167 | 		return height, true
168 | 	}
169 | 	return 0, false
170 | }
171 | 
172 | func (n *node) insert(pp *pbft.SealedProposal) error {
173 | 	err := n.c.insertFinalProposal(pp)
174 | 	if err != nil {
175 | 		panic(err)
176 | 	}
177 | 	return nil
178 | }
179 | 
180 | // setFaultyNode sets flag indicating that the node should be faulty or not
181 | // 0 is for not being faulty
182 | func (n *node) setFaultyNode(b bool) {
183 | 	if b {
184 | 		atomic.StoreUint64(&n.faulty, 1)
185 | 	} else {
186 | 		atomic.StoreUint64(&n.faulty, 0)
187 | 	}
188 | }
189 | 
190 | // isFaulty checks if the node should be faulty or not depending on the stored value
191 | // 0 is for not being faulty
192 | func (n *node) isFaulty() bool {
193 | 	return atomic.LoadUint64(&n.faulty) != 0
194 | }
195 | 
196 | func (n *node) restart() {
197 | 	n.Stop()
198 | 	n.Start()
199 | }
200 | 


--------------------------------------------------------------------------------
/e2e/notifier/default.go:
--------------------------------------------------------------------------------
 1 | package notifier
 2 | 
 3 | import "github.com/0xPolygon/pbft-consensus"
 4 | 
 5 | // DefaultNotifier is a null object implementation of Notifier interface
 6 | type DefaultNotifier struct {
 7 | }
 8 | 
 9 | // HandleTimeout implements StateNotifier interface
10 | func (n *DefaultNotifier) HandleTimeout(to pbft.NodeID, msgType pbft.MsgType, view *pbft.View) {
11 | }
12 | 
13 | // ReadNextMessage is an implementation of StateNotifier interface
14 | func (n *DefaultNotifier) ReadNextMessage(p *pbft.Pbft) (*pbft.MessageReq, []*pbft.MessageReq) {
15 | 	return p.ReadMessageWithDiscards()
16 | }
17 | 
18 | // SaveMetaData is an implementation of ReplayNotifier interface
19 | func (n *DefaultNotifier) SaveMetaData(nodeNames *[]string) error { return nil }
20 | 
21 | // SaveState is an implementation of ReplayNotifier interface
22 | func (n *DefaultNotifier) SaveState() error { return nil }
23 | 
24 | // HandleMessage is an implementation of ReplayNotifier interface
25 | func (n *DefaultNotifier) HandleMessage(to pbft.NodeID, message *pbft.MessageReq) {}
26 | 


--------------------------------------------------------------------------------
/e2e/notifier/notifier.go:
--------------------------------------------------------------------------------
 1 | package notifier
 2 | 
 3 | import "github.com/0xPolygon/pbft-consensus"
 4 | 
 5 | // Notifier is an interface that expands the pbft.StateNotifier with additional methods for saving and loading replay messages
 6 | type Notifier interface {
 7 | 	pbft.StateNotifier
 8 | 	SaveMetaData(nodeNames *[]string) error
 9 | 	SaveState() error
10 | 	HandleMessage(to pbft.NodeID, message *pbft.MessageReq)
11 | }
12 | 


--------------------------------------------------------------------------------
/e2e/otel-jaeger-config.yaml:
--------------------------------------------------------------------------------
 1 | receivers:
 2 |   # Make sure to add the otlp receiver.
 3 |   # This will open up the receiver on port 4317
 4 |   otlp:
 5 |     protocols:
 6 |       grpc:
 7 |         endpoint: "0.0.0.0:4317"
 8 | processors:
 9 | extensions:
10 |   health_check: {}
11 | exporters:
12 |   jaeger:
13 |     endpoint: "127.0.0.1:14250"
14 |     insecure: true
15 | service:
16 |   pipelines:
17 |     traces:
18 |       receivers: [otlp]
19 |       processors: []
20 |       exporters: [jaeger]


--------------------------------------------------------------------------------
/e2e/rapid_test.go:
--------------------------------------------------------------------------------
  1 | package e2e
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"crypto/sha1"
  6 | 	"errors"
  7 | 	"fmt"
  8 | 	"io"
  9 | 	"log"
 10 | 	"math"
 11 | 	"strconv"
 12 | 	"sync"
 13 | 	"testing"
 14 | 	"time"
 15 | 
 16 | 	"go.opentelemetry.io/otel/trace"
 17 | 	"golang.org/x/sync/errgroup"
 18 | 	"pgregory.net/rapid"
 19 | 
 20 | 	"github.com/0xPolygon/pbft-consensus"
 21 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
 22 | 	"github.com/stretchr/testify/require"
 23 | )
 24 | 
 25 | const waitDuration = 50 * time.Millisecond
 26 | 
 27 | func TestProperty_SeveralHonestNodesCanAchiveAgreement(t *testing.T) {
 28 | 	rapid.Check(t, func(t *rapid.T) {
 29 | 		numOfNodes := rapid.IntRange(4, 30).Draw(t, "num of nodes").(int)
 30 | 		ft := &pbft.TransportStub{}
 31 | 		cluster, timeoutsChan := generateCluster(numOfNodes, ft, nil)
 32 | 		for i := range cluster {
 33 | 			cluster[i].SetInitialState(context.Background())
 34 | 		}
 35 | 
 36 | 		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
 37 | 		defer cancel()
 38 | 
 39 | 		err := runCluster(ctx,
 40 | 			cluster,
 41 | 			sendTimeoutIfNNodesStucked(t, timeoutsChan, numOfNodes),
 42 | 			func(doneList *helper.BoolSlice) bool {
 43 | 				//everything done. All nodes in done state
 44 | 				return doneList.CalculateNum(true) == numOfNodes
 45 | 			}, func(maxRound uint64) bool {
 46 | 				// something went wrong.
 47 | 				if maxRound > 3 {
 48 | 					t.Error("Infinite rounds")
 49 | 					return true
 50 | 				}
 51 | 				return false
 52 | 			}, 100)
 53 | 		if err != nil {
 54 | 			t.Fatal(err)
 55 | 		}
 56 | 	})
 57 | }
 58 | 
 59 | func TestProperty_SeveralNodesCanAchiveAgreementWithFailureNodes(t *testing.T) {
 60 | 	rapid.Check(t, func(t *rapid.T) {
 61 | 		numOfNodes := rapid.IntRange(4, 30).Draw(t, "num of nodes").(int)
 62 | 		routingMapGenerator := rapid.MapOfN(
 63 | 			rapid.Uint64Range(0, uint64(numOfNodes)-1),
 64 | 			// not used
 65 | 			rapid.Bool(),
 66 | 			2*numOfNodes/3+1,
 67 | 			numOfNodes-1,
 68 | 		).Filter(func(m map[uint64]bool) bool {
 69 | 			_, ok := m[0]
 70 | 			return ok
 71 | 		})
 72 | 
 73 | 		routingMap := routingMapGenerator.Draw(t, "generate routing").(map[uint64]bool)
 74 | 		ft := &pbft.TransportStub{
 75 | 			GossipFunc: func(ft *pbft.TransportStub, msg *pbft.MessageReq) error {
 76 | 				from, err := strconv.ParseUint(string(msg.From), 10, 64)
 77 | 				if err != nil {
 78 | 					t.Fatal(err)
 79 | 				}
 80 | 
 81 | 				if _, ok := routingMap[from]; ok {
 82 | 					for i := range routingMap {
 83 | 						if i == from {
 84 | 							continue
 85 | 						}
 86 | 						ft.Nodes[i].PushMessage(msg)
 87 | 					}
 88 | 				}
 89 | 
 90 | 				return nil
 91 | 			},
 92 | 		}
 93 | 		cluster, timeoutsChan := generateCluster(numOfNodes, ft, nil)
 94 | 		for i := range cluster {
 95 | 			cluster[i].SetInitialState(context.Background())
 96 | 		}
 97 | 
 98 | 		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
 99 | 		defer cancel()
100 | 
101 | 		err := runCluster(ctx,
102 | 			cluster,
103 | 			sendTimeoutIfNNodesStucked(t, timeoutsChan, numOfNodes),
104 | 			func(doneList *helper.BoolSlice) bool {
105 | 				//check that 3 node switched to done state
106 | 				return doneList.CalculateNum(true) >= numOfNodes*2/3+1
107 | 			}, func(maxRound uint64) bool {
108 | 				if maxRound > 10 {
109 | 					t.Error("Infinite rounds")
110 | 					return true
111 | 				}
112 | 				return false
113 | 			}, 100)
114 | 		if err != nil {
115 | 			t.Fatal(err)
116 | 		}
117 | 	})
118 | }
119 | 
120 | func TestProperty_4NodesCanAchiveAgreementIfWeLockButNotCommitProposer_Fails(t *testing.T) {
121 | 	t.Skip("Unskip when fix")
122 | 	numOfNodes := 4
123 | 	rounds := map[uint64]map[int][]int{
124 | 		0: {
125 | 			0: {0, 1, 3, 2},
126 | 			1: {0, 1, 3},
127 | 			2: {0, 1, 2, 3},
128 | 		},
129 | 	}
130 | 
131 | 	countPrepare := 0
132 | 	ft := &pbft.TransportStub{
133 | 		// for round 0 we have a routing from routing map without commit messages and
134 | 		// for other rounds we dont send messages to node 3
135 | 		GossipFunc: func(ft *pbft.TransportStub, msg *pbft.MessageReq) error {
136 | 			routing, changed := rounds[msg.View.Round]
137 | 			if changed {
138 | 				from, err := strconv.Atoi(string(msg.From))
139 | 				if err != nil {
140 | 					t.Fatal(err)
141 | 				}
142 | 				for _, nodeId := range routing[from] {
143 | 					// restrict prepare messages to node 3 in round 0
144 | 					if msg.Type == pbft.MessageReq_Prepare && nodeId == 3 {
145 | 						countPrepare++
146 | 						if countPrepare == 3 {
147 | 							fmt.Println("Ignoring prepare 3")
148 | 							continue
149 | 						}
150 | 					}
151 | 					// do not send commit for round 0
152 | 					if msg.Type == pbft.MessageReq_Commit {
153 | 						continue
154 | 					}
155 | 
156 | 					ft.Nodes[nodeId].PushMessage(msg)
157 | 				}
158 | 			} else {
159 | 				for i := range ft.Nodes {
160 | 					from, _ := strconv.Atoi(string(msg.From))
161 | 					// for rounds >0 do not send messages to/from node 3
162 | 					if i == 3 || from == 3 {
163 | 						continue
164 | 					} else {
165 | 						ft.Nodes[i].PushMessage(msg)
166 | 					}
167 | 				}
168 | 			}
169 | 
170 | 			return nil
171 | 		},
172 | 	}
173 | 	cluster, timeoutsChan := generateCluster(numOfNodes, ft, nil)
174 | 	for i := range cluster {
175 | 		cluster[i].SetInitialState(context.Background())
176 | 	}
177 | 	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
178 | 	defer cancel()
179 | 	err := runCluster(ctx,
180 | 		cluster,
181 | 		sendTimeoutIfNNodesStucked(t, timeoutsChan, numOfNodes),
182 | 		func(doneList *helper.BoolSlice) bool {
183 | 			return doneList.CalculateNum(true) >= 3
184 | 		}, func(maxRound uint64) bool {
185 | 			if maxRound > 5 {
186 | 				t.Error("Liveness issue")
187 | 				return true
188 | 			}
189 | 			return false
190 | 		}, 50)
191 | 	if err != nil {
192 | 		t.Fatal(err)
193 | 	}
194 | }
195 | 
196 | func TestProperty_FiveNodesCanAchiveAgreementIfWeLockTwoNodesOnDifferentProposals(t *testing.T) {
197 | 	numOfNodes := 5
198 | 	rounds := map[uint64]map[int][]int{
199 | 		0: {
200 | 			0: {0, 1, 2},
201 | 			1: {0},
202 | 			2: {0, 3},
203 | 			3: {0},
204 | 		},
205 | 		1: {
206 | 			0: {1},
207 | 			1: {1, 2, 3},
208 | 			2: {1},
209 | 			3: {1},
210 | 		},
211 | 	}
212 | 
213 | 	ft := &pbft.TransportStub{
214 | 		GossipFunc: func(ft *pbft.TransportStub, msg *pbft.MessageReq) error {
215 | 			routing, changed := rounds[msg.View.Round]
216 | 			if changed {
217 | 				from, err := strconv.Atoi(string(msg.From))
218 | 				if err != nil {
219 | 					t.Fatal(err)
220 | 				}
221 | 				for _, nodeId := range routing[from] {
222 | 					ft.Nodes[nodeId].PushMessage(msg)
223 | 				}
224 | 			} else {
225 | 				for i := range ft.Nodes {
226 | 					ft.Nodes[i].PushMessage(msg)
227 | 				}
228 | 			}
229 | 			return nil
230 | 		},
231 | 	}
232 | 	cluster, timeoutsChan := generateCluster(numOfNodes, ft, nil)
233 | 	for i := range cluster {
234 | 		cluster[i].SetInitialState(context.Background())
235 | 	}
236 | 
237 | 	err := runCluster(context.Background(),
238 | 		cluster,
239 | 		sendTimeoutIfNNodesStucked(t, timeoutsChan, numOfNodes),
240 | 		func(doneList *helper.BoolSlice) bool {
241 | 			return doneList.CalculateNum(true) > 3
242 | 		}, func(maxNodeRound uint64) bool {
243 | 			if maxNodeRound > 20 {
244 | 				t.Fatal("too many rounds")
245 | 			}
246 | 			return true
247 | 		}, 0)
248 | 	if err != nil {
249 | 		t.Fatal(err)
250 | 	}
251 | }
252 | 
253 | func TestProperty_NodeDoubleSign(t *testing.T) {
254 | 	t.Skip("Unskip when fix")
255 | 	rapid.Check(t, func(t *rapid.T) {
256 | 		numOfNodes := rapid.IntRange(4, 7).Draw(t, "num of nodes").(int)
257 | 		// sign different message to up to 1/2 of the nodes
258 | 		maliciousMessagesToNodes := rapid.IntRange(0, numOfNodes/2).Draw(t, "malicious message to nodes").(int)
259 | 		weightedNodes := make(map[pbft.NodeID]uint64, numOfNodes)
260 | 		for i := 0; i < numOfNodes; i++ {
261 | 			weightedNodes[pbft.NodeID(fmt.Sprintf("NODE_%s", strconv.Itoa(i)))] = 1
262 | 		}
263 | 		maxFaultyVotingPower, _, err := pbft.CalculateQuorum(weightedNodes)
264 | 		require.NoError(t, err)
265 | 		faultyNodes := rapid.IntRange(1, int(maxFaultyVotingPower)).Draw(t, "malicious nodes").(int)
266 | 		maliciousNodes := generateMaliciousProposers(faultyNodes)
267 | 		votingPower := make(map[pbft.NodeID]uint64, numOfNodes)
268 | 
269 | 		for i := 0; i < numOfNodes; i++ {
270 | 			votingPower[pbft.NodeID(strconv.Itoa(i))] = 1
271 | 		}
272 | 
273 | 		ft := &pbft.TransportStub{
274 | 			GossipFunc: func(ft *pbft.TransportStub, msg *pbft.MessageReq) error {
275 | 				for to := range ft.Nodes {
276 | 					modifiedMessage := msg.Copy()
277 | 					// faulty node modifies proposal and sends to subset of nodes (maliciousMessagesToNodes)
278 | 					from, _ := strconv.Atoi(string(modifiedMessage.From))
279 | 					if from < len(maliciousNodes) {
280 | 						if maliciousMessagesToNodes > to {
281 | 							modifiedMessage.Proposal = maliciousNodes[from].maliciousProposal
282 | 							modifiedMessage.Hash = maliciousNodes[from].maliciousProposalHash
283 | 						}
284 | 					}
285 | 					ft.Nodes[to].PushMessage(modifiedMessage)
286 | 				}
287 | 
288 | 				return nil
289 | 			},
290 | 		}
291 | 		cluster, timeoutsChan := generateCluster(numOfNodes, ft, votingPower)
292 | 		for i := range cluster {
293 | 			cluster[i].SetInitialState(context.Background())
294 | 		}
295 | 
296 | 		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
297 | 		defer cancel()
298 | 
299 | 		err = runCluster(ctx,
300 | 			cluster,
301 | 			sendTimeoutIfNNodesStucked(t, timeoutsChan, numOfNodes),
302 | 			func(doneList *helper.BoolSlice) bool {
303 | 				return doneList.CalculateNum(true) >= 2*numOfNodes/3+1
304 | 			}, func(maxRound uint64) bool {
305 | 				if maxRound > 10 {
306 | 					t.Error("Infinite rounds")
307 | 					return true
308 | 				}
309 | 				return false
310 | 			}, 100)
311 | 		if err != nil {
312 | 			// fail if node inserts different proposal
313 | 			t.Fatalf("%v\n", err)
314 | 		}
315 | 	})
316 | }
317 | 
318 | func TestProperty_SeveralHonestNodesWithVotingPowerCanAchiveAgreement(t *testing.T) {
319 | 	rapid.Check(t, func(t *rapid.T) {
320 | 		numOfNodes := rapid.IntRange(4, 10).Draw(t, "num of nodes").(int)
321 | 		votingPowerSlice := rapid.SliceOfN(rapid.Uint64Range(1, math.MaxUint64/uint64(numOfNodes)), numOfNodes, numOfNodes).Draw(t, "voting power").([]uint64)
322 | 		votingPower := make(map[pbft.NodeID]uint64, numOfNodes)
323 | 		for i := 0; i < numOfNodes; i++ {
324 | 			votingPower[pbft.NodeID(strconv.Itoa(i))] = votingPowerSlice[i]
325 | 		}
326 | 		ft := &pbft.TransportStub{}
327 | 		cluster, timeoutsChan := generateCluster(numOfNodes, ft, votingPower)
328 | 		for i := range cluster {
329 | 			cluster[i].SetInitialState(context.Background())
330 | 		}
331 | 
332 | 		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
333 | 		defer cancel()
334 | 
335 | 		err := runCluster(ctx,
336 | 			cluster,
337 | 			sendTimeoutIfNNodesStucked(t, timeoutsChan, numOfNodes),
338 | 			func(doneList *helper.BoolSlice) bool {
339 | 				// everything done. All nodes in done state
340 | 				return doneList.CalculateNum(true) == numOfNodes
341 | 			}, func(maxRound uint64) bool {
342 | 				// something went wrong.
343 | 				if maxRound > 3 {
344 | 					t.Error("Infinite rounds")
345 | 					return true
346 | 				}
347 | 				return false
348 | 			}, 100)
349 | 		if err != nil {
350 | 			t.Fatal(err)
351 | 		}
352 | 	})
353 | }
354 | 
355 | func TestProperty_NodesWithMajorityOfVotingPowerCanAchiveAgreement(t *testing.T) {
356 | 	rapid.Check(t, func(t *rapid.T) {
357 | 		numOfNodes := rapid.IntRange(5, 12).Draw(t, "num of nodes").(int)
358 | 		stake := rapid.SliceOfN(rapid.Uint64Range(5, 10), numOfNodes, numOfNodes).Draw(t, "Generate stake").([]uint64)
359 | 		votingPower := make(map[pbft.NodeID]uint64, numOfNodes)
360 | 		for i := range stake {
361 | 			votingPower[pbft.NodeID(strconv.Itoa(i))] = stake[i]
362 | 		}
363 | 		_, quorumSize, err := pbft.CalculateQuorum(votingPower)
364 | 		require.NoError(t, err)
365 | 
366 | 		connectionsList := rapid.SliceOfDistinct(rapid.IntRange(0, numOfNodes-1), func(v int) int {
367 | 			return v
368 | 		}).Filter(func(votes []int) bool {
369 | 			var votesVP uint64
370 | 			for i := range votes {
371 | 				votesVP += stake[votes[i]]
372 | 			}
373 | 			return votesVP >= quorumSize
374 | 		}).Draw(t, "Select arbitrary nodes that have majority of voting power").([]int)
375 | 
376 | 		connections := map[pbft.NodeID]struct{}{}
377 | 		var topologyVotingPower uint64
378 | 		for _, nodeIDInt := range connectionsList {
379 | 			connections[pbft.NodeID(strconv.Itoa(nodeIDInt))] = struct{}{}
380 | 			topologyVotingPower += stake[nodeIDInt]
381 | 		}
382 | 
383 | 		ft := &pbft.TransportStub{
384 | 			GossipFunc: func(ft *pbft.TransportStub, msg *pbft.MessageReq) error {
385 | 				for _, node := range ft.Nodes {
386 | 					// skip faulty nodes
387 | 					if _, ok := connections[msg.From]; !ok {
388 | 						continue
389 | 					}
390 | 					if msg.From != node.GetValidatorId() {
391 | 						node.PushMessage(msg.Copy())
392 | 					}
393 | 				}
394 | 				return nil
395 | 			},
396 | 		}
397 | 		cluster, timeoutsChan := generateCluster(numOfNodes, ft, votingPower)
398 | 		for i := range cluster {
399 | 			cluster[i].SetInitialState(context.Background())
400 | 		}
401 | 
402 | 		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
403 | 		defer cancel()
404 | 
405 | 		err = runCluster(ctx,
406 | 			cluster,
407 | 			sendTimeoutIfNNodesStucked(t, timeoutsChan, numOfNodes),
408 | 			func(doneList *helper.BoolSlice) bool {
409 | 				accumulatedVotingPower := uint64(0)
410 | 				// enough nodes (by their respective voting power) are in done state
411 | 				doneList.Iterate(func(index int, isDone bool) {
412 | 					if isDone {
413 | 						accumulatedVotingPower += votingPower[cluster[index].GetValidatorId()]
414 | 					}
415 | 				})
416 | 				return accumulatedVotingPower >= topologyVotingPower
417 | 			}, func(maxRound uint64) bool {
418 | 				// something went wrong.
419 | 				if maxRound > 3 {
420 | 					for i := range cluster {
421 | 						fmt.Println(i, cluster[i].GetState())
422 | 					}
423 | 					t.Error("Infinite rounds")
424 | 					return true
425 | 				}
426 | 				return false
427 | 			}, 100)
428 | 		if err != nil {
429 | 			for i := range cluster {
430 | 				fmt.Println(i, cluster[i].GetState())
431 | 			}
432 | 			t.Fatal(err)
433 | 		}
434 | 	})
435 | }
436 | 
437 | type maliciousProposer struct {
438 | 	nodeID                pbft.NodeID
439 | 	maliciousProposal     []byte
440 | 	maliciousProposalHash []byte
441 | }
442 | 
443 | func generateMaliciousProposers(num int) []maliciousProposer {
444 | 	maliciousProposers := make([]maliciousProposer, num)
445 | 	for i := 0; i < num; i++ {
446 | 		maliciousProposal := helper.GenerateProposal()
447 | 		h := sha1.New()
448 | 		h.Write(maliciousProposal)
449 | 		maliciousProposalHash := h.Sum(nil)
450 | 		malicious := maliciousProposer{pbft.NodeID(strconv.Itoa(i)), maliciousProposal, maliciousProposalHash}
451 | 		maliciousProposers[i] = malicious
452 | 	}
453 | 
454 | 	return maliciousProposers
455 | }
456 | 
457 | func getMaxClusterRound(cluster []*pbft.Pbft) uint64 {
458 | 	var maxRound uint64
459 | 	for i := range cluster {
460 | 		localRound := cluster[i].Round()
461 | 		if localRound > maxRound {
462 | 			maxRound = localRound
463 | 		}
464 | 	}
465 | 	return maxRound
466 | }
467 | 
468 | func generateNode(id int, transport *pbft.TransportStub) (*pbft.Pbft, chan time.Time) {
469 | 	timeoutChan := make(chan time.Time)
470 | 	node := pbft.New(pbft.ValidatorKeyMock(strconv.Itoa(id)), transport,
471 | 		pbft.WithTracer(trace.NewNoopTracerProvider().Tracer("")),
472 | 		pbft.WithLogger(log.New(io.Discard, "", 0)),
473 | 		pbft.WithRoundTimeout(func(_ uint64) <-chan time.Time {
474 | 			return timeoutChan
475 | 		}),
476 | 	)
477 | 
478 | 	transport.Nodes = append(transport.Nodes, node)
479 | 	return node, timeoutChan
480 | }
481 | 
482 | func generateCluster(numOfNodes int, transport *pbft.TransportStub, votingPower map[pbft.NodeID]uint64) ([]*pbft.Pbft, []chan time.Time) {
483 | 	nodes := make([]string, numOfNodes)
484 | 	timeoutsChan := make([]chan time.Time, numOfNodes)
485 | 	ip := &finalProposal{
486 | 		lock: sync.Mutex{},
487 | 		bc:   make(map[uint64]pbft.Proposal),
488 | 	}
489 | 	cluster := make([]*pbft.Pbft, numOfNodes)
490 | 	for i := 0; i < numOfNodes; i++ {
491 | 		cluster[i], timeoutsChan[i] = generateNode(i, transport)
492 | 		nodes[i] = strconv.Itoa(i)
493 | 	}
494 | 
495 | 	for _, nd := range cluster {
496 | 		_ = nd.SetBackend(&BackendFake{
497 | 			nodes:          nodes,
498 | 			votingPowerMap: votingPower,
499 | 			insertFunc: func(proposal *pbft.SealedProposal) error {
500 | 				return ip.Insert(*proposal)
501 | 			},
502 | 			isStuckFunc: func(num uint64) (uint64, bool) {
503 | 				return 0, false
504 | 			},
505 | 		})
506 | 	}
507 | 
508 | 	return cluster, timeoutsChan
509 | }
510 | 
511 | func runClusterCycle(cluster []*pbft.Pbft, _ int, stuckList, doneList *helper.BoolSlice) error {
512 | 	wg := errgroup.Group{}
513 | 	for i := range cluster {
514 | 		i := i
515 | 		state := cluster[i].GetState()
516 | 		isLocked := cluster[i].IsLocked()
517 | 		wg.Go(func() (err1 error) {
518 | 			wgTime := time.Now()
519 | 			exitCh := make(chan struct{})
520 | 			deadlineTimeout := waitDuration
521 | 			deadline := time.After(deadlineTimeout)
522 | 			errCh := make(chan error)
523 | 			if stuckList.Get(i) {
524 | 				return nil
525 | 			}
526 | 
527 | 			if doneList.Get(i) {
528 | 				return nil
529 | 			}
530 | 
531 | 			go func() {
532 | 				defer func() {
533 | 					if r := recover(); r != nil {
534 | 						errCh <- fmt.Errorf("%v", r)
535 | 					}
536 | 				}()
537 | 				stuckList.Set(i, true)
538 | 				defer func() {
539 | 					stuckList.Set(i, false)
540 | 				}()
541 | 				cluster[i].RunCycle(context.Background())
542 | 				close(exitCh)
543 | 			}()
544 | 			select {
545 | 			case <-exitCh:
546 | 			case <-deadline:
547 | 			case er := <-errCh:
548 | 				err1 = er
549 | 			}
550 | 
551 | 			// useful for debug
552 | 			_, _, _ = state, wgTime, isLocked
553 | 			// if time.Since(wgTime) > waitDuration {
554 | 			//	fmt.Println("wgitme ", state, i, callNumber, time.Since(wgTime), err1, isLocked)
555 | 			// }
556 | 			//
557 | 			return err1
558 | 		})
559 | 	}
560 | 
561 | 	return wg.Wait()
562 | }
563 | 
564 | func setDoneOnDoneState(cluster []*pbft.Pbft, doneList *helper.BoolSlice) {
565 | 	for i, node := range cluster {
566 | 		state := node.GetState()
567 | 		if state == pbft.DoneState {
568 | 			doneList.Set(i, true)
569 | 		}
570 | 	}
571 | }
572 | 
573 | type Errorer interface {
574 | 	Error(args ...interface{})
575 | }
576 | 
577 | func sendTimeoutIfNNodesStucked(t Errorer, timeoutsChan []chan time.Time, numOfNodes int) func(stuckList *helper.BoolSlice) bool {
578 | 	return func(stuckList *helper.BoolSlice) bool {
579 | 		if stuckList.CalculateNum(true) == numOfNodes {
580 | 			c := time.After(time.Second * 5)
581 | 			for i := range timeoutsChan {
582 | 				select {
583 | 				case timeoutsChan[i] <- time.Now():
584 | 				case <-c:
585 | 					t.Error(i, "node timeout stucked")
586 | 					return true
587 | 				}
588 | 			}
589 | 		}
590 | 		return false
591 | 	}
592 | }
593 | 
594 | func runCluster(ctx context.Context,
595 | 	cluster []*pbft.Pbft,
596 | 	handleStuckList func(*helper.BoolSlice) bool,
597 | 	handleDoneList func(*helper.BoolSlice) bool,
598 | 	handleMaxRoundNumber func(uint64) bool,
599 | 	limitCallNumber int,
600 | ) error {
601 | 	for i := range cluster {
602 | 		cluster[i].SetInitialState(context.Background())
603 | 	}
604 | 
605 | 	stuckList := helper.NewBoolSlice(len(cluster))
606 | 	doneList := helper.NewBoolSlice(len(cluster))
607 | 
608 | 	callNumber := 0
609 | 	for {
610 | 		callNumber++
611 | 		err := runClusterCycle(cluster, callNumber, stuckList, doneList)
612 | 		if err != nil {
613 | 			return err
614 | 		}
615 | 
616 | 		setDoneOnDoneState(cluster, doneList)
617 | 		if handleStuckList(stuckList) {
618 | 			return nil
619 | 		}
620 | 		if handleDoneList(doneList) {
621 | 			return nil
622 | 		}
623 | 
624 | 		if handleMaxRoundNumber(getMaxClusterRound(cluster)) {
625 | 			return nil
626 | 		}
627 | 		if callNumber > limitCallNumber {
628 | 			return errors.New("callnumber limit")
629 | 		}
630 | 		select {
631 | 		case <-ctx.Done():
632 | 			return ctx.Err()
633 | 		default:
634 | 
635 | 		}
636 | 	}
637 | }
638 | 
639 | // finalProposal struct contains inserted final proposals for the node
640 | type finalProposal struct {
641 | 	lock sync.Mutex
642 | 	bc   map[uint64]pbft.Proposal
643 | }
644 | 
645 | func (i *finalProposal) Insert(proposal pbft.SealedProposal) error {
646 | 	i.lock.Lock()
647 | 	defer i.lock.Unlock()
648 | 	if p, ok := i.bc[proposal.Number]; ok {
649 | 		if !p.Equal(proposal.Proposal) {
650 | 			panic("wrong proposal inserted")
651 | 		}
652 | 	} else {
653 | 		i.bc[proposal.Number] = *proposal.Proposal
654 | 	}
655 | 	return nil
656 | }
657 | 


--------------------------------------------------------------------------------
/e2e/replay/msg.go:
--------------------------------------------------------------------------------
 1 | package replay
 2 | 
 3 | import (
 4 | 	"github.com/0xPolygon/pbft-consensus"
 5 | )
 6 | 
 7 | // message is a struct that represents a single json object in .flow file
 8 | type message struct {
 9 | 	To      pbft.NodeID      `json:"to"`
10 | 	Message *pbft.MessageReq `json:"message"`
11 | }
12 | 
13 | // newMessage creates a new message to be written to .flow file
14 | func newMessage(to pbft.NodeID, messageReq *pbft.MessageReq) *message {
15 | 	return &message{
16 | 		To:      to,
17 | 		Message: messageReq,
18 | 	}
19 | }
20 | 
21 | // newTimeoutMessage creates a new timeout to be written to .flow file
22 | func newTimeoutMessage(to pbft.NodeID, msgType pbft.MsgType, view *pbft.View) *message {
23 | 	return &message{
24 | 		To: to,
25 | 		Message: &pbft.MessageReq{
26 | 			Type: msgType,
27 | 			View: view,
28 | 		},
29 | 	}
30 | }
31 | 


--------------------------------------------------------------------------------
/e2e/replay/msg_middleware.go:
--------------------------------------------------------------------------------
 1 | package replay
 2 | 
 3 | import (
 4 | 	"github.com/0xPolygon/pbft-consensus"
 5 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
 6 | )
 7 | 
 8 | const (
 9 | 	FileName = "messages"
10 | )
11 | 
12 | // MessagesMiddleware is a struct that implements Notifier interface
13 | type MessagesMiddleware struct {
14 | 	messagePersister *messagePersister
15 | 	messageReader    *MessageReader
16 | }
17 | 
18 | // NewMessagesMiddlewareWithPersister creates a new messages notifier with messages persister (required when fuzz-run is executed to save messages to file)
19 | func NewMessagesMiddlewareWithPersister() *MessagesMiddleware {
20 | 	return &MessagesMiddleware{
21 | 		messagePersister: &messagePersister{},
22 | 	}
23 | }
24 | 
25 | // NewMessagesMiddlewareWithReader creates a new messages notifier with messages reader (required when replay-messages is executed to read messages from file)
26 | func NewMessagesMiddlewareWithReader(r *MessageReader) *MessagesMiddleware {
27 | 	return &MessagesMiddleware{
28 | 		messageReader:    r,
29 | 		messagePersister: &messagePersister{},
30 | 	}
31 | }
32 | 
33 | // SaveMetaData saves node meta data to .flow file
34 | func (r *MessagesMiddleware) SaveMetaData(nodeNames *[]string) error {
35 | 	return r.messagePersister.saveMetaData(nodeNames)
36 | }
37 | 
38 | // SaveState saves currently cached messages and timeouts to .flow file
39 | func (r *MessagesMiddleware) SaveState() error {
40 | 	return r.messagePersister.saveCachedMessages()
41 | }
42 | 
43 | // HandleMessage caches processed message to be saved later in .flow file
44 | func (r *MessagesMiddleware) HandleMessage(to pbft.NodeID, message *pbft.MessageReq) {
45 | 	r.messagePersister.addMessage(newMessage(to, message))
46 | }
47 | 
48 | // HandleTimeout is an implementation of StateNotifier interface
49 | func (r *MessagesMiddleware) HandleTimeout(to pbft.NodeID, msgType pbft.MsgType, view *pbft.View) {
50 | 	r.messagePersister.addMessage(newTimeoutMessage(to, msgType, view))
51 | }
52 | 
53 | // ReadNextMessage is an implementation of StateNotifier interface
54 | func (r *MessagesMiddleware) ReadNextMessage(p *pbft.Pbft) (*pbft.MessageReq, []*pbft.MessageReq) {
55 | 	msg, discards := p.ReadMessageWithDiscards()
56 | 
57 | 	if r.messageReader != nil && msg != nil {
58 | 		if helper.IsTimeoutMessage(msg) {
59 | 			return nil, nil
60 | 		} else {
61 | 			r.messageReader.checkIfDoneWithExecution(p.GetValidatorId(), msg)
62 | 		}
63 | 	}
64 | 
65 | 	return msg, discards
66 | }
67 | 
68 | // CloseFile closes file created by the ReplayMessagesHandler if it is open
69 | func (r *MessagesMiddleware) CloseFile() error {
70 | 	return r.messagePersister.closeFile()
71 | }
72 | 


--------------------------------------------------------------------------------
/e2e/replay/msg_persister.go:
--------------------------------------------------------------------------------
  1 | package replay
  2 | 
  3 | import (
  4 | 	"bufio"
  5 | 	"encoding/json"
  6 | 	"fmt"
  7 | 	"os"
  8 | 	"path/filepath"
  9 | 	"sync"
 10 | 	"time"
 11 | )
 12 | 
 13 | const directoryPath = "../SavedState"
 14 | 
 15 | // messagePersister encapsulates logic for saving messages in .flow file
 16 | type messagePersister struct {
 17 | 	lock     sync.Mutex
 18 | 	messages []*message
 19 | 	file     *os.File
 20 | }
 21 | 
 22 | // saveMetaData saves node meta data to .flow file
 23 | func (r *messagePersister) saveMetaData(nodeNames *[]string) error {
 24 | 	var err error
 25 | 	if err = r.createFile(); err != nil {
 26 | 		return err
 27 | 	}
 28 | 
 29 | 	bufWriter := bufio.NewWriter(r.file)
 30 | 	defer bufWriter.Flush()
 31 | 
 32 | 	currentRawMessage, err := json.Marshal(nodeNames)
 33 | 	if err != nil {
 34 | 		return err
 35 | 	}
 36 | 
 37 | 	_, err = bufWriter.Write(currentRawMessage)
 38 | 	if err != nil {
 39 | 		return err
 40 | 	}
 41 | 
 42 | 	_, err = bufWriter.Write([]byte("\n"))
 43 | 
 44 | 	return err
 45 | }
 46 | 
 47 | // saveCachedMessages saves currently cached messages and timeouts to .flow file
 48 | func (r *messagePersister) saveCachedMessages() error {
 49 | 	r.lock.Lock()
 50 | 	defer r.lock.Unlock()
 51 | 
 52 | 	var err error
 53 | 	if err = r.createFile(); err != nil {
 54 | 		return err
 55 | 	}
 56 | 
 57 | 	if r.messages != nil {
 58 | 		err = r.saveMessages(r.file)
 59 | 	}
 60 | 
 61 | 	return err
 62 | }
 63 | 
 64 | // createFile creates a .flow file to save messages and timeouts on the predifined location
 65 | func (r *messagePersister) createFile() error {
 66 | 	if r.file == nil {
 67 | 		if _, err := os.Stat(directoryPath); os.IsNotExist(err) {
 68 | 			err := os.Mkdir(directoryPath, 0777)
 69 | 			if err != nil {
 70 | 				return err
 71 | 			}
 72 | 		}
 73 | 
 74 | 		path, err := filepath.Abs(directoryPath)
 75 | 		if err != nil {
 76 | 			return err
 77 | 		}
 78 | 
 79 | 		file, err := os.OpenFile(filepath.Join(path, fmt.Sprintf("%v_%v.flow", FileName, time.Now().Format(time.RFC3339))), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
 80 | 		if err != nil {
 81 | 			return err
 82 | 		}
 83 | 		r.file = file
 84 | 	}
 85 | 
 86 | 	return nil
 87 | }
 88 | 
 89 | // closeFile closes file created by the ReplayMessagesHandler if it is open
 90 | func (r *messagePersister) closeFile() error {
 91 | 	if r.file != nil {
 92 | 		return r.file.Close()
 93 | 	}
 94 | 	return nil
 95 | }
 96 | 
 97 | // addMessage adds a message from sequence to message cache that will be written to .flow file
 98 | func (r *messagePersister) addMessage(message *message) {
 99 | 	r.lock.Lock()
100 | 	r.messages = append(r.messages, message)
101 | 	r.lock.Unlock()
102 | }
103 | 
104 | // saveMessages saves ReplayMessages to the JSON file within the pre-defined directory.
105 | func (r *messagePersister) saveMessages(fileWriter *os.File) error {
106 | 	rawMessages, err := convertToByteArrays(r.messages)
107 | 	if err != nil {
108 | 		return err
109 | 	}
110 | 
111 | 	bufWriter := bufio.NewWriterSize(fileWriter, maxCharactersPerLine)
112 | 	defer bufWriter.Flush()
113 | 
114 | 	for _, rawMessage := range rawMessages {
115 | 		_, err = bufWriter.Write(rawMessage)
116 | 		if err != nil {
117 | 			return err
118 | 		}
119 | 
120 | 		_, err = bufWriter.Write([]byte("\n"))
121 | 		if err != nil {
122 | 			return err
123 | 		}
124 | 	}
125 | 
126 | 	r.messages = nil
127 | 	return nil
128 | }
129 | 
130 | // convertToByteArrays converts message slice to JSON representation and return it back as slice of byte arrays
131 | func convertToByteArrays(messages []*message) ([][]byte, error) {
132 | 	var allRawMessages [][]byte
133 | 	for _, message := range messages {
134 | 		currentRawMessage, err := json.Marshal(message)
135 | 		if err != nil {
136 | 			return allRawMessages, err
137 | 		}
138 | 		allRawMessages = append(allRawMessages, currentRawMessage)
139 | 	}
140 | 	return allRawMessages, nil
141 | }
142 | 


--------------------------------------------------------------------------------
/e2e/replay/msg_reader.go:
--------------------------------------------------------------------------------
  1 | package replay
  2 | 
  3 | import (
  4 | 	"bufio"
  5 | 	"encoding/json"
  6 | 	"errors"
  7 | 	"log"
  8 | 	"os"
  9 | 	"sync"
 10 | 
 11 | 	"github.com/0xPolygon/pbft-consensus"
 12 | 	"github.com/0xPolygon/pbft-consensus/e2e/helper"
 13 | )
 14 | 
 15 | const (
 16 | 	maxCharactersPerLine = 2048 * 1024 // Increase Scanner buffer size to 2MB per line
 17 | 	messageChunkSize     = 200
 18 | )
 19 | 
 20 | type sequenceMessages struct {
 21 | 	sequence uint64
 22 | 	messages []*pbft.MessageReq
 23 | }
 24 | 
 25 | // Node represents a behavior of a cluster node
 26 | type Node interface {
 27 | 	GetName() string
 28 | 	PushMessageInternal(message *pbft.MessageReq)
 29 | }
 30 | 
 31 | // MessageReader encapsulates logic for reading messages from flow file
 32 | type MessageReader struct {
 33 | 	lock                   sync.Mutex
 34 | 	file                   *os.File
 35 | 	scanner                *bufio.Scanner
 36 | 	msgProcessingDone      chan string
 37 | 	nodesDoneWithExecution map[pbft.NodeID]bool
 38 | 	lastSequenceMessages   map[pbft.NodeID]*sequenceMessages
 39 | 	prePrepareMessages     map[uint64]*pbft.MessageReq
 40 | }
 41 | 
 42 | // NewMessageReader is the constructor of MessageReader
 43 | func NewMessageReader() *MessageReader {
 44 | 	return &MessageReader{
 45 | 		msgProcessingDone: make(chan string),
 46 | 	}
 47 | }
 48 | 
 49 | // ProcessingDone returns msgProcessingDone chan
 50 | func (r *MessageReader) ProcessingDone() <-chan string {
 51 | 	return r.msgProcessingDone
 52 | }
 53 | 
 54 | // OpenFile opens the file on provided location
 55 | func (r *MessageReader) OpenFile(filePath string) error {
 56 | 	_, err := os.Stat(filePath)
 57 | 	if err != nil {
 58 | 		return err
 59 | 	}
 60 | 
 61 | 	r.file, err = os.Open(filePath)
 62 | 	if err != nil {
 63 | 		return err
 64 | 	}
 65 | 
 66 | 	r.scanner = bufio.NewScanner(r.file)
 67 | 
 68 | 	buffer := []byte{}
 69 | 	r.scanner.Buffer(buffer, maxCharactersPerLine)
 70 | 
 71 | 	return nil
 72 | }
 73 | 
 74 | // CloseFile closes the opened .flow file
 75 | func (r *MessageReader) CloseFile() error {
 76 | 	if r.file != nil {
 77 | 		return r.file.Close()
 78 | 	}
 79 | 	return nil
 80 | }
 81 | 
 82 | // ReadNodeMetaData reads the first line of .flow file which should be a list of nodes
 83 | func (r *MessageReader) ReadNodeMetaData() ([]string, error) {
 84 | 	var nodeNames []string
 85 | 	r.scanner.Scan() // first line carries the node names needed to create appropriate number of nodes for replay
 86 | 	err := json.Unmarshal(r.scanner.Bytes(), &nodeNames)
 87 | 	if err != nil {
 88 | 		return nil, err
 89 | 	} else if len(nodeNames) == 0 {
 90 | 		err = errors.New("no nodes were found in .flow file, so no cluster will be started")
 91 | 	}
 92 | 
 93 | 	return nodeNames, err
 94 | }
 95 | 
 96 | // ReadMessages reads messages from open .flow file and pushes them to appropriate nodes
 97 | func (r *MessageReader) ReadMessages(nodes map[string]Node) {
 98 | 	nodesCount := len(nodes)
 99 | 	r.nodesDoneWithExecution = make(map[pbft.NodeID]bool, nodesCount)
100 | 	r.lastSequenceMessages = make(map[pbft.NodeID]*sequenceMessages, nodesCount)
101 | 	r.prePrepareMessages = make(map[uint64]*pbft.MessageReq)
102 | 
103 | 	messagesChannel := make(chan []*message)
104 | 	doneChannel := make(chan struct{})
105 | 
106 | 	r.startChunkReading(messagesChannel, doneChannel)
107 | 
108 | 	nodeMessages := make(map[pbft.NodeID]map[uint64][]*pbft.MessageReq, nodesCount)
109 | 	for _, n := range nodes {
110 | 		nodeMessages[pbft.NodeID(n.GetName())] = make(map[uint64][]*pbft.MessageReq)
111 | 	}
112 | 
113 | 	isDone := false
114 | LOOP:
115 | 	for !isDone {
116 | 		select {
117 | 		case messages := <-messagesChannel:
118 | 			for _, message := range messages {
119 | 				node, exists := nodes[string(message.To)]
120 | 				if !exists {
121 | 					log.Printf("[WARNING] Could not find node: %v to push message from .flow file.\n", message.To)
122 | 				} else {
123 | 					node.PushMessageInternal(message.Message)
124 | 					nodeMessages[message.To][message.Message.View.Sequence] = append(nodeMessages[message.To][message.Message.View.Sequence], message.Message)
125 | 
126 | 					if !helper.IsTimeoutMessage(message.Message) && message.Message.Type == pbft.MessageReq_Preprepare {
127 | 						if _, isPrePrepareAdded := r.prePrepareMessages[message.Message.View.Sequence]; !isPrePrepareAdded {
128 | 							r.prePrepareMessages[message.Message.View.Sequence] = message.Message
129 | 						}
130 | 					}
131 | 				}
132 | 			}
133 | 		case <-doneChannel:
134 | 			for name, n := range nodeMessages {
135 | 				nodeLastSequence := uint64(0)
136 | 				for sequence := range n {
137 | 					if nodeLastSequence < sequence {
138 | 						nodeLastSequence = sequence
139 | 					}
140 | 				}
141 | 
142 | 				r.lastSequenceMessages[name] = &sequenceMessages{
143 | 					sequence: nodeLastSequence,
144 | 					messages: nodeMessages[name][nodeLastSequence],
145 | 				}
146 | 			}
147 | 			break LOOP
148 | 		}
149 | 	}
150 | }
151 | 
152 | // GetPrePrepareMessages reads messages from .flow file in chunks
153 | func (r *MessageReader) GetPrePrepareMessages(sequence uint64) (*pbft.MessageReq, bool) {
154 | 	msg, ok := r.prePrepareMessages[sequence]
155 | 	return msg, ok
156 | }
157 | 
158 | // startChunkReading reads messages from .flow file in chunks
159 | func (r *MessageReader) startChunkReading(messagesChannel chan []*message, doneChannel chan struct{}) {
160 | 	go func() {
161 | 		messages := make([]*message, 0)
162 | 		i := 0
163 | 		for r.scanner.Scan() {
164 | 			var msg *message
165 | 			if err := json.Unmarshal(r.scanner.Bytes(), &msg); err != nil {
166 | 				log.Printf("[ERROR] Error happened on unmarshalling a message in .flow file. Reason: %v.\n", err)
167 | 				return
168 | 			}
169 | 
170 | 			messages = append(messages, msg)
171 | 			i++
172 | 
173 | 			if i%messageChunkSize == 0 {
174 | 				messagesChannel <- messages
175 | 				messages = nil
176 | 			}
177 | 		}
178 | 
179 | 		if len(messages) > 0 {
180 | 			//its the leftover of messages
181 | 			messagesChannel <- messages
182 | 		}
183 | 
184 | 		doneChannel <- struct{}{}
185 | 	}()
186 | }
187 | 
188 | // checkIfDoneWithExecution checks if node finished with processing all the messages from .flow file
189 | func (r *MessageReader) checkIfDoneWithExecution(validatorId pbft.NodeID, msg *pbft.MessageReq) {
190 | 	if msg.View.Sequence > r.lastSequenceMessages[validatorId].sequence ||
191 | 		(msg.View.Sequence == r.lastSequenceMessages[validatorId].sequence && r.areMessagesFromLastSequenceProcessed(msg, validatorId)) {
192 | 		r.lock.Lock()
193 | 		if _, isDone := r.nodesDoneWithExecution[validatorId]; !isDone {
194 | 			r.nodesDoneWithExecution[validatorId] = true
195 | 			r.msgProcessingDone <- string(validatorId)
196 | 		}
197 | 		r.lock.Unlock()
198 | 	}
199 | }
200 | 
201 | // areMessagesFromLastSequenceProcessed checks if all the messages from the last sequence of given node are processed so that the node can be stoped
202 | func (r *MessageReader) areMessagesFromLastSequenceProcessed(msg *pbft.MessageReq, validatorId pbft.NodeID) bool {
203 | 	lastSequenceMessages := r.lastSequenceMessages[validatorId]
204 | 
205 | 	lastSequenceMessagesCount := len(lastSequenceMessages.messages)
206 | 	if lastSequenceMessagesCount > 0 {
207 | 		messageIndexToRemove := -1
208 | 		for i, message := range lastSequenceMessages.messages {
209 | 			if msg.Equal(message) {
210 | 				messageIndexToRemove = i
211 | 				break
212 | 			}
213 | 		}
214 | 
215 | 		if messageIndexToRemove != -1 {
216 | 			lastSequenceMessages.messages = append(lastSequenceMessages.messages[:messageIndexToRemove], lastSequenceMessages.messages[messageIndexToRemove+1:]...)
217 | 			lastSequenceMessagesCount = len(lastSequenceMessages.messages)
218 | 		}
219 | 	}
220 | 
221 | 	return lastSequenceMessagesCount == 0
222 | }
223 | 


--------------------------------------------------------------------------------
/e2e/transport/gossip.go:
--------------------------------------------------------------------------------
 1 | package transport
 2 | 
 3 | import "github.com/0xPolygon/pbft-consensus"
 4 | 
 5 | type Sender pbft.NodeID
 6 | type Receivers []pbft.NodeID
 7 | 
 8 | // RoundMetadata encapsulates message routing for certain round
 9 | type RoundMetadata struct {
10 | 	Round      uint64
11 | 	RoutingMap map[Sender]Receivers
12 | }
13 | 
14 | // GossipHandler is the callback func which enables determining which message should be gossiped
15 | type GossipHandler func(sender, receiver pbft.NodeID, msg *pbft.MessageReq) bool
16 | 
17 | // GenericGossip is the transport implementation which enables specifying custom gossiping logic
18 | type GenericGossip struct {
19 | 	flowMap       map[uint64]RoundMetadata
20 | 	gossipHandler GossipHandler
21 | }
22 | 
23 | // NewGenericGossip initializes new generic gossip transport
24 | func NewGenericGossip() *GenericGossip {
25 | 	defaultGossipHandler := func(sender, receiver pbft.NodeID, msg *pbft.MessageReq) bool {
26 | 		return true
27 | 	}
28 | 	return &GenericGossip{
29 | 		flowMap:       make(map[uint64]RoundMetadata),
30 | 		gossipHandler: defaultGossipHandler,
31 | 	}
32 | }
33 | 
34 | // WithGossipHandler attaches gossip handler
35 | func (t *GenericGossip) WithGossipHandler(gossipHandler GossipHandler) *GenericGossip {
36 | 	t.gossipHandler = gossipHandler
37 | 	return t
38 | }
39 | 
40 | // WithFlowMap sets message routing per round mapping
41 | func (t *GenericGossip) WithFlowMap(flowMap map[uint64]RoundMetadata) *GenericGossip {
42 | 	t.flowMap = flowMap
43 | 	return t
44 | }
45 | 
46 | // ShouldGossipBasedOnMsgFlowMap determines whether a message should be gossiped, based on provided flow map, which describes messages routing per round.
47 | func (t *GenericGossip) ShouldGossipBasedOnMsgFlowMap(msg *pbft.MessageReq, senderId pbft.NodeID, receiverId pbft.NodeID) bool {
48 | 	roundMedatada, ok := t.flowMap[msg.View.Round]
49 | 	if !ok {
50 | 		return false
51 | 	}
52 | 
53 | 	if roundMedatada.Round == msg.View.Round {
54 | 		receivers, ok := roundMedatada.RoutingMap[Sender(senderId)]
55 | 		if !ok {
56 | 			return false
57 | 		}
58 | 
59 | 		foundReceiver := false
60 | 		for _, v := range receivers {
61 | 			if v == receiverId {
62 | 				foundReceiver = true
63 | 				break
64 | 			}
65 | 		}
66 | 		return foundReceiver
67 | 	}
68 | 	return true
69 | }
70 | 
71 | func (t *GenericGossip) Gossip(from, to pbft.NodeID, msg *pbft.MessageReq) bool {
72 | 	if t.gossipHandler != nil {
73 | 		return t.gossipHandler(from, to, msg)
74 | 	}
75 | 	return true
76 | }
77 | 
78 | func (t *GenericGossip) Connects(from, to pbft.NodeID) bool {
79 | 	return true
80 | }
81 | 
82 | func (t *GenericGossip) Reset() {
83 | 	t.gossipHandler = nil
84 | }
85 | 
86 | func (t *GenericGossip) GetPartitions() map[string][]string {
87 | 	return nil
88 | }
89 | 


--------------------------------------------------------------------------------
/e2e/transport/helper.go:
--------------------------------------------------------------------------------
 1 | package transport
 2 | 
 3 | import (
 4 | 	"math/rand"
 5 | 	"time"
 6 | )
 7 | 
 8 | func timeJitter(jitterMax time.Duration) time.Duration {
 9 | 	return time.Duration(uint64(rand.Int63()) % uint64(jitterMax)) //nolint:golint,gosec
10 | }
11 | 


--------------------------------------------------------------------------------
/e2e/transport/partition.go:
--------------------------------------------------------------------------------
 1 | package transport
 2 | 
 3 | import (
 4 | 	"sync"
 5 | 	"time"
 6 | 
 7 | 	"github.com/0xPolygon/pbft-consensus"
 8 | )
 9 | 
10 | type Partition struct {
11 | 	jitterMax time.Duration
12 | 	lock      sync.Mutex
13 | 	subsets   map[string][]string
14 | }
15 | 
16 | func NewPartition(jitterMax time.Duration) *Partition {
17 | 	return &Partition{jitterMax: jitterMax}
18 | }
19 | 
20 | func (p *Partition) isConnected(from, to pbft.NodeID) bool {
21 | 	if p.subsets == nil {
22 | 		return true
23 | 	}
24 | 
25 | 	subset, ok := p.subsets[string(from)]
26 | 	if !ok {
27 | 		// if not set, they are connected
28 | 		return true
29 | 	}
30 | 
31 | 	found := false
32 | 	for _, i := range subset {
33 | 		if i == string(to) {
34 | 			found = true
35 | 			break
36 | 		}
37 | 	}
38 | 	return found
39 | }
40 | 
41 | func (p *Partition) Connects(from, to pbft.NodeID) bool {
42 | 	p.lock.Lock()
43 | 	defer p.lock.Unlock()
44 | 
45 | 	return p.isConnected(from, to)
46 | }
47 | 
48 | func (p *Partition) Reset() {
49 | 	p.lock.Lock()
50 | 	defer p.lock.Unlock()
51 | 
52 | 	p.subsets = nil
53 | }
54 | 
55 | func (p *Partition) GetPartitions() map[string][]string {
56 | 	return p.subsets
57 | }
58 | 
59 | func (p *Partition) addSubset(from string, to []string) {
60 | 	if p.subsets == nil {
61 | 		p.subsets = map[string][]string{}
62 | 	}
63 | 	if p.subsets[from] == nil {
64 | 		p.subsets[from] = []string{}
65 | 	}
66 | 	p.subsets[from] = append(p.subsets[from], to...)
67 | }
68 | 
69 | func (p *Partition) Partition(subsets ...[]string) {
70 | 	p.lock.Lock()
71 | 	for _, subset := range subsets {
72 | 		for _, i := range subset {
73 | 			p.addSubset(i, subset)
74 | 		}
75 | 	}
76 | 	p.lock.Unlock()
77 | }
78 | 
79 | func (p *Partition) Gossip(from, to pbft.NodeID, msg *pbft.MessageReq) bool {
80 | 	p.lock.Lock()
81 | 	isConnected := p.isConnected(from, to)
82 | 	p.lock.Unlock()
83 | 
84 | 	if !isConnected {
85 | 		return false
86 | 	}
87 | 
88 | 	time.Sleep(timeJitter(p.jitterMax))
89 | 	return true
90 | }
91 | 


--------------------------------------------------------------------------------
/e2e/transport/random.go:
--------------------------------------------------------------------------------
 1 | package transport
 2 | 
 3 | import (
 4 | 	"time"
 5 | 
 6 | 	"github.com/0xPolygon/pbft-consensus"
 7 | )
 8 | 
 9 | // random is the latency transport
10 | type random struct {
11 | 	jitterMax time.Duration
12 | }
13 | 
14 | func NewRandom(jitterMax time.Duration) Hook {
15 | 	return &random{jitterMax: jitterMax}
16 | }
17 | 
18 | func (r *random) Connects(from, to pbft.NodeID) bool {
19 | 	return true
20 | }
21 | 
22 | func (r *random) Gossip(from, to pbft.NodeID, msg *pbft.MessageReq) bool {
23 | 	// adds random latency between the queries
24 | 	if r.jitterMax != 0 {
25 | 		tt := timeJitter(r.jitterMax)
26 | 		time.Sleep(tt)
27 | 	}
28 | 	return true
29 | }
30 | 
31 | func (r *random) Reset() {
32 | 	// no impl
33 | }
34 | 
35 | func (r *random) GetPartitions() map[string][]string {
36 | 	return nil
37 | }
38 | 


--------------------------------------------------------------------------------
/e2e/transport/transport.go:
--------------------------------------------------------------------------------
 1 | package transport
 2 | 
 3 | import (
 4 | 	"log"
 5 | 	"sync"
 6 | 
 7 | 	"github.com/0xPolygon/pbft-consensus"
 8 | )
 9 | 
10 | type Hook interface {
11 | 	Connects(from, to pbft.NodeID) bool
12 | 	Gossip(from, to pbft.NodeID, msg *pbft.MessageReq) bool
13 | 	Reset()
14 | 	GetPartitions() map[string][]string
15 | }
16 | 
17 | type Handler func(pbft.NodeID, *pbft.MessageReq)
18 | 
19 | type Transport struct {
20 | 	lock   sync.Mutex
21 | 	logger *log.Logger
22 | 	nodes  map[pbft.NodeID]Handler
23 | 	hook   Hook
24 | }
25 | 
26 | func (t *Transport) SetLogger(logger *log.Logger) {
27 | 	t.lock.Lock()
28 | 	defer t.lock.Unlock()
29 | 	t.logger = logger
30 | }
31 | 
32 | func (t *Transport) AddHook(hook Hook) {
33 | 	t.lock.Lock()
34 | 	defer t.lock.Unlock()
35 | 	t.hook = hook
36 | }
37 | 
38 | func (t *Transport) GetHook() Hook {
39 | 	t.lock.Lock()
40 | 	defer t.lock.Unlock()
41 | 	return t.hook
42 | }
43 | 
44 | func (t *Transport) Register(name pbft.NodeID, handler Handler) {
45 | 	if t.nodes == nil {
46 | 		t.nodes = map[pbft.NodeID]Handler{}
47 | 	}
48 | 	t.nodes[name] = handler
49 | }
50 | 
51 | func (t *Transport) Gossip(msg *pbft.MessageReq) error {
52 | 	for to, handler := range t.nodes {
53 | 		if msg.From == to {
54 | 			continue
55 | 		}
56 | 		go func(to pbft.NodeID, handler Handler) {
57 | 			send := true
58 | 			if hook := t.GetHook(); hook != nil {
59 | 				send = hook.Gossip(msg.From, to, msg)
60 | 			}
61 | 			if send {
62 | 				handler(to, msg)
63 | 				t.logger.Printf("[TRACE] Message sent to %s - %s", to, msg)
64 | 			} else {
65 | 				t.logger.Printf("[TRACE] Message not sent to %s - %s", to, msg)
66 | 			}
67 | 		}(to, handler)
68 | 	}
69 | 	return nil
70 | }
71 | 


--------------------------------------------------------------------------------
/e2e/validator_set.go:
--------------------------------------------------------------------------------
 1 | package e2e
 2 | 
 3 | import "github.com/0xPolygon/pbft-consensus"
 4 | 
 5 | type ValidatorSet struct {
 6 | 	Nodes        []pbft.NodeID
 7 | 	LastProposer pbft.NodeID
 8 | }
 9 | 
10 | func (n *ValidatorSet) CalcProposer(round uint64) pbft.NodeID {
11 | 	seed := uint64(0)
12 | 	if n.LastProposer == "" {
13 | 		seed = round
14 | 	} else {
15 | 		offset := 0
16 | 		if indx := n.Index(n.LastProposer); indx != -1 {
17 | 			offset = indx
18 | 		}
19 | 		seed = uint64(offset) + round + 1
20 | 	}
21 | 
22 | 	pick := seed % uint64(n.Len())
23 | 
24 | 	return (n.Nodes)[pick]
25 | }
26 | 
27 | func (n *ValidatorSet) Index(addr pbft.NodeID) int {
28 | 	for indx, i := range n.Nodes {
29 | 		if i == addr {
30 | 			return indx
31 | 		}
32 | 	}
33 | 	return -1
34 | }
35 | 
36 | func (n *ValidatorSet) Includes(id pbft.NodeID) bool {
37 | 	for _, i := range n.Nodes {
38 | 		if i == id {
39 | 			return true
40 | 		}
41 | 	}
42 | 	return false
43 | }
44 | 
45 | func (n *ValidatorSet) Len() int {
46 | 	return len(n.Nodes)
47 | }
48 | 
49 | func (n *ValidatorSet) VotingPower() map[pbft.NodeID]uint64 {
50 | 	return pbft.CreateEqualVotingPowerMap(n.Nodes)
51 | }
52 | 


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
 1 | module github.com/0xPolygon/pbft-consensus
 2 | 
 3 | go 1.17
 4 | 
 5 | require (
 6 | 	github.com/stretchr/testify v1.7.0
 7 | 	go.opentelemetry.io/otel v1.1.0
 8 | 	go.opentelemetry.io/otel/trace v1.1.0
 9 | 	pgregory.net/rapid v0.4.7
10 | )
11 | 
12 | require (
13 | 	github.com/davecgh/go-spew v1.1.0 // indirect
14 | 	github.com/kr/pretty v0.1.0 // indirect
15 | 	github.com/pmezard/go-difflib v1.0.0 // indirect
16 | 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
17 | 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
18 | )
19 | 


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 3 | github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 4 | github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 5 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 6 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 7 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 8 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 9 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
10 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
11 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
12 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
13 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
14 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
15 | go.opentelemetry.io/otel v1.1.0 h1:8p0uMLcyyIx0KHNTgO8o3CW8A1aA+dJZJW6PvnMz0Wc=
16 | go.opentelemetry.io/otel v1.1.0/go.mod h1:7cww0OW51jQ8IaZChIEdqLwgh+44+7uiTdWsAL0wQpA=
17 | go.opentelemetry.io/otel/trace v1.1.0 h1:N25T9qCL0+7IpOT8RrRy0WYlL7y6U0WiUJzXcVdXY/o=
18 | go.opentelemetry.io/otel/trace v1.1.0/go.mod h1:i47XtdcBQiktu5IsrPqOHe8w+sBmnLwwHt8wiUsWGTI=
19 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
20 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
21 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
22 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
23 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
24 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
25 | pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g=
26 | pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
27 | 


--------------------------------------------------------------------------------
/interfaces.go:
--------------------------------------------------------------------------------
 1 | package pbft
 2 | 
 3 | // ValidatorSet represents the validator set bahavior
 4 | type ValidatorSet interface {
 5 | 	CalcProposer(round uint64) NodeID
 6 | 	Includes(id NodeID) bool
 7 | 	Len() int
 8 | 	VotingPower() map[NodeID]uint64
 9 | }
10 | 
11 | // Logger represents logger behavior
12 | type Logger interface {
13 | 	Printf(format string, args ...interface{})
14 | 	Print(args ...interface{})
15 | }
16 | 
17 | // Transport is a generic interface for a gossip transport protocol
18 | type Transport interface {
19 | 	// Gossip broadcast the message to the network
20 | 	Gossip(msg *MessageReq) error
21 | }
22 | 
23 | // SignKey represents the behavior of the signing key
24 | type SignKey interface {
25 | 	NodeID() NodeID
26 | 	Sign(b []byte) ([]byte, error)
27 | }
28 | 
29 | // StateNotifier enables custom logic encapsulation related to internal triggers within PBFT state machine (namely receiving timeouts).
30 | type StateNotifier interface {
31 | 	// HandleTimeout notifies that a timeout occurred while getting next message
32 | 	HandleTimeout(to NodeID, msgType MsgType, view *View)
33 | 
34 | 	// ReadNextMessage reads the next message from message queue of the state machine
35 | 	ReadNextMessage(p *Pbft) (*MessageReq, []*MessageReq)
36 | }
37 | 
38 | // Backend represents the backend behavior
39 | type Backend interface {
40 | 	// BuildProposal builds a proposal for the current round (used if proposer)
41 | 	BuildProposal() (*Proposal, error)
42 | 
43 | 	// Height returns the height for the current round
44 | 	Height() uint64
45 | 
46 | 	// Init is used to signal the backend that a new round is going to start.
47 | 	Init(*RoundInfo)
48 | 
49 | 	// Insert inserts the sealed proposal
50 | 	Insert(p *SealedProposal) error
51 | 
52 | 	// IsStuck returns whether the pbft is stucked
53 | 	IsStuck(num uint64) (uint64, bool)
54 | 
55 | 	// Validate validates a raw proposal (used if non-proposer)
56 | 	Validate(*Proposal) error
57 | 
58 | 	// ValidatorSet returns the validator set for the current round
59 | 	ValidatorSet() ValidatorSet
60 | 
61 | 	// ValidateCommit is used to validate that a given commit is valid
62 | 	ValidateCommit(from NodeID, seal []byte) error
63 | }
64 | 


--------------------------------------------------------------------------------
/message.go:
--------------------------------------------------------------------------------
  1 | package pbft
  2 | 
  3 | import (
  4 | 	"bytes"
  5 | 	"fmt"
  6 | )
  7 | 
  8 | type MsgType int32
  9 | 
 10 | const (
 11 | 	MessageReq_RoundChange MsgType = 0
 12 | 	MessageReq_Preprepare  MsgType = 1
 13 | 	MessageReq_Commit      MsgType = 2
 14 | 	MessageReq_Prepare     MsgType = 3
 15 | )
 16 | 
 17 | func (m MsgType) String() string {
 18 | 	switch m {
 19 | 	case MessageReq_RoundChange:
 20 | 		return "RoundChange"
 21 | 	case MessageReq_Preprepare:
 22 | 		return "Preprepare"
 23 | 	case MessageReq_Commit:
 24 | 		return "Commit"
 25 | 	case MessageReq_Prepare:
 26 | 		return "Prepare"
 27 | 	default:
 28 | 		panic(fmt.Sprintf("BUG: Bad msgtype %d", m))
 29 | 	}
 30 | }
 31 | 
 32 | type MessageReq struct {
 33 | 	// type is the type of the message
 34 | 	Type MsgType `json:"type"`
 35 | 
 36 | 	// from is the address of the sender
 37 | 	From NodeID `json:"from"`
 38 | 
 39 | 	// seal is the committed seal for the proposal (only for commit messages)
 40 | 	Seal []byte `json:"seal"`
 41 | 
 42 | 	// view is the view assigned to the message
 43 | 	View *View `json:"view"`
 44 | 
 45 | 	// hash of the proposal
 46 | 	Hash []byte `json:"hash"`
 47 | 
 48 | 	// proposal is the arbitrary data proposal (only for preprepare messages)
 49 | 	Proposal []byte `json:"proposal"`
 50 | }
 51 | 
 52 | func (m MessageReq) String() string {
 53 | 	return fmt.Sprintf("message - type: %s from: %s, view: %v, proposal: %v, hash: %v, seal: %v", m.Type, m.From, m.View, m.Proposal, m.Hash, m.Seal)
 54 | }
 55 | 
 56 | func (m *MessageReq) Validate() error {
 57 | 	// Hash field has to exist for state != RoundStateChange
 58 | 	if m.Type != MessageReq_RoundChange {
 59 | 		if m.Hash == nil {
 60 | 			return fmt.Errorf("hash is empty for type %s", m.Type.String())
 61 | 		}
 62 | 	}
 63 | 
 64 | 	// TODO
 65 | 	return nil
 66 | }
 67 | 
 68 | func (m *MessageReq) SetProposal(proposal []byte) {
 69 | 	m.Proposal = append([]byte{}, proposal...)
 70 | }
 71 | 
 72 | func (m *MessageReq) Copy() *MessageReq {
 73 | 	mm := new(MessageReq)
 74 | 	*mm = *m
 75 | 	if m.View != nil {
 76 | 		mm.View = m.View.Copy()
 77 | 	}
 78 | 
 79 | 	if m.Proposal != nil {
 80 | 		mm.SetProposal(m.Proposal)
 81 | 	}
 82 | 
 83 | 	if m.Seal != nil {
 84 | 		mm.Seal = append([]byte{}, m.Seal...)
 85 | 	}
 86 | 
 87 | 	return mm
 88 | }
 89 | 
 90 | // Equal compares if two messages are equal
 91 | func (m *MessageReq) Equal(other *MessageReq) bool {
 92 | 	return other != nil &&
 93 | 		m.Type == other.Type && m.From == other.From &&
 94 | 		bytes.Equal(m.Proposal, other.Proposal) &&
 95 | 		bytes.Equal(m.Hash, other.Hash) &&
 96 | 		bytes.Equal(m.Seal, other.Seal) &&
 97 | 		m.View.Round == other.View.Round &&
 98 | 		m.View.Sequence == other.View.Sequence
 99 | }
100 | 


--------------------------------------------------------------------------------
/msg_queue.go:
--------------------------------------------------------------------------------
  1 | package pbft
  2 | 
  3 | import (
  4 | 	"container/heap"
  5 | 	"sync"
  6 | )
  7 | 
  8 | // msgQueue defines the structure that holds message queues for different PBFT states
  9 | type msgQueue struct {
 10 | 	// Heap implementation for the round change message queue
 11 | 	roundChangeStateQueue msgQueueImpl
 12 | 
 13 | 	// Heap implementation for the accept state message queue
 14 | 	acceptStateQueue msgQueueImpl
 15 | 
 16 | 	// Heap implementation for the validate state message queue
 17 | 	validateStateQueue msgQueueImpl
 18 | 
 19 | 	queueLock sync.Mutex
 20 | }
 21 | 
 22 | // pushMessage adds a new message to a message queue
 23 | func (m *msgQueue) pushMessage(message *MessageReq) {
 24 | 	m.queueLock.Lock()
 25 | 	defer m.queueLock.Unlock()
 26 | 
 27 | 	queue := m.getQueue(msgToState(message.Type))
 28 | 	heap.Push(queue, message)
 29 | }
 30 | 
 31 | // readMessage reads the message from a message queue, based on the current state and view
 32 | func (m *msgQueue) readMessage(st State, current *View) *MessageReq {
 33 | 	msg, _ := m.readMessageWithDiscards(st, current)
 34 | 	return msg
 35 | }
 36 | 
 37 | func (m *msgQueue) readMessageWithDiscards(st State, current *View) (*MessageReq, []*MessageReq) {
 38 | 	m.queueLock.Lock()
 39 | 	defer m.queueLock.Unlock()
 40 | 
 41 | 	discarded := []*MessageReq{}
 42 | 	queue := m.getQueue(st)
 43 | 
 44 | 	for {
 45 | 		if queue.Len() == 0 {
 46 | 			return nil, discarded
 47 | 		}
 48 | 		msg := queue.head()
 49 | 
 50 | 		// check if the message is from the future
 51 | 		if st == RoundChangeState {
 52 | 			// if we are in RoundChangeState we only care about sequence
 53 | 			// since we are interested in knowing all the possible rounds
 54 | 			if msg.View.Sequence > current.Sequence {
 55 | 				// future message
 56 | 				return nil, discarded
 57 | 			}
 58 | 		} else {
 59 | 			// otherwise, we compare both sequence and round
 60 | 			if cmpView(msg.View, current) > 0 {
 61 | 				// future message
 62 | 				return nil, discarded
 63 | 			}
 64 | 		}
 65 | 
 66 | 		// at this point, 'msg' is good or old, in either case
 67 | 		// we have to remove it from the queue
 68 | 		heap.Pop(queue)
 69 | 
 70 | 		if cmpView(msg.View, current) < 0 {
 71 | 			// old value, try again
 72 | 			discarded = append(discarded, msg)
 73 | 			continue
 74 | 		}
 75 | 
 76 | 		// good value, return it
 77 | 		return msg, discarded
 78 | 	}
 79 | }
 80 | 
 81 | // getQueue checks the passed in state, and returns the corresponding message queue
 82 | func (m *msgQueue) getQueue(st State) *msgQueueImpl {
 83 | 	if st == RoundChangeState {
 84 | 		// round change
 85 | 		return &m.roundChangeStateQueue
 86 | 	} else if st == AcceptState {
 87 | 		// preprepare
 88 | 		return &m.acceptStateQueue
 89 | 	} else {
 90 | 		// prepare and commit
 91 | 		return &m.validateStateQueue
 92 | 	}
 93 | }
 94 | 
 95 | // newMsgQueue creates a new message queue structure
 96 | func newMsgQueue() *msgQueue {
 97 | 	return &msgQueue{
 98 | 		roundChangeStateQueue: msgQueueImpl{},
 99 | 		acceptStateQueue:      msgQueueImpl{},
100 | 		validateStateQueue:    msgQueueImpl{},
101 | 	}
102 | }
103 | 
104 | // msgToState converts the message type to an State
105 | func msgToState(msg MsgType) State {
106 | 	if msg == MessageReq_RoundChange {
107 | 		// round change
108 | 		return RoundChangeState
109 | 	} else if msg == MessageReq_Preprepare {
110 | 		// preprepare
111 | 		return AcceptState
112 | 	} else if msg == MessageReq_Prepare || msg == MessageReq_Commit {
113 | 		// prepare and commit
114 | 		return ValidateState
115 | 	}
116 | 
117 | 	panic("BUG: not expected")
118 | }
119 | 
120 | func stateToMsg(st State) MsgType {
121 | 	switch st {
122 | 	case RoundChangeState:
123 | 		return MessageReq_RoundChange
124 | 	case AcceptState:
125 | 		return MessageReq_Preprepare
126 | 	case ValidateState:
127 | 		return MessageReq_Prepare
128 | 	default:
129 | 		panic("BUG: not expected")
130 | 	}
131 | }
132 | 
133 | type msgQueueImpl []*MessageReq
134 | 
135 | // head returns the head of the queue
136 | func (m msgQueueImpl) head() *MessageReq {
137 | 	return m[0]
138 | }
139 | 
140 | // Len returns the length of the queue
141 | func (m msgQueueImpl) Len() int {
142 | 	return len(m)
143 | }
144 | 
145 | // Less compares the priorities of two items at the passed in indexes (A < B)
146 | func (m msgQueueImpl) Less(i, j int) bool {
147 | 	ti, tj := m[i], m[j]
148 | 	// sort by sequence
149 | 	if ti.View.Sequence != tj.View.Sequence {
150 | 		return ti.View.Sequence < tj.View.Sequence
151 | 	}
152 | 	// sort by round
153 | 	if ti.View.Round != tj.View.Round {
154 | 		return ti.View.Round < tj.View.Round
155 | 	}
156 | 	// sort by message
157 | 	return ti.Type < tj.Type
158 | }
159 | 
160 | // Swap swaps the places of the items at the passed-in indexes
161 | func (m msgQueueImpl) Swap(i, j int) {
162 | 	m[i], m[j] = m[j], m[i]
163 | }
164 | 
165 | // Push adds a new item to the queue
166 | func (m *msgQueueImpl) Push(x interface{}) {
167 | 	*m = append(*m, x.(*MessageReq))
168 | }
169 | 
170 | // Pop removes an item from the queue
171 | func (m *msgQueueImpl) Pop() interface{} {
172 | 	old := *m
173 | 	n := len(old)
174 | 	item := old[n-1]
175 | 	old[n-1] = nil
176 | 	*m = old[0 : n-1]
177 | 	return item
178 | }
179 | 
180 | // cmpView compares two proto views.
181 | //
182 | // If v.Sequence == y.Sequence && v.Round == y.Round => 0
183 | //
184 | // If v.Sequence < y.Sequence => -1 ELSE => 1
185 | //
186 | // If v.Round < y.Round => -1 ELSE 1
187 | func cmpView(v, y *View) int {
188 | 	if v.Sequence != y.Sequence {
189 | 		if v.Sequence < y.Sequence {
190 | 			return -1
191 | 		} else {
192 | 			return 1
193 | 		}
194 | 	}
195 | 	if v.Round != y.Round {
196 | 		if v.Round < y.Round {
197 | 			return -1
198 | 		} else {
199 | 			return 1
200 | 		}
201 | 	}
202 | 
203 | 	return 0
204 | }
205 | 


--------------------------------------------------------------------------------
/msg_queue_test.go:
--------------------------------------------------------------------------------
  1 | package pbft
  2 | 
  3 | import (
  4 | 	"testing"
  5 | 
  6 | 	"github.com/stretchr/testify/assert"
  7 | )
  8 | 
  9 | func TestMsgQueue_RoundChangeState(t *testing.T) {
 10 | 	m := newMsgQueue()
 11 | 
 12 | 	// insert non round change messages
 13 | 	{
 14 | 		m.pushMessage(createMessage("A", MessageReq_Prepare, ViewMsg(1, 1)))
 15 | 		m.pushMessage(createMessage("B", MessageReq_Commit, ViewMsg(1, 1)))
 16 | 
 17 | 		// we only read round state messages
 18 | 		assert.Nil(t, m.readMessage(RoundChangeState, ViewMsg(1, 0)))
 19 | 	}
 20 | 
 21 | 	// insert old round change messages
 22 | 	{
 23 | 		m.pushMessage(createMessage("C", MessageReq_RoundChange, ViewMsg(1, 1)))
 24 | 
 25 | 		// the round change message is old
 26 | 		assert.Nil(t, m.readMessage(RoundChangeState, ViewMsg(2, 0)))
 27 | 		assert.Zero(t, m.roundChangeStateQueue.Len())
 28 | 	}
 29 | 
 30 | 	// insert two valid round change messages with an old one in the middle
 31 | 	{
 32 | 		m.pushMessage(createMessage("D", MessageReq_RoundChange, ViewMsg(2, 2)))
 33 | 		m.pushMessage(createMessage("E", MessageReq_RoundChange, ViewMsg(1, 1)))
 34 | 		m.pushMessage(createMessage("F", MessageReq_RoundChange, ViewMsg(2, 1)))
 35 | 
 36 | 		msg1 := m.readMessage(RoundChangeState, ViewMsg(2, 0))
 37 | 		assert.NotNil(t, msg1)
 38 | 		assert.Equal(t, msg1.From, NodeID("F"))
 39 | 
 40 | 		msg2 := m.readMessage(RoundChangeState, ViewMsg(2, 0))
 41 | 		assert.NotNil(t, msg2)
 42 | 		assert.Equal(t, msg2.From, NodeID("D"))
 43 | 	}
 44 | 
 45 | 	// insert future messages to the queue => such messages should not be retrieved back
 46 | 	{
 47 | 		m = newMsgQueue()
 48 | 		m.pushMessage(createMessage("A", MessageReq_RoundChange, ViewMsg(3, 1)))
 49 | 		assert.Nil(t, m.readMessage(RoundChangeState, ViewMsg(1, 1)))
 50 | 
 51 | 		m.pushMessage(createMessage("A", MessageReq_Commit, ViewMsg(3, 1)))
 52 | 		assert.Nil(t, m.readMessage(CommitState, ViewMsg(1, 1)))
 53 | 	}
 54 | }
 55 | 
 56 | func Test_msgToState(t *testing.T) {
 57 | 	expectedResult := map[MsgType]State{
 58 | 		MessageReq_RoundChange: RoundChangeState,
 59 | 		MessageReq_Preprepare:  AcceptState,
 60 | 		MessageReq_Prepare:     ValidateState,
 61 | 		MessageReq_Commit:      ValidateState,
 62 | 	}
 63 | 	for msgType, st := range expectedResult {
 64 | 		assert.Equal(t, st, msgToState(msgType))
 65 | 	}
 66 | }
 67 | 
 68 | func TestCmpView(t *testing.T) {
 69 | 	var cases = []struct {
 70 | 		x, y           *View
 71 | 		expectedResult int
 72 | 	}{
 73 | 		{
 74 | 			&View{
 75 | 				Sequence: 1,
 76 | 				Round:    1,
 77 | 			},
 78 | 			&View{
 79 | 				Sequence: 2,
 80 | 				Round:    1,
 81 | 			},
 82 | 			-1,
 83 | 		},
 84 | 		{
 85 | 			&View{
 86 | 				Sequence: 2,
 87 | 				Round:    1,
 88 | 			},
 89 | 			&View{
 90 | 				Sequence: 1,
 91 | 				Round:    1,
 92 | 			},
 93 | 			1,
 94 | 		},
 95 | 		{
 96 | 			&View{
 97 | 				Sequence: 1,
 98 | 				Round:    1,
 99 | 			},
100 | 			&View{
101 | 				Sequence: 1,
102 | 				Round:    2,
103 | 			},
104 | 			-1,
105 | 		},
106 | 		{
107 | 			&View{
108 | 				Sequence: 1,
109 | 				Round:    2,
110 | 			},
111 | 			&View{
112 | 				Sequence: 1,
113 | 				Round:    1,
114 | 			},
115 | 			1,
116 | 		},
117 | 		{
118 | 			&View{
119 | 				Sequence: 1,
120 | 				Round:    1,
121 | 			},
122 | 			&View{
123 | 				Sequence: 1,
124 | 				Round:    1,
125 | 			},
126 | 			0,
127 | 		},
128 | 	}
129 | 
130 | 	for _, c := range cases {
131 | 		assert.Equal(t, cmpView(c.x, c.y), c.expectedResult)
132 | 	}
133 | }
134 | 


--------------------------------------------------------------------------------
/proposal.go:
--------------------------------------------------------------------------------
 1 | package pbft
 2 | 
 3 | import (
 4 | 	"bytes"
 5 | 	"time"
 6 | )
 7 | 
 8 | // Proposal is the default proposal
 9 | type Proposal struct {
10 | 	// Data is an arbitrary set of data to approve in consensus
11 | 	Data []byte
12 | 
13 | 	// Time is the time to create the proposal
14 | 	Time time.Time
15 | 
16 | 	// Hash is the digest of the data to seal
17 | 	Hash []byte
18 | }
19 | 
20 | // Equal compares whether two proposals have the same hash
21 | func (p *Proposal) Equal(pp *Proposal) bool {
22 | 	return bytes.Equal(p.Hash, pp.Hash)
23 | }
24 | 
25 | // Copy makes a copy of the Proposal
26 | func (p *Proposal) Copy() *Proposal {
27 | 	pp := new(Proposal)
28 | 	*pp = *p
29 | 
30 | 	pp.Data = append([]byte{}, p.Data...)
31 | 	pp.Hash = append([]byte{}, p.Hash...)
32 | 
33 | 	return pp
34 | }
35 | 


--------------------------------------------------------------------------------
/state.go:
--------------------------------------------------------------------------------
  1 | package pbft
  2 | 
  3 | import (
  4 | 	"sync/atomic"
  5 | 	"time"
  6 | )
  7 | 
  8 | // state defines the current state object in PBFT
  9 | type state struct {
 10 | 	// validators represent the current validator set
 11 | 	validators ValidatorSet
 12 | 
 13 | 	// state is the current state
 14 | 	state uint64
 15 | 
 16 | 	// proposal stores information about the height proposal
 17 | 	proposal *Proposal
 18 | 
 19 | 	// The selected proposer
 20 | 	proposer NodeID
 21 | 
 22 | 	// Current view
 23 | 	view *View
 24 | 
 25 | 	// List of prepared messages
 26 | 	prepared *messages
 27 | 
 28 | 	// List of committed messages
 29 | 	committed *messages
 30 | 
 31 | 	// List of round change messages
 32 | 	roundMessages map[uint64]*messages
 33 | 
 34 | 	// maxFaultyVotingPower represents max tolerable faulty voting power in order to have Byzantine fault tollerance property satisfied
 35 | 	maxFaultyVotingPower uint64
 36 | 
 37 | 	// quorumSize represents minimum accumulated voting power needed to proceed to next PBFT state
 38 | 	quorumSize uint64
 39 | 
 40 | 	// Locked signals whether the proposal is locked
 41 | 	locked uint64
 42 | 
 43 | 	// timeout tracks the time left for this round
 44 | 	timeoutChan <-chan time.Time
 45 | 
 46 | 	// Describes whether there has been an error during the computation
 47 | 	err error
 48 | }
 49 | 
 50 | // newState creates a new state with reset round messages
 51 | func newState() *state {
 52 | 	c := &state{
 53 | 		// this is a default value, it will get reset
 54 | 		// at every iteration
 55 | 		timeoutChan: nil,
 56 | 	}
 57 | 
 58 | 	c.resetRoundMsgs()
 59 | 
 60 | 	return c
 61 | }
 62 | 
 63 | // initializeVotingInfo populates voting information: maximum faulty voting power and quorum size,
 64 | // based on the provided voting power map from ValidatorSet
 65 | func (s *state) initializeVotingInfo() error {
 66 | 	maxFaultyVotingPower, quorumSize, err := CalculateQuorum(s.validators.VotingPower())
 67 | 	if err != nil {
 68 | 		return err
 69 | 	}
 70 | 	s.maxFaultyVotingPower = maxFaultyVotingPower
 71 | 	s.quorumSize = quorumSize
 72 | 	return nil
 73 | }
 74 | 
 75 | // getQuorumSize calculates quorum size (namely the number of required messages of some type in order to proceed to the next state in PBFT state machine).
 76 | // It is calculated by formula:
 77 | // 2 * F + 1, where F denotes maximum count of faulty nodes in order to have Byzantine fault tollerant property satisfied.
 78 | func (s *state) getQuorumSize() uint64 {
 79 | 	return s.quorumSize
 80 | }
 81 | 
 82 | // getMaxFaultyVotingPower is calculated as at most 1/3 of total voting power of the entire validator set.
 83 | func (s *state) getMaxFaultyVotingPower() uint64 {
 84 | 	return s.maxFaultyVotingPower
 85 | }
 86 | 
 87 | func (s *state) IsLocked() bool {
 88 | 	return atomic.LoadUint64(&s.locked) == 1
 89 | }
 90 | 
 91 | func (s *state) GetSequence() uint64 {
 92 | 	return s.view.Sequence
 93 | }
 94 | 
 95 | func (s *state) getCommittedSeals() []CommittedSeal {
 96 | 	committedSeals := make([]CommittedSeal, 0, len(s.committed.messageMap))
 97 | 	for nodeId, commit := range s.committed.messageMap {
 98 | 		committedSeals = append(committedSeals, CommittedSeal{Signature: commit.Seal, NodeID: nodeId})
 99 | 	}
100 | 
101 | 	return committedSeals
102 | }
103 | 
104 | // getState returns the current state
105 | func (s *state) getState() State {
106 | 	stateAddr := &s.state
107 | 
108 | 	return State(atomic.LoadUint64(stateAddr))
109 | }
110 | 
111 | // setState sets the current state
112 | func (s *state) setState(st State) {
113 | 	stateAddr := &s.state
114 | 
115 | 	atomic.StoreUint64(stateAddr, uint64(st))
116 | }
117 | 
118 | // getErr returns the current error, if any, and consumes it
119 | func (s *state) getErr() error {
120 | 	err := s.err
121 | 	s.err = nil
122 | 
123 | 	return err
124 | }
125 | 
126 | // maxRound tries to resolve the round node should fast-track, based on round change messages.
127 | // Quorum size for fast-track higher round is F+1 round change messages (where F denotes max faulty voting power)
128 | func (s *state) maxRound() (maxRound uint64, found bool) {
129 | 	for currentRound, messages := range s.roundMessages {
130 | 		if messages.getAccumulatedVotingPower() < s.getMaxFaultyVotingPower()+1 {
131 | 			continue
132 | 		}
133 | 		if maxRound < currentRound {
134 | 			maxRound = currentRound
135 | 			found = true
136 | 		}
137 | 	}
138 | 
139 | 	return
140 | }
141 | 
142 | // resetRoundMsgs resets the prepared, committed and round messages in the current state
143 | func (s *state) resetRoundMsgs() {
144 | 	s.prepared = newMessages()
145 | 	s.committed = newMessages()
146 | 	s.roundMessages = map[uint64]*messages{}
147 | }
148 | 
149 | // CalcProposer calculates the proposer and sets it to the state
150 | func (s *state) CalcProposer() {
151 | 	s.proposer = s.validators.CalcProposer(s.view.Round)
152 | }
153 | 
154 | func (s *state) lock() {
155 | 	atomic.StoreUint64(&s.locked, 1)
156 | }
157 | 
158 | func (s *state) unlock() {
159 | 	s.proposal = nil
160 | 	atomic.StoreUint64(&s.locked, 0)
161 | }
162 | 
163 | // cleanRound deletes the specific round messages
164 | func (s *state) cleanRound(round uint64) {
165 | 	delete(s.roundMessages, round)
166 | }
167 | 
168 | // addRoundChangeMsg adds a ROUND-CHANGE message to the round, and returns the round message size
169 | func (s *state) addRoundChangeMsg(msg *MessageReq) {
170 | 	if msg.Type != MessageReq_RoundChange {
171 | 		return
172 | 	}
173 | 
174 | 	s.addMessage(msg)
175 | }
176 | 
177 | // addPrepareMsg adds a PREPARE message
178 | func (s *state) addPrepareMsg(msg *MessageReq) {
179 | 	if msg.Type != MessageReq_Prepare {
180 | 		return
181 | 	}
182 | 
183 | 	s.addMessage(msg)
184 | }
185 | 
186 | // addCommitMsg adds a COMMIT message
187 | func (s *state) addCommitMsg(msg *MessageReq) {
188 | 	if msg.Type != MessageReq_Commit {
189 | 		return
190 | 	}
191 | 
192 | 	s.addMessage(msg)
193 | }
194 | 
195 | // addMessage adds a new message to one of the following message lists: committed, prepared, roundMessages
196 | func (s *state) addMessage(msg *MessageReq) {
197 | 	addr := msg.From
198 | 	if !s.validators.Includes(addr) {
199 | 		// only include messages from validators
200 | 		return
201 | 	}
202 | 
203 | 	votingPower := s.validators.VotingPower()[msg.From]
204 | 	if msg.Type == MessageReq_Commit {
205 | 		s.committed.addMessage(msg, votingPower)
206 | 	} else if msg.Type == MessageReq_Prepare {
207 | 		s.prepared.addMessage(msg, votingPower)
208 | 	} else if msg.Type == MessageReq_RoundChange {
209 | 		view := msg.View
210 | 		roundChangeMessages, exists := s.roundMessages[view.Round]
211 | 		if !exists {
212 | 			roundChangeMessages = newMessages()
213 | 			s.roundMessages[view.Round] = roundChangeMessages
214 | 		}
215 | 		roundChangeMessages.addMessage(msg, votingPower)
216 | 	}
217 | }
218 | 
219 | // numPrepared returns the number of messages in the prepared message list
220 | func (s *state) numPrepared() int {
221 | 	return s.prepared.length()
222 | }
223 | 
224 | // numCommitted returns the number of messages in the committed message list
225 | func (s *state) numCommitted() int {
226 | 	return s.committed.length()
227 | }
228 | 
229 | func (s *state) GetCurrentRound() uint64 {
230 | 	return atomic.LoadUint64(&s.view.Round)
231 | }
232 | 
233 | func (s *state) SetCurrentRound(round uint64) {
234 | 	atomic.StoreUint64(&s.view.Round, round)
235 | }
236 | 
237 | type messages struct {
238 | 	messageMap             map[NodeID]*MessageReq
239 | 	accumulatedVotingPower uint64
240 | }
241 | 
242 | func newMessages() *messages {
243 | 	return &messages{
244 | 		messageMap:             make(map[NodeID]*MessageReq),
245 | 		accumulatedVotingPower: 0,
246 | 	}
247 | }
248 | 
249 | func (m *messages) addMessage(message *MessageReq, votingPower uint64) {
250 | 	if _, exists := m.messageMap[message.From]; exists {
251 | 		return
252 | 	}
253 | 	m.messageMap[message.From] = message
254 | 	m.accumulatedVotingPower += votingPower
255 | }
256 | 
257 | func (m messages) getAccumulatedVotingPower() uint64 {
258 | 	return m.accumulatedVotingPower
259 | }
260 | 
261 | func (m messages) length() int {
262 | 	return len(m.messageMap)
263 | }
264 | 


--------------------------------------------------------------------------------
/state_test.go:
--------------------------------------------------------------------------------
  1 | package pbft
  2 | 
  3 | import (
  4 | 	"crypto/ecdsa"
  5 | 	"crypto/elliptic"
  6 | 	crand "crypto/rand"
  7 | 	"fmt"
  8 | 	mrand "math/rand"
  9 | 	"strconv"
 10 | 	"testing"
 11 | 	"time"
 12 | 
 13 | 	"github.com/stretchr/testify/assert"
 14 | 	"github.com/stretchr/testify/require"
 15 | )
 16 | 
 17 | func init() {
 18 | 	mrand.Seed(time.Now().UnixNano())
 19 | }
 20 | 
 21 | // Helper function which enables creation of MessageReq.
 22 | func createMessage(sender NodeID, messageType MsgType, view *View) *MessageReq {
 23 | 	if view == nil {
 24 | 		view = ViewMsg(1, 0)
 25 | 	}
 26 | 	msg := &MessageReq{
 27 | 		From: sender,
 28 | 		Type: messageType,
 29 | 		View: view,
 30 | 	}
 31 | 	switch msg.Type {
 32 | 	case MessageReq_Preprepare:
 33 | 		msg.Proposal = mockProposal
 34 | 		msg.Hash = digest
 35 | 
 36 | 	case MessageReq_Commit:
 37 | 		seal := make([]byte, 2)
 38 | 		mrand.Read(seal)
 39 | 		msg.Seal = seal
 40 | 	}
 41 | 	return msg
 42 | }
 43 | 
 44 | func TestState_AddMessages(t *testing.T) {
 45 | 	pool := newTesterAccountPool()
 46 | 	validatorIds := []NodeID{"A", "B", "C", "D"}
 47 | 	pool.addAccounts(CreateEqualVotingPowerMap(validatorIds))
 48 | 
 49 | 	s, err := initState(pool)
 50 | 	require.NoError(t, err)
 51 | 	s.validators = pool.validatorSet()
 52 | 
 53 | 	// Send message from node which is not amongst validator nodes
 54 | 	s.addMessage(createMessage("E", MessageReq_Prepare, ViewMsg(1, 0)))
 55 | 	assert.Empty(t, s.committed.messageMap)
 56 | 	assert.Empty(t, s.prepared.messageMap)
 57 | 	assert.Empty(t, s.roundMessages)
 58 | 
 59 | 	// -- test committed messages --
 60 | 	s.addMessage(pool.createMessage("A", MessageReq_Commit))
 61 | 	s.addMessage(pool.createMessage("B", MessageReq_Commit))
 62 | 	s.addMessage(pool.createMessage("B", MessageReq_Commit))
 63 | 
 64 | 	assert.Equal(t, 2, s.numCommitted())
 65 | 
 66 | 	// -- test prepare messages --
 67 | 	s.addMessage(pool.createMessage("C", MessageReq_Prepare))
 68 | 	s.addMessage(pool.createMessage("C", MessageReq_Prepare))
 69 | 	s.addMessage(pool.createMessage("D", MessageReq_Prepare))
 70 | 
 71 | 	assert.Equal(t, 2, s.numPrepared())
 72 | 
 73 | 	// -- test round change messages --
 74 | 	rounds := 2
 75 | 	for round := 0; round < rounds; round++ {
 76 | 		for i := 0; i < s.validators.Len(); i++ {
 77 | 			s.addMessage(pool.createMessage(validatorIds[i], MessageReq_RoundChange, uint64(round)))
 78 | 		}
 79 | 	}
 80 | 
 81 | 	for round := 0; round < rounds; round++ {
 82 | 		assert.Equal(t, rounds, len(s.roundMessages))
 83 | 		msgsPerRound := s.roundMessages[uint64(round)]
 84 | 		assert.Equal(t, s.validators.Len(), msgsPerRound.length())
 85 | 	}
 86 | }
 87 | 
 88 | func TestState_MaxRound_Found(t *testing.T) {
 89 | 	const (
 90 | 		validatorsCount = 5
 91 | 		roundsCount     = 6
 92 | 	)
 93 | 
 94 | 	validatorIds := make([]NodeID, validatorsCount)
 95 | 	for i := 0; i < validatorsCount; i++ {
 96 | 		validatorId := fmt.Sprintf("validator_%d", i)
 97 | 		validatorIds[i] = NodeID(validatorId)
 98 | 	}
 99 | 	pool := newTesterAccountPool()
100 | 	pool.addAccounts(CreateEqualVotingPowerMap(validatorIds))
101 | 	s, err := initState(pool)
102 | 	require.NoError(t, err)
103 | 
104 | 	for round := 0; round < roundsCount; round++ {
105 | 		if round%2 == 1 {
106 | 			for _, validatorId := range validatorIds {
107 | 				s.addMessage(pool.createMessage(validatorId, MessageReq_RoundChange, uint64(round)))
108 | 			}
109 | 		} else {
110 | 			s.addMessage(pool.createMessage(validatorIds[mrand.Intn(validatorsCount)], MessageReq_RoundChange, uint64(round)))
111 | 		}
112 | 	}
113 | 
114 | 	maxRound, found := s.maxRound()
115 | 	assert.Equal(t, uint64(5), maxRound)
116 | 	assert.Equal(t, true, found)
117 | }
118 | 
119 | func TestState_MaxRound_NotFound(t *testing.T) {
120 | 	validatorsCount := 7
121 | 
122 | 	validatorIds := make([]NodeID, validatorsCount)
123 | 	for i := 0; i < validatorsCount; i++ {
124 | 		validatorIds[i] = NodeID(fmt.Sprintf("validator_%d", i))
125 | 	}
126 | 	pool := newTesterAccountPool()
127 | 	pool.addAccounts(CreateEqualVotingPowerMap(validatorIds))
128 | 	s, err := initState(pool)
129 | 	require.NoError(t, err)
130 | 
131 | 	// Send wrong message type from some validator, whereas roundMessages map is empty
132 | 	s.addMessage(createMessage(validatorIds[0], MessageReq_Preprepare, ViewMsg(1, 1)))
133 | 
134 | 	maxRound, found := s.maxRound()
135 | 	assert.Equal(t, maxRound, uint64(0))
136 | 	assert.Equal(t, found, false)
137 | 
138 | 	// Seed insufficient "RoundChange" messages count, so that maxRound isn't going to be found
139 | 	for round := range validatorIds {
140 | 		if round%2 == 0 {
141 | 			// Each even round should populate more than one "RoundChange" messages, but just enough that we don't reach census (max faulty nodes+1)
142 | 			for i := 0; i < int(s.getMaxFaultyVotingPower()); i++ {
143 | 				s.addMessage(createMessage(validatorIds[mrand.Intn(validatorsCount)], MessageReq_RoundChange, ViewMsg(1, uint64(round))))
144 | 			}
145 | 		} else {
146 | 			s.addMessage(createMessage(validatorIds[mrand.Intn(validatorsCount)], MessageReq_RoundChange, ViewMsg(1, uint64(round))))
147 | 		}
148 | 	}
149 | 
150 | 	maxRound, found = s.maxRound()
151 | 	assert.Equal(t, uint64(0), maxRound)
152 | 	assert.Equal(t, false, found)
153 | }
154 | 
155 | func TestState_AddRoundMessage(t *testing.T) {
156 | 	s := newState()
157 | 	validatorIds := []NodeID{"A", "B"}
158 | 	s.validators = NewValStringStub(validatorIds, CreateEqualVotingPowerMap(validatorIds))
159 | 
160 | 	// commit message isn't added to the round change messages queue
161 | 	s.addRoundChangeMsg(createMessage("A", MessageReq_Commit, ViewMsg(1, 0)))
162 | 	assert.Empty(t, s.roundMessages)
163 | 	assert.Nil(t, s.roundMessages[0])
164 | 
165 | 	s.addRoundChangeMsg(createMessage("A", MessageReq_RoundChange, ViewMsg(1, 0)))
166 | 	s.addRoundChangeMsg(createMessage("A", MessageReq_RoundChange, ViewMsg(1, 1)))
167 | 	s.addRoundChangeMsg(createMessage("A", MessageReq_RoundChange, ViewMsg(1, 2)))
168 | 
169 | 	s.addRoundChangeMsg(createMessage("B", MessageReq_RoundChange, ViewMsg(1, 2)))
170 | 	assert.Equal(t, 2, s.roundMessages[2].length())
171 | 
172 | 	s.addRoundChangeMsg(createMessage("B", MessageReq_RoundChange, ViewMsg(1, 3)))
173 | 	assert.Len(t, s.roundMessages, 4)
174 | 
175 | 	assert.Empty(t, s.prepared.messageMap)
176 | 	assert.Empty(t, s.committed.messageMap)
177 | }
178 | 
179 | func TestState_addPrepared(t *testing.T) {
180 | 	s := newState()
181 | 	validatorIds := []NodeID{"A", "B"}
182 | 	s.validators = NewValStringStub(validatorIds, CreateEqualVotingPowerMap(validatorIds))
183 | 
184 | 	s.addPrepareMsg(createMessage("A", MessageReq_Commit, ViewMsg(1, 1)))
185 | 	assert.Equal(t, 0, s.prepared.length())
186 | 
187 | 	s.addPrepareMsg(createMessage("A", MessageReq_Prepare, ViewMsg(1, 1)))
188 | 	s.addPrepareMsg(createMessage("B", MessageReq_Prepare, ViewMsg(1, 1)))
189 | 
190 | 	assert.Equal(t, len(validatorIds), s.prepared.length())
191 | 	assert.True(t, s.committed.length() == 0)
192 | 	assert.Empty(t, s.roundMessages)
193 | }
194 | 
195 | func TestState_addCommitted(t *testing.T) {
196 | 	s := newState()
197 | 	validatorIds := []NodeID{"A", "B"}
198 | 	s.validators = NewValStringStub(validatorIds, CreateEqualVotingPowerMap(validatorIds))
199 | 
200 | 	s.addCommitMsg(createMessage("A", MessageReq_Prepare, ViewMsg(1, 1)))
201 | 	assert.True(t, s.committed.length() == 0)
202 | 
203 | 	s.addCommitMsg(createMessage("A", MessageReq_Commit, ViewMsg(1, 1)))
204 | 	s.addCommitMsg(createMessage("B", MessageReq_Commit, ViewMsg(1, 1)))
205 | 
206 | 	assert.Equal(t, len(validatorIds), s.committed.length())
207 | 	assert.True(t, s.prepared.length() == 0)
208 | 	assert.Empty(t, s.roundMessages)
209 | }
210 | 
211 | func TestState_Copy(t *testing.T) {
212 | 	originalMsg := createMessage("A", MessageReq_Preprepare, ViewMsg(1, 0))
213 | 	copyMsg := originalMsg.Copy()
214 | 	assert.NotSame(t, originalMsg, copyMsg)
215 | 	assert.Equal(t, originalMsg, copyMsg)
216 | }
217 | 
218 | func TestState_Lock_Unlock(t *testing.T) {
219 | 	s := newState()
220 | 	proposalData := make([]byte, 2)
221 | 	mrand.Read(proposalData)
222 | 	s.proposal = &Proposal{
223 | 		Data: proposalData,
224 | 		Time: time.Now(),
225 | 	}
226 | 	s.lock()
227 | 	assert.True(t, s.IsLocked())
228 | 	assert.NotNil(t, s.proposal)
229 | 
230 | 	s.unlock()
231 | 	assert.False(t, s.IsLocked())
232 | 	assert.Nil(t, s.proposal)
233 | }
234 | 
235 | func TestState_GetSequence(t *testing.T) {
236 | 	s := newState()
237 | 	s.view = &View{Sequence: 3, Round: 0}
238 | 	assert.True(t, s.GetSequence() == 3)
239 | }
240 | 
241 | func TestState_getCommittedSeals(t *testing.T) {
242 | 	pool := newTesterAccountPool()
243 | 	pool.addAccounts(CreateEqualVotingPowerMap([]NodeID{"A", "B", "C", "D", "E"}))
244 | 
245 | 	s := newState()
246 | 	s.validators = pool.validatorSet()
247 | 
248 | 	s.addCommitMsg(createMessage("A", MessageReq_Commit, ViewMsg(1, 0)))
249 | 	s.addCommitMsg(createMessage("B", MessageReq_Commit, ViewMsg(1, 0)))
250 | 	s.addCommitMsg(createMessage("C", MessageReq_Commit, ViewMsg(1, 0)))
251 | 	committedSeals := s.getCommittedSeals()
252 | 
253 | 	assert.Len(t, committedSeals, 3)
254 | 	processed := map[NodeID]struct{}{}
255 | 	for _, commSeal := range committedSeals {
256 | 		_, exists := processed[commSeal.NodeID]
257 | 		assert.False(t, exists) // all entries in committedSeals should be different
258 | 		processed[commSeal.NodeID] = struct{}{}
259 | 		msg := s.committed.messageMap[commSeal.NodeID]
260 | 		assert.NotNil(t, msg)                         // there should be entry in currentState.committed...
261 | 		assert.Equal(t, commSeal.Signature, msg.Seal) // ...and signatures should match
262 | 	}
263 | }
264 | 
265 | func TestMsgType_ToString(t *testing.T) {
266 | 	expectedMapping := map[MsgType]string{
267 | 		MessageReq_RoundChange: "RoundChange",
268 | 		MessageReq_Preprepare:  "Preprepare",
269 | 		MessageReq_Commit:      "Commit",
270 | 		MessageReq_Prepare:     "Prepare",
271 | 	}
272 | 
273 | 	for msgType, expected := range expectedMapping {
274 | 		assert.Equal(t, expected, msgType.String())
275 | 	}
276 | }
277 | 
278 | func TestState_ToString(t *testing.T) {
279 | 	expectedMapping := map[State]string{
280 | 		AcceptState:      "AcceptState",
281 | 		RoundChangeState: "RoundChangeState",
282 | 		ValidateState:    "ValidateState",
283 | 		CommitState:      "CommitState",
284 | 		SyncState:        "SyncState",
285 | 		DoneState:        "DoneState",
286 | 	}
287 | 
288 | 	for st, expected := range expectedMapping {
289 | 		assert.Equal(t, expected, st.String())
290 | 	}
291 | }
292 | 
293 | func TestState_MaxFaultyVotingPower_EqualVotingPower(t *testing.T) {
294 | 	cases := []struct {
295 | 		nodesCount, faultyNodesCount uint
296 | 	}{
297 | 		{1, 0},
298 | 		{2, 0},
299 | 		{3, 0},
300 | 		{4, 1},
301 | 		{5, 1},
302 | 		{6, 1},
303 | 		{7, 2},
304 | 		{8, 2},
305 | 		{9, 2},
306 | 		{10, 3},
307 | 		{99, 32},
308 | 		{100, 33},
309 | 	}
310 | 	for _, c := range cases {
311 | 		pool := newTesterAccountPool(int(c.nodesCount))
312 | 		state, err := initState(pool)
313 | 		require.NoError(t, err)
314 | 		assert.Equal(t, c.faultyNodesCount, uint(state.getMaxFaultyVotingPower()))
315 | 	}
316 | }
317 | 
318 | func TestState_QuorumSize_EqualVotingPower(t *testing.T) {
319 | 	cases := []struct {
320 | 		nodesCount uint
321 | 		quorumSize uint64
322 | 	}{
323 | 		{1, 1},
324 | 		{2, 1},
325 | 		{3, 1},
326 | 		{4, 3},
327 | 		{5, 3},
328 | 		{6, 3},
329 | 		{7, 5},
330 | 		{8, 5},
331 | 		{9, 5},
332 | 		{10, 7},
333 | 		{100, 67},
334 | 	}
335 | 
336 | 	for _, c := range cases {
337 | 		pool := newTesterAccountPool(int(c.nodesCount))
338 | 		state, err := initState(pool)
339 | 		require.NoError(t, err)
340 | 		assert.Equal(t, c.quorumSize, state.getQuorumSize())
341 | 	}
342 | }
343 | 
344 | func TestState_MaxFaultyVotingPower_MixedVotingPower(t *testing.T) {
345 | 	cases := []struct {
346 | 		votingPower    map[NodeID]uint64
347 | 		maxFaultyNodes uint64
348 | 	}{
349 | 		{map[NodeID]uint64{"A": 5, "B": 5, "C": 6}, 5},
350 | 		{map[NodeID]uint64{"A": 5, "B": 5, "C": 5, "D": 5}, 6},
351 | 		{map[NodeID]uint64{"A": 50, "B": 25, "C": 10, "D": 15}, 33},
352 | 	}
353 | 	for _, c := range cases {
354 | 		pool := newTesterAccountPool()
355 | 		pool.addAccounts(c.votingPower)
356 | 		state, err := initState(pool)
357 | 		require.NoError(t, err)
358 | 		assert.Equal(t, c.maxFaultyNodes, state.getMaxFaultyVotingPower())
359 | 	}
360 | }
361 | 
362 | func TestState_QuorumSize_MixedVotingPower(t *testing.T) {
363 | 	cases := []struct {
364 | 		votingPower map[NodeID]uint64
365 | 		quorumSize  uint64
366 | 	}{
367 | 		{map[NodeID]uint64{"A": 5, "B": 5, "C": 5, "D": 5}, 13},
368 | 		{map[NodeID]uint64{"A": 5, "B": 5, "C": 6}, 11},
369 | 		{map[NodeID]uint64{"A": 50, "B": 25, "C": 10, "D": 15}, 67},
370 | 	}
371 | 	for _, c := range cases {
372 | 		pool := newTesterAccountPool()
373 | 		pool.addAccounts(c.votingPower)
374 | 		state, err := initState(pool)
375 | 		require.NoError(t, err)
376 | 		assert.Equal(t, c.quorumSize, state.getQuorumSize())
377 | 	}
378 | }
379 | 
380 | type signDelegate func([]byte) ([]byte, error)
381 | type testerAccount struct {
382 | 	alias       NodeID
383 | 	priv        *ecdsa.PrivateKey
384 | 	votingPower uint64
385 | 	signFn      signDelegate
386 | }
387 | 
388 | func (t *testerAccount) NodeID() NodeID {
389 | 	return t.alias
390 | }
391 | 
392 | func (t *testerAccount) Sign(b []byte) ([]byte, error) {
393 | 	if t.signFn != nil {
394 | 		return t.signFn(b)
395 | 	}
396 | 	return nil, nil
397 | }
398 | 
399 | type testerAccountPool struct {
400 | 	accounts []*testerAccount
401 | }
402 | 
403 | func newTesterAccountPool(num ...int) *testerAccountPool {
404 | 	t := &testerAccountPool{
405 | 		accounts: []*testerAccount{},
406 | 	}
407 | 	if len(num) == 1 {
408 | 		for i := 0; i < num[0]; i++ {
409 | 			t.accounts = append(t.accounts, &testerAccount{
410 | 				alias:       NodeID(strconv.Itoa(i)),
411 | 				priv:        generateKey(),
412 | 				votingPower: 1,
413 | 			})
414 | 		}
415 | 	}
416 | 	return t
417 | }
418 | 
419 | func (ap *testerAccountPool) addAccounts(votingPowerMap map[NodeID]uint64) {
420 | 	for alias, votingPower := range votingPowerMap {
421 | 		if acct := ap.get(alias); acct != nil {
422 | 			continue
423 | 		}
424 | 		ap.accounts = append(ap.accounts, &testerAccount{
425 | 			alias:       alias,
426 | 			priv:        generateKey(),
427 | 			votingPower: votingPower,
428 | 		})
429 | 	}
430 | }
431 | 
432 | func (ap *testerAccountPool) get(alias NodeID) *testerAccount {
433 | 	for _, account := range ap.accounts {
434 | 		if account.alias == alias {
435 | 			return account
436 | 		}
437 | 	}
438 | 	return nil
439 | }
440 | 
441 | func (ap *testerAccountPool) validatorSet() ValidatorSet {
442 | 	validatorIds := make([]NodeID, len(ap.accounts))
443 | 	votingPowerMap := make(map[NodeID]uint64, len(ap.accounts))
444 | 	for i, acc := range ap.accounts {
445 | 		validatorIds[i] = acc.alias
446 | 		votingPowerMap[acc.alias] = acc.votingPower
447 | 	}
448 | 	return NewValStringStub(validatorIds, votingPowerMap)
449 | }
450 | 
451 | // Helper function which enables creation of MessageReq.
452 | // Note: sender needs to be seeded to the pool before invoking, otherwise senderId will be set to provided sender parameter.
453 | func (ap *testerAccountPool) createMessage(sender NodeID, messageType MsgType, round ...uint64) *MessageReq {
454 | 	poolSender := ap.get(sender)
455 | 	if poolSender != nil {
456 | 		sender = poolSender.alias
457 | 	}
458 | 	r := uint64(0)
459 | 	if len(round) == 1 {
460 | 		r = round[0]
461 | 	}
462 | 	return createMessage(sender, messageType, ViewMsg(1, r))
463 | }
464 | 
465 | func generateKey() *ecdsa.PrivateKey {
466 | 	prv, err := ecdsa.GenerateKey(elliptic.P384(), crand.Reader)
467 | 	if err != nil {
468 | 		panic(err)
469 | 	}
470 | 	return prv
471 | }
472 | 
473 | func initState(accountPool *testerAccountPool) (*state, error) {
474 | 	s := newState()
475 | 	s.validators = accountPool.validatorSet()
476 | 	err := s.initializeVotingInfo()
477 | 	if err != nil {
478 | 		return nil, err
479 | 	}
480 | 	return s, nil
481 | }
482 | 


--------------------------------------------------------------------------------
/stats/stats.go:
--------------------------------------------------------------------------------
 1 | package stats
 2 | 
 3 | import (
 4 | 	"sync"
 5 | 	"time"
 6 | )
 7 | 
 8 | type Stats struct {
 9 | 	lock *sync.Mutex
10 | 
11 | 	round    uint64
12 | 	sequence uint64
13 | 
14 | 	msgCount       map[string]uint64
15 | 	msgVotingPower map[string]uint64
16 | 	stateDuration  map[string]time.Duration
17 | }
18 | 
19 | func NewStats() *Stats {
20 | 	return &Stats{
21 | 		lock:           &sync.Mutex{},
22 | 		msgCount:       make(map[string]uint64),
23 | 		msgVotingPower: make(map[string]uint64),
24 | 		stateDuration:  make(map[string]time.Duration),
25 | 	}
26 | }
27 | 
28 | func (s *Stats) SetView(sequence uint64, round uint64) {
29 | 	s.lock.Lock()
30 | 	defer s.lock.Unlock()
31 | 	s.sequence = sequence
32 | 	s.round = round
33 | }
34 | 
35 | func (s *Stats) IncrMsgCount(msgType string, votingPower uint64) {
36 | 	s.lock.Lock()
37 | 	defer s.lock.Unlock()
38 | 	s.msgCount[msgType]++
39 | 	s.msgVotingPower[msgType] += votingPower
40 | }
41 | 
42 | func (s *Stats) StateDuration(state string, t time.Time) {
43 | 	s.lock.Lock()
44 | 	defer s.lock.Unlock()
45 | 	s.stateDuration[state] = time.Since(t)
46 | }
47 | 
48 | func (s *Stats) Snapshot() Stats {
49 | 	// Allocate a new stats struct
50 | 	stats := NewStats()
51 | 	s.lock.Lock()
52 | 	defer s.lock.Unlock()
53 | 
54 | 	stats.round = s.round
55 | 	stats.sequence = s.sequence
56 | 
57 | 	for msgType, count := range s.msgCount {
58 | 		stats.msgCount[msgType] = count
59 | 	}
60 | 
61 | 	for msgType, votingPower := range s.msgVotingPower {
62 | 		stats.msgVotingPower[msgType] = votingPower
63 | 	}
64 | 
65 | 	for msgType, duration := range s.stateDuration {
66 | 		stats.stateDuration[msgType] = duration
67 | 	}
68 | 
69 | 	return *stats
70 | }
71 | 
72 | func (s *Stats) Reset() {
73 | 	s.lock.Lock()
74 | 	defer s.lock.Unlock()
75 | 
76 | 	s.msgCount = make(map[string]uint64)
77 | 	s.msgVotingPower = make(map[string]uint64)
78 | 	s.stateDuration = make(map[string]time.Duration)
79 | }
80 | 


--------------------------------------------------------------------------------
/stats/stats_test.go:
--------------------------------------------------------------------------------
 1 | package stats
 2 | 
 3 | import (
 4 | 	"sync"
 5 | 	"testing"
 6 | 
 7 | 	"github.com/stretchr/testify/assert"
 8 | )
 9 | 
10 | func TestIncrMsgCount(t *testing.T) {
11 | 	stats := NewStats()
12 | 	var wg sync.WaitGroup
13 | 
14 | 	roundChange := "RoundChange"
15 | 	preprepare := "Preprepare"
16 | 	commit := "Commit"
17 | 	prepare := "Prepare"
18 | 
19 | 	// increment Round Change Message count
20 | 	wg.Add(1)
21 | 	go func() {
22 | 		defer wg.Done()
23 | 		stats.IncrMsgCount(roundChange, 1)
24 | 	}()
25 | 	// increment Prepepare Message count
26 | 	wg.Add(1)
27 | 	go func() {
28 | 		defer wg.Done()
29 | 		stats.IncrMsgCount(preprepare, 3)
30 | 		stats.IncrMsgCount(preprepare, 5)
31 | 	}()
32 | 	// increment Commit Message count
33 | 	wg.Add(1)
34 | 	go func() {
35 | 		defer wg.Done()
36 | 		stats.IncrMsgCount(commit, 3)
37 | 	}()
38 | 	// increment Prepare Message count
39 | 	wg.Add(1)
40 | 	go func() {
41 | 		defer wg.Done()
42 | 		stats.IncrMsgCount(prepare, 2)
43 | 		stats.IncrMsgCount(prepare, 4)
44 | 		stats.IncrMsgCount(prepare, 6)
45 | 	}()
46 | 
47 | 	wg.Wait()
48 | 
49 | 	// assert Round change messages
50 | 	assert.Equal(t, uint64(1), stats.msgCount[roundChange])
51 | 	assert.Equal(t, uint64(1), stats.msgVotingPower[roundChange])
52 | 	// assert Pre-prepare messages
53 | 	assert.Equal(t, uint64(2), stats.msgCount[preprepare])
54 | 	assert.Equal(t, uint64(8), stats.msgVotingPower[preprepare])
55 | 	// assert Commit messages
56 | 	assert.Equal(t, uint64(1), stats.msgCount[commit])
57 | 	assert.Equal(t, uint64(3), stats.msgVotingPower[commit])
58 | 	// assert Prepare messages
59 | 	assert.Equal(t, uint64(3), stats.msgCount[prepare])
60 | 	assert.Equal(t, uint64(12), stats.msgVotingPower[prepare])
61 | }
62 | 
63 | func TestSetView(t *testing.T) {
64 | 	stats := NewStats()
65 | 
66 | 	stats.SetView(uint64(1), uint64(1))
67 | 	// assert Sequence
68 | 	assert.Equal(t, uint64(1), stats.sequence)
69 | 	// assert Round
70 | 	assert.Equal(t, uint64(1), stats.round)
71 | }
72 | 
73 | func TestReset(t *testing.T) {
74 | 	stats := NewStats()
75 | 	preprepare := "Preprepare"
76 | 
77 | 	stats.IncrMsgCount(preprepare, 1)
78 | 	stats.IncrMsgCount(preprepare, 1)
79 | 	stats.Reset()
80 | 
81 | 	assert.Equal(t, uint64(0), stats.msgCount[preprepare])
82 | 	assert.Equal(t, uint64(0), stats.msgVotingPower[preprepare])
83 | }
84 | 


--------------------------------------------------------------------------------
/test_helpers.go:
--------------------------------------------------------------------------------
 1 | package pbft
 2 | 
 3 | type ValidatorKeyMock string
 4 | 
 5 | func (k ValidatorKeyMock) NodeID() NodeID {
 6 | 	return NodeID(k)
 7 | }
 8 | 
 9 | func (k ValidatorKeyMock) Sign(b []byte) ([]byte, error) {
10 | 	return b, nil
11 | }
12 | 
13 | type TransportStub struct {
14 | 	Nodes      []*Pbft
15 | 	GossipFunc func(ft *TransportStub, msg *MessageReq) error
16 | }
17 | 
18 | func (ft *TransportStub) Gossip(msg *MessageReq) error {
19 | 	if ft.GossipFunc != nil {
20 | 		return ft.GossipFunc(ft, msg)
21 | 	}
22 | 
23 | 	for _, node := range ft.Nodes {
24 | 		if msg.From != node.GetValidatorId() {
25 | 			node.PushMessage(msg.Copy())
26 | 		}
27 | 	}
28 | 	return nil
29 | }
30 | 
31 | func NewValStringStub(nodes []NodeID, votingPowerMap map[NodeID]uint64) *ValStringStub {
32 | 	return &ValStringStub{
33 | 		Nodes:          nodes,
34 | 		VotingPowerMap: votingPowerMap,
35 | 	}
36 | }
37 | 
38 | type ValStringStub struct {
39 | 	Nodes          []NodeID
40 | 	VotingPowerMap map[NodeID]uint64
41 | }
42 | 
43 | func (v *ValStringStub) CalcProposer(round uint64) NodeID {
44 | 	seed := uint64(0)
45 | 
46 | 	offset := 0
47 | 	// add last proposer
48 | 
49 | 	seed = uint64(offset) + round
50 | 	pick := seed % uint64(v.Len())
51 | 
52 | 	return (v.Nodes)[pick]
53 | }
54 | 
55 | func (v *ValStringStub) Index(id NodeID) int {
56 | 	for i, currentId := range v.Nodes {
57 | 		if currentId == id {
58 | 			return i
59 | 		}
60 | 	}
61 | 
62 | 	return -1
63 | }
64 | 
65 | func (v *ValStringStub) Includes(id NodeID) bool {
66 | 	for _, currentId := range v.Nodes {
67 | 		if currentId == id {
68 | 			return true
69 | 		}
70 | 	}
71 | 	return false
72 | }
73 | 
74 | func (v *ValStringStub) Len() int {
75 | 	return len(v.Nodes)
76 | }
77 | 
78 | func (v *ValStringStub) VotingPower() map[NodeID]uint64 {
79 | 	return v.VotingPowerMap
80 | }
81 | 
82 | // CreateEqualVotingPowerMap is a helper function which creates map with same weight for every validator id in the provided slice
83 | func CreateEqualVotingPowerMap(validatorIds []NodeID) map[NodeID]uint64 {
84 | 	weightedValidators := make(map[NodeID]uint64, len(validatorIds))
85 | 	for _, validatorId := range validatorIds {
86 | 		weightedValidators[validatorId] = 1
87 | 	}
88 | 	return weightedValidators
89 | }
90 | 


--------------------------------------------------------------------------------
/view.go:
--------------------------------------------------------------------------------
 1 | package pbft
 2 | 
 3 | import "fmt"
 4 | 
 5 | type View struct {
 6 | 	// round is the current round/height being finalized
 7 | 	Round uint64 `json:"round"`
 8 | 
 9 | 	// Sequence is a sequence number inside the round
10 | 	Sequence uint64 `json:"sequence"`
11 | }
12 | 
13 | // ViewMsg is the constructor of View
14 | func ViewMsg(sequence, round uint64) *View {
15 | 	return &View{
16 | 		Round:    round,
17 | 		Sequence: sequence,
18 | 	}
19 | }
20 | 
21 | func (v *View) Copy() *View {
22 | 	vv := new(View)
23 | 	*vv = *v
24 | 	return vv
25 | }
26 | 
27 | func (v *View) String() string {
28 | 	return fmt.Sprintf("(Sequence=%d, Round=%d)", v.Sequence, v.Round)
29 | }
30 | 


--------------------------------------------------------------------------------