├── .github ├── dependabot.yml └── workflows │ ├── codeql-analysis.yml │ ├── gh-workflow-approve.yaml │ ├── govuln.yaml │ ├── static-analysis.yaml │ ├── test_amd64.yaml │ └── test_template.yaml ├── .gitignore ├── .go-version ├── .golangci.yaml ├── CHANGELOG └── CHANGELOG-3.6.md ├── LICENSE ├── Makefile ├── OWNERS ├── README.md ├── bootstrap.go ├── code-of-conduct.md ├── confchange ├── confchange.go ├── datadriven_test.go ├── quick_test.go ├── restore.go ├── restore_test.go └── testdata │ ├── joint_autoleave.txt │ ├── joint_idempotency.txt │ ├── joint_learners_next.txt │ ├── joint_safety.txt │ ├── simple_idempotency.txt │ ├── simple_promote_demote.txt │ ├── simple_safety.txt │ ├── update.txt │ └── zero.txt ├── design.md ├── diff_test.go ├── doc.go ├── example_test.go ├── go.mod ├── go.sum ├── interaction_test.go ├── log.go ├── log_test.go ├── log_unstable.go ├── log_unstable_test.go ├── logger.go ├── node.go ├── node_bench_test.go ├── node_test.go ├── node_util_test.go ├── quorum ├── bench_test.go ├── datadriven_test.go ├── joint.go ├── majority.go ├── quick_test.go ├── quorum.go ├── testdata │ ├── joint_commit.txt │ ├── joint_vote.txt │ ├── majority_commit.txt │ └── majority_vote.txt └── voteresult_string.go ├── raft.go ├── raft_flow_control_test.go ├── raft_paper_test.go ├── raft_snap_test.go ├── raft_test.go ├── raftpb ├── confchange.go ├── confstate.go ├── confstate_test.go ├── raft.pb.go ├── raft.proto └── raft_test.go ├── rafttest ├── doc.go ├── interaction_env.go ├── interaction_env_handler.go ├── interaction_env_handler_add_nodes.go ├── interaction_env_handler_campaign.go ├── interaction_env_handler_compact.go ├── interaction_env_handler_deliver_msgs.go ├── interaction_env_handler_forget_leader.go ├── interaction_env_handler_log_level.go ├── interaction_env_handler_process_append_thread.go ├── interaction_env_handler_process_apply_thread.go ├── interaction_env_handler_process_ready.go ├── interaction_env_handler_propose.go ├── interaction_env_handler_propose_conf_change.go ├── interaction_env_handler_raft_log.go ├── interaction_env_handler_raftstate.go ├── interaction_env_handler_report_unreachable.go ├── interaction_env_handler_send_snapshot.go ├── interaction_env_handler_set_randomized_election_timeout.go ├── interaction_env_handler_stabilize.go ├── interaction_env_handler_status.go ├── interaction_env_handler_tick.go ├── interaction_env_handler_transfer_leadership.go ├── interaction_env_logger.go ├── network.go ├── network_test.go ├── node.go ├── node_bench_test.go └── node_test.go ├── rawnode.go ├── rawnode_test.go ├── read_only.go ├── scripts ├── fix.sh ├── genproto.sh ├── test.sh ├── test_lib.sh └── verify_genproto.sh ├── state_trace.go ├── state_trace_nop.go ├── status.go ├── storage.go ├── storage_test.go ├── testdata ├── async_storage_writes.txt ├── async_storage_writes_append_aba_race.txt ├── campaign.txt ├── campaign_learner_must_vote.txt ├── checkquorum.txt ├── confchange_disable_validation.txt ├── confchange_v1_add_single.txt ├── confchange_v1_remove_leader.txt ├── confchange_v1_remove_leader_stepdown.txt ├── confchange_v2_add_double_auto.txt ├── confchange_v2_add_double_implicit.txt ├── confchange_v2_add_single_auto.txt ├── confchange_v2_add_single_explicit.txt ├── confchange_v2_replace_leader.txt ├── confchange_v2_replace_leader_stepdown.txt ├── forget_leader.txt ├── forget_leader_prevote_checkquorum.txt ├── forget_leader_read_only_lease_based.txt ├── heartbeat_resp_recovers_from_probing.txt ├── lagging_commit.txt ├── prevote.txt ├── prevote_checkquorum.txt ├── probe_and_replicate.txt ├── replicate_pause.txt ├── single_node.txt ├── slow_follower_after_compaction.txt ├── snapshot_succeed_via_app_resp.txt └── snapshot_succeed_via_app_resp_behind.txt ├── tla ├── MCetcdraft.cfg ├── MCetcdraft.tla ├── README.md ├── Traceetcdraft.cfg ├── Traceetcdraft.tla ├── etcdraft.cfg ├── etcdraft.tla ├── example.ndjson ├── validate-model.sh └── validate.sh ├── tools └── mod │ ├── go.mod │ ├── go.sum │ ├── install_all.sh │ ├── libs.go │ └── tools.go ├── tracker ├── inflights.go ├── inflights_test.go ├── progress.go ├── progress_test.go ├── state.go └── tracker.go ├── types.go ├── types_test.go ├── util.go └── util_test.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: / 5 | schedule: 6 | interval: weekly 7 | 8 | - package-ecosystem: gomod 9 | directory: / 10 | schedule: 11 | interval: weekly 12 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ main ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ main ] 20 | schedule: 21 | - cron: '20 14 * * 5' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 37 | # Learn more: 38 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 39 | 40 | steps: 41 | - name: Checkout repository 42 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 43 | 44 | # Initializes the CodeQL tools for scanning. 45 | - name: Initialize CodeQL 46 | uses: github/codeql-action/init@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 47 | with: 48 | languages: ${{ matrix.language }} 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 72 | -------------------------------------------------------------------------------- /.github/workflows/gh-workflow-approve.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Approve GitHub Workflows 3 | 4 | on: 5 | pull_request_target: 6 | types: 7 | - labeled 8 | - synchronize 9 | branches: 10 | - main 11 | 12 | jobs: 13 | approve: 14 | name: Approve ok-to-test 15 | if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') 16 | runs-on: ubuntu-latest 17 | permissions: 18 | actions: write 19 | steps: 20 | - name: Update PR 21 | uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 22 | continue-on-error: true 23 | with: 24 | github-token: ${{ secrets.GITHUB_TOKEN }} 25 | debug: ${{ secrets.ACTIONS_RUNNER_DEBUG == 'true' }} 26 | script: | 27 | const result = await github.rest.actions.listWorkflowRunsForRepo({ 28 | owner: context.repo.owner, 29 | repo: context.repo.repo, 30 | event: "pull_request", 31 | status: "action_required", 32 | head_sha: context.payload.pull_request.head.sha, 33 | per_page: 100 34 | }); 35 | 36 | for (var run of result.data.workflow_runs) { 37 | await github.rest.actions.approveWorkflowRun({ 38 | owner: context.repo.owner, 39 | repo: context.repo.repo, 40 | run_id: run.id 41 | }); 42 | } 43 | -------------------------------------------------------------------------------- /.github/workflows/govuln.yaml: -------------------------------------------------------------------------------- 1 | name: Go Vulnerability Checker 2 | on: [push, pull_request] 3 | jobs: 4 | test: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 8 | - id: goversion 9 | run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" 10 | - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 11 | with: 12 | go-version: ${{ steps.goversion.outputs.goversion }} 13 | - run: date 14 | - run: go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./... 15 | -------------------------------------------------------------------------------- /.github/workflows/static-analysis.yaml: -------------------------------------------------------------------------------- 1 | name: Static Analysis 2 | on: [push, pull_request] 3 | jobs: 4 | run: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 8 | - id: goversion 9 | run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" 10 | - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 11 | with: 12 | go-version: ${{ steps.goversion.outputs.goversion }} 13 | - name: golangci-lint 14 | uses: golangci/golangci-lint-action@051d91933864810ecd5e2ea2cfd98f6a5bca5347 # v6.3.2 15 | with: 16 | version: v1.64.5 17 | - name: protoc 18 | uses: arduino/setup-protoc@149f6c87b92550901b26acd1632e11c3662e381f # v1.3.0 19 | with: 20 | version: '3.20.3' 21 | - run: make verify 22 | -------------------------------------------------------------------------------- /.github/workflows/test_amd64.yaml: -------------------------------------------------------------------------------- 1 | name: Test AMD64 2 | permissions: read-all 3 | on: [push, pull_request] 4 | jobs: 5 | test-linux-amd64: 6 | uses: ./.github/workflows/test_template.yaml 7 | with: 8 | targets: "['linux-amd64-unit-4-cpu-race']" 9 | test-linux-386: 10 | uses: ./.github/workflows/test_template.yaml 11 | with: 12 | targets: "['linux-386-unit-1-cpu']" 13 | 14 | coverage: 15 | needs: 16 | - test-linux-amd64 17 | - test-linux-386 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 21 | - id: goversion 22 | run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" 23 | - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 24 | with: 25 | go-version: ${{ steps.goversion.outputs.goversion }} 26 | - run: make test 27 | -------------------------------------------------------------------------------- /.github/workflows/test_template.yaml: -------------------------------------------------------------------------------- 1 | name: Workflow Test Template 2 | on: 3 | workflow_call: 4 | inputs: 5 | runs-on: 6 | required: false 7 | type: string 8 | default: ubuntu-latest 9 | targets: 10 | required: false 11 | type: string 12 | default: "[]" 13 | permissions: read-all 14 | 15 | jobs: 16 | run: 17 | runs-on: ${{ inputs.runs-on }} 18 | # this is to prevent arm64 jobs from running at forked projects 19 | if: inputs.arch != 'arm64' || github.repository == 'etcd-io/raft' 20 | strategy: 21 | fail-fast: false 22 | matrix: 23 | target: ${{ fromJSON(inputs.targets) }} 24 | steps: 25 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 26 | - id: goversion 27 | run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" 28 | - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 29 | with: 30 | go-version: ${{ steps.goversion.outputs.goversion }} 31 | - env: 32 | TARGET: ${{ matrix.target }} 33 | run: | 34 | go clean -testcache 35 | case "${TARGET}" in 36 | linux-amd64-unit-4-cpu-race) 37 | GOARCH=amd64 PASSES='unit' RACE='true' CPU='4' ./scripts/test.sh -p=2 38 | ;; 39 | linux-386-unit-1-cpu) 40 | GOARCH=386 PASSES='unit' RACE='false' CPU='1' ./scripts/test.sh -p=4 41 | ;; 42 | linux-arm64-unit-4-cpu-race) 43 | GOARCH=arm64 PASSES='unit' RACE='true' CPU='4' ./scripts/test.sh -p=2 44 | ;; 45 | *) 46 | echo "Failed to find target" 47 | exit 1 48 | ;; 49 | esac 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | -------------------------------------------------------------------------------- /.go-version: -------------------------------------------------------------------------------- 1 | 1.24.2 2 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 30m 3 | skip-files: 4 | - "^zz_generated.*" 5 | 6 | issues: 7 | max-same-issues: 0 8 | # Excluding configuration per-path, per-linter, per-text and per-source 9 | exclude-rules: 10 | # exclude ineffassing linter for generated files for conversion 11 | - path: conversion\.go 12 | linters: 13 | - ineffassign 14 | 15 | linters: 16 | disable-all: true 17 | enable: # please keep this alphabetized 18 | # Don't use soon to deprecated[1] linters that lead to false 19 | # https://github.com/golangci/golangci-lint/issues/1841 20 | # - deadcode 21 | # - structcheck 22 | # - varcheck 23 | - goimports 24 | - ineffassign 25 | - revive 26 | - staticcheck 27 | - stylecheck 28 | - unused 29 | - unconvert # Remove unnecessary type conversions 30 | 31 | linters-settings: # please keep this alphabetized 32 | goimports: 33 | local-prefixes: go.etcd.io # Put imports beginning with prefix after 3rd-party packages. 34 | staticcheck: 35 | checks: 36 | - "all" 37 | - "-SA1019" # TODO(fix) Using a deprecated function, variable, constant or field 38 | - "-SA2002" # TODO(fix) Called testing.T.FailNow or SkipNow in a goroutine, which isn’t allowed 39 | stylecheck: 40 | checks: 41 | - "ST1019" # Importing the same package multiple times. 42 | -------------------------------------------------------------------------------- /CHANGELOG/CHANGELOG-3.6.md: -------------------------------------------------------------------------------- 1 | Note that we start to track changes starting from v3.6. 2 | 3 |
4 | 5 | ## v3.6.0(2025-02-05) 6 | 7 | ### Changelog since v3.6.0-beta.0 8 | There isn't any production code change since v3.6.0-beta.0. Only some dependencies 9 | are bumped, and also a minor update on readme in https://github.com/etcd-io/raft/pull/227. 10 | 11 |
12 | 13 | ## v3.6.0-beta.0(2024-11-20) 14 | 15 | ### Changelog since v3.6.0-alpha.0 16 | - [Minor refactoring `raft.maybeSendAppend`](https://github.com/etcd-io/raft/pull/136) 17 | - [Add entryID and logSlice types](https://github.com/etcd-io/raft/pull/145) 18 | - [Fix next index might be smaller than match index](https://github.com/etcd-io/raft/pull/149) 19 | - [Minor refactoring raftLog initialization](https://github.com/etcd-io/raft/pull/151) 20 | - [cleanup Match, Next and MaybeUpdate](https://github.com/etcd-io/raft/pull/165) 21 | - [tracker: track in-flight commit index](https://github.com/etcd-io/raft/pull/171) 22 | - [Replace sort.Slice with slices.Sort, slices.SortFunc](https://github.com/etcd-io/raft/pull/221) 23 | 24 | ### Others 25 | - [Introduce TLA+ trace validation](https://github.com/etcd-io/raft/pull/113) 26 | 27 |
28 | 29 | ## v3.6.0-alpha.0(2024-01-12) 30 | 31 | ### Features 32 | - [Add MaxInflightBytes setting in `raft.Config` for better flow control of entries](https://github.com/etcd-io/etcd/pull/14624) 33 | - [Send empty `MsgApp` when entry in-flight limits are exceeded](https://github.com/etcd-io/etcd/pull/14633) 34 | - [Support asynchronous storage writes](https://github.com/etcd-io/raft/pull/8) 35 | - [Paginate the unapplied config changes scan](https://github.com/etcd-io/raft/pull/32) 36 | - [Add ForgetLeader](https://github.com/etcd-io/raft/pull/78) 37 | - [Add StepDownOnRemoval](https://github.com/etcd-io/raft/pull/79) 38 | - [Accept any snapshot that allows replication](https://github.com/etcd-io/raft/pull/110) 39 | 40 | ### Others 41 | - [Deprecate RawNode.TickQuiesced()](https://github.com/etcd-io/raft/pull/62) 42 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GO_TEST_FLAGS?= 2 | 3 | .PHONY: verify 4 | verify: verify-gofmt verify-dep verify-lint verify-mod-tidy verify-genproto 5 | 6 | .PHONY: verify-gofmt 7 | verify-gofmt: 8 | PASSES="gofmt" ./scripts/test.sh 9 | 10 | .PHONY: verify-dep 11 | verify-dep: 12 | PASSES="dep" ./scripts/test.sh 13 | 14 | .PHONY: verify-lint 15 | verify-lint: 16 | golangci-lint run 17 | 18 | .PHONY: verify-mod-tidy 19 | verify-mod-tidy: 20 | PASSES="mod_tidy" ./scripts/test.sh 21 | 22 | .PHONY: verify-genproto 23 | verify-genproto: 24 | PASSES="genproto" ./scripts/test.sh 25 | 26 | .PHONY: test 27 | test: 28 | PASSES="unit" ./scripts/test.sh $(GO_TEST_FLAGS) 29 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | 3 | approvers: 4 | - ahrtr # Benjamin Wang 5 | - serathius # Marek Siarkowicz 6 | - ptabor # Piotr Tabor 7 | - spzala # Sahdev Zala 8 | reviewers: 9 | - pav-kv # Pavel Kalinnikov 10 | emeritus_approvers: 11 | - tbg # Tobias Grieger 12 | -------------------------------------------------------------------------------- /bootstrap.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raft 16 | 17 | import ( 18 | "errors" 19 | 20 | pb "go.etcd.io/raft/v3/raftpb" 21 | ) 22 | 23 | // Bootstrap initializes the RawNode for first use by appending configuration 24 | // changes for the supplied peers. This method returns an error if the Storage 25 | // is nonempty. 26 | // 27 | // It is recommended that instead of calling this method, applications bootstrap 28 | // their state manually by setting up a Storage that has a first index > 1 and 29 | // which stores the desired ConfState as its InitialState. 30 | func (rn *RawNode) Bootstrap(peers []Peer) error { 31 | if len(peers) == 0 { 32 | return errors.New("must provide at least one peer to Bootstrap") 33 | } 34 | lastIndex, err := rn.raft.raftLog.storage.LastIndex() 35 | if err != nil { 36 | return err 37 | } 38 | 39 | if lastIndex != 0 { 40 | return errors.New("can't bootstrap a nonempty Storage") 41 | } 42 | 43 | // We've faked out initial entries above, but nothing has been 44 | // persisted. Start with an empty HardState (thus the first Ready will 45 | // emit a HardState update for the app to persist). 46 | rn.prevHardSt = emptyState 47 | 48 | // TODO(tbg): remove StartNode and give the application the right tools to 49 | // bootstrap the initial membership in a cleaner way. 50 | rn.raft.becomeFollower(1, None) 51 | ents := make([]pb.Entry, len(peers)) 52 | for i, peer := range peers { 53 | cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context} 54 | data, err := cc.Marshal() 55 | if err != nil { 56 | return err 57 | } 58 | 59 | ents[i] = pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: uint64(i + 1), Data: data} 60 | } 61 | rn.raft.raftLog.append(ents...) 62 | 63 | // Now apply them, mainly so that the application can call Campaign 64 | // immediately after StartNode in tests. Note that these nodes will 65 | // be added to raft twice: here and when the application's Ready 66 | // loop calls ApplyConfChange. The calls to addNode must come after 67 | // all calls to raftLog.append so progress.next is set after these 68 | // bootstrapping entries (it is an error if we try to append these 69 | // entries since they have already been committed). 70 | // We do not set raftLog.applied so the application will be able 71 | // to observe all conf changes via Ready.CommittedEntries. 72 | // 73 | // TODO(bdarnell): These entries are still unstable; do we need to preserve 74 | // the invariant that committed < unstable? 75 | rn.raft.raftLog.committed = uint64(len(ents)) 76 | for _, peer := range peers { 77 | rn.raft.applyConfChange(pb.ConfChange{NodeID: peer.ID, Type: pb.ConfChangeAddNode}.AsV2()) 78 | } 79 | return nil 80 | } 81 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # etcd Community Code of Conduct 2 | 3 | Please refer to [etcd Community Code of Conduct](https://github.com/etcd-io/etcd/blob/main/code-of-conduct.md). 4 | -------------------------------------------------------------------------------- /confchange/datadriven_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package confchange 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "strconv" 21 | "strings" 22 | "testing" 23 | 24 | "github.com/cockroachdb/datadriven" 25 | 26 | pb "go.etcd.io/raft/v3/raftpb" 27 | "go.etcd.io/raft/v3/tracker" 28 | ) 29 | 30 | func TestConfChangeDataDriven(t *testing.T) { 31 | datadriven.Walk(t, "testdata", func(t *testing.T, path string) { 32 | tr := tracker.MakeProgressTracker(10, 0) 33 | c := Changer{ 34 | Tracker: tr, 35 | LastIndex: 0, // incremented in this test with each cmd 36 | } 37 | 38 | // The test files use the commands 39 | // - simple: run a simple conf change (i.e. no joint consensus), 40 | // - enter-joint: enter a joint config, and 41 | // - leave-joint: leave a joint config. 42 | // The first two take a list of config changes, which have the following 43 | // syntax: 44 | // - vn: make n a voter, 45 | // - ln: make n a learner, 46 | // - rn: remove n, and 47 | // - un: update n. 48 | datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string { 49 | defer func() { 50 | c.LastIndex++ 51 | }() 52 | var ccs []pb.ConfChangeSingle 53 | toks := strings.Split(strings.TrimSpace(d.Input), " ") 54 | if toks[0] == "" { 55 | toks = nil 56 | } 57 | for _, tok := range toks { 58 | if len(tok) < 2 { 59 | return fmt.Sprintf("unknown token %s", tok) 60 | } 61 | var cc pb.ConfChangeSingle 62 | switch tok[0] { 63 | case 'v': 64 | cc.Type = pb.ConfChangeAddNode 65 | case 'l': 66 | cc.Type = pb.ConfChangeAddLearnerNode 67 | case 'r': 68 | cc.Type = pb.ConfChangeRemoveNode 69 | case 'u': 70 | cc.Type = pb.ConfChangeUpdateNode 71 | default: 72 | return fmt.Sprintf("unknown input: %s", tok) 73 | } 74 | id, err := strconv.ParseUint(tok[1:], 10, 64) 75 | if err != nil { 76 | return err.Error() 77 | } 78 | cc.NodeID = id 79 | ccs = append(ccs, cc) 80 | } 81 | 82 | var cfg tracker.Config 83 | var trk tracker.ProgressMap 84 | var err error 85 | switch d.Cmd { 86 | case "simple": 87 | cfg, trk, err = c.Simple(ccs...) 88 | case "enter-joint": 89 | var autoLeave bool 90 | if len(d.CmdArgs) > 0 { 91 | d.ScanArgs(t, "autoleave", &autoLeave) 92 | } 93 | cfg, trk, err = c.EnterJoint(autoLeave, ccs...) 94 | case "leave-joint": 95 | if len(ccs) > 0 { 96 | err = errors.New("this command takes no input") 97 | } else { 98 | cfg, trk, err = c.LeaveJoint() 99 | } 100 | default: 101 | return "unknown command" 102 | } 103 | if err != nil { 104 | return err.Error() + "\n" 105 | } 106 | c.Tracker.Config, c.Tracker.Progress = cfg, trk 107 | return fmt.Sprintf("%s\n%s", c.Tracker.Config, c.Tracker.Progress) 108 | }) 109 | }) 110 | } 111 | -------------------------------------------------------------------------------- /confchange/restore_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package confchange 16 | 17 | import ( 18 | "math/rand" 19 | "reflect" 20 | "slices" 21 | "testing" 22 | "testing/quick" 23 | 24 | "github.com/stretchr/testify/assert" 25 | 26 | pb "go.etcd.io/raft/v3/raftpb" 27 | "go.etcd.io/raft/v3/tracker" 28 | ) 29 | 30 | type rndConfChange pb.ConfState 31 | 32 | // Generate creates a random (valid) ConfState for use with quickcheck. 33 | func (rndConfChange) Generate(rand *rand.Rand, _ int) reflect.Value { 34 | conv := func(sl []int) []uint64 { 35 | // We want IDs but the incoming slice is zero-indexed, so add one to 36 | // each. 37 | out := make([]uint64, len(sl)) 38 | for i := range sl { 39 | out[i] = uint64(sl[i] + 1) 40 | } 41 | return out 42 | } 43 | var cs pb.ConfState 44 | // NB: never generate the empty ConfState, that one should be unit tested. 45 | nVoters := 1 + rand.Intn(5) 46 | 47 | nLearners := rand.Intn(5) 48 | // The number of voters that are in the outgoing config but not in the 49 | // incoming one. (We'll additionally retain a random number of the 50 | // incoming voters below). 51 | nRemovedVoters := rand.Intn(3) 52 | 53 | // Voters, learners, and removed voters must not overlap. A "removed voter" 54 | // is one that we have in the outgoing config but not the incoming one. 55 | ids := conv(rand.Perm(2 * (nVoters + nLearners + nRemovedVoters))) 56 | 57 | cs.Voters = ids[:nVoters] 58 | ids = ids[nVoters:] 59 | 60 | if nLearners > 0 { 61 | cs.Learners = ids[:nLearners] 62 | ids = ids[nLearners:] 63 | } 64 | 65 | // Roll the dice on how many of the incoming voters we decide were also 66 | // previously voters. 67 | // 68 | // NB: this code avoids creating non-nil empty slices (here and below). 69 | nOutgoingRetainedVoters := rand.Intn(nVoters + 1) 70 | if nOutgoingRetainedVoters > 0 || nRemovedVoters > 0 { 71 | cs.VotersOutgoing = append([]uint64(nil), cs.Voters[:nOutgoingRetainedVoters]...) 72 | cs.VotersOutgoing = append(cs.VotersOutgoing, ids[:nRemovedVoters]...) 73 | } 74 | // Only outgoing voters that are not also incoming voters can be in 75 | // LearnersNext (they represent demotions). 76 | if nRemovedVoters > 0 { 77 | if nLearnersNext := rand.Intn(nRemovedVoters + 1); nLearnersNext > 0 { 78 | cs.LearnersNext = ids[:nLearnersNext] 79 | } 80 | } 81 | 82 | cs.AutoLeave = len(cs.VotersOutgoing) > 0 && rand.Intn(2) == 1 83 | return reflect.ValueOf(rndConfChange(cs)) 84 | } 85 | 86 | func TestRestore(t *testing.T) { 87 | cfg := quick.Config{MaxCount: 1000} 88 | 89 | f := func(cs pb.ConfState) bool { 90 | chg := Changer{ 91 | Tracker: tracker.MakeProgressTracker(20, 0), 92 | LastIndex: 10, 93 | } 94 | cfg, trk, err := Restore(chg, cs) 95 | if !assert.NoError(t, err) { 96 | return false 97 | } 98 | chg.Tracker.Config = cfg 99 | chg.Tracker.Progress = trk 100 | 101 | for _, sl := range [][]uint64{ 102 | cs.Voters, 103 | cs.Learners, 104 | cs.VotersOutgoing, 105 | cs.LearnersNext, 106 | } { 107 | slices.Sort(sl) 108 | } 109 | 110 | cs2 := chg.Tracker.ConfState() 111 | // NB: cs.Equivalent does the same "sorting" dance internally, but let's 112 | // test it a bit here instead of relying on it. 113 | if assert.Equal(t, cs, cs2) && assert.NoError(t, cs.Equivalent(cs2)) && assert.NoError(t, cs2.Equivalent(cs)) { 114 | return true // success 115 | } 116 | return false 117 | } 118 | 119 | ids := func(sl ...uint64) []uint64 { 120 | return sl 121 | } 122 | 123 | // Unit tests. 124 | for _, cs := range []pb.ConfState{ 125 | {}, 126 | {Voters: ids(1, 2, 3)}, 127 | {Voters: ids(1, 2, 3), Learners: ids(4, 5, 6)}, 128 | {Voters: ids(1, 2, 3), Learners: ids(5), VotersOutgoing: ids(1, 2, 4, 6), LearnersNext: ids(4)}, 129 | } { 130 | if !f(cs) { 131 | t.FailNow() // f() already logged a nice t.Error() 132 | } 133 | } 134 | 135 | assert.NoError(t, quick.Check(func(cs rndConfChange) bool { 136 | return f(pb.ConfState(cs)) 137 | }, &cfg)) 138 | } 139 | -------------------------------------------------------------------------------- /confchange/testdata/joint_autoleave.txt: -------------------------------------------------------------------------------- 1 | # Test the autoleave argument to EnterJoint. It defaults to false in the 2 | # datadriven tests. The flag has no associated semantics in this package, 3 | # it is simply passed through. 4 | simple 5 | v1 6 | ---- 7 | voters=(1) 8 | 1: StateProbe match=0 next=1 9 | 10 | # Autoleave is reflected in the config. 11 | enter-joint autoleave=true 12 | v2 v3 13 | ---- 14 | voters=(1 2 3)&&(1) autoleave 15 | 1: StateProbe match=0 next=1 16 | 2: StateProbe match=0 next=1 17 | 3: StateProbe match=0 next=1 18 | 19 | # Can't enter-joint twice, even if autoleave changes. 20 | enter-joint autoleave=false 21 | ---- 22 | config is already joint 23 | 24 | leave-joint 25 | ---- 26 | voters=(1 2 3) 27 | 1: StateProbe match=0 next=1 28 | 2: StateProbe match=0 next=1 29 | 3: StateProbe match=0 next=1 30 | -------------------------------------------------------------------------------- /confchange/testdata/joint_idempotency.txt: -------------------------------------------------------------------------------- 1 | # Verify that operations upon entering the joint state are idempotent, i.e. 2 | # removing an absent node is fine, etc. 3 | 4 | simple 5 | v1 6 | ---- 7 | voters=(1) 8 | 1: StateProbe match=0 next=1 9 | 10 | enter-joint 11 | r1 r2 r9 v2 v3 v4 v2 v3 v4 l2 l2 r4 r4 l1 l1 12 | ---- 13 | voters=(3)&&(1) learners=(2) learners_next=(1) 14 | 1: StateProbe match=0 next=1 15 | 2: StateProbe match=0 next=1 learner 16 | 3: StateProbe match=0 next=1 17 | 18 | leave-joint 19 | ---- 20 | voters=(3) learners=(1 2) 21 | 1: StateProbe match=0 next=1 learner 22 | 2: StateProbe match=0 next=1 learner 23 | 3: StateProbe match=0 next=1 24 | -------------------------------------------------------------------------------- /confchange/testdata/joint_learners_next.txt: -------------------------------------------------------------------------------- 1 | # Verify that when a voter is demoted in a joint config, it will show up in 2 | # learners_next until the joint config is left, and only then will the progress 3 | # turn into that of a learner, without resetting the progress. Note that this 4 | # last fact is verified by `next`, which can tell us which "round" the progress 5 | # was originally created in. 6 | 7 | simple 8 | v1 9 | ---- 10 | voters=(1) 11 | 1: StateProbe match=0 next=1 12 | 13 | enter-joint 14 | v2 l1 15 | ---- 16 | voters=(2)&&(1) learners_next=(1) 17 | 1: StateProbe match=0 next=1 18 | 2: StateProbe match=0 next=1 19 | 20 | leave-joint 21 | ---- 22 | voters=(2) learners=(1) 23 | 1: StateProbe match=0 next=1 learner 24 | 2: StateProbe match=0 next=1 25 | -------------------------------------------------------------------------------- /confchange/testdata/joint_safety.txt: -------------------------------------------------------------------------------- 1 | leave-joint 2 | ---- 3 | can't leave a non-joint config 4 | 5 | enter-joint 6 | ---- 7 | can't make a zero-voter config joint 8 | 9 | enter-joint 10 | v1 11 | ---- 12 | can't make a zero-voter config joint 13 | 14 | simple 15 | v1 16 | ---- 17 | voters=(1) 18 | 1: StateProbe match=0 next=3 19 | 20 | leave-joint 21 | ---- 22 | can't leave a non-joint config 23 | 24 | # Can enter into joint config. 25 | enter-joint 26 | ---- 27 | voters=(1)&&(1) 28 | 1: StateProbe match=0 next=3 29 | 30 | enter-joint 31 | ---- 32 | config is already joint 33 | 34 | leave-joint 35 | ---- 36 | voters=(1) 37 | 1: StateProbe match=0 next=3 38 | 39 | leave-joint 40 | ---- 41 | can't leave a non-joint config 42 | 43 | # Can enter again, this time with some ops. 44 | enter-joint 45 | r1 v2 v3 l4 46 | ---- 47 | voters=(2 3)&&(1) learners=(4) 48 | 1: StateProbe match=0 next=3 49 | 2: StateProbe match=0 next=9 50 | 3: StateProbe match=0 next=9 51 | 4: StateProbe match=0 next=9 learner 52 | 53 | enter-joint 54 | ---- 55 | config is already joint 56 | 57 | enter-joint 58 | v12 59 | ---- 60 | config is already joint 61 | 62 | simple 63 | l15 64 | ---- 65 | can't apply simple config change in joint config 66 | 67 | leave-joint 68 | ---- 69 | voters=(2 3) learners=(4) 70 | 2: StateProbe match=0 next=9 71 | 3: StateProbe match=0 next=9 72 | 4: StateProbe match=0 next=9 learner 73 | 74 | simple 75 | l9 76 | ---- 77 | voters=(2 3) learners=(4 9) 78 | 2: StateProbe match=0 next=9 79 | 3: StateProbe match=0 next=9 80 | 4: StateProbe match=0 next=9 learner 81 | 9: StateProbe match=0 next=14 learner 82 | -------------------------------------------------------------------------------- /confchange/testdata/simple_idempotency.txt: -------------------------------------------------------------------------------- 1 | simple 2 | v1 3 | ---- 4 | voters=(1) 5 | 1: StateProbe match=0 next=1 6 | 7 | simple 8 | v1 9 | ---- 10 | voters=(1) 11 | 1: StateProbe match=0 next=1 12 | 13 | simple 14 | v2 15 | ---- 16 | voters=(1 2) 17 | 1: StateProbe match=0 next=1 18 | 2: StateProbe match=0 next=2 19 | 20 | simple 21 | l1 22 | ---- 23 | voters=(2) learners=(1) 24 | 1: StateProbe match=0 next=1 learner 25 | 2: StateProbe match=0 next=2 26 | 27 | simple 28 | l1 29 | ---- 30 | voters=(2) learners=(1) 31 | 1: StateProbe match=0 next=1 learner 32 | 2: StateProbe match=0 next=2 33 | 34 | simple 35 | r1 36 | ---- 37 | voters=(2) 38 | 2: StateProbe match=0 next=2 39 | 40 | simple 41 | r1 42 | ---- 43 | voters=(2) 44 | 2: StateProbe match=0 next=2 45 | 46 | simple 47 | v3 48 | ---- 49 | voters=(2 3) 50 | 2: StateProbe match=0 next=2 51 | 3: StateProbe match=0 next=7 52 | 53 | simple 54 | r3 55 | ---- 56 | voters=(2) 57 | 2: StateProbe match=0 next=2 58 | 59 | simple 60 | r3 61 | ---- 62 | voters=(2) 63 | 2: StateProbe match=0 next=2 64 | 65 | simple 66 | r4 67 | ---- 68 | voters=(2) 69 | 2: StateProbe match=0 next=2 70 | -------------------------------------------------------------------------------- /confchange/testdata/simple_promote_demote.txt: -------------------------------------------------------------------------------- 1 | # Set up three voters for this test. 2 | 3 | simple 4 | v1 5 | ---- 6 | voters=(1) 7 | 1: StateProbe match=0 next=1 8 | 9 | simple 10 | v2 11 | ---- 12 | voters=(1 2) 13 | 1: StateProbe match=0 next=1 14 | 2: StateProbe match=0 next=1 15 | 16 | simple 17 | v3 18 | ---- 19 | voters=(1 2 3) 20 | 1: StateProbe match=0 next=1 21 | 2: StateProbe match=0 next=1 22 | 3: StateProbe match=0 next=2 23 | 24 | # Can atomically demote and promote without a hitch. 25 | # This is pointless, but possible. 26 | simple 27 | l1 v1 28 | ---- 29 | voters=(1 2 3) 30 | 1: StateProbe match=0 next=1 31 | 2: StateProbe match=0 next=1 32 | 3: StateProbe match=0 next=2 33 | 34 | # Can demote a voter. 35 | simple 36 | l2 37 | ---- 38 | voters=(1 3) learners=(2) 39 | 1: StateProbe match=0 next=1 40 | 2: StateProbe match=0 next=1 learner 41 | 3: StateProbe match=0 next=2 42 | 43 | # Can atomically promote and demote the same voter. 44 | # This is pointless, but possible. 45 | simple 46 | v2 l2 47 | ---- 48 | voters=(1 3) learners=(2) 49 | 1: StateProbe match=0 next=1 50 | 2: StateProbe match=0 next=1 learner 51 | 3: StateProbe match=0 next=2 52 | 53 | # Can promote a voter. 54 | simple 55 | v2 56 | ---- 57 | voters=(1 2 3) 58 | 1: StateProbe match=0 next=1 59 | 2: StateProbe match=0 next=1 60 | 3: StateProbe match=0 next=2 61 | -------------------------------------------------------------------------------- /confchange/testdata/simple_safety.txt: -------------------------------------------------------------------------------- 1 | simple 2 | l1 3 | ---- 4 | removed all voters 5 | 6 | simple 7 | v1 8 | ---- 9 | voters=(1) 10 | 1: StateProbe match=0 next=1 11 | 12 | simple 13 | v2 l3 14 | ---- 15 | voters=(1 2) learners=(3) 16 | 1: StateProbe match=0 next=1 17 | 2: StateProbe match=0 next=2 18 | 3: StateProbe match=0 next=2 learner 19 | 20 | simple 21 | r1 v5 22 | ---- 23 | more than one voter changed without entering joint config 24 | 25 | simple 26 | r1 r2 27 | ---- 28 | removed all voters 29 | 30 | simple 31 | v3 v4 32 | ---- 33 | more than one voter changed without entering joint config 34 | 35 | simple 36 | l1 v5 37 | ---- 38 | more than one voter changed without entering joint config 39 | 40 | simple 41 | l1 l2 42 | ---- 43 | removed all voters 44 | 45 | simple 46 | l2 l3 l4 l5 47 | ---- 48 | voters=(1) learners=(2 3 4 5) 49 | 1: StateProbe match=0 next=1 50 | 2: StateProbe match=0 next=2 learner 51 | 3: StateProbe match=0 next=2 learner 52 | 4: StateProbe match=0 next=8 learner 53 | 5: StateProbe match=0 next=8 learner 54 | 55 | simple 56 | r1 57 | ---- 58 | removed all voters 59 | 60 | simple 61 | r2 r3 r4 r5 62 | ---- 63 | voters=(1) 64 | 1: StateProbe match=0 next=1 65 | -------------------------------------------------------------------------------- /confchange/testdata/update.txt: -------------------------------------------------------------------------------- 1 | # Nobody cares about ConfChangeUpdateNode, but at least use it once. It is used 2 | # by etcd as a convenient way to pass a blob through their conf change machinery 3 | # that updates information tracked outside of raft. 4 | 5 | simple 6 | v1 7 | ---- 8 | voters=(1) 9 | 1: StateProbe match=0 next=1 10 | 11 | simple 12 | v2 u1 13 | ---- 14 | voters=(1 2) 15 | 1: StateProbe match=0 next=1 16 | 2: StateProbe match=0 next=1 17 | 18 | simple 19 | u1 u2 u3 u1 u2 u3 20 | ---- 21 | voters=(1 2) 22 | 1: StateProbe match=0 next=1 23 | 2: StateProbe match=0 next=1 24 | -------------------------------------------------------------------------------- /confchange/testdata/zero.txt: -------------------------------------------------------------------------------- 1 | # NodeID zero is ignored. 2 | simple 3 | v1 r0 v0 l0 4 | ---- 5 | voters=(1) 6 | 1: StateProbe match=0 next=1 7 | -------------------------------------------------------------------------------- /diff_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raft 16 | 17 | import ( 18 | "fmt" 19 | "io" 20 | "os" 21 | "os/exec" 22 | "strings" 23 | ) 24 | 25 | func diffu(a, b string) string { 26 | if a == b { 27 | return "" 28 | } 29 | aname, bname := mustTemp("base", a), mustTemp("other", b) 30 | defer os.Remove(aname) 31 | defer os.Remove(bname) 32 | cmd := exec.Command("diff", "-u", aname, bname) 33 | buf, err := cmd.CombinedOutput() 34 | if err != nil { 35 | if _, ok := err.(*exec.ExitError); ok { 36 | // do nothing 37 | return string(buf) 38 | } 39 | panic(err) 40 | } 41 | return string(buf) 42 | } 43 | 44 | func mustTemp(pre, body string) string { 45 | f, err := os.CreateTemp("", pre) 46 | if err != nil { 47 | panic(err) 48 | } 49 | _, err = io.Copy(f, strings.NewReader(body)) 50 | if err != nil { 51 | panic(err) 52 | } 53 | f.Close() 54 | return f.Name() 55 | } 56 | 57 | func ltoa(l *raftLog) string { 58 | s := fmt.Sprintf("lastIndex: %d\n", l.lastIndex()) 59 | s += fmt.Sprintf("applied: %d\n", l.applied) 60 | s += fmt.Sprintf("applying: %d\n", l.applying) 61 | for i, e := range l.allEntries() { 62 | s += fmt.Sprintf("#%d: %+v\n", i, e) 63 | } 64 | return s 65 | } 66 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raft 16 | 17 | import ( 18 | pb "go.etcd.io/raft/v3/raftpb" 19 | ) 20 | 21 | func applyToStore(_ []pb.Entry) {} 22 | func sendMessages(_ []pb.Message) {} 23 | func saveStateToDisk(_ pb.HardState) {} 24 | func saveToDisk(_ []pb.Entry) {} 25 | 26 | func ExampleNode() { 27 | c := &Config{} 28 | n := StartNode(c, nil) 29 | defer n.Stop() 30 | 31 | // stuff to n happens in other goroutines 32 | 33 | // the last known state 34 | var prev pb.HardState 35 | for { 36 | // Ready blocks until there is new state ready. 37 | rd := <-n.Ready() 38 | if !isHardStateEqual(prev, rd.HardState) { 39 | saveStateToDisk(rd.HardState) 40 | prev = rd.HardState 41 | } 42 | 43 | saveToDisk(rd.Entries) 44 | go applyToStore(rd.CommittedEntries) 45 | sendMessages(rd.Messages) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module go.etcd.io/raft/v3 2 | 3 | go 1.24 4 | 5 | toolchain go1.24.3 6 | 7 | require ( 8 | github.com/cockroachdb/datadriven v1.0.2 9 | github.com/gogo/protobuf v1.3.2 10 | github.com/golang/protobuf v1.5.4 11 | github.com/stretchr/testify v1.10.0 12 | ) 13 | 14 | require ( 15 | github.com/davecgh/go-spew v1.1.1 // indirect 16 | github.com/google/go-cmp v0.5.8 // indirect 17 | github.com/pmezard/go-difflib v1.0.0 // indirect 18 | google.golang.org/protobuf v1.33.0 // indirect 19 | gopkg.in/yaml.v3 v3.0.1 // indirect 20 | ) 21 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= 2 | github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 6 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 7 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 8 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 9 | github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= 10 | github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 11 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 12 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 13 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 14 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 15 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 16 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 17 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 18 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 19 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 20 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 21 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 22 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 23 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 24 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 25 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 26 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 27 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 28 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 29 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 30 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 31 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 32 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 33 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 34 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 35 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 36 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 37 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 38 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 39 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 40 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 41 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 42 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 43 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 44 | google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= 45 | google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 46 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 47 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 48 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 49 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 50 | -------------------------------------------------------------------------------- /interaction_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raft_test 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/cockroachdb/datadriven" 21 | 22 | "go.etcd.io/raft/v3" 23 | "go.etcd.io/raft/v3/rafttest" 24 | ) 25 | 26 | func TestInteraction(t *testing.T) { 27 | // NB: if this test fails, run `go test ./raft -rewrite` and inspect the 28 | // diff. Only commit the changes if you understand what caused them and if 29 | // they are desired. 30 | datadriven.Walk(t, "testdata", func(t *testing.T, path string) { 31 | env := rafttest.NewInteractionEnv(&rafttest.InteractionOpts{ 32 | SetRandomizedElectionTimeout: raft.SetRandomizedElectionTimeout, 33 | }) 34 | datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string { 35 | return env.Handle(t, *d) 36 | }) 37 | }) 38 | } 39 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raft 16 | 17 | import ( 18 | "fmt" 19 | "io" 20 | "log" 21 | "os" 22 | "sync" 23 | ) 24 | 25 | type Logger interface { 26 | Debug(v ...interface{}) 27 | Debugf(format string, v ...interface{}) 28 | 29 | Error(v ...interface{}) 30 | Errorf(format string, v ...interface{}) 31 | 32 | Info(v ...interface{}) 33 | Infof(format string, v ...interface{}) 34 | 35 | Warning(v ...interface{}) 36 | Warningf(format string, v ...interface{}) 37 | 38 | Fatal(v ...interface{}) 39 | Fatalf(format string, v ...interface{}) 40 | 41 | Panic(v ...interface{}) 42 | Panicf(format string, v ...interface{}) 43 | } 44 | 45 | func SetLogger(l Logger) { 46 | raftLoggerMu.Lock() 47 | raftLogger = l 48 | raftLoggerMu.Unlock() 49 | } 50 | 51 | func ResetDefaultLogger() { 52 | SetLogger(defaultLogger) 53 | } 54 | 55 | func getLogger() Logger { 56 | raftLoggerMu.Lock() 57 | defer raftLoggerMu.Unlock() 58 | return raftLogger 59 | } 60 | 61 | var ( 62 | defaultLogger = &DefaultLogger{Logger: log.New(os.Stderr, "raft", log.LstdFlags)} 63 | discardLogger = &DefaultLogger{Logger: log.New(io.Discard, "", 0)} 64 | raftLoggerMu sync.Mutex 65 | raftLogger = Logger(defaultLogger) 66 | ) 67 | 68 | const ( 69 | calldepth = 2 70 | ) 71 | 72 | // DefaultLogger is a default implementation of the Logger interface. 73 | type DefaultLogger struct { 74 | *log.Logger 75 | debug bool 76 | } 77 | 78 | func (l *DefaultLogger) EnableTimestamps() { 79 | l.SetFlags(l.Flags() | log.Ldate | log.Ltime) 80 | } 81 | 82 | func (l *DefaultLogger) EnableDebug() { 83 | l.debug = true 84 | } 85 | 86 | func (l *DefaultLogger) Debug(v ...interface{}) { 87 | if l.debug { 88 | l.Output(calldepth, header("DEBUG", fmt.Sprint(v...))) 89 | } 90 | } 91 | 92 | func (l *DefaultLogger) Debugf(format string, v ...interface{}) { 93 | if l.debug { 94 | l.Output(calldepth, header("DEBUG", fmt.Sprintf(format, v...))) 95 | } 96 | } 97 | 98 | func (l *DefaultLogger) Info(v ...interface{}) { 99 | l.Output(calldepth, header("INFO", fmt.Sprint(v...))) 100 | } 101 | 102 | func (l *DefaultLogger) Infof(format string, v ...interface{}) { 103 | l.Output(calldepth, header("INFO", fmt.Sprintf(format, v...))) 104 | } 105 | 106 | func (l *DefaultLogger) Error(v ...interface{}) { 107 | l.Output(calldepth, header("ERROR", fmt.Sprint(v...))) 108 | } 109 | 110 | func (l *DefaultLogger) Errorf(format string, v ...interface{}) { 111 | l.Output(calldepth, header("ERROR", fmt.Sprintf(format, v...))) 112 | } 113 | 114 | func (l *DefaultLogger) Warning(v ...interface{}) { 115 | l.Output(calldepth, header("WARN", fmt.Sprint(v...))) 116 | } 117 | 118 | func (l *DefaultLogger) Warningf(format string, v ...interface{}) { 119 | l.Output(calldepth, header("WARN", fmt.Sprintf(format, v...))) 120 | } 121 | 122 | func (l *DefaultLogger) Fatal(v ...interface{}) { 123 | l.Output(calldepth, header("FATAL", fmt.Sprint(v...))) 124 | os.Exit(1) 125 | } 126 | 127 | func (l *DefaultLogger) Fatalf(format string, v ...interface{}) { 128 | l.Output(calldepth, header("FATAL", fmt.Sprintf(format, v...))) 129 | os.Exit(1) 130 | } 131 | 132 | func (l *DefaultLogger) Panic(v ...interface{}) { 133 | l.Logger.Panic(v...) 134 | } 135 | 136 | func (l *DefaultLogger) Panicf(format string, v ...interface{}) { 137 | l.Logger.Panicf(format, v...) 138 | } 139 | 140 | func header(lvl, msg string) string { 141 | return fmt.Sprintf("%s: %s", lvl, msg) 142 | } 143 | -------------------------------------------------------------------------------- /node_bench_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raft 16 | 17 | import ( 18 | "context" 19 | "testing" 20 | "time" 21 | ) 22 | 23 | func BenchmarkOneNode(b *testing.B) { 24 | ctx, cancel := context.WithCancel(context.Background()) 25 | defer cancel() 26 | 27 | s := newTestMemoryStorage(withPeers(1)) 28 | rn := newTestRawNode(1, 10, 1, s) 29 | n := newNode(rn) 30 | go n.run() 31 | 32 | defer n.Stop() 33 | 34 | n.Campaign(ctx) 35 | go func() { 36 | for i := 0; i < b.N; i++ { 37 | n.Propose(ctx, []byte("foo")) 38 | } 39 | }() 40 | 41 | for { 42 | rd := <-n.Ready() 43 | s.Append(rd.Entries) 44 | // a reasonable disk sync latency 45 | time.Sleep(1 * time.Millisecond) 46 | n.Advance() 47 | if rd.HardState.Commit == uint64(b.N+1) { 48 | return 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /node_util_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raft 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "testing" 21 | "time" 22 | ) 23 | 24 | type nodeTestHarness struct { 25 | *node 26 | t *testing.T 27 | } 28 | 29 | func (l *nodeTestHarness) Debug(v ...interface{}) { 30 | l.t.Log(v...) 31 | } 32 | 33 | func (l *nodeTestHarness) Debugf(format string, v ...interface{}) { 34 | l.t.Logf(format, v...) 35 | } 36 | 37 | func (l *nodeTestHarness) Error(v ...interface{}) { 38 | l.t.Error(v...) 39 | } 40 | 41 | func (l *nodeTestHarness) Errorf(format string, v ...interface{}) { 42 | l.t.Errorf(format, v...) 43 | } 44 | 45 | func (l *nodeTestHarness) Info(v ...interface{}) { 46 | l.t.Log(v...) 47 | } 48 | 49 | func (l *nodeTestHarness) Infof(format string, v ...interface{}) { 50 | l.t.Logf(format, v...) 51 | } 52 | 53 | func (l *nodeTestHarness) Warning(v ...interface{}) { 54 | l.t.Log(v...) 55 | } 56 | 57 | func (l *nodeTestHarness) Warningf(format string, v ...interface{}) { 58 | l.t.Logf(format, v...) 59 | } 60 | 61 | func (l *nodeTestHarness) Fatal(v ...interface{}) { 62 | l.t.Error(v...) 63 | panic(fmt.Sprint(v...)) 64 | } 65 | 66 | func (l *nodeTestHarness) Fatalf(format string, v ...interface{}) { 67 | l.t.Errorf(format, v...) 68 | panic(fmt.Sprintf(format, v...)) 69 | } 70 | 71 | func (l *nodeTestHarness) Panic(v ...interface{}) { 72 | l.t.Log(v...) 73 | panic(fmt.Sprint(v...)) 74 | } 75 | 76 | func (l *nodeTestHarness) Panicf(format string, v ...interface{}) { 77 | l.t.Errorf(format, v...) 78 | panic(fmt.Sprintf(format, v...)) 79 | } 80 | 81 | func newNodeTestHarness(ctx context.Context, t *testing.T, cfg *Config, peers ...Peer) (_ context.Context, cancel func(), _ *nodeTestHarness) { 82 | // Wrap context in a 10s timeout to make tests more robust. Otherwise, 83 | // it's likely that deadlock will occur unless Node behaves exactly as 84 | // expected - when you expect a Ready and start waiting on the channel 85 | // but no Ready ever shows up, for example. 86 | ctx, cancel = context.WithTimeout(ctx, 10*time.Second) 87 | var n *node 88 | if len(peers) > 0 { 89 | n = setupNode(cfg, peers) 90 | } else { 91 | rn, err := NewRawNode(cfg) 92 | if err != nil { 93 | t.Fatal(err) 94 | } 95 | nn := newNode(rn) 96 | n = &nn 97 | } 98 | go func() { 99 | defer func() { 100 | if r := recover(); r != nil { 101 | t.Error(r) 102 | } 103 | }() 104 | defer cancel() 105 | defer n.Stop() 106 | n.run() 107 | }() 108 | t.Cleanup(n.Stop) 109 | return ctx, cancel, &nodeTestHarness{node: n, t: t} 110 | } 111 | -------------------------------------------------------------------------------- /quorum/bench_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package quorum 16 | 17 | import ( 18 | "fmt" 19 | "math" 20 | "math/rand" 21 | "testing" 22 | ) 23 | 24 | func BenchmarkMajorityConfig_CommittedIndex(b *testing.B) { 25 | // go test -run - -bench . -benchmem ./raft/quorum 26 | for _, n := range []int{1, 3, 5, 7, 9, 11} { 27 | b.Run(fmt.Sprintf("voters=%d", n), func(b *testing.B) { 28 | c := MajorityConfig{} 29 | l := mapAckIndexer{} 30 | for i := uint64(0); i < uint64(n); i++ { 31 | c[i+1] = struct{}{} 32 | l[i+1] = Index(rand.Int63n(math.MaxInt64)) 33 | } 34 | 35 | for i := 0; i < b.N; i++ { 36 | _ = c.CommittedIndex(l) 37 | } 38 | }) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /quorum/joint.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package quorum 16 | 17 | // JointConfig is a configuration of two groups of (possibly overlapping) 18 | // majority configurations. Decisions require the support of both majorities. 19 | type JointConfig [2]MajorityConfig 20 | 21 | func (c JointConfig) String() string { 22 | if len(c[1]) > 0 { 23 | return c[0].String() + "&&" + c[1].String() 24 | } 25 | return c[0].String() 26 | } 27 | 28 | // IDs returns a newly initialized map representing the set of voters present 29 | // in the joint configuration. 30 | func (c JointConfig) IDs() map[uint64]struct{} { 31 | m := map[uint64]struct{}{} 32 | for _, cc := range c { 33 | for id := range cc { 34 | m[id] = struct{}{} 35 | } 36 | } 37 | return m 38 | } 39 | 40 | // Describe returns a (multi-line) representation of the commit indexes for the 41 | // given lookuper. 42 | func (c JointConfig) Describe(l AckedIndexer) string { 43 | return MajorityConfig(c.IDs()).Describe(l) 44 | } 45 | 46 | // CommittedIndex returns the largest committed index for the given joint 47 | // quorum. An index is jointly committed if it is committed in both constituent 48 | // majorities. 49 | func (c JointConfig) CommittedIndex(l AckedIndexer) Index { 50 | idx0 := c[0].CommittedIndex(l) 51 | idx1 := c[1].CommittedIndex(l) 52 | if idx0 < idx1 { 53 | return idx0 54 | } 55 | return idx1 56 | } 57 | 58 | // VoteResult takes a mapping of voters to yes/no (true/false) votes and returns 59 | // a result indicating whether the vote is pending, lost, or won. A joint quorum 60 | // requires both majority quorums to vote in favor. 61 | func (c JointConfig) VoteResult(votes map[uint64]bool) VoteResult { 62 | r1 := c[0].VoteResult(votes) 63 | r2 := c[1].VoteResult(votes) 64 | 65 | if r1 == r2 { 66 | // If they agree, return the agreed state. 67 | return r1 68 | } 69 | if r1 == VoteLost || r2 == VoteLost { 70 | // If either config has lost, loss is the only possible outcome. 71 | return VoteLost 72 | } 73 | // One side won, the other one is pending, so the whole outcome is. 74 | return VotePending 75 | } 76 | -------------------------------------------------------------------------------- /quorum/quick_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package quorum 16 | 17 | import ( 18 | "math" 19 | "math/rand" 20 | "reflect" 21 | "testing" 22 | "testing/quick" 23 | 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | // TestQuick uses quickcheck to heuristically assert that the main 28 | // implementation of (MajorityConfig).CommittedIndex agrees with a "dumb" 29 | // alternative version. 30 | func TestQuick(t *testing.T) { 31 | cfg := &quick.Config{ 32 | MaxCount: 50000, 33 | } 34 | 35 | t.Run("majority_commit", func(t *testing.T) { 36 | fn1 := func(c memberMap, l idxMap) uint64 { 37 | return uint64(MajorityConfig(c).CommittedIndex(mapAckIndexer(l))) 38 | } 39 | fn2 := func(c memberMap, l idxMap) uint64 { 40 | return uint64(alternativeMajorityCommittedIndex(MajorityConfig(c), mapAckIndexer(l))) 41 | } 42 | require.NoError(t, quick.CheckEqual(fn1, fn2, cfg)) 43 | }) 44 | } 45 | 46 | // smallRandIdxMap returns a reasonably sized map of ids to commit indexes. 47 | func smallRandIdxMap(rand *rand.Rand, _ int) map[uint64]Index { 48 | // Hard-code a reasonably small size here (quick will hard-code 50, which 49 | // is not useful here). 50 | size := 10 51 | 52 | n := rand.Intn(size) 53 | ids := rand.Perm(2 * n)[:n] 54 | idxs := make([]int, len(ids)) 55 | for i := range idxs { 56 | idxs[i] = rand.Intn(n) 57 | } 58 | 59 | m := map[uint64]Index{} 60 | for i := range ids { 61 | m[uint64(ids[i])] = Index(idxs[i]) 62 | } 63 | return m 64 | } 65 | 66 | type idxMap map[uint64]Index 67 | 68 | func (idxMap) Generate(rand *rand.Rand, size int) reflect.Value { 69 | m := smallRandIdxMap(rand, size) 70 | return reflect.ValueOf(m) 71 | } 72 | 73 | type memberMap map[uint64]struct{} 74 | 75 | func (memberMap) Generate(rand *rand.Rand, size int) reflect.Value { 76 | m := smallRandIdxMap(rand, size) 77 | mm := map[uint64]struct{}{} 78 | for id := range m { 79 | mm[id] = struct{}{} 80 | } 81 | return reflect.ValueOf(mm) 82 | } 83 | 84 | // This is an alternative implementation of (MajorityConfig).CommittedIndex(l). 85 | func alternativeMajorityCommittedIndex(c MajorityConfig, l AckedIndexer) Index { 86 | if len(c) == 0 { 87 | return math.MaxUint64 88 | } 89 | 90 | idToIdx := map[uint64]Index{} 91 | for id := range c { 92 | if idx, ok := l.AckedIndex(id); ok { 93 | idToIdx[id] = idx 94 | } 95 | } 96 | 97 | // Build a map from index to voters who have acked that or any higher index. 98 | idxToVotes := map[Index]int{} 99 | for _, idx := range idToIdx { 100 | idxToVotes[idx] = 0 101 | } 102 | 103 | for _, idx := range idToIdx { 104 | for idy := range idxToVotes { 105 | if idy > idx { 106 | continue 107 | } 108 | idxToVotes[idy]++ 109 | } 110 | } 111 | 112 | // Find the maximum index that has achieved quorum. 113 | q := len(c)/2 + 1 114 | var maxQuorumIdx Index 115 | for idx, n := range idxToVotes { 116 | if n >= q && idx > maxQuorumIdx { 117 | maxQuorumIdx = idx 118 | } 119 | } 120 | 121 | return maxQuorumIdx 122 | } 123 | -------------------------------------------------------------------------------- /quorum/quorum.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package quorum 16 | 17 | import ( 18 | "math" 19 | "strconv" 20 | ) 21 | 22 | // Index is a Raft log position. 23 | type Index uint64 24 | 25 | func (i Index) String() string { 26 | if i == math.MaxUint64 { 27 | return "∞" 28 | } 29 | return strconv.FormatUint(uint64(i), 10) 30 | } 31 | 32 | // AckedIndexer allows looking up a commit index for a given ID of a voter 33 | // from a corresponding MajorityConfig. 34 | type AckedIndexer interface { 35 | AckedIndex(voterID uint64) (idx Index, found bool) 36 | } 37 | 38 | type mapAckIndexer map[uint64]Index 39 | 40 | func (m mapAckIndexer) AckedIndex(id uint64) (Index, bool) { 41 | idx, ok := m[id] 42 | return idx, ok 43 | } 44 | 45 | // VoteResult indicates the outcome of a vote. 46 | // 47 | //go:generate stringer -type=VoteResult 48 | type VoteResult uint8 49 | 50 | const ( 51 | // VotePending indicates that the decision of the vote depends on future 52 | // votes, i.e. neither "yes" or "no" has reached quorum yet. 53 | VotePending VoteResult = 1 + iota 54 | // VoteLost indicates that the quorum has voted "no". 55 | VoteLost 56 | // VoteWon indicates that the quorum has voted "yes". 57 | VoteWon 58 | ) 59 | -------------------------------------------------------------------------------- /quorum/testdata/joint_vote.txt: -------------------------------------------------------------------------------- 1 | # Empty joint config wins all votes. This isn't used in production. Note that 2 | # by specifying cfgj explicitly we tell the test harness to treat the input as 3 | # a joint quorum and not a majority quorum. 4 | vote cfgj=zero 5 | ---- 6 | VoteWon 7 | 8 | # More examples with close to trivial configs. 9 | 10 | vote cfg=(1) cfgj=zero votes=(_) 11 | ---- 12 | VotePending 13 | 14 | vote cfg=(1) cfgj=zero votes=(y) 15 | ---- 16 | VoteWon 17 | 18 | vote cfg=(1) cfgj=zero votes=(n) 19 | ---- 20 | VoteLost 21 | 22 | vote cfg=(1) cfgj=(1) votes=(_) 23 | ---- 24 | VotePending 25 | 26 | vote cfg=(1) cfgj=(1) votes=(y) 27 | ---- 28 | VoteWon 29 | 30 | vote cfg=(1) cfgj=(1) votes=(n) 31 | ---- 32 | VoteLost 33 | 34 | vote cfg=(1) cfgj=(2) votes=(_,_) 35 | ---- 36 | VotePending 37 | 38 | vote cfg=(1) cfgj=(2) votes=(y,_) 39 | ---- 40 | VotePending 41 | 42 | vote cfg=(1) cfgj=(2) votes=(y,y) 43 | ---- 44 | VoteWon 45 | 46 | vote cfg=(1) cfgj=(2) votes=(y,n) 47 | ---- 48 | VoteLost 49 | 50 | vote cfg=(1) cfgj=(2) votes=(n,_) 51 | ---- 52 | VoteLost 53 | 54 | vote cfg=(1) cfgj=(2) votes=(n,n) 55 | ---- 56 | VoteLost 57 | 58 | vote cfg=(1) cfgj=(2) votes=(n,y) 59 | ---- 60 | VoteLost 61 | 62 | # Two node configs. 63 | 64 | vote cfg=(1,2) cfgj=(3,4) votes=(_,_,_,_) 65 | ---- 66 | VotePending 67 | 68 | vote cfg=(1,2) cfgj=(3,4) votes=(y,_,_,_) 69 | ---- 70 | VotePending 71 | 72 | vote cfg=(1,2) cfgj=(3,4) votes=(y,y,_,_) 73 | ---- 74 | VotePending 75 | 76 | vote cfg=(1,2) cfgj=(3,4) votes=(y,y,n,_) 77 | ---- 78 | VoteLost 79 | 80 | vote cfg=(1,2) cfgj=(3,4) votes=(y,y,n,n) 81 | ---- 82 | VoteLost 83 | 84 | vote cfg=(1,2) cfgj=(3,4) votes=(y,y,y,n) 85 | ---- 86 | VoteLost 87 | 88 | vote cfg=(1,2) cfgj=(3,4) votes=(y,y,y,y) 89 | ---- 90 | VoteWon 91 | 92 | vote cfg=(1,2) cfgj=(2,3) votes=(_,_,_) 93 | ---- 94 | VotePending 95 | 96 | vote cfg=(1,2) cfgj=(2,3) votes=(_,n,_) 97 | ---- 98 | VoteLost 99 | 100 | vote cfg=(1,2) cfgj=(2,3) votes=(y,y,_) 101 | ---- 102 | VotePending 103 | 104 | vote cfg=(1,2) cfgj=(2,3) votes=(y,y,n) 105 | ---- 106 | VoteLost 107 | 108 | vote cfg=(1,2) cfgj=(2,3) votes=(y,y,y) 109 | ---- 110 | VoteWon 111 | 112 | vote cfg=(1,2) cfgj=(1,2) votes=(_,_) 113 | ---- 114 | VotePending 115 | 116 | vote cfg=(1,2) cfgj=(1,2) votes=(y,_) 117 | ---- 118 | VotePending 119 | 120 | vote cfg=(1,2) cfgj=(1,2) votes=(y,n) 121 | ---- 122 | VoteLost 123 | 124 | vote cfg=(1,2) cfgj=(1,2) votes=(n,_) 125 | ---- 126 | VoteLost 127 | 128 | vote cfg=(1,2) cfgj=(1,2) votes=(n,n) 129 | ---- 130 | VoteLost 131 | 132 | 133 | # Simple example for overlapping three node configs. 134 | 135 | vote cfg=(1,2,3) cfgj=(2,3,4) votes=(_,_,_,_) 136 | ---- 137 | VotePending 138 | 139 | vote cfg=(1,2,3) cfgj=(2,3,4) votes=(_,n,_,_) 140 | ---- 141 | VotePending 142 | 143 | vote cfg=(1,2,3) cfgj=(2,3,4) votes=(_,n,n,_) 144 | ---- 145 | VoteLost 146 | 147 | vote cfg=(1,2,3) cfgj=(2,3,4) votes=(_,y,y,_) 148 | ---- 149 | VoteWon 150 | 151 | vote cfg=(1,2,3) cfgj=(2,3,4) votes=(y,y,_,_) 152 | ---- 153 | VotePending 154 | 155 | vote cfg=(1,2,3) cfgj=(2,3,4) votes=(y,y,n,_) 156 | ---- 157 | VotePending 158 | 159 | vote cfg=(1,2,3) cfgj=(2,3,4) votes=(y,y,n,n) 160 | ---- 161 | VoteLost 162 | 163 | vote cfg=(1,2,3) cfgj=(2,3,4) votes=(y,y,n,y) 164 | ---- 165 | VoteWon 166 | -------------------------------------------------------------------------------- /quorum/testdata/majority_commit.txt: -------------------------------------------------------------------------------- 1 | # The empty quorum commits "everything". This is useful for its use in joint 2 | # quorums. 3 | committed 4 | ---- 5 | ∞ 6 | 7 | 8 | 9 | # A single voter quorum is not final when no index is known. 10 | committed cfg=(1) idx=(_) 11 | ---- 12 | idx 13 | ? 0 (id=1) 14 | 0 15 | 16 | # When an index is known, that's the committed index, and that's final. 17 | committed cfg=(1) idx=(12) 18 | ---- 19 | idx 20 | > 12 (id=1) 21 | 12 22 | 23 | 24 | 25 | 26 | # With two nodes, start out similarly. 27 | committed cfg=(1, 2) idx=(_,_) 28 | ---- 29 | idx 30 | ? 0 (id=1) 31 | ? 0 (id=2) 32 | 0 33 | 34 | # The first committed index becomes known (for n1). Nothing changes in the 35 | # output because idx=12 is not known to be on a quorum (which is both nodes). 36 | committed cfg=(1, 2) idx=(12,_) 37 | ---- 38 | idx 39 | x> 12 (id=1) 40 | ? 0 (id=2) 41 | 0 42 | 43 | # The second index comes in and finalize the decision. The result will be the 44 | # smaller of the two indexes. 45 | committed cfg=(1,2) idx=(12,5) 46 | ---- 47 | idx 48 | x> 12 (id=1) 49 | > 5 (id=2) 50 | 5 51 | 52 | 53 | 54 | 55 | # No surprises for three nodes. 56 | committed cfg=(1,2,3) idx=(_,_,_) 57 | ---- 58 | idx 59 | ? 0 (id=1) 60 | ? 0 (id=2) 61 | ? 0 (id=3) 62 | 0 63 | 64 | committed cfg=(1,2,3) idx=(12,_,_) 65 | ---- 66 | idx 67 | xx> 12 (id=1) 68 | ? 0 (id=2) 69 | ? 0 (id=3) 70 | 0 71 | 72 | # We see a committed index, but a higher committed index for the last pending 73 | # votes could change (increment) the outcome, so not final yet. 74 | committed cfg=(1,2,3) idx=(12,5,_) 75 | ---- 76 | idx 77 | xx> 12 (id=1) 78 | x> 5 (id=2) 79 | ? 0 (id=3) 80 | 5 81 | 82 | # a) the case in which it does: 83 | committed cfg=(1,2,3) idx=(12,5,6) 84 | ---- 85 | idx 86 | xx> 12 (id=1) 87 | > 5 (id=2) 88 | x> 6 (id=3) 89 | 6 90 | 91 | # b) the case in which it does not: 92 | committed cfg=(1,2,3) idx=(12,5,4) 93 | ---- 94 | idx 95 | xx> 12 (id=1) 96 | x> 5 (id=2) 97 | > 4 (id=3) 98 | 5 99 | 100 | # c) a different case in which the last index is pending but it has no chance of 101 | # swaying the outcome (because nobody in the current quorum agrees on anything 102 | # higher than the candidate): 103 | committed cfg=(1,2,3) idx=(5,5,_) 104 | ---- 105 | idx 106 | x> 5 (id=1) 107 | > 5 (id=2) 108 | ? 0 (id=3) 109 | 5 110 | 111 | # c) continued: Doesn't matter what shows up last. The result is final. 112 | committed cfg=(1,2,3) idx=(5,5,12) 113 | ---- 114 | idx 115 | > 5 (id=1) 116 | > 5 (id=2) 117 | xx> 12 (id=3) 118 | 5 119 | 120 | # With all committed idx known, the result is final. 121 | committed cfg=(1, 2, 3) idx=(100, 101, 103) 122 | ---- 123 | idx 124 | > 100 (id=1) 125 | x> 101 (id=2) 126 | xx> 103 (id=3) 127 | 101 128 | 129 | 130 | 131 | # Some more complicated examples. Similar to case c) above. The result is 132 | # already final because no index higher than 103 is one short of quorum. 133 | committed cfg=(1, 2, 3, 4, 5) idx=(101, 104, 103, 103,_) 134 | ---- 135 | idx 136 | x> 101 (id=1) 137 | xxxx> 104 (id=2) 138 | xx> 103 (id=3) 139 | > 103 (id=4) 140 | ? 0 (id=5) 141 | 103 142 | 143 | # A similar case which is not final because another vote for >= 103 would change 144 | # the outcome. 145 | committed cfg=(1, 2, 3, 4, 5) idx=(101, 102, 103, 103,_) 146 | ---- 147 | idx 148 | x> 101 (id=1) 149 | xx> 102 (id=2) 150 | xxx> 103 (id=3) 151 | > 103 (id=4) 152 | ? 0 (id=5) 153 | 102 154 | -------------------------------------------------------------------------------- /quorum/testdata/majority_vote.txt: -------------------------------------------------------------------------------- 1 | # The empty config always announces a won vote. 2 | vote 3 | ---- 4 | VoteWon 5 | 6 | vote cfg=(1) votes=(_) 7 | ---- 8 | VotePending 9 | 10 | vote cfg=(1) votes=(n) 11 | ---- 12 | VoteLost 13 | 14 | vote cfg=(123) votes=(y) 15 | ---- 16 | VoteWon 17 | 18 | 19 | 20 | 21 | vote cfg=(4,8) votes=(_,_) 22 | ---- 23 | VotePending 24 | 25 | # With two voters, a single rejection loses the vote. 26 | vote cfg=(4,8) votes=(n,_) 27 | ---- 28 | VoteLost 29 | 30 | vote cfg=(4,8) votes=(y,_) 31 | ---- 32 | VotePending 33 | 34 | vote cfg=(4,8) votes=(n,y) 35 | ---- 36 | VoteLost 37 | 38 | vote cfg=(4,8) votes=(y,y) 39 | ---- 40 | VoteWon 41 | 42 | 43 | 44 | vote cfg=(2,4,7) votes=(_,_,_) 45 | ---- 46 | VotePending 47 | 48 | vote cfg=(2,4,7) votes=(n,_,_) 49 | ---- 50 | VotePending 51 | 52 | vote cfg=(2,4,7) votes=(y,_,_) 53 | ---- 54 | VotePending 55 | 56 | vote cfg=(2,4,7) votes=(n,n,_) 57 | ---- 58 | VoteLost 59 | 60 | vote cfg=(2,4,7) votes=(y,n,_) 61 | ---- 62 | VotePending 63 | 64 | vote cfg=(2,4,7) votes=(y,y,_) 65 | ---- 66 | VoteWon 67 | 68 | vote cfg=(2,4,7) votes=(y,y,n) 69 | ---- 70 | VoteWon 71 | 72 | vote cfg=(2,4,7) votes=(n,y,n) 73 | ---- 74 | VoteLost 75 | 76 | 77 | 78 | # Test some random example with seven nodes (why not). 79 | vote cfg=(1,2,3,4,5,6,7) votes=(y,y,n,y,_,_,_) 80 | ---- 81 | VotePending 82 | 83 | vote cfg=(1,2,3,4,5,6,7) votes=(_,y,y,_,n,y,n) 84 | ---- 85 | VotePending 86 | 87 | vote cfg=(1,2,3,4,5,6,7) votes=(y,y,n,y,_,n,y) 88 | ---- 89 | VoteWon 90 | 91 | vote cfg=(1,2,3,4,5,6,7) votes=(y,y,_,n,y,n,n) 92 | ---- 93 | VotePending 94 | 95 | vote cfg=(1,2,3,4,5,6,7) votes=(y,y,n,y,n,n,n) 96 | ---- 97 | VoteLost 98 | -------------------------------------------------------------------------------- /quorum/voteresult_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=VoteResult"; DO NOT EDIT. 2 | 3 | package quorum 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[VotePending-1] 12 | _ = x[VoteLost-2] 13 | _ = x[VoteWon-3] 14 | } 15 | 16 | const _VoteResult_name = "VotePendingVoteLostVoteWon" 17 | 18 | var _VoteResult_index = [...]uint8{0, 11, 19, 26} 19 | 20 | func (i VoteResult) String() string { 21 | i -= 1 22 | if i >= VoteResult(len(_VoteResult_index)-1) { 23 | return "VoteResult(" + strconv.FormatInt(int64(i+1), 10) + ")" 24 | } 25 | return _VoteResult_name[_VoteResult_index[i]:_VoteResult_index[i+1]] 26 | } 27 | -------------------------------------------------------------------------------- /raft_snap_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raft 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/stretchr/testify/assert" 21 | "github.com/stretchr/testify/require" 22 | 23 | pb "go.etcd.io/raft/v3/raftpb" 24 | ) 25 | 26 | var ( 27 | testingSnap = pb.Snapshot{ 28 | Metadata: pb.SnapshotMetadata{ 29 | Index: 11, // magic number 30 | Term: 11, // magic number 31 | ConfState: pb.ConfState{Voters: []uint64{1, 2}}, 32 | }, 33 | } 34 | ) 35 | 36 | func TestSendingSnapshotSetPendingSnapshot(t *testing.T) { 37 | storage := newTestMemoryStorage(withPeers(1)) 38 | sm := newTestRaft(1, 10, 1, storage) 39 | sm.restore(testingSnap) 40 | 41 | sm.becomeCandidate() 42 | sm.becomeLeader() 43 | 44 | // force set the next of node 2, so that 45 | // node 2 needs a snapshot 46 | sm.trk.Progress[2].Next = sm.raftLog.firstIndex() 47 | 48 | sm.Step(pb.Message{From: 2, To: 1, Type: pb.MsgAppResp, Index: sm.trk.Progress[2].Next - 1, Reject: true}) 49 | require.Equal(t, uint64(11), sm.trk.Progress[2].PendingSnapshot) 50 | } 51 | 52 | func TestPendingSnapshotPauseReplication(t *testing.T) { 53 | storage := newTestMemoryStorage(withPeers(1, 2)) 54 | sm := newTestRaft(1, 10, 1, storage) 55 | sm.restore(testingSnap) 56 | 57 | sm.becomeCandidate() 58 | sm.becomeLeader() 59 | 60 | sm.trk.Progress[2].BecomeSnapshot(11) 61 | 62 | sm.Step(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("somedata")}}}) 63 | msgs := sm.readMessages() 64 | require.Empty(t, msgs) 65 | } 66 | 67 | func TestSnapshotFailure(t *testing.T) { 68 | storage := newTestMemoryStorage(withPeers(1, 2)) 69 | sm := newTestRaft(1, 10, 1, storage) 70 | sm.restore(testingSnap) 71 | 72 | sm.becomeCandidate() 73 | sm.becomeLeader() 74 | 75 | sm.trk.Progress[2].Next = 1 76 | sm.trk.Progress[2].BecomeSnapshot(11) 77 | 78 | sm.Step(pb.Message{From: 2, To: 1, Type: pb.MsgSnapStatus, Reject: true}) 79 | require.Zero(t, sm.trk.Progress[2].PendingSnapshot) 80 | require.Equal(t, uint64(1), sm.trk.Progress[2].Next) 81 | assert.True(t, sm.trk.Progress[2].MsgAppFlowPaused) 82 | } 83 | 84 | func TestSnapshotSucceed(t *testing.T) { 85 | storage := newTestMemoryStorage(withPeers(1, 2)) 86 | sm := newTestRaft(1, 10, 1, storage) 87 | sm.restore(testingSnap) 88 | 89 | sm.becomeCandidate() 90 | sm.becomeLeader() 91 | 92 | sm.trk.Progress[2].Next = 1 93 | sm.trk.Progress[2].BecomeSnapshot(11) 94 | 95 | sm.Step(pb.Message{From: 2, To: 1, Type: pb.MsgSnapStatus, Reject: false}) 96 | require.Zero(t, sm.trk.Progress[2].PendingSnapshot) 97 | require.Equal(t, uint64(12), sm.trk.Progress[2].Next) 98 | assert.True(t, sm.trk.Progress[2].MsgAppFlowPaused) 99 | } 100 | 101 | func TestSnapshotAbort(t *testing.T) { 102 | storage := newTestMemoryStorage(withPeers(1, 2)) 103 | sm := newTestRaft(1, 10, 1, storage) 104 | sm.restore(testingSnap) 105 | 106 | sm.becomeCandidate() 107 | sm.becomeLeader() 108 | 109 | sm.trk.Progress[2].Next = 1 110 | sm.trk.Progress[2].BecomeSnapshot(11) 111 | 112 | // A successful msgAppResp that has a higher/equal index than the 113 | // pending snapshot should abort the pending snapshot. 114 | sm.Step(pb.Message{From: 2, To: 1, Type: pb.MsgAppResp, Index: 11}) 115 | require.Zero(t, sm.trk.Progress[2].PendingSnapshot) 116 | // The follower entered StateReplicate and the leader send an append 117 | // and optimistically updated the progress (so we see 13 instead of 12). 118 | // There is something to append because the leader appended an empty entry 119 | // to the log at index 12 when it assumed leadership. 120 | require.Equal(t, uint64(13), sm.trk.Progress[2].Next) 121 | require.Equal(t, 1, sm.trk.Progress[2].Inflights.Count()) 122 | } 123 | -------------------------------------------------------------------------------- /raftpb/confstate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raftpb 16 | 17 | import ( 18 | "fmt" 19 | "reflect" 20 | "slices" 21 | ) 22 | 23 | // Equivalent returns a nil error if the inputs describe the same configuration. 24 | // On mismatch, returns a descriptive error showing the differences. 25 | func (cs ConfState) Equivalent(cs2 ConfState) error { 26 | cs1 := cs 27 | orig1, orig2 := cs1, cs2 28 | s := func(sl *[]uint64) { 29 | *sl = append([]uint64(nil), *sl...) 30 | slices.Sort(*sl) 31 | } 32 | 33 | for _, cs := range []*ConfState{&cs1, &cs2} { 34 | s(&cs.Voters) 35 | s(&cs.Learners) 36 | s(&cs.VotersOutgoing) 37 | s(&cs.LearnersNext) 38 | } 39 | 40 | if !reflect.DeepEqual(cs1, cs2) { 41 | return fmt.Errorf("ConfStates not equivalent after sorting:\n%+#v\n%+#v\nInputs were:\n%+#v\n%+#v", cs1, cs2, orig1, orig2) 42 | } 43 | return nil 44 | } 45 | -------------------------------------------------------------------------------- /raftpb/confstate_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raftpb 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/stretchr/testify/require" 21 | ) 22 | 23 | func TestConfState_Equivalent(t *testing.T) { 24 | type testCase struct { 25 | cs, cs2 ConfState 26 | ok bool 27 | } 28 | 29 | testCases := []testCase{ 30 | // Reordered voters and learners. 31 | {ConfState{ 32 | Voters: []uint64{1, 2, 3}, 33 | Learners: []uint64{5, 4, 6}, 34 | VotersOutgoing: []uint64{9, 8, 7}, 35 | LearnersNext: []uint64{10, 20, 15}, 36 | }, ConfState{ 37 | Voters: []uint64{1, 2, 3}, 38 | Learners: []uint64{4, 5, 6}, 39 | VotersOutgoing: []uint64{7, 9, 8}, 40 | LearnersNext: []uint64{20, 10, 15}, 41 | }, true}, 42 | // Not sensitive to nil vs empty slice. 43 | {ConfState{Voters: []uint64{}}, ConfState{Voters: []uint64(nil)}, true}, 44 | // Non-equivalent voters. 45 | {ConfState{Voters: []uint64{1, 2, 3, 4}}, ConfState{Voters: []uint64{2, 1, 3}}, false}, 46 | {ConfState{Voters: []uint64{1, 4, 3}}, ConfState{Voters: []uint64{2, 1, 3}}, false}, 47 | // Non-equivalent learners. 48 | {ConfState{Voters: []uint64{1, 2, 3, 4}}, ConfState{Voters: []uint64{2, 1, 3}}, false}, 49 | // Sensitive to AutoLeave flag. 50 | {ConfState{AutoLeave: true}, ConfState{}, false}, 51 | } 52 | 53 | for _, tc := range testCases { 54 | t.Run("", func(t *testing.T) { 55 | require.Equal(t, tc.ok, tc.cs.Equivalent(tc.cs2) == nil) 56 | }) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /raftpb/raft_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raftpb 16 | 17 | import ( 18 | "math/bits" 19 | "testing" 20 | "unsafe" 21 | 22 | "github.com/stretchr/testify/assert" 23 | ) 24 | 25 | func TestProtoMemorySizes(t *testing.T) { 26 | if64Bit := func(yes, no uintptr) uintptr { 27 | if bits.UintSize == 64 { 28 | return yes 29 | } 30 | return no 31 | } 32 | 33 | var e Entry 34 | assert.Equal(t, if64Bit(48, 32), unsafe.Sizeof(e), "Entry size check") 35 | 36 | var sm SnapshotMetadata 37 | assert.Equal(t, if64Bit(120, 68), unsafe.Sizeof(sm), "SnapshotMetadata size check") 38 | 39 | var s Snapshot 40 | assert.Equal(t, if64Bit(144, 80), unsafe.Sizeof(s), "Snapshot size check") 41 | 42 | var m Message 43 | assert.Equal(t, if64Bit(160, 112), unsafe.Sizeof(m), "Message size check") 44 | 45 | var hs HardState 46 | assert.Equal(t, uintptr(24), unsafe.Sizeof(hs), "HardState size check") 47 | 48 | var cs ConfState 49 | assert.Equal(t, if64Bit(104, 52), unsafe.Sizeof(cs), "ConfState size check") 50 | 51 | var cc ConfChange 52 | assert.Equal(t, if64Bit(48, 32), unsafe.Sizeof(cc), "ConfChange size check") 53 | 54 | var ccs ConfChangeSingle 55 | assert.Equal(t, if64Bit(16, 12), unsafe.Sizeof(ccs), "ConfChangeSingle size check") 56 | 57 | var ccv2 ConfChangeV2 58 | assert.Equal(t, if64Bit(56, 28), unsafe.Sizeof(ccv2), "ConfChangeV2 size check") 59 | } 60 | -------------------------------------------------------------------------------- /rafttest/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package rafttest provides functional tests for etcd's raft implementation. 16 | package rafttest 17 | -------------------------------------------------------------------------------- /rafttest/interaction_env.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "bufio" 19 | "fmt" 20 | "math" 21 | "strings" 22 | 23 | "go.etcd.io/raft/v3" 24 | pb "go.etcd.io/raft/v3/raftpb" 25 | ) 26 | 27 | // InteractionOpts groups the options for an InteractionEnv. 28 | type InteractionOpts struct { 29 | OnConfig func(*raft.Config) 30 | 31 | // SetRandomizedElectionTimeout is used to plumb this function down from the 32 | // raft test package. 33 | SetRandomizedElectionTimeout func(node *raft.RawNode, timeout int) 34 | } 35 | 36 | // Node is a member of a raft group tested via an InteractionEnv. 37 | type Node struct { 38 | *raft.RawNode 39 | Storage 40 | 41 | Config *raft.Config 42 | AppendWork []pb.Message // []MsgStorageAppend 43 | ApplyWork []pb.Message // []MsgStorageApply 44 | History []pb.Snapshot 45 | } 46 | 47 | // InteractionEnv facilitates testing of complex interactions between the 48 | // members of a raft group. 49 | type InteractionEnv struct { 50 | Options *InteractionOpts 51 | Nodes []Node 52 | Messages []pb.Message // in-flight messages 53 | 54 | Output *RedirectLogger 55 | } 56 | 57 | // NewInteractionEnv initializes an InteractionEnv. opts may be nil. 58 | func NewInteractionEnv(opts *InteractionOpts) *InteractionEnv { 59 | if opts == nil { 60 | opts = &InteractionOpts{} 61 | } 62 | return &InteractionEnv{ 63 | Options: opts, 64 | Output: &RedirectLogger{ 65 | Builder: &strings.Builder{}, 66 | }, 67 | } 68 | } 69 | 70 | func (env *InteractionEnv) withIndent(f func()) { 71 | orig := env.Output.Builder 72 | env.Output.Builder = &strings.Builder{} 73 | f() 74 | 75 | scanner := bufio.NewScanner(strings.NewReader(env.Output.Builder.String())) 76 | for scanner.Scan() { 77 | orig.WriteString(" " + scanner.Text() + "\n") 78 | } 79 | env.Output.Builder = orig 80 | } 81 | 82 | // Storage is the interface used by InteractionEnv. It is comprised of raft's 83 | // Storage interface plus access to operations that maintain the log and drive 84 | // the Ready handling loop. 85 | type Storage interface { 86 | raft.Storage 87 | SetHardState(state pb.HardState) error 88 | ApplySnapshot(pb.Snapshot) error 89 | Compact(newFirstIndex uint64) error 90 | Append([]pb.Entry) error 91 | } 92 | 93 | // raftConfigStub sets up a raft.Config stub with reasonable testing defaults. 94 | // In particular, no limits are set. It is not a complete config: ID and Storage 95 | // must be set for each node using the stub as a template. 96 | func raftConfigStub() raft.Config { 97 | return raft.Config{ 98 | ElectionTick: 3, 99 | HeartbeatTick: 1, 100 | MaxSizePerMsg: math.MaxUint64, 101 | MaxInflightMsgs: math.MaxInt32, 102 | } 103 | } 104 | 105 | func defaultEntryFormatter(b []byte) string { 106 | return fmt.Sprintf("%q", b) 107 | } 108 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_campaign.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/cockroachdb/datadriven" 21 | ) 22 | 23 | func (env *InteractionEnv) handleCampaign(t *testing.T, d datadriven.TestData) error { 24 | idx := firstAsNodeIdx(t, d) 25 | return env.Campaign(idx) 26 | } 27 | 28 | // Campaign the node at the given index. 29 | func (env *InteractionEnv) Campaign(idx int) error { 30 | return env.Nodes[idx].Campaign() 31 | } 32 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_compact.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "strconv" 19 | "testing" 20 | 21 | "github.com/cockroachdb/datadriven" 22 | ) 23 | 24 | func (env *InteractionEnv) handleCompact(t *testing.T, d datadriven.TestData) error { 25 | idx := firstAsNodeIdx(t, d) 26 | newFirstIndex, err := strconv.ParseUint(d.CmdArgs[1].Key, 10, 64) 27 | if err != nil { 28 | return err 29 | } 30 | return env.Compact(idx, newFirstIndex) 31 | } 32 | 33 | // Compact truncates the log on the node at index idx so that the supplied new 34 | // first index results. 35 | func (env *InteractionEnv) Compact(idx int, newFirstIndex uint64) error { 36 | if err := env.Nodes[idx].Compact(newFirstIndex); err != nil { 37 | return err 38 | } 39 | return env.RaftLog(idx) 40 | } 41 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_deliver_msgs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "fmt" 19 | "strconv" 20 | "testing" 21 | 22 | "github.com/cockroachdb/datadriven" 23 | 24 | "go.etcd.io/raft/v3" 25 | "go.etcd.io/raft/v3/raftpb" 26 | ) 27 | 28 | func (env *InteractionEnv) handleDeliverMsgs(t *testing.T, d datadriven.TestData) error { 29 | var typ raftpb.MessageType = -1 30 | var rs []Recipient 31 | for _, arg := range d.CmdArgs { 32 | if len(arg.Vals) == 0 { 33 | id, err := strconv.ParseUint(arg.Key, 10, 64) 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | rs = append(rs, Recipient{ID: id}) 38 | } 39 | for i := range arg.Vals { 40 | switch arg.Key { 41 | case "drop": 42 | var id uint64 43 | arg.Scan(t, i, &id) 44 | var found bool 45 | for _, r := range rs { 46 | if r.ID == id { 47 | found = true 48 | } 49 | } 50 | if found { 51 | t.Fatalf("can't both deliver and drop msgs to %d", id) 52 | } 53 | rs = append(rs, Recipient{ID: id, Drop: true}) 54 | case "type": 55 | var s string 56 | arg.Scan(t, i, &s) 57 | v, ok := raftpb.MessageType_value[s] 58 | if !ok { 59 | t.Fatalf("unknown message type %s", s) 60 | } 61 | typ = raftpb.MessageType(v) 62 | } 63 | } 64 | } 65 | 66 | if n := env.DeliverMsgs(typ, rs...); n == 0 { 67 | env.Output.WriteString("no messages\n") 68 | } 69 | return nil 70 | } 71 | 72 | type Recipient struct { 73 | ID uint64 74 | Drop bool 75 | } 76 | 77 | // DeliverMsgs goes through env.Messages and, depending on the Drop flag, 78 | // delivers or drops messages to the specified Recipients. Only messages of type 79 | // typ are delivered (-1 for all types). Returns the number of messages handled 80 | // (i.e. delivered or dropped). A handled message is removed from env.Messages. 81 | func (env *InteractionEnv) DeliverMsgs(typ raftpb.MessageType, rs ...Recipient) int { 82 | var n int 83 | for _, r := range rs { 84 | var msgs []raftpb.Message 85 | msgs, env.Messages = splitMsgs(env.Messages, r.ID, typ, r.Drop) 86 | n += len(msgs) 87 | for _, msg := range msgs { 88 | if r.Drop { 89 | fmt.Fprint(env.Output, "dropped: ") 90 | } 91 | fmt.Fprintln(env.Output, raft.DescribeMessage(msg, defaultEntryFormatter)) 92 | if r.Drop { 93 | // NB: it's allowed to drop messages to nodes that haven't been instantiated yet, 94 | // we haven't used msg.To yet. 95 | continue 96 | } 97 | toIdx := int(msg.To - 1) 98 | if err := env.Nodes[toIdx].Step(msg); err != nil { 99 | fmt.Fprintln(env.Output, err) 100 | } 101 | } 102 | } 103 | return n 104 | } 105 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_forget_leader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/cockroachdb/datadriven" 21 | ) 22 | 23 | func (env *InteractionEnv) handleForgetLeader(t *testing.T, d datadriven.TestData) error { 24 | idx := firstAsNodeIdx(t, d) 25 | env.ForgetLeader(idx) 26 | return nil 27 | } 28 | 29 | // ForgetLeader makes the follower at the given index forget its leader. 30 | func (env *InteractionEnv) ForgetLeader(idx int) { 31 | env.Nodes[idx].ForgetLeader() 32 | } 33 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_log_level.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | 21 | "github.com/cockroachdb/datadriven" 22 | ) 23 | 24 | func (env *InteractionEnv) handleLogLevel(d datadriven.TestData) error { 25 | return env.LogLevel(d.CmdArgs[0].Key) 26 | } 27 | 28 | func (env *InteractionEnv) LogLevel(name string) error { 29 | for i, s := range lvlNames { 30 | if strings.EqualFold(s, name) { 31 | env.Output.Lvl = i 32 | return nil 33 | } 34 | } 35 | return fmt.Errorf("log levels must be either of %v", lvlNames) 36 | } 37 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_process_append_thread.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "testing" 21 | 22 | "github.com/cockroachdb/datadriven" 23 | 24 | "go.etcd.io/raft/v3" 25 | "go.etcd.io/raft/v3/raftpb" 26 | ) 27 | 28 | func (env *InteractionEnv) handleProcessAppendThread(t *testing.T, d datadriven.TestData) error { 29 | idxs := nodeIdxs(t, d) 30 | for _, idx := range idxs { 31 | var err error 32 | if len(idxs) > 1 { 33 | fmt.Fprintf(env.Output, "> %d processing append thread\n", idx+1) 34 | env.withIndent(func() { err = env.ProcessAppendThread(idx) }) 35 | } else { 36 | err = env.ProcessAppendThread(idx) 37 | } 38 | if err != nil { 39 | return err 40 | } 41 | } 42 | return nil 43 | } 44 | 45 | // ProcessAppendThread runs processes a single message on the "append" thread of 46 | // the node with the given index. 47 | func (env *InteractionEnv) ProcessAppendThread(idx int) error { 48 | n := &env.Nodes[idx] 49 | if len(n.AppendWork) == 0 { 50 | env.Output.WriteString("no append work to perform") 51 | return nil 52 | } 53 | m := n.AppendWork[0] 54 | n.AppendWork = n.AppendWork[1:] 55 | 56 | resps := m.Responses 57 | m.Responses = nil 58 | env.Output.WriteString("Processing:\n") 59 | env.Output.WriteString(raft.DescribeMessage(m, defaultEntryFormatter) + "\n") 60 | st := raftpb.HardState{ 61 | Term: m.Term, 62 | Vote: m.Vote, 63 | Commit: m.Commit, 64 | } 65 | var snap raftpb.Snapshot 66 | if m.Snapshot != nil { 67 | snap = *m.Snapshot 68 | } 69 | if err := processAppend(n, st, m.Entries, snap); err != nil { 70 | return err 71 | } 72 | 73 | env.Output.WriteString("Responses:\n") 74 | for _, m := range resps { 75 | env.Output.WriteString(raft.DescribeMessage(m, defaultEntryFormatter) + "\n") 76 | } 77 | env.Messages = append(env.Messages, resps...) 78 | return nil 79 | } 80 | 81 | func processAppend(n *Node, st raftpb.HardState, ents []raftpb.Entry, snap raftpb.Snapshot) error { 82 | // TODO(tbg): the order of operations here is not necessarily safe. See: 83 | // https://github.com/etcd-io/etcd/pull/10861 84 | s := n.Storage 85 | if !raft.IsEmptyHardState(st) { 86 | if err := s.SetHardState(st); err != nil { 87 | return err 88 | } 89 | } 90 | if !raft.IsEmptySnap(snap) { 91 | if len(ents) > 0 { 92 | return errors.New("can't apply snapshot and entries at the same time") 93 | } 94 | return s.ApplySnapshot(snap) 95 | } 96 | return s.Append(ents) 97 | } 98 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_process_apply_thread.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | 21 | "github.com/cockroachdb/datadriven" 22 | 23 | "go.etcd.io/raft/v3" 24 | "go.etcd.io/raft/v3/raftpb" 25 | ) 26 | 27 | func (env *InteractionEnv) handleProcessApplyThread(t *testing.T, d datadriven.TestData) error { 28 | idxs := nodeIdxs(t, d) 29 | for _, idx := range idxs { 30 | var err error 31 | if len(idxs) > 1 { 32 | fmt.Fprintf(env.Output, "> %d processing apply thread\n", idx+1) 33 | env.withIndent(func() { err = env.ProcessApplyThread(idx) }) 34 | } else { 35 | err = env.ProcessApplyThread(idx) 36 | } 37 | if err != nil { 38 | return err 39 | } 40 | } 41 | return nil 42 | } 43 | 44 | // ProcessApplyThread runs processes a single message on the "apply" thread of 45 | // the node with the given index. 46 | func (env *InteractionEnv) ProcessApplyThread(idx int) error { 47 | n := &env.Nodes[idx] 48 | if len(n.ApplyWork) == 0 { 49 | env.Output.WriteString("no apply work to perform") 50 | return nil 51 | } 52 | m := n.ApplyWork[0] 53 | n.ApplyWork = n.ApplyWork[1:] 54 | 55 | resps := m.Responses 56 | m.Responses = nil 57 | env.Output.WriteString("Processing:\n") 58 | env.Output.WriteString(raft.DescribeMessage(m, defaultEntryFormatter) + "\n") 59 | if err := processApply(n, m.Entries); err != nil { 60 | return err 61 | } 62 | 63 | env.Output.WriteString("Responses:\n") 64 | for _, m := range resps { 65 | env.Output.WriteString(raft.DescribeMessage(m, defaultEntryFormatter) + "\n") 66 | } 67 | env.Messages = append(env.Messages, resps...) 68 | return nil 69 | } 70 | 71 | func processApply(n *Node, ents []raftpb.Entry) error { 72 | for _, ent := range ents { 73 | var update []byte 74 | var cs *raftpb.ConfState 75 | switch ent.Type { 76 | case raftpb.EntryConfChange: 77 | var cc raftpb.ConfChange 78 | if err := cc.Unmarshal(ent.Data); err != nil { 79 | return err 80 | } 81 | update = cc.Context 82 | cs = n.RawNode.ApplyConfChange(cc) 83 | case raftpb.EntryConfChangeV2: 84 | var cc raftpb.ConfChangeV2 85 | if err := cc.Unmarshal(ent.Data); err != nil { 86 | return err 87 | } 88 | cs = n.RawNode.ApplyConfChange(cc) 89 | update = cc.Context 90 | default: 91 | update = ent.Data 92 | } 93 | 94 | // Record the new state by starting with the current state and applying 95 | // the command. 96 | lastSnap := n.History[len(n.History)-1] 97 | var snap raftpb.Snapshot 98 | snap.Data = append(snap.Data, lastSnap.Data...) 99 | // NB: this hard-codes an "appender" state machine. 100 | snap.Data = append(snap.Data, update...) 101 | snap.Metadata.Index = ent.Index 102 | snap.Metadata.Term = ent.Term 103 | if cs == nil { 104 | sl := n.History 105 | cs = &sl[len(sl)-1].Metadata.ConfState 106 | } 107 | snap.Metadata.ConfState = *cs 108 | n.History = append(n.History, snap) 109 | } 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_process_ready.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | 21 | "github.com/cockroachdb/datadriven" 22 | 23 | "go.etcd.io/raft/v3" 24 | "go.etcd.io/raft/v3/raftpb" 25 | ) 26 | 27 | func (env *InteractionEnv) handleProcessReady(t *testing.T, d datadriven.TestData) error { 28 | idxs := nodeIdxs(t, d) 29 | for _, idx := range idxs { 30 | var err error 31 | if len(idxs) > 1 { 32 | fmt.Fprintf(env.Output, "> %d handling Ready\n", idx+1) 33 | env.withIndent(func() { err = env.ProcessReady(idx) }) 34 | } else { 35 | err = env.ProcessReady(idx) 36 | } 37 | if err != nil { 38 | return err 39 | } 40 | } 41 | return nil 42 | } 43 | 44 | // ProcessReady runs Ready handling on the node with the given index. 45 | func (env *InteractionEnv) ProcessReady(idx int) error { 46 | // TODO(tbg): Allow simulating crashes here. 47 | n := &env.Nodes[idx] 48 | rd := n.Ready() 49 | env.Output.WriteString(raft.DescribeReady(rd, defaultEntryFormatter)) 50 | 51 | if !n.Config.AsyncStorageWrites { 52 | if err := processAppend(n, rd.HardState, rd.Entries, rd.Snapshot); err != nil { 53 | return err 54 | } 55 | if err := processApply(n, rd.CommittedEntries); err != nil { 56 | return err 57 | } 58 | } 59 | 60 | for _, m := range rd.Messages { 61 | if raft.IsLocalMsgTarget(m.To) { 62 | if !n.Config.AsyncStorageWrites { 63 | panic("unexpected local msg target") 64 | } 65 | switch m.Type { 66 | case raftpb.MsgStorageAppend: 67 | n.AppendWork = append(n.AppendWork, m) 68 | case raftpb.MsgStorageApply: 69 | n.ApplyWork = append(n.ApplyWork, m) 70 | default: 71 | panic(fmt.Sprintf("unexpected message type %s", m.Type)) 72 | } 73 | } else { 74 | env.Messages = append(env.Messages, m) 75 | } 76 | } 77 | 78 | if !n.Config.AsyncStorageWrites { 79 | n.Advance(rd) 80 | } 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_propose.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/cockroachdb/datadriven" 21 | ) 22 | 23 | func (env *InteractionEnv) handlePropose(t *testing.T, d datadriven.TestData) error { 24 | idx := firstAsNodeIdx(t, d) 25 | if len(d.CmdArgs) != 2 || len(d.CmdArgs[1].Vals) > 0 { 26 | t.Fatalf("expected exactly one key with no vals: %+v", d.CmdArgs[1:]) 27 | } 28 | return env.Propose(idx, []byte(d.CmdArgs[1].Key)) 29 | } 30 | 31 | // Propose a regular entry. 32 | func (env *InteractionEnv) Propose(idx int, data []byte) error { 33 | return env.Nodes[idx].Propose(data) 34 | } 35 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_propose_conf_change.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "fmt" 19 | "strconv" 20 | "testing" 21 | 22 | "github.com/cockroachdb/datadriven" 23 | 24 | "go.etcd.io/raft/v3/raftpb" 25 | ) 26 | 27 | func (env *InteractionEnv) handleProposeConfChange(t *testing.T, d datadriven.TestData) error { 28 | idx := firstAsNodeIdx(t, d) 29 | var v1 bool 30 | transition := raftpb.ConfChangeTransitionAuto 31 | for _, arg := range d.CmdArgs[1:] { 32 | for _, val := range arg.Vals { 33 | switch arg.Key { 34 | case "v1": 35 | var err error 36 | v1, err = strconv.ParseBool(val) 37 | if err != nil { 38 | return err 39 | } 40 | case "transition": 41 | switch val { 42 | case "auto": 43 | transition = raftpb.ConfChangeTransitionAuto 44 | case "implicit": 45 | transition = raftpb.ConfChangeTransitionJointImplicit 46 | case "explicit": 47 | transition = raftpb.ConfChangeTransitionJointExplicit 48 | default: 49 | return fmt.Errorf("unknown transition %s", val) 50 | } 51 | default: 52 | return fmt.Errorf("unknown command %s", arg.Key) 53 | } 54 | } 55 | } 56 | 57 | ccs, err := raftpb.ConfChangesFromString(d.Input) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | var c raftpb.ConfChangeI 63 | if v1 { 64 | if len(ccs) > 1 || transition != raftpb.ConfChangeTransitionAuto { 65 | return fmt.Errorf("v1 conf change can only have one operation and no transition") 66 | } 67 | c = raftpb.ConfChange{ 68 | Type: ccs[0].Type, 69 | NodeID: ccs[0].NodeID, 70 | } 71 | } else { 72 | c = raftpb.ConfChangeV2{ 73 | Transition: transition, 74 | Changes: ccs, 75 | } 76 | } 77 | return env.ProposeConfChange(idx, c) 78 | } 79 | 80 | // ProposeConfChange proposes a configuration change on the node with the given index. 81 | func (env *InteractionEnv) ProposeConfChange(idx int, c raftpb.ConfChangeI) error { 82 | return env.Nodes[idx].ProposeConfChange(c) 83 | } 84 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_raft_log.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "fmt" 19 | "math" 20 | "testing" 21 | 22 | "github.com/cockroachdb/datadriven" 23 | 24 | "go.etcd.io/raft/v3" 25 | ) 26 | 27 | func (env *InteractionEnv) handleRaftLog(t *testing.T, d datadriven.TestData) error { 28 | idx := firstAsNodeIdx(t, d) 29 | return env.RaftLog(idx) 30 | } 31 | 32 | // RaftLog pretty prints the raft log to the output buffer. 33 | func (env *InteractionEnv) RaftLog(idx int) error { 34 | s := env.Nodes[idx].Storage 35 | fi, err := s.FirstIndex() 36 | if err != nil { 37 | return err 38 | } 39 | li, err := s.LastIndex() 40 | if err != nil { 41 | return err 42 | } 43 | if li < fi { 44 | // TODO(tbg): this is what MemoryStorage returns, but unclear if it's 45 | // the "correct" thing to do. 46 | fmt.Fprintf(env.Output, "log is empty: first index=%d, last index=%d", fi, li) 47 | return nil 48 | } 49 | ents, err := s.Entries(fi, li+1, math.MaxUint64) 50 | if err != nil { 51 | return err 52 | } 53 | env.Output.WriteString(raft.DescribeEntries(ents, defaultEntryFormatter)) 54 | return err 55 | } 56 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_raftstate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "fmt" 19 | 20 | "go.etcd.io/raft/v3" 21 | ) 22 | 23 | // isVoter checks whether node id is in the voter list within st. 24 | func isVoter(id uint64, st raft.Status) bool { 25 | idMap := st.Config.Voters.IDs() 26 | for idx := range idMap { 27 | if id == idx { 28 | return true 29 | } 30 | } 31 | return false 32 | } 33 | 34 | // handleRaftState pretty-prints the raft state for all nodes to the output buffer. 35 | // For each node, the information is based on its own configuration view. 36 | func (env *InteractionEnv) handleRaftState() error { 37 | for _, n := range env.Nodes { 38 | st := n.Status() 39 | var voterStatus string 40 | if isVoter(st.ID, st) { 41 | voterStatus = "(Voter)" 42 | } else { 43 | voterStatus = "(Non-Voter)" 44 | } 45 | fmt.Fprintf(env.Output, "%d: %s %s Term:%d Lead:%d\n", 46 | st.ID, st.RaftState, voterStatus, st.Term, st.Lead) 47 | } 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_report_unreachable.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package rafttest 15 | 16 | import ( 17 | "errors" 18 | "testing" 19 | 20 | "github.com/cockroachdb/datadriven" 21 | ) 22 | 23 | func (env *InteractionEnv) handleReportUnreachable(t *testing.T, d datadriven.TestData) error { 24 | sl := nodeIdxs(t, d) 25 | if len(sl) != 2 { 26 | return errors.New("must specify exactly two node indexes: node on which to report, and reported node") 27 | } 28 | env.Nodes[sl[0]].ReportUnreachable(env.Nodes[sl[1]].Config.ID) 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_send_snapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/cockroachdb/datadriven" 21 | "github.com/stretchr/testify/require" 22 | 23 | "go.etcd.io/raft/v3" 24 | "go.etcd.io/raft/v3/raftpb" 25 | ) 26 | 27 | func (env *InteractionEnv) handleSendSnapshot(t *testing.T, d datadriven.TestData) error { 28 | idxs := nodeIdxs(t, d) 29 | require.Len(t, idxs, 2) 30 | return env.SendSnapshot(idxs[0], idxs[1]) 31 | } 32 | 33 | // SendSnapshot sends a snapshot. 34 | func (env *InteractionEnv) SendSnapshot(fromIdx, toIdx int) error { 35 | snap, err := env.Nodes[fromIdx].Snapshot() 36 | if err != nil { 37 | return err 38 | } 39 | from, to := uint64(fromIdx+1), uint64(toIdx+1) 40 | msg := raftpb.Message{ 41 | Type: raftpb.MsgSnap, 42 | Term: env.Nodes[fromIdx].BasicStatus().Term, 43 | From: from, 44 | To: to, 45 | Snapshot: &snap, 46 | } 47 | env.Messages = append(env.Messages, msg) 48 | _, _ = env.Output.WriteString(raft.DescribeMessage(msg, nil)) 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_set_randomized_election_timeout.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/cockroachdb/datadriven" 21 | "github.com/stretchr/testify/require" 22 | ) 23 | 24 | func (env *InteractionEnv) handleSetRandomizedElectionTimeout( 25 | t *testing.T, d datadriven.TestData, 26 | ) error { 27 | idx := firstAsNodeIdx(t, d) 28 | var timeout int 29 | d.ScanArgs(t, "timeout", &timeout) 30 | require.NotZero(t, timeout) 31 | 32 | env.Options.SetRandomizedElectionTimeout(env.Nodes[idx].RawNode, timeout) 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_stabilize.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | 21 | "github.com/cockroachdb/datadriven" 22 | 23 | "go.etcd.io/raft/v3" 24 | "go.etcd.io/raft/v3/raftpb" 25 | ) 26 | 27 | func (env *InteractionEnv) handleStabilize(t *testing.T, d datadriven.TestData) error { 28 | idxs := nodeIdxs(t, d) // skips key=value args 29 | for _, arg := range d.CmdArgs { 30 | for i := range arg.Vals { 31 | switch arg.Key { 32 | case "log-level": 33 | defer func(old int) { 34 | env.Output.Lvl = old 35 | }(env.Output.Lvl) 36 | var level string 37 | arg.Scan(t, i, &level) 38 | if err := env.LogLevel(level); err != nil { 39 | return err 40 | } 41 | } 42 | } 43 | } 44 | return env.Stabilize(idxs...) 45 | } 46 | 47 | // Stabilize repeatedly runs Ready handling on and message delivery to the set 48 | // of nodes specified via the idxs slice until reaching a fixed point. 49 | func (env *InteractionEnv) Stabilize(idxs ...int) error { 50 | var nodes []*Node 51 | if len(idxs) != 0 { 52 | for _, idx := range idxs { 53 | nodes = append(nodes, &env.Nodes[idx]) 54 | } 55 | } else { 56 | for i := range env.Nodes { 57 | nodes = append(nodes, &env.Nodes[i]) 58 | } 59 | } 60 | 61 | for { 62 | done := true 63 | for _, rn := range nodes { 64 | if rn.HasReady() { 65 | idx := int(rn.Status().ID - 1) 66 | fmt.Fprintf(env.Output, "> %d handling Ready\n", idx+1) 67 | var err error 68 | env.withIndent(func() { err = env.ProcessReady(idx) }) 69 | if err != nil { 70 | return err 71 | } 72 | done = false 73 | } 74 | } 75 | for _, rn := range nodes { 76 | id := rn.Status().ID 77 | // NB: we grab the messages just to see whether to print the header. 78 | // DeliverMsgs will do it again. 79 | if msgs, _ := splitMsgs(env.Messages, id, -1 /* typ */, false /* drop */); len(msgs) > 0 { 80 | fmt.Fprintf(env.Output, "> %d receiving messages\n", id) 81 | env.withIndent(func() { env.DeliverMsgs(-1 /* typ */, Recipient{ID: id}) }) 82 | done = false 83 | } 84 | } 85 | for _, rn := range nodes { 86 | idx := int(rn.Status().ID - 1) 87 | if len(rn.AppendWork) > 0 { 88 | fmt.Fprintf(env.Output, "> %d processing append thread\n", idx+1) 89 | for len(rn.AppendWork) > 0 { 90 | var err error 91 | env.withIndent(func() { err = env.ProcessAppendThread(idx) }) 92 | if err != nil { 93 | return err 94 | } 95 | } 96 | done = false 97 | } 98 | } 99 | for _, rn := range nodes { 100 | idx := int(rn.Status().ID - 1) 101 | if len(rn.ApplyWork) > 0 { 102 | fmt.Fprintf(env.Output, "> %d processing apply thread\n", idx+1) 103 | for len(rn.ApplyWork) > 0 { 104 | env.withIndent(func() { env.ProcessApplyThread(idx) }) 105 | } 106 | done = false 107 | } 108 | } 109 | if done { 110 | return nil 111 | } 112 | } 113 | } 114 | 115 | // splitMsgs extracts messages for the given recipient of the given type (-1 for 116 | // all types) from msgs, and returns them along with the remainder of msgs. 117 | func splitMsgs(msgs []raftpb.Message, to uint64, typ raftpb.MessageType, drop bool) (toMsgs []raftpb.Message, rmdr []raftpb.Message) { 118 | // NB: this method does not reorder messages. 119 | for _, msg := range msgs { 120 | if msg.To == to && !(drop && isLocalMsg(msg)) && (typ < 0 || msg.Type == typ) { 121 | toMsgs = append(toMsgs, msg) 122 | } else { 123 | rmdr = append(rmdr, msg) 124 | } 125 | } 126 | return toMsgs, rmdr 127 | } 128 | 129 | // Don't drop local messages, which require reliable delivery. 130 | func isLocalMsg(msg raftpb.Message) bool { 131 | return msg.From == msg.To || raft.IsLocalMsgTarget(msg.From) || raft.IsLocalMsgTarget(msg.To) 132 | } 133 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | 21 | "github.com/cockroachdb/datadriven" 22 | 23 | "go.etcd.io/raft/v3/tracker" 24 | ) 25 | 26 | func (env *InteractionEnv) handleStatus(t *testing.T, d datadriven.TestData) error { 27 | idx := firstAsNodeIdx(t, d) 28 | return env.Status(idx) 29 | } 30 | 31 | // Status pretty-prints the raft status for the node at the given index to the output 32 | // buffer. 33 | func (env *InteractionEnv) Status(idx int) error { 34 | // TODO(tbg): actually print the full status. 35 | st := env.Nodes[idx].Status() 36 | m := tracker.ProgressMap{} 37 | for id, pr := range st.Progress { 38 | pr := pr // loop-local copy 39 | m[id] = &pr 40 | } 41 | fmt.Fprint(env.Output, m) 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_tick.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/cockroachdb/datadriven" 21 | ) 22 | 23 | func (env *InteractionEnv) handleTickElection(t *testing.T, d datadriven.TestData) error { 24 | idx := firstAsNodeIdx(t, d) 25 | return env.Tick(idx, env.Nodes[idx].Config.ElectionTick) 26 | } 27 | 28 | func (env *InteractionEnv) handleTickHeartbeat(t *testing.T, d datadriven.TestData) error { 29 | idx := firstAsNodeIdx(t, d) 30 | return env.Tick(idx, env.Nodes[idx].Config.HeartbeatTick) 31 | } 32 | 33 | // Tick the node at the given index the given number of times. 34 | func (env *InteractionEnv) Tick(idx int, num int) error { 35 | for i := 0; i < num; i++ { 36 | env.Nodes[idx].Tick() 37 | } 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /rafttest/interaction_env_handler_transfer_leadership.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/cockroachdb/datadriven" 21 | ) 22 | 23 | func (env *InteractionEnv) handleTransferLeadership(t *testing.T, d datadriven.TestData) error { 24 | var from, to uint64 25 | d.ScanArgs(t, "from", &from) 26 | d.ScanArgs(t, "to", &to) 27 | if from == 0 || from > uint64(len(env.Nodes)) { 28 | t.Fatalf(`expected valid "from" argument`) 29 | } 30 | if to == 0 || to > uint64(len(env.Nodes)) { 31 | t.Fatalf(`expected valid "to" argument`) 32 | } 33 | return env.transferLeadership(from, to) 34 | } 35 | 36 | // Initiate leadership transfer. 37 | func (env *InteractionEnv) transferLeadership(from, to uint64) error { 38 | fromIdx := from - 1 39 | env.Nodes[fromIdx].TransferLeader(to) 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /rafttest/interaction_env_logger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | 21 | "go.etcd.io/raft/v3" 22 | ) 23 | 24 | type logLevels [6]string 25 | 26 | var lvlNames logLevels = [...]string{"DEBUG", "INFO", "WARN", "ERROR", "FATAL", "NONE"} 27 | 28 | type RedirectLogger struct { 29 | *strings.Builder 30 | Lvl int // 0 = DEBUG, 1 = INFO, 2 = WARNING, 3 = ERROR, 4 = FATAL, 5 = NONE 31 | } 32 | 33 | var _ raft.Logger = (*RedirectLogger)(nil) 34 | 35 | func (l *RedirectLogger) printf(lvl int, format string, args ...interface{}) { 36 | if l.Lvl <= lvl { 37 | fmt.Fprint(l, lvlNames[lvl], " ") 38 | fmt.Fprintf(l, format, args...) 39 | if n := len(format); n > 0 && format[n-1] != '\n' { 40 | l.WriteByte('\n') 41 | } 42 | } 43 | } 44 | func (l *RedirectLogger) print(lvl int, args ...interface{}) { 45 | if l.Lvl <= lvl { 46 | fmt.Fprint(l, lvlNames[lvl], " ") 47 | fmt.Fprintln(l, args...) 48 | } 49 | } 50 | 51 | func (l *RedirectLogger) Debug(v ...interface{}) { 52 | l.print(0, v...) 53 | } 54 | 55 | func (l *RedirectLogger) Debugf(format string, v ...interface{}) { 56 | l.printf(0, format, v...) 57 | } 58 | 59 | func (l *RedirectLogger) Info(v ...interface{}) { 60 | l.print(1, v...) 61 | } 62 | 63 | func (l *RedirectLogger) Infof(format string, v ...interface{}) { 64 | l.printf(1, format, v...) 65 | } 66 | 67 | func (l *RedirectLogger) Warning(v ...interface{}) { 68 | l.print(2, v...) 69 | } 70 | 71 | func (l *RedirectLogger) Warningf(format string, v ...interface{}) { 72 | l.printf(2, format, v...) 73 | } 74 | 75 | func (l *RedirectLogger) Error(v ...interface{}) { 76 | l.print(3, v...) 77 | } 78 | 79 | func (l *RedirectLogger) Errorf(format string, v ...interface{}) { 80 | l.printf(3, format, v...) 81 | } 82 | 83 | func (l *RedirectLogger) Fatal(v ...interface{}) { 84 | l.print(4, v...) 85 | panic(fmt.Sprint(v...)) 86 | } 87 | 88 | func (l *RedirectLogger) Fatalf(format string, v ...interface{}) { 89 | l.printf(4, format, v...) 90 | panic(fmt.Sprintf(format, v...)) 91 | } 92 | 93 | func (l *RedirectLogger) Panic(v ...interface{}) { 94 | l.print(4, v...) 95 | panic(fmt.Sprint(v...)) 96 | } 97 | 98 | func (l *RedirectLogger) Panicf(format string, v ...interface{}) { 99 | l.printf(4, format, v...) 100 | // TODO(pavelkalinnikov): catch the panic gracefully in datadriven package. 101 | // This would allow observing all the intermediate logging while debugging, 102 | // and testing the cases when panic is expected. 103 | panic(fmt.Sprintf(format, v...)) 104 | } 105 | 106 | // Override StringBuilder write methods to silence them under NONE. 107 | 108 | func (l *RedirectLogger) Quiet() bool { 109 | return l.Lvl == len(lvlNames)-1 110 | } 111 | 112 | func (l *RedirectLogger) Write(p []byte) (int, error) { 113 | if l.Quiet() { 114 | return 0, nil 115 | } 116 | return l.Builder.Write(p) 117 | } 118 | 119 | func (l *RedirectLogger) WriteByte(c byte) error { 120 | if l.Quiet() { 121 | return nil 122 | } 123 | return l.Builder.WriteByte(c) 124 | } 125 | 126 | func (l *RedirectLogger) WriteRune(r rune) (int, error) { 127 | if l.Quiet() { 128 | return 0, nil 129 | } 130 | return l.Builder.WriteRune(r) 131 | } 132 | 133 | func (l *RedirectLogger) WriteString(s string) (int, error) { 134 | if l.Quiet() { 135 | return 0, nil 136 | } 137 | return l.Builder.WriteString(s) 138 | } 139 | -------------------------------------------------------------------------------- /rafttest/network.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "math/rand" 19 | "sync" 20 | "time" 21 | 22 | "go.etcd.io/raft/v3/raftpb" 23 | ) 24 | 25 | // a network interface 26 | type iface interface { 27 | send(m raftpb.Message) 28 | recv() chan raftpb.Message 29 | disconnect() 30 | connect() 31 | } 32 | 33 | type raftNetwork struct { 34 | rand *rand.Rand 35 | mu sync.Mutex 36 | disconnected map[uint64]bool 37 | dropmap map[conn]float64 38 | delaymap map[conn]delay 39 | recvQueues map[uint64]chan raftpb.Message 40 | } 41 | 42 | type conn struct { 43 | from, to uint64 44 | } 45 | 46 | type delay struct { 47 | d time.Duration 48 | rate float64 49 | } 50 | 51 | func newRaftNetwork(nodes ...uint64) *raftNetwork { 52 | pn := &raftNetwork{ 53 | rand: rand.New(rand.NewSource(1)), 54 | recvQueues: make(map[uint64]chan raftpb.Message), 55 | dropmap: make(map[conn]float64), 56 | delaymap: make(map[conn]delay), 57 | disconnected: make(map[uint64]bool), 58 | } 59 | 60 | for _, n := range nodes { 61 | pn.recvQueues[n] = make(chan raftpb.Message, 1024) 62 | } 63 | return pn 64 | } 65 | 66 | func (rn *raftNetwork) nodeNetwork(id uint64) iface { 67 | return &nodeNetwork{id: id, raftNetwork: rn} 68 | } 69 | 70 | func (rn *raftNetwork) send(m raftpb.Message) { 71 | rn.mu.Lock() 72 | to := rn.recvQueues[m.To] 73 | if rn.disconnected[m.To] { 74 | to = nil 75 | } 76 | drop := rn.dropmap[conn{m.From, m.To}] 77 | dl := rn.delaymap[conn{m.From, m.To}] 78 | rn.mu.Unlock() 79 | 80 | if to == nil { 81 | return 82 | } 83 | if drop != 0 && rn.rand.Float64() < drop { 84 | return 85 | } 86 | // TODO: shall we dl without blocking the send call? 87 | if dl.d != 0 && rn.rand.Float64() < dl.rate { 88 | rd := rn.rand.Int63n(int64(dl.d)) 89 | time.Sleep(time.Duration(rd)) 90 | } 91 | 92 | // use marshal/unmarshal to copy message to avoid data race. 93 | b, err := m.Marshal() 94 | if err != nil { 95 | panic(err) 96 | } 97 | 98 | var cm raftpb.Message 99 | err = cm.Unmarshal(b) 100 | if err != nil { 101 | panic(err) 102 | } 103 | 104 | select { 105 | case to <- cm: 106 | default: 107 | // drop messages when the receiver queue is full. 108 | } 109 | } 110 | 111 | func (rn *raftNetwork) recvFrom(from uint64) chan raftpb.Message { 112 | rn.mu.Lock() 113 | fromc := rn.recvQueues[from] 114 | if rn.disconnected[from] { 115 | fromc = nil 116 | } 117 | rn.mu.Unlock() 118 | 119 | return fromc 120 | } 121 | 122 | func (rn *raftNetwork) drop(from, to uint64, rate float64) { 123 | rn.mu.Lock() 124 | defer rn.mu.Unlock() 125 | rn.dropmap[conn{from, to}] = rate 126 | } 127 | 128 | func (rn *raftNetwork) delay(from, to uint64, d time.Duration, rate float64) { 129 | rn.mu.Lock() 130 | defer rn.mu.Unlock() 131 | rn.delaymap[conn{from, to}] = delay{d, rate} 132 | } 133 | 134 | func (rn *raftNetwork) disconnect(id uint64) { 135 | rn.mu.Lock() 136 | defer rn.mu.Unlock() 137 | rn.disconnected[id] = true 138 | } 139 | 140 | func (rn *raftNetwork) connect(id uint64) { 141 | rn.mu.Lock() 142 | defer rn.mu.Unlock() 143 | rn.disconnected[id] = false 144 | } 145 | 146 | type nodeNetwork struct { 147 | id uint64 148 | *raftNetwork 149 | } 150 | 151 | func (nt *nodeNetwork) connect() { 152 | nt.raftNetwork.connect(nt.id) 153 | } 154 | 155 | func (nt *nodeNetwork) disconnect() { 156 | nt.raftNetwork.disconnect(nt.id) 157 | } 158 | 159 | func (nt *nodeNetwork) send(m raftpb.Message) { 160 | nt.raftNetwork.send(m) 161 | } 162 | 163 | func (nt *nodeNetwork) recv() chan raftpb.Message { 164 | return nt.recvFrom(nt.id) 165 | } 166 | -------------------------------------------------------------------------------- /rafttest/network_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/stretchr/testify/assert" 22 | 23 | "go.etcd.io/raft/v3/raftpb" 24 | ) 25 | 26 | func TestNetworkDrop(t *testing.T) { 27 | // drop around 10% messages 28 | sent := 1000 29 | droprate := 0.1 30 | nt := newRaftNetwork(1, 2) 31 | nt.drop(1, 2, droprate) 32 | for i := 0; i < sent; i++ { 33 | nt.send(raftpb.Message{From: 1, To: 2}) 34 | } 35 | 36 | c := nt.recvFrom(2) 37 | 38 | received := 0 39 | done := false 40 | for !done { 41 | select { 42 | case <-c: 43 | received++ 44 | default: 45 | done = true 46 | } 47 | } 48 | 49 | drop := sent - received 50 | assert.LessOrEqual(t, drop, int((droprate+0.1)*float64(sent))) 51 | assert.GreaterOrEqual(t, drop, int((droprate-0.1)*float64(sent))) 52 | } 53 | 54 | func TestNetworkDelay(t *testing.T) { 55 | sent := 1000 56 | delay := time.Millisecond 57 | delayrate := 0.1 58 | nt := newRaftNetwork(1, 2) 59 | 60 | nt.delay(1, 2, delay, delayrate) 61 | var total time.Duration 62 | for i := 0; i < sent; i++ { 63 | s := time.Now() 64 | nt.send(raftpb.Message{From: 1, To: 2}) 65 | total += time.Since(s) 66 | } 67 | 68 | w := time.Duration(float64(sent)*delayrate/2) * delay 69 | // there is some overhead in the send call since it generates random numbers. 70 | assert.GreaterOrEqual(t, total, w) 71 | } 72 | -------------------------------------------------------------------------------- /rafttest/node.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "context" 19 | "log" 20 | "math/rand" 21 | "sync" 22 | "time" 23 | 24 | "go.etcd.io/raft/v3" 25 | "go.etcd.io/raft/v3/raftpb" 26 | ) 27 | 28 | type node struct { 29 | raft.Node 30 | id uint64 31 | iface iface 32 | stopc chan struct{} 33 | pausec chan bool 34 | 35 | // stable 36 | storage *raft.MemoryStorage 37 | 38 | mu sync.Mutex // guards state 39 | state raftpb.HardState 40 | } 41 | 42 | func startNode(id uint64, peers []raft.Peer, iface iface) *node { 43 | st := raft.NewMemoryStorage() 44 | c := &raft.Config{ 45 | ID: id, 46 | ElectionTick: 10, 47 | HeartbeatTick: 1, 48 | Storage: st, 49 | MaxSizePerMsg: 1024 * 1024, 50 | MaxInflightMsgs: 256, 51 | MaxUncommittedEntriesSize: 1 << 30, 52 | } 53 | rn := raft.StartNode(c, peers) 54 | n := &node{ 55 | Node: rn, 56 | id: id, 57 | storage: st, 58 | iface: iface, 59 | pausec: make(chan bool), 60 | } 61 | n.start() 62 | return n 63 | } 64 | 65 | func (n *node) start() { 66 | n.stopc = make(chan struct{}) 67 | ticker := time.NewTicker(5 * time.Millisecond).C 68 | 69 | go func() { 70 | for { 71 | select { 72 | case <-ticker: 73 | n.Tick() 74 | case rd := <-n.Ready(): 75 | if !raft.IsEmptyHardState(rd.HardState) { 76 | n.mu.Lock() 77 | n.state = rd.HardState 78 | n.mu.Unlock() 79 | n.storage.SetHardState(n.state) 80 | } 81 | n.storage.Append(rd.Entries) 82 | time.Sleep(time.Millisecond) 83 | 84 | // simulate async send, more like real world... 85 | for _, m := range rd.Messages { 86 | mlocal := m 87 | go func() { 88 | time.Sleep(time.Duration(rand.Int63n(10)) * time.Millisecond) 89 | n.iface.send(mlocal) 90 | }() 91 | } 92 | n.Advance() 93 | case m := <-n.iface.recv(): 94 | go n.Step(context.TODO(), m) 95 | case <-n.stopc: 96 | n.Stop() 97 | log.Printf("raft.%d: stop", n.id) 98 | n.Node = nil 99 | close(n.stopc) 100 | return 101 | case p := <-n.pausec: 102 | recvms := make([]raftpb.Message, 0) 103 | for p { 104 | select { 105 | case m := <-n.iface.recv(): 106 | recvms = append(recvms, m) 107 | case p = <-n.pausec: 108 | } 109 | } 110 | // step all pending messages 111 | for _, m := range recvms { 112 | n.Step(context.TODO(), m) 113 | } 114 | } 115 | } 116 | }() 117 | } 118 | 119 | // stop stops the node. stop a stopped node might panic. 120 | // All in memory state of node is discarded. 121 | // All stable MUST be unchanged. 122 | func (n *node) stop() { 123 | n.iface.disconnect() 124 | n.stopc <- struct{}{} 125 | // wait for the shutdown 126 | <-n.stopc 127 | } 128 | 129 | // restart restarts the node. restart a started node 130 | // blocks and might affect the future stop operation. 131 | func (n *node) restart() { 132 | // wait for the shutdown 133 | <-n.stopc 134 | c := &raft.Config{ 135 | ID: n.id, 136 | ElectionTick: 10, 137 | HeartbeatTick: 1, 138 | Storage: n.storage, 139 | MaxSizePerMsg: 1024 * 1024, 140 | MaxInflightMsgs: 256, 141 | MaxUncommittedEntriesSize: 1 << 30, 142 | } 143 | n.Node = raft.RestartNode(c) 144 | n.start() 145 | n.iface.connect() 146 | } 147 | 148 | // pause pauses the node. 149 | // The paused node buffers the received messages and replies 150 | // all of them when it resumes. 151 | func (n *node) pause() { 152 | n.pausec <- true 153 | } 154 | 155 | // resume resumes the paused node. 156 | func (n *node) resume() { 157 | n.pausec <- false 158 | } 159 | -------------------------------------------------------------------------------- /rafttest/node_bench_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "context" 19 | "testing" 20 | "time" 21 | 22 | "go.etcd.io/raft/v3" 23 | ) 24 | 25 | func BenchmarkProposal3Nodes(b *testing.B) { 26 | peers := []raft.Peer{{ID: 1, Context: nil}, {ID: 2, Context: nil}, {ID: 3, Context: nil}} 27 | nt := newRaftNetwork(1, 2, 3) 28 | 29 | nodes := make([]*node, 0) 30 | 31 | for i := 1; i <= 3; i++ { 32 | n := startNode(uint64(i), peers, nt.nodeNetwork(uint64(i))) 33 | nodes = append(nodes, n) 34 | } 35 | // get ready and warm up 36 | time.Sleep(50 * time.Millisecond) 37 | 38 | b.ResetTimer() 39 | for i := 0; i < b.N; i++ { 40 | nodes[0].Propose(context.TODO(), []byte("somedata")) 41 | } 42 | 43 | for _, n := range nodes { 44 | if n.state.Commit != uint64(b.N+4) { 45 | continue 46 | } 47 | } 48 | b.StopTimer() 49 | 50 | for _, n := range nodes { 51 | n.stop() 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /rafttest/node_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rafttest 16 | 17 | import ( 18 | "context" 19 | "testing" 20 | "time" 21 | 22 | "github.com/stretchr/testify/assert" 23 | 24 | "go.etcd.io/raft/v3" 25 | ) 26 | 27 | func TestBasicProgress(t *testing.T) { 28 | peers := []raft.Peer{{ID: 1, Context: nil}, {ID: 2, Context: nil}, {ID: 3, Context: nil}, {ID: 4, Context: nil}, {ID: 5, Context: nil}} 29 | nt := newRaftNetwork(1, 2, 3, 4, 5) 30 | 31 | nodes := make([]*node, 0) 32 | 33 | for i := 1; i <= 5; i++ { 34 | n := startNode(uint64(i), peers, nt.nodeNetwork(uint64(i))) 35 | nodes = append(nodes, n) 36 | } 37 | 38 | waitLeader(nodes) 39 | 40 | for i := 0; i < 100; i++ { 41 | nodes[0].Propose(context.TODO(), []byte("somedata")) 42 | } 43 | 44 | assert.True(t, waitCommitConverge(nodes, 100)) 45 | 46 | for _, n := range nodes { 47 | n.stop() 48 | } 49 | } 50 | 51 | func TestRestart(t *testing.T) { 52 | peers := []raft.Peer{{ID: 1, Context: nil}, {ID: 2, Context: nil}, {ID: 3, Context: nil}, {ID: 4, Context: nil}, {ID: 5, Context: nil}} 53 | nt := newRaftNetwork(1, 2, 3, 4, 5) 54 | 55 | nodes := make([]*node, 0) 56 | 57 | for i := 1; i <= 5; i++ { 58 | n := startNode(uint64(i), peers, nt.nodeNetwork(uint64(i))) 59 | nodes = append(nodes, n) 60 | } 61 | 62 | l := waitLeader(nodes) 63 | k1, k2 := (l+1)%5, (l+2)%5 64 | 65 | for i := 0; i < 30; i++ { 66 | nodes[l].Propose(context.TODO(), []byte("somedata")) 67 | } 68 | nodes[k1].stop() 69 | for i := 0; i < 30; i++ { 70 | nodes[(l+3)%5].Propose(context.TODO(), []byte("somedata")) 71 | } 72 | nodes[k2].stop() 73 | for i := 0; i < 30; i++ { 74 | nodes[(l+4)%5].Propose(context.TODO(), []byte("somedata")) 75 | } 76 | nodes[k2].restart() 77 | for i := 0; i < 30; i++ { 78 | nodes[l].Propose(context.TODO(), []byte("somedata")) 79 | } 80 | nodes[k1].restart() 81 | 82 | assert.True(t, waitCommitConverge(nodes, 120)) 83 | 84 | for _, n := range nodes { 85 | n.stop() 86 | } 87 | } 88 | 89 | func TestPause(t *testing.T) { 90 | peers := []raft.Peer{{ID: 1, Context: nil}, {ID: 2, Context: nil}, {ID: 3, Context: nil}, {ID: 4, Context: nil}, {ID: 5, Context: nil}} 91 | nt := newRaftNetwork(1, 2, 3, 4, 5) 92 | 93 | nodes := make([]*node, 0) 94 | 95 | for i := 1; i <= 5; i++ { 96 | n := startNode(uint64(i), peers, nt.nodeNetwork(uint64(i))) 97 | nodes = append(nodes, n) 98 | } 99 | 100 | waitLeader(nodes) 101 | 102 | for i := 0; i < 30; i++ { 103 | nodes[0].Propose(context.TODO(), []byte("somedata")) 104 | } 105 | nodes[1].pause() 106 | for i := 0; i < 30; i++ { 107 | nodes[0].Propose(context.TODO(), []byte("somedata")) 108 | } 109 | nodes[2].pause() 110 | for i := 0; i < 30; i++ { 111 | nodes[0].Propose(context.TODO(), []byte("somedata")) 112 | } 113 | nodes[2].resume() 114 | for i := 0; i < 30; i++ { 115 | nodes[0].Propose(context.TODO(), []byte("somedata")) 116 | } 117 | nodes[1].resume() 118 | 119 | assert.True(t, waitCommitConverge(nodes, 120)) 120 | 121 | for _, n := range nodes { 122 | n.stop() 123 | } 124 | } 125 | 126 | func waitLeader(ns []*node) int { 127 | var l map[uint64]struct{} 128 | var lindex = -1 129 | 130 | for { 131 | l = make(map[uint64]struct{}) 132 | 133 | for i, n := range ns { 134 | lead := n.Status().SoftState.Lead 135 | if lead != 0 { 136 | l[lead] = struct{}{} 137 | if n.id == lead { 138 | lindex = i 139 | } 140 | } 141 | } 142 | 143 | if len(l) == 1 && lindex != -1 { 144 | return lindex 145 | } 146 | } 147 | } 148 | 149 | func waitCommitConverge(ns []*node, target uint64) bool { 150 | var c map[uint64]struct{} 151 | 152 | for i := 0; i < 50; i++ { 153 | c = make(map[uint64]struct{}) 154 | var good int 155 | 156 | for _, n := range ns { 157 | commit := n.Node.Status().HardState.Commit 158 | c[commit] = struct{}{} 159 | if commit > target { 160 | good++ 161 | } 162 | } 163 | 164 | if len(c) == 1 && good == len(ns) { 165 | return true 166 | } 167 | time.Sleep(100 * time.Millisecond) 168 | } 169 | 170 | return false 171 | } 172 | -------------------------------------------------------------------------------- /read_only.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raft 16 | 17 | import pb "go.etcd.io/raft/v3/raftpb" 18 | 19 | // ReadState provides state for read only query. 20 | // It's caller's responsibility to call ReadIndex first before getting 21 | // this state from ready, it's also caller's duty to differentiate if this 22 | // state is what it requests through RequestCtx, eg. given a unique id as 23 | // RequestCtx 24 | type ReadState struct { 25 | Index uint64 26 | RequestCtx []byte 27 | } 28 | 29 | type readIndexStatus struct { 30 | req pb.Message 31 | index uint64 32 | // NB: this never records 'false', but it's more convenient to use this 33 | // instead of a map[uint64]struct{} due to the API of quorum.VoteResult. If 34 | // this becomes performance sensitive enough (doubtful), quorum.VoteResult 35 | // can change to an API that is closer to that of CommittedIndex. 36 | acks map[uint64]bool 37 | } 38 | 39 | type readOnly struct { 40 | option ReadOnlyOption 41 | pendingReadIndex map[string]*readIndexStatus 42 | readIndexQueue []string 43 | } 44 | 45 | func newReadOnly(option ReadOnlyOption) *readOnly { 46 | return &readOnly{ 47 | option: option, 48 | pendingReadIndex: make(map[string]*readIndexStatus), 49 | } 50 | } 51 | 52 | // addRequest adds a read only request into readonly struct. 53 | // `index` is the commit index of the raft state machine when it received 54 | // the read only request. 55 | // `m` is the original read only request message from the local or remote node. 56 | func (ro *readOnly) addRequest(index uint64, m pb.Message) { 57 | s := string(m.Entries[0].Data) 58 | if _, ok := ro.pendingReadIndex[s]; ok { 59 | return 60 | } 61 | ro.pendingReadIndex[s] = &readIndexStatus{index: index, req: m, acks: make(map[uint64]bool)} 62 | ro.readIndexQueue = append(ro.readIndexQueue, s) 63 | } 64 | 65 | // recvAck notifies the readonly struct that the raft state machine received 66 | // an acknowledgment of the heartbeat that attached with the read only request 67 | // context. 68 | func (ro *readOnly) recvAck(id uint64, context []byte) map[uint64]bool { 69 | rs, ok := ro.pendingReadIndex[string(context)] 70 | if !ok { 71 | return nil 72 | } 73 | 74 | rs.acks[id] = true 75 | return rs.acks 76 | } 77 | 78 | // advance advances the read only request queue kept by the readonly struct. 79 | // It dequeues the requests until it finds the read only request that has 80 | // the same context as the given `m`. 81 | func (ro *readOnly) advance(m pb.Message) []*readIndexStatus { 82 | var ( 83 | i int 84 | found bool 85 | ) 86 | 87 | ctx := string(m.Context) 88 | var rss []*readIndexStatus 89 | 90 | for _, okctx := range ro.readIndexQueue { 91 | i++ 92 | rs, ok := ro.pendingReadIndex[okctx] 93 | if !ok { 94 | panic("cannot find corresponding read state from pending map") 95 | } 96 | rss = append(rss, rs) 97 | if okctx == ctx { 98 | found = true 99 | break 100 | } 101 | } 102 | 103 | if found { 104 | ro.readIndexQueue = ro.readIndexQueue[i:] 105 | for _, rs := range rss { 106 | delete(ro.pendingReadIndex, string(rs.req.Entries[0].Data)) 107 | } 108 | return rss 109 | } 110 | 111 | return nil 112 | } 113 | 114 | // lastPendingRequestCtx returns the context of the last pending read only 115 | // request in readonly struct. 116 | func (ro *readOnly) lastPendingRequestCtx() string { 117 | if len(ro.readIndexQueue) == 0 { 118 | return "" 119 | } 120 | return ro.readIndexQueue[len(ro.readIndexQueue)-1] 121 | } 122 | -------------------------------------------------------------------------------- /scripts/fix.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eo pipefail 4 | 5 | source ./scripts/test_lib.sh 6 | 7 | function mod_tidy_fix { 8 | rm ./go.sum 9 | go mod tidy || return 2 10 | } 11 | 12 | function go_fmt_fix { 13 | go fmt -n . || return 2 14 | } 15 | 16 | log_callout -e "\\nFixing raft code for you...\\n" 17 | 18 | mod_tidy_fix || exit 2 19 | go_fmt_fix || exit 2 20 | 21 | log_success -e "\\nSUCCESS: raft code is fixed :)" 22 | 23 | -------------------------------------------------------------------------------- /scripts/genproto.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Generate all etcd protobuf bindings. 4 | # Run from repository root directory named etcd. 5 | # 6 | set -e 7 | shopt -s globstar 8 | 9 | if ! [[ "$0" =~ scripts/genproto.sh ]]; then 10 | echo "must be run from repository root" 11 | exit 255 12 | fi 13 | 14 | source ./scripts/test_lib.sh 15 | 16 | if [[ $(protoc --version | cut -f2 -d' ') != "3.20.3" ]]; then 17 | echo "could not find protoc 3.20.3, is it installed + in PATH?" 18 | exit 255 19 | fi 20 | 21 | GOFAST_BIN=$(tool_get_bin github.com/gogo/protobuf/protoc-gen-gofast) 22 | GOGOPROTO_ROOT="$(tool_pkg_dir github.com/gogo/protobuf/proto)/.." 23 | 24 | echo 25 | echo "Resolved binary and packages versions:" 26 | echo " - protoc-gen-gofast: ${GOFAST_BIN}" 27 | echo " - gogoproto-root: ${GOGOPROTO_ROOT}" 28 | GOGOPROTO_PATH="${GOGOPROTO_ROOT}:${GOGOPROTO_ROOT}/protobuf" 29 | 30 | # directories containing protos to be built 31 | DIRS="./raftpb" 32 | 33 | log_callout -e "\\nRunning gofast (gogo) proto generation..." 34 | 35 | for dir in ${DIRS}; do 36 | pushd "${dir}" 37 | protoc --gofast_out=. -I=".:${GOGOPROTO_PATH}:${RAFT_ROOT_DIR}/..:${RAFT_ROOT_DIR}" \ 38 | --plugin="${GOFAST_BIN}" ./**/*.proto 39 | 40 | sed -i.bak -E 's|"raft/raftpb"|"go.etcd.io/etcd/raft/v3/raftpb"|g' ./**/*.pb.go 41 | sed -i.bak -E 's|"google/protobuf"|"github.com/gogo/protobuf/protoc-gen-gogo/descriptor"|g' ./**/*.pb.go 42 | 43 | rm -f ./**/*.bak 44 | gofmt -s -w ./**/*.pb.go 45 | run_go_tool "golang.org/x/tools/cmd/goimports" -w ./**/*.pb.go 46 | popd 47 | done 48 | 49 | log_success -e "\\n./genproto SUCCESS" 50 | -------------------------------------------------------------------------------- /scripts/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eo pipefail 4 | 5 | source ./scripts/test_lib.sh 6 | 7 | # set default GOARCH if not set 8 | if [ -z "$GOARCH" ]; then 9 | GOARCH=$(go env GOARCH); 10 | fi 11 | 12 | # determine whether target supports race detection 13 | if [ -z "${RACE}" ] ; then 14 | if [ "$GOARCH" == "amd64" ]; then 15 | RACE="--race" 16 | else 17 | RACE="--race=false" 18 | fi 19 | else 20 | RACE="--race=${RACE:-true}" 21 | fi 22 | 23 | # This options make sense for cases where SUT (System Under Test) is compiled by test. 24 | COMMON_TEST_FLAGS=("${RACE}") 25 | if [[ -n "${CPU}" ]]; then 26 | COMMON_TEST_FLAGS+=("--cpu=${CPU}") 27 | fi 28 | 29 | ######### Code formatting checkers ############################################# 30 | 31 | # generic_checker [cmd...] 32 | # executes given command in the current module, and clearly fails if it 33 | # failed or returned output. 34 | function generic_checker { 35 | local cmd=("$@") 36 | if ! output=$("${cmd[@]}"); then 37 | echo "${output}" 38 | log_error -e "FAIL: '${cmd[*]}' checking failed (!=0 return code)" 39 | return 255 40 | fi 41 | if [ -n "${output}" ]; then 42 | echo "${output}" 43 | log_error -e "FAIL: '${cmd[*]}' checking failed (printed output)" 44 | return 255 45 | fi 46 | } 47 | 48 | function go_fmt_for_package { 49 | # We utilize 'go fmt' to find all files suitable for formatting, 50 | # but reuse full power gofmt to perform just RO check. 51 | go fmt -n . | sed 's| -w | -d |g' | sh 52 | } 53 | 54 | function gofmt_pass { 55 | generic_checker go_fmt_for_package 56 | } 57 | 58 | function genproto_pass { 59 | "${RAFT_ROOT_DIR}/scripts/verify_genproto.sh" 60 | } 61 | 62 | ######## VARIOUS CHECKERS ###################################################### 63 | 64 | function dump_deps_of_module() { 65 | local module 66 | if ! module=$(go list -m); then 67 | return 255 68 | fi 69 | go list -f "{{if not .Indirect}}{{if .Version}}{{.Path}},{{.Version}},${module}{{end}}{{end}}" -m all 70 | } 71 | 72 | # Checks whether dependencies are consistent across modules 73 | function dep_pass { 74 | local all_dependencies 75 | all_dependencies=$(dump_deps_of_module | sort) || return 2 76 | 77 | local duplicates 78 | duplicates=$(echo "${all_dependencies}" | cut -d ',' -f 1,2 | sort | uniq | cut -d ',' -f 1 | sort | uniq -d) || return 2 79 | 80 | for dup in ${duplicates}; do 81 | log_error "FAIL: inconsistent versions for depencency: ${dup}" 82 | echo "${all_dependencies}" | grep "${dup}" | sed "s|\\([^,]*\\),\\([^,]*\\),\\([^,]*\\)| - \\1@\\2 from: \\3|g" 83 | done 84 | if [[ -n "${duplicates}" ]]; then 85 | log_error "FAIL: inconsistent dependencies" 86 | return 2 87 | else 88 | log_success "SUCCESS: dependencies are consistent across modules" 89 | fi 90 | } 91 | 92 | function mod_tidy_for_module { 93 | # Watch for upstream solution: https://github.com/golang/go/issues/27005 94 | local tmpModDir 95 | tmpModDir=$(mktemp -d -t 'tmpModDir.XXXXXX') 96 | cp "./go.mod" "${tmpModDir}" || return 2 97 | 98 | # Guarantees keeping go.sum minimal 99 | # If this is causing too much problems, we should 100 | # stop controlling go.sum at all. 101 | rm go.sum 102 | go mod tidy || return 2 103 | 104 | set +e 105 | local tmpFileGoModInSync 106 | diff -C 5 "${tmpModDir}/go.mod" "./go.mod" 107 | tmpFileGoModInSync="$?" 108 | 109 | # Bring back initial state 110 | mv "${tmpModDir}/go.mod" "./go.mod" 111 | 112 | if [ "${tmpFileGoModInSync}" -ne 0 ]; then 113 | log_error "${PWD}/go.mod is not in sync with 'go mod tidy'" 114 | return 255 115 | fi 116 | } 117 | 118 | function mod_tidy_pass { 119 | mod_tidy_for_module 120 | } 121 | 122 | ################# REGULAR TESTS ################################################ 123 | 124 | # run_unit_tests [pkgs] runs unit tests for a current module and givesn set of [pkgs] 125 | function run_unit_tests { 126 | shift 1 127 | # shellcheck disable=SC2086 128 | GOLANG_TEST_SHORT=true go test ./... -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" 129 | } 130 | 131 | function unit_pass { 132 | run_unit_tests "$@" 133 | } 134 | 135 | ########### MAIN ############################################################### 136 | 137 | function run_pass { 138 | local pass="${1}" 139 | shift 1 140 | log_callout -e "\\n'${pass}' started at $(date)" 141 | if "${pass}_pass" "$@" ; then 142 | log_success "'${pass}' completed at $(date)" 143 | else 144 | log_error "FAIL: '${pass}' failed at $(date)" 145 | exit 255 146 | fi 147 | } 148 | 149 | log_callout "Starting at: $(date)" 150 | for pass in $PASSES; do 151 | run_pass "${pass}" "${@}" 152 | done 153 | 154 | log_success "SUCCESS" 155 | 156 | -------------------------------------------------------------------------------- /scripts/test_lib.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ROOT_MODULE="go.etcd.io/raft" 4 | 5 | function set_root_dir { 6 | RAFT_ROOT_DIR=$(go list -f '{{.Dir}}' "${ROOT_MODULE}/v3") 7 | } 8 | 9 | set_root_dir 10 | 11 | #### Convenient IO methods ##### 12 | 13 | COLOR_RED='\033[0;31m' 14 | COLOR_ORANGE='\033[0;33m' 15 | COLOR_GREEN='\033[0;32m' 16 | COLOR_LIGHTCYAN='\033[0;36m' 17 | COLOR_BLUE='\033[0;94m' 18 | COLOR_MAGENTA='\033[95m' 19 | COLOR_BOLD='\033[1m' 20 | COLOR_NONE='\033[0m' # No Color 21 | 22 | 23 | function log_error { 24 | >&2 echo -n -e "${COLOR_BOLD}${COLOR_RED}" 25 | >&2 echo "$@" 26 | >&2 echo -n -e "${COLOR_NONE}" 27 | } 28 | 29 | function log_warning { 30 | >&2 echo -n -e "${COLOR_ORANGE}" 31 | >&2 echo "$@" 32 | >&2 echo -n -e "${COLOR_NONE}" 33 | } 34 | 35 | function log_callout { 36 | >&2 echo -n -e "${COLOR_LIGHTCYAN}" 37 | >&2 echo "$@" 38 | >&2 echo -n -e "${COLOR_NONE}" 39 | } 40 | 41 | function log_cmd { 42 | >&2 echo -n -e "${COLOR_BLUE}" 43 | >&2 echo "$@" 44 | >&2 echo -n -e "${COLOR_NONE}" 45 | } 46 | 47 | function log_success { 48 | >&2 echo -n -e "${COLOR_GREEN}" 49 | >&2 echo "$@" 50 | >&2 echo -n -e "${COLOR_NONE}" 51 | } 52 | 53 | function log_info { 54 | >&2 echo -n -e "${COLOR_NONE}" 55 | >&2 echo "$@" 56 | >&2 echo -n -e "${COLOR_NONE}" 57 | } 58 | 59 | # run_for_module [module] [cmd] 60 | # executes given command in the given module for given pkgs. 61 | # module_name - "." (in future: tests, client, server) 62 | # cmd - cmd to be executed - that takes package as last argument 63 | function run_for_module { 64 | local module=${1:-"."} 65 | shift 1 66 | ( 67 | cd "${RAFT_ROOT_DIR}/${module}" && "$@" 68 | ) 69 | } 70 | 71 | # tool_pkg_dir [pkg] - returns absolute path to a directory that stores given pkg. 72 | # The pkg versions must be defined in ./tools/mod directory. 73 | function tool_pkg_dir { 74 | run_for_module ./tools/mod go list -f '{{.Dir}}' "${1}" 75 | } 76 | 77 | # tool_get_bin [tool] - returns absolute path to a tool binary (or returns error) 78 | function tool_get_bin { 79 | local tool="$1" 80 | local pkg_part="$1" 81 | if [[ "$tool" == *"@"* ]]; then 82 | pkg_part=$(echo "${tool}" | cut -d'@' -f1) 83 | # shellcheck disable=SC2086 84 | go install ${GOBINARGS:-} "${tool}" || return 2 85 | else 86 | # shellcheck disable=SC2086 87 | run_for_module ./tools/mod go install ${GOBINARGS:-} "${tool}" || return 2 88 | fi 89 | 90 | # remove the version suffix, such as removing "/v3" from "go.etcd.io/etcd/v3". 91 | local cmd_base_name 92 | cmd_base_name=$(basename "${pkg_part}") 93 | if [[ ${cmd_base_name} =~ ^v[0-9]*$ ]]; then 94 | pkg_part=$(dirname "${pkg_part}") 95 | fi 96 | 97 | run_for_module ./tools/mod go list -f '{{.Target}}' "${pkg_part}" 98 | } 99 | 100 | # tool_get_bin [tool] 101 | function run_go_tool { 102 | local cmdbin 103 | if ! cmdbin=$(GOARCH="" tool_get_bin "${1}"); then 104 | log_warning "Failed to install tool '${1}'" 105 | return 2 106 | fi 107 | shift 1 108 | GOARCH="" "${cmdbin}" "$@" || return 2 109 | } 110 | -------------------------------------------------------------------------------- /scripts/verify_genproto.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # This scripts is automatically run by CI to prevent pull requests missing running genproto.sh 3 | # after changing *.proto file. 4 | 5 | set -o errexit 6 | set -o nounset 7 | set -o pipefail 8 | 9 | tmpWorkDir=$(mktemp -d -t 'twd.XXXXXX') 10 | mkdir "$tmpWorkDir/raft" 11 | tmpWorkDir="$tmpWorkDir/raft" 12 | cp -r . "$tmpWorkDir" 13 | pushd "$tmpWorkDir" 14 | git add -A 15 | git commit -m init || true # maybe fail because nothing to commit 16 | ./scripts/genproto.sh 17 | diff=$(git diff --numstat | awk '{print $3}') 18 | popd 19 | if [ -z "$diff" ]; then 20 | echo "PASSED genproto-verification!" 21 | exit 0 22 | fi 23 | echo "Failed genproto-verification!" >&2 24 | printf "* Found changed files:\n%s\n" "$diff" >&2 25 | echo "* Please rerun genproto.sh after changing *.proto file" >&2 26 | echo "* Run ./scripts/genproto.sh" >&2 27 | exit 1 28 | -------------------------------------------------------------------------------- /state_trace_nop.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !with_tla 16 | 17 | package raft 18 | 19 | import ( 20 | "go.etcd.io/raft/v3/raftpb" 21 | "go.etcd.io/raft/v3/tracker" 22 | ) 23 | 24 | const StateTraceDeployed = false 25 | 26 | type TraceLogger interface{} 27 | 28 | type TracingEvent struct{} 29 | 30 | func traceInitState(*raft) {} 31 | 32 | func traceReady(*raft) {} 33 | 34 | func traceCommit(*raft) {} 35 | 36 | func traceReplicate(*raft, ...raftpb.Entry) {} 37 | 38 | func traceBecomeFollower(*raft) {} 39 | 40 | func traceBecomeCandidate(*raft) {} 41 | 42 | func traceBecomeLeader(*raft) {} 43 | 44 | func traceChangeConfEvent(raftpb.ConfChangeI, *raft) {} 45 | 46 | func traceConfChangeEvent(tracker.Config, *raft) {} 47 | 48 | func traceSendMessage(*raft, *raftpb.Message) {} 49 | 50 | func traceReceiveMessage(*raft, *raftpb.Message) {} 51 | -------------------------------------------------------------------------------- /status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raft 16 | 17 | import ( 18 | "fmt" 19 | 20 | pb "go.etcd.io/raft/v3/raftpb" 21 | "go.etcd.io/raft/v3/tracker" 22 | ) 23 | 24 | // Status contains information about this Raft peer and its view of the system. 25 | // The Progress is only populated on the leader. 26 | type Status struct { 27 | BasicStatus 28 | Config tracker.Config 29 | Progress map[uint64]tracker.Progress 30 | } 31 | 32 | // BasicStatus contains basic information about the Raft peer. It does not allocate. 33 | type BasicStatus struct { 34 | ID uint64 35 | 36 | pb.HardState 37 | SoftState 38 | 39 | Applied uint64 40 | 41 | LeadTransferee uint64 42 | } 43 | 44 | func getProgressCopy(r *raft) map[uint64]tracker.Progress { 45 | m := make(map[uint64]tracker.Progress) 46 | r.trk.Visit(func(id uint64, pr *tracker.Progress) { 47 | p := *pr 48 | p.Inflights = pr.Inflights.Clone() 49 | pr = nil 50 | 51 | m[id] = p 52 | }) 53 | return m 54 | } 55 | 56 | func getBasicStatus(r *raft) BasicStatus { 57 | s := BasicStatus{ 58 | ID: r.id, 59 | LeadTransferee: r.leadTransferee, 60 | } 61 | s.HardState = r.hardState() 62 | s.SoftState = r.softState() 63 | s.Applied = r.raftLog.applied 64 | return s 65 | } 66 | 67 | // getStatus gets a copy of the current raft status. 68 | func getStatus(r *raft) Status { 69 | var s Status 70 | s.BasicStatus = getBasicStatus(r) 71 | if s.RaftState == StateLeader { 72 | s.Progress = getProgressCopy(r) 73 | } 74 | s.Config = r.trk.Config.Clone() 75 | return s 76 | } 77 | 78 | // MarshalJSON translates the raft status into JSON. 79 | // TODO: try to simplify this by introducing ID type into raft 80 | func (s Status) MarshalJSON() ([]byte, error) { 81 | j := fmt.Sprintf(`{"id":"%x","term":%d,"vote":"%x","commit":%d,"lead":"%x","raftState":%q,"applied":%d,"progress":{`, 82 | s.ID, s.Term, s.Vote, s.Commit, s.Lead, s.RaftState, s.Applied) 83 | 84 | if len(s.Progress) == 0 { 85 | j += "}," 86 | } else { 87 | for k, v := range s.Progress { 88 | subj := fmt.Sprintf(`"%x":{"match":%d,"next":%d,"state":%q},`, k, v.Match, v.Next, v.State) 89 | j += subj 90 | } 91 | // remove the trailing "," 92 | j = j[:len(j)-1] + "}," 93 | } 94 | 95 | j += fmt.Sprintf(`"leadtransferee":"%x"}`, s.LeadTransferee) 96 | return []byte(j), nil 97 | } 98 | 99 | func (s Status) String() string { 100 | b, err := s.MarshalJSON() 101 | if err != nil { 102 | getLogger().Panicf("unexpected error: %v", err) 103 | } 104 | return string(b) 105 | } 106 | -------------------------------------------------------------------------------- /testdata/campaign.txt: -------------------------------------------------------------------------------- 1 | log-level info 2 | ---- 3 | ok 4 | 5 | add-nodes 3 voters=(1,2,3) index=2 6 | ---- 7 | INFO 1 switched to configuration voters=(1 2 3) 8 | INFO 1 became follower at term 0 9 | INFO newRaft 1 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] 10 | INFO 2 switched to configuration voters=(1 2 3) 11 | INFO 2 became follower at term 0 12 | INFO newRaft 2 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] 13 | INFO 3 switched to configuration voters=(1 2 3) 14 | INFO 3 became follower at term 0 15 | INFO newRaft 3 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] 16 | 17 | campaign 1 18 | ---- 19 | INFO 1 is starting a new election at term 0 20 | INFO 1 became candidate at term 1 21 | INFO 1 [logterm: 1, index: 2] sent MsgVote request to 2 at term 1 22 | INFO 1 [logterm: 1, index: 2] sent MsgVote request to 3 at term 1 23 | 24 | stabilize 25 | ---- 26 | > 1 handling Ready 27 | Ready MustSync=true: 28 | Lead:0 State:StateCandidate 29 | HardState Term:1 Vote:1 Commit:2 30 | Messages: 31 | 1->2 MsgVote Term:1 Log:1/2 32 | 1->3 MsgVote Term:1 Log:1/2 33 | INFO 1 received MsgVoteResp from 1 at term 1 34 | INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections 35 | > 2 receiving messages 36 | 1->2 MsgVote Term:1 Log:1/2 37 | INFO 2 [term: 0] received a MsgVote message with higher term from 1 [term: 1] 38 | INFO 2 became follower at term 1 39 | INFO 2 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 40 | > 3 receiving messages 41 | 1->3 MsgVote Term:1 Log:1/2 42 | INFO 3 [term: 0] received a MsgVote message with higher term from 1 [term: 1] 43 | INFO 3 became follower at term 1 44 | INFO 3 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 45 | > 2 handling Ready 46 | Ready MustSync=true: 47 | HardState Term:1 Vote:1 Commit:2 48 | Messages: 49 | 2->1 MsgVoteResp Term:1 Log:0/0 50 | > 3 handling Ready 51 | Ready MustSync=true: 52 | HardState Term:1 Vote:1 Commit:2 53 | Messages: 54 | 3->1 MsgVoteResp Term:1 Log:0/0 55 | > 1 receiving messages 56 | 2->1 MsgVoteResp Term:1 Log:0/0 57 | INFO 1 received MsgVoteResp from 2 at term 1 58 | INFO 1 has received 2 MsgVoteResp votes and 0 vote rejections 59 | INFO 1 became leader at term 1 60 | 3->1 MsgVoteResp Term:1 Log:0/0 61 | > 1 handling Ready 62 | Ready MustSync=true: 63 | Lead:1 State:StateLeader 64 | Entries: 65 | 1/3 EntryNormal "" 66 | Messages: 67 | 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] 68 | 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] 69 | > 2 receiving messages 70 | 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] 71 | > 3 receiving messages 72 | 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] 73 | > 2 handling Ready 74 | Ready MustSync=true: 75 | Lead:1 State:StateFollower 76 | Entries: 77 | 1/3 EntryNormal "" 78 | Messages: 79 | 2->1 MsgAppResp Term:1 Log:0/3 80 | > 3 handling Ready 81 | Ready MustSync=true: 82 | Lead:1 State:StateFollower 83 | Entries: 84 | 1/3 EntryNormal "" 85 | Messages: 86 | 3->1 MsgAppResp Term:1 Log:0/3 87 | > 1 receiving messages 88 | 2->1 MsgAppResp Term:1 Log:0/3 89 | 3->1 MsgAppResp Term:1 Log:0/3 90 | > 1 handling Ready 91 | Ready MustSync=false: 92 | HardState Term:1 Vote:1 Commit:3 93 | CommittedEntries: 94 | 1/3 EntryNormal "" 95 | Messages: 96 | 1->2 MsgApp Term:1 Log:1/3 Commit:3 97 | 1->3 MsgApp Term:1 Log:1/3 Commit:3 98 | > 2 receiving messages 99 | 1->2 MsgApp Term:1 Log:1/3 Commit:3 100 | > 3 receiving messages 101 | 1->3 MsgApp Term:1 Log:1/3 Commit:3 102 | > 2 handling Ready 103 | Ready MustSync=false: 104 | HardState Term:1 Vote:1 Commit:3 105 | CommittedEntries: 106 | 1/3 EntryNormal "" 107 | Messages: 108 | 2->1 MsgAppResp Term:1 Log:0/3 109 | > 3 handling Ready 110 | Ready MustSync=false: 111 | HardState Term:1 Vote:1 Commit:3 112 | CommittedEntries: 113 | 1/3 EntryNormal "" 114 | Messages: 115 | 3->1 MsgAppResp Term:1 Log:0/3 116 | > 1 receiving messages 117 | 2->1 MsgAppResp Term:1 Log:0/3 118 | 3->1 MsgAppResp Term:1 Log:0/3 119 | -------------------------------------------------------------------------------- /testdata/confchange_disable_validation.txt: -------------------------------------------------------------------------------- 1 | # This test verifies the DisableConfChangeValidation setting. 2 | # With it set, raft should allow configuration changes to enter the log even 3 | # if they appear to be incompatible with the currently active configuration. 4 | # 5 | # The test sets up a single-voter group that applies entries one at a time. 6 | # Then it proposes a bogus entry followed by a conf change. When the bogus entry 7 | # has applied, a second (compatible, but the node doesn't know this yet) 8 | # configuration change is proposed. That configuration change is accepted into 9 | # the log since due to DisableConfChangeValidation=true. 10 | add-nodes 1 voters=(1) index=2 max-committed-size-per-ready=1 disable-conf-change-validation=true 11 | ---- 12 | INFO 1 switched to configuration voters=(1) 13 | INFO 1 became follower at term 0 14 | INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] 15 | 16 | campaign 1 17 | ---- 18 | INFO 1 is starting a new election at term 0 19 | INFO 1 became candidate at term 1 20 | 21 | stabilize log-level=none 22 | ---- 23 | ok 24 | 25 | # Dummy entry. 26 | propose 1 foo 27 | ---- 28 | ok 29 | 30 | propose-conf-change 1 transition=explicit 31 | l2 l3 32 | ---- 33 | ok 34 | 35 | # Entries both get appended. 36 | process-ready 1 37 | ---- 38 | Ready MustSync=true: 39 | Entries: 40 | 1/4 EntryNormal "foo" 41 | 1/5 EntryConfChangeV2 l2 l3 42 | 43 | # Dummy entry comes up for application. 44 | process-ready 1 45 | ---- 46 | Ready MustSync=false: 47 | HardState Term:1 Vote:1 Commit:5 48 | CommittedEntries: 49 | 1/4 EntryNormal "foo" 50 | 51 | # Propose new config change. Note how it isn't rejected, 52 | # which is due to DisableConfChangeValidation=true. 53 | propose-conf-change 1 54 | ---- 55 | ok 56 | 57 | # Turn on autopilot: the first config change applies, the 58 | # second one gets committed and also applies. 59 | stabilize 60 | ---- 61 | > 1 handling Ready 62 | Ready MustSync=true: 63 | Entries: 64 | 1/6 EntryConfChangeV2 65 | CommittedEntries: 66 | 1/5 EntryConfChangeV2 l2 l3 67 | INFO 1 switched to configuration voters=(1)&&(1) learners=(2 3) 68 | > 1 handling Ready 69 | Ready MustSync=false: 70 | HardState Term:1 Vote:1 Commit:6 71 | CommittedEntries: 72 | 1/6 EntryConfChangeV2 73 | Messages: 74 | 1->2 MsgApp Term:1 Log:1/5 Commit:5 Entries:[1/6 EntryConfChangeV2] 75 | 1->3 MsgApp Term:1 Log:1/5 Commit:5 Entries:[1/6 EntryConfChangeV2] 76 | INFO 1 switched to configuration voters=(1) learners=(2 3) 77 | -------------------------------------------------------------------------------- /testdata/confchange_v1_add_single.txt: -------------------------------------------------------------------------------- 1 | # Run a V1 membership change that adds a single voter. 2 | 3 | # Bootstrap n1. 4 | add-nodes 1 voters=(1) index=2 5 | ---- 6 | INFO 1 switched to configuration voters=(1) 7 | INFO 1 became follower at term 0 8 | INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] 9 | 10 | campaign 1 11 | ---- 12 | INFO 1 is starting a new election at term 0 13 | INFO 1 became candidate at term 1 14 | 15 | process-ready 1 16 | ---- 17 | Ready MustSync=true: 18 | Lead:0 State:StateCandidate 19 | HardState Term:1 Vote:1 Commit:2 20 | INFO 1 received MsgVoteResp from 1 at term 1 21 | INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections 22 | INFO 1 became leader at term 1 23 | 24 | # Add v2 (with an auto transition). 25 | propose-conf-change 1 v1=true 26 | v2 27 | ---- 28 | ok 29 | 30 | # Pull n2 out of thin air. 31 | add-nodes 1 32 | ---- 33 | INFO 2 switched to configuration voters=() 34 | INFO 2 became follower at term 0 35 | INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] 36 | 37 | # n1 commits the conf change using itself as commit quorum, immediately transitions into 38 | # the final config, and catches up n2. Note that it's using an EntryConfChange, not an 39 | # EntryConfChangeV2, so this is compatible with nodes that don't know about V2 conf changes. 40 | stabilize 41 | ---- 42 | > 1 handling Ready 43 | Ready MustSync=true: 44 | Lead:1 State:StateLeader 45 | Entries: 46 | 1/3 EntryNormal "" 47 | 1/4 EntryConfChange v2 48 | > 1 handling Ready 49 | Ready MustSync=false: 50 | HardState Term:1 Vote:1 Commit:4 51 | CommittedEntries: 52 | 1/3 EntryNormal "" 53 | 1/4 EntryConfChange v2 54 | INFO 1 switched to configuration voters=(1 2) 55 | > 1 handling Ready 56 | Ready MustSync=false: 57 | Messages: 58 | 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChange v2] 59 | > 2 receiving messages 60 | 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChange v2] 61 | INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] 62 | INFO 2 became follower at term 1 63 | DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 64 | > 2 handling Ready 65 | Ready MustSync=true: 66 | Lead:1 State:StateFollower 67 | HardState Term:1 Commit:0 68 | Messages: 69 | 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) 70 | > 1 receiving messages 71 | 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) 72 | DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 73 | DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] 74 | DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] 75 | DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] 76 | > 1 handling Ready 77 | Ready MustSync=false: 78 | Messages: 79 | 1->2 MsgSnap Term:1 Log:0/0 80 | Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false 81 | > 2 receiving messages 82 | 1->2 MsgSnap Term:1 Log:0/0 83 | Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false 84 | INFO log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] 85 | INFO 2 switched to configuration voters=(1 2) 86 | INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] 87 | INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] 88 | > 2 handling Ready 89 | Ready MustSync=false: 90 | HardState Term:1 Commit:4 91 | Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false 92 | Messages: 93 | 2->1 MsgAppResp Term:1 Log:0/4 94 | > 1 receiving messages 95 | 2->1 MsgAppResp Term:1 Log:0/4 96 | DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] 97 | -------------------------------------------------------------------------------- /testdata/confchange_v2_add_single_auto.txt: -------------------------------------------------------------------------------- 1 | # Run a V2 membership change that adds a single voter in auto mode, which means 2 | # that joint consensus is not used but a direct transition into the new config 3 | # takes place. 4 | 5 | # Bootstrap n1. 6 | add-nodes 1 voters=(1) index=2 7 | ---- 8 | INFO 1 switched to configuration voters=(1) 9 | INFO 1 became follower at term 0 10 | INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] 11 | 12 | campaign 1 13 | ---- 14 | INFO 1 is starting a new election at term 0 15 | INFO 1 became candidate at term 1 16 | 17 | process-ready 1 18 | ---- 19 | Ready MustSync=true: 20 | Lead:0 State:StateCandidate 21 | HardState Term:1 Vote:1 Commit:2 22 | INFO 1 received MsgVoteResp from 1 at term 1 23 | INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections 24 | INFO 1 became leader at term 1 25 | 26 | # Add v2 (with an auto transition). 27 | propose-conf-change 1 28 | v2 29 | ---- 30 | ok 31 | 32 | # Pull n2 out of thin air. 33 | add-nodes 1 34 | ---- 35 | INFO 2 switched to configuration voters=() 36 | INFO 2 became follower at term 0 37 | INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] 38 | 39 | # n1 commits the conf change using itself as commit quorum, immediately transitions into 40 | # the final config, and catches up n2. 41 | stabilize 42 | ---- 43 | > 1 handling Ready 44 | Ready MustSync=true: 45 | Lead:1 State:StateLeader 46 | Entries: 47 | 1/3 EntryNormal "" 48 | 1/4 EntryConfChangeV2 v2 49 | > 1 handling Ready 50 | Ready MustSync=false: 51 | HardState Term:1 Vote:1 Commit:4 52 | CommittedEntries: 53 | 1/3 EntryNormal "" 54 | 1/4 EntryConfChangeV2 v2 55 | INFO 1 switched to configuration voters=(1 2) 56 | > 1 handling Ready 57 | Ready MustSync=false: 58 | Messages: 59 | 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] 60 | > 2 receiving messages 61 | 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] 62 | INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] 63 | INFO 2 became follower at term 1 64 | DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 65 | > 2 handling Ready 66 | Ready MustSync=true: 67 | Lead:1 State:StateFollower 68 | HardState Term:1 Commit:0 69 | Messages: 70 | 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) 71 | > 1 receiving messages 72 | 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) 73 | DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 74 | DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] 75 | DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] 76 | DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] 77 | > 1 handling Ready 78 | Ready MustSync=false: 79 | Messages: 80 | 1->2 MsgSnap Term:1 Log:0/0 81 | Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false 82 | > 2 receiving messages 83 | 1->2 MsgSnap Term:1 Log:0/0 84 | Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false 85 | INFO log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] 86 | INFO 2 switched to configuration voters=(1 2) 87 | INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] 88 | INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] 89 | > 2 handling Ready 90 | Ready MustSync=false: 91 | HardState Term:1 Commit:4 92 | Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false 93 | Messages: 94 | 2->1 MsgAppResp Term:1 Log:0/4 95 | > 1 receiving messages 96 | 2->1 MsgAppResp Term:1 Log:0/4 97 | DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] 98 | -------------------------------------------------------------------------------- /testdata/forget_leader_read_only_lease_based.txt: -------------------------------------------------------------------------------- 1 | log-level none 2 | ---- 3 | ok 4 | 5 | add-nodes 3 voters=(1,2,3) index=10 checkquorum=true read-only=lease-based 6 | ---- 7 | ok 8 | 9 | campaign 1 10 | ---- 11 | ok 12 | 13 | stabilize 14 | ---- 15 | ok 16 | 17 | log-level debug 18 | ---- 19 | ok 20 | 21 | # ForgetLeader fails with lease-based reads, as it's not safe. 22 | forget-leader 2 23 | ---- 24 | ERROR ignoring MsgForgetLeader due to ReadOnlyLeaseBased 25 | 26 | raft-state 27 | ---- 28 | 1: StateLeader (Voter) Term:1 Lead:1 29 | 2: StateFollower (Voter) Term:1 Lead:1 30 | 3: StateFollower (Voter) Term:1 Lead:1 31 | -------------------------------------------------------------------------------- /testdata/heartbeat_resp_recovers_from_probing.txt: -------------------------------------------------------------------------------- 1 | # This test checks that if a fully caught-up follower transitions 2 | # into StateProbe (for example due to a call to ReportUnreachable), the 3 | # leader will react to a subsequent heartbeat response from the probing 4 | # follower by sending an empty MsgApp, the response of which restores 5 | # StateReplicate for the follower. In other words, we don't end up in 6 | # a stable state with a fully caught up follower in StateProbe. 7 | 8 | # Turn off output during the setup of the test. 9 | log-level none 10 | ---- 11 | ok 12 | 13 | add-nodes 3 voters=(1,2,3) index=10 14 | ---- 15 | ok 16 | 17 | campaign 1 18 | ---- 19 | ok 20 | 21 | stabilize 22 | ---- 23 | ok 24 | 25 | log-level debug 26 | ---- 27 | ok 28 | 29 | status 1 30 | ---- 31 | 1: StateReplicate match=11 next=12 32 | 2: StateReplicate match=11 next=12 33 | 3: StateReplicate match=11 next=12 34 | 35 | # On the first replica, report the second one as not reachable. 36 | report-unreachable 1 2 37 | ---- 38 | DEBUG 1 failed to send message to 2 because it is unreachable [StateProbe match=11 next=12] 39 | 40 | status 1 41 | ---- 42 | 1: StateReplicate match=11 next=12 43 | 2: StateProbe match=11 next=12 44 | 3: StateReplicate match=11 next=12 45 | 46 | tick-heartbeat 1 47 | ---- 48 | ok 49 | 50 | # Heartbeat -> HeartbeatResp -> MsgApp -> MsgAppResp -> StateReplicate. 51 | stabilize 52 | ---- 53 | > 1 handling Ready 54 | Ready MustSync=false: 55 | Messages: 56 | 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 57 | 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 58 | > 2 receiving messages 59 | 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 60 | > 3 receiving messages 61 | 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 62 | > 2 handling Ready 63 | Ready MustSync=false: 64 | Messages: 65 | 2->1 MsgHeartbeatResp Term:1 Log:0/0 66 | > 3 handling Ready 67 | Ready MustSync=false: 68 | Messages: 69 | 3->1 MsgHeartbeatResp Term:1 Log:0/0 70 | > 1 receiving messages 71 | 2->1 MsgHeartbeatResp Term:1 Log:0/0 72 | 3->1 MsgHeartbeatResp Term:1 Log:0/0 73 | > 1 handling Ready 74 | Ready MustSync=false: 75 | Messages: 76 | 1->2 MsgApp Term:1 Log:1/11 Commit:11 77 | > 2 receiving messages 78 | 1->2 MsgApp Term:1 Log:1/11 Commit:11 79 | > 2 handling Ready 80 | Ready MustSync=false: 81 | Messages: 82 | 2->1 MsgAppResp Term:1 Log:0/11 83 | > 1 receiving messages 84 | 2->1 MsgAppResp Term:1 Log:0/11 85 | 86 | status 1 87 | ---- 88 | 1: StateReplicate match=11 next=12 89 | 2: StateReplicate match=11 next=12 90 | 3: StateReplicate match=11 next=12 91 | -------------------------------------------------------------------------------- /testdata/replicate_pause.txt: -------------------------------------------------------------------------------- 1 | # This test ensures that MsgApp stream to a follower is paused when the 2 | # in-flight state exceeds the configured limits. This is a regression test for 3 | # the issue fixed by https://github.com/etcd-io/etcd/pull/14633. 4 | 5 | # Turn off output during the setup of the test. 6 | log-level none 7 | ---- 8 | ok 9 | 10 | # Start with 3 nodes, with a limited in-flight capacity. 11 | add-nodes 3 voters=(1,2,3) index=10 inflight=3 12 | ---- 13 | ok 14 | 15 | campaign 1 16 | ---- 17 | ok 18 | 19 | stabilize 20 | ---- 21 | ok 22 | 23 | # Propose 3 entries. 24 | propose 1 prop_1_12 25 | ---- 26 | ok 27 | 28 | propose 1 prop_1_13 29 | ---- 30 | ok 31 | 32 | propose 1 prop_1_14 33 | ---- 34 | ok 35 | 36 | # Store entries and send proposals. 37 | process-ready 1 38 | ---- 39 | ok 40 | 41 | # Re-enable log messages. 42 | log-level debug 43 | ---- 44 | ok 45 | 46 | # Expect that in-flight tracking to nodes 2 and 3 is saturated. 47 | status 1 48 | ---- 49 | 1: StateReplicate match=14 next=15 50 | 2: StateReplicate match=11 next=15 paused inflight=3[full] 51 | 3: StateReplicate match=11 next=15 paused inflight=3[full] 52 | 53 | log-level none 54 | ---- 55 | ok 56 | 57 | # Commit entries between nodes 1 and 2. 58 | stabilize 1 2 59 | ---- 60 | ok 61 | 62 | log-level debug 63 | ---- 64 | ok 65 | 66 | # Expect that the entries are committed and stored on nodes 1 and 2. 67 | status 1 68 | ---- 69 | 1: StateReplicate match=14 next=15 70 | 2: StateReplicate match=14 next=15 71 | 3: StateReplicate match=11 next=15 paused inflight=3[full] 72 | 73 | # Drop append messages to node 3. 74 | deliver-msgs drop=3 75 | ---- 76 | dropped: 1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1_12"] 77 | dropped: 1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_1_13"] 78 | dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:11 Entries:[1/14 EntryNormal "prop_1_14"] 79 | 80 | 81 | # Repeat committing 3 entries. 82 | propose 1 prop_1_15 83 | ---- 84 | ok 85 | 86 | propose 1 prop_1_16 87 | ---- 88 | ok 89 | 90 | propose 1 prop_1_17 91 | ---- 92 | ok 93 | 94 | # In-flight tracking to nodes 2 and 3 is saturated, but node 3 is behind. 95 | status 1 96 | ---- 97 | 1: StateReplicate match=14 next=15 98 | 2: StateReplicate match=14 next=18 paused inflight=3[full] 99 | 3: StateReplicate match=11 next=15 paused inflight=3[full] 100 | 101 | log-level none 102 | ---- 103 | ok 104 | 105 | # Commit entries between nodes 1 and 2 again. 106 | stabilize 1 2 107 | ---- 108 | ok 109 | 110 | log-level debug 111 | ---- 112 | ok 113 | 114 | # Expect that the entries are committed and stored only on nodes 1 and 2. 115 | status 1 116 | ---- 117 | 1: StateReplicate match=17 next=18 118 | 2: StateReplicate match=17 next=18 119 | 3: StateReplicate match=11 next=15 paused inflight=3[full] 120 | 121 | # Make a heartbeat roundtrip. 122 | tick-heartbeat 1 123 | ---- 124 | ok 125 | 126 | stabilize 1 127 | ---- 128 | > 1 handling Ready 129 | Ready MustSync=false: 130 | Messages: 131 | 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:17 132 | 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 133 | 134 | stabilize 2 3 135 | ---- 136 | > 2 receiving messages 137 | 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:17 138 | > 3 receiving messages 139 | 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 140 | > 2 handling Ready 141 | Ready MustSync=false: 142 | Messages: 143 | 2->1 MsgHeartbeatResp Term:1 Log:0/0 144 | > 3 handling Ready 145 | Ready MustSync=false: 146 | Messages: 147 | 3->1 MsgHeartbeatResp Term:1 Log:0/0 148 | 149 | # After handling heartbeat responses, node 1 sends an empty MsgApp to a 150 | # throttled node 3 because it hasn't yet replied to a single MsgApp, and the 151 | # in-flight tracker is still saturated. 152 | stabilize 1 153 | ---- 154 | > 1 receiving messages 155 | 2->1 MsgHeartbeatResp Term:1 Log:0/0 156 | 3->1 MsgHeartbeatResp Term:1 Log:0/0 157 | > 1 handling Ready 158 | Ready MustSync=false: 159 | Messages: 160 | 1->3 MsgApp Term:1 Log:1/14 Commit:17 161 | 162 | # Node 3 finally receives a MsgApp, but there was a gap, so it rejects it. 163 | stabilize 3 164 | ---- 165 | > 3 receiving messages 166 | 1->3 MsgApp Term:1 Log:1/14 Commit:17 167 | DEBUG 3 [logterm: 0, index: 14] rejected MsgApp [logterm: 1, index: 14] from 1 168 | > 3 handling Ready 169 | Ready MustSync=false: 170 | Messages: 171 | 3->1 MsgAppResp Term:1 Log:1/14 Rejected (Hint: 11) 172 | 173 | log-level none 174 | ---- 175 | ok 176 | 177 | stabilize 178 | ---- 179 | ok 180 | 181 | log-level debug 182 | ---- 183 | ok 184 | 185 | # Eventually all nodes catch up on the committed state. 186 | status 1 187 | ---- 188 | 1: StateReplicate match=17 next=18 189 | 2: StateReplicate match=17 next=18 190 | 3: StateReplicate match=17 next=18 191 | -------------------------------------------------------------------------------- /testdata/single_node.txt: -------------------------------------------------------------------------------- 1 | log-level info 2 | ---- 3 | ok 4 | 5 | add-nodes 1 voters=(1) index=3 6 | ---- 7 | INFO 1 switched to configuration voters=(1) 8 | INFO 1 became follower at term 0 9 | INFO newRaft 1 [peers: [1], term: 0, commit: 3, applied: 3, lastindex: 3, lastterm: 1] 10 | 11 | campaign 1 12 | ---- 13 | INFO 1 is starting a new election at term 0 14 | INFO 1 became candidate at term 1 15 | 16 | stabilize 17 | ---- 18 | > 1 handling Ready 19 | Ready MustSync=true: 20 | Lead:0 State:StateCandidate 21 | HardState Term:1 Vote:1 Commit:3 22 | INFO 1 received MsgVoteResp from 1 at term 1 23 | INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections 24 | INFO 1 became leader at term 1 25 | > 1 handling Ready 26 | Ready MustSync=true: 27 | Lead:1 State:StateLeader 28 | Entries: 29 | 1/4 EntryNormal "" 30 | > 1 handling Ready 31 | Ready MustSync=false: 32 | HardState Term:1 Vote:1 Commit:4 33 | CommittedEntries: 34 | 1/4 EntryNormal "" 35 | -------------------------------------------------------------------------------- /testdata/slow_follower_after_compaction.txt: -------------------------------------------------------------------------------- 1 | # This is a regression test for https://github.com/etcd-io/raft/pull/31. 2 | 3 | # Turn off output during the setup of the test. 4 | log-level none 5 | ---- 6 | ok 7 | 8 | # Start with 3 nodes, with a limited in-flight capacity. 9 | add-nodes 3 voters=(1,2,3) index=10 inflight=2 10 | ---- 11 | ok 12 | 13 | campaign 1 14 | ---- 15 | ok 16 | 17 | stabilize 18 | ---- 19 | ok 20 | 21 | # Propose 3 entries. 22 | propose 1 prop_1_12 23 | ---- 24 | ok 25 | 26 | propose 1 prop_1_13 27 | ---- 28 | ok 29 | 30 | propose 1 prop_1_14 31 | ---- 32 | ok 33 | 34 | stabilize 35 | ---- 36 | ok 37 | 38 | # Re-enable log messages. 39 | log-level debug 40 | ---- 41 | ok 42 | 43 | # All nodes up-to-date. 44 | status 1 45 | ---- 46 | 1: StateReplicate match=14 next=15 47 | 2: StateReplicate match=14 next=15 48 | 3: StateReplicate match=14 next=15 49 | 50 | log-level none 51 | ---- 52 | ok 53 | 54 | propose 1 prop_1_15 55 | ---- 56 | ok 57 | 58 | propose 1 prop_1_16 59 | ---- 60 | ok 61 | 62 | propose 1 prop_1_17 63 | ---- 64 | ok 65 | 66 | propose 1 prop_1_18 67 | ---- 68 | ok 69 | 70 | # Commit entries on nodes 1 and 2. 71 | stabilize 1 2 72 | ---- 73 | ok 74 | 75 | log-level debug 76 | ---- 77 | ok 78 | 79 | # Nodes 1 and 2 up-to-date, 3 is behind and MsgApp flow is throttled. 80 | status 1 81 | ---- 82 | 1: StateReplicate match=18 next=19 83 | 2: StateReplicate match=18 next=19 84 | 3: StateReplicate match=14 next=17 paused inflight=2[full] 85 | 86 | # Break the MsgApp flow from the leader to node 3. 87 | deliver-msgs drop=3 88 | ---- 89 | dropped: 1->3 MsgApp Term:1 Log:1/14 Commit:14 Entries:[1/15 EntryNormal "prop_1_15"] 90 | dropped: 1->3 MsgApp Term:1 Log:1/15 Commit:14 Entries:[1/16 EntryNormal "prop_1_16"] 91 | 92 | # Truncate the leader's log beyond node 3 log size. 93 | compact 1 17 94 | ---- 95 | 1/18 EntryNormal "prop_1_18" 96 | 97 | # Trigger a round of empty MsgApp "probe" from leader. It will reach node 3 98 | # which will reply with a rejection MsgApp because it sees a gap in the log. 99 | # Node 1 will reset the MsgApp flow and send a snapshot to catch node 3 up. 100 | tick-heartbeat 1 101 | ---- 102 | ok 103 | 104 | log-level none 105 | ---- 106 | ok 107 | 108 | stabilize 109 | ---- 110 | ok 111 | 112 | log-level debug 113 | ---- 114 | ok 115 | 116 | # All nodes caught up. 117 | status 1 118 | ---- 119 | 1: StateReplicate match=18 next=19 120 | 2: StateReplicate match=18 next=19 121 | 3: StateReplicate match=18 next=19 122 | -------------------------------------------------------------------------------- /tla/MCetcdraft.cfg: -------------------------------------------------------------------------------- 1 | \* Copyright 2024 The etcd Authors 2 | \* 3 | \* Licensed under the Apache License, Version 2.0 (the "License"); 4 | \* you may not use this file except in compliance with the License. 5 | \* You may obtain a copy of the License at 6 | \* 7 | \* http://www.apache.org/licenses/LICENSE-2.0 8 | \* 9 | \* Unless required by applicable law or agreed to in writing, software 10 | \* distributed under the License is distributed on an "AS IS" BASIS, 11 | \* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | \* See the License for the specific language governing permissions and 13 | \* limitations under the License. 14 | \* 15 | SPECIFICATION mc_etcdSpec 16 | 17 | CONSTANTS 18 | s1 = 1 19 | s2 = 2 20 | s3 = 3 21 | s4 = 4 22 | s5 = 5 23 | 24 | InitServer = {s1, s2, s3} 25 | Server = {s1, s2, s3, s4} 26 | 27 | ReconfigurationLimit = 2 28 | MaxTermLimit = 10 29 | RequestLimit = 5 30 | 31 | Timeout <- MCTimeout 32 | Send <- MCSend 33 | ClientRequest <- MCClientRequest 34 | AddNewServer <- MCAddNewServer 35 | DeleteServer <- MCDeleteServer 36 | AddLearner <- MCAddLearner 37 | 38 | InitServerVars <- etcdInitServerVars 39 | InitLogVars <- etcdInitLogVars 40 | InitConfigVars <- etcdInitConfigVars 41 | 42 | Nil = 0 43 | 44 | ValueEntry = "ValueEntry" 45 | ConfigEntry = "ConfigEntry" 46 | 47 | Follower = "Follower" 48 | Candidate = "Candidate" 49 | Leader = "Leader" 50 | RequestVoteRequest = "RequestVoteRequest" 51 | RequestVoteResponse = "RequestVoteResponse" 52 | AppendEntriesRequest = "AppendEntriesRequest" 53 | AppendEntriesResponse = "AppendEntriesResponse" 54 | 55 | SYMMETRY Symmetry 56 | VIEW View 57 | 58 | CHECK_DEADLOCK 59 | FALSE 60 | 61 | INVARIANTS 62 | LogInv 63 | MoreThanOneLeaderInv 64 | ElectionSafetyInv 65 | LogMatchingInv 66 | QuorumLogInv 67 | MoreUpToDateCorrectInv 68 | LeaderCompletenessInv 69 | CommittedIsDurableInv -------------------------------------------------------------------------------- /tla/Traceetcdraft.cfg: -------------------------------------------------------------------------------- 1 | \* Copyright 2024 The etcd Authors 2 | \* 3 | \* Licensed under the Apache License, Version 2.0 (the "License"); 4 | \* you may not use this file except in compliance with the License. 5 | \* You may obtain a copy of the License at 6 | \* 7 | \* http://www.apache.org/licenses/LICENSE-2.0 8 | \* 9 | \* Unless required by applicable law or agreed to in writing, software 10 | \* distributed under the License is distributed on an "AS IS" BASIS, 11 | \* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | \* See the License for the specific language governing permissions and 13 | \* limitations under the License. 14 | \* 15 | 16 | \* A trace specification defines a set of one or more traces, wherein the value of some 17 | \* variables in each state aligns with the values specified on the corresponding line 18 | \* of the log or trace file. Commonly, the log file might be incomplete, failing to 19 | \* provide the value of all variables. In such instances, the omitted values are 20 | \* determined non-deterministically, adhering to the high-level specification's parameters. 21 | \* Furthermore, a trace may not be complete such that it only matches a prefix of the log. 22 | SPECIFICATION 23 | TraceSpec 24 | 25 | VIEW 26 | TraceView 27 | 28 | \* TLA has only limited support for hyperproperties. The following property is a poorman's 29 | \* hyperproperty that asserts that TLC generated *at least one* trace that fully matches the 30 | \* log file. 31 | PROPERTIES 32 | TraceMatched 33 | 34 | PROPERTIES 35 | etcdSpec 36 | 37 | \* Checking for deadlocks during trace validation is disabled, as it may lead to false 38 | \* counterexamples. A trace specification defines a set of traces, where at least one 39 | \* trace is expected to match the log file in terms of variable values and length. 40 | \* However, partial matches may occur where the trace cannot be extended to fully 41 | \* correspond with the log file. In such cases, deadlock checking would report the first 42 | \* of these traces. 43 | CHECK_DEADLOCK 44 | FALSE 45 | 46 | CONSTANTS 47 | Nil = "0" 48 | 49 | ValueEntry = "ValueEntry" 50 | ConfigEntry = "ConfigEntry" 51 | 52 | Follower = "StateFollower" 53 | Candidate = "StateCandidate" 54 | Leader = "StateLeader" 55 | RequestVoteRequest = "RequestVoteRequest" 56 | RequestVoteResponse = "RequestVoteResponse" 57 | AppendEntriesRequest = "AppendEntriesRequest" 58 | AppendEntriesResponse = "AppendEntriesResponse" 59 | 60 | InitServerVars <- TraceInitServerVars 61 | InitLogVars <- TraceInitLogVars 62 | InitConfigVars <- TraceInitConfigVars 63 | 64 | InitServer <- TraceInitServer 65 | Server <- TraceServer 66 | 67 | INVARIANTS 68 | LogInv 69 | MoreThanOneLeaderInv 70 | ElectionSafetyInv 71 | LogMatchingInv 72 | QuorumLogInv 73 | MoreUpToDateCorrectInv 74 | LeaderCompletenessInv 75 | CommittedIsDurableInv -------------------------------------------------------------------------------- /tla/etcdraft.cfg: -------------------------------------------------------------------------------- 1 | \* Copyright 2024 The etcd Authors 2 | \* 3 | \* Licensed under the Apache License, Version 2.0 (the "License"); 4 | \* you may not use this file except in compliance with the License. 5 | \* You may obtain a copy of the License at 6 | \* 7 | \* http://www.apache.org/licenses/LICENSE-2.0 8 | \* 9 | \* Unless required by applicable law or agreed to in writing, software 10 | \* distributed under the License is distributed on an "AS IS" BASIS, 11 | \* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | \* See the License for the specific language governing permissions and 13 | \* limitations under the License. 14 | \* 15 | 16 | CONSTANTS 17 | s1 = 1 18 | s2 = 2 19 | s3 = 3 20 | s4 = 4 21 | s5 = 5 22 | 23 | InitServer = {s1, s2, s3} 24 | Server = {s1, s2, s3} 25 | 26 | Nil = 0 27 | 28 | ValueEntry = "ValueEntry" 29 | ConfigEntry = "ConfigEntry" 30 | 31 | Follower = "Follower" 32 | Candidate = "Candidate" 33 | Leader = "Leader" 34 | RequestVoteRequest = "RequestVoteRequest" 35 | RequestVoteResponse = "RequestVoteResponse" 36 | AppendEntriesRequest = "AppendEntriesRequest" 37 | AppendEntriesResponse = "AppendEntriesResponse" 38 | 39 | 40 | VIEW vars 41 | 42 | INIT Init 43 | NEXT NextAsyncCrash 44 | 45 | \* Raft properties 46 | INVARIANTS 47 | LogInv 48 | MoreThanOneLeaderInv 49 | ElectionSafetyInv 50 | LogMatchingInv 51 | QuorumLogInv 52 | MoreUpToDateCorrectInv 53 | LeaderCompletenessInv 54 | -------------------------------------------------------------------------------- /tla/validate-model.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | WORKDIR="$(mktemp -d)" 4 | TOOLDIR="${WORKDIR}/tool" 5 | STATEDIR="${WORKDIR}/state" 6 | FAILFAST=false 7 | PARALLEL=$(nproc) 8 | 9 | function show_usage { 10 | echo "usage: validate-model.sh -s -c ">&2 11 | } 12 | 13 | function install_tlaplus { 14 | echo -n "Downloading TLA+ tools ... " 15 | wget -qN https://nightly.tlapl.us/dist/tla2tools.jar -P ${TOOLDIR} 16 | wget -qN https://github.com/tlaplus/CommunityModules/releases/latest/download/CommunityModules-deps.jar -P ${TOOLDIR} 17 | echo "done." 18 | } 19 | 20 | function validate { 21 | local spec=${1} 22 | local config=${2} 23 | local tooldir=${3} 24 | local statedir=${4} 25 | 26 | set -o pipefail 27 | java -XX:+UseParallelGC -cp ${tooldir}/tla2tools.jar:${tooldir}/CommunityModules-deps.jar tlc2.TLC -config "${config}" "${spec}" -lncheck final -metadir "${statedir}" -fpmem 0.9 28 | } 29 | 30 | while getopts :hs:c:p: flag 31 | do 32 | case "${flag}" in 33 | s) SPEC=${OPTARG};; 34 | c) CONFIG=${OPTARG};; 35 | h|*) show_usage; exit 1;; 36 | esac 37 | done 38 | 39 | if [ ! "$SPEC" ] || [ ! "$CONFIG" ] 40 | then 41 | show_usage 42 | exit 1 43 | fi 44 | 45 | echo "spec: ${SPEC}" 46 | echo "config: ${CONFIG}" 47 | 48 | install_tlaplus 49 | 50 | validate $SPEC $CONFIG $TOOLDIR $STATEDIR 51 | 52 | -------------------------------------------------------------------------------- /tools/mod/go.mod: -------------------------------------------------------------------------------- 1 | module go.etcd.io/raft/tools/v3 2 | 3 | go 1.24 4 | 5 | toolchain go1.24.3 6 | 7 | require ( 8 | github.com/alexkohler/nakedret v1.0.0 9 | github.com/chzchzchz/goword v0.0.0-20170907005317-a9744cb52b03 10 | github.com/coreos/license-bill-of-materials v0.0.0-20190913234955-13baff47494e 11 | github.com/gogo/protobuf v1.3.2 12 | github.com/google/addlicense v1.0.0 13 | github.com/gordonklaus/ineffassign v0.0.0-20210914165742-4cc7213b9bc8 14 | github.com/gyuho/gocovmerge v0.0.0-20171205171859-50c7e6afd535 15 | github.com/hexfusion/schwag v0.0.0-20211117114134-3ceb0191ccbf 16 | github.com/mdempsky/unconvert v0.0.0-20200228143138-95ecdbfc0b5f 17 | github.com/mgechev/revive v1.2.1 18 | github.com/mikefarah/yq/v4 v4.24.2 19 | go.etcd.io/gofail v0.0.0-20221125214112-fc21f61ba88a 20 | go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116 21 | gotest.tools/gotestsum v1.7.0 22 | gotest.tools/v3 v3.1.0 23 | honnef.co/go/tools v0.3.0 24 | mvdan.cc/unparam v0.0.0-20220316160445-06cc5682983b 25 | ) 26 | 27 | require ( 28 | github.com/BurntSushi/toml v1.1.0 // indirect 29 | github.com/PuerkitoBio/purell v1.1.1 // indirect 30 | github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect 31 | github.com/a8m/envsubst v1.3.0 // indirect 32 | github.com/akhenakh/hunspellgo v0.0.0-20160221122622-9db38fa26e19 // indirect 33 | github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect 34 | github.com/bmatcuk/doublestar/v4 v4.0.2 // indirect 35 | github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4 // indirect 36 | github.com/dnephin/pflag v1.0.7 // indirect 37 | github.com/elliotchance/orderedmap v1.4.0 // indirect 38 | github.com/fatih/color v1.13.0 // indirect 39 | github.com/fatih/structtag v1.2.0 // indirect 40 | github.com/fsnotify/fsnotify v1.4.9 // indirect 41 | github.com/go-openapi/analysis v0.21.2 // indirect 42 | github.com/go-openapi/errors v0.19.9 // indirect 43 | github.com/go-openapi/jsonpointer v0.19.5 // indirect 44 | github.com/go-openapi/jsonreference v0.19.6 // indirect 45 | github.com/go-openapi/loads v0.21.1 // indirect 46 | github.com/go-openapi/spec v0.20.4 // indirect 47 | github.com/go-openapi/strfmt v0.21.0 // indirect 48 | github.com/go-openapi/swag v0.19.15 // indirect 49 | github.com/go-stack/stack v1.8.0 // indirect 50 | github.com/goccy/go-yaml v1.9.5 // indirect 51 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect 52 | github.com/google/uuid v1.1.2 // indirect 53 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 54 | github.com/jinzhu/copier v0.3.5 // indirect 55 | github.com/jonboulle/clockwork v0.2.2 // indirect 56 | github.com/josharian/intern v1.0.0 // indirect 57 | github.com/magiconair/properties v1.8.6 // indirect 58 | github.com/mailru/easyjson v0.7.6 // indirect 59 | github.com/mattn/go-colorable v0.1.12 // indirect 60 | github.com/mattn/go-isatty v0.0.14 // indirect 61 | github.com/mattn/go-runewidth v0.0.9 // indirect 62 | github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517 // indirect 63 | github.com/mitchellh/go-homedir v1.1.0 // indirect 64 | github.com/mitchellh/mapstructure v1.4.1 // indirect 65 | github.com/oklog/ulid v1.3.1 // indirect 66 | github.com/olekukonko/tablewriter v0.0.5 // indirect 67 | github.com/pkg/errors v0.9.1 // indirect 68 | github.com/spf13/cobra v1.4.0 // indirect 69 | github.com/spf13/pflag v1.0.5 // indirect 70 | github.com/timtadh/data-structures v0.5.3 // indirect 71 | github.com/timtadh/lexmachine v0.2.2 // indirect 72 | github.com/trustmaster/go-aspell v0.0.0-20200701131845-c2b1f55bec8f // indirect 73 | go.mongodb.org/mongo-driver v1.7.3 // indirect 74 | golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e // indirect 75 | golang.org/x/mod v0.17.0 // indirect 76 | golang.org/x/net v0.38.0 // indirect 77 | golang.org/x/sync v0.12.0 // indirect 78 | golang.org/x/sys v0.31.0 // indirect 79 | golang.org/x/term v0.30.0 // indirect 80 | golang.org/x/text v0.23.0 // indirect 81 | golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect 82 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 83 | gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 // indirect 84 | gopkg.in/yaml.v2 v2.4.0 // indirect 85 | gopkg.in/yaml.v3 v3.0.1 // indirect 86 | ) 87 | -------------------------------------------------------------------------------- /tools/mod/install_all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | cd ./tools/mod || exit 2 4 | go list --tags tools -f '{{ join .Imports "\n" }}' | xargs go install 5 | -------------------------------------------------------------------------------- /tools/mod/libs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build libs 16 | // +build libs 17 | 18 | // This file implements that pattern: 19 | // https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module 20 | // for etcd. Thanks to this file 'go mod tidy' does not removes dependencies. 21 | 22 | package libs 23 | 24 | import ( 25 | _ "github.com/gogo/protobuf/proto" 26 | ) 27 | -------------------------------------------------------------------------------- /tools/mod/tools.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build tools 16 | // +build tools 17 | 18 | // This file implements that pattern: 19 | // https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module 20 | // for raft. Thanks to this file 'go mod tidy' does not remove dependencies. 21 | 22 | package tools 23 | 24 | import ( 25 | _ "github.com/alexkohler/nakedret" 26 | _ "github.com/chzchzchz/goword" 27 | _ "github.com/coreos/license-bill-of-materials" 28 | _ "github.com/google/addlicense" 29 | _ "github.com/gordonklaus/ineffassign" 30 | _ "github.com/gyuho/gocovmerge" 31 | _ "github.com/hexfusion/schwag" 32 | _ "github.com/mdempsky/unconvert" 33 | _ "github.com/mgechev/revive" 34 | _ "github.com/mikefarah/yq/v4" 35 | _ "go.etcd.io/gofail" 36 | _ "go.etcd.io/protodoc" 37 | _ "gotest.tools/gotestsum" 38 | _ "gotest.tools/v3" 39 | _ "honnef.co/go/tools/cmd/staticcheck" 40 | _ "mvdan.cc/unparam" 41 | ) 42 | -------------------------------------------------------------------------------- /tracker/state.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tracker 16 | 17 | // StateType is the state of a tracked follower. 18 | type StateType uint64 19 | 20 | const ( 21 | // StateProbe indicates a follower whose last index isn't known. Such a 22 | // follower is "probed" (i.e. an append sent periodically) to narrow down 23 | // its last index. In the ideal (and common) case, only one round of probing 24 | // is necessary as the follower will react with a hint. Followers that are 25 | // probed over extended periods of time are often offline. 26 | StateProbe StateType = iota 27 | // StateReplicate is the state steady in which a follower eagerly receives 28 | // log entries to append to its log. 29 | StateReplicate 30 | // StateSnapshot indicates a follower that needs log entries not available 31 | // from the leader's Raft log. Such a follower needs a full snapshot to 32 | // return to StateReplicate. 33 | StateSnapshot 34 | ) 35 | 36 | var prstmap = [...]string{ 37 | "StateProbe", 38 | "StateReplicate", 39 | "StateSnapshot", 40 | } 41 | 42 | func (st StateType) String() string { return prstmap[st] } 43 | -------------------------------------------------------------------------------- /types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raft 16 | 17 | import ( 18 | "fmt" 19 | 20 | pb "go.etcd.io/raft/v3/raftpb" 21 | ) 22 | 23 | // entryID uniquely identifies a raft log entry. 24 | // 25 | // Every entry is associated with a leadership term which issued this entry and 26 | // initially appended it to the log. There can only be one leader at any term, 27 | // and a leader never issues two entries with the same index. 28 | type entryID struct { 29 | term uint64 30 | index uint64 31 | } 32 | 33 | // pbEntryID returns the ID of the given pb.Entry. 34 | func pbEntryID(entry *pb.Entry) entryID { 35 | return entryID{term: entry.Term, index: entry.Index} 36 | } 37 | 38 | // logSlice describes a correct slice of a raft log. 39 | // 40 | // Every log slice is considered in a context of a specific leader term. This 41 | // term does not necessarily match entryID.term of the entries, since a leader 42 | // log contains both entries from its own term, and some earlier terms. 43 | // 44 | // Two slices with a matching logSlice.term are guaranteed to be consistent, 45 | // i.e. they never contain two different entries at the same index. The reverse 46 | // is not true: two slices with different logSlice.term may contain both 47 | // matching and mismatching entries. Specifically, logs at two different leader 48 | // terms share a common prefix, after which they *permanently* diverge. 49 | // 50 | // A well-formed logSlice conforms to raft safety properties. It provides the 51 | // following guarantees: 52 | // 53 | // 1. entries[i].Index == prev.index + 1 + i, 54 | // 2. prev.term <= entries[0].Term, 55 | // 3. entries[i-1].Term <= entries[i].Term, 56 | // 4. entries[len-1].Term <= term. 57 | // 58 | // Property (1) means the slice is contiguous. Properties (2) and (3) mean that 59 | // the terms of the entries in a log never regress. Property (4) means that a 60 | // leader log at a specific term never has entries from higher terms. 61 | // 62 | // Users of this struct can assume the invariants hold true. Exception is the 63 | // "gateway" code that initially constructs logSlice, such as when its content 64 | // is sourced from a message that was received via transport, or from Storage, 65 | // or in a test code that manually hard-codes this struct. In these cases, the 66 | // invariants should be validated using the valid() method. 67 | type logSlice struct { 68 | // term is the leader term containing the given entries in its log. 69 | term uint64 70 | // prev is the ID of the entry immediately preceding the entries. 71 | prev entryID 72 | // entries contains the consecutive entries representing this slice. 73 | entries []pb.Entry 74 | } 75 | 76 | // lastIndex returns the index of the last entry in this log slice. Returns 77 | // prev.index if there are no entries. 78 | func (s logSlice) lastIndex() uint64 { 79 | return s.prev.index + uint64(len(s.entries)) 80 | } 81 | 82 | // lastEntryID returns the ID of the last entry in this log slice, or prev if 83 | // there are no entries. 84 | func (s logSlice) lastEntryID() entryID { 85 | if ln := len(s.entries); ln != 0 { 86 | return pbEntryID(&s.entries[ln-1]) 87 | } 88 | return s.prev 89 | } 90 | 91 | // valid returns nil iff the logSlice is a well-formed log slice. See logSlice 92 | // comment for details on what constitutes a valid raft log slice. 93 | func (s logSlice) valid() error { 94 | prev := s.prev 95 | for i := range s.entries { 96 | id := pbEntryID(&s.entries[i]) 97 | if id.term < prev.term || id.index != prev.index+1 { 98 | return fmt.Errorf("leader term %d: entries %+v and %+v not consistent", s.term, prev, id) 99 | } 100 | prev = id 101 | } 102 | if s.term < prev.term { 103 | return fmt.Errorf("leader term %d: entry %+v has a newer term", s.term, prev) 104 | } 105 | return nil 106 | } 107 | -------------------------------------------------------------------------------- /types_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raft 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/stretchr/testify/require" 21 | 22 | pb "go.etcd.io/raft/v3/raftpb" 23 | ) 24 | 25 | func TestEntryID(t *testing.T) { 26 | // Some obvious checks first. 27 | require.Equal(t, entryID{term: 5, index: 10}, entryID{term: 5, index: 10}) 28 | require.NotEqual(t, entryID{term: 4, index: 10}, entryID{term: 5, index: 10}) 29 | require.NotEqual(t, entryID{term: 5, index: 9}, entryID{term: 5, index: 10}) 30 | 31 | for _, tt := range []struct { 32 | entry pb.Entry 33 | want entryID 34 | }{ 35 | {entry: pb.Entry{}, want: entryID{term: 0, index: 0}}, 36 | {entry: pb.Entry{Term: 1, Index: 2, Data: []byte("data")}, want: entryID{term: 1, index: 2}}, 37 | {entry: pb.Entry{Term: 10, Index: 123}, want: entryID{term: 10, index: 123}}, 38 | } { 39 | require.Equal(t, tt.want, pbEntryID(&tt.entry)) 40 | } 41 | } 42 | 43 | func TestLogSlice(t *testing.T) { 44 | id := func(index, term uint64) entryID { 45 | return entryID{term: term, index: index} 46 | } 47 | e := func(index, term uint64) pb.Entry { 48 | return pb.Entry{Term: term, Index: index} 49 | } 50 | for _, tt := range []struct { 51 | term uint64 52 | prev entryID 53 | entries []pb.Entry 54 | 55 | notOk bool 56 | last entryID 57 | }{ 58 | // Empty "dummy" slice, starting at (0, 0) origin of the log. 59 | {last: id(0, 0)}, 60 | // Empty slice with a given prev ID. Valid only if term >= prev.term. 61 | {prev: id(123, 10), notOk: true}, 62 | {term: 9, prev: id(123, 10), notOk: true}, 63 | {term: 10, prev: id(123, 10), last: id(123, 10)}, 64 | {term: 11, prev: id(123, 10), last: id(123, 10)}, 65 | // A single entry. 66 | {term: 0, entries: []pb.Entry{e(1, 1)}, notOk: true}, 67 | {term: 1, entries: []pb.Entry{e(1, 1)}, last: id(1, 1)}, 68 | {term: 2, entries: []pb.Entry{e(1, 1)}, last: id(1, 1)}, 69 | // Multiple entries. 70 | {term: 2, entries: []pb.Entry{e(2, 1), e(3, 1), e(4, 2)}, notOk: true}, 71 | {term: 1, prev: id(1, 1), entries: []pb.Entry{e(2, 1), e(3, 1), e(4, 2)}, notOk: true}, 72 | {term: 2, prev: id(1, 1), entries: []pb.Entry{e(2, 1), e(3, 1), e(4, 2)}, last: id(4, 2)}, 73 | // First entry inconsistent with prev. 74 | {term: 10, prev: id(123, 5), entries: []pb.Entry{e(111, 5)}, notOk: true}, 75 | {term: 10, prev: id(123, 5), entries: []pb.Entry{e(124, 4)}, notOk: true}, 76 | {term: 10, prev: id(123, 5), entries: []pb.Entry{e(234, 6)}, notOk: true}, 77 | {term: 10, prev: id(123, 5), entries: []pb.Entry{e(124, 6)}, last: id(124, 6)}, 78 | // Inconsistent entries. 79 | {term: 10, prev: id(12, 2), entries: []pb.Entry{e(13, 2), e(12, 2)}, notOk: true}, 80 | {term: 10, prev: id(12, 2), entries: []pb.Entry{e(13, 2), e(15, 2)}, notOk: true}, 81 | {term: 10, prev: id(12, 2), entries: []pb.Entry{e(13, 2), e(14, 1)}, notOk: true}, 82 | {term: 10, prev: id(12, 2), entries: []pb.Entry{e(13, 2), e(14, 3)}, last: id(14, 3)}, 83 | } { 84 | t.Run("", func(t *testing.T) { 85 | s := logSlice{term: tt.term, prev: tt.prev, entries: tt.entries} 86 | require.Equal(t, tt.notOk, s.valid() != nil) 87 | if !tt.notOk { 88 | last := s.lastEntryID() 89 | require.Equal(t, tt.last, last) 90 | require.Equal(t, last.index, s.lastIndex()) 91 | } 92 | }) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The etcd Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package raft 16 | 17 | import ( 18 | "fmt" 19 | "math" 20 | "strings" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | "github.com/stretchr/testify/require" 25 | 26 | pb "go.etcd.io/raft/v3/raftpb" 27 | ) 28 | 29 | var testFormatter EntryFormatter = func(data []byte) string { 30 | return strings.ToUpper(string(data)) 31 | } 32 | 33 | func TestDescribeEntry(t *testing.T) { 34 | entry := pb.Entry{ 35 | Term: 1, 36 | Index: 2, 37 | Type: pb.EntryNormal, 38 | Data: []byte("hello\x00world"), 39 | } 40 | require.Equal(t, `1/2 EntryNormal "hello\x00world"`, DescribeEntry(entry, nil)) 41 | require.Equal(t, "1/2 EntryNormal HELLO\x00WORLD", DescribeEntry(entry, testFormatter)) 42 | } 43 | 44 | func TestLimitSize(t *testing.T) { 45 | ents := []pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 5}, {Index: 6, Term: 6}} 46 | prefix := func(size int) []pb.Entry { 47 | return append([]pb.Entry{}, ents[:size]...) // protect the original slice 48 | } 49 | for _, tt := range []struct { 50 | maxSize uint64 51 | want []pb.Entry 52 | }{ 53 | {math.MaxUint64, prefix(len(ents))}, // all entries are returned 54 | // Even if maxSize is zero, the first entry should be returned. 55 | {0, prefix(1)}, 56 | // Limit to 2. 57 | {uint64(ents[0].Size() + ents[1].Size()), prefix(2)}, 58 | {uint64(ents[0].Size() + ents[1].Size() + ents[2].Size()/2), prefix(2)}, 59 | {uint64(ents[0].Size() + ents[1].Size() + ents[2].Size() - 1), prefix(2)}, 60 | // All. 61 | {uint64(ents[0].Size() + ents[1].Size() + ents[2].Size()), prefix(3)}, 62 | } { 63 | t.Run("", func(t *testing.T) { 64 | got := limitSize(ents, entryEncodingSize(tt.maxSize)) 65 | require.Equal(t, tt.want, got) 66 | size := entsSize(got) 67 | require.True(t, len(got) == 1 || size <= entryEncodingSize(tt.maxSize)) 68 | }) 69 | } 70 | } 71 | 72 | func TestIsLocalMsg(t *testing.T) { 73 | tests := []struct { 74 | msgt pb.MessageType 75 | isLocal bool 76 | }{ 77 | {pb.MsgHup, true}, 78 | {pb.MsgBeat, true}, 79 | {pb.MsgUnreachable, true}, 80 | {pb.MsgSnapStatus, true}, 81 | {pb.MsgCheckQuorum, true}, 82 | {pb.MsgTransferLeader, false}, 83 | {pb.MsgProp, false}, 84 | {pb.MsgApp, false}, 85 | {pb.MsgAppResp, false}, 86 | {pb.MsgVote, false}, 87 | {pb.MsgVoteResp, false}, 88 | {pb.MsgSnap, false}, 89 | {pb.MsgHeartbeat, false}, 90 | {pb.MsgHeartbeatResp, false}, 91 | {pb.MsgTimeoutNow, false}, 92 | {pb.MsgReadIndex, false}, 93 | {pb.MsgReadIndexResp, false}, 94 | {pb.MsgPreVote, false}, 95 | {pb.MsgPreVoteResp, false}, 96 | {pb.MsgStorageAppend, true}, 97 | {pb.MsgStorageAppendResp, true}, 98 | {pb.MsgStorageApply, true}, 99 | {pb.MsgStorageApplyResp, true}, 100 | } 101 | 102 | for _, tt := range tests { 103 | t.Run(fmt.Sprint(tt.msgt), func(t *testing.T) { 104 | require.Equal(t, tt.isLocal, IsLocalMsg(tt.msgt)) 105 | }) 106 | } 107 | } 108 | 109 | func TestIsResponseMsg(t *testing.T) { 110 | tests := []struct { 111 | msgt pb.MessageType 112 | isResponse bool 113 | }{ 114 | {pb.MsgHup, false}, 115 | {pb.MsgBeat, false}, 116 | {pb.MsgUnreachable, true}, 117 | {pb.MsgSnapStatus, false}, 118 | {pb.MsgCheckQuorum, false}, 119 | {pb.MsgTransferLeader, false}, 120 | {pb.MsgProp, false}, 121 | {pb.MsgApp, false}, 122 | {pb.MsgAppResp, true}, 123 | {pb.MsgVote, false}, 124 | {pb.MsgVoteResp, true}, 125 | {pb.MsgSnap, false}, 126 | {pb.MsgHeartbeat, false}, 127 | {pb.MsgHeartbeatResp, true}, 128 | {pb.MsgTimeoutNow, false}, 129 | {pb.MsgReadIndex, false}, 130 | {pb.MsgReadIndexResp, true}, 131 | {pb.MsgPreVote, false}, 132 | {pb.MsgPreVoteResp, true}, 133 | {pb.MsgStorageAppend, false}, 134 | {pb.MsgStorageAppendResp, true}, 135 | {pb.MsgStorageApply, false}, 136 | {pb.MsgStorageApplyResp, true}, 137 | } 138 | 139 | for i, tt := range tests { 140 | got := IsResponseMsg(tt.msgt) 141 | assert.Equal(t, tt.isResponse, got, "#%d", i) 142 | } 143 | } 144 | 145 | // TestPayloadSizeOfEmptyEntry ensures that payloadSize of empty entry is always zero. 146 | // This property is important because new leaders append an empty entry to their log, 147 | // and we don't want this to count towards the uncommitted log quota. 148 | func TestPayloadSizeOfEmptyEntry(t *testing.T) { 149 | e := pb.Entry{Data: nil} 150 | require.Equal(t, 0, int(payloadSize(e))) 151 | } 152 | --------------------------------------------------------------------------------