├── .github ├── CODEOWNERS └── workflows │ └── check.yml ├── .gitignore ├── .go-version ├── .golangci.yml ├── LICENSE ├── Makefile ├── README.md ├── alive_delegate.go ├── awareness.go ├── awareness_test.go ├── broadcast.go ├── broadcast_test.go ├── config.go ├── config_test.go ├── conflict_delegate.go ├── delegate.go ├── event_delegate.go ├── go.mod ├── go.sum ├── integ_test.go ├── internal └── retry │ ├── retry.go │ └── retry_test.go ├── keyring.go ├── keyring_test.go ├── label.go ├── label_test.go ├── logging.go ├── logging_test.go ├── memberlist.go ├── memberlist_test.go ├── merge_delegate.go ├── mock_transport.go ├── net.go ├── net_test.go ├── net_transport.go ├── peeked_conn.go ├── ping_delegate.go ├── queue.go ├── queue_test.go ├── security.go ├── security_test.go ├── state.go ├── state_test.go ├── suspicion.go ├── suspicion_test.go ├── tag.sh ├── test └── setup_subnet.sh ├── todo.md ├── transport.go ├── transport_test.go ├── util.go └── util_test.go /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Each line is a file pattern followed by one or more owners. 2 | # More on CODEOWNERS files: https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners 3 | 4 | # Default owner 5 | * @hashicorp/team-ip-compliance 6 | 7 | # Add override rules below. Each line is a file/folder pattern followed by one or more owners. 8 | # Being an owner means those groups or individuals will be added as reviewers to PRs affecting 9 | # those areas of the code. 10 | # Examples: 11 | # /docs/ @docs-team 12 | # *.js @js-team 13 | # *.go @go-team -------------------------------------------------------------------------------- /.github/workflows/check.yml: -------------------------------------------------------------------------------- 1 | name: Checks 2 | 3 | on: 4 | pull_request: 5 | branches: [master] 6 | 7 | # This workflow runs for not-yet-reviewed external contributions and so it 8 | # intentionally has no write access and only limited read access to the 9 | # repository. 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | lint: 15 | name: "Run golangci-lint" 16 | runs-on: ubuntu-latest 17 | strategy: 18 | matrix: 19 | GO_VERSION: [ "1.20","1.21" ] 20 | steps: 21 | - name: "Fetch source code" 22 | uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 23 | 24 | - name: Install Go toolchain 25 | uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 26 | with: 27 | go-version: ${{ matrix.GO_VERSION }} 28 | - name: Cache Go modules 29 | uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 30 | with: 31 | path: | 32 | ~/.cache/go-build 33 | ~/go/pkg/mod 34 | key: go-mod-${{ matrix.GO_VERSION }}-${{ hashFiles('go.sum') }} 35 | restore-keys: | 36 | go-mod-${{ matrix.GO_VERSION }} 37 | - name: "Download golang-lint" 38 | run: | 39 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin 40 | golangci-lint --version 41 | - name: "lint" 42 | run: | 43 | golangci-lint run -v 44 | unit-tests: 45 | name: "Unit Tests" 46 | runs-on: ubuntu-latest 47 | strategy: 48 | matrix: 49 | GO_VERSION: [ "1.20","1.21" ] 50 | steps: 51 | - name: "Fetch source code" 52 | uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 53 | 54 | - name: Install Go toolchain 55 | uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 56 | with: 57 | go-version: ${{ matrix.GO_VERSION }} 58 | 59 | # NOTE: This cache is shared so the following step must always be 60 | # identical across the unit-tests, e2e-tests, and consistency-checks 61 | # jobs, or else weird things could happen. 62 | - name: Cache Go modules 63 | uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 64 | with: 65 | path: | 66 | ~/.cache/go-build 67 | ~/go/pkg/mod 68 | key: go-mod-${{ matrix.GO_VERSION }}-${{ hashFiles('go.sum') }} 69 | restore-keys: | 70 | go-mod-${{ matrix.GO_VERSION }} 71 | - name: "Unit tests and generate coverage report" 72 | run: | 73 | go test ./... 74 | make cov 75 | - name: Upload coveragee report 76 | uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 77 | with: 78 | path: coverage.out 79 | name: Coverage-report-${{matrix.GO_VERSION}} 80 | - name: Display Coverage report 81 | run: go tool cover -func=coverage.out 82 | - name: Build go 83 | run: go build ./... 84 | 85 | unit-tests-race: 86 | name: "Unit Tests Race" 87 | runs-on: ubuntu-latest 88 | strategy: 89 | matrix: 90 | GO_VERSION: [ "1.20","1.21" ] 91 | steps: 92 | - name: "Fetch source code" 93 | uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 94 | 95 | - name: Install Go toolchain 96 | uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 97 | with: 98 | go-version: ${{ matrix.GO_VERSION }} 99 | 100 | # NOTE: This cache is shared so the following step must always be 101 | # identical across the unit-tests, e2e-tests, and consistency-checks 102 | # jobs, or else weird things could happen. 103 | - name: Cache Go modules 104 | uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 105 | with: 106 | path: | 107 | ~/.cache/go-build 108 | ~/go/pkg/mod 109 | key: go-mod-${{ matrix.GO_VERSION }}-${{ hashFiles('go.sum') }} 110 | restore-keys: | 111 | go-mod-${{ matrix.GO_VERSION }} 112 | - name: "Race Unit tests" 113 | run: | 114 | go test -race ./... 115 | - name: Race Build 116 | run: go build -race ./... 117 | 118 | consistency-checks: 119 | name: "Code Consistency Checks" 120 | runs-on: ubuntu-latest 121 | strategy: 122 | matrix: 123 | GO_VERSION: [ "1.20","1.21" ] 124 | steps: 125 | - name: "Fetch source code" 126 | uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 127 | 128 | - name: Install Go toolchain 129 | uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 130 | with: 131 | go-version: ${{ matrix.GO_VERSION }} 132 | 133 | # NOTE: This cache is shared so the following step must always be 134 | # identical across the unit-tests and consistency-checks 135 | # jobs, or else weird things could happen. 136 | - name: Cache Go modules 137 | uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 138 | with: 139 | path: | 140 | ~/.cache/go-build 141 | ~/go/pkg/mod 142 | key: go-mod-${{ matrix.GO_VERSION }}-${{ hashFiles('go.sum') }} 143 | restore-keys: | 144 | go-mod-${{ matrix.GO_VERSION }} 145 | - name: "go.mod and go.sum consistency check" 146 | run: | 147 | go mod tidy 148 | if [[ -n "$(git status --porcelain)" ]]; then 149 | echo >&2 "ERROR: go.mod/go.sum are not up-to-date. Run 'go mod tidy' and then commit the updated files." 150 | exit 1 151 | fi 152 | - name: "go vet" 153 | run: | 154 | go vet ./... 155 | - name: "go fmt check" 156 | run: | 157 | files=$(go fmt ./...) 158 | if [ -n "$files" ]; then 159 | echo "The following file(s) do not conform to go fmt:" 160 | echo "$files" 161 | exit 1 162 | fi -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | .vagrant/ 25 | 26 | -------------------------------------------------------------------------------- /.go-version: -------------------------------------------------------------------------------- 1 | 1.20 2 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) HashiCorp, Inc. 2 | # SPDX-License-Identifier: MPL-2.0 3 | 4 | linters: 5 | disable-all: true 6 | enable: 7 | # (TODO) uncomment after fixing the lint issues 8 | # - gofmt 9 | - govet 10 | # - unconvert 11 | # - staticcheck 12 | # - ineffassign 13 | # - unparam 14 | - forbidigo 15 | 16 | issues: 17 | # Disable the default exclude list so that all excludes are explicitly 18 | # defined in this file. 19 | exclude-use-default: false 20 | 21 | exclude-rules: 22 | # Temp Ignore SA9004: only the first constant in this group has an explicit type 23 | # https://staticcheck.io/docs/checks#SA9004 24 | - linters: [staticcheck] 25 | text: 'SA9004:' 26 | 27 | - linters: [staticcheck] 28 | text: 'SA1019: Package github.com/golang/protobuf/jsonpb is deprecated' 29 | 30 | - linters: [staticcheck] 31 | text: 'SA1019: Package github.com/golang/protobuf/proto is deprecated' 32 | 33 | - linters: [staticcheck] 34 | text: 'SA1019: ptypes.MarshalAny is deprecated' 35 | 36 | - linters: [staticcheck] 37 | text: 'SA1019: ptypes.UnmarshalAny is deprecated' 38 | 39 | - linters: [staticcheck] 40 | text: 'SA1019: package github.com/golang/protobuf/ptypes is deprecated' 41 | 42 | # An argument that always receives the same value is often not a problem. 43 | - linters: [unparam] 44 | text: 'always receives' 45 | 46 | # Often functions will implement an interface that returns an error without 47 | # needing to return an error. Sometimes the error return value is unnecessary 48 | # but a linter can not tell the difference. 49 | - linters: [unparam] 50 | text: 'result \d+ \(error\) is always nil' 51 | 52 | # Allow unused parameters to start with an underscore. Arguments with a name 53 | # of '_' are already ignored. 54 | # Ignoring longer names that start with underscore allow for better 55 | # self-documentation than a single underscore by itself. Underscore arguments 56 | # should generally only be used when a function is implementing an interface. 57 | - linters: [unparam] 58 | text: '`_[^`]*` is unused' 59 | 60 | # Temp ignore some common unused parameters so that unparam can be added 61 | # incrementally. 62 | - linters: [unparam] 63 | text: '`(t|resp|req|entMeta)` is unused' 64 | 65 | linters-settings: 66 | gofmt: 67 | simplify: true 68 | forbidigo: 69 | # Forbid the following identifiers (list of regexp). 70 | forbid: 71 | - '\brequire\.New\b(# Use package-level functions with explicit TestingT)?' 72 | - '\bassert\.New\b(# Use package-level functions with explicit TestingT)?' 73 | - '\bmetrics\.IncrCounter\b(# Use labeled metrics)?' 74 | - '\bmetrics\.AddSample\b(# Use labeled metrics)?' 75 | - '\bmetrics\.MeasureSince\b(# Use labeled metrics)?' 76 | - '\bmetrics\.SetGauge\b(# Use labeled metrics)?' 77 | # Exclude godoc examples from forbidigo checks. 78 | # Default: true 79 | exclude_godoc_examples: false 80 | 81 | run: 82 | timeout: 10m 83 | concurrency: 4 84 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 HashiCorp, Inc. 2 | 3 | Mozilla Public License, version 2.0 4 | 5 | 1. Definitions 6 | 7 | 1.1. “Contributor” 8 | 9 | means each individual or legal entity that creates, contributes to the 10 | creation of, or owns Covered Software. 11 | 12 | 1.2. “Contributor Version” 13 | 14 | means the combination of the Contributions of others (if any) used by a 15 | Contributor and that particular Contributor’s Contribution. 16 | 17 | 1.3. “Contribution” 18 | 19 | means Covered Software of a particular Contributor. 20 | 21 | 1.4. “Covered Software” 22 | 23 | means Source Code Form to which the initial Contributor has attached the 24 | notice in Exhibit A, the Executable Form of such Source Code Form, and 25 | Modifications of such Source Code Form, in each case including portions 26 | thereof. 27 | 28 | 1.5. “Incompatible With Secondary Licenses” 29 | means 30 | 31 | a. that the initial Contributor has attached the notice described in 32 | Exhibit B to the Covered Software; or 33 | 34 | b. that the Covered Software was made available under the terms of version 35 | 1.1 or earlier of the License, but not also under the terms of a 36 | Secondary License. 37 | 38 | 1.6. “Executable Form” 39 | 40 | means any form of the work other than Source Code Form. 41 | 42 | 1.7. “Larger Work” 43 | 44 | means a work that combines Covered Software with other material, in a separate 45 | file or files, that is not Covered Software. 46 | 47 | 1.8. “License” 48 | 49 | means this document. 50 | 51 | 1.9. “Licensable” 52 | 53 | means having the right to grant, to the maximum extent possible, whether at the 54 | time of the initial grant or subsequently, any and all of the rights conveyed by 55 | this License. 56 | 57 | 1.10. “Modifications” 58 | 59 | means any of the following: 60 | 61 | a. any file in Source Code Form that results from an addition to, deletion 62 | from, or modification of the contents of Covered Software; or 63 | 64 | b. any new file in Source Code Form that contains any Covered Software. 65 | 66 | 1.11. “Patent Claims” of a Contributor 67 | 68 | means any patent claim(s), including without limitation, method, process, 69 | and apparatus claims, in any patent Licensable by such Contributor that 70 | would be infringed, but for the grant of the License, by the making, 71 | using, selling, offering for sale, having made, import, or transfer of 72 | either its Contributions or its Contributor Version. 73 | 74 | 1.12. “Secondary License” 75 | 76 | means either the GNU General Public License, Version 2.0, the GNU Lesser 77 | General Public License, Version 2.1, the GNU Affero General Public 78 | License, Version 3.0, or any later versions of those licenses. 79 | 80 | 1.13. “Source Code Form” 81 | 82 | means the form of the work preferred for making modifications. 83 | 84 | 1.14. “You” (or “Your”) 85 | 86 | means an individual or a legal entity exercising rights under this 87 | License. For legal entities, “You” includes any entity that controls, is 88 | controlled by, or is under common control with You. For purposes of this 89 | definition, “control” means (a) the power, direct or indirect, to cause 90 | the direction or management of such entity, whether by contract or 91 | otherwise, or (b) ownership of more than fifty percent (50%) of the 92 | outstanding shares or beneficial ownership of such entity. 93 | 94 | 95 | 2. License Grants and Conditions 96 | 97 | 2.1. Grants 98 | 99 | Each Contributor hereby grants You a world-wide, royalty-free, 100 | non-exclusive license: 101 | 102 | a. under intellectual property rights (other than patent or trademark) 103 | Licensable by such Contributor to use, reproduce, make available, 104 | modify, display, perform, distribute, and otherwise exploit its 105 | Contributions, either on an unmodified basis, with Modifications, or as 106 | part of a Larger Work; and 107 | 108 | b. under Patent Claims of such Contributor to make, use, sell, offer for 109 | sale, have made, import, and otherwise transfer either its Contributions 110 | or its Contributor Version. 111 | 112 | 2.2. Effective Date 113 | 114 | The licenses granted in Section 2.1 with respect to any Contribution become 115 | effective for each Contribution on the date the Contributor first distributes 116 | such Contribution. 117 | 118 | 2.3. Limitations on Grant Scope 119 | 120 | The licenses granted in this Section 2 are the only rights granted under this 121 | License. No additional rights or licenses will be implied from the distribution 122 | or licensing of Covered Software under this License. Notwithstanding Section 123 | 2.1(b) above, no patent license is granted by a Contributor: 124 | 125 | a. for any code that a Contributor has removed from Covered Software; or 126 | 127 | b. for infringements caused by: (i) Your and any other third party’s 128 | modifications of Covered Software, or (ii) the combination of its 129 | Contributions with other software (except as part of its Contributor 130 | Version); or 131 | 132 | c. under Patent Claims infringed by Covered Software in the absence of its 133 | Contributions. 134 | 135 | This License does not grant any rights in the trademarks, service marks, or 136 | logos of any Contributor (except as may be necessary to comply with the 137 | notice requirements in Section 3.4). 138 | 139 | 2.4. Subsequent Licenses 140 | 141 | No Contributor makes additional grants as a result of Your choice to 142 | distribute the Covered Software under a subsequent version of this License 143 | (see Section 10.2) or under the terms of a Secondary License (if permitted 144 | under the terms of Section 3.3). 145 | 146 | 2.5. Representation 147 | 148 | Each Contributor represents that the Contributor believes its Contributions 149 | are its original creation(s) or it has sufficient rights to grant the 150 | rights to its Contributions conveyed by this License. 151 | 152 | 2.6. Fair Use 153 | 154 | This License is not intended to limit any rights You have under applicable 155 | copyright doctrines of fair use, fair dealing, or other equivalents. 156 | 157 | 2.7. Conditions 158 | 159 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in 160 | Section 2.1. 161 | 162 | 163 | 3. Responsibilities 164 | 165 | 3.1. Distribution of Source Form 166 | 167 | All distribution of Covered Software in Source Code Form, including any 168 | Modifications that You create or to which You contribute, must be under the 169 | terms of this License. You must inform recipients that the Source Code Form 170 | of the Covered Software is governed by the terms of this License, and how 171 | they can obtain a copy of this License. You may not attempt to alter or 172 | restrict the recipients’ rights in the Source Code Form. 173 | 174 | 3.2. Distribution of Executable Form 175 | 176 | If You distribute Covered Software in Executable Form then: 177 | 178 | a. such Covered Software must also be made available in Source Code Form, 179 | as described in Section 3.1, and You must inform recipients of the 180 | Executable Form how they can obtain a copy of such Source Code Form by 181 | reasonable means in a timely manner, at a charge no more than the cost 182 | of distribution to the recipient; and 183 | 184 | b. You may distribute such Executable Form under the terms of this License, 185 | or sublicense it under different terms, provided that the license for 186 | the Executable Form does not attempt to limit or alter the recipients’ 187 | rights in the Source Code Form under this License. 188 | 189 | 3.3. Distribution of a Larger Work 190 | 191 | You may create and distribute a Larger Work under terms of Your choice, 192 | provided that You also comply with the requirements of this License for the 193 | Covered Software. If the Larger Work is a combination of Covered Software 194 | with a work governed by one or more Secondary Licenses, and the Covered 195 | Software is not Incompatible With Secondary Licenses, this License permits 196 | You to additionally distribute such Covered Software under the terms of 197 | such Secondary License(s), so that the recipient of the Larger Work may, at 198 | their option, further distribute the Covered Software under the terms of 199 | either this License or such Secondary License(s). 200 | 201 | 3.4. Notices 202 | 203 | You may not remove or alter the substance of any license notices (including 204 | copyright notices, patent notices, disclaimers of warranty, or limitations 205 | of liability) contained within the Source Code Form of the Covered 206 | Software, except that You may alter any license notices to the extent 207 | required to remedy known factual inaccuracies. 208 | 209 | 3.5. Application of Additional Terms 210 | 211 | You may choose to offer, and to charge a fee for, warranty, support, 212 | indemnity or liability obligations to one or more recipients of Covered 213 | Software. However, You may do so only on Your own behalf, and not on behalf 214 | of any Contributor. You must make it absolutely clear that any such 215 | warranty, support, indemnity, or liability obligation is offered by You 216 | alone, and You hereby agree to indemnify every Contributor for any 217 | liability incurred by such Contributor as a result of warranty, support, 218 | indemnity or liability terms You offer. You may include additional 219 | disclaimers of warranty and limitations of liability specific to any 220 | jurisdiction. 221 | 222 | 4. Inability to Comply Due to Statute or Regulation 223 | 224 | If it is impossible for You to comply with any of the terms of this License 225 | with respect to some or all of the Covered Software due to statute, judicial 226 | order, or regulation then You must: (a) comply with the terms of this License 227 | to the maximum extent possible; and (b) describe the limitations and the code 228 | they affect. Such description must be placed in a text file included with all 229 | distributions of the Covered Software under this License. Except to the 230 | extent prohibited by statute or regulation, such description must be 231 | sufficiently detailed for a recipient of ordinary skill to be able to 232 | understand it. 233 | 234 | 5. Termination 235 | 236 | 5.1. The rights granted under this License will terminate automatically if You 237 | fail to comply with any of its terms. However, if You become compliant, 238 | then the rights granted under this License from a particular Contributor 239 | are reinstated (a) provisionally, unless and until such Contributor 240 | explicitly and finally terminates Your grants, and (b) on an ongoing basis, 241 | if such Contributor fails to notify You of the non-compliance by some 242 | reasonable means prior to 60 days after You have come back into compliance. 243 | Moreover, Your grants from a particular Contributor are reinstated on an 244 | ongoing basis if such Contributor notifies You of the non-compliance by 245 | some reasonable means, this is the first time You have received notice of 246 | non-compliance with this License from such Contributor, and You become 247 | compliant prior to 30 days after Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, counter-claims, 251 | and cross-claims) alleging that a Contributor Version directly or 252 | indirectly infringes any patent, then the rights granted to You by any and 253 | all Contributors for the Covered Software under Section 2.1 of this License 254 | shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user 257 | license agreements (excluding distributors and resellers) which have been 258 | validly granted by You or Your distributors under this License prior to 259 | termination shall survive termination. 260 | 261 | 6. Disclaimer of Warranty 262 | 263 | Covered Software is provided under this License on an “as is” basis, without 264 | warranty of any kind, either expressed, implied, or statutory, including, 265 | without limitation, warranties that the Covered Software is free of defects, 266 | merchantable, fit for a particular purpose or non-infringing. The entire 267 | risk as to the quality and performance of the Covered Software is with You. 268 | Should any Covered Software prove defective in any respect, You (not any 269 | Contributor) assume the cost of any necessary servicing, repair, or 270 | correction. This disclaimer of warranty constitutes an essential part of this 271 | License. No use of any Covered Software is authorized under this License 272 | except under this disclaimer. 273 | 274 | 7. Limitation of Liability 275 | 276 | Under no circumstances and under no legal theory, whether tort (including 277 | negligence), contract, or otherwise, shall any Contributor, or anyone who 278 | distributes Covered Software as permitted above, be liable to You for any 279 | direct, indirect, special, incidental, or consequential damages of any 280 | character including, without limitation, damages for lost profits, loss of 281 | goodwill, work stoppage, computer failure or malfunction, or any and all 282 | other commercial damages or losses, even if such party shall have been 283 | informed of the possibility of such damages. This limitation of liability 284 | shall not apply to liability for death or personal injury resulting from such 285 | party’s negligence to the extent applicable law prohibits such limitation. 286 | Some jurisdictions do not allow the exclusion or limitation of incidental or 287 | consequential damages, so this exclusion and limitation may not apply to You. 288 | 289 | 8. Litigation 290 | 291 | Any litigation relating to this License may be brought only in the courts of 292 | a jurisdiction where the defendant maintains its principal place of business 293 | and such litigation shall be governed by laws of that jurisdiction, without 294 | reference to its conflict-of-law provisions. Nothing in this Section shall 295 | prevent a party’s ability to bring cross-claims or counter-claims. 296 | 297 | 9. Miscellaneous 298 | 299 | This License represents the complete agreement concerning the subject matter 300 | hereof. If any provision of this License is held to be unenforceable, such 301 | provision shall be reformed only to the extent necessary to make it 302 | enforceable. Any law or regulation which provides that the language of a 303 | contract shall be construed against the drafter shall not be used to construe 304 | this License against a Contributor. 305 | 306 | 307 | 10. Versions of the License 308 | 309 | 10.1. New Versions 310 | 311 | Mozilla Foundation is the license steward. Except as provided in Section 312 | 10.3, no one other than the license steward has the right to modify or 313 | publish new versions of this License. Each version will be given a 314 | distinguishing version number. 315 | 316 | 10.2. Effect of New Versions 317 | 318 | You may distribute the Covered Software under the terms of the version of 319 | the License under which You originally received the Covered Software, or 320 | under the terms of any subsequent version published by the license 321 | steward. 322 | 323 | 10.3. Modified Versions 324 | 325 | If you create software not governed by this License, and you want to 326 | create a new license for such software, you may create and use a modified 327 | version of this License if you rename the license and remove any 328 | references to the name of the license steward (except to note that such 329 | modified license differs from this License). 330 | 331 | 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses 332 | If You choose to distribute Source Code Form that is Incompatible With 333 | Secondary Licenses under the terms of this version of the License, the 334 | notice described in Exhibit B of this License must be attached. 335 | 336 | Exhibit A - Source Code Form License Notice 337 | 338 | This Source Code Form is subject to the 339 | terms of the Mozilla Public License, v. 340 | 2.0. If a copy of the MPL was not 341 | distributed with this file, You can 342 | obtain one at 343 | http://mozilla.org/MPL/2.0/. 344 | 345 | If it is not possible or desirable to put the notice in a particular file, then 346 | You may include the notice in a location (such as a LICENSE file in a relevant 347 | directory) where a recipient would be likely to look for such a notice. 348 | 349 | You may add additional accurate notices of copyright ownership. 350 | 351 | Exhibit B - “Incompatible With Secondary Licenses” Notice 352 | 353 | This Source Code Form is “Incompatible 354 | With Secondary Licenses”, as defined by 355 | the Mozilla Public License, v. 2.0. 356 | 357 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := bash 2 | 3 | GOFILES ?= $(shell go list ./... | grep -v /vendor/) 4 | 5 | default: test 6 | 7 | test: vet subnet 8 | go test ./... 9 | 10 | integ: subnet 11 | INTEG_TESTS=yes go test ./... 12 | 13 | subnet: 14 | ./test/setup_subnet.sh 15 | 16 | cov: 17 | go test ./... -coverprofile=coverage.out 18 | go tool cover -html=coverage.out 19 | 20 | format: 21 | @echo "--> Running go fmt" 22 | @go fmt $(GOFILES) 23 | 24 | vet: 25 | @echo "--> Running go vet" 26 | @go vet -tags '$(GOTAGS)' $(GOFILES); if [ $$? -eq 1 ]; then \ 27 | echo ""; \ 28 | echo "Vet found suspicious constructs. Please check the reported constructs"; \ 29 | echo "and fix them if necessary before submitting the code for review."; \ 30 | exit 1; \ 31 | fi 32 | 33 | .PHONY: default test integ subnet cov format vet 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # memberlist [![GoDoc](https://godoc.org/github.com/hashicorp/memberlist?status.png)](https://godoc.org/github.com/hashicorp/memberlist) [![CircleCI](https://circleci.com/gh/hashicorp/memberlist.svg?style=svg)](https://circleci.com/gh/hashicorp/memberlist) 2 | 3 | memberlist is a [Go](http://www.golang.org) library that manages cluster 4 | membership and member failure detection using a gossip based protocol. 5 | 6 | The use cases for such a library are far-reaching: all distributed systems 7 | require membership, and memberlist is a re-usable solution to managing 8 | cluster membership and node failure detection. 9 | 10 | memberlist is eventually consistent but converges quickly on average. 11 | The speed at which it converges can be heavily tuned via various knobs 12 | on the protocol. Node failures are detected and network partitions are partially 13 | tolerated by attempting to communicate to potentially dead nodes through 14 | multiple routes. 15 | 16 | ## Building 17 | 18 | If you wish to build memberlist you'll need Go version 1.2+ installed. 19 | 20 | Please check your installation with: 21 | 22 | ``` 23 | go version 24 | ``` 25 | 26 | ## Usage 27 | 28 | Memberlist is surprisingly simple to use. An example is shown below: 29 | 30 | ```go 31 | /* Create the initial memberlist from a safe configuration. 32 | Please reference the godoc for other default config types. 33 | http://godoc.org/github.com/hashicorp/memberlist#Config 34 | */ 35 | list, err := memberlist.Create(memberlist.DefaultLocalConfig()) 36 | if err != nil { 37 | panic("Failed to create memberlist: " + err.Error()) 38 | } 39 | 40 | // Join an existing cluster by specifying at least one known member. 41 | n, err := list.Join([]string{"1.2.3.4"}) 42 | if err != nil { 43 | panic("Failed to join cluster: " + err.Error()) 44 | } 45 | 46 | // Ask for members of the cluster 47 | for _, member := range list.Members() { 48 | fmt.Printf("Member: %s %s\n", member.Name, member.Addr) 49 | } 50 | 51 | // Continue doing whatever you need, memberlist will maintain membership 52 | // information in the background. Delegates can be used for receiving 53 | // events when members join or leave. 54 | ``` 55 | 56 | The most difficult part of memberlist is configuring it since it has many 57 | available knobs in order to tune state propagation delay and convergence times. 58 | Memberlist provides a default configuration that offers a good starting point, 59 | but errs on the side of caution, choosing values that are optimized for 60 | higher convergence at the cost of higher bandwidth usage. 61 | 62 | For complete documentation, see the associated [Godoc](http://godoc.org/github.com/hashicorp/memberlist). 63 | 64 | ## Protocol 65 | 66 | memberlist is based on ["SWIM: Scalable Weakly-consistent Infection-style Process Group Membership Protocol"](http://ieeexplore.ieee.org/document/1028914/). However, we extend the protocol in a number of ways: 67 | 68 | * Several extensions are made to increase propagation speed and 69 | convergence rate. 70 | * Another set of extensions, that we call Lifeguard, are made to make memberlist more robust in the presence of slow message processing (due to factors such as CPU starvation, and network delay or loss). 71 | 72 | For details on all of these extensions, please read our paper "[Lifeguard : SWIM-ing with Situational Awareness](https://arxiv.org/abs/1707.00788)", along with the memberlist source. We welcome any questions related 73 | to the protocol on our issue tracker. 74 | 75 | ## Metrics Emission and Compatibility 76 | 77 | This library can emit metrics using either `github.com/armon/go-metrics` or `github.com/hashicorp/go-metrics`. Choosing between the libraries is controlled via build tags. 78 | 79 | **Build Tags** 80 | * `armonmetrics` - Using this tag will cause metrics to be routed to `armon/go-metrics` 81 | * `hashicorpmetrics` - Using this tag will cause all metrics to be routed to `hashicorp/go-metrics` 82 | 83 | If no build tag is specified, the default behavior is to use `armon/go-metrics`. 84 | 85 | **Deprecating `armon/go-metrics`** 86 | 87 | Emitting metrics to `armon/go-metrics` is officially deprecated. Usage of `armon/go-metrics` will remain the default until mid-2025 with opt-in support continuing to the end of 2025. 88 | 89 | **Migration** 90 | To migrate an application currently using the older `armon/go-metrics` to instead use `hashicorp/go-metrics` the following should be done. 91 | 92 | 1. Upgrade libraries using `armon/go-metrics` to consume `hashicorp/go-metrics/compat` instead. This should involve only changing import statements. All repositories in the `hashicorp` namespace 93 | 2. Update an applications library dependencies to those that have the compatibility layer configured. 94 | 3. Update the application to use `hashicorp/go-metrics` for configuring metrics export instead of `armon/go-metrics` 95 | * Replace all application imports of `github.com/armon/go-metrics` with `github.com/hashicorp/go-metrics` 96 | * Instrument your build system to build with the `hashicorpmetrics` tag. 97 | 98 | Eventually once the default behavior changes to use `hashicorp/go-metrics` by default (mid-2025), you can drop the `hashicorpmetrics` build tag. 99 | -------------------------------------------------------------------------------- /alive_delegate.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | // AliveDelegate is used to involve a client in processing 7 | // a node "alive" message. When a node joins, either through 8 | // a UDP gossip or TCP push/pull, we update the state of 9 | // that node via an alive message. This can be used to filter 10 | // a node out and prevent it from being considered a peer 11 | // using application specific logic. 12 | type AliveDelegate interface { 13 | // NotifyAlive is invoked when a message about a live 14 | // node is received from the network. Returning a non-nil 15 | // error prevents the node from being considered a peer. 16 | NotifyAlive(peer *Node) error 17 | } 18 | -------------------------------------------------------------------------------- /awareness.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "sync" 8 | "time" 9 | 10 | "github.com/hashicorp/go-metrics/compat" 11 | ) 12 | 13 | // awareness manages a simple metric for tracking the estimated health of the 14 | // local node. Health is primary the node's ability to respond in the soft 15 | // real-time manner required for correct health checking of other nodes in the 16 | // cluster. 17 | type awareness struct { 18 | sync.RWMutex 19 | 20 | // max is the upper threshold for the timeout scale (the score will be 21 | // constrained to be from 0 <= score < max). 22 | max int 23 | 24 | // score is the current awareness score. Lower values are healthier and 25 | // zero is the minimum value. 26 | score int 27 | 28 | // metricLabels is the slice of labels to put on all emitted metrics 29 | metricLabels []metrics.Label 30 | } 31 | 32 | // newAwareness returns a new awareness object. 33 | func newAwareness(max int, metricLabels []metrics.Label) *awareness { 34 | return &awareness{ 35 | max: max, 36 | score: 0, 37 | metricLabels: metricLabels, 38 | } 39 | } 40 | 41 | // ApplyDelta takes the given delta and applies it to the score in a thread-safe 42 | // manner. It also enforces a floor of zero and a max of max, so deltas may not 43 | // change the overall score if it's railed at one of the extremes. 44 | func (a *awareness) ApplyDelta(delta int) { 45 | a.Lock() 46 | initial := a.score 47 | a.score += delta 48 | if a.score < 0 { 49 | a.score = 0 50 | } else if a.score > (a.max - 1) { 51 | a.score = (a.max - 1) 52 | } 53 | final := a.score 54 | a.Unlock() 55 | 56 | if initial != final { 57 | metrics.SetGaugeWithLabels([]string{"memberlist", "health", "score"}, float32(final), a.metricLabels) 58 | } 59 | } 60 | 61 | // GetHealthScore returns the raw health score. 62 | func (a *awareness) GetHealthScore() int { 63 | a.RLock() 64 | score := a.score 65 | a.RUnlock() 66 | return score 67 | } 68 | 69 | // ScaleTimeout takes the given duration and scales it based on the current 70 | // score. Less healthyness will lead to longer timeouts. 71 | func (a *awareness) ScaleTimeout(timeout time.Duration) time.Duration { 72 | a.RLock() 73 | score := a.score 74 | a.RUnlock() 75 | return timeout * (time.Duration(score) + 1) 76 | } 77 | -------------------------------------------------------------------------------- /awareness_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestAwareness(t *testing.T) { 12 | cases := []struct { 13 | delta int 14 | score int 15 | timeout time.Duration 16 | }{ 17 | {0, 0, 1 * time.Second}, 18 | {-1, 0, 1 * time.Second}, 19 | {-10, 0, 1 * time.Second}, 20 | {1, 1, 2 * time.Second}, 21 | {-1, 0, 1 * time.Second}, 22 | {10, 7, 8 * time.Second}, 23 | {-1, 6, 7 * time.Second}, 24 | {-1, 5, 6 * time.Second}, 25 | {-1, 4, 5 * time.Second}, 26 | {-1, 3, 4 * time.Second}, 27 | {-1, 2, 3 * time.Second}, 28 | {-1, 1, 2 * time.Second}, 29 | {-1, 0, 1 * time.Second}, 30 | {-1, 0, 1 * time.Second}, 31 | } 32 | 33 | a := newAwareness(8, nil) 34 | for i, c := range cases { 35 | a.ApplyDelta(c.delta) 36 | if a.GetHealthScore() != c.score { 37 | t.Errorf("case %d: score mismatch %d != %d", i, a.score, c.score) 38 | } 39 | if timeout := a.ScaleTimeout(1 * time.Second); timeout != c.timeout { 40 | t.Errorf("case %d: scaled timeout mismatch %9.6f != %9.6f", 41 | i, timeout.Seconds(), c.timeout.Seconds()) 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /broadcast.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | /* 7 | The broadcast mechanism works by maintaining a sorted list of messages to be 8 | sent out. When a message is to be broadcast, the retransmit count 9 | is set to zero and appended to the queue. The retransmit count serves 10 | as the "priority", ensuring that newer messages get sent first. Once 11 | a message hits the retransmit limit, it is removed from the queue. 12 | 13 | Additionally, older entries can be invalidated by new messages that 14 | are contradictory. For example, if we send "{suspect M1 inc: 1}, 15 | then a following {alive M1 inc: 2} will invalidate that message 16 | */ 17 | 18 | type memberlistBroadcast struct { 19 | node string 20 | msg []byte 21 | notify chan struct{} 22 | } 23 | 24 | func (b *memberlistBroadcast) Invalidates(other Broadcast) bool { 25 | // Check if that broadcast is a memberlist type 26 | mb, ok := other.(*memberlistBroadcast) 27 | if !ok { 28 | return false 29 | } 30 | 31 | // Invalidates any message about the same node 32 | return b.node == mb.node 33 | } 34 | 35 | // memberlist.NamedBroadcast optional interface 36 | func (b *memberlistBroadcast) Name() string { 37 | return b.node 38 | } 39 | 40 | func (b *memberlistBroadcast) Message() []byte { 41 | return b.msg 42 | } 43 | 44 | func (b *memberlistBroadcast) Finished() { 45 | select { 46 | case b.notify <- struct{}{}: 47 | default: 48 | } 49 | } 50 | 51 | // encodeAndBroadcast encodes a message and enqueues it for broadcast. Fails 52 | // silently if there is an encoding error. 53 | func (m *Memberlist) encodeAndBroadcast(node string, msgType messageType, msg interface{}) { 54 | m.encodeBroadcastNotify(node, msgType, msg, nil) 55 | } 56 | 57 | // encodeBroadcastNotify encodes a message and enqueues it for broadcast 58 | // and notifies the given channel when transmission is finished. Fails 59 | // silently if there is an encoding error. 60 | func (m *Memberlist) encodeBroadcastNotify(node string, msgType messageType, msg interface{}, notify chan struct{}) { 61 | buf, err := encode(msgType, msg, m.config.MsgpackUseNewTimeFormat) 62 | if err != nil { 63 | m.logger.Printf("[ERR] memberlist: Failed to encode message for broadcast: %s", err) 64 | } else { 65 | m.queueBroadcast(node, buf.Bytes(), notify) 66 | } 67 | } 68 | 69 | // queueBroadcast is used to start dissemination of a message. It will be 70 | // sent up to a configured number of times. The message could potentially 71 | // be invalidated by a future message about the same node 72 | func (m *Memberlist) queueBroadcast(node string, msg []byte, notify chan struct{}) { 73 | b := &memberlistBroadcast{node, msg, notify} 74 | m.broadcasts.QueueBroadcast(b) 75 | } 76 | 77 | // getBroadcasts is used to return a slice of broadcasts to send up to 78 | // a maximum byte size, while imposing a per-broadcast overhead. This is used 79 | // to fill a UDP packet with piggybacked data 80 | func (m *Memberlist) getBroadcasts(overhead, limit int) [][]byte { 81 | // Get memberlist messages first 82 | toSend := m.broadcasts.GetBroadcasts(overhead, limit) 83 | 84 | // Check if the user has anything to broadcast 85 | d := m.config.Delegate 86 | if d != nil { 87 | // Determine the bytes used already 88 | bytesUsed := 0 89 | for _, msg := range toSend { 90 | bytesUsed += len(msg) + overhead 91 | } 92 | 93 | // Check space remaining for user messages 94 | avail := limit - bytesUsed 95 | if avail > overhead+userMsgOverhead { 96 | userMsgs := d.GetBroadcasts(overhead+userMsgOverhead, avail) 97 | 98 | // Frame each user message 99 | for _, msg := range userMsgs { 100 | buf := make([]byte, 1, len(msg)+1) 101 | buf[0] = byte(userMsg) 102 | buf = append(buf, msg...) 103 | toSend = append(toSend, buf) 104 | } 105 | } 106 | } 107 | return toSend 108 | } 109 | -------------------------------------------------------------------------------- /broadcast_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "reflect" 8 | "testing" 9 | ) 10 | 11 | func TestMemberlistBroadcast_Invalidates(t *testing.T) { 12 | m1 := &memberlistBroadcast{"test", nil, nil} 13 | m2 := &memberlistBroadcast{"foo", nil, nil} 14 | 15 | if m1.Invalidates(m2) || m2.Invalidates(m1) { 16 | t.Fatalf("unexpected invalidation") 17 | } 18 | 19 | if !m1.Invalidates(m1) { 20 | t.Fatalf("expected invalidation") 21 | } 22 | } 23 | 24 | func TestMemberlistBroadcast_Message(t *testing.T) { 25 | m1 := &memberlistBroadcast{"test", []byte("test"), nil} 26 | msg := m1.Message() 27 | if !reflect.DeepEqual(msg, []byte("test")) { 28 | t.Fatalf("messages do not match") 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "fmt" 8 | "io" 9 | "log" 10 | "net" 11 | "os" 12 | "strings" 13 | "time" 14 | 15 | "github.com/hashicorp/go-metrics/compat" 16 | "github.com/hashicorp/go-multierror" 17 | ) 18 | 19 | type Config struct { 20 | // The name of this node. This must be unique in the cluster. 21 | Name string 22 | 23 | // Transport is a hook for providing custom code to communicate with 24 | // other nodes. If this is left nil, then memberlist will by default 25 | // make a NetTransport using BindAddr and BindPort from this structure. 26 | Transport Transport 27 | 28 | // Label is an optional set of bytes to include on the outside of each 29 | // packet and stream. 30 | // 31 | // If gossip encryption is enabled and this is set it is treated as GCM 32 | // authenticated data. 33 | Label string 34 | 35 | // SkipInboundLabelCheck skips the check that inbound packets and gossip 36 | // streams need to be label prefixed. 37 | SkipInboundLabelCheck bool 38 | 39 | // Configuration related to what address to bind to and ports to 40 | // listen on. The port is used for both UDP and TCP gossip. It is 41 | // assumed other nodes are running on this port, but they do not need 42 | // to. 43 | BindAddr string 44 | BindPort int 45 | 46 | // Configuration related to what address to advertise to other 47 | // cluster members. Used for nat traversal. 48 | AdvertiseAddr string 49 | AdvertisePort int 50 | 51 | // ProtocolVersion is the configured protocol version that we 52 | // will _speak_. This must be between ProtocolVersionMin and 53 | // ProtocolVersionMax. 54 | ProtocolVersion uint8 55 | 56 | // TCPTimeout is the timeout for establishing a stream connection with 57 | // a remote node for a full state sync, and for stream read and write 58 | // operations. This is a legacy name for backwards compatibility, but 59 | // should really be called StreamTimeout now that we have generalized 60 | // the transport. 61 | TCPTimeout time.Duration 62 | 63 | // IndirectChecks is the number of nodes that will be asked to perform 64 | // an indirect probe of a node in the case a direct probe fails. Memberlist 65 | // waits for an ack from any single indirect node, so increasing this 66 | // number will increase the likelihood that an indirect probe will succeed 67 | // at the expense of bandwidth. 68 | IndirectChecks int 69 | 70 | // RetransmitMult is the multiplier for the number of retransmissions 71 | // that are attempted for messages broadcasted over gossip. The actual 72 | // count of retransmissions is calculated using the formula: 73 | // 74 | // Retransmits = RetransmitMult * log(N+1) 75 | // 76 | // This allows the retransmits to scale properly with cluster size. The 77 | // higher the multiplier, the more likely a failed broadcast is to converge 78 | // at the expense of increased bandwidth. 79 | RetransmitMult int 80 | 81 | // SuspicionMult is the multiplier for determining the time an 82 | // inaccessible node is considered suspect before declaring it dead. 83 | // The actual timeout is calculated using the formula: 84 | // 85 | // SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval 86 | // 87 | // This allows the timeout to scale properly with expected propagation 88 | // delay with a larger cluster size. The higher the multiplier, the longer 89 | // an inaccessible node is considered part of the cluster before declaring 90 | // it dead, giving that suspect node more time to refute if it is indeed 91 | // still alive. 92 | SuspicionMult int 93 | 94 | // SuspicionMaxTimeoutMult is the multiplier applied to the 95 | // SuspicionTimeout used as an upper bound on detection time. This max 96 | // timeout is calculated using the formula: 97 | // 98 | // SuspicionMaxTimeout = SuspicionMaxTimeoutMult * SuspicionTimeout 99 | // 100 | // If everything is working properly, confirmations from other nodes will 101 | // accelerate suspicion timers in a manner which will cause the timeout 102 | // to reach the base SuspicionTimeout before that elapses, so this value 103 | // will typically only come into play if a node is experiencing issues 104 | // communicating with other nodes. It should be set to a something fairly 105 | // large so that a node having problems will have a lot of chances to 106 | // recover before falsely declaring other nodes as failed, but short 107 | // enough for a legitimately isolated node to still make progress marking 108 | // nodes failed in a reasonable amount of time. 109 | SuspicionMaxTimeoutMult int 110 | 111 | // PushPullInterval is the interval between complete state syncs. 112 | // Complete state syncs are done with a single node over TCP and are 113 | // quite expensive relative to standard gossiped messages. Setting this 114 | // to zero will disable state push/pull syncs completely. 115 | // 116 | // Setting this interval lower (more frequent) will increase convergence 117 | // speeds across larger clusters at the expense of increased bandwidth 118 | // usage. 119 | PushPullInterval time.Duration 120 | 121 | // ProbeInterval and ProbeTimeout are used to configure probing 122 | // behavior for memberlist. 123 | // 124 | // ProbeInterval is the interval between random node probes. Setting 125 | // this lower (more frequent) will cause the memberlist cluster to detect 126 | // failed nodes more quickly at the expense of increased bandwidth usage. 127 | // 128 | // ProbeTimeout is the timeout to wait for an ack from a probed node 129 | // before assuming it is unhealthy. This should be set to 99-percentile 130 | // of RTT (round-trip time) on your network. 131 | ProbeInterval time.Duration 132 | ProbeTimeout time.Duration 133 | 134 | // DisableTcpPings will turn off the fallback TCP pings that are attempted 135 | // if the direct UDP ping fails. These get pipelined along with the 136 | // indirect UDP pings. 137 | DisableTcpPings bool 138 | 139 | // DisableTcpPingsForNode is like DisableTcpPings, but lets you control 140 | // whether to perform TCP pings on a node-by-node basis. 141 | DisableTcpPingsForNode func(nodeName string) bool 142 | 143 | // AwarenessMaxMultiplier will increase the probe interval if the node 144 | // becomes aware that it might be degraded and not meeting the soft real 145 | // time requirements to reliably probe other nodes. 146 | AwarenessMaxMultiplier int 147 | 148 | // GossipInterval and GossipNodes are used to configure the gossip 149 | // behavior of memberlist. 150 | // 151 | // GossipInterval is the interval between sending messages that need 152 | // to be gossiped that haven't been able to piggyback on probing messages. 153 | // If this is set to zero, non-piggyback gossip is disabled. By lowering 154 | // this value (more frequent) gossip messages are propagated across 155 | // the cluster more quickly at the expense of increased bandwidth. 156 | // 157 | // GossipNodes is the number of random nodes to send gossip messages to 158 | // per GossipInterval. Increasing this number causes the gossip messages 159 | // to propagate across the cluster more quickly at the expense of 160 | // increased bandwidth. 161 | // 162 | // GossipToTheDeadTime is the interval after which a node has died that 163 | // we will still try to gossip to it. This gives it a chance to refute. 164 | GossipInterval time.Duration 165 | GossipNodes int 166 | GossipToTheDeadTime time.Duration 167 | 168 | // GossipVerifyIncoming controls whether to enforce encryption for incoming 169 | // gossip. It is used for upshifting from unencrypted to encrypted gossip on 170 | // a running cluster. 171 | GossipVerifyIncoming bool 172 | 173 | // GossipVerifyOutgoing controls whether to enforce encryption for outgoing 174 | // gossip. It is used for upshifting from unencrypted to encrypted gossip on 175 | // a running cluster. 176 | GossipVerifyOutgoing bool 177 | 178 | // EnableCompression is used to control message compression. This can 179 | // be used to reduce bandwidth usage at the cost of slightly more CPU 180 | // utilization. This is only available starting at protocol version 1. 181 | EnableCompression bool 182 | 183 | // SecretKey is used to initialize the primary encryption key in a keyring. 184 | // The primary encryption key is the only key used to encrypt messages and 185 | // the first key used while attempting to decrypt messages. Providing a 186 | // value for this primary key will enable message-level encryption and 187 | // verification, and automatically install the key onto the keyring. 188 | // The value should be either 16, 24, or 32 bytes to select AES-128, 189 | // AES-192, or AES-256. 190 | SecretKey []byte 191 | 192 | // The keyring holds all of the encryption keys used internally. It is 193 | // automatically initialized using the SecretKey and SecretKeys values. 194 | Keyring *Keyring 195 | 196 | // Delegate and Events are delegates for receiving and providing 197 | // data to memberlist via callback mechanisms. For Delegate, see 198 | // the Delegate interface. For Events, see the EventDelegate interface. 199 | // 200 | // The DelegateProtocolMin/Max are used to guarantee protocol-compatibility 201 | // for any custom messages that the delegate might do (broadcasts, 202 | // local/remote state, etc.). If you don't set these, then the protocol 203 | // versions will just be zero, and version compliance won't be done. 204 | Delegate Delegate 205 | DelegateProtocolVersion uint8 206 | DelegateProtocolMin uint8 207 | DelegateProtocolMax uint8 208 | Events EventDelegate 209 | Conflict ConflictDelegate 210 | Merge MergeDelegate 211 | Ping PingDelegate 212 | Alive AliveDelegate 213 | 214 | // DNSConfigPath points to the system's DNS config file, usually located 215 | // at /etc/resolv.conf. It can be overridden via config for easier testing. 216 | DNSConfigPath string 217 | 218 | // LogOutput is the writer where logs should be sent. If this is not 219 | // set, logging will go to stderr by default. You cannot specify both LogOutput 220 | // and Logger at the same time. 221 | LogOutput io.Writer 222 | 223 | // Logger is a custom logger which you provide. If Logger is set, it will use 224 | // this for the internal logger. If Logger is not set, it will fall back to the 225 | // behavior for using LogOutput. You cannot specify both LogOutput and Logger 226 | // at the same time. 227 | Logger *log.Logger 228 | 229 | // Size of Memberlist's internal channel which handles UDP messages. The 230 | // size of this determines the size of the queue which Memberlist will keep 231 | // while UDP messages are handled. 232 | HandoffQueueDepth int 233 | 234 | // Maximum number of bytes that memberlist will put in a packet (this 235 | // will be for UDP packets by default with a NetTransport). A safe value 236 | // for this is typically 1400 bytes (which is the default). However, 237 | // depending on your network's MTU (Maximum Transmission Unit) you may 238 | // be able to increase this to get more content into each gossip packet. 239 | // This is a legacy name for backward compatibility but should really be 240 | // called PacketBufferSize now that we have generalized the transport. 241 | UDPBufferSize int 242 | 243 | // DeadNodeReclaimTime controls the time before a dead node's name can be 244 | // reclaimed by one with a different address or port. By default, this is 0, 245 | // meaning nodes cannot be reclaimed this way. 246 | DeadNodeReclaimTime time.Duration 247 | 248 | // RequireNodeNames controls if the name of a node is required when sending 249 | // a message to that node. 250 | RequireNodeNames bool 251 | 252 | // CIDRsAllowed If nil, allow any connection (default), otherwise specify all networks 253 | // allowed to connect (you must specify IPv6/IPv4 separately) 254 | // Using [] will block all connections. 255 | CIDRsAllowed []net.IPNet 256 | 257 | // MetricLabels is a map of optional labels to apply to all metrics emitted. 258 | MetricLabels []metrics.Label 259 | 260 | // QueueCheckInterval is the interval at which we check the message 261 | // queue to apply the warning and max depth. 262 | QueueCheckInterval time.Duration 263 | 264 | // MsgpackUseNewTimeFormat when set to true, force the underlying msgpack 265 | // codec to use the new format of time.Time when encoding (used in 266 | // go-msgpack v1.1.5 by default). Decoding is not affected, as all 267 | // go-msgpack v2.1.0+ decoders know how to decode both formats. 268 | MsgpackUseNewTimeFormat bool 269 | } 270 | 271 | // ParseCIDRs return a possible empty list of all Network that have been parsed 272 | // In case of error, it returns succesfully parsed CIDRs and the last error found 273 | func ParseCIDRs(v []string) ([]net.IPNet, error) { 274 | nets := make([]net.IPNet, 0) 275 | if v == nil { 276 | return nets, nil 277 | } 278 | var errs error 279 | hasErrors := false 280 | for _, p := range v { 281 | _, net, err := net.ParseCIDR(strings.TrimSpace(p)) 282 | if err != nil { 283 | err = fmt.Errorf("invalid cidr: %s", p) 284 | errs = multierror.Append(errs, err) 285 | hasErrors = true 286 | } else { 287 | nets = append(nets, *net) 288 | } 289 | } 290 | if !hasErrors { 291 | errs = nil 292 | } 293 | return nets, errs 294 | } 295 | 296 | // DefaultLANConfig returns a sane set of configurations for Memberlist. 297 | // It uses the hostname as the node name, and otherwise sets very conservative 298 | // values that are sane for most LAN environments. The default configuration 299 | // errs on the side of caution, choosing values that are optimized 300 | // for higher convergence at the cost of higher bandwidth usage. Regardless, 301 | // these values are a good starting point when getting started with memberlist. 302 | func DefaultLANConfig() *Config { 303 | hostname, _ := os.Hostname() 304 | return &Config{ 305 | Name: hostname, 306 | BindAddr: "0.0.0.0", 307 | BindPort: 7946, 308 | AdvertiseAddr: "", 309 | AdvertisePort: 7946, 310 | ProtocolVersion: ProtocolVersion2Compatible, 311 | TCPTimeout: 10 * time.Second, // Timeout after 10 seconds 312 | IndirectChecks: 3, // Use 3 nodes for the indirect ping 313 | RetransmitMult: 4, // Retransmit a message 4 * log(N+1) nodes 314 | SuspicionMult: 4, // Suspect a node for 4 * log(N+1) * Interval 315 | SuspicionMaxTimeoutMult: 6, // For 10k nodes this will give a max timeout of 120 seconds 316 | PushPullInterval: 30 * time.Second, // Low frequency 317 | ProbeTimeout: 500 * time.Millisecond, // Reasonable RTT time for LAN 318 | ProbeInterval: 1 * time.Second, // Failure check every second 319 | DisableTcpPings: false, // TCP pings are safe, even with mixed versions 320 | AwarenessMaxMultiplier: 8, // Probe interval backs off to 8 seconds 321 | 322 | GossipNodes: 3, // Gossip to 3 nodes 323 | GossipInterval: 200 * time.Millisecond, // Gossip more rapidly 324 | GossipToTheDeadTime: 30 * time.Second, // Same as push/pull 325 | GossipVerifyIncoming: true, 326 | GossipVerifyOutgoing: true, 327 | 328 | EnableCompression: true, // Enable compression by default 329 | 330 | SecretKey: nil, 331 | Keyring: nil, 332 | 333 | DNSConfigPath: "/etc/resolv.conf", 334 | 335 | HandoffQueueDepth: 1024, 336 | UDPBufferSize: 1400, 337 | CIDRsAllowed: nil, // same as allow all 338 | 339 | QueueCheckInterval: 30 * time.Second, 340 | } 341 | } 342 | 343 | // DefaultWANConfig works like DefaultConfig, however it returns a configuration 344 | // that is optimized for most WAN environments. The default configuration is 345 | // still very conservative and errs on the side of caution. 346 | func DefaultWANConfig() *Config { 347 | conf := DefaultLANConfig() 348 | conf.TCPTimeout = 30 * time.Second 349 | conf.SuspicionMult = 6 350 | conf.PushPullInterval = 60 * time.Second 351 | conf.ProbeTimeout = 3 * time.Second 352 | conf.ProbeInterval = 5 * time.Second 353 | conf.GossipNodes = 4 // Gossip less frequently, but to an additional node 354 | conf.GossipInterval = 500 * time.Millisecond 355 | conf.GossipToTheDeadTime = 60 * time.Second 356 | return conf 357 | } 358 | 359 | // IPMustBeChecked return true if IPAllowed must be called 360 | func (c *Config) IPMustBeChecked() bool { 361 | return len(c.CIDRsAllowed) > 0 362 | } 363 | 364 | // IPAllowed return an error if access to memberlist is denied 365 | func (c *Config) IPAllowed(ip net.IP) error { 366 | if !c.IPMustBeChecked() { 367 | return nil 368 | } 369 | for _, n := range c.CIDRsAllowed { 370 | if n.Contains(ip) { 371 | return nil 372 | } 373 | } 374 | return fmt.Errorf("%s is not allowed", ip) 375 | } 376 | 377 | // DefaultLocalConfig works like DefaultConfig, however it returns a configuration 378 | // that is optimized for a local loopback environments. The default configuration is 379 | // still very conservative and errs on the side of caution. 380 | func DefaultLocalConfig() *Config { 381 | conf := DefaultLANConfig() 382 | conf.TCPTimeout = time.Second 383 | conf.IndirectChecks = 1 384 | conf.RetransmitMult = 2 385 | conf.SuspicionMult = 3 386 | conf.PushPullInterval = 15 * time.Second 387 | conf.ProbeTimeout = 200 * time.Millisecond 388 | conf.ProbeInterval = time.Second 389 | conf.GossipInterval = 100 * time.Millisecond 390 | conf.GossipToTheDeadTime = 15 * time.Second 391 | return conf 392 | } 393 | 394 | // Returns whether or not encryption is enabled 395 | func (c *Config) EncryptionEnabled() bool { 396 | return c.Keyring != nil && len(c.Keyring.GetKeys()) > 0 397 | } 398 | -------------------------------------------------------------------------------- /config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "net" 8 | "testing" 9 | ) 10 | 11 | func Test_IsValidAddressDefaults(t *testing.T) { 12 | tests := []string{ 13 | "127.0.0.1", 14 | "127.0.0.5", 15 | "10.0.0.9", 16 | "172.16.0.7", 17 | "192.168.2.1", 18 | "fe80::aede:48ff:fe00:1122", 19 | "::1", 20 | } 21 | config := DefaultLANConfig() 22 | for _, ip := range tests { 23 | localV4 := net.ParseIP(ip) 24 | if err := config.IPAllowed(localV4); err != nil { 25 | t.Fatalf("IP %s Localhost Should be accepted for LAN", ip) 26 | } 27 | } 28 | config = DefaultWANConfig() 29 | for _, ip := range tests { 30 | localV4 := net.ParseIP(ip) 31 | if err := config.IPAllowed(localV4); err != nil { 32 | t.Fatalf("IP %s Localhost Should be accepted for WAN", ip) 33 | } 34 | } 35 | } 36 | 37 | func Test_IsValidAddressOverride(t *testing.T) { 38 | t.Parallel() 39 | cases := []struct { 40 | name string 41 | allow []string 42 | success []string 43 | fail []string 44 | }{ 45 | { 46 | name: "Default, nil allows all", 47 | allow: nil, 48 | success: []string{"127.0.0.5", "10.0.0.9", "192.168.1.7", "::1"}, 49 | fail: []string{}, 50 | }, 51 | { 52 | name: "Only IPv4", 53 | allow: []string{"0.0.0.0/0"}, 54 | success: []string{"127.0.0.5", "10.0.0.9", "192.168.1.7"}, 55 | fail: []string{"fe80::38bc:4dff:fe62:b1ae", "::1"}, 56 | }, 57 | { 58 | name: "Only IPv6", 59 | allow: []string{"::0/0"}, 60 | success: []string{"fe80::38bc:4dff:fe62:b1ae", "::1"}, 61 | fail: []string{"127.0.0.5", "10.0.0.9", "192.168.1.7"}, 62 | }, 63 | { 64 | name: "Only 127.0.0.0/8 and ::1", 65 | allow: []string{"::1/128", "127.0.0.0/8"}, 66 | success: []string{"127.0.0.5", "::1"}, 67 | fail: []string{"::2", "178.250.0.187", "10.0.0.9", "192.168.1.7", "fe80::38bc:4dff:fe62:b1ae"}, 68 | }, 69 | } 70 | 71 | for _, testCase := range cases { 72 | t.Run(testCase.name, func(t *testing.T) { 73 | config := DefaultLANConfig() 74 | var err error 75 | config.CIDRsAllowed, err = ParseCIDRs(testCase.allow) 76 | if err != nil { 77 | t.Fatalf("failed parsing %s", testCase.allow) 78 | } 79 | for _, ips := range testCase.success { 80 | ip := net.ParseIP(ips) 81 | if err := config.IPAllowed(ip); err != nil { 82 | t.Fatalf("Test case with %s should pass", ip) 83 | } 84 | } 85 | for _, ips := range testCase.fail { 86 | ip := net.ParseIP(ips) 87 | if err := config.IPAllowed(ip); err == nil { 88 | t.Fatalf("Test case with %s should fail", ip) 89 | } 90 | } 91 | }) 92 | 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /conflict_delegate.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | // ConflictDelegate is a used to inform a client that 7 | // a node has attempted to join which would result in a 8 | // name conflict. This happens if two clients are configured 9 | // with the same name but different addresses. 10 | type ConflictDelegate interface { 11 | // NotifyConflict is invoked when a name conflict is detected 12 | NotifyConflict(existing, other *Node) 13 | } 14 | -------------------------------------------------------------------------------- /delegate.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | // Delegate is the interface that clients must implement if they want to hook 7 | // into the gossip layer of Memberlist. All the methods must be thread-safe, 8 | // as they can and generally will be called concurrently. 9 | type Delegate interface { 10 | // NodeMeta is used to retrieve meta-data about the current node 11 | // when broadcasting an alive message. It's length is limited to 12 | // the given byte size. This metadata is available in the Node structure. 13 | NodeMeta(limit int) []byte 14 | 15 | // NotifyMsg is called when a user-data message is received. 16 | // Care should be taken that this method does not block, since doing 17 | // so would block the entire UDP packet receive loop. Additionally, the byte 18 | // slice may be modified after the call returns, so it should be copied if needed 19 | NotifyMsg([]byte) 20 | 21 | // GetBroadcasts is called when user data messages can be broadcast. 22 | // It can return a list of buffers to send. Each buffer should assume an 23 | // overhead as provided with a limit on the total byte size allowed. 24 | // The total byte size of the resulting data to send must not exceed 25 | // the limit. Care should be taken that this method does not block, 26 | // since doing so would block the entire UDP packet receive loop. 27 | GetBroadcasts(overhead, limit int) [][]byte 28 | 29 | // LocalState is used for a TCP Push/Pull. This is sent to 30 | // the remote side in addition to the membership information. Any 31 | // data can be sent here. See MergeRemoteState as well. The `join` 32 | // boolean indicates this is for a join instead of a push/pull. 33 | LocalState(join bool) []byte 34 | 35 | // MergeRemoteState is invoked after a TCP Push/Pull. This is the 36 | // state received from the remote side and is the result of the 37 | // remote side's LocalState call. The 'join' 38 | // boolean indicates this is for a join instead of a push/pull. 39 | MergeRemoteState(buf []byte, join bool) 40 | } 41 | -------------------------------------------------------------------------------- /event_delegate.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | // EventDelegate is a simpler delegate that is used only to receive 7 | // notifications about members joining and leaving. The methods in this 8 | // delegate may be called by multiple goroutines, but never concurrently. 9 | // This allows you to reason about ordering. 10 | type EventDelegate interface { 11 | // NotifyJoin is invoked when a node is detected to have joined. 12 | // The Node argument must not be modified. 13 | NotifyJoin(*Node) 14 | 15 | // NotifyLeave is invoked when a node is detected to have left. 16 | // The Node argument must not be modified. 17 | NotifyLeave(*Node) 18 | 19 | // NotifyUpdate is invoked when a node is detected to have 20 | // updated, usually involving the meta data. The Node argument 21 | // must not be modified. 22 | NotifyUpdate(*Node) 23 | } 24 | 25 | // ChannelEventDelegate is used to enable an application to receive 26 | // events about joins and leaves over a channel instead of a direct 27 | // function call. 28 | // 29 | // Care must be taken that events are processed in a timely manner from 30 | // the channel, since this delegate will block until an event can be sent. 31 | type ChannelEventDelegate struct { 32 | Ch chan<- NodeEvent 33 | } 34 | 35 | // NodeEventType are the types of events that can be sent from the 36 | // ChannelEventDelegate. 37 | type NodeEventType int 38 | 39 | const ( 40 | NodeJoin NodeEventType = iota 41 | NodeLeave 42 | NodeUpdate 43 | ) 44 | 45 | // NodeEvent is a single event related to node activity in the memberlist. 46 | // The Node member of this struct must not be directly modified. It is passed 47 | // as a pointer to avoid unnecessary copies. If you wish to modify the node, 48 | // make a copy first. 49 | type NodeEvent struct { 50 | Event NodeEventType 51 | Node *Node 52 | } 53 | 54 | func (c *ChannelEventDelegate) NotifyJoin(n *Node) { 55 | node := *n 56 | c.Ch <- NodeEvent{NodeJoin, &node} 57 | } 58 | 59 | func (c *ChannelEventDelegate) NotifyLeave(n *Node) { 60 | node := *n 61 | c.Ch <- NodeEvent{NodeLeave, &node} 62 | } 63 | 64 | func (c *ChannelEventDelegate) NotifyUpdate(n *Node) { 65 | node := *n 66 | c.Ch <- NodeEvent{NodeUpdate, &node} 67 | } 68 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/hashicorp/memberlist 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c 7 | github.com/hashicorp/go-metrics v0.5.4 8 | github.com/hashicorp/go-msgpack/v2 v2.1.1 9 | github.com/hashicorp/go-multierror v1.0.0 10 | github.com/hashicorp/go-sockaddr v1.0.0 11 | github.com/miekg/dns v1.1.26 12 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 13 | github.com/stretchr/testify v1.4.0 14 | ) 15 | 16 | require ( 17 | github.com/armon/go-metrics v0.4.1 // indirect 18 | github.com/davecgh/go-spew v1.1.1 // indirect 19 | github.com/hashicorp/errwrap v1.0.0 // indirect 20 | github.com/hashicorp/go-immutable-radix v1.0.0 // indirect 21 | github.com/hashicorp/golang-lru v0.5.0 // indirect 22 | github.com/pmezard/go-difflib v1.0.0 // indirect 23 | golang.org/x/crypto v0.32.0 // indirect 24 | golang.org/x/net v0.34.0 // indirect 25 | golang.org/x/sys v0.29.0 // indirect 26 | gopkg.in/yaml.v2 v2.4.0 // indirect 27 | ) 28 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= 3 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 4 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 5 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 6 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 7 | github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= 8 | github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= 9 | github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= 10 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 11 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 12 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 13 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 14 | github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= 15 | github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= 16 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 17 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 18 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 19 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 20 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 21 | github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= 22 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 23 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 24 | github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= 25 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 26 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 27 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 28 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 29 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 30 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 31 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 32 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 33 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 34 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 35 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 36 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 37 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= 38 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 39 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 40 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 41 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 42 | github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 43 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 44 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 45 | github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= 46 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 47 | github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 48 | github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= 49 | github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 50 | github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY= 51 | github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI= 52 | github.com/hashicorp/go-msgpack/v2 v2.1.1 h1:xQEY9yB2wnHitoSzk/B9UjXWRQ67QKu5AOm8aFp8N3I= 53 | github.com/hashicorp/go-msgpack/v2 v2.1.1/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4= 54 | github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= 55 | github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= 56 | github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= 57 | github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= 58 | github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= 59 | github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= 60 | github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 61 | github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= 62 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 63 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 64 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 65 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 66 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 67 | github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 68 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 69 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 70 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 71 | github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 72 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 73 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 74 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 75 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 76 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 77 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 78 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 79 | github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= 80 | github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= 81 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 82 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 83 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 84 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 85 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 86 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 87 | github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= 88 | github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 89 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 90 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 91 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 92 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 93 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 94 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 95 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 96 | github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= 97 | github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= 98 | github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= 99 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 100 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 101 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 102 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 103 | github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= 104 | github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= 105 | github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= 106 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 107 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 108 | github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= 109 | github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 110 | github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= 111 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= 112 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= 113 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 114 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 115 | github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= 116 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 117 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 118 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 119 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 120 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 121 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 122 | github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= 123 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 124 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 125 | golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= 126 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 127 | golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= 128 | golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= 129 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 130 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 131 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 132 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 133 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 134 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 135 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 136 | golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 137 | golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= 138 | golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= 139 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 140 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 141 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 142 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 143 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 144 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= 145 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 146 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 147 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 148 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 149 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 150 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 151 | golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 152 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 153 | golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 154 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 155 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 156 | golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 157 | golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 158 | golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 159 | golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 160 | golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= 161 | golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 162 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 163 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 164 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 165 | golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 166 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 167 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 168 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 169 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 170 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 171 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 172 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 173 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 174 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 175 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 176 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 177 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 178 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 179 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 180 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 181 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 182 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 183 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 184 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 185 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 186 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 187 | -------------------------------------------------------------------------------- /integ_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "fmt" 8 | "log" 9 | "os" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | // CheckInteg will skip a test if integration testing is not enabled. 15 | func CheckInteg(t *testing.T) { 16 | if !IsInteg() { 17 | t.SkipNow() 18 | } 19 | } 20 | 21 | // IsInteg returns a boolean telling you if we're in integ testing mode. 22 | func IsInteg() bool { 23 | return os.Getenv("INTEG_TESTS") != "" 24 | } 25 | 26 | // Tests the memberlist by creating a cluster of 100 nodes 27 | // and checking that we get strong convergence of changes. 28 | func TestMemberlist_Integ(t *testing.T) { 29 | CheckInteg(t) 30 | 31 | num := 16 32 | var members []*Memberlist 33 | 34 | secret := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} 35 | eventCh := make(chan NodeEvent, num) 36 | 37 | addr := "127.0.0.1" 38 | for i := 0; i < num; i++ { 39 | c := DefaultLANConfig() 40 | c.Name = fmt.Sprintf("%s:%d", addr, 12345+i) 41 | c.BindAddr = addr 42 | c.BindPort = 12345 + i 43 | c.ProbeInterval = 20 * time.Millisecond 44 | c.ProbeTimeout = 100 * time.Millisecond 45 | c.GossipInterval = 20 * time.Millisecond 46 | c.PushPullInterval = 200 * time.Millisecond 47 | c.SecretKey = secret 48 | c.Logger = log.New(os.Stderr, c.Name, log.LstdFlags) 49 | 50 | if i == 0 { 51 | c.Events = &ChannelEventDelegate{eventCh} 52 | } 53 | 54 | m, err := Create(c) 55 | if err != nil { 56 | t.Fatalf("unexpected err: %s", err) 57 | } 58 | defer m.Shutdown() 59 | 60 | members = append(members, m) 61 | 62 | if i > 0 { 63 | last := members[i-1] 64 | num, err := m.Join([]string{last.config.Name + "/" + last.config.Name}) 65 | if num == 0 || err != nil { 66 | t.Fatalf("unexpected err: %s", err) 67 | } 68 | } 69 | } 70 | 71 | // Wait and print debug info 72 | breakTimer := time.After(250 * time.Millisecond) 73 | WAIT: 74 | for { 75 | select { 76 | case e := <-eventCh: 77 | if e.Event == NodeJoin { 78 | t.Logf("[DEBUG] Node join: %v (%d)", *e.Node, members[0].NumMembers()) 79 | } else { 80 | t.Logf("[DEBUG] Node leave: %v (%d)", *e.Node, members[0].NumMembers()) 81 | } 82 | case <-breakTimer: 83 | break WAIT 84 | } 85 | } 86 | 87 | for idx, m := range members { 88 | got := m.NumMembers() 89 | if got != num { 90 | t.Errorf("bad num members at idx %d. Expected %d. Got %d.", 91 | idx, num, got) 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /internal/retry/retry.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | // Package retry provides support for repeating operations in tests. 5 | // 6 | // A sample retry operation looks like this: 7 | // 8 | // func TestX(t *testing.T) { 9 | // retry.Run(t, func(r *retry.R) { 10 | // if err := foo(); err != nil { 11 | // r.Fatal("f: ", err) 12 | // } 13 | // }) 14 | // } 15 | package retry 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "runtime" 21 | "strings" 22 | "sync" 23 | "time" 24 | ) 25 | 26 | // Failer is an interface compatible with testing.T. 27 | type Failer interface { 28 | // Log is called for the final test output 29 | Log(args ...interface{}) 30 | 31 | // FailNow is called when the retrying is abandoned. 32 | FailNow() 33 | } 34 | 35 | // R provides context for the retryer. 36 | type R struct { 37 | fail bool 38 | output []string 39 | } 40 | 41 | func (r *R) FailNow() { 42 | r.fail = true 43 | runtime.Goexit() 44 | } 45 | 46 | func (r *R) Fatal(args ...interface{}) { 47 | r.log(fmt.Sprint(args...)) 48 | r.FailNow() 49 | } 50 | 51 | func (r *R) Fatalf(format string, args ...interface{}) { 52 | r.log(fmt.Sprintf(format, args...)) 53 | r.FailNow() 54 | } 55 | 56 | func (r *R) Error(args ...interface{}) { 57 | r.log(fmt.Sprint(args...)) 58 | r.fail = true 59 | } 60 | 61 | func (r *R) Errorf(format string, args ...interface{}) { 62 | r.log(fmt.Sprintf(format, args...)) 63 | r.fail = true 64 | } 65 | 66 | func (r *R) Check(err error) { 67 | if err != nil { 68 | r.log(err.Error()) 69 | r.FailNow() 70 | } 71 | } 72 | 73 | func (r *R) log(s string) { 74 | r.output = append(r.output, decorate(s)) 75 | } 76 | 77 | func decorate(s string) string { 78 | _, file, line, ok := runtime.Caller(3) 79 | if ok { 80 | n := strings.LastIndex(file, "/") 81 | if n >= 0 { 82 | file = file[n+1:] 83 | } 84 | } else { 85 | file = "???" 86 | line = 1 87 | } 88 | return fmt.Sprintf("%s:%d: %s", file, line, s) 89 | } 90 | 91 | func Run(t Failer, f func(r *R)) { 92 | run(DefaultFailer(), t, f) 93 | } 94 | 95 | func RunWith(r Retryer, t Failer, f func(r *R)) { 96 | run(r, t, f) 97 | } 98 | 99 | func dedup(a []string) string { 100 | if len(a) == 0 { 101 | return "" 102 | } 103 | m := map[string]int{} 104 | for _, s := range a { 105 | m[s] = m[s] + 1 106 | } 107 | var b bytes.Buffer 108 | for _, s := range a { 109 | if _, ok := m[s]; ok { 110 | b.WriteString(s) 111 | b.WriteRune('\n') 112 | delete(m, s) 113 | } 114 | } 115 | return b.String() 116 | } 117 | 118 | func run(r Retryer, t Failer, f func(r *R)) { 119 | rr := &R{} 120 | fail := func() { 121 | out := dedup(rr.output) 122 | if out != "" { 123 | t.Log(out) 124 | } 125 | t.FailNow() 126 | } 127 | for r.NextOr(fail) { 128 | var wg sync.WaitGroup 129 | wg.Add(1) 130 | go func() { 131 | defer wg.Done() 132 | f(rr) 133 | }() 134 | wg.Wait() 135 | if rr.fail { 136 | rr.fail = false 137 | continue 138 | } 139 | break 140 | } 141 | } 142 | 143 | // DefaultFailer provides default retry.Run() behavior for unit tests. 144 | func DefaultFailer() *Timer { 145 | return &Timer{Timeout: 7 * time.Second, Wait: 25 * time.Millisecond} 146 | } 147 | 148 | // TwoSeconds repeats an operation for two seconds and waits 25ms in between. 149 | func TwoSeconds() *Timer { 150 | return &Timer{Timeout: 2 * time.Second, Wait: 25 * time.Millisecond} 151 | } 152 | 153 | // ThreeTimes repeats an operation three times and waits 25ms in between. 154 | func ThreeTimes() *Counter { 155 | return &Counter{Count: 3, Wait: 25 * time.Millisecond} 156 | } 157 | 158 | // Retryer provides an interface for repeating operations 159 | // until they succeed or an exit condition is met. 160 | type Retryer interface { 161 | // NextOr returns true if the operation should be repeated. 162 | // Otherwise, it calls fail and returns false. 163 | NextOr(fail func()) bool 164 | } 165 | 166 | // Counter repeats an operation a given number of 167 | // times and waits between subsequent operations. 168 | type Counter struct { 169 | Count int 170 | Wait time.Duration 171 | 172 | count int 173 | } 174 | 175 | func (r *Counter) NextOr(fail func()) bool { 176 | if r.count == r.Count { 177 | fail() 178 | return false 179 | } 180 | if r.count > 0 { 181 | time.Sleep(r.Wait) 182 | } 183 | r.count++ 184 | return true 185 | } 186 | 187 | // Timer repeats an operation for a given amount 188 | // of time and waits between subsequent operations. 189 | type Timer struct { 190 | Timeout time.Duration 191 | Wait time.Duration 192 | 193 | // stop is the timeout deadline. 194 | // Set on the first invocation of Next(). 195 | stop time.Time 196 | } 197 | 198 | func (r *Timer) NextOr(fail func()) bool { 199 | if r.stop.IsZero() { 200 | r.stop = time.Now().Add(r.Timeout) 201 | return true 202 | } 203 | if time.Now().After(r.stop) { 204 | fail() 205 | return false 206 | } 207 | time.Sleep(r.Wait) 208 | return true 209 | } 210 | -------------------------------------------------------------------------------- /internal/retry/retry_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package retry 5 | 6 | import ( 7 | "testing" 8 | "time" 9 | ) 10 | 11 | // delta defines the time band a test run should complete in. 12 | var delta = 25 * time.Millisecond 13 | 14 | func TestRetryer(t *testing.T) { 15 | tests := []struct { 16 | desc string 17 | r Retryer 18 | }{ 19 | {"counter", &Counter{Count: 3, Wait: 100 * time.Millisecond}}, 20 | {"timer", &Timer{Timeout: 200 * time.Millisecond, Wait: 100 * time.Millisecond}}, 21 | } 22 | 23 | for _, tt := range tests { 24 | t.Run(tt.desc, func(t *testing.T) { 25 | var iters, fails int 26 | fail := func() { fails++ } 27 | start := time.Now() 28 | for tt.r.NextOr(fail) { 29 | iters++ 30 | } 31 | dur := time.Since(start) 32 | if got, want := iters, 3; got != want { 33 | t.Fatalf("got %d retries want %d", got, want) 34 | } 35 | if got, want := fails, 1; got != want { 36 | t.Fatalf("got %d FailNow calls want %d", got, want) 37 | } 38 | // since the first iteration happens immediately 39 | // the retryer waits only twice for three iterations. 40 | // order of events: (true, (wait) true, (wait) true, false) 41 | if got, want := dur, 200*time.Millisecond; got < (want-delta) || got > (want+delta) { 42 | t.Fatalf("loop took %v want %v (+/- %v)", got, want, delta) 43 | } 44 | }) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /keyring.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "sync" 10 | ) 11 | 12 | type Keyring struct { 13 | // Keys stores the key data used during encryption and decryption. It is 14 | // ordered in such a way where the first key (index 0) is the primary key, 15 | // which is used for encrypting messages, and is the first key tried during 16 | // message decryption. 17 | keys [][]byte 18 | 19 | // The keyring lock is used while performing IO operations on the keyring. 20 | l sync.Mutex 21 | } 22 | 23 | // Init allocates substructures 24 | func (k *Keyring) init() { 25 | k.keys = make([][]byte, 0) 26 | } 27 | 28 | // NewKeyring constructs a new container for a set of encryption keys. The 29 | // keyring contains all key data used internally by memberlist. 30 | // 31 | // While creating a new keyring, you must do one of: 32 | // - Omit keys and primary key, effectively disabling encryption 33 | // - Pass a set of keys plus the primary key 34 | // - Pass only a primary key 35 | // 36 | // If only a primary key is passed, then it will be automatically added to the 37 | // keyring. If creating a keyring with multiple keys, one key must be designated 38 | // primary by passing it as the primaryKey. If the primaryKey does not exist in 39 | // the list of secondary keys, it will be automatically added at position 0. 40 | // 41 | // A key should be either 16, 24, or 32 bytes to select AES-128, 42 | // AES-192, or AES-256. 43 | func NewKeyring(keys [][]byte, primaryKey []byte) (*Keyring, error) { 44 | keyring := &Keyring{} 45 | keyring.init() 46 | 47 | if len(keys) > 0 || len(primaryKey) > 0 { 48 | if len(primaryKey) == 0 { 49 | return nil, fmt.Errorf("Empty primary key not allowed") 50 | } 51 | if err := keyring.AddKey(primaryKey); err != nil { 52 | return nil, err 53 | } 54 | for _, key := range keys { 55 | if err := keyring.AddKey(key); err != nil { 56 | return nil, err 57 | } 58 | } 59 | } 60 | 61 | return keyring, nil 62 | } 63 | 64 | // ValidateKey will check to see if the key is valid and returns an error if not. 65 | // 66 | // key should be either 16, 24, or 32 bytes to select AES-128, 67 | // AES-192, or AES-256. 68 | func ValidateKey(key []byte) error { 69 | if l := len(key); l != 16 && l != 24 && l != 32 { 70 | return fmt.Errorf("key size must be 16, 24 or 32 bytes") 71 | } 72 | return nil 73 | } 74 | 75 | // AddKey will install a new key on the ring. Adding a key to the ring will make 76 | // it available for use in decryption. If the key already exists on the ring, 77 | // this function will just return noop. 78 | // 79 | // key should be either 16, 24, or 32 bytes to select AES-128, 80 | // AES-192, or AES-256. 81 | func (k *Keyring) AddKey(key []byte) error { 82 | if err := ValidateKey(key); err != nil { 83 | return err 84 | } 85 | 86 | // No-op if key is already installed 87 | for _, installedKey := range k.keys { 88 | if bytes.Equal(installedKey, key) { 89 | return nil 90 | } 91 | } 92 | 93 | keys := append(k.keys, key) 94 | primaryKey := k.GetPrimaryKey() 95 | if primaryKey == nil { 96 | primaryKey = key 97 | } 98 | k.installKeys(keys, primaryKey) 99 | return nil 100 | } 101 | 102 | // UseKey changes the key used to encrypt messages. This is the only key used to 103 | // encrypt messages, so peers should know this key before this method is called. 104 | func (k *Keyring) UseKey(key []byte) error { 105 | for _, installedKey := range k.keys { 106 | if bytes.Equal(key, installedKey) { 107 | k.installKeys(k.keys, key) 108 | return nil 109 | } 110 | } 111 | return fmt.Errorf("Requested key is not in the keyring") 112 | } 113 | 114 | // RemoveKey drops a key from the keyring. This will return an error if the key 115 | // requested for removal is currently at position 0 (primary key). 116 | func (k *Keyring) RemoveKey(key []byte) error { 117 | if bytes.Equal(key, k.keys[0]) { 118 | return fmt.Errorf("Removing the primary key is not allowed") 119 | } 120 | for i, installedKey := range k.keys { 121 | if bytes.Equal(key, installedKey) { 122 | keys := append(k.keys[:i], k.keys[i+1:]...) 123 | k.installKeys(keys, k.keys[0]) 124 | } 125 | } 126 | return nil 127 | } 128 | 129 | // installKeys will take out a lock on the keyring, and replace the keys with a 130 | // new set of keys. The key indicated by primaryKey will be installed as the new 131 | // primary key. 132 | func (k *Keyring) installKeys(keys [][]byte, primaryKey []byte) { 133 | k.l.Lock() 134 | defer k.l.Unlock() 135 | 136 | newKeys := [][]byte{primaryKey} 137 | for _, key := range keys { 138 | if !bytes.Equal(key, primaryKey) { 139 | newKeys = append(newKeys, key) 140 | } 141 | } 142 | k.keys = newKeys 143 | } 144 | 145 | // GetKeys returns the current set of keys on the ring. 146 | func (k *Keyring) GetKeys() [][]byte { 147 | k.l.Lock() 148 | defer k.l.Unlock() 149 | 150 | return k.keys 151 | } 152 | 153 | // GetPrimaryKey returns the key on the ring at position 0. This is the key used 154 | // for encrypting messages, and is the first key tried for decrypting messages. 155 | func (k *Keyring) GetPrimaryKey() (key []byte) { 156 | k.l.Lock() 157 | defer k.l.Unlock() 158 | 159 | if len(k.keys) > 0 { 160 | key = k.keys[0] 161 | } 162 | return 163 | } 164 | -------------------------------------------------------------------------------- /keyring_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "bytes" 8 | "testing" 9 | ) 10 | 11 | var TestKeys [][]byte = [][]byte{ 12 | []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, 13 | []byte{15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}, 14 | []byte{8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7}, 15 | } 16 | 17 | func TestKeyring_EmptyRing(t *testing.T) { 18 | // Keyrings can be created with no encryption keys (disabled encryption) 19 | keyring, err := NewKeyring(nil, nil) 20 | if err != nil { 21 | t.Fatalf("err: %s", err) 22 | } 23 | 24 | keys := keyring.GetKeys() 25 | if len(keys) != 0 { 26 | t.Fatalf("Expected 0 keys but have %d", len(keys)) 27 | } 28 | } 29 | 30 | func TestKeyring_PrimaryOnly(t *testing.T) { 31 | // Keyrings can be created using only a primary key 32 | keyring, err := NewKeyring(nil, TestKeys[0]) 33 | if err != nil { 34 | t.Fatalf("err: %s", err) 35 | } 36 | 37 | keys := keyring.GetKeys() 38 | if len(keys) != 1 { 39 | t.Fatalf("Expected 1 key but have %d", len(keys)) 40 | } 41 | } 42 | 43 | func TestKeyring_GetPrimaryKey(t *testing.T) { 44 | keyring, err := NewKeyring(TestKeys, TestKeys[1]) 45 | if err != nil { 46 | t.Fatalf("err: %s", err) 47 | } 48 | 49 | // GetPrimaryKey returns correct key 50 | primaryKey := keyring.GetPrimaryKey() 51 | if !bytes.Equal(primaryKey, TestKeys[1]) { 52 | t.Fatalf("Unexpected primary key: %v", primaryKey) 53 | } 54 | } 55 | 56 | func TestKeyring_AddRemoveUse(t *testing.T) { 57 | keyring, err := NewKeyring(nil, TestKeys[1]) 58 | if err != nil { 59 | t.Fatalf("err :%s", err) 60 | } 61 | 62 | // Use non-existent key throws error 63 | if err := keyring.UseKey(TestKeys[2]); err == nil { 64 | t.Fatalf("Expected key not installed error") 65 | } 66 | 67 | // Add key to ring 68 | if err := keyring.AddKey(TestKeys[2]); err != nil { 69 | t.Fatalf("err: %s", err) 70 | } 71 | 72 | keys := keyring.GetKeys() 73 | if !bytes.Equal(keys[0], TestKeys[1]) { 74 | t.Fatalf("Unexpected primary key change") 75 | } 76 | 77 | if len(keys) != 2 { 78 | t.Fatalf("Expected 2 keys but have %d", len(keys)) 79 | } 80 | 81 | // Use key that exists should succeed 82 | if err := keyring.UseKey(TestKeys[2]); err != nil { 83 | t.Fatalf("err: %s", err) 84 | } 85 | 86 | primaryKey := keyring.GetPrimaryKey() 87 | if !bytes.Equal(primaryKey, TestKeys[2]) { 88 | t.Fatalf("Unexpected primary key: %v", primaryKey) 89 | } 90 | 91 | // Removing primary key should fail 92 | if err := keyring.RemoveKey(TestKeys[2]); err == nil { 93 | t.Fatalf("Expected primary key removal error") 94 | } 95 | 96 | // Removing non-primary key should succeed 97 | if err := keyring.RemoveKey(TestKeys[1]); err != nil { 98 | t.Fatalf("err: %s", err) 99 | } 100 | 101 | keys = keyring.GetKeys() 102 | if len(keys) != 1 { 103 | t.Fatalf("Expected 1 key but have %d", len(keys)) 104 | } 105 | } 106 | 107 | func TestKeyRing_MultiKeyEncryptDecrypt(t *testing.T) { 108 | plaintext := []byte("this is a plain text message") 109 | extra := []byte("random data") 110 | 111 | keyring, err := NewKeyring(TestKeys, TestKeys[0]) 112 | if err != nil { 113 | t.Fatalf("err: %s", err) 114 | } 115 | 116 | // First encrypt using the primary key and make sure we can decrypt 117 | var buf bytes.Buffer 118 | err = encryptPayload(1, TestKeys[0], plaintext, extra, &buf) 119 | if err != nil { 120 | t.Fatalf("err: %v", err) 121 | } 122 | 123 | msg, err := decryptPayload(keyring.GetKeys(), buf.Bytes(), extra) 124 | if err != nil { 125 | t.Fatalf("err: %v", err) 126 | } 127 | 128 | if !bytes.Equal(msg, plaintext) { 129 | t.Fatalf("bad: %v", msg) 130 | } 131 | 132 | // Now encrypt with a secondary key and try decrypting again. 133 | buf.Reset() 134 | err = encryptPayload(1, TestKeys[2], plaintext, extra, &buf) 135 | if err != nil { 136 | t.Fatalf("err: %v", err) 137 | } 138 | 139 | msg, err = decryptPayload(keyring.GetKeys(), buf.Bytes(), extra) 140 | if err != nil { 141 | t.Fatalf("err: %v", err) 142 | } 143 | 144 | if !bytes.Equal(msg, plaintext) { 145 | t.Fatalf("bad: %v", msg) 146 | } 147 | 148 | // Remove a key from the ring, and then try decrypting again 149 | if err := keyring.RemoveKey(TestKeys[2]); err != nil { 150 | t.Fatalf("err: %s", err) 151 | } 152 | 153 | msg, err = decryptPayload(keyring.GetKeys(), buf.Bytes(), extra) 154 | if err == nil { 155 | t.Fatalf("Expected no keys to decrypt message") 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /label.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "bufio" 8 | "fmt" 9 | "io" 10 | "net" 11 | ) 12 | 13 | // General approach is to prefix all packets and streams with the same structure: 14 | // 15 | // magic type byte (244): uint8 16 | // length of label name: uint8 (because labels can't be longer than 255 bytes) 17 | // label name: []uint8 18 | 19 | // LabelMaxSize is the maximum length of a packet or stream label. 20 | const LabelMaxSize = 255 21 | 22 | // AddLabelHeaderToPacket prefixes outgoing packets with the correct header if 23 | // the label is not empty. 24 | func AddLabelHeaderToPacket(buf []byte, label string) ([]byte, error) { 25 | if label == "" { 26 | return buf, nil 27 | } 28 | if len(label) > LabelMaxSize { 29 | return nil, fmt.Errorf("label %q is too long", label) 30 | } 31 | 32 | return makeLabelHeader(label, buf), nil 33 | } 34 | 35 | // RemoveLabelHeaderFromPacket removes any label header from the provided 36 | // packet and returns it along with the remaining packet contents. 37 | func RemoveLabelHeaderFromPacket(buf []byte) (newBuf []byte, label string, err error) { 38 | if len(buf) == 0 { 39 | return buf, "", nil // can't possibly be labeled 40 | } 41 | 42 | // [type:byte] [size:byte] [size bytes] 43 | 44 | msgType := messageType(buf[0]) 45 | if msgType != hasLabelMsg { 46 | return buf, "", nil 47 | } 48 | 49 | if len(buf) < 2 { 50 | return nil, "", fmt.Errorf("cannot decode label; packet has been truncated") 51 | } 52 | 53 | size := int(buf[1]) 54 | if size < 1 { 55 | return nil, "", fmt.Errorf("label header cannot be empty when present") 56 | } 57 | 58 | if len(buf) < 2+size { 59 | return nil, "", fmt.Errorf("cannot decode label; packet has been truncated") 60 | } 61 | 62 | label = string(buf[2 : 2+size]) 63 | newBuf = buf[2+size:] 64 | 65 | return newBuf, label, nil 66 | } 67 | 68 | // AddLabelHeaderToStream prefixes outgoing streams with the correct header if 69 | // the label is not empty. 70 | func AddLabelHeaderToStream(conn net.Conn, label string) error { 71 | if label == "" { 72 | return nil 73 | } 74 | if len(label) > LabelMaxSize { 75 | return fmt.Errorf("label %q is too long", label) 76 | } 77 | 78 | header := makeLabelHeader(label, nil) 79 | 80 | _, err := conn.Write(header) 81 | return err 82 | } 83 | 84 | // RemoveLabelHeaderFromStream removes any label header from the beginning of 85 | // the stream if present and returns it along with an updated conn with that 86 | // header removed. 87 | // 88 | // Note that on error it is the caller's responsibility to close the 89 | // connection. 90 | func RemoveLabelHeaderFromStream(conn net.Conn) (net.Conn, string, error) { 91 | br := bufio.NewReader(conn) 92 | 93 | // First check for the type byte. 94 | peeked, err := br.Peek(1) 95 | if err != nil { 96 | if err == io.EOF { 97 | // It is safe to return the original net.Conn at this point because 98 | // it never contained any data in the first place so we don't have 99 | // to splice the buffer into the conn because both are empty. 100 | return conn, "", nil 101 | } 102 | return nil, "", err 103 | } 104 | 105 | msgType := messageType(peeked[0]) 106 | if msgType != hasLabelMsg { 107 | conn, err = newPeekedConnFromBufferedReader(conn, br, 0) 108 | return conn, "", err 109 | } 110 | 111 | // We are guaranteed to get a size byte as well. 112 | peeked, err = br.Peek(2) 113 | if err != nil { 114 | if err == io.EOF { 115 | return nil, "", fmt.Errorf("cannot decode label; stream has been truncated") 116 | } 117 | return nil, "", err 118 | } 119 | 120 | size := int(peeked[1]) 121 | if size < 1 { 122 | return nil, "", fmt.Errorf("label header cannot be empty when present") 123 | } 124 | // NOTE: we don't have to check this against LabelMaxSize because a byte 125 | // already has a max value of 255. 126 | 127 | // Once we know the size we can peek the label as well. Note that since we 128 | // are using the default bufio.Reader size of 4096, the entire label header 129 | // fits in the initial buffer fill so this should be free. 130 | peeked, err = br.Peek(2 + size) 131 | if err != nil { 132 | if err == io.EOF { 133 | return nil, "", fmt.Errorf("cannot decode label; stream has been truncated") 134 | } 135 | return nil, "", err 136 | } 137 | 138 | label := string(peeked[2 : 2+size]) 139 | 140 | conn, err = newPeekedConnFromBufferedReader(conn, br, 2+size) 141 | if err != nil { 142 | return nil, "", err 143 | } 144 | 145 | return conn, label, nil 146 | } 147 | 148 | // newPeekedConnFromBufferedReader will splice the buffer contents after the 149 | // offset into the provided net.Conn and return the result so that the rest of 150 | // the buffer contents are returned first when reading from the returned 151 | // peekedConn before moving on to the unbuffered conn contents. 152 | func newPeekedConnFromBufferedReader(conn net.Conn, br *bufio.Reader, offset int) (*peekedConn, error) { 153 | // Extract any of the readahead buffer. 154 | peeked, err := br.Peek(br.Buffered()) 155 | if err != nil { 156 | return nil, err 157 | } 158 | 159 | return &peekedConn{ 160 | Peeked: peeked[offset:], 161 | Conn: conn, 162 | }, nil 163 | } 164 | 165 | func makeLabelHeader(label string, rest []byte) []byte { 166 | newBuf := make([]byte, 2, 2+len(label)+len(rest)) 167 | newBuf[0] = byte(hasLabelMsg) 168 | newBuf[1] = byte(len(label)) 169 | newBuf = append(newBuf, []byte(label)...) 170 | if len(rest) > 0 { 171 | newBuf = append(newBuf, []byte(rest)...) 172 | } 173 | return newBuf 174 | } 175 | 176 | func labelOverhead(label string) int { 177 | if label == "" { 178 | return 0 179 | } 180 | return 2 + len(label) 181 | } 182 | -------------------------------------------------------------------------------- /label_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "bytes" 8 | "io" 9 | "net" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func TestAddLabelHeaderToPacket(t *testing.T) { 17 | type testcase struct { 18 | buf []byte 19 | label string 20 | expectPacket []byte 21 | expectErr string 22 | } 23 | 24 | run := func(t *testing.T, tc testcase) { 25 | got, err := AddLabelHeaderToPacket(tc.buf, tc.label) 26 | if tc.expectErr != "" { 27 | require.Error(t, err) 28 | require.Contains(t, err.Error(), tc.expectErr) 29 | } else { 30 | require.NoError(t, err) 31 | require.Equal(t, tc.expectPacket, got) 32 | } 33 | } 34 | 35 | longLabel := strings.Repeat("a", 255) 36 | 37 | cases := map[string]testcase{ 38 | "nil buf with no label": testcase{ 39 | buf: nil, 40 | label: "", 41 | expectPacket: nil, 42 | }, 43 | "nil buf with label": testcase{ 44 | buf: nil, 45 | label: "foo", 46 | expectPacket: append([]byte{byte(hasLabelMsg), 3}, []byte("foo")...), 47 | }, 48 | "message with label": testcase{ 49 | buf: []byte("something"), 50 | label: "foo", 51 | expectPacket: append([]byte{byte(hasLabelMsg), 3}, []byte("foosomething")...), 52 | }, 53 | "message with no label": testcase{ 54 | buf: []byte("something"), 55 | label: "", 56 | expectPacket: []byte("something"), 57 | }, 58 | "message with almost too long label": testcase{ 59 | buf: []byte("something"), 60 | label: longLabel, 61 | expectPacket: append([]byte{byte(hasLabelMsg), 255}, []byte(longLabel+"something")...), 62 | }, 63 | "label too long by one byte": testcase{ 64 | buf: []byte("something"), 65 | label: longLabel + "x", 66 | expectErr: `label "` + longLabel + `x" is too long`, 67 | }, 68 | } 69 | 70 | for name, tc := range cases { 71 | t.Run(name, func(t *testing.T) { 72 | run(t, tc) 73 | }) 74 | } 75 | } 76 | 77 | func TestRemoveLabelHeaderFromPacket(t *testing.T) { 78 | type testcase struct { 79 | buf []byte 80 | expectLabel string 81 | expectPacket []byte 82 | expectErr string 83 | } 84 | 85 | run := func(t *testing.T, tc testcase) { 86 | gotBuf, gotLabel, err := RemoveLabelHeaderFromPacket(tc.buf) 87 | if tc.expectErr != "" { 88 | require.Error(t, err) 89 | require.Contains(t, err.Error(), tc.expectErr) 90 | } else { 91 | require.NoError(t, err) 92 | require.Equal(t, tc.expectPacket, gotBuf) 93 | require.Equal(t, tc.expectLabel, gotLabel) 94 | } 95 | } 96 | 97 | cases := map[string]testcase{ 98 | "empty buf": testcase{ 99 | buf: []byte{}, 100 | expectLabel: "", 101 | expectPacket: []byte{}, 102 | }, 103 | "ping with no label": testcase{ 104 | buf: buildBuffer(t, pingMsg, "blah"), 105 | expectLabel: "", 106 | expectPacket: buildBuffer(t, pingMsg, "blah"), 107 | }, 108 | "error with no label": testcase{ // 2021-10: largest standard message type 109 | buf: buildBuffer(t, errMsg, "blah"), 110 | expectLabel: "", 111 | expectPacket: buildBuffer(t, errMsg, "blah"), 112 | }, 113 | "v1 encrypt with no label": testcase{ // 2021-10: highest encryption version 114 | buf: buildBuffer(t, maxEncryptionVersion, "blah"), 115 | expectLabel: "", 116 | expectPacket: buildBuffer(t, maxEncryptionVersion, "blah"), 117 | }, 118 | "buf too small for label": testcase{ 119 | buf: buildBuffer(t, hasLabelMsg, "x"), 120 | expectErr: `cannot decode label; packet has been truncated`, 121 | }, 122 | "buf too small for label size": testcase{ 123 | buf: buildBuffer(t, hasLabelMsg), 124 | expectErr: `cannot decode label; packet has been truncated`, 125 | }, 126 | "label empty": testcase{ 127 | buf: buildBuffer(t, hasLabelMsg, 0, "x"), 128 | expectErr: `label header cannot be empty when present`, 129 | }, 130 | "label truncated": testcase{ 131 | buf: buildBuffer(t, hasLabelMsg, 2, "x"), 132 | expectErr: `cannot decode label; packet has been truncated`, 133 | }, 134 | "ping with label": testcase{ 135 | buf: buildBuffer(t, hasLabelMsg, 3, "abc", pingMsg, "blah"), 136 | expectLabel: "abc", 137 | expectPacket: buildBuffer(t, pingMsg, "blah"), 138 | }, 139 | "error with label": testcase{ // 2021-10: largest standard message type 140 | buf: buildBuffer(t, hasLabelMsg, 3, "abc", errMsg, "blah"), 141 | expectLabel: "abc", 142 | expectPacket: buildBuffer(t, errMsg, "blah"), 143 | }, 144 | "v1 encrypt with label": testcase{ // 2021-10: highest encryption version 145 | buf: buildBuffer(t, hasLabelMsg, 3, "abc", maxEncryptionVersion, "blah"), 146 | expectLabel: "abc", 147 | expectPacket: buildBuffer(t, maxEncryptionVersion, "blah"), 148 | }, 149 | } 150 | 151 | for name, tc := range cases { 152 | t.Run(name, func(t *testing.T) { 153 | run(t, tc) 154 | }) 155 | } 156 | } 157 | 158 | func TestAddLabelHeaderToStream(t *testing.T) { 159 | type testcase struct { 160 | label string 161 | expectData []byte 162 | expectErr string 163 | } 164 | 165 | suffixData := "EXTRA DATA" 166 | 167 | run := func(t *testing.T, tc testcase) { 168 | server, client := net.Pipe() 169 | defer server.Close() 170 | defer client.Close() 171 | 172 | var ( 173 | dataCh = make(chan []byte, 1) 174 | errCh = make(chan error, 1) 175 | ) 176 | go func() { 177 | var buf bytes.Buffer 178 | _, err := io.Copy(&buf, server) 179 | if err != nil { 180 | errCh <- err 181 | } 182 | dataCh <- buf.Bytes() 183 | }() 184 | 185 | err := AddLabelHeaderToStream(client, tc.label) 186 | if tc.expectErr != "" { 187 | require.Error(t, err) 188 | require.Contains(t, err.Error(), tc.expectErr) 189 | return 190 | } 191 | require.NoError(t, err) 192 | 193 | client.Write([]byte(suffixData)) 194 | client.Close() 195 | 196 | expect := make([]byte, 0, len(suffixData)+len(tc.expectData)) 197 | expect = append(expect, tc.expectData...) 198 | expect = append(expect, suffixData...) 199 | 200 | select { 201 | case err := <-errCh: 202 | require.NoError(t, err) 203 | case got := <-dataCh: 204 | require.Equal(t, expect, got) 205 | } 206 | } 207 | 208 | longLabel := strings.Repeat("a", 255) 209 | 210 | cases := map[string]testcase{ 211 | "no label": testcase{ 212 | label: "", 213 | expectData: nil, 214 | }, 215 | "with label": testcase{ 216 | label: "foo", 217 | expectData: buildBuffer(t, hasLabelMsg, 3, "foo"), 218 | }, 219 | "almost too long label": testcase{ 220 | label: longLabel, 221 | expectData: buildBuffer(t, hasLabelMsg, 255, longLabel), 222 | }, 223 | "label too long by one byte": testcase{ 224 | label: longLabel + "x", 225 | expectErr: `label "` + longLabel + `x" is too long`, 226 | }, 227 | } 228 | 229 | for name, tc := range cases { 230 | t.Run(name, func(t *testing.T) { 231 | run(t, tc) 232 | }) 233 | } 234 | } 235 | 236 | func TestRemoveLabelHeaderFromStream(t *testing.T) { 237 | type testcase struct { 238 | buf []byte 239 | expectLabel string 240 | expectData []byte 241 | expectErr string 242 | } 243 | 244 | run := func(t *testing.T, tc testcase) { 245 | server, client := net.Pipe() 246 | defer server.Close() 247 | defer client.Close() 248 | 249 | var ( 250 | errCh = make(chan error, 1) 251 | ) 252 | go func() { 253 | _, err := server.Write(tc.buf) 254 | if err != nil { 255 | errCh <- err 256 | } 257 | server.Close() 258 | }() 259 | 260 | newConn, gotLabel, err := RemoveLabelHeaderFromStream(client) 261 | if tc.expectErr != "" { 262 | require.Error(t, err) 263 | require.Contains(t, err.Error(), tc.expectErr) 264 | return 265 | } 266 | require.NoError(t, err) 267 | 268 | gotBuf, err := io.ReadAll(newConn) 269 | require.NoError(t, err) 270 | 271 | require.Equal(t, tc.expectData, gotBuf) 272 | require.Equal(t, tc.expectLabel, gotLabel) 273 | } 274 | 275 | cases := map[string]testcase{ 276 | "empty buf": testcase{ 277 | buf: []byte{}, 278 | expectLabel: "", 279 | expectData: []byte{}, 280 | }, 281 | "ping with no label": testcase{ 282 | buf: buildBuffer(t, pingMsg, "blah"), 283 | expectLabel: "", 284 | expectData: buildBuffer(t, pingMsg, "blah"), 285 | }, 286 | "error with no label": testcase{ // 2021-10: largest standard message type 287 | buf: buildBuffer(t, errMsg, "blah"), 288 | expectLabel: "", 289 | expectData: buildBuffer(t, errMsg, "blah"), 290 | }, 291 | "v1 encrypt with no label": testcase{ // 2021-10: highest encryption version 292 | buf: buildBuffer(t, maxEncryptionVersion, "blah"), 293 | expectLabel: "", 294 | expectData: buildBuffer(t, maxEncryptionVersion, "blah"), 295 | }, 296 | "buf too small for label": testcase{ 297 | buf: buildBuffer(t, hasLabelMsg, "x"), 298 | expectErr: `cannot decode label; stream has been truncated`, 299 | }, 300 | "buf too small for label size": testcase{ 301 | buf: buildBuffer(t, hasLabelMsg), 302 | expectErr: `cannot decode label; stream has been truncated`, 303 | }, 304 | "label empty": testcase{ 305 | buf: buildBuffer(t, hasLabelMsg, 0, "x"), 306 | expectErr: `label header cannot be empty when present`, 307 | }, 308 | "label truncated": testcase{ 309 | buf: buildBuffer(t, hasLabelMsg, 2, "x"), 310 | expectErr: `cannot decode label; stream has been truncated`, 311 | }, 312 | "ping with label": testcase{ 313 | buf: buildBuffer(t, hasLabelMsg, 3, "abc", pingMsg, "blah"), 314 | expectLabel: "abc", 315 | expectData: buildBuffer(t, pingMsg, "blah"), 316 | }, 317 | "error with label": testcase{ // 2021-10: largest standard message type 318 | buf: buildBuffer(t, hasLabelMsg, 3, "abc", errMsg, "blah"), 319 | expectLabel: "abc", 320 | expectData: buildBuffer(t, errMsg, "blah"), 321 | }, 322 | "v1 encrypt with label": testcase{ // 2021-10: highest encryption version 323 | buf: buildBuffer(t, hasLabelMsg, 3, "abc", maxEncryptionVersion, "blah"), 324 | expectLabel: "abc", 325 | expectData: buildBuffer(t, maxEncryptionVersion, "blah"), 326 | }, 327 | } 328 | 329 | for name, tc := range cases { 330 | t.Run(name, func(t *testing.T) { 331 | run(t, tc) 332 | }) 333 | } 334 | } 335 | 336 | func buildBuffer(t *testing.T, stuff ...interface{}) []byte { 337 | var buf bytes.Buffer 338 | for _, item := range stuff { 339 | switch x := item.(type) { 340 | case int: 341 | x2 := uint(x) 342 | if x2 > 255 { 343 | t.Fatalf("int is too big") 344 | } 345 | buf.WriteByte(byte(x2)) 346 | case byte: 347 | buf.WriteByte(byte(x)) 348 | case messageType: 349 | buf.WriteByte(byte(x)) 350 | case encryptionVersion: 351 | buf.WriteByte(byte(x)) 352 | case string: 353 | buf.Write([]byte(x)) 354 | case []byte: 355 | buf.Write(x) 356 | default: 357 | t.Fatalf("unexpected type %T", item) 358 | } 359 | } 360 | return buf.Bytes() 361 | } 362 | 363 | func TestLabelOverhead(t *testing.T) { 364 | require.Equal(t, 0, labelOverhead("")) 365 | require.Equal(t, 3, labelOverhead("a")) 366 | require.Equal(t, 9, labelOverhead("abcdefg")) 367 | } 368 | -------------------------------------------------------------------------------- /logging.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "fmt" 8 | "net" 9 | ) 10 | 11 | func LogAddress(addr net.Addr) string { 12 | if addr == nil { 13 | return "from=" 14 | } 15 | 16 | return fmt.Sprintf("from=%s", addr.String()) 17 | } 18 | 19 | func LogStringAddress(addr string) string { 20 | if addr == "" { 21 | return "from=" 22 | } 23 | 24 | return fmt.Sprintf("from=%s", addr) 25 | } 26 | 27 | func LogConn(conn net.Conn) string { 28 | if conn == nil { 29 | return LogAddress(nil) 30 | } 31 | 32 | return LogAddress(conn.RemoteAddr()) 33 | } 34 | -------------------------------------------------------------------------------- /logging_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "fmt" 8 | "net" 9 | "testing" 10 | ) 11 | 12 | func TestLogging_Address(t *testing.T) { 13 | s := LogAddress(nil) 14 | if s != "from=" { 15 | t.Fatalf("bad: %s", s) 16 | } 17 | 18 | addr, err := net.ResolveIPAddr("ip4", "127.0.0.1") 19 | if err != nil { 20 | t.Fatalf("err: %v", err) 21 | } 22 | 23 | s = LogAddress(addr) 24 | if s != "from=127.0.0.1" { 25 | t.Fatalf("bad: %s", s) 26 | } 27 | } 28 | 29 | func TestLogging_Conn(t *testing.T) { 30 | s := LogConn(nil) 31 | if s != "from=" { 32 | t.Fatalf("bad: %s", s) 33 | } 34 | 35 | ln, err := net.Listen("tcp", ":0") 36 | if err != nil { 37 | t.Fatalf("err: %v", err) 38 | } 39 | 40 | conn, err := net.Dial("tcp", ln.Addr().String()) 41 | if err != nil { 42 | t.Fatalf("err: %v", err) 43 | } 44 | defer conn.Close() 45 | 46 | s = LogConn(conn) 47 | if s != fmt.Sprintf("from=%s", conn.RemoteAddr().String()) { 48 | t.Fatalf("bad: %s", s) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /merge_delegate.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | // MergeDelegate is used to involve a client in 7 | // a potential cluster merge operation. Namely, when 8 | // a node does a TCP push/pull (as part of a join), 9 | // the delegate is involved and allowed to cancel the join 10 | // based on custom logic. The merge delegate is NOT invoked 11 | // as part of the push-pull anti-entropy. 12 | type MergeDelegate interface { 13 | // NotifyMerge is invoked when a merge could take place. 14 | // Provides a list of the nodes known by the peer. If 15 | // the return value is non-nil, the merge is canceled. 16 | NotifyMerge(peers []*Node) error 17 | } 18 | -------------------------------------------------------------------------------- /mock_transport.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "io" 10 | "net" 11 | "strconv" 12 | "time" 13 | ) 14 | 15 | // MockNetwork is used as a factory that produces MockTransport instances which 16 | // are uniquely addressed and wired up to talk to each other. 17 | type MockNetwork struct { 18 | transportsByAddr map[string]*MockTransport 19 | transportsByName map[string]*MockTransport 20 | port int 21 | } 22 | 23 | // NewTransport returns a new MockTransport with a unique address, wired up to 24 | // talk to the other transports in the MockNetwork. 25 | func (n *MockNetwork) NewTransport(name string) *MockTransport { 26 | n.port += 1 27 | addr := fmt.Sprintf("127.0.0.1:%d", n.port) 28 | transport := &MockTransport{ 29 | net: n, 30 | addr: &MockAddress{addr, name}, 31 | packetCh: make(chan *Packet), 32 | streamCh: make(chan net.Conn), 33 | } 34 | 35 | if n.transportsByAddr == nil { 36 | n.transportsByAddr = make(map[string]*MockTransport) 37 | } 38 | n.transportsByAddr[addr] = transport 39 | 40 | if n.transportsByName == nil { 41 | n.transportsByName = make(map[string]*MockTransport) 42 | } 43 | n.transportsByName[name] = transport 44 | 45 | return transport 46 | } 47 | 48 | // MockAddress is a wrapper which adds the net.Addr interface to our mock 49 | // address scheme. 50 | type MockAddress struct { 51 | addr string 52 | name string 53 | } 54 | 55 | // See net.Addr. 56 | func (a *MockAddress) Network() string { 57 | return "mock" 58 | } 59 | 60 | // See net.Addr. 61 | func (a *MockAddress) String() string { 62 | return a.addr 63 | } 64 | 65 | // MockTransport directly plumbs messages to other transports its MockNetwork. 66 | type MockTransport struct { 67 | net *MockNetwork 68 | addr *MockAddress 69 | packetCh chan *Packet 70 | streamCh chan net.Conn 71 | } 72 | 73 | var _ NodeAwareTransport = (*MockTransport)(nil) 74 | 75 | // See Transport. 76 | func (t *MockTransport) FinalAdvertiseAddr(string, int) (net.IP, int, error) { 77 | host, portStr, err := net.SplitHostPort(t.addr.String()) 78 | if err != nil { 79 | return nil, 0, err 80 | } 81 | 82 | ip := net.ParseIP(host) 83 | if ip == nil { 84 | return nil, 0, fmt.Errorf("Failed to parse IP %q", host) 85 | } 86 | 87 | port, err := strconv.ParseInt(portStr, 10, 16) 88 | if err != nil { 89 | return nil, 0, err 90 | } 91 | 92 | return ip, int(port), nil 93 | } 94 | 95 | // See Transport. 96 | func (t *MockTransport) WriteTo(b []byte, addr string) (time.Time, error) { 97 | a := Address{Addr: addr, Name: ""} 98 | return t.WriteToAddress(b, a) 99 | } 100 | 101 | // See NodeAwareTransport. 102 | func (t *MockTransport) WriteToAddress(b []byte, a Address) (time.Time, error) { 103 | dest, err := t.getPeer(a) 104 | if err != nil { 105 | return time.Time{}, err 106 | } 107 | 108 | now := time.Now() 109 | dest.packetCh <- &Packet{ 110 | Buf: b, 111 | From: t.addr, 112 | Timestamp: now, 113 | } 114 | return now, nil 115 | } 116 | 117 | // See Transport. 118 | func (t *MockTransport) PacketCh() <-chan *Packet { 119 | return t.packetCh 120 | } 121 | 122 | // See NodeAwareTransport. 123 | func (t *MockTransport) IngestPacket(conn net.Conn, addr net.Addr, now time.Time, shouldClose bool) error { 124 | if shouldClose { 125 | defer conn.Close() 126 | } 127 | 128 | // Copy everything from the stream into packet buffer. 129 | var buf bytes.Buffer 130 | if _, err := io.Copy(&buf, conn); err != nil { 131 | return fmt.Errorf("failed to read packet: %v", err) 132 | } 133 | 134 | // Check the length - it needs to have at least one byte to be a proper 135 | // message. This is checked elsewhere for writes coming in directly from 136 | // the UDP socket. 137 | if n := buf.Len(); n < 1 { 138 | return fmt.Errorf("packet too short (%d bytes) %s", n, LogAddress(addr)) 139 | } 140 | 141 | // Inject the packet. 142 | t.packetCh <- &Packet{ 143 | Buf: buf.Bytes(), 144 | From: addr, 145 | Timestamp: now, 146 | } 147 | return nil 148 | } 149 | 150 | // See Transport. 151 | func (t *MockTransport) DialTimeout(addr string, timeout time.Duration) (net.Conn, error) { 152 | a := Address{Addr: addr, Name: ""} 153 | return t.DialAddressTimeout(a, timeout) 154 | } 155 | 156 | // See NodeAwareTransport. 157 | func (t *MockTransport) DialAddressTimeout(a Address, timeout time.Duration) (net.Conn, error) { 158 | dest, err := t.getPeer(a) 159 | if err != nil { 160 | return nil, err 161 | } 162 | 163 | p1, p2 := net.Pipe() 164 | dest.streamCh <- p1 165 | return p2, nil 166 | } 167 | 168 | // See Transport. 169 | func (t *MockTransport) StreamCh() <-chan net.Conn { 170 | return t.streamCh 171 | } 172 | 173 | // See NodeAwareTransport. 174 | func (t *MockTransport) IngestStream(conn net.Conn) error { 175 | t.streamCh <- conn 176 | return nil 177 | } 178 | 179 | // See Transport. 180 | func (t *MockTransport) Shutdown() error { 181 | return nil 182 | } 183 | 184 | func (t *MockTransport) getPeer(a Address) (*MockTransport, error) { 185 | var ( 186 | dest *MockTransport 187 | ok bool 188 | ) 189 | if a.Name != "" { 190 | dest, ok = t.net.transportsByName[a.Name] 191 | } else { 192 | dest, ok = t.net.transportsByAddr[a.Addr] 193 | } 194 | if !ok { 195 | return nil, fmt.Errorf("No route to %s", a) 196 | } 197 | return dest, nil 198 | } 199 | -------------------------------------------------------------------------------- /net_transport.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "io" 10 | "log" 11 | "net" 12 | "sync" 13 | "sync/atomic" 14 | "time" 15 | 16 | "github.com/hashicorp/go-metrics/compat" 17 | sockaddr "github.com/hashicorp/go-sockaddr" 18 | ) 19 | 20 | const ( 21 | // udpPacketBufSize is used to buffer incoming packets during read 22 | // operations. 23 | udpPacketBufSize = 65536 24 | 25 | // udpRecvBufSize is a large buffer size that we attempt to set UDP 26 | // sockets to in order to handle a large volume of messages. 27 | udpRecvBufSize = 2 * 1024 * 1024 28 | ) 29 | 30 | // NetTransportConfig is used to configure a net transport. 31 | type NetTransportConfig struct { 32 | // BindAddrs is a list of addresses to bind to for both TCP and UDP 33 | // communications. 34 | BindAddrs []string 35 | 36 | // BindPort is the port to listen on, for each address above. 37 | BindPort int 38 | 39 | // Logger is a logger for operator messages. 40 | Logger *log.Logger 41 | 42 | // MetricLabels is a map of optional labels to apply to all metrics 43 | // emitted by this transport. 44 | MetricLabels []metrics.Label 45 | } 46 | 47 | // NetTransport is a Transport implementation that uses connectionless UDP for 48 | // packet operations, and ad-hoc TCP connections for stream operations. 49 | type NetTransport struct { 50 | config *NetTransportConfig 51 | packetCh chan *Packet 52 | streamCh chan net.Conn 53 | logger *log.Logger 54 | wg sync.WaitGroup 55 | tcpListeners []*net.TCPListener 56 | udpListeners []*net.UDPConn 57 | shutdown int32 58 | 59 | metricLabels []metrics.Label 60 | } 61 | 62 | var _ NodeAwareTransport = (*NetTransport)(nil) 63 | 64 | // NewNetTransport returns a net transport with the given configuration. On 65 | // success all the network listeners will be created and listening. 66 | func NewNetTransport(config *NetTransportConfig) (*NetTransport, error) { 67 | // If we reject the empty list outright we can assume that there's at 68 | // least one listener of each type later during operation. 69 | if len(config.BindAddrs) == 0 { 70 | return nil, fmt.Errorf("At least one bind address is required") 71 | } 72 | 73 | // Build out the new transport. 74 | var ok bool 75 | t := NetTransport{ 76 | config: config, 77 | packetCh: make(chan *Packet), 78 | streamCh: make(chan net.Conn), 79 | logger: config.Logger, 80 | metricLabels: config.MetricLabels, 81 | } 82 | 83 | // Clean up listeners if there's an error. 84 | defer func() { 85 | if !ok { 86 | t.Shutdown() 87 | } 88 | }() 89 | 90 | // Build all the TCP and UDP listeners. 91 | port := config.BindPort 92 | for _, addr := range config.BindAddrs { 93 | ip := net.ParseIP(addr) 94 | 95 | tcpAddr := &net.TCPAddr{IP: ip, Port: port} 96 | tcpLn, err := net.ListenTCP("tcp", tcpAddr) 97 | if err != nil { 98 | return nil, fmt.Errorf("Failed to start TCP listener on %q port %d: %v", addr, port, err) 99 | } 100 | t.tcpListeners = append(t.tcpListeners, tcpLn) 101 | 102 | // If the config port given was zero, use the first TCP listener 103 | // to pick an available port and then apply that to everything 104 | // else. 105 | if port == 0 { 106 | port = tcpLn.Addr().(*net.TCPAddr).Port 107 | } 108 | 109 | udpAddr := &net.UDPAddr{IP: ip, Port: port} 110 | udpLn, err := net.ListenUDP("udp", udpAddr) 111 | if err != nil { 112 | return nil, fmt.Errorf("Failed to start UDP listener on %q port %d: %v", addr, port, err) 113 | } 114 | if err := setUDPRecvBuf(udpLn); err != nil { 115 | return nil, fmt.Errorf("Failed to resize UDP buffer: %v", err) 116 | } 117 | t.udpListeners = append(t.udpListeners, udpLn) 118 | } 119 | 120 | // Fire them up now that we've been able to create them all. 121 | for i := 0; i < len(config.BindAddrs); i++ { 122 | t.wg.Add(2) 123 | go t.tcpListen(t.tcpListeners[i]) 124 | go t.udpListen(t.udpListeners[i]) 125 | } 126 | 127 | ok = true 128 | return &t, nil 129 | } 130 | 131 | // GetAutoBindPort returns the bind port that was automatically given by the 132 | // kernel, if a bind port of 0 was given. 133 | func (t *NetTransport) GetAutoBindPort() int { 134 | // We made sure there's at least one TCP listener, and that one's 135 | // port was applied to all the others for the dynamic bind case. 136 | return t.tcpListeners[0].Addr().(*net.TCPAddr).Port 137 | } 138 | 139 | // See Transport. 140 | func (t *NetTransport) FinalAdvertiseAddr(ip string, port int) (net.IP, int, error) { 141 | var advertiseAddr net.IP 142 | var advertisePort int 143 | if ip != "" { 144 | // If they've supplied an address, use that. 145 | advertiseAddr = net.ParseIP(ip) 146 | if advertiseAddr == nil { 147 | return nil, 0, fmt.Errorf("Failed to parse advertise address %q", ip) 148 | } 149 | 150 | // Ensure IPv4 conversion if necessary. 151 | if ip4 := advertiseAddr.To4(); ip4 != nil { 152 | advertiseAddr = ip4 153 | } 154 | advertisePort = port 155 | } else { 156 | if t.config.BindAddrs[0] == "0.0.0.0" { 157 | // Otherwise, if we're not bound to a specific IP, let's 158 | // use a suitable private IP address. 159 | var err error 160 | ip, err = sockaddr.GetPrivateIP() 161 | if err != nil { 162 | return nil, 0, fmt.Errorf("Failed to get interface addresses: %v", err) 163 | } 164 | if ip == "" { 165 | return nil, 0, fmt.Errorf("No private IP address found, and explicit IP not provided") 166 | } 167 | 168 | advertiseAddr = net.ParseIP(ip) 169 | if advertiseAddr == nil { 170 | return nil, 0, fmt.Errorf("Failed to parse advertise address: %q", ip) 171 | } 172 | } else { 173 | // Use the IP that we're bound to, based on the first 174 | // TCP listener, which we already ensure is there. 175 | advertiseAddr = t.tcpListeners[0].Addr().(*net.TCPAddr).IP 176 | } 177 | 178 | // Use the port we are bound to. 179 | advertisePort = t.GetAutoBindPort() 180 | } 181 | 182 | return advertiseAddr, advertisePort, nil 183 | } 184 | 185 | // See Transport. 186 | func (t *NetTransport) WriteTo(b []byte, addr string) (time.Time, error) { 187 | a := Address{Addr: addr, Name: ""} 188 | return t.WriteToAddress(b, a) 189 | } 190 | 191 | // See NodeAwareTransport. 192 | func (t *NetTransport) WriteToAddress(b []byte, a Address) (time.Time, error) { 193 | addr := a.Addr 194 | 195 | udpAddr, err := net.ResolveUDPAddr("udp", addr) 196 | if err != nil { 197 | return time.Time{}, err 198 | } 199 | 200 | // We made sure there's at least one UDP listener, so just use the 201 | // packet sending interface on the first one. Take the time after the 202 | // write call comes back, which will underestimate the time a little, 203 | // but help account for any delays before the write occurs. 204 | _, err = t.udpListeners[0].WriteTo(b, udpAddr) 205 | return time.Now(), err 206 | } 207 | 208 | // See Transport. 209 | func (t *NetTransport) PacketCh() <-chan *Packet { 210 | return t.packetCh 211 | } 212 | 213 | // See IngestionAwareTransport. 214 | func (t *NetTransport) IngestPacket(conn net.Conn, addr net.Addr, now time.Time, shouldClose bool) error { 215 | if shouldClose { 216 | defer conn.Close() 217 | } 218 | 219 | // Copy everything from the stream into packet buffer. 220 | var buf bytes.Buffer 221 | if _, err := io.Copy(&buf, conn); err != nil { 222 | return fmt.Errorf("failed to read packet: %v", err) 223 | } 224 | 225 | // Check the length - it needs to have at least one byte to be a proper 226 | // message. This is checked elsewhere for writes coming in directly from 227 | // the UDP socket. 228 | if n := buf.Len(); n < 1 { 229 | return fmt.Errorf("packet too short (%d bytes) %s", n, LogAddress(addr)) 230 | } 231 | 232 | // Inject the packet. 233 | t.packetCh <- &Packet{ 234 | Buf: buf.Bytes(), 235 | From: addr, 236 | Timestamp: now, 237 | } 238 | return nil 239 | } 240 | 241 | // See Transport. 242 | func (t *NetTransport) DialTimeout(addr string, timeout time.Duration) (net.Conn, error) { 243 | a := Address{Addr: addr, Name: ""} 244 | return t.DialAddressTimeout(a, timeout) 245 | } 246 | 247 | // See NodeAwareTransport. 248 | func (t *NetTransport) DialAddressTimeout(a Address, timeout time.Duration) (net.Conn, error) { 249 | addr := a.Addr 250 | 251 | dialer := net.Dialer{Timeout: timeout} 252 | return dialer.Dial("tcp", addr) 253 | } 254 | 255 | // See Transport. 256 | func (t *NetTransport) StreamCh() <-chan net.Conn { 257 | return t.streamCh 258 | } 259 | 260 | // See IngestionAwareTransport. 261 | func (t *NetTransport) IngestStream(conn net.Conn) error { 262 | t.streamCh <- conn 263 | return nil 264 | } 265 | 266 | // See Transport. 267 | func (t *NetTransport) Shutdown() error { 268 | // This will avoid log spam about errors when we shut down. 269 | atomic.StoreInt32(&t.shutdown, 1) 270 | 271 | // Rip through all the connections and shut them down. 272 | for _, conn := range t.tcpListeners { 273 | conn.Close() 274 | } 275 | for _, conn := range t.udpListeners { 276 | conn.Close() 277 | } 278 | 279 | // Block until all the listener threads have died. 280 | t.wg.Wait() 281 | return nil 282 | } 283 | 284 | // tcpListen is a long running goroutine that accepts incoming TCP connections 285 | // and hands them off to the stream channel. 286 | func (t *NetTransport) tcpListen(tcpLn *net.TCPListener) { 287 | defer t.wg.Done() 288 | 289 | // baseDelay is the initial delay after an AcceptTCP() error before attempting again 290 | const baseDelay = 5 * time.Millisecond 291 | 292 | // maxDelay is the maximum delay after an AcceptTCP() error before attempting again. 293 | // In the case that tcpListen() is error-looping, it will delay the shutdown check. 294 | // Therefore, changes to maxDelay may have an effect on the latency of shutdown. 295 | const maxDelay = 1 * time.Second 296 | 297 | var loopDelay time.Duration 298 | for { 299 | conn, err := tcpLn.AcceptTCP() 300 | if err != nil { 301 | if s := atomic.LoadInt32(&t.shutdown); s == 1 { 302 | break 303 | } 304 | 305 | if loopDelay == 0 { 306 | loopDelay = baseDelay 307 | } else { 308 | loopDelay *= 2 309 | } 310 | 311 | if loopDelay > maxDelay { 312 | loopDelay = maxDelay 313 | } 314 | 315 | t.logger.Printf("[ERR] memberlist: Error accepting TCP connection: %v", err) 316 | time.Sleep(loopDelay) 317 | continue 318 | } 319 | // No error, reset loop delay 320 | loopDelay = 0 321 | 322 | t.streamCh <- conn 323 | } 324 | } 325 | 326 | // udpListen is a long running goroutine that accepts incoming UDP packets and 327 | // hands them off to the packet channel. 328 | func (t *NetTransport) udpListen(udpLn *net.UDPConn) { 329 | defer t.wg.Done() 330 | for { 331 | // Do a blocking read into a fresh buffer. Grab a time stamp as 332 | // close as possible to the I/O. 333 | buf := make([]byte, udpPacketBufSize) 334 | n, addr, err := udpLn.ReadFrom(buf) 335 | ts := time.Now() 336 | if err != nil { 337 | if s := atomic.LoadInt32(&t.shutdown); s == 1 { 338 | break 339 | } 340 | 341 | t.logger.Printf("[ERR] memberlist: Error reading UDP packet: %v", err) 342 | continue 343 | } 344 | 345 | // Check the length - it needs to have at least one byte to be a 346 | // proper message. 347 | if n < 1 { 348 | t.logger.Printf("[ERR] memberlist: UDP packet too short (%d bytes) %s", 349 | len(buf), LogAddress(addr)) 350 | continue 351 | } 352 | 353 | // Ingest the packet. 354 | metrics.IncrCounterWithLabels([]string{"memberlist", "udp", "received"}, float32(n), t.metricLabels) 355 | t.packetCh <- &Packet{ 356 | Buf: buf[:n], 357 | From: addr, 358 | Timestamp: ts, 359 | } 360 | } 361 | } 362 | 363 | // setUDPRecvBuf is used to resize the UDP receive window. The function 364 | // attempts to set the read buffer to `udpRecvBuf` but backs off until 365 | // the read buffer can be set. 366 | func setUDPRecvBuf(c *net.UDPConn) error { 367 | size := udpRecvBufSize 368 | var err error 369 | for size > 0 { 370 | if err = c.SetReadBuffer(size); err == nil { 371 | return nil 372 | } 373 | size = size / 2 374 | } 375 | return err 376 | } 377 | -------------------------------------------------------------------------------- /peeked_conn.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Originally from: https://github.com/google/tcpproxy/blob/master/tcpproxy.go 16 | // at f5c09fbedceb69e4b238dec52cdf9f2fe9a815e2 17 | 18 | package memberlist 19 | 20 | import "net" 21 | 22 | // peekedConn is an incoming connection that has had some bytes read from it 23 | // to determine how to route the connection. The Read method stitches 24 | // the peeked bytes and unread bytes back together. 25 | type peekedConn struct { 26 | // Peeked are the bytes that have been read from Conn for the 27 | // purposes of route matching, but have not yet been consumed 28 | // by Read calls. It set to nil by Read when fully consumed. 29 | Peeked []byte 30 | 31 | // Conn is the underlying connection. 32 | // It can be type asserted against *net.TCPConn or other types 33 | // as needed. It should not be read from directly unless 34 | // Peeked is nil. 35 | net.Conn 36 | } 37 | 38 | func (c *peekedConn) Read(p []byte) (n int, err error) { 39 | if len(c.Peeked) > 0 { 40 | n = copy(p, c.Peeked) 41 | c.Peeked = c.Peeked[n:] 42 | if len(c.Peeked) == 0 { 43 | c.Peeked = nil 44 | } 45 | return n, nil 46 | } 47 | return c.Conn.Read(p) 48 | } 49 | -------------------------------------------------------------------------------- /ping_delegate.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import "time" 7 | 8 | // PingDelegate is used to notify an observer how long it took for a ping message to 9 | // complete a round trip. It can also be used for writing arbitrary byte slices 10 | // into ack messages. Note that in order to be meaningful for RTT estimates, this 11 | // delegate does not apply to indirect pings, nor fallback pings sent over TCP. 12 | type PingDelegate interface { 13 | // AckPayload is invoked when an ack is being sent; the returned bytes will be appended to the ack 14 | AckPayload() []byte 15 | // NotifyPing is invoked when an ack for a ping is received 16 | NotifyPingComplete(other *Node, rtt time.Duration, payload []byte) 17 | } 18 | -------------------------------------------------------------------------------- /queue.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "math" 8 | "sync" 9 | 10 | "github.com/google/btree" 11 | ) 12 | 13 | // TransmitLimitedQueue is used to queue messages to broadcast to 14 | // the cluster (via gossip) but limits the number of transmits per 15 | // message. It also prioritizes messages with lower transmit counts 16 | // (hence newer messages). 17 | type TransmitLimitedQueue struct { 18 | // NumNodes returns the number of nodes in the cluster. This is 19 | // used to determine the retransmit count, which is calculated 20 | // based on the log of this. 21 | NumNodes func() int 22 | 23 | // RetransmitMult is the multiplier used to determine the maximum 24 | // number of retransmissions attempted. 25 | RetransmitMult int 26 | 27 | mu sync.Mutex 28 | tq *btree.BTree // stores *limitedBroadcast as btree.Item 29 | tm map[string]*limitedBroadcast 30 | idGen int64 31 | } 32 | 33 | type limitedBroadcast struct { 34 | transmits int // btree-key[0]: Number of transmissions attempted. 35 | msgLen int64 // btree-key[1]: copied from len(b.Message()) 36 | id int64 // btree-key[2]: unique incrementing id stamped at submission time 37 | b Broadcast 38 | 39 | name string // set if Broadcast is a NamedBroadcast 40 | } 41 | 42 | // Less tests whether the current item is less than the given argument. 43 | // 44 | // This must provide a strict weak ordering. 45 | // If !a.Less(b) && !b.Less(a), we treat this to mean a == b (i.e. we can only 46 | // hold one of either a or b in the tree). 47 | // 48 | // default ordering is 49 | // - [transmits=0, ..., transmits=inf] 50 | // - [transmits=0:len=999, ..., transmits=0:len=2, ...] 51 | // - [transmits=0:len=999,id=999, ..., transmits=0:len=999:id=1, ...] 52 | func (b *limitedBroadcast) Less(than btree.Item) bool { 53 | o := than.(*limitedBroadcast) 54 | if b.transmits < o.transmits { 55 | return true 56 | } else if b.transmits > o.transmits { 57 | return false 58 | } 59 | if b.msgLen > o.msgLen { 60 | return true 61 | } else if b.msgLen < o.msgLen { 62 | return false 63 | } 64 | return b.id > o.id 65 | } 66 | 67 | // for testing; emits in transmit order if reverse=false 68 | func (q *TransmitLimitedQueue) orderedView(reverse bool) []*limitedBroadcast { 69 | q.mu.Lock() 70 | defer q.mu.Unlock() 71 | 72 | out := make([]*limitedBroadcast, 0, q.lenLocked()) 73 | q.walkReadOnlyLocked(reverse, func(cur *limitedBroadcast) bool { 74 | out = append(out, cur) 75 | return true 76 | }) 77 | 78 | return out 79 | } 80 | 81 | // walkReadOnlyLocked calls f for each item in the queue traversing it in 82 | // natural order (by Less) when reverse=false and the opposite when true. You 83 | // must hold the mutex. 84 | // 85 | // This method panics if you attempt to mutate the item during traversal. The 86 | // underlying btree should also not be mutated during traversal. 87 | func (q *TransmitLimitedQueue) walkReadOnlyLocked(reverse bool, f func(*limitedBroadcast) bool) { 88 | if q.lenLocked() == 0 { 89 | return 90 | } 91 | 92 | iter := func(item btree.Item) bool { 93 | cur := item.(*limitedBroadcast) 94 | 95 | prevTransmits := cur.transmits 96 | prevMsgLen := cur.msgLen 97 | prevID := cur.id 98 | 99 | keepGoing := f(cur) 100 | 101 | if prevTransmits != cur.transmits || prevMsgLen != cur.msgLen || prevID != cur.id { 102 | panic("edited queue while walking read only") 103 | } 104 | 105 | return keepGoing 106 | } 107 | 108 | if reverse { 109 | q.tq.Descend(iter) // end with transmit 0 110 | } else { 111 | q.tq.Ascend(iter) // start with transmit 0 112 | } 113 | } 114 | 115 | // Broadcast is something that can be broadcasted via gossip to 116 | // the memberlist cluster. 117 | type Broadcast interface { 118 | // Invalidates checks if enqueuing the current broadcast 119 | // invalidates a previous broadcast 120 | Invalidates(b Broadcast) bool 121 | 122 | // Returns a byte form of the message 123 | Message() []byte 124 | 125 | // Finished is invoked when the message will no longer 126 | // be broadcast, either due to invalidation or to the 127 | // transmit limit being reached 128 | Finished() 129 | } 130 | 131 | // NamedBroadcast is an optional extension of the Broadcast interface that 132 | // gives each message a unique string name, and that is used to optimize 133 | // 134 | // You shoud ensure that Invalidates() checks the same uniqueness as the 135 | // example below: 136 | // 137 | // func (b *foo) Invalidates(other Broadcast) bool { 138 | // nb, ok := other.(NamedBroadcast) 139 | // if !ok { 140 | // return false 141 | // } 142 | // return b.Name() == nb.Name() 143 | // } 144 | // 145 | // Invalidates() isn't currently used for NamedBroadcasts, but that may change 146 | // in the future. 147 | type NamedBroadcast interface { 148 | Broadcast 149 | // The unique identity of this broadcast message. 150 | Name() string 151 | } 152 | 153 | // UniqueBroadcast is an optional interface that indicates that each message is 154 | // intrinsically unique and there is no need to scan the broadcast queue for 155 | // duplicates. 156 | // 157 | // You should ensure that Invalidates() always returns false if implementing 158 | // this interface. Invalidates() isn't currently used for UniqueBroadcasts, but 159 | // that may change in the future. 160 | type UniqueBroadcast interface { 161 | Broadcast 162 | // UniqueBroadcast is just a marker method for this interface. 163 | UniqueBroadcast() 164 | } 165 | 166 | // QueueBroadcast is used to enqueue a broadcast 167 | func (q *TransmitLimitedQueue) QueueBroadcast(b Broadcast) { 168 | q.queueBroadcast(b, 0) 169 | } 170 | 171 | // lazyInit initializes internal data structures the first time they are 172 | // needed. You must already hold the mutex. 173 | func (q *TransmitLimitedQueue) lazyInit() { 174 | if q.tq == nil { 175 | q.tq = btree.New(32) 176 | } 177 | if q.tm == nil { 178 | q.tm = make(map[string]*limitedBroadcast) 179 | } 180 | } 181 | 182 | // queueBroadcast is like QueueBroadcast but you can use a nonzero value for 183 | // the initial transmit tier assigned to the message. This is meant to be used 184 | // for unit testing. 185 | func (q *TransmitLimitedQueue) queueBroadcast(b Broadcast, initialTransmits int) { 186 | q.mu.Lock() 187 | defer q.mu.Unlock() 188 | 189 | q.lazyInit() 190 | 191 | if q.idGen == math.MaxInt64 { 192 | // it's super duper unlikely to wrap around within the retransmit limit 193 | q.idGen = 1 194 | } else { 195 | q.idGen++ 196 | } 197 | id := q.idGen 198 | 199 | lb := &limitedBroadcast{ 200 | transmits: initialTransmits, 201 | msgLen: int64(len(b.Message())), 202 | id: id, 203 | b: b, 204 | } 205 | unique := false 206 | if nb, ok := b.(NamedBroadcast); ok { 207 | lb.name = nb.Name() 208 | } else if _, ok := b.(UniqueBroadcast); ok { 209 | unique = true 210 | } 211 | 212 | // Check if this message invalidates another. 213 | if lb.name != "" { 214 | if old, ok := q.tm[lb.name]; ok { 215 | old.b.Finished() 216 | q.deleteItem(old) 217 | } 218 | } else if !unique { 219 | // Slow path, hopefully nothing hot hits this. 220 | var remove []*limitedBroadcast 221 | q.tq.Ascend(func(item btree.Item) bool { 222 | cur := item.(*limitedBroadcast) 223 | 224 | // Special Broadcasts can only invalidate each other. 225 | switch cur.b.(type) { 226 | case NamedBroadcast: 227 | // noop 228 | case UniqueBroadcast: 229 | // noop 230 | default: 231 | if b.Invalidates(cur.b) { 232 | cur.b.Finished() 233 | remove = append(remove, cur) 234 | } 235 | } 236 | return true 237 | }) 238 | for _, cur := range remove { 239 | q.deleteItem(cur) 240 | } 241 | } 242 | 243 | // Append to the relevant queue. 244 | q.addItem(lb) 245 | } 246 | 247 | // deleteItem removes the given item from the overall datastructure. You 248 | // must already hold the mutex. 249 | func (q *TransmitLimitedQueue) deleteItem(cur *limitedBroadcast) { 250 | _ = q.tq.Delete(cur) 251 | if cur.name != "" { 252 | delete(q.tm, cur.name) 253 | } 254 | 255 | if q.tq.Len() == 0 { 256 | // At idle there's no reason to let the id generator keep going 257 | // indefinitely. 258 | q.idGen = 0 259 | } 260 | } 261 | 262 | // addItem adds the given item into the overall datastructure. You must already 263 | // hold the mutex. 264 | func (q *TransmitLimitedQueue) addItem(cur *limitedBroadcast) { 265 | _ = q.tq.ReplaceOrInsert(cur) 266 | if cur.name != "" { 267 | q.tm[cur.name] = cur 268 | } 269 | } 270 | 271 | // getTransmitRange returns a pair of min/max values for transmit values 272 | // represented by the current queue contents. Both values represent actual 273 | // transmit values on the interval [0, len). You must already hold the mutex. 274 | func (q *TransmitLimitedQueue) getTransmitRange() (minTransmit, maxTransmit int) { 275 | if q.lenLocked() == 0 { 276 | return 0, 0 277 | } 278 | minItem, maxItem := q.tq.Min(), q.tq.Max() 279 | if minItem == nil || maxItem == nil { 280 | return 0, 0 281 | } 282 | 283 | min := minItem.(*limitedBroadcast).transmits 284 | max := maxItem.(*limitedBroadcast).transmits 285 | 286 | return min, max 287 | } 288 | 289 | // GetBroadcasts is used to get a number of broadcasts, up to a byte limit 290 | // and applying a per-message overhead as provided. 291 | func (q *TransmitLimitedQueue) GetBroadcasts(overhead, limit int) [][]byte { 292 | q.mu.Lock() 293 | defer q.mu.Unlock() 294 | 295 | // Fast path the default case 296 | if q.lenLocked() == 0 { 297 | return nil 298 | } 299 | 300 | transmitLimit := retransmitLimit(q.RetransmitMult, q.NumNodes()) 301 | 302 | var ( 303 | bytesUsed int 304 | toSend [][]byte 305 | reinsert []*limitedBroadcast 306 | ) 307 | 308 | // Visit fresher items first, but only look at stuff that will fit. 309 | // We'll go tier by tier, grabbing the largest items first. 310 | minTr, maxTr := q.getTransmitRange() 311 | for transmits := minTr; transmits <= maxTr; /*do not advance automatically*/ { 312 | free := int64(limit - bytesUsed - overhead) 313 | if free <= 0 { 314 | break // bail out early 315 | } 316 | 317 | // Search for the least element on a given tier (by transmit count) as 318 | // defined in the limitedBroadcast.Less function that will fit into our 319 | // remaining space. 320 | greaterOrEqual := &limitedBroadcast{ 321 | transmits: transmits, 322 | msgLen: free, 323 | id: math.MaxInt64, 324 | } 325 | lessThan := &limitedBroadcast{ 326 | transmits: transmits + 1, 327 | msgLen: math.MaxInt64, 328 | id: math.MaxInt64, 329 | } 330 | var keep *limitedBroadcast 331 | q.tq.AscendRange(greaterOrEqual, lessThan, func(item btree.Item) bool { 332 | cur := item.(*limitedBroadcast) 333 | // Check if this is within our limits 334 | if int64(len(cur.b.Message())) > free { 335 | // If this happens it's a bug in the datastructure or 336 | // surrounding use doing something like having len(Message()) 337 | // change over time. There's enough going on here that it's 338 | // probably sane to just skip it and move on for now. 339 | return true 340 | } 341 | keep = cur 342 | return false 343 | }) 344 | if keep == nil { 345 | // No more items of an appropriate size in the tier. 346 | transmits++ 347 | continue 348 | } 349 | 350 | msg := keep.b.Message() 351 | 352 | // Add to slice to send 353 | bytesUsed += overhead + len(msg) 354 | toSend = append(toSend, msg) 355 | 356 | // Check if we should stop transmission 357 | q.deleteItem(keep) 358 | if keep.transmits+1 >= transmitLimit { 359 | keep.b.Finished() 360 | } else { 361 | // We need to bump this item down to another transmit tier, but 362 | // because it would be in the same direction that we're walking the 363 | // tiers, we will have to delay the reinsertion until we are 364 | // finished our search. Otherwise we'll possibly re-add the message 365 | // when we ascend to the next tier. 366 | keep.transmits++ 367 | reinsert = append(reinsert, keep) 368 | } 369 | } 370 | 371 | for _, cur := range reinsert { 372 | q.addItem(cur) 373 | } 374 | 375 | return toSend 376 | } 377 | 378 | // NumQueued returns the number of queued messages 379 | func (q *TransmitLimitedQueue) NumQueued() int { 380 | q.mu.Lock() 381 | defer q.mu.Unlock() 382 | return q.lenLocked() 383 | } 384 | 385 | // lenLocked returns the length of the overall queue datastructure. You must 386 | // hold the mutex. 387 | func (q *TransmitLimitedQueue) lenLocked() int { 388 | if q.tq == nil { 389 | return 0 390 | } 391 | return q.tq.Len() 392 | } 393 | 394 | // Reset clears all the queued messages. Should only be used for tests. 395 | func (q *TransmitLimitedQueue) Reset() { 396 | q.mu.Lock() 397 | defer q.mu.Unlock() 398 | 399 | q.walkReadOnlyLocked(false, func(cur *limitedBroadcast) bool { 400 | cur.b.Finished() 401 | return true 402 | }) 403 | 404 | q.tq = nil 405 | q.tm = nil 406 | q.idGen = 0 407 | } 408 | 409 | // Prune will retain the maxRetain latest messages, and the rest 410 | // will be discarded. This can be used to prevent unbounded queue sizes 411 | func (q *TransmitLimitedQueue) Prune(maxRetain int) { 412 | q.mu.Lock() 413 | defer q.mu.Unlock() 414 | 415 | // Do nothing if queue size is less than the limit 416 | for q.tq.Len() > maxRetain { 417 | item := q.tq.Max() 418 | if item == nil { 419 | break 420 | } 421 | cur := item.(*limitedBroadcast) 422 | cur.b.Finished() 423 | q.deleteItem(cur) 424 | } 425 | } 426 | -------------------------------------------------------------------------------- /queue_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/google/btree" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestLimitedBroadcastLess(t *testing.T) { 14 | cases := []struct { 15 | Name string 16 | A *limitedBroadcast // lesser 17 | B *limitedBroadcast 18 | }{ 19 | { 20 | "diff-transmits", 21 | &limitedBroadcast{transmits: 0, msgLen: 10, id: 100}, 22 | &limitedBroadcast{transmits: 1, msgLen: 10, id: 100}, 23 | }, 24 | { 25 | "same-transmits--diff-len", 26 | &limitedBroadcast{transmits: 0, msgLen: 12, id: 100}, 27 | &limitedBroadcast{transmits: 0, msgLen: 10, id: 100}, 28 | }, 29 | { 30 | "same-transmits--same-len--diff-id", 31 | &limitedBroadcast{transmits: 0, msgLen: 12, id: 100}, 32 | &limitedBroadcast{transmits: 0, msgLen: 12, id: 90}, 33 | }, 34 | } 35 | 36 | for _, c := range cases { 37 | t.Run(c.Name, func(t *testing.T) { 38 | a, b := c.A, c.B 39 | 40 | require.True(t, a.Less(b)) 41 | 42 | tree := btree.New(32) 43 | 44 | tree.ReplaceOrInsert(b) 45 | tree.ReplaceOrInsert(a) 46 | 47 | min := tree.Min().(*limitedBroadcast) 48 | require.Equal(t, a.transmits, min.transmits) 49 | require.Equal(t, a.msgLen, min.msgLen) 50 | require.Equal(t, a.id, min.id) 51 | 52 | max := tree.Max().(*limitedBroadcast) 53 | require.Equal(t, b.transmits, max.transmits) 54 | require.Equal(t, b.msgLen, max.msgLen) 55 | require.Equal(t, b.id, max.id) 56 | }) 57 | } 58 | } 59 | 60 | func TestTransmitLimited_Queue(t *testing.T) { 61 | q := &TransmitLimitedQueue{RetransmitMult: 1, NumNodes: func() int { return 1 }} 62 | q.QueueBroadcast(&memberlistBroadcast{"test", nil, nil}) 63 | q.QueueBroadcast(&memberlistBroadcast{"foo", nil, nil}) 64 | q.QueueBroadcast(&memberlistBroadcast{"bar", nil, nil}) 65 | 66 | if q.NumQueued() != 3 { 67 | t.Fatalf("bad len") 68 | } 69 | dump := q.orderedView(true) 70 | if dump[0].b.(*memberlistBroadcast).node != "test" { 71 | t.Fatalf("missing test") 72 | } 73 | if dump[1].b.(*memberlistBroadcast).node != "foo" { 74 | t.Fatalf("missing foo") 75 | } 76 | if dump[2].b.(*memberlistBroadcast).node != "bar" { 77 | t.Fatalf("missing bar") 78 | } 79 | 80 | // Should invalidate previous message 81 | q.QueueBroadcast(&memberlistBroadcast{"test", nil, nil}) 82 | 83 | if q.NumQueued() != 3 { 84 | t.Fatalf("bad len") 85 | } 86 | dump = q.orderedView(true) 87 | if dump[0].b.(*memberlistBroadcast).node != "foo" { 88 | t.Fatalf("missing foo") 89 | } 90 | if dump[1].b.(*memberlistBroadcast).node != "bar" { 91 | t.Fatalf("missing bar") 92 | } 93 | if dump[2].b.(*memberlistBroadcast).node != "test" { 94 | t.Fatalf("missing test") 95 | } 96 | } 97 | 98 | func TestTransmitLimited_GetBroadcasts(t *testing.T) { 99 | q := &TransmitLimitedQueue{RetransmitMult: 3, NumNodes: func() int { return 10 }} 100 | 101 | // 18 bytes per message 102 | q.QueueBroadcast(&memberlistBroadcast{"test", []byte("1. this is a test."), nil}) 103 | q.QueueBroadcast(&memberlistBroadcast{"foo", []byte("2. this is a test."), nil}) 104 | q.QueueBroadcast(&memberlistBroadcast{"bar", []byte("3. this is a test."), nil}) 105 | q.QueueBroadcast(&memberlistBroadcast{"baz", []byte("4. this is a test."), nil}) 106 | 107 | // 2 byte overhead per message, should get all 4 messages 108 | all := q.GetBroadcasts(2, 80) 109 | require.Equal(t, 4, len(all), "missing messages: %v", prettyPrintMessages(all)) 110 | 111 | // 3 byte overhead, should only get 3 messages back 112 | partial := q.GetBroadcasts(3, 80) 113 | require.Equal(t, 3, len(partial), "missing messages: %v", prettyPrintMessages(partial)) 114 | } 115 | 116 | func TestTransmitLimited_GetBroadcasts_Limit(t *testing.T) { 117 | q := &TransmitLimitedQueue{RetransmitMult: 1, NumNodes: func() int { return 10 }} 118 | 119 | require.Equal(t, int64(0), q.idGen, "the id generator seed starts at zero") 120 | require.Equal(t, 2, retransmitLimit(q.RetransmitMult, q.NumNodes()), "sanity check transmit limits") 121 | 122 | // 18 bytes per message 123 | q.QueueBroadcast(&memberlistBroadcast{"test", []byte("1. this is a test."), nil}) 124 | q.QueueBroadcast(&memberlistBroadcast{"foo", []byte("2. this is a test."), nil}) 125 | q.QueueBroadcast(&memberlistBroadcast{"bar", []byte("3. this is a test."), nil}) 126 | q.QueueBroadcast(&memberlistBroadcast{"baz", []byte("4. this is a test."), nil}) 127 | 128 | require.Equal(t, int64(4), q.idGen, "we handed out 4 IDs") 129 | 130 | // 3 byte overhead, should only get 3 messages back 131 | partial1 := q.GetBroadcasts(3, 80) 132 | require.Equal(t, 3, len(partial1), "missing messages: %v", prettyPrintMessages(partial1)) 133 | 134 | require.Equal(t, int64(4), q.idGen, "id generator doesn't reset until empty") 135 | 136 | partial2 := q.GetBroadcasts(3, 80) 137 | require.Equal(t, 3, len(partial2), "missing messages: %v", prettyPrintMessages(partial2)) 138 | 139 | require.Equal(t, int64(4), q.idGen, "id generator doesn't reset until empty") 140 | 141 | // Only two not expired 142 | partial3 := q.GetBroadcasts(3, 80) 143 | require.Equal(t, 2, len(partial3), "missing messages: %v", prettyPrintMessages(partial3)) 144 | 145 | require.Equal(t, int64(0), q.idGen, "id generator resets on empty") 146 | 147 | // Should get nothing 148 | partial5 := q.GetBroadcasts(3, 80) 149 | require.Equal(t, 0, len(partial5), "missing messages: %v", prettyPrintMessages(partial5)) 150 | 151 | require.Equal(t, int64(0), q.idGen, "id generator resets on empty") 152 | } 153 | 154 | func prettyPrintMessages(msgs [][]byte) []string { 155 | var out []string 156 | for _, msg := range msgs { 157 | out = append(out, "'"+string(msg)+"'") 158 | } 159 | return out 160 | } 161 | 162 | func TestTransmitLimited_Prune(t *testing.T) { 163 | q := &TransmitLimitedQueue{RetransmitMult: 1, NumNodes: func() int { return 10 }} 164 | 165 | ch1 := make(chan struct{}, 1) 166 | ch2 := make(chan struct{}, 1) 167 | 168 | // 18 bytes per message 169 | q.QueueBroadcast(&memberlistBroadcast{"test", []byte("1. this is a test."), ch1}) 170 | q.QueueBroadcast(&memberlistBroadcast{"foo", []byte("2. this is a test."), ch2}) 171 | q.QueueBroadcast(&memberlistBroadcast{"bar", []byte("3. this is a test."), nil}) 172 | q.QueueBroadcast(&memberlistBroadcast{"baz", []byte("4. this is a test."), nil}) 173 | 174 | // Keep only 2 175 | q.Prune(2) 176 | 177 | require.Equal(t, 2, q.NumQueued()) 178 | 179 | // Should notify the first two 180 | select { 181 | case <-ch1: 182 | default: 183 | t.Fatalf("expected invalidation") 184 | } 185 | select { 186 | case <-ch2: 187 | default: 188 | t.Fatalf("expected invalidation") 189 | } 190 | 191 | dump := q.orderedView(true) 192 | 193 | if dump[0].b.(*memberlistBroadcast).node != "bar" { 194 | t.Fatalf("missing bar") 195 | } 196 | if dump[1].b.(*memberlistBroadcast).node != "baz" { 197 | t.Fatalf("missing baz") 198 | } 199 | } 200 | 201 | func TestTransmitLimited_ordering(t *testing.T) { 202 | q := &TransmitLimitedQueue{RetransmitMult: 1, NumNodes: func() int { return 10 }} 203 | 204 | insert := func(name string, transmits int) { 205 | q.queueBroadcast(&memberlistBroadcast{name, []byte(name), make(chan struct{})}, transmits) 206 | } 207 | 208 | insert("node0", 0) 209 | insert("node1", 10) 210 | insert("node2", 3) 211 | insert("node3", 4) 212 | insert("node4", 7) 213 | 214 | dump := q.orderedView(true) 215 | 216 | if dump[0].transmits != 10 { 217 | t.Fatalf("bad val %v, %d", dump[0].b.(*memberlistBroadcast).node, dump[0].transmits) 218 | } 219 | if dump[1].transmits != 7 { 220 | t.Fatalf("bad val %v, %d", dump[7].b.(*memberlistBroadcast).node, dump[7].transmits) 221 | } 222 | if dump[2].transmits != 4 { 223 | t.Fatalf("bad val %v, %d", dump[2].b.(*memberlistBroadcast).node, dump[2].transmits) 224 | } 225 | if dump[3].transmits != 3 { 226 | t.Fatalf("bad val %v, %d", dump[3].b.(*memberlistBroadcast).node, dump[3].transmits) 227 | } 228 | if dump[4].transmits != 0 { 229 | t.Fatalf("bad val %v, %d", dump[4].b.(*memberlistBroadcast).node, dump[4].transmits) 230 | } 231 | } 232 | -------------------------------------------------------------------------------- /security.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "bytes" 8 | "crypto/aes" 9 | "crypto/cipher" 10 | "crypto/rand" 11 | "fmt" 12 | "io" 13 | ) 14 | 15 | /* 16 | Encrypted messages are prefixed with an encryptionVersion byte 17 | that is used for us to be able to properly encode/decode. We 18 | currently support the following versions: 19 | 20 | 0 - AES-GCM 128, using PKCS7 padding 21 | 1 - AES-GCM 128, no padding. Padding not needed, caused bloat. 22 | */ 23 | type encryptionVersion uint8 24 | 25 | const ( 26 | minEncryptionVersion encryptionVersion = 0 27 | maxEncryptionVersion encryptionVersion = 1 28 | ) 29 | 30 | const ( 31 | versionSize = 1 32 | nonceSize = 12 33 | tagSize = 16 34 | maxPadOverhead = 16 35 | blockSize = aes.BlockSize 36 | ) 37 | 38 | // pkcs7encode is used to pad a byte buffer to a specific block size using 39 | // the PKCS7 algorithm. "Ignores" some bytes to compensate for IV 40 | func pkcs7encode(buf *bytes.Buffer, ignore, blockSize int) { 41 | n := buf.Len() - ignore 42 | more := blockSize - (n % blockSize) 43 | for i := 0; i < more; i++ { 44 | buf.WriteByte(byte(more)) 45 | } 46 | } 47 | 48 | // pkcs7decode is used to decode a buffer that has been padded 49 | func pkcs7decode(buf []byte, blockSize int) []byte { 50 | if len(buf) == 0 { 51 | panic("Cannot decode a PKCS7 buffer of zero length") 52 | } 53 | n := len(buf) 54 | last := buf[n-1] 55 | n -= int(last) 56 | return buf[:n] 57 | } 58 | 59 | // encryptOverhead returns the maximum possible overhead of encryption by version 60 | func encryptOverhead(vsn encryptionVersion) int { 61 | switch vsn { 62 | case 0: 63 | return 45 // Version: 1, IV: 12, Padding: 16, Tag: 16 64 | case 1: 65 | return 29 // Version: 1, IV: 12, Tag: 16 66 | default: 67 | panic("unsupported version") 68 | } 69 | } 70 | 71 | // encryptedLength is used to compute the buffer size needed 72 | // for a message of given length 73 | func encryptedLength(vsn encryptionVersion, inp int) int { 74 | // If we are on version 1, there is no padding 75 | if vsn >= 1 { 76 | return versionSize + nonceSize + inp + tagSize 77 | } 78 | 79 | // Determine the padding size 80 | padding := blockSize - (inp % blockSize) 81 | 82 | // Sum the extra parts to get total size 83 | return versionSize + nonceSize + inp + padding + tagSize 84 | } 85 | 86 | // encryptPayload is used to encrypt a message with a given key. 87 | // We make use of AES-128 in GCM mode. New byte buffer is the version, 88 | // nonce, ciphertext and tag 89 | func encryptPayload(vsn encryptionVersion, key []byte, msg []byte, data []byte, dst *bytes.Buffer) error { 90 | // Get the AES block cipher 91 | aesBlock, err := aes.NewCipher(key) 92 | if err != nil { 93 | return err 94 | } 95 | 96 | // Get the GCM cipher mode 97 | gcm, err := cipher.NewGCM(aesBlock) 98 | if err != nil { 99 | return err 100 | } 101 | 102 | // Grow the buffer to make room for everything 103 | offset := dst.Len() 104 | dst.Grow(encryptedLength(vsn, len(msg))) 105 | 106 | // Write the encryption version 107 | dst.WriteByte(byte(vsn)) 108 | 109 | // Add a random nonce 110 | _, err = io.CopyN(dst, rand.Reader, nonceSize) 111 | if err != nil { 112 | return err 113 | } 114 | afterNonce := dst.Len() 115 | 116 | // Ensure we are correctly padded (only version 0) 117 | if vsn == 0 { 118 | io.Copy(dst, bytes.NewReader(msg)) 119 | pkcs7encode(dst, offset+versionSize+nonceSize, aes.BlockSize) 120 | } 121 | 122 | // Encrypt message using GCM 123 | slice := dst.Bytes()[offset:] 124 | nonce := slice[versionSize : versionSize+nonceSize] 125 | 126 | // Message source depends on the encryption version. 127 | // Version 0 uses padding, version 1 does not 128 | var src []byte 129 | if vsn == 0 { 130 | src = slice[versionSize+nonceSize:] 131 | } else { 132 | src = msg 133 | } 134 | out := gcm.Seal(nil, nonce, src, data) 135 | 136 | // Truncate the plaintext, and write the cipher text 137 | dst.Truncate(afterNonce) 138 | dst.Write(out) 139 | return nil 140 | } 141 | 142 | // decryptMessage performs the actual decryption of ciphertext. This is in its 143 | // own function to allow it to be called on all keys easily. 144 | func decryptMessage(key, msg []byte, data []byte) ([]byte, error) { 145 | // Get the AES block cipher 146 | aesBlock, err := aes.NewCipher(key) 147 | if err != nil { 148 | return nil, err 149 | } 150 | 151 | // Get the GCM cipher mode 152 | gcm, err := cipher.NewGCM(aesBlock) 153 | if err != nil { 154 | return nil, err 155 | } 156 | 157 | // Decrypt the message 158 | nonce := msg[versionSize : versionSize+nonceSize] 159 | ciphertext := msg[versionSize+nonceSize:] 160 | plain, err := gcm.Open(nil, nonce, ciphertext, data) 161 | if err != nil { 162 | return nil, err 163 | } 164 | 165 | // Success! 166 | return plain, nil 167 | } 168 | 169 | // decryptPayload is used to decrypt a message with a given key, 170 | // and verify it's contents. Any padding will be removed, and a 171 | // slice to the plaintext is returned. Decryption is done IN PLACE! 172 | func decryptPayload(keys [][]byte, msg []byte, data []byte) ([]byte, error) { 173 | // Ensure we have at least one byte 174 | if len(msg) == 0 { 175 | return nil, fmt.Errorf("Cannot decrypt empty payload") 176 | } 177 | 178 | // Verify the version 179 | vsn := encryptionVersion(msg[0]) 180 | if vsn > maxEncryptionVersion { 181 | return nil, fmt.Errorf("Unsupported encryption version %d", msg[0]) 182 | } 183 | 184 | // Ensure the length is sane 185 | if len(msg) < encryptedLength(vsn, 0) { 186 | return nil, fmt.Errorf("Payload is too small to decrypt: %d", len(msg)) 187 | } 188 | 189 | for _, key := range keys { 190 | plain, err := decryptMessage(key, msg, data) 191 | if err == nil { 192 | // Remove the PKCS7 padding for vsn 0 193 | if vsn == 0 { 194 | return pkcs7decode(plain, aes.BlockSize), nil 195 | } else { 196 | return plain, nil 197 | } 198 | } 199 | } 200 | 201 | return nil, fmt.Errorf("No installed keys could decrypt the message") 202 | } 203 | 204 | func appendBytes(first []byte, second []byte) []byte { 205 | hasFirst := len(first) > 0 206 | hasSecond := len(second) > 0 207 | 208 | switch { 209 | case hasFirst && hasSecond: 210 | out := make([]byte, 0, len(first)+len(second)) 211 | out = append(out, first...) 212 | out = append(out, second...) 213 | return out 214 | case hasFirst: 215 | return first 216 | case hasSecond: 217 | return second 218 | default: 219 | return nil 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /security_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "bytes" 8 | "reflect" 9 | "testing" 10 | ) 11 | 12 | func TestPKCS7(t *testing.T) { 13 | for i := 0; i <= 255; i++ { 14 | // Make a buffer of size i 15 | buf := []byte{} 16 | for j := 0; j < i; j++ { 17 | buf = append(buf, byte(i)) 18 | } 19 | 20 | // Copy to bytes buffer 21 | inp := bytes.NewBuffer(nil) 22 | inp.Write(buf) 23 | 24 | // Pad this out 25 | pkcs7encode(inp, 0, 16) 26 | 27 | // Unpad 28 | dec := pkcs7decode(inp.Bytes(), 16) 29 | 30 | // Ensure equivilence 31 | if !reflect.DeepEqual(buf, dec) { 32 | t.Fatalf("mismatch: %v %v", buf, dec) 33 | } 34 | } 35 | 36 | } 37 | 38 | func TestEncryptDecrypt_V0(t *testing.T) { 39 | encryptDecryptVersioned(0, t) 40 | } 41 | 42 | func TestEncryptDecrypt_V1(t *testing.T) { 43 | encryptDecryptVersioned(1, t) 44 | } 45 | 46 | func encryptDecryptVersioned(vsn encryptionVersion, t *testing.T) { 47 | k1 := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} 48 | plaintext := []byte("this is a plain text message") 49 | extra := []byte("random data") 50 | 51 | var buf bytes.Buffer 52 | err := encryptPayload(vsn, k1, plaintext, extra, &buf) 53 | if err != nil { 54 | t.Fatalf("err: %v", err) 55 | } 56 | 57 | expLen := encryptedLength(vsn, len(plaintext)) 58 | if buf.Len() != expLen { 59 | t.Fatalf("output length is unexpected %d %d %d", len(plaintext), buf.Len(), expLen) 60 | } 61 | 62 | msg, err := decryptPayload([][]byte{k1}, buf.Bytes(), extra) 63 | if err != nil { 64 | t.Fatalf("err: %v", err) 65 | } 66 | 67 | cmp := bytes.Compare(msg, plaintext) 68 | if cmp != 0 { 69 | t.Errorf("len %d %v", len(msg), msg) 70 | t.Errorf("len %d %v", len(plaintext), plaintext) 71 | t.Fatalf("encrypt/decrypt failed! %d '%s' '%s'", cmp, msg, plaintext) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /suspicion.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "math" 8 | "sync/atomic" 9 | "time" 10 | ) 11 | 12 | // suspicion manages the suspect timer for a node and provides an interface 13 | // to accelerate the timeout as we get more independent confirmations that 14 | // a node is suspect. 15 | type suspicion struct { 16 | // n is the number of independent confirmations we've seen. This must 17 | // be updated using atomic instructions to prevent contention with the 18 | // timer callback. 19 | n int32 20 | 21 | // k is the number of independent confirmations we'd like to see in 22 | // order to drive the timer to its minimum value. 23 | k int32 24 | 25 | // min is the minimum timer value. 26 | min time.Duration 27 | 28 | // max is the maximum timer value. 29 | max time.Duration 30 | 31 | // start captures the timestamp when we began the timer. This is used 32 | // so we can calculate durations to feed the timer during updates in 33 | // a way the achieves the overall time we'd like. 34 | start time.Time 35 | 36 | // timer is the underlying timer that implements the timeout. 37 | timer *time.Timer 38 | 39 | // f is the function to call when the timer expires. We hold on to this 40 | // because there are cases where we call it directly. 41 | timeoutFn func() 42 | 43 | // confirmations is a map of "from" nodes that have confirmed a given 44 | // node is suspect. This prevents double counting. 45 | confirmations map[string]struct{} 46 | } 47 | 48 | // newSuspicion returns a timer started with the max time, and that will drive 49 | // to the min time after seeing k or more confirmations. The from node will be 50 | // excluded from confirmations since we might get our own suspicion message 51 | // gossiped back to us. The minimum time will be used if no confirmations are 52 | // called for (k <= 0). 53 | func newSuspicion(from string, k int, min time.Duration, max time.Duration, fn func(int)) *suspicion { 54 | s := &suspicion{ 55 | k: int32(k), 56 | min: min, 57 | max: max, 58 | confirmations: make(map[string]struct{}), 59 | } 60 | 61 | // Exclude the from node from any confirmations. 62 | s.confirmations[from] = struct{}{} 63 | 64 | // Pass the number of confirmations into the timeout function for 65 | // easy telemetry. 66 | s.timeoutFn = func() { 67 | fn(int(atomic.LoadInt32(&s.n))) 68 | } 69 | 70 | // If there aren't any confirmations to be made then take the min 71 | // time from the start. 72 | timeout := max 73 | if k < 1 { 74 | timeout = min 75 | } 76 | s.timer = time.AfterFunc(timeout, s.timeoutFn) 77 | 78 | // Capture the start time right after starting the timer above so 79 | // we should always err on the side of a little longer timeout if 80 | // there's any preemption that separates this and the step above. 81 | s.start = time.Now() 82 | return s 83 | } 84 | 85 | // remainingSuspicionTime takes the state variables of the suspicion timer and 86 | // calculates the remaining time to wait before considering a node dead. The 87 | // return value can be negative, so be prepared to fire the timer immediately in 88 | // that case. 89 | func remainingSuspicionTime(n, k int32, elapsed time.Duration, min, max time.Duration) time.Duration { 90 | frac := math.Log(float64(n)+1.0) / math.Log(float64(k)+1.0) 91 | raw := max.Seconds() - frac*(max.Seconds()-min.Seconds()) 92 | timeout := time.Duration(math.Floor(1000.0*raw)) * time.Millisecond 93 | if timeout < min { 94 | timeout = min 95 | } 96 | 97 | // We have to take into account the amount of time that has passed so 98 | // far, so we get the right overall timeout. 99 | return timeout - elapsed 100 | } 101 | 102 | // Confirm registers that a possibly new peer has also determined the given 103 | // node is suspect. This returns true if this was new information, and false 104 | // if it was a duplicate confirmation, or if we've got enough confirmations to 105 | // hit the minimum. 106 | func (s *suspicion) Confirm(from string) bool { 107 | // If we've got enough confirmations then stop accepting them. 108 | if atomic.LoadInt32(&s.n) >= s.k { 109 | return false 110 | } 111 | 112 | // Only allow one confirmation from each possible peer. 113 | if _, ok := s.confirmations[from]; ok { 114 | return false 115 | } 116 | s.confirmations[from] = struct{}{} 117 | 118 | // Compute the new timeout given the current number of confirmations and 119 | // adjust the timer. If the timeout becomes negative *and* we can cleanly 120 | // stop the timer then we will call the timeout function directly from 121 | // here. 122 | n := atomic.AddInt32(&s.n, 1) 123 | elapsed := time.Since(s.start) 124 | remaining := remainingSuspicionTime(n, s.k, elapsed, s.min, s.max) 125 | if s.timer.Stop() { 126 | if remaining > 0 { 127 | s.timer.Reset(remaining) 128 | } else { 129 | go s.timeoutFn() 130 | } 131 | } 132 | return true 133 | } 134 | -------------------------------------------------------------------------------- /suspicion_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestSuspicion_remainingSuspicionTime(t *testing.T) { 12 | cases := []struct { 13 | n int32 14 | k int32 15 | elapsed time.Duration 16 | min time.Duration 17 | max time.Duration 18 | expected time.Duration 19 | }{ 20 | {0, 3, 0, 2 * time.Second, 30 * time.Second, 30 * time.Second}, 21 | {1, 3, 2 * time.Second, 2 * time.Second, 30 * time.Second, 14 * time.Second}, 22 | {2, 3, 3 * time.Second, 2 * time.Second, 30 * time.Second, 4810 * time.Millisecond}, 23 | {3, 3, 4 * time.Second, 2 * time.Second, 30 * time.Second, -2 * time.Second}, 24 | {4, 3, 5 * time.Second, 2 * time.Second, 30 * time.Second, -3 * time.Second}, 25 | {5, 3, 10 * time.Second, 2 * time.Second, 30 * time.Second, -8 * time.Second}, 26 | } 27 | for i, c := range cases { 28 | remaining := remainingSuspicionTime(c.n, c.k, c.elapsed, c.min, c.max) 29 | if remaining != c.expected { 30 | t.Errorf("case %d: remaining %9.6f != expected %9.6f", i, remaining.Seconds(), c.expected.Seconds()) 31 | } 32 | } 33 | } 34 | 35 | func TestSuspicion_Timer(t *testing.T) { 36 | const k = 3 37 | const min = 500 * time.Millisecond 38 | const max = 2 * time.Second 39 | 40 | type pair struct { 41 | from string 42 | newInfo bool 43 | } 44 | cases := []struct { 45 | numConfirmations int 46 | from string 47 | confirmations []pair 48 | expected time.Duration 49 | }{ 50 | { 51 | 0, 52 | "me", 53 | []pair{}, 54 | max, 55 | }, 56 | { 57 | 1, 58 | "me", 59 | []pair{ 60 | pair{"me", false}, 61 | pair{"foo", true}, 62 | }, 63 | 1250 * time.Millisecond, 64 | }, 65 | { 66 | 1, 67 | "me", 68 | []pair{ 69 | pair{"me", false}, 70 | pair{"foo", true}, 71 | pair{"foo", false}, 72 | pair{"foo", false}, 73 | }, 74 | 1250 * time.Millisecond, 75 | }, 76 | { 77 | 2, 78 | "me", 79 | []pair{ 80 | pair{"me", false}, 81 | pair{"foo", true}, 82 | pair{"bar", true}, 83 | }, 84 | 810 * time.Millisecond, 85 | }, 86 | { 87 | 3, 88 | "me", 89 | []pair{ 90 | pair{"me", false}, 91 | pair{"foo", true}, 92 | pair{"bar", true}, 93 | pair{"baz", true}, 94 | }, 95 | min, 96 | }, 97 | { 98 | 3, 99 | "me", 100 | []pair{ 101 | pair{"me", false}, 102 | pair{"foo", true}, 103 | pair{"bar", true}, 104 | pair{"baz", true}, 105 | pair{"zoo", false}, 106 | }, 107 | min, 108 | }, 109 | } 110 | for i, c := range cases { 111 | ch := make(chan time.Duration, 1) 112 | start := time.Now() 113 | f := func(numConfirmations int) { 114 | if numConfirmations != c.numConfirmations { 115 | t.Errorf("case %d: bad %d != %d", i, numConfirmations, c.numConfirmations) 116 | } 117 | 118 | ch <- time.Now().Sub(start) 119 | } 120 | 121 | // Create the timer and add the requested confirmations. Wait 122 | // the fudge amount to help make sure we calculate the timeout 123 | // overall, and don't accumulate extra time. 124 | s := newSuspicion(c.from, k, min, max, f) 125 | fudge := 25 * time.Millisecond 126 | for _, p := range c.confirmations { 127 | time.Sleep(fudge) 128 | if s.Confirm(p.from) != p.newInfo { 129 | t.Fatalf("case %d: newInfo mismatch for %s", i, p.from) 130 | } 131 | } 132 | 133 | // Wait until right before the timeout and make sure the 134 | // timer hasn't fired. 135 | already := time.Duration(len(c.confirmations)) * fudge 136 | time.Sleep(c.expected - already - fudge) 137 | select { 138 | case d := <-ch: 139 | t.Fatalf("case %d: should not have fired (%9.6f)", i, d.Seconds()) 140 | default: 141 | } 142 | 143 | // Wait through the timeout and a little after and make sure it 144 | // fires. 145 | time.Sleep(2 * fudge) 146 | select { 147 | case <-ch: 148 | default: 149 | t.Fatalf("case %d: should have fired", i) 150 | } 151 | 152 | // Confirm after to make sure it handles a negative remaining 153 | // time correctly and doesn't fire again. 154 | s.Confirm("late") 155 | time.Sleep(c.expected + 2*fudge) 156 | select { 157 | case d := <-ch: 158 | t.Fatalf("case %d: should not have fired (%9.6f)", i, d.Seconds()) 159 | default: 160 | } 161 | } 162 | } 163 | 164 | func TestSuspicion_Timer_ZeroK(t *testing.T) { 165 | ch := make(chan struct{}, 1) 166 | f := func(int) { 167 | ch <- struct{}{} 168 | } 169 | 170 | // This should select the min time since there are no expected 171 | // confirmations to accelerate the timer. 172 | s := newSuspicion("me", 0, 25*time.Millisecond, 30*time.Second, f) 173 | if s.Confirm("foo") { 174 | t.Fatalf("should not provide new information") 175 | } 176 | 177 | select { 178 | case <-ch: 179 | case <-time.After(50 * time.Millisecond): 180 | t.Fatalf("should have fired") 181 | } 182 | } 183 | 184 | func TestSuspicion_Timer_Immediate(t *testing.T) { 185 | ch := make(chan struct{}, 1) 186 | f := func(int) { 187 | ch <- struct{}{} 188 | } 189 | 190 | // This should underflow the timeout and fire immediately. 191 | s := newSuspicion("me", 1, 100*time.Millisecond, 30*time.Second, f) 192 | time.Sleep(200 * time.Millisecond) 193 | s.Confirm("foo") 194 | 195 | // Wait a little while since the function gets called in a goroutine. 196 | select { 197 | case <-ch: 198 | case <-time.After(25 * time.Millisecond): 199 | t.Fatalf("should have fired") 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /tag.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) HashiCorp, Inc. 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | set -e 6 | 7 | # The version must be supplied from the environment. Do not include the 8 | # leading "v". 9 | if [ -z $VERSION ]; then 10 | echo "Please specify a version." 11 | exit 1 12 | fi 13 | 14 | # Generate the tag. 15 | echo "==> Tagging version $VERSION..." 16 | git commit --allow-empty -a --gpg-sign=348FFC4C -m "Release v$VERSION" 17 | git tag -a -m "Version $VERSION" -s -u 348FFC4C "v${VERSION}" master 18 | 19 | exit 0 20 | -------------------------------------------------------------------------------- /test/setup_subnet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) HashiCorp, Inc. 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | # 6 | # This script makes sure that 127.0.0.x is routable. On Darwin, there 7 | # is a bug that it isn't routable and this causes errors. 8 | # 9 | 10 | action=${1:-up} 11 | 12 | if [ "$action" = "up" ] 13 | then 14 | # Check if loopback is setup 15 | ping -c 1 -W 10 127.0.0.2 > /dev/null 2>&1 16 | if [ $? -eq 0 ] 17 | then 18 | exit 19 | fi 20 | fi 21 | 22 | # If we're not on OS X, then error 23 | case $OSTYPE in 24 | darwin*) 25 | ;; 26 | *) 27 | echo "Can't setup interfaces on non-Mac. Error!" 28 | exit 1 29 | ;; 30 | esac 31 | 32 | # Setup loopback 33 | for j in 0 1 2 34 | do 35 | for ((i=2;i<256;i++)) 36 | do 37 | if [ "$action" = "up" ] 38 | then 39 | sudo ifconfig lo0 alias 127.0.$j.$i up 40 | else 41 | sudo ifconfig lo0 127.0.$j.$i delete 42 | fi 43 | done 44 | done 45 | -------------------------------------------------------------------------------- /todo.md: -------------------------------------------------------------------------------- 1 | # TODO 2 | * Dynamic RTT discovery 3 | * Compute 99th percentile for ping/ack 4 | * Better lower bound for ping/ack, faster failure detection 5 | * Dynamic MTU discovery 6 | * Prevent lost updates, increases efficiency 7 | -------------------------------------------------------------------------------- /transport.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "fmt" 8 | "net" 9 | "time" 10 | ) 11 | 12 | // Packet is used to provide some metadata about incoming packets from peers 13 | // over a packet connection, as well as the packet payload. 14 | type Packet struct { 15 | // Buf has the raw contents of the packet. 16 | Buf []byte 17 | 18 | // From has the address of the peer. This is an actual net.Addr so we 19 | // can expose some concrete details about incoming packets. 20 | From net.Addr 21 | 22 | // Timestamp is the time when the packet was received. This should be 23 | // taken as close as possible to the actual receipt time to help make an 24 | // accurate RTT measurement during probes. 25 | Timestamp time.Time 26 | } 27 | 28 | // Transport is used to abstract over communicating with other peers. The packet 29 | // interface is assumed to be best-effort and the stream interface is assumed to 30 | // be reliable. 31 | type Transport interface { 32 | // FinalAdvertiseAddr is given the user's configured values (which 33 | // might be empty) and returns the desired IP and port to advertise to 34 | // the rest of the cluster. 35 | FinalAdvertiseAddr(ip string, port int) (net.IP, int, error) 36 | 37 | // WriteTo is a packet-oriented interface that fires off the given 38 | // payload to the given address in a connectionless fashion. This should 39 | // return a time stamp that's as close as possible to when the packet 40 | // was transmitted to help make accurate RTT measurements during probes. 41 | // 42 | // This is similar to net.PacketConn, though we didn't want to expose 43 | // that full set of required methods to keep assumptions about the 44 | // underlying plumbing to a minimum. We also treat the address here as a 45 | // string, similar to Dial, so it's network neutral, so this usually is 46 | // in the form of "host:port". 47 | WriteTo(b []byte, addr string) (time.Time, error) 48 | 49 | // PacketCh returns a channel that can be read to receive incoming 50 | // packets from other peers. How this is set up for listening is left as 51 | // an exercise for the concrete transport implementations. 52 | PacketCh() <-chan *Packet 53 | 54 | // DialTimeout is used to create a connection that allows us to perform 55 | // two-way communication with a peer. This is generally more expensive 56 | // than packet connections so is used for more infrequent operations 57 | // such as anti-entropy or fallback probes if the packet-oriented probe 58 | // failed. 59 | DialTimeout(addr string, timeout time.Duration) (net.Conn, error) 60 | 61 | // StreamCh returns a channel that can be read to handle incoming stream 62 | // connections from other peers. How this is set up for listening is 63 | // left as an exercise for the concrete transport implementations. 64 | StreamCh() <-chan net.Conn 65 | 66 | // Shutdown is called when memberlist is shutting down; this gives the 67 | // transport a chance to clean up any listeners. 68 | Shutdown() error 69 | } 70 | 71 | type Address struct { 72 | // Addr is a network address as a string, similar to Dial. This usually is 73 | // in the form of "host:port". This is required. 74 | Addr string 75 | 76 | // Name is the name of the node being addressed. This is optional but 77 | // transports may require it. 78 | Name string 79 | } 80 | 81 | func (a *Address) String() string { 82 | if a.Name != "" { 83 | return fmt.Sprintf("%s (%s)", a.Name, a.Addr) 84 | } 85 | return a.Addr 86 | } 87 | 88 | // IngestionAwareTransport is not used. 89 | // 90 | // Deprecated: IngestionAwareTransport is not used and may be removed in a future 91 | // version. Define the interface locally instead of referencing this exported 92 | // interface. 93 | type IngestionAwareTransport interface { 94 | IngestPacket(conn net.Conn, addr net.Addr, now time.Time, shouldClose bool) error 95 | IngestStream(conn net.Conn) error 96 | } 97 | 98 | type NodeAwareTransport interface { 99 | Transport 100 | WriteToAddress(b []byte, addr Address) (time.Time, error) 101 | DialAddressTimeout(addr Address, timeout time.Duration) (net.Conn, error) 102 | } 103 | 104 | type shimNodeAwareTransport struct { 105 | Transport 106 | } 107 | 108 | var _ NodeAwareTransport = (*shimNodeAwareTransport)(nil) 109 | 110 | func (t *shimNodeAwareTransport) WriteToAddress(b []byte, addr Address) (time.Time, error) { 111 | return t.WriteTo(b, addr.Addr) 112 | } 113 | 114 | func (t *shimNodeAwareTransport) DialAddressTimeout(addr Address, timeout time.Duration) (net.Conn, error) { 115 | return t.DialTimeout(addr.Addr, timeout) 116 | } 117 | 118 | type labelWrappedTransport struct { 119 | label string 120 | NodeAwareTransport 121 | } 122 | 123 | var _ NodeAwareTransport = (*labelWrappedTransport)(nil) 124 | 125 | func (t *labelWrappedTransport) WriteToAddress(buf []byte, addr Address) (time.Time, error) { 126 | var err error 127 | buf, err = AddLabelHeaderToPacket(buf, t.label) 128 | if err != nil { 129 | return time.Time{}, fmt.Errorf("failed to add label header to packet: %w", err) 130 | } 131 | return t.NodeAwareTransport.WriteToAddress(buf, addr) 132 | } 133 | 134 | func (t *labelWrappedTransport) WriteTo(buf []byte, addr string) (time.Time, error) { 135 | var err error 136 | buf, err = AddLabelHeaderToPacket(buf, t.label) 137 | if err != nil { 138 | return time.Time{}, err 139 | } 140 | return t.NodeAwareTransport.WriteTo(buf, addr) 141 | } 142 | 143 | func (t *labelWrappedTransport) DialAddressTimeout(addr Address, timeout time.Duration) (net.Conn, error) { 144 | conn, err := t.NodeAwareTransport.DialAddressTimeout(addr, timeout) 145 | if err != nil { 146 | return nil, err 147 | } 148 | if err := AddLabelHeaderToStream(conn, t.label); err != nil { 149 | return nil, fmt.Errorf("failed to add label header to stream: %w", err) 150 | } 151 | return conn, nil 152 | } 153 | 154 | func (t *labelWrappedTransport) DialTimeout(addr string, timeout time.Duration) (net.Conn, error) { 155 | conn, err := t.NodeAwareTransport.DialTimeout(addr, timeout) 156 | if err != nil { 157 | return nil, err 158 | } 159 | if err := AddLabelHeaderToStream(conn, t.label); err != nil { 160 | return nil, fmt.Errorf("failed to add label header to stream: %w", err) 161 | } 162 | return conn, nil 163 | } 164 | -------------------------------------------------------------------------------- /transport_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "log" 8 | "net" 9 | "strings" 10 | "sync/atomic" 11 | "testing" 12 | "time" 13 | 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | func TestTransport_Join(t *testing.T) { 18 | net := &MockNetwork{} 19 | 20 | t1 := net.NewTransport("node1") 21 | 22 | c1 := DefaultLANConfig() 23 | c1.Name = "node1" 24 | c1.Transport = t1 25 | m1, err := Create(c1) 26 | if err != nil { 27 | t.Fatalf("err: %v", err) 28 | } 29 | m1.setAlive() 30 | m1.schedule() 31 | defer m1.Shutdown() 32 | 33 | c2 := DefaultLANConfig() 34 | c2.Name = "node2" 35 | c2.Transport = net.NewTransport("node2") 36 | m2, err := Create(c2) 37 | if err != nil { 38 | t.Fatalf("err: %v", err) 39 | } 40 | m2.setAlive() 41 | m2.schedule() 42 | defer m2.Shutdown() 43 | 44 | num, err := m2.Join([]string{c1.Name + "/" + t1.addr.String()}) 45 | if num != 1 { 46 | t.Fatalf("bad: %d", num) 47 | } 48 | if err != nil { 49 | t.Fatalf("err: %v", err) 50 | } 51 | 52 | if len(m2.Members()) != 2 { 53 | t.Fatalf("bad: %v", m2.Members()) 54 | } 55 | if m2.estNumNodes() != 2 { 56 | t.Fatalf("bad: %v", m2.Members()) 57 | } 58 | 59 | } 60 | 61 | func TestTransport_Send(t *testing.T) { 62 | net := &MockNetwork{} 63 | 64 | t1 := net.NewTransport("node1") 65 | d1 := &MockDelegate{} 66 | 67 | c1 := DefaultLANConfig() 68 | c1.Name = "node1" 69 | c1.Transport = t1 70 | c1.Delegate = d1 71 | m1, err := Create(c1) 72 | if err != nil { 73 | t.Fatalf("err: %v", err) 74 | } 75 | m1.setAlive() 76 | m1.schedule() 77 | defer m1.Shutdown() 78 | 79 | c2 := DefaultLANConfig() 80 | c2.Name = "node2" 81 | c2.Transport = net.NewTransport("node2") 82 | m2, err := Create(c2) 83 | if err != nil { 84 | t.Fatalf("err: %v", err) 85 | } 86 | m2.setAlive() 87 | m2.schedule() 88 | defer m2.Shutdown() 89 | 90 | num, err := m2.Join([]string{c1.Name + "/" + t1.addr.String()}) 91 | if num != 1 { 92 | t.Fatalf("bad: %d", num) 93 | } 94 | if err != nil { 95 | t.Fatalf("err: %v", err) 96 | } 97 | 98 | if err := m2.SendTo(t1.addr, []byte("SendTo")); err != nil { 99 | t.Fatalf("err: %v", err) 100 | } 101 | 102 | var n1 *Node 103 | for _, n := range m2.Members() { 104 | if n.Name == c1.Name { 105 | n1 = n 106 | break 107 | } 108 | } 109 | if n1 == nil { 110 | t.Fatalf("bad") 111 | } 112 | 113 | if err := m2.SendToUDP(n1, []byte("SendToUDP")); err != nil { 114 | t.Fatalf("err: %v", err) 115 | } 116 | if err := m2.SendToTCP(n1, []byte("SendToTCP")); err != nil { 117 | t.Fatalf("err: %v", err) 118 | } 119 | if err := m2.SendBestEffort(n1, []byte("SendBestEffort")); err != nil { 120 | t.Fatalf("err: %v", err) 121 | } 122 | if err := m2.SendReliable(n1, []byte("SendReliable")); err != nil { 123 | t.Fatalf("err: %v", err) 124 | } 125 | time.Sleep(100 * time.Millisecond) 126 | 127 | expected := []string{"SendTo", "SendToUDP", "SendToTCP", "SendBestEffort", "SendReliable"} 128 | 129 | msgs1 := d1.getMessages() 130 | 131 | received := make([]string, len(msgs1)) 132 | for i, bs := range msgs1 { 133 | received[i] = string(bs) 134 | } 135 | // Some of these are UDP so often get re-ordered making the test flaky if we 136 | // assert send ordering. Sort both slices to be tolerant of re-ordering. 137 | require.ElementsMatch(t, expected, received) 138 | } 139 | 140 | type testCountingWriter struct { 141 | t *testing.T 142 | numCalls *int32 143 | } 144 | 145 | func (tw testCountingWriter) Write(p []byte) (n int, err error) { 146 | atomic.AddInt32(tw.numCalls, 1) 147 | if !strings.Contains(string(p), "memberlist: Error accepting TCP connection") { 148 | tw.t.Error("did not receive expected log message") 149 | } 150 | tw.t.Log("countingWriter:", string(p)) 151 | return len(p), nil 152 | } 153 | 154 | // TestTransport_TcpListenBackoff tests that AcceptTCP() errors in NetTransport#tcpListen() 155 | // do not result in a tight loop and spam the log. We verify this here by counting the number 156 | // of entries logged in a given time period. 157 | func TestTransport_TcpListenBackoff(t *testing.T) { 158 | 159 | // testTime is the amount of time we will allow NetTransport#tcpListen() to run 160 | // This needs to be long enough that to verify that maxDelay is in force, 161 | // but not so long as to be obnoxious when running the test suite. 162 | const testTime = 4 * time.Second 163 | 164 | var numCalls int32 165 | countingWriter := testCountingWriter{t, &numCalls} 166 | countingLogger := log.New(countingWriter, "test", log.LstdFlags) 167 | transport := NetTransport{ 168 | streamCh: make(chan net.Conn), 169 | logger: countingLogger, 170 | } 171 | transport.wg.Add(1) 172 | 173 | // create a listener that will cause AcceptTCP calls to fail 174 | listener, _ := net.ListenTCP("tcp", nil) 175 | listener.Close() 176 | go transport.tcpListen(listener) 177 | 178 | // sleep (+yield) for testTime seconds before asking the accept loop to shut down 179 | time.Sleep(testTime) 180 | atomic.StoreInt32(&transport.shutdown, 1) 181 | 182 | // Verify that the wg was completed on exit (but without blocking this test) 183 | // maxDelay == 1s, so we will give the routine 1.25s to loop around and shut down. 184 | c := make(chan struct{}) 185 | go func() { 186 | defer close(c) 187 | transport.wg.Wait() 188 | }() 189 | select { 190 | case <-c: 191 | case <-time.After(1250 * time.Millisecond): 192 | t.Error("timed out waiting for transport waitgroup to be done after flagging shutdown") 193 | } 194 | 195 | // In testTime==4s, we expect to loop approximately 12 times (and log approximately 11 errors), 196 | // with the following delays (in ms): 197 | // 0+5+10+20+40+80+160+320+640+1000+1000+1000 == 4275 ms 198 | // Too few calls suggests that the minDelay is not in force; too many calls suggests that the 199 | // maxDelay is not in force or that the back-off isn't working at all. 200 | // We'll leave a little flex; the important thing here is the asymptotic behavior. 201 | // If the minDelay or maxDelay in NetTransport#tcpListen() are modified, this test may fail 202 | // and need to be adjusted. 203 | require.True(t, numCalls > 8) 204 | require.True(t, numCalls < 14) 205 | 206 | // no connections should have been accepted and sent to the channel 207 | require.Equal(t, len(transport.streamCh), 0) 208 | } 209 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "bytes" 8 | "compress/lzw" 9 | "encoding/binary" 10 | "fmt" 11 | "io" 12 | "math" 13 | "math/rand" 14 | "net" 15 | "strconv" 16 | "strings" 17 | "time" 18 | 19 | "github.com/hashicorp/go-msgpack/v2/codec" 20 | "github.com/sean-/seed" 21 | ) 22 | 23 | // pushPullScale is the minimum number of nodes 24 | // before we start scaling the push/pull timing. The scale 25 | // effect is the log2(Nodes) - log2(pushPullScale). This means 26 | // that the 33rd node will cause us to double the interval, 27 | // while the 65th will triple it. 28 | const pushPullScaleThreshold = 32 29 | 30 | const ( 31 | // Constant litWidth 2-8 32 | lzwLitWidth = 8 33 | ) 34 | 35 | func init() { 36 | seed.Init() 37 | } 38 | 39 | // Decode reverses the encode operation on a byte slice input 40 | func decode(buf []byte, out interface{}) error { 41 | r := bytes.NewReader(buf) 42 | hd := codec.MsgpackHandle{} 43 | dec := codec.NewDecoder(r, &hd) 44 | return dec.Decode(out) 45 | } 46 | 47 | // Encode writes an encoded object to a new bytes buffer 48 | func encode(msgType messageType, in interface{}, msgpackUseNewTimeFormat bool) (*bytes.Buffer, error) { 49 | buf := bytes.NewBuffer(nil) 50 | buf.WriteByte(uint8(msgType)) 51 | hd := codec.MsgpackHandle{ 52 | BasicHandle: codec.BasicHandle{ 53 | TimeNotBuiltin: !msgpackUseNewTimeFormat, 54 | }, 55 | } 56 | enc := codec.NewEncoder(buf, &hd) 57 | err := enc.Encode(in) 58 | return buf, err 59 | } 60 | 61 | // Returns a random offset between 0 and n 62 | func randomOffset(n int) int { 63 | if n == 0 { 64 | return 0 65 | } 66 | return int(rand.Uint32() % uint32(n)) 67 | } 68 | 69 | // suspicionTimeout computes the timeout that should be used when 70 | // a node is suspected 71 | func suspicionTimeout(suspicionMult, n int, interval time.Duration) time.Duration { 72 | nodeScale := math.Max(1.0, math.Log10(math.Max(1.0, float64(n)))) 73 | // multiply by 1000 to keep some precision because time.Duration is an int64 type 74 | timeout := time.Duration(suspicionMult) * time.Duration(nodeScale*1000) * interval / 1000 75 | return timeout 76 | } 77 | 78 | // retransmitLimit computes the limit of retransmissions 79 | func retransmitLimit(retransmitMult, n int) int { 80 | nodeScale := math.Ceil(math.Log10(float64(n + 1))) 81 | limit := retransmitMult * int(nodeScale) 82 | return limit 83 | } 84 | 85 | // shuffleNodes randomly shuffles the input nodes using the Fisher-Yates shuffle 86 | func shuffleNodes(nodes []*nodeState) { 87 | n := len(nodes) 88 | rand.Shuffle(n, func(i, j int) { 89 | nodes[i], nodes[j] = nodes[j], nodes[i] 90 | }) 91 | } 92 | 93 | // pushPushScale is used to scale the time interval at which push/pull 94 | // syncs take place. It is used to prevent network saturation as the 95 | // cluster size grows 96 | func pushPullScale(interval time.Duration, n int) time.Duration { 97 | // Don't scale until we cross the threshold 98 | if n <= pushPullScaleThreshold { 99 | return interval 100 | } 101 | 102 | multiplier := math.Ceil(math.Log2(float64(n))-math.Log2(pushPullScaleThreshold)) + 1.0 103 | return time.Duration(multiplier) * interval 104 | } 105 | 106 | // moveDeadNodes moves dead and left nodes that that have not changed during the gossipToTheDeadTime interval 107 | // to the end of the slice and returns the index of the first moved node. 108 | func moveDeadNodes(nodes []*nodeState, gossipToTheDeadTime time.Duration) int { 109 | numDead := 0 110 | n := len(nodes) 111 | for i := 0; i < n-numDead; i++ { 112 | if !nodes[i].DeadOrLeft() { 113 | continue 114 | } 115 | 116 | // Respect the gossip to the dead interval 117 | if time.Since(nodes[i].StateChange) <= gossipToTheDeadTime { 118 | continue 119 | } 120 | 121 | // Move this node to the end 122 | nodes[i], nodes[n-numDead-1] = nodes[n-numDead-1], nodes[i] 123 | numDead++ 124 | i-- 125 | } 126 | return n - numDead 127 | } 128 | 129 | // kRandomNodes is used to select up to k random Nodes, excluding any nodes where 130 | // the exclude function returns true. It is possible that less than k nodes are 131 | // returned. 132 | func kRandomNodes(k int, nodes []*nodeState, exclude func(*nodeState) bool) []Node { 133 | n := len(nodes) 134 | kNodes := make([]Node, 0, k) 135 | OUTER: 136 | // Probe up to 3*n times, with large n this is not necessary 137 | // since k << n, but with small n we want search to be 138 | // exhaustive 139 | for i := 0; i < 3*n && len(kNodes) < k; i++ { 140 | // Get random nodeState 141 | idx := randomOffset(n) 142 | state := nodes[idx] 143 | 144 | // Give the filter a shot at it. 145 | if exclude != nil && exclude(state) { 146 | continue OUTER 147 | } 148 | 149 | // Check if we have this node already 150 | for j := 0; j < len(kNodes); j++ { 151 | if state.Node.Name == kNodes[j].Name { 152 | continue OUTER 153 | } 154 | } 155 | 156 | // Append the node 157 | kNodes = append(kNodes, state.Node) 158 | } 159 | return kNodes 160 | } 161 | 162 | // makeCompoundMessages takes a list of messages and packs 163 | // them into one or multiple messages based on the limitations 164 | // of compound messages (255 messages each). 165 | func makeCompoundMessages(msgs [][]byte) []*bytes.Buffer { 166 | const maxMsgs = 255 167 | bufs := make([]*bytes.Buffer, 0, (len(msgs)+(maxMsgs-1))/maxMsgs) 168 | 169 | for ; len(msgs) > maxMsgs; msgs = msgs[maxMsgs:] { 170 | bufs = append(bufs, makeCompoundMessage(msgs[:maxMsgs])) 171 | } 172 | if len(msgs) > 0 { 173 | bufs = append(bufs, makeCompoundMessage(msgs)) 174 | } 175 | 176 | return bufs 177 | } 178 | 179 | // makeCompoundMessage takes a list of messages and generates 180 | // a single compound message containing all of them 181 | func makeCompoundMessage(msgs [][]byte) *bytes.Buffer { 182 | // Create a local buffer 183 | buf := bytes.NewBuffer(nil) 184 | 185 | // Write out the type 186 | buf.WriteByte(uint8(compoundMsg)) 187 | 188 | // Write out the number of message 189 | buf.WriteByte(uint8(len(msgs))) 190 | 191 | // Add the message lengths 192 | for _, m := range msgs { 193 | binary.Write(buf, binary.BigEndian, uint16(len(m))) 194 | } 195 | 196 | // Append the messages 197 | for _, m := range msgs { 198 | buf.Write(m) 199 | } 200 | 201 | return buf 202 | } 203 | 204 | // decodeCompoundMessage splits a compound message and returns 205 | // the slices of individual messages. Also returns the number 206 | // of truncated messages and any potential error 207 | func decodeCompoundMessage(buf []byte) (trunc int, parts [][]byte, err error) { 208 | if len(buf) < 1 { 209 | err = fmt.Errorf("missing compound length byte") 210 | return 211 | } 212 | numParts := int(buf[0]) 213 | buf = buf[1:] 214 | 215 | // Check we have enough bytes 216 | if len(buf) < numParts*2 { 217 | err = fmt.Errorf("truncated len slice") 218 | return 219 | } 220 | 221 | // Decode the lengths 222 | lengths := make([]uint16, numParts) 223 | for i := 0; i < numParts; i++ { 224 | lengths[i] = binary.BigEndian.Uint16(buf[i*2 : i*2+2]) 225 | } 226 | buf = buf[numParts*2:] 227 | 228 | // Split each message 229 | for idx, msgLen := range lengths { 230 | if len(buf) < int(msgLen) { 231 | trunc = numParts - idx 232 | return 233 | } 234 | 235 | // Extract the slice, seek past on the buffer 236 | slice := buf[:msgLen] 237 | buf = buf[msgLen:] 238 | parts = append(parts, slice) 239 | } 240 | return 241 | } 242 | 243 | // compressPayload takes an opaque input buffer, compresses it 244 | // and wraps it in a compress{} message that is encoded. 245 | func compressPayload(inp []byte, msgpackUseNewTimeFormat bool) (*bytes.Buffer, error) { 246 | var buf bytes.Buffer 247 | compressor := lzw.NewWriter(&buf, lzw.LSB, lzwLitWidth) 248 | 249 | _, err := compressor.Write(inp) 250 | if err != nil { 251 | return nil, err 252 | } 253 | 254 | // Ensure we flush everything out 255 | if err := compressor.Close(); err != nil { 256 | return nil, err 257 | } 258 | 259 | // Create a compressed message 260 | c := compress{ 261 | Algo: lzwAlgo, 262 | Buf: buf.Bytes(), 263 | } 264 | return encode(compressMsg, &c, msgpackUseNewTimeFormat) 265 | } 266 | 267 | // decompressPayload is used to unpack an encoded compress{} 268 | // message and return its payload uncompressed 269 | func decompressPayload(msg []byte) ([]byte, error) { 270 | // Decode the message 271 | var c compress 272 | if err := decode(msg, &c); err != nil { 273 | return nil, err 274 | } 275 | return decompressBuffer(&c) 276 | } 277 | 278 | // decompressBuffer is used to decompress the buffer of 279 | // a single compress message, handling multiple algorithms 280 | func decompressBuffer(c *compress) ([]byte, error) { 281 | // Verify the algorithm 282 | if c.Algo != lzwAlgo { 283 | return nil, fmt.Errorf("Cannot decompress unknown algorithm %d", c.Algo) 284 | } 285 | 286 | // Create a uncompressor 287 | uncomp := lzw.NewReader(bytes.NewReader(c.Buf), lzw.LSB, lzwLitWidth) 288 | defer uncomp.Close() 289 | 290 | // Read all the data 291 | var b bytes.Buffer 292 | _, err := io.Copy(&b, uncomp) 293 | if err != nil { 294 | return nil, err 295 | } 296 | 297 | // Return the uncompressed bytes 298 | return b.Bytes(), nil 299 | } 300 | 301 | // joinHostPort returns the host:port form of an address, for use with a 302 | // transport. 303 | func joinHostPort(host string, port uint16) string { 304 | return net.JoinHostPort(host, strconv.Itoa(int(port))) 305 | } 306 | 307 | // hasPort is given a string of the form "host", "host:port", "ipv6::address", 308 | // or "[ipv6::address]:port", and returns true if the string includes a port. 309 | func hasPort(s string) bool { 310 | // IPv6 address in brackets. 311 | if strings.LastIndex(s, "[") == 0 { 312 | return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") 313 | } 314 | 315 | // Otherwise the presence of a single colon determines if there's a port 316 | // since IPv6 addresses outside of brackets (count > 1) can't have a 317 | // port. 318 | return strings.Count(s, ":") == 1 319 | } 320 | 321 | // ensurePort makes sure the given string has a port number on it, otherwise it 322 | // appends the given port as a default. 323 | func ensurePort(s string, port int) string { 324 | if hasPort(s) { 325 | return s 326 | } 327 | 328 | // If this is an IPv6 address, the join call will add another set of 329 | // brackets, so we have to trim before we add the default port. 330 | s = strings.Trim(s, "[]") 331 | s = net.JoinHostPort(s, strconv.Itoa(port)) 332 | return s 333 | } 334 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package memberlist 5 | 6 | import ( 7 | "fmt" 8 | "reflect" 9 | "testing" 10 | "time" 11 | 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestUtil_PortFunctions(t *testing.T) { 16 | tests := []struct { 17 | addr string 18 | hasPort bool 19 | ensurePort string 20 | }{ 21 | {"1.2.3.4", false, "1.2.3.4:8301"}, 22 | {"1.2.3.4:1234", true, "1.2.3.4:1234"}, 23 | {"2600:1f14:e22:1501:f9a:2e0c:a167:67e8", false, "[2600:1f14:e22:1501:f9a:2e0c:a167:67e8]:8301"}, 24 | {"[2600:1f14:e22:1501:f9a:2e0c:a167:67e8]", false, "[2600:1f14:e22:1501:f9a:2e0c:a167:67e8]:8301"}, 25 | {"[2600:1f14:e22:1501:f9a:2e0c:a167:67e8]:1234", true, "[2600:1f14:e22:1501:f9a:2e0c:a167:67e8]:1234"}, 26 | {"localhost", false, "localhost:8301"}, 27 | {"localhost:1234", true, "localhost:1234"}, 28 | {"hashicorp.com", false, "hashicorp.com:8301"}, 29 | {"hashicorp.com:1234", true, "hashicorp.com:1234"}, 30 | } 31 | for _, tt := range tests { 32 | t.Run(tt.addr, func(t *testing.T) { 33 | if got, want := hasPort(tt.addr), tt.hasPort; got != want { 34 | t.Fatalf("got %v want %v", got, want) 35 | } 36 | if got, want := ensurePort(tt.addr, 8301), tt.ensurePort; got != want { 37 | t.Fatalf("got %v want %v", got, want) 38 | } 39 | }) 40 | } 41 | } 42 | 43 | func TestEncodeDecode(t *testing.T) { 44 | msg := &ping{SeqNo: 100} 45 | buf, err := encode(pingMsg, msg, false) 46 | if err != nil { 47 | t.Fatalf("unexpected err: %s", err) 48 | } 49 | var out ping 50 | if err := decode(buf.Bytes()[1:], &out); err != nil { 51 | t.Fatalf("unexpected err: %s", err) 52 | } 53 | if msg.SeqNo != out.SeqNo { 54 | t.Fatalf("bad sequence no") 55 | } 56 | } 57 | 58 | func TestRandomOffset(t *testing.T) { 59 | vals := make(map[int]struct{}) 60 | for i := 0; i < 100; i++ { 61 | offset := randomOffset(2 << 30) 62 | if _, ok := vals[offset]; ok { 63 | t.Fatalf("got collision") 64 | } 65 | vals[offset] = struct{}{} 66 | } 67 | } 68 | 69 | func TestRandomOffset_Zero(t *testing.T) { 70 | offset := randomOffset(0) 71 | if offset != 0 { 72 | t.Fatalf("bad offset") 73 | } 74 | } 75 | 76 | func TestSuspicionTimeout(t *testing.T) { 77 | timeouts := map[int]time.Duration{ 78 | 5: 1000 * time.Millisecond, 79 | 10: 1000 * time.Millisecond, 80 | 50: 1698 * time.Millisecond, 81 | 100: 2000 * time.Millisecond, 82 | 500: 2698 * time.Millisecond, 83 | 1000: 3000 * time.Millisecond, 84 | } 85 | for n, expected := range timeouts { 86 | timeout := suspicionTimeout(3, n, time.Second) / 3 87 | if timeout != expected { 88 | t.Fatalf("bad: %v, %v", expected, timeout) 89 | } 90 | } 91 | } 92 | 93 | func TestRetransmitLimit(t *testing.T) { 94 | lim := retransmitLimit(3, 0) 95 | if lim != 0 { 96 | t.Fatalf("bad val %v", lim) 97 | } 98 | lim = retransmitLimit(3, 1) 99 | if lim != 3 { 100 | t.Fatalf("bad val %v", lim) 101 | } 102 | lim = retransmitLimit(3, 99) 103 | if lim != 6 { 104 | t.Fatalf("bad val %v", lim) 105 | } 106 | } 107 | 108 | func TestShuffleNodes(t *testing.T) { 109 | orig := []*nodeState{ 110 | &nodeState{ 111 | State: StateDead, 112 | }, 113 | &nodeState{ 114 | State: StateAlive, 115 | }, 116 | &nodeState{ 117 | State: StateAlive, 118 | }, 119 | &nodeState{ 120 | State: StateDead, 121 | }, 122 | &nodeState{ 123 | State: StateAlive, 124 | }, 125 | &nodeState{ 126 | State: StateAlive, 127 | }, 128 | &nodeState{ 129 | State: StateDead, 130 | }, 131 | &nodeState{ 132 | State: StateAlive, 133 | }, 134 | } 135 | nodes := make([]*nodeState, len(orig)) 136 | copy(nodes[:], orig[:]) 137 | 138 | if !reflect.DeepEqual(nodes, orig) { 139 | t.Fatalf("should match") 140 | } 141 | 142 | shuffleNodes(nodes) 143 | 144 | if reflect.DeepEqual(nodes, orig) { 145 | t.Fatalf("should not match") 146 | } 147 | } 148 | 149 | func TestPushPullScale(t *testing.T) { 150 | sec := time.Second 151 | for i := 0; i <= 32; i++ { 152 | if s := pushPullScale(sec, i); s != sec { 153 | t.Fatalf("Bad time scale: %v", s) 154 | } 155 | } 156 | for i := 33; i <= 64; i++ { 157 | if s := pushPullScale(sec, i); s != 2*sec { 158 | t.Fatalf("Bad time scale: %v", s) 159 | } 160 | } 161 | for i := 65; i <= 128; i++ { 162 | if s := pushPullScale(sec, i); s != 3*sec { 163 | t.Fatalf("Bad time scale: %v", s) 164 | } 165 | } 166 | } 167 | 168 | func TestMoveDeadNodes(t *testing.T) { 169 | nodes := []*nodeState{ 170 | &nodeState{ 171 | State: StateDead, 172 | StateChange: time.Now().Add(-20 * time.Second), 173 | }, 174 | &nodeState{ 175 | State: StateAlive, 176 | StateChange: time.Now().Add(-20 * time.Second), 177 | }, 178 | // This dead node should not be moved, as its state changed 179 | // less than the specified GossipToTheDead time ago 180 | &nodeState{ 181 | State: StateDead, 182 | StateChange: time.Now().Add(-10 * time.Second), 183 | }, 184 | // This left node should not be moved, as its state changed 185 | // less than the specified GossipToTheDead time ago 186 | &nodeState{ 187 | State: StateLeft, 188 | StateChange: time.Now().Add(-10 * time.Second), 189 | }, 190 | &nodeState{ 191 | State: StateLeft, 192 | StateChange: time.Now().Add(-20 * time.Second), 193 | }, 194 | &nodeState{ 195 | State: StateAlive, 196 | StateChange: time.Now().Add(-20 * time.Second), 197 | }, 198 | &nodeState{ 199 | State: StateDead, 200 | StateChange: time.Now().Add(-20 * time.Second), 201 | }, 202 | &nodeState{ 203 | State: StateAlive, 204 | StateChange: time.Now().Add(-20 * time.Second), 205 | }, 206 | &nodeState{ 207 | State: StateLeft, 208 | StateChange: time.Now().Add(-20 * time.Second), 209 | }, 210 | } 211 | 212 | idx := moveDeadNodes(nodes, (15 * time.Second)) 213 | if idx != 5 { 214 | t.Fatalf("bad index") 215 | } 216 | for i := 0; i < idx; i++ { 217 | switch i { 218 | case 2: 219 | // Recently dead node remains at index 2, 220 | // since nodes are swapped out to move to end. 221 | if nodes[i].State != StateDead { 222 | t.Fatalf("Bad state %d", i) 223 | } 224 | case 3: 225 | //Recently left node should remain at 3 226 | if nodes[i].State != StateLeft { 227 | t.Fatalf("Bad State %d", i) 228 | } 229 | default: 230 | if nodes[i].State != StateAlive { 231 | t.Fatalf("Bad state %d", i) 232 | } 233 | } 234 | } 235 | for i := idx; i < len(nodes); i++ { 236 | if !nodes[i].DeadOrLeft() { 237 | t.Fatalf("Bad state %d", i) 238 | } 239 | } 240 | } 241 | 242 | func TestKRandomNodes(t *testing.T) { 243 | nodes := []*nodeState{} 244 | for i := 0; i < 90; i++ { 245 | // Half the nodes are in a bad state 246 | state := StateAlive 247 | switch i % 3 { 248 | case 0: 249 | state = StateAlive 250 | case 1: 251 | state = StateSuspect 252 | case 2: 253 | state = StateDead 254 | } 255 | nodes = append(nodes, &nodeState{ 256 | Node: Node{ 257 | Name: fmt.Sprintf("test%d", i), 258 | }, 259 | State: state, 260 | }) 261 | } 262 | 263 | filterFunc := func(n *nodeState) bool { 264 | if n.Name == "test0" || n.State != StateAlive { 265 | return true 266 | } 267 | return false 268 | } 269 | 270 | s1 := kRandomNodes(3, nodes, filterFunc) 271 | s2 := kRandomNodes(3, nodes, filterFunc) 272 | s3 := kRandomNodes(3, nodes, filterFunc) 273 | 274 | if reflect.DeepEqual(s1, s2) { 275 | t.Fatalf("unexpected equal") 276 | } 277 | if reflect.DeepEqual(s1, s3) { 278 | t.Fatalf("unexpected equal") 279 | } 280 | if reflect.DeepEqual(s2, s3) { 281 | t.Fatalf("unexpected equal") 282 | } 283 | 284 | for _, s := range [][]Node{s1, s2, s3} { 285 | if len(s) != 3 { 286 | t.Fatalf("bad len") 287 | } 288 | for _, n := range s { 289 | if n.Name == "test0" { 290 | t.Fatalf("Bad name") 291 | } 292 | if n.State != StateAlive { 293 | t.Fatalf("Bad state") 294 | } 295 | } 296 | } 297 | } 298 | 299 | func TestMakeCompoundMessage(t *testing.T) { 300 | msg := &ping{SeqNo: 100} 301 | buf, err := encode(pingMsg, msg, false) 302 | if err != nil { 303 | t.Fatalf("unexpected err: %s", err) 304 | } 305 | 306 | msgs := [][]byte{buf.Bytes(), buf.Bytes(), buf.Bytes()} 307 | compound := makeCompoundMessage(msgs) 308 | 309 | if compound.Len() != 3*buf.Len()+3*compoundOverhead+compoundHeaderOverhead { 310 | t.Fatalf("bad len") 311 | } 312 | } 313 | 314 | func TestDecodeCompoundMessage(t *testing.T) { 315 | msg := &ping{SeqNo: 100} 316 | buf, err := encode(pingMsg, msg, false) 317 | if err != nil { 318 | t.Fatalf("unexpected err: %s", err) 319 | } 320 | 321 | msgs := [][]byte{buf.Bytes(), buf.Bytes(), buf.Bytes()} 322 | compound := makeCompoundMessage(msgs) 323 | 324 | trunc, parts, err := decodeCompoundMessage(compound.Bytes()[1:]) 325 | if err != nil { 326 | t.Fatalf("unexpected err: %s", err) 327 | } 328 | if trunc != 0 { 329 | t.Fatalf("should not truncate") 330 | } 331 | if len(parts) != 3 { 332 | t.Fatalf("bad parts") 333 | } 334 | for _, p := range parts { 335 | if len(p) != buf.Len() { 336 | t.Fatalf("bad part len") 337 | } 338 | } 339 | } 340 | 341 | func TestDecodeCompoundMessage_NumberOfPartsOverflow(t *testing.T) { 342 | buf := []byte{0x80} 343 | _, _, err := decodeCompoundMessage(buf) 344 | require.Error(t, err) 345 | require.Equal(t, err.Error(), "truncated len slice") 346 | } 347 | 348 | func TestDecodeCompoundMessage_Trunc(t *testing.T) { 349 | msg := &ping{SeqNo: 100} 350 | buf, err := encode(pingMsg, msg, false) 351 | if err != nil { 352 | t.Fatalf("unexpected err: %s", err) 353 | } 354 | 355 | msgs := [][]byte{buf.Bytes(), buf.Bytes(), buf.Bytes()} 356 | compound := makeCompoundMessage(msgs) 357 | 358 | trunc, parts, err := decodeCompoundMessage(compound.Bytes()[1:38]) 359 | if err != nil { 360 | t.Fatalf("unexpected err: %s", err) 361 | } 362 | if trunc != 1 { 363 | t.Fatalf("truncate: %d", trunc) 364 | } 365 | if len(parts) != 2 { 366 | t.Fatalf("bad parts") 367 | } 368 | for _, p := range parts { 369 | if len(p) != buf.Len() { 370 | t.Fatalf("bad part len") 371 | } 372 | } 373 | } 374 | 375 | func TestCompressDecompressPayload(t *testing.T) { 376 | buf, err := compressPayload([]byte("testing"), false) 377 | if err != nil { 378 | t.Fatalf("unexpected err: %s", err) 379 | } 380 | 381 | decomp, err := decompressPayload(buf.Bytes()[1:]) 382 | if err != nil { 383 | t.Fatalf("unexpected err: %s", err) 384 | } 385 | 386 | if !reflect.DeepEqual(decomp, []byte("testing")) { 387 | t.Fatalf("bad payload: %v", decomp) 388 | } 389 | } 390 | --------------------------------------------------------------------------------