├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── broadcast.go ├── broadcast_test.go ├── bytes.go ├── bytes_test.go ├── docker-compose.yml ├── events.go ├── events_test.go ├── go.mod ├── go.sum ├── log.go ├── logo └── logo.png ├── membership.go ├── membership_test.go ├── message.go ├── messageVerb.go ├── message_test.go ├── node.go ├── nodeMap.go ├── nodeStatus.go ├── pingData.go ├── properties.go ├── properties_test.go ├── registry.go ├── registry_test.go └── smudge ├── README.md └── smudge.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool 12 | *.out 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.x 5 | 6 | script: 7 | - go test -coverprofile cp.out -v ./... 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # This is a multi-stage Dockerfile. The first part executes a build in a Golang 2 | # container, and the second retrieves the binary from the build container and 3 | # inserts it into a "scratch" image. 4 | 5 | # Part 1: Compile the binary in a containerized Golang environment 6 | # 7 | FROM golang:1.14 as test 8 | 9 | COPY . /go/src/github.com/clockworksoul/smudge 10 | 11 | WORKDIR /go/src/github.com/clockworksoul/smudge 12 | 13 | RUN go test -v ./... 14 | 15 | 16 | # Part 2: Compile the binary in a containerized Go environment 17 | # 18 | FROM golang:1.14 as build 19 | 20 | COPY . /go/src/github.com/clockworksoul/smudge 21 | 22 | RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o /smudge github.com/clockworksoul/smudge/smudge 23 | 24 | 25 | # Part 3: Build the Smudge image proper 26 | # 27 | FROM scratch as image 28 | 29 | COPY --from=build /smudge . 30 | 31 | EXPOSE 9999 32 | 33 | CMD ["/smudge"] 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := help 2 | 3 | ############################ 4 | ## Project Info 5 | ############################ 6 | PROJECT = smudge 7 | GIT_URL = github.com 8 | GIT_ORGANIZATION = clockworksoul 9 | GIT_REPOSITORY = $(GIT_URL)/$(GIT_ORGANIZATION)/$(PROJECT) 10 | 11 | ############################ 12 | ## Docker Registry Info 13 | ############################ 14 | REGISTRY_URL = clockworksoul 15 | IMAGE_NAME = $(REGISTRY_URL)/$(PROJECT) 16 | IMAGE_TAG = latest 17 | 18 | help: 19 | # Commands: 20 | # make help - Show this message 21 | # 22 | # Dev commands: 23 | # make clean - Remove generated files 24 | # make install - Install requirements (none atm) 25 | # make test - Run Go tests 26 | # make build - Build go binary 27 | # 28 | # Docker commands: 29 | # make image - Build Docker image with current version tag 30 | # make run - Run Docker image with current version tag 31 | # 32 | # Deployment commands: 33 | # make push - Push current version tag to registry 34 | 35 | clean: 36 | if [ -d "bin" ]; then rm -R bin; fi 37 | 38 | install: 39 | echo 40 | 41 | test: 42 | @docker build --target test -t foo_$(PROJECT)_foo . 43 | @docker rmi foo_$(PROJECT)_foo 44 | 45 | build: clean 46 | mkdir -p bin 47 | @go build -a -installsuffix cgo -o bin/$(PROJECT) $(GIT_REPOSITORY) 48 | 49 | image: 50 | @docker build -t $(IMAGE_NAME):$(IMAGE_TAG) . 51 | 52 | push: 53 | @docker push $(IMAGE_NAME):$(IMAGE_TAG) 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Smudge 2 | 3 | [![GoDoc](https://godoc.org/github.com/clockworksoul/smudge?status.svg)](https://godoc.org/github.com/clockworksoul/smudge) 4 | [![Build Status](https://travis-ci.org/clockworksoul/smudge.svg?branch=master)](https://travis-ci.org/clockworksoul/smudge) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/clockworksoul/smudge)](https://goreportcard.com/report/github.com/clockworksoul/smudge) 6 | [![Maintainability](https://api.codeclimate.com/v1/badges/651f423082d8fc38b399/maintainability)](https://codeclimate.com/github/clockworksoul/smudge/maintainability) 7 | 8 | 9 | 10 | ## Introduction 11 | Smudge is a minimalist Go implementation of the [SWIM](https://pdfs.semanticscholar.org/8712/3307869ac84fc16122043a4a313604bd948f.pdf) (Scalable Weakly-consistent Infection-style Membership) protocol for cluster node membership, status dissemination, and failure detection developed at Cornell University by Motivala, et al. It isn't a distributed data store in its own right, but rather a framework intended to facilitate the construction of such systems. 12 | 13 | Smudge also extends the standard SWIM protocol so that in addition to the standard membership status functionality it also allows the transmission of broadcasts containing a small amount (256 bytes) of arbitrary content to all present healthy members. This maximum is related to the limit imposed on maximum safe UDP packet size by RFC 791 and RFC 2460. We recognize that some systems allow larger packets, however, and although that can risk fragmentation and dropped packets the maximum payload size is configurable. 14 | 15 | Smudge was conceived with space-sensitive systems (mobile, IoT, containers) in mind, and therefore was developed with a minimalist philosophy of doing a few things well. As such, its feature set is relatively small and mostly limited to functionality around adding and removing nodes and detecting status changes on the cluster. 16 | 17 | Complete documentation is available from [the associated Godoc](https://godoc.org/github.com/clockworksoul/smudge). 18 | 19 | 20 | ## Features 21 | * Uses gossip (i.e., epidemic) protocol for dissemination, the latency of which grows logarithmically with the number of members. 22 | * Low-bandwidth UDP-based failure detection and status dissemination. 23 | * Imposes a constant message load per group member, regardless of the number of members. 24 | * Member status changes are eventually detected by all non-faulty members of the cluster (strong completeness). 25 | * Supports transmission of short broadcasts that are propagated at most once to all present, healthy members. 26 | * Supports both IPv4 and IPv6. 27 | * Pluggable logging 28 | 29 | ## Known issues 30 | * Broadcasts are limited to 256 bytes, or 512 bytes when using IPv6. 31 | * No WAN support: only local-network, private IPs are supported. 32 | 33 | ### Deviations from [Motivala, et al](https://pdfs.semanticscholar.org/8712/3307869ac84fc16122043a4a313604bd948f.pdf) 34 | 35 | * Dead nodes are not immediately removed, but are instead periodically re-tried (with exponential backoff) for a time before finally being removed. 36 | * Smudge allows the transmission of short, arbitrary-content broadcasts to all healthy nodes. 37 | 38 | ## How broadcasts work 39 | 40 | TL;DR a broadcast can be added to the local node by either calling a function or by receiving it from a remote node. A broadcast is send to other nodes a couple of times, piggybacked on membership messages. Then after a while the broadcast is removed from the node. 41 | 42 | ### Emit counter 43 | 44 | The emit counter represents the number of times a broadcast message must be send to other nodes. An emit counter is calculated with the following formula: `int(2.5 * log(number of nodes) + 0.5)`. The larger the network the higher the emit counter will be, but the larger the network the slower the emit counter will grow. 45 | 46 | Examples: 47 | 48 | * 2 nodes: `int(2.5 * log(2) + 0.5) = 2` 49 | * 10 nodes: `int(2.5 * log(10) + 0.5) = 6` 50 | * 20 nodes: `int(2.5 * log(20) + 0.5) = 8` 51 | 52 | ### Broadcasts 53 | 54 | When a broadcast is added to Smudge, either because it is added locally (by calling a function of the library) or is received from a remote node, an emit counter initialized with the formula above. The emit counter and the broadcast are then saved to a local buffer. 55 | 56 | The emit counter is used to track how many times a broadcast must be send to other nodes in the network. When the emit counter gets below a certain, large negative, thresh-hold the broadcast is removed from the buffer. Only broadcasts with a positive emit counter will be send when they are selected. 57 | 58 | When Smudge is about to send a membership message it looks for the broadcast with the largest emit counter. If multiple broadcasts have the same emit counter value, one is arbitrarily chosen. The selected broadcast can have a negative emit counter. If the emit counter is larger then `0` Smudge adds that broadcast to the membership message that will be send. In any case the emit counter is lowered by `1`. 59 | 60 | When a broadcast is received from another node and that broadcast is already in the buffer it will be ignored. To achieve this the origin IP of the node that added the broadcast to the network is saved as part of the broadcast. 61 | 62 | ## How to build 63 | 64 | Although Smudge is intended to be directly extended, a Dockerfile is provided for testing and proofs-of-function. 65 | 66 | The Dockerfile uses a multi-stage build, so Docker 17.05 or higher is required. The build compiles the code in a dedicated Golang container and drops the resulting binary into a `scratch` image for execution. This makes a `Makefile` or `build.sh` largely superfluous and removed the need to configure a local environment. 67 | 68 | ### Running the tests 69 | 70 | To run the tests in a containerized environment, which requires only that you have Docker installed (not Go), you can do: 71 | 72 | ```bash 73 | make test 74 | ``` 75 | 76 | Or, if you'd rather not use a Makefile: 77 | 78 | ```bash 79 | go test -v github.com/clockworksoul/smudge 80 | ``` 81 | 82 | 83 | ### Building the Docker image 84 | 85 | To execute the build, you simply need to do the following: 86 | 87 | ```bash 88 | make image 89 | ``` 90 | 91 | Or, if you'd rather not use a Makefile: 92 | 93 | ```bash 94 | docker build -t clockworksoul/smudge:latest . 95 | ``` 96 | 97 | 98 | ### Testing the Docker image 99 | 100 | You can test Smudge locally using the Docker image. First create a network to use for your Smudge nodes and then add some nodes to the network. 101 | 102 | For IPv4 you can use the following commands: 103 | 104 | ```bash 105 | docker network create smudge 106 | docker run -i -t --network smudge --rm clockworksoul/smudge:latest /smudge 107 | # you can add nodes with the following command 108 | docker run -i -t --network smudge --rm clockworksoul/smudge:latest /smudge -node 172.20.0.2 109 | ``` 110 | 111 | To try out Smudge with IPv6 you can use the following commands: 112 | 113 | ```bash 114 | docker network create --ipv6 --subnet fd02:6b8:b010:9020:1::/80 smudge6 115 | docker run -i -t --network smudge6 --rm clockworksoul/smudge:latest /smudge 116 | # you can add nodes with the following command 117 | docker run -i -t --network smudge6 --rm clockworksoul/smudge:latest /smudge -node [fd02:6b8:b010:9020:1::2]:9999 118 | ``` 119 | 120 | ### Building the binary with the Go compiler 121 | 122 | #### Set up your Golang environment 123 | 124 | If you already have a `$GOPATH` set up, you can skip to the following section. 125 | 126 | First, you'll need to decide where your Go code and binaries will live. This will be your Gopath. You simply need to export this as `GOPATH`: 127 | 128 | ```bash 129 | export GOPATH=~/go/ 130 | ``` 131 | 132 | Change it to whatever works for you. You'll want to add this to your `.bashrc` or `.bash_profile`. 133 | 134 | #### Clone the repo into your GOPATH 135 | 136 | Clone the code into `$GOPATH/src/github.com/clockworksoul/smudge`. Using the full-qualified path structure makes it possible to import the code into other libraries, as well as Smudge's own `main()` function. 137 | 138 | ```bash 139 | git clone git@github.com:clockworksoul/smudge.git $GOPATH/src/github.com/clockworksoul/smudge 140 | ``` 141 | 142 | #### Execute your build 143 | 144 | Once you have a `$GOPATH` already configured and the repository correctly cloned into `$GOPATH/src/github.com/clockworksoul/smudge`, you can execute the following: 145 | 146 | ```bash 147 | make build 148 | ``` 149 | 150 | When using the Makefile, the compiled binary will be present in the `/bin` directory in the code directory. 151 | 152 | If you'd rather not use a Makefile: 153 | 154 | ```bash 155 | go build -a -installsuffix cgo -o smudge github.com/clockworksoul/smudge/smudge 156 | ``` 157 | 158 | The binary, compiled for your current environment, will be present in your present working directory. 159 | 160 | 161 | ## How to use 162 | To use the code, you simply specify a few configuration options (or use the defaults), create and add a node status change listener, and call the `smudge.Begin()` function. 163 | 164 | 165 | ### Configuring the node with environment variables 166 | Perhaps the simplest way of directing the behavior of the SWIM driver is by setting the appropriate system environment variables, which is useful when making use of Smudge inside of a container. 167 | 168 | The following variables and their default values are as follows: 169 | 170 | ``` 171 | Variable | Default | Description 172 | ---------------------------------- | --------------- | ------------------------------- 173 | SMUDGE_CLUSTER_NAME | smudge | Cluster name for for multicast discovery 174 | SMUDGE_HEARTBEAT_MILLIS | 250 | Milliseconds between heartbeats 175 | SMUDGE_INITIAL_HOSTS | | Comma-delimmited list of known members as IP or IP:PORT 176 | SMUDGE_LISTEN_PORT | 9999 | UDP port to listen on 177 | SMUDGE_LISTEN_IP | 127.0.0.1 | IP address to listen on 178 | SMUDGE_MAX_BROADCAST_BYTES | 256 | Maximum byte length of broadcast payloads 179 | SMUDGE_MULTICAST_ENABLED | true | Multicast announce on startup; listen for multicast announcements 180 | SMUDGE_MULTICAST_ANNOUNCE_INTERVAL | 0 | Seconds between multicast announcements, 0 will disable subsequent anouncements 181 | SMUDGE_MULTICAST_ADDRESS | See description | The multicast broadcast address. Default: `224.0.0.0` (IPv4) or `[ff02::1]` (IPv6) 182 | SMUDGE_MULTICAST_PORT | 9998 | The multicast listen port 183 | ``` 184 | 185 | 186 | ### Configuring the node with API calls 187 | If you prefer to direct the behavior of the service using the API, the calls are relatively straight-forward. Note that setting the application properties using this method overrides the behavior of environment variables. 188 | 189 | ```go 190 | smudge.SetListenPort(9999) 191 | smudge.SetHeartbeatMillis(250) 192 | smudge.SetListenIP(net.ParseIP("127.0.0.1")) 193 | smudge.SetMaxBroadcastBytes(256) // set to 512 when using IPv6 194 | ``` 195 | 196 | ### Creating and adding a status change listener 197 | Creating a status change listener is very straight-forward: 198 | 199 | ```go 200 | type MyStatusListener struct { 201 | smudge.StatusListener 202 | } 203 | 204 | func (m MyStatusListener) OnChange(node *smudge.Node, status smudge.NodeStatus) { 205 | fmt.Printf("Node %s is now status %s\n", node.Address(), status) 206 | } 207 | 208 | func main() { 209 | smudge.AddStatusListener(MyStatusListener{}) 210 | } 211 | ``` 212 | 213 | ### Creating and adding a broadcast listener 214 | Adding a broadcast listener is very similar to creating a status listener: 215 | 216 | ```go 217 | type MyBroadcastListener struct { 218 | smudge.BroadcastListener 219 | } 220 | 221 | func (m MyBroadcastListener) OnBroadcast(b *smudge.Broadcast) { 222 | fmt.Printf("Received broadcast from %v: %s\n", 223 | b.Origin().Address(), 224 | string(b.Bytes())) 225 | } 226 | 227 | func main() { 228 | smudge.AddBroadcastListener(MyBroadcastListener{}) 229 | } 230 | ``` 231 | 232 | ### Adding a new member to the "known nodes" list 233 | Adding a new member to your known nodes list will also make that node aware of the adding server. To join an existing cluster without using multicast (or on a network where multicast is disabled) you must use this method to add at least one of that cluster's healthy member nodes. 234 | 235 | ```go 236 | node, err := smudge.CreateNodeByAddress("localhost:10000") 237 | if err == nil { 238 | smudge.AddNode(node) 239 | } 240 | ``` 241 | 242 | ### Starting the server 243 | Once everything else is done, starting the server is trivial: 244 | 245 | Simply call: `smudge.Begin()` 246 | 247 | ### Transmitting a broadcast 248 | To transmit a broadcast to all healthy nodes currenty in the cluster you can use one of the [`BroadcastBytes(bytes []byte)`](https://godoc.org/github.com/clockworksoul/smudge#BroadcastBytes) or [`BroadcastString(str string)`](https://godoc.org/github.com/clockworksoul/smudge#BroadcastString) functions. 249 | 250 | Be aware of the following caveats: 251 | * Attempting to send a broadcast before the server has been started will cause a panic. 252 | * The broadcast _will not_ be received by the originating member; `BroadcastListener`s on the originating member will not be triggered. 253 | * Nodes that join the cluster after the broadcast has been fully propagated will not receive the broadcast; nodes that join after the initial transmission but before complete proagation may or may not receive the broadcast. 254 | 255 | ### Getting a list of nodes 256 | The [`AllNodes()`](https://godoc.org/github.com/clockworksoul/smudge#AllNodes) can be used to get all known nodes; [`HealthyNodes()`](https://godoc.org/github.com/clockworksoul/smudge#HealthyNodes) works similarly, but returns only healthy nodes (defined as nodes with a [status](https://godoc.org/github.com/clockworksoul/smudge#NodeStatus) of "alive"). 257 | 258 | ### Everything in one place 259 | 260 | ```go 261 | package main 262 | 263 | import "github.com/clockworksoul/smudge" 264 | import "fmt" 265 | import "net" 266 | 267 | type MyStatusListener struct { 268 | smudge.StatusListener 269 | } 270 | 271 | func (m MyStatusListener) OnChange(node *smudge.Node, status smudge.NodeStatus) { 272 | fmt.Printf("Node %s is now status %s\n", node.Address(), status) 273 | } 274 | 275 | type MyBroadcastListener struct { 276 | smudge.BroadcastListener 277 | } 278 | 279 | func (m MyBroadcastListener) OnBroadcast(b *smudge.Broadcast) { 280 | fmt.Printf("Received broadcast from %s: %s\n", 281 | b.Origin().Address(), 282 | string(b.Bytes())) 283 | } 284 | 285 | func main() { 286 | heartbeatMillis := 500 287 | listenPort := 9999 288 | 289 | // Set configuration options 290 | smudge.SetListenPort(listenPort) 291 | smudge.SetHeartbeatMillis(heartbeatMillis) 292 | smudge.SetListenIP(net.ParseIP("127.0.0.1")) 293 | 294 | // Add the status listener 295 | smudge.AddStatusListener(MyStatusListener{}) 296 | 297 | // Add the broadcast listener 298 | smudge.AddBroadcastListener(MyBroadcastListener{}) 299 | 300 | // Add a new remote node. Currently, to join an existing cluster you must 301 | // add at least one of its healthy member nodes. 302 | node, err := smudge.CreateNodeByAddress("localhost:10000") 303 | if err == nil { 304 | smudge.AddNode(node) 305 | } 306 | 307 | // Start the server! 308 | smudge.Begin() 309 | } 310 | ``` 311 | 312 | ### Bringing your own logger 313 | 314 | Smudge comes with a `DefaultLogger` that writes log messages to `stderr`. You can plug in your own logger by implementing the functions of the `Logger` interface and setting the logger by calling `smudge.SetLogger(MyCoolLogger)`. 315 | 316 | -------------------------------------------------------------------------------- /broadcast.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "net" 23 | "sort" 24 | "sync" 25 | ) 26 | 27 | const ( 28 | // Emit counters for broadcasts can be less than 0. We transmit positive 29 | // numbers, and decrement all the others. At this value, the broadcast 30 | // is removed from the map all together. This ensures broadcasts are 31 | // emitted briefly, but retained long enough to not be received twice. 32 | broadcastRemoveValue int8 = int8(-100) 33 | ) 34 | 35 | // The index counter value for the next broadcast message 36 | var indexCounter uint32 = 1 37 | 38 | // Emitted broadcasts. Once they are added here, the membership machinery will 39 | // pick them up and piggyback them onto standard messages. 40 | var broadcasts = struct { 41 | sync.RWMutex 42 | m map[string]*Broadcast 43 | }{m: make(map[string]*Broadcast)} 44 | 45 | // Broadcast represents a packet of bytes emitted across the cluster on top of 46 | // the status update infrastructure. Although useful, its payload is limited 47 | // to only 256 bytes. 48 | type Broadcast struct { 49 | bytes []byte 50 | origin *Node 51 | index uint32 52 | label string 53 | emitCounter int8 54 | } 55 | 56 | // Bytes returns a copy of this broadcast's bytes. Manipulating the contents 57 | // of this slice will not be reflected in the contents of the broadcast. 58 | func (b *Broadcast) Bytes() []byte { 59 | length := len(b.bytes) 60 | bytesCopy := make([]byte, length, length) 61 | copy(bytesCopy, b.bytes) 62 | 63 | return bytesCopy 64 | } 65 | 66 | // Index returns the origin message index for this broadcast. This value is 67 | // incremented for each broadcast. The combination of 68 | // originIP:originPort:Index is unique. 69 | func (b *Broadcast) Index() uint32 { 70 | return b.index 71 | } 72 | 73 | // Label returns a unique label string composed of originIP:originPort:Index. 74 | func (b *Broadcast) Label() string { 75 | if b.label == "" { 76 | b.label = fmt.Sprintf("%s:%d:%d", 77 | b.origin.ip.String(), 78 | b.origin.port, 79 | b.index) 80 | } 81 | 82 | return b.label 83 | } 84 | 85 | // Origin returns the node that this broadcast originated from. 86 | func (b *Broadcast) Origin() *Node { 87 | return b.origin 88 | } 89 | 90 | // BroadcastBytes allows a user to emit a short broadcast in the form of a byte 91 | // slice, which will be transmitted at most once to all other healthy current 92 | // members. Members that join after the broadcast has already propagated 93 | // through the cluster will not receive the message. The maximum broadcast 94 | // length is 256 bytes. 95 | func BroadcastBytes(bytes []byte) error { 96 | if len(bytes) > GetMaxBroadcastBytes() { 97 | emsg := fmt.Sprintf( 98 | "broadcast payload length exceeds %d bytes", 99 | GetMaxBroadcastBytes()) 100 | 101 | return errors.New(emsg) 102 | } 103 | 104 | broadcasts.Lock() 105 | 106 | bcast := Broadcast{ 107 | origin: thisHost, 108 | index: indexCounter, 109 | bytes: bytes, 110 | emitCounter: int8(emitCount())} 111 | 112 | broadcasts.m[bcast.Label()] = &bcast 113 | 114 | indexCounter++ 115 | 116 | broadcasts.Unlock() 117 | 118 | return nil 119 | } 120 | 121 | // BroadcastString allows a user to emit a short broadcast in the form of a 122 | // string, which will be transmitted at most once to all other healthy current 123 | // members. Members that join after the broadcast has already propagated 124 | // through the cluster will not receive the message. The maximum broadcast 125 | // length is 256 bytes. 126 | func BroadcastString(str string) error { 127 | return BroadcastBytes([]byte(str)) 128 | } 129 | 130 | // Message contents for IPv6 131 | // Bytes Content 132 | // ------------------------ 133 | // Bytes 00-15 Origin IP (00-03 for IPv4) 134 | // Bytes 16-17 Origin response port (04-05 for IPv4) 135 | // Bytes 18-21 Origin broadcast counter (06-09 for IPv4) 136 | // Bytes 22-23 Payload length (bytes) (10-11 for IPv4) 137 | // Bytes 24-NN Payload (12-NN for IPv4) 138 | func (b *Broadcast) encode() []byte { 139 | size := 8 + ipLen + len(b.bytes) 140 | bytes := make([]byte, size, size) 141 | 142 | // Index pointer 143 | p := 0 144 | 145 | // Bytes 00-15: Origin IP 146 | ip := b.origin.IP() 147 | if ip.To4() != nil { 148 | ip = ip.To4() 149 | } 150 | 151 | for i := 0; i < ipLen; i++ { 152 | bytes[p+i] = ip[i] 153 | } 154 | p += ipLen 155 | 156 | // Bytes 16-17 Origin response port 157 | p += encodeUint16(b.origin.Port(), bytes, p) 158 | 159 | // Bytes 18-21 Origin broadcast counter 160 | p += encodeUint32(b.index, bytes, p) 161 | 162 | // Bytes 22-23 Payload length (bytes) 163 | p += encodeUint16(uint16(len(b.bytes)), bytes, p) 164 | 165 | // Bytes 24-NN Payload 166 | for i, by := range b.bytes { 167 | bytes[i+p] = by 168 | } 169 | 170 | return bytes 171 | } 172 | 173 | // Message contents 174 | // Bytes Content 175 | // ------------------------ 176 | // Bytes 00-15 Origin IP (00-03 on IPv4) 177 | // Bytes 16-17 Origin response port (04-05 on IPv4) 178 | // Bytes 18-21 Origin broadcast counter (06-09 on IPv4) 179 | // Bytes 22-23 Payload length (bytes) (10-11 on IPv4) 180 | // Bytes 24-NN Payload (12-NN on IPv4) 181 | func decodeBroadcast(bytes []byte) (*Broadcast, error) { 182 | var index uint32 183 | var port uint16 184 | var ip net.IP 185 | var length uint16 186 | 187 | // An index pointer 188 | p := 0 189 | 190 | if ipLen == net.IPv6len { 191 | // Bytes 00-15 Origin IP 192 | ip = make(net.IP, net.IPv6len) 193 | copy(ip, bytes[p:p+16]) 194 | } else { 195 | // Bytes 00-03 Origin IPv4 196 | ip = net.IPv4(bytes[p+0], bytes[p+1], bytes[p+2], bytes[p+3]) 197 | } 198 | 199 | p += ipLen 200 | 201 | // Bytes 16-17 Origin response port 202 | port, p = decodeUint16(bytes, p) 203 | 204 | // Bytes 18-21 Origin broadcast counter 205 | index, p = decodeUint32(bytes, p) 206 | 207 | // Bytes 22-23 Payload length (bytes) 208 | length, p = decodeUint16(bytes, p) 209 | 210 | // Now that we have the IP and port, we can find the Node. 211 | origin := knownNodes.getByIP(ip, port) 212 | 213 | // We don't know this node, so create a new one! 214 | if origin == nil { 215 | origin, _ = CreateNodeByIP(ip, port) 216 | } 217 | 218 | bcast := Broadcast{ 219 | origin: origin, 220 | index: index, 221 | bytes: bytes[p : p+int(length)], 222 | emitCounter: int8(emitCount())} 223 | 224 | err := checkOrigin(origin) 225 | if err != nil { 226 | logWarn(err) 227 | return &bcast, err 228 | } 229 | 230 | if int(length) > GetMaxBroadcastBytes() { 231 | return &bcast, 232 | errors.New("message length exceeds maximum length") 233 | } 234 | 235 | return &bcast, nil 236 | } 237 | 238 | // getBroadcastToEmit identifies the single known broadcast with the highest 239 | // emitCounter value (which can be negative), and returns it. If multiple 240 | // broadcasts have the same value, one is arbitrarily chosen. 241 | func getBroadcastToEmit() *Broadcast { 242 | // Get all broadcast messages. 243 | values := make([]*Broadcast, 0, 0) 244 | broadcasts.RLock() 245 | for _, v := range broadcasts.m { 246 | values = append(values, v) 247 | } 248 | broadcasts.RUnlock() 249 | 250 | // Remove all overly-emitted messages from the list 251 | broadcastSlice := make([]*Broadcast, 0, 0) 252 | broadcasts.Lock() 253 | for _, b := range values { 254 | if b.emitCounter <= broadcastRemoveValue { 255 | logDebug("Removing", b.Label(), "from recently updated list") 256 | delete(broadcasts.m, b.Label()) 257 | } else { 258 | broadcastSlice = append(broadcastSlice, b) 259 | } 260 | } 261 | broadcasts.Unlock() 262 | 263 | if len(broadcastSlice) > 0 { 264 | // Put the newest nodes on top. 265 | sort.Sort(byBroadcastEmitCounter(broadcastSlice)) 266 | return broadcastSlice[0] 267 | } 268 | 269 | return nil 270 | } 271 | 272 | // receiveBroadcast is called by receiveMessageUDP when a broadcast payload 273 | // is found in a message. 274 | func receiveBroadcast(broadcast *Broadcast) { 275 | if broadcast == nil { 276 | return 277 | } 278 | 279 | err := checkOrigin(broadcast.Origin()) 280 | if err != nil { 281 | logWarn(err) 282 | return 283 | } 284 | 285 | label := broadcast.Label() 286 | 287 | broadcasts.Lock() 288 | _, contains := broadcasts.m[label] 289 | if !contains { 290 | broadcasts.m[label] = broadcast 291 | } 292 | broadcasts.Unlock() 293 | 294 | if !contains { 295 | logfInfo("Broadcast [%s]=%s", 296 | label, 297 | string(broadcast.Bytes())) 298 | 299 | doBroadcastUpdate(broadcast) 300 | } 301 | } 302 | 303 | // checkBroadcastOrigin checks wether the origin is set correctly 304 | func checkOrigin(origin *Node) error { 305 | // normalize to IPv4 or IPv6 to check below 306 | ip := origin.IP() 307 | if ip.To4() != nil { 308 | ip = ip.To4() 309 | } 310 | 311 | if (ip[0] == 0) || origin.Port() == 0 { 312 | return errors.New("Received originless broadcast") 313 | } 314 | return nil 315 | } 316 | 317 | // byBroadcastEmitCounter implements sort.Interface for []*Broadcast based on 318 | // the emitCounter field. 319 | type byBroadcastEmitCounter []*Broadcast 320 | 321 | func (a byBroadcastEmitCounter) Len() int { 322 | return len(a) 323 | } 324 | 325 | func (a byBroadcastEmitCounter) Swap(i, j int) { 326 | a[i], a[j] = a[j], a[i] 327 | } 328 | 329 | func (a byBroadcastEmitCounter) Less(i, j int) bool { 330 | return a[i].emitCounter > a[j].emitCounter 331 | } 332 | -------------------------------------------------------------------------------- /broadcast_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "fmt" 21 | "net" 22 | "testing" 23 | 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | var ( 28 | expectedBytes = []byte("The quick brown fox jumps over the lazy dog") 29 | 30 | expectedIndex = uint32(1) 31 | 32 | expectedEmitCounter = int8(64) 33 | 34 | expectedOriginIP = net.IPv4(10, 9, 8, 7) 35 | 36 | expectedOriginPort = uint16(1234) 37 | 38 | expectedLabel = fmt.Sprintf("%s:%d:%d", 39 | expectedOriginIP.String(), 40 | expectedOriginPort, 41 | expectedIndex) 42 | ) 43 | 44 | func testNode() *Node { 45 | return &Node{ 46 | ip: expectedOriginIP, 47 | port: expectedOriginPort, 48 | } 49 | } 50 | 51 | func testBroadcast() *Broadcast { 52 | node := testNode() 53 | 54 | return &Broadcast{ 55 | origin: node, 56 | bytes: expectedBytes, 57 | index: expectedIndex, 58 | emitCounter: expectedEmitCounter, 59 | } 60 | } 61 | 62 | func TestBytes(t *testing.T) { 63 | bc := testBroadcast() 64 | out := bc.Bytes() 65 | 66 | require.Equal(t, expectedBytes, out, "Value mismatch") 67 | } 68 | 69 | func TestIndex(t *testing.T) { 70 | bc := testBroadcast() 71 | out := bc.Index() 72 | 73 | require.Equal(t, expectedIndex, out, "Value mismatch") 74 | } 75 | 76 | func TestLabel(t *testing.T) { 77 | bc := testBroadcast() 78 | out := bc.Label() 79 | 80 | require.Equal(t, expectedLabel, out, "Value mismatch") 81 | } 82 | 83 | func TestGetBroadcastToEmit(t *testing.T) { 84 | bca, bcb, bcc := testBroadcast(), testBroadcast(), testBroadcast() 85 | 86 | bca.emitCounter = 5 87 | bca.index = 1 88 | bcb.emitCounter = 15 89 | bcb.index = 2 90 | bcc.emitCounter = 10 91 | bcc.index = 3 92 | 93 | broadcasts.m["a"] = bca 94 | broadcasts.m["b"] = bcb 95 | broadcasts.m["c"] = bcc 96 | 97 | bc1 := getBroadcastToEmit() 98 | require.Equal(t, bcb, bc1, fmt.Sprintf("Expected %v, got %v", bcb, bc1)) 99 | 100 | delete(broadcasts.m, "b") 101 | bc2 := getBroadcastToEmit() 102 | require.Equal(t, bcc, bc2, fmt.Sprintf("Expected %v, got %v", bcc, bc2)) 103 | 104 | delete(broadcasts.m, "c") 105 | bc3 := getBroadcastToEmit() 106 | require.Equal(t, bca, bc3, fmt.Sprintf("Expected %v, got %v", bcc, bc3)) 107 | 108 | delete(broadcasts.m, "a") 109 | } 110 | 111 | func TestBroadcastBytes(t *testing.T) { 112 | h := testNode() 113 | thisHost = h 114 | 115 | require.Empty(t, broadcasts.m, "Broadcasts map isn't empty") 116 | 117 | err := BroadcastBytes(expectedBytes) 118 | require.Nil(t, err, "Should have been no error!") 119 | 120 | bc := broadcasts.m[expectedLabel] 121 | 122 | require.Equal(t, expectedBytes, bc.Bytes(), "Message contents mismatch") 123 | 124 | delete(broadcasts.m, expectedLabel) 125 | } 126 | 127 | func TestBroadcastBytesTooLong(t *testing.T) { 128 | bytesTooLong := make([]byte, GetMaxBroadcastBytes()*2, GetMaxBroadcastBytes()*2) 129 | err := BroadcastBytes(bytesTooLong) 130 | require.NotNil(t, err, "Should have been too long!") 131 | } 132 | 133 | func TestReceiveBroadcast(t *testing.T) { 134 | bc := testBroadcast() 135 | 136 | require.Empty(t, broadcasts.m, "Broadcasts map isn't empty") 137 | 138 | receiveBroadcast(bc) 139 | 140 | require.True(t, broadcasts.m[expectedLabel] != nil, "Broadcast value is nil") 141 | 142 | receiveBroadcast(bc) 143 | 144 | require.True(t, len(broadcasts.m) == 1, "Added another where it shouldn't have") 145 | } 146 | -------------------------------------------------------------------------------- /bytes.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // This file contains a variety of methods to encode/decode various primitive 18 | // data types into a byte slice using a fixed length encoding scheme. We 19 | // chose fixed lengths (as opposed to the variable length encoding provided by 20 | // the encoding/binary package) because it makes decoding easier. It's also 21 | // (very slightly) more efficient. 22 | // 23 | // These functions are used mostly by the components defined in message.go. 24 | 25 | package smudge 26 | 27 | func decodeByte(bytes []byte, startIndex int) (byte, int) { 28 | return bytes[startIndex], startIndex + 1 29 | } 30 | 31 | func decodeUint8(bytes []byte, startIndex int) (uint8, int) { 32 | n, i := decodeByte(bytes, startIndex) 33 | 34 | return byte(n), i 35 | } 36 | 37 | func decodeUint16(bytes []byte, startIndex int) (uint16, int) { 38 | var number uint16 39 | 40 | number = uint16(bytes[startIndex+1])<<8 | 41 | uint16(bytes[startIndex+0]) 42 | 43 | return number, startIndex + 2 44 | } 45 | 46 | func decodeUint32(bytes []byte, startIndex int) (uint32, int) { 47 | var number uint32 48 | 49 | number = uint32(bytes[startIndex+3])<<24 | 50 | uint32(bytes[startIndex+2])<<16 | 51 | uint32(bytes[startIndex+1])<<8 | 52 | uint32(bytes[startIndex+0]) 53 | 54 | return number, startIndex + 4 55 | } 56 | 57 | func decodeUint64(bytes []byte, startIndex int) (uint64, int) { 58 | var number uint64 59 | 60 | number = uint64(bytes[startIndex+7])<<56 | 61 | uint64(bytes[startIndex+6])<<48 | 62 | uint64(bytes[startIndex+5])<<40 | 63 | uint64(bytes[startIndex+4])<<32 | 64 | uint64(bytes[startIndex+3])<<24 | 65 | uint64(bytes[startIndex+2])<<16 | 66 | uint64(bytes[startIndex+1])<<8 | 67 | uint64(bytes[startIndex+0]) 68 | 69 | return number, startIndex + 8 70 | } 71 | 72 | func encodeByte(number byte, bytes []byte, startIndex int) int { 73 | bytes[startIndex+0] = number 74 | 75 | return 1 76 | } 77 | 78 | func encodeUint8(number uint8, bytes []byte, startIndex int) int { 79 | return encodeByte(byte(number), bytes, startIndex) 80 | } 81 | 82 | func encodeUint16(number uint16, bytes []byte, startIndex int) int { 83 | bytes[startIndex+0] = byte(number) 84 | bytes[startIndex+1] = byte(number >> 8) 85 | 86 | return 2 87 | } 88 | 89 | func encodeUint32(number uint32, bytes []byte, startIndex int) int { 90 | bytes[startIndex+0] = byte(number) 91 | bytes[startIndex+1] = byte(number >> 8) 92 | bytes[startIndex+2] = byte(number >> 16) 93 | bytes[startIndex+3] = byte(number >> 24) 94 | 95 | return 4 96 | } 97 | 98 | func encodeUint64(number uint64, bytes []byte, startIndex int) int { 99 | bytes[startIndex+0] = byte(number) 100 | bytes[startIndex+1] = byte(number >> 8) 101 | bytes[startIndex+2] = byte(number >> 16) 102 | bytes[startIndex+3] = byte(number >> 24) 103 | bytes[startIndex+4] = byte(number >> 32) 104 | bytes[startIndex+5] = byte(number >> 40) 105 | bytes[startIndex+6] = byte(number >> 48) 106 | bytes[startIndex+7] = byte(number >> 56) 107 | 108 | return 8 109 | } 110 | -------------------------------------------------------------------------------- /bytes_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | func TestEncodeDecodeUint16A(t *testing.T) { 24 | bytes := make([]byte, 2, 2) 25 | initial := uint16(0x0000) 26 | 27 | encodeUint16(initial, bytes, 0) 28 | 29 | backAgain, p := decodeUint16(bytes, 0) 30 | 31 | if p != 2 { 32 | t.Fail() 33 | } 34 | 35 | if initial != backAgain { 36 | t.Errorf("%d != %d", initial, backAgain) 37 | } 38 | } 39 | 40 | func TestEncodeDecodeUint16B(t *testing.T) { 41 | bytes := make([]byte, 2, 2) 42 | initial := uint16(0x048C) 43 | 44 | encodeUint16(initial, bytes, 0) 45 | 46 | backAgain, p := decodeUint16(bytes, 0) 47 | 48 | if p != 2 { 49 | t.Fail() 50 | } 51 | 52 | if initial != backAgain { 53 | t.Errorf("%d != %d", initial, backAgain) 54 | } 55 | } 56 | 57 | func TestEncodeDecodeUint16C(t *testing.T) { 58 | bytes := make([]byte, 2, 2) 59 | initial := uint16(0x159D) 60 | 61 | encodeUint16(initial, bytes, 0) 62 | 63 | backAgain, p := decodeUint16(bytes, 0) 64 | 65 | if p != 2 { 66 | t.Fail() 67 | } 68 | 69 | if initial != backAgain { 70 | t.Errorf("%d != %d", initial, backAgain) 71 | } 72 | } 73 | 74 | func TestEncodeDecodeUint16D(t *testing.T) { 75 | bytes := make([]byte, 2, 2) 76 | initial := uint16(0xFFFF) 77 | 78 | encodeUint16(initial, bytes, 0) 79 | 80 | backAgain, p := decodeUint16(bytes, 0) 81 | 82 | if p != 2 { 83 | t.Fail() 84 | } 85 | 86 | if initial != backAgain { 87 | t.Errorf("%d != %d", initial, backAgain) 88 | } 89 | } 90 | 91 | func TestEncodeDecodeUint32A(t *testing.T) { 92 | bytes := make([]byte, 4, 4) 93 | initial := uint32(0x00000000) 94 | 95 | encodeUint32(initial, bytes, 0) 96 | 97 | backAgain, p := decodeUint32(bytes, 0) 98 | 99 | if p != 4 { 100 | t.Fail() 101 | } 102 | 103 | if initial != backAgain { 104 | t.Errorf("%d != %d", initial, backAgain) 105 | } 106 | } 107 | 108 | func TestEncodeDecodeUint32B(t *testing.T) { 109 | bytes := make([]byte, 4, 4) 110 | initial := uint32(0x02468ACE) 111 | 112 | encodeUint32(initial, bytes, 0) 113 | 114 | backAgain, p := decodeUint32(bytes, 0) 115 | 116 | if p != 4 { 117 | t.Fail() 118 | } 119 | 120 | if initial != backAgain { 121 | t.Errorf("%d != %d", initial, backAgain) 122 | } 123 | } 124 | 125 | func TestEncodeDecodeUint32C(t *testing.T) { 126 | bytes := make([]byte, 4, 4) 127 | initial := uint32(0x13579BDF) 128 | 129 | encodeUint32(initial, bytes, 0) 130 | 131 | backAgain, p := decodeUint32(bytes, 0) 132 | 133 | if p != 4 { 134 | t.Fail() 135 | } 136 | 137 | if initial != backAgain { 138 | t.Errorf("%d != %d", initial, backAgain) 139 | } 140 | } 141 | 142 | func TestEncodeDecodeUint32D(t *testing.T) { 143 | bytes := make([]byte, 4, 4) 144 | initial := uint32(0xFFFFFFFF) 145 | 146 | encodeUint32(initial, bytes, 0) 147 | 148 | backAgain, p := decodeUint32(bytes, 0) 149 | 150 | if p != 4 { 151 | t.Fail() 152 | } 153 | 154 | if initial != backAgain { 155 | t.Errorf("%d != %d", initial, backAgain) 156 | } 157 | } 158 | 159 | func TestEncodeDecodeUint64A(t *testing.T) { 160 | bytes := make([]byte, 8, 8) 161 | initial := uint64(0x000000000000000) 162 | 163 | encodeUint64(initial, bytes, 0) 164 | 165 | backAgain, p := decodeUint64(bytes, 0) 166 | 167 | if p != 8 { 168 | t.Fail() 169 | } 170 | 171 | if initial != backAgain { 172 | t.Errorf("%d != %d", initial, backAgain) 173 | } 174 | } 175 | 176 | func TestEncodeDecodeUint64B(t *testing.T) { 177 | bytes := make([]byte, 8, 8) 178 | initial := uint64(0x02468ACE02468ACE) 179 | 180 | encodeUint64(initial, bytes, 0) 181 | 182 | backAgain, p := decodeUint64(bytes, 0) 183 | 184 | if p != 8 { 185 | t.Fail() 186 | } 187 | 188 | if initial != backAgain { 189 | t.Errorf("%d != %d", initial, backAgain) 190 | } 191 | } 192 | 193 | func TestEncodeDecodeUint64C(t *testing.T) { 194 | bytes := make([]byte, 8, 8) 195 | initial := uint64(0x13579BDF13579BDF) 196 | 197 | encodeUint64(initial, bytes, 0) 198 | 199 | backAgain, p := decodeUint64(bytes, 0) 200 | 201 | if p != 8 { 202 | t.Fail() 203 | } 204 | 205 | if initial != backAgain { 206 | t.Errorf("%d != %d", initial, backAgain) 207 | } 208 | } 209 | 210 | func TestEncodeDecodeUint64D(t *testing.T) { 211 | bytes := make([]byte, 8, 8) 212 | initial := uint64(0xFFFFFFFFFFFFFFFF) 213 | 214 | encodeUint64(initial, bytes, 0) 215 | 216 | backAgain, p := decodeUint64(bytes, 0) 217 | 218 | if p != 8 { 219 | t.Fail() 220 | } 221 | 222 | if initial != backAgain { 223 | t.Errorf("%d != %d", initial, backAgain) 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | node000: 5 | image: clockworksoul/smudge:latest 6 | command: "/smudge" 7 | networks: 8 | - smudgeNetwork 9 | 10 | node001: 11 | image: clockworksoul/smudge:latest 12 | command: "/smudge" 13 | networks: 14 | - smudgeNetwork 15 | 16 | node002: 17 | image: clockworksoul/smudge:latest 18 | command: "/smudge" 19 | networks: 20 | - smudgeNetwork 21 | 22 | node003: 23 | image: clockworksoul/smudge:latest 24 | command: "/smudge" 25 | networks: 26 | - smudgeNetwork 27 | 28 | node004: 29 | image: clockworksoul/smudge:latest 30 | command: "/smudge" 31 | networks: 32 | - smudgeNetwork 33 | 34 | node005: 35 | image: clockworksoul/smudge:latest 36 | command: "/smudge" 37 | networks: 38 | - smudgeNetwork 39 | 40 | node006: 41 | image: clockworksoul/smudge:latest 42 | command: "/smudge" 43 | networks: 44 | - smudgeNetwork 45 | 46 | node007: 47 | image: clockworksoul/smudge:latest 48 | command: "/smudge" 49 | networks: 50 | - smudgeNetwork 51 | 52 | node008: 53 | image: clockworksoul/smudge:latest 54 | command: "/smudge" 55 | networks: 56 | - smudgeNetwork 57 | 58 | node009: 59 | image: clockworksoul/smudge:latest 60 | command: "/smudge" 61 | networks: 62 | - smudgeNetwork 63 | 64 | node010: 65 | image: clockworksoul/smudge:latest 66 | command: "/smudge" 67 | networks: 68 | - smudgeNetwork 69 | 70 | node011: 71 | image: clockworksoul/smudge:latest 72 | command: "/smudge" 73 | networks: 74 | - smudgeNetwork 75 | 76 | node012: 77 | image: clockworksoul/smudge:latest 78 | command: "/smudge" 79 | networks: 80 | - smudgeNetwork 81 | 82 | node013: 83 | image: clockworksoul/smudge:latest 84 | command: "/smudge" 85 | networks: 86 | - smudgeNetwork 87 | 88 | node014: 89 | image: clockworksoul/smudge:latest 90 | command: "/smudge" 91 | networks: 92 | - smudgeNetwork 93 | 94 | node015: 95 | image: clockworksoul/smudge:latest 96 | command: "/smudge" 97 | networks: 98 | - smudgeNetwork 99 | 100 | node016: 101 | image: clockworksoul/smudge:latest 102 | command: "/smudge" 103 | networks: 104 | - smudgeNetwork 105 | 106 | node017: 107 | image: clockworksoul/smudge:latest 108 | command: "/smudge" 109 | networks: 110 | - smudgeNetwork 111 | 112 | node018: 113 | image: clockworksoul/smudge:latest 114 | command: "/smudge" 115 | networks: 116 | - smudgeNetwork 117 | 118 | node019: 119 | image: clockworksoul/smudge:latest 120 | command: "/smudge" 121 | networks: 122 | - smudgeNetwork 123 | 124 | node020: 125 | image: clockworksoul/smudge:latest 126 | command: "/smudge" 127 | networks: 128 | - smudgeNetwork 129 | 130 | node021: 131 | image: clockworksoul/smudge:latest 132 | command: "/smudge" 133 | networks: 134 | - smudgeNetwork 135 | 136 | node022: 137 | image: clockworksoul/smudge:latest 138 | command: "/smudge" 139 | networks: 140 | - smudgeNetwork 141 | 142 | node023: 143 | image: clockworksoul/smudge:latest 144 | command: "/smudge" 145 | networks: 146 | - smudgeNetwork 147 | 148 | node024: 149 | image: clockworksoul/smudge:latest 150 | command: "/smudge" 151 | networks: 152 | - smudgeNetwork 153 | 154 | node025: 155 | image: clockworksoul/smudge:latest 156 | command: "/smudge" 157 | networks: 158 | - smudgeNetwork 159 | 160 | node026: 161 | image: clockworksoul/smudge:latest 162 | command: "/smudge" 163 | networks: 164 | - smudgeNetwork 165 | 166 | node027: 167 | image: clockworksoul/smudge:latest 168 | command: "/smudge" 169 | networks: 170 | - smudgeNetwork 171 | 172 | node028: 173 | image: clockworksoul/smudge:latest 174 | command: "/smudge" 175 | networks: 176 | - smudgeNetwork 177 | 178 | node029: 179 | image: clockworksoul/smudge:latest 180 | command: "/smudge" 181 | networks: 182 | - smudgeNetwork 183 | networks: 184 | smudgeNetwork: 185 | -------------------------------------------------------------------------------- /events.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import "sync" 20 | 21 | var broadcastListeners = struct { 22 | sync.RWMutex 23 | s []BroadcastListener 24 | }{s: make([]BroadcastListener, 0, 16)} 25 | 26 | var statusListeners = struct { 27 | sync.RWMutex 28 | s []StatusListener 29 | }{s: make([]StatusListener, 0, 16)} 30 | 31 | // BroadcastListener is the interface that must be implemented to take advantage 32 | // of the cluster member status update notification functionality provided by 33 | // the AddBroadcastListener() function. 34 | type BroadcastListener interface { 35 | // The OnBroadcast() function is called whenever the node is notified of 36 | // an incoming broadcast message. 37 | OnBroadcast(broadcast *Broadcast) 38 | } 39 | 40 | // AddBroadcastListener allows the submission of a BroadcastListener implementation 41 | // whose OnChange() function will be called whenever the node is notified of any 42 | // change in the status of a cluster member. 43 | func AddBroadcastListener(listener BroadcastListener) { 44 | broadcastListeners.Lock() 45 | broadcastListeners.s = append(broadcastListeners.s, listener) 46 | broadcastListeners.Unlock() 47 | } 48 | 49 | func doBroadcastUpdate(broadcast *Broadcast) { 50 | broadcastListeners.RLock() 51 | for _, sl := range broadcastListeners.s { 52 | sl.OnBroadcast(broadcast) 53 | } 54 | broadcastListeners.RUnlock() 55 | } 56 | 57 | // StatusListener is the interface that must be implemented to take advantage 58 | // of the cluster member status update notification functionality provided by 59 | // the AddStatusListener() function. 60 | type StatusListener interface { 61 | // The OnChange() function is called whenever the node is notified of any 62 | // change in the status of a cluster member. 63 | OnChange(node *Node, status NodeStatus) 64 | } 65 | 66 | // AddStatusListener allows the submission of a StatusListener implementation 67 | // whose OnChange() function will be called whenever the node is notified of any 68 | // change in the status of a cluster member. 69 | func AddStatusListener(listener StatusListener) { 70 | statusListeners.Lock() 71 | statusListeners.s = append(statusListeners.s, listener) 72 | statusListeners.Unlock() 73 | } 74 | 75 | func doStatusUpdate(node *Node, status NodeStatus) { 76 | statusListeners.RLock() 77 | for _, sl := range statusListeners.s { 78 | sl.OnChange(node, status) 79 | } 80 | statusListeners.RUnlock() 81 | } 82 | -------------------------------------------------------------------------------- /events_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | type TestBroadcastListener struct { 26 | broadcast *Broadcast 27 | } 28 | 29 | func (l *TestBroadcastListener) OnBroadcast(broadcast *Broadcast) { 30 | l.broadcast = broadcast 31 | } 32 | 33 | func TestBroadcastListeners(t *testing.T) { 34 | require.Empty(t, broadcastListeners.s) 35 | 36 | l := &TestBroadcastListener{} 37 | 38 | AddBroadcastListener(l) 39 | require.Equal(t, 1, len(broadcastListeners.s)) 40 | 41 | require.Nil(t, l.broadcast) 42 | 43 | bc := testBroadcast() 44 | doBroadcastUpdate(bc) 45 | 46 | require.NotNil(t, l.broadcast) 47 | 48 | require.Equal(t, bc, l.broadcast) 49 | } 50 | 51 | type TestStatusListener struct { 52 | node *Node 53 | status NodeStatus 54 | } 55 | 56 | func (l *TestStatusListener) OnChange(node *Node, status NodeStatus) { 57 | l.node = node 58 | l.status = status 59 | } 60 | 61 | func TestStatusListeners(t *testing.T) { 62 | require.Empty(t, statusListeners.s) 63 | 64 | l := &TestStatusListener{} 65 | 66 | AddStatusListener(l) 67 | require.Equal(t, 1, len(statusListeners.s)) 68 | 69 | require.Nil(t, l.node) 70 | require.Equal(t, StatusUnknown, l.status) 71 | 72 | n := testNode() 73 | s := StatusAlive 74 | 75 | doStatusUpdate(n, s) 76 | 77 | require.Equal(t, s, l.status) 78 | require.Equal(t, n, l.node) 79 | } 80 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/clockworksoul/smudge 2 | 3 | go 1.13 4 | 5 | require github.com/stretchr/testify v1.6.1 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 4 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 5 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 6 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 7 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 8 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 9 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 10 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 11 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 12 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "time" 23 | ) 24 | 25 | // LogLevel represents a logging levels to be used as a parameter passed to 26 | // the SetLogThreshhold() function. 27 | type LogLevel byte 28 | 29 | const ( 30 | // LogAll allows all log output of all levels to be emitted. 31 | LogAll LogLevel = iota 32 | 33 | // LogTrace restricts log output to trace level and above. 34 | LogTrace 35 | 36 | // LogDebug restricts log output to debug level and above. 37 | LogDebug 38 | 39 | // LogInfo restricts log output to info level and above. 40 | LogInfo 41 | 42 | // LogWarn restricts log output to warn level and above. 43 | LogWarn 44 | 45 | // LogError restricts log output to error level and above. 46 | LogError 47 | 48 | // LogFatal restricts log output to fatal level. 49 | LogFatal 50 | 51 | // LogOff prevents all log output entirely. 52 | LogOff 53 | ) 54 | 55 | func (s LogLevel) String() string { 56 | switch s { 57 | case LogAll: 58 | return "All" 59 | case LogTrace: 60 | return "Trace" 61 | case LogDebug: 62 | return "Debug" 63 | case LogInfo: 64 | return "Info" 65 | case LogWarn: 66 | return "Warn" 67 | case LogError: 68 | return "Error" 69 | case LogFatal: 70 | return "Fatal" 71 | case LogOff: 72 | return "Off" 73 | default: 74 | return "Unknown" 75 | } 76 | } 77 | 78 | // Logger should be implemented by Logger's that are passed via SetLogger. 79 | type Logger interface { 80 | Log(level LogLevel, a ...interface{}) (int, error) 81 | Logf(level LogLevel, format string, a ...interface{}) (int, error) 82 | } 83 | 84 | // DefaultLogger is the default logger that is included with Smudge. 85 | type DefaultLogger struct{} 86 | 87 | var ( 88 | logThreshhold LogLevel 89 | logger Logger 90 | ) 91 | 92 | // SetLogThreshold allows the output noise level to be adjusted by setting 93 | // the logging priority threshold. 94 | func SetLogThreshold(level LogLevel) { 95 | logThreshhold = level 96 | } 97 | 98 | // SetLogger plugs in another logger to control the output of the library 99 | func SetLogger(l Logger) { 100 | logger = l 101 | } 102 | 103 | // Log writes a log message of a certain level to the logger 104 | func (d DefaultLogger) Log(level LogLevel, a ...interface{}) (n int, err error) { 105 | if level >= logThreshhold { 106 | fmt.Fprint(os.Stderr, prefix(level)+" ") 107 | return fmt.Fprintln(os.Stderr, a...) 108 | } 109 | return 0, nil 110 | } 111 | 112 | // Logf writes a log message with a specific format to the logger 113 | func (d DefaultLogger) Logf(level LogLevel, format string, a ...interface{}) (n int, err error) { 114 | if level >= logThreshhold { 115 | return fmt.Fprintf(os.Stderr, prefix(level)+" "+format+"\n", a...) 116 | } 117 | 118 | return 0, nil 119 | } 120 | 121 | func init() { 122 | SetLogger(DefaultLogger{}) 123 | SetLogThreshold(LogInfo) 124 | } 125 | 126 | func log(level LogLevel, a ...interface{}) (n int, err error) { 127 | if level >= logThreshhold { 128 | return logger.Log(level, a...) 129 | } 130 | return 0, nil 131 | } 132 | func logf(level LogLevel, format string, a ...interface{}) (n int, err error) { 133 | if level >= logThreshhold { 134 | return logger.Logf(level, format, a...) 135 | } 136 | return 0, nil 137 | } 138 | 139 | func prefix(level LogLevel) string { 140 | f := time.Now().Format("02/Jan/2006:15:04:05 MST") 141 | 142 | return fmt.Sprintf("%5s %s -", level.String(), f) 143 | } 144 | 145 | func logTrace(a ...interface{}) (n int, err error) { 146 | return log(LogTrace, a...) 147 | } 148 | 149 | func logDebug(a ...interface{}) (n int, err error) { 150 | return log(LogDebug, a...) 151 | } 152 | 153 | func logInfo(a ...interface{}) (n int, err error) { 154 | return log(LogInfo, a...) 155 | } 156 | 157 | func logWarn(a ...interface{}) (n int, err error) { 158 | return log(LogWarn, a...) 159 | } 160 | 161 | func logError(a ...interface{}) (n int, err error) { 162 | return log(LogError, a...) 163 | } 164 | 165 | func logFatal(a ...interface{}) (n int, err error) { 166 | return log(LogFatal, a...) 167 | } 168 | 169 | func logfTrace(format string, a ...interface{}) (n int, err error) { 170 | return logf(LogTrace, format, a...) 171 | } 172 | 173 | func logfDebug(format string, a ...interface{}) (n int, err error) { 174 | return logf(LogDebug, format, a...) 175 | } 176 | 177 | func logfInfo(format string, a ...interface{}) (n int, err error) { 178 | return logf(LogInfo, format, a...) 179 | } 180 | 181 | func logfWarn(format string, a ...interface{}) (n int, err error) { 182 | return logf(LogWarn, format, a...) 183 | } 184 | 185 | func logfError(format string, a ...interface{}) (n int, err error) { 186 | return logf(LogError, format, a...) 187 | } 188 | 189 | func logfFatal(format string, a ...interface{}) (n int, err error) { 190 | return logf(LogFatal, format, a...) 191 | } 192 | -------------------------------------------------------------------------------- /logo/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clockworksoul/smudge/936065ee451b9a935a5f0fbcb1c289e312468cbc/logo/logo.png -------------------------------------------------------------------------------- /membership.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "errors" 21 | "math" 22 | "net" 23 | "strconv" 24 | "sync" 25 | "time" 26 | ) 27 | 28 | // A scalar value used to calculate a variety of limits 29 | const lambda = 2.5 30 | 31 | // How many standard deviations beyond the mean PING/ACK response time we 32 | // allow before timing out an ACK. 33 | const timeoutToleranceSigmas = 3.0 34 | 35 | const defaultIPv4MulticastAddress = "224.0.0.0" 36 | 37 | const defaultIPv6MulticastAddress = "[ff02::1]" 38 | 39 | var currentHeartbeat uint32 40 | 41 | var pendingAcks = struct { 42 | sync.RWMutex 43 | m map[string]*pendingAck 44 | }{m: make(map[string]*pendingAck)} 45 | 46 | var thisHostAddress string 47 | 48 | var thisHost *Node 49 | 50 | var ipLen = net.IPv4len 51 | 52 | // This flag is set whenever a known node is added or removed. 53 | var knownNodesModifiedFlag = false 54 | 55 | var pingdata = newPingData(GetPingHistoryFrontload(), 50) 56 | 57 | /****************************************************************************** 58 | * Exported functions (for public consumption) 59 | *****************************************************************************/ 60 | 61 | // Begin starts the server by opening a UDP port and beginning the heartbeat. 62 | // Note that this is a blocking function, so act appropriately. 63 | func Begin() { 64 | // Add this host. 65 | logfInfo("Using listen IP: %s", listenIP) 66 | 67 | // Use IPv6 address length if the listen IP is not an IPv4 address 68 | if GetListenIP().To4() == nil { 69 | ipLen = net.IPv6len 70 | } 71 | 72 | initGlobalHostEnvironment() 73 | 74 | logInfo("My host address:", thisHostAddress) 75 | 76 | // Add this node's status. Don't update any other node's statuses: they'll 77 | // report those back to us. 78 | updateNodeStatus(thisHost, StatusAlive, 0, thisHost) 79 | AddNode(thisHost) 80 | 81 | go listenUDP(GetListenPort()) 82 | 83 | // Add initial hosts as specified by the SMUDGE_INITIAL_HOSTS property 84 | for _, address := range GetInitialHosts() { 85 | n, err := CreateNodeByAddress(address) 86 | if err != nil { 87 | logfError("Could not create node %s: %v", address, err) 88 | } else { 89 | AddNode(n) 90 | } 91 | } 92 | 93 | if GetMulticastEnabled() { 94 | go listenUDPMulticast(GetMulticastPort()) 95 | go multicastAnnounce(GetMulticastAddress()) 96 | } 97 | 98 | go startTimeoutCheckLoop() 99 | 100 | // Loop over a randomized list of all known nodes (except for this host 101 | // node), pinging one at a time. If the knownNodesModifiedFlag is set to 102 | // true by AddNode() or RemoveNode(), the we get a fresh list and start 103 | // again. 104 | 105 | for { 106 | var randomAllNodes = knownNodes.getRandomNodes(0, thisHost) 107 | var pingCounter int 108 | 109 | for _, node := range randomAllNodes { 110 | // Exponential backoff of dead nodes, until such time as they are removed. 111 | if node.status == StatusDead { 112 | var dnc *deadNodeCounter 113 | var ok bool 114 | 115 | deadNodeRetries.Lock() 116 | if dnc, ok = deadNodeRetries.m[node.Address()]; !ok { 117 | dnc = &deadNodeCounter{retry: 1, retryCountdown: 2} 118 | deadNodeRetries.m[node.Address()] = dnc 119 | } 120 | deadNodeRetries.Unlock() 121 | 122 | dnc.retryCountdown-- 123 | 124 | if dnc.retryCountdown <= 0 { 125 | dnc.retry++ 126 | dnc.retryCountdown = int(math.Pow(2.0, float64(dnc.retry))) 127 | 128 | if dnc.retry > maxDeadNodeRetries { 129 | logDebug("Forgetting dead node", node.Address()) 130 | 131 | deadNodeRetries.Lock() 132 | delete(deadNodeRetries.m, node.Address()) 133 | deadNodeRetries.Unlock() 134 | 135 | RemoveNode(node) 136 | continue 137 | } 138 | } else { 139 | continue 140 | } 141 | } 142 | 143 | currentHeartbeat++ 144 | 145 | logfTrace("%d - hosts=%d (announce=%d forward=%d)", 146 | currentHeartbeat, 147 | len(randomAllNodes), 148 | emitCount(), 149 | pingRequestCount()) 150 | 151 | PingNode(node) 152 | pingCounter++ 153 | 154 | time.Sleep(time.Millisecond * time.Duration(GetHeartbeatMillis())) 155 | 156 | if knownNodesModifiedFlag { 157 | knownNodesModifiedFlag = false 158 | break 159 | } 160 | } 161 | 162 | if pingCounter == 0 { 163 | logDebug("No nodes to ping. So lonely. :(") 164 | time.Sleep(time.Millisecond * time.Duration(GetHeartbeatMillis())) 165 | } 166 | } 167 | } 168 | 169 | // PingNode can be used to explicitly ping a node. Calls the low-level 170 | // doPingNode(), and outputs a message (and returns an error) if it fails. 171 | func PingNode(node *Node) error { 172 | err := transmitVerbPingUDP(node, currentHeartbeat) 173 | if err != nil { 174 | logInfo("Failure to ping", node, "->", err) 175 | } 176 | 177 | return err 178 | } 179 | 180 | /****************************************************************************** 181 | * Private functions (for internal use only) 182 | *****************************************************************************/ 183 | 184 | // Multicast announcements are constructed as: 185 | // Byte 0 - 1 byte character byte length N 186 | // Bytes 1 to N - Cluster name bytes 187 | // Bytes N+1... - A message (without members) 188 | func decodeMulticastAnnounceBytes(bytes []byte) (string, []byte, error) { 189 | nameBytesLen := int(bytes[0]) 190 | 191 | if nameBytesLen+1 > len(bytes) { 192 | return "", nil, errors.New("Invalid multicast message received") 193 | } 194 | 195 | nameBytes := bytes[1 : nameBytesLen+1] 196 | name := string(nameBytes) 197 | msgBytes := bytes[nameBytesLen+1 : len(bytes)] 198 | 199 | return name, msgBytes, nil 200 | } 201 | 202 | func doForwardOnTimeout(pack *pendingAck) { 203 | filteredNodes := getTargetNodes(pingRequestCount(), thisHost, pack.node) 204 | 205 | if len(filteredNodes) == 0 { 206 | logDebug(thisHost.Address(), "Cannot forward ping request: no more nodes") 207 | 208 | updateNodeStatus(pack.node, StatusDead, currentHeartbeat, thisHost) 209 | } else { 210 | for i, n := range filteredNodes { 211 | logfDebug("(%d/%d) Requesting indirect ping of %s via %s", 212 | i+1, 213 | len(filteredNodes), 214 | pack.node.Address(), 215 | n.Address()) 216 | 217 | transmitVerbForwardUDP(n, pack.node, currentHeartbeat) 218 | } 219 | } 220 | } 221 | 222 | // The number of times any node's new status should be emitted after changes. 223 | // Currently set to (lambda * log(node count)). 224 | func emitCount() int { 225 | logn := math.Log(float64(knownNodes.length())) 226 | mult := (lambda * logn) + 0.5 227 | 228 | return int(mult) 229 | } 230 | 231 | // Multicast announcements are constructed as: 232 | // Byte 0 - 1 byte character byte length N 233 | // Bytes 1 to N - Cluster name bytes 234 | // Bytes N+1... - A message (without members) 235 | func encodeMulticastAnnounceBytes() []byte { 236 | nameBytes := []byte(GetClusterName()) 237 | nameBytesLen := len(nameBytes) 238 | 239 | if nameBytesLen > 0xFF { 240 | panic("Cluster name too long: " + 241 | strconv.FormatInt(int64(nameBytesLen), 10) + 242 | " bytes (max 254)") 243 | } 244 | 245 | msg := newMessage(verbPing, thisHost, currentHeartbeat) 246 | msgBytes := msg.encode() 247 | msgBytesLen := len(msgBytes) 248 | 249 | totalByteCount := 1 + nameBytesLen + msgBytesLen 250 | 251 | bytes := make([]byte, totalByteCount, totalByteCount) 252 | 253 | // Add name length byte 254 | bytes[0] = byte(nameBytesLen) 255 | 256 | // Copy the name bytes 257 | copy(bytes[1:nameBytesLen+1], nameBytes) 258 | 259 | // Copy the message proper 260 | copy(bytes[nameBytesLen+1:totalByteCount], msgBytes) 261 | 262 | return bytes 263 | } 264 | 265 | func guessMulticastAddress() string { 266 | if multicastAddress == "" { 267 | if ipLen == net.IPv6len { 268 | multicastAddress = defaultIPv6MulticastAddress 269 | } else if ipLen == net.IPv4len { 270 | multicastAddress = defaultIPv4MulticastAddress 271 | } else { 272 | logFatal("Failed to determine IPv4/IPv6") 273 | } 274 | } 275 | 276 | return multicastAddress 277 | } 278 | 279 | // getListenInterface gets the network interface for the listen IP 280 | func getListenInterface() (*net.Interface, error) { 281 | ifaces, err := net.Interfaces() 282 | if err == nil { 283 | for _, iface := range ifaces { 284 | addrs, err := iface.Addrs() 285 | if err != nil { 286 | logfWarn("Can not get addresses of interface %s", iface.Name) 287 | continue 288 | } 289 | for _, addr := range addrs { 290 | ip, _, err := net.ParseCIDR(addr.String()) 291 | if err != nil { 292 | continue 293 | } 294 | if ip.String() == GetListenIP().String() { 295 | logfInfo("Found interface with listen IP: %s", iface.Name) 296 | return &iface, nil 297 | } 298 | } 299 | } 300 | } 301 | return nil, errors.New("Could not determine the interface of the listen IP address") 302 | } 303 | 304 | // Returns a random slice of valid ping/forward request targets; i.e., not 305 | // this node, and not dead. 306 | func getTargetNodes(count int, exclude ...*Node) []*Node { 307 | randomNodes := knownNodes.getRandomNodes(0, exclude...) 308 | filteredNodes := make([]*Node, 0, count) 309 | 310 | for _, n := range randomNodes { 311 | if len(filteredNodes) >= count { 312 | break 313 | } 314 | 315 | if n.status == StatusDead { 316 | continue 317 | } 318 | 319 | filteredNodes = append(filteredNodes, n) 320 | } 321 | 322 | return filteredNodes 323 | } 324 | 325 | func initGlobalHostEnvironment() { 326 | thisHost = &Node{ 327 | ip: GetListenIP(), 328 | port: uint16(GetListenPort()), 329 | timestamp: GetNowInMillis(), 330 | pingMillis: PingNoData, 331 | } 332 | 333 | thisHostAddress = thisHost.Address() 334 | } 335 | 336 | func listenUDP(port int) error { 337 | listenAddress, err := net.ResolveUDPAddr("udp", ":"+strconv.FormatInt(int64(port), 10)) 338 | if err != nil { 339 | return err 340 | } 341 | 342 | /* Now listen at selected port */ 343 | c, err := net.ListenUDP("udp", listenAddress) 344 | if err != nil { 345 | return err 346 | } 347 | defer c.Close() 348 | 349 | for { 350 | buf := make([]byte, 2048) // big enough to fit 1280 IPv6 UDP message 351 | n, addr, err := c.ReadFromUDP(buf) 352 | if err != nil { 353 | logError("UDP read error: ", err) 354 | } 355 | 356 | go func(addr *net.UDPAddr, msg []byte) { 357 | err = receiveMessageUDP(addr, buf[0:n]) 358 | if err != nil { 359 | logError(err) 360 | } 361 | }(addr, buf[0:n]) 362 | } 363 | } 364 | 365 | func listenUDPMulticast(port int) error { 366 | addr := GetMulticastAddress() 367 | if addr == "" { 368 | addr = guessMulticastAddress() 369 | } 370 | 371 | listenAddress, err := net.ResolveUDPAddr("udp", addr+":"+strconv.FormatInt(int64(port), 10)) 372 | if err != nil { 373 | return err 374 | } 375 | 376 | /* Now listen at selected port */ 377 | iface, err := getListenInterface() 378 | if err != nil { 379 | return err 380 | } 381 | c, err := net.ListenMulticastUDP("udp", iface, listenAddress) 382 | if err != nil { 383 | return err 384 | } 385 | defer c.Close() 386 | 387 | for { 388 | buf := make([]byte, 2048) // big enough to fit 1280 IPv6 UDP message 389 | n, addr, err := c.ReadFromUDP(buf) 390 | if err != nil { 391 | logError("UDP read error:", err) 392 | } 393 | 394 | go func(addr *net.UDPAddr, bytes []byte) { 395 | name, msgBytes, err := decodeMulticastAnnounceBytes(bytes) 396 | 397 | if err != nil { 398 | logDebug("Ignoring unexpected multicast message.") 399 | } else { 400 | if GetClusterName() == name { 401 | msg, err := decodeMessage(addr.IP, msgBytes) 402 | if err == nil { 403 | logfTrace("Got multicast %v from %v code=%d", 404 | msg.verb, 405 | msg.sender.Address(), 406 | msg.senderHeartbeat) 407 | 408 | // Update statuses of the sender. 409 | updateStatusesFromMessage(msg) 410 | } else { 411 | logError(err) 412 | } 413 | } 414 | } 415 | }(addr, buf[0:n]) 416 | } 417 | } 418 | 419 | // multicastAnnounce is called when the server first starts to broadcast its 420 | // presence to all listening servers within the specified subnet and continues 421 | // to broadcast its presence every multicastAnnounceIntervalSeconds in case 422 | // this value is larger than zero. 423 | func multicastAnnounce(addr string) error { 424 | if addr == "" { 425 | addr = guessMulticastAddress() 426 | } 427 | 428 | fullAddr := addr + ":" + strconv.FormatInt(int64(GetMulticastPort()), 10) 429 | 430 | logInfo("Announcing presence on", fullAddr) 431 | 432 | address, err := net.ResolveUDPAddr("udp", fullAddr) 433 | if err != nil { 434 | logError(err) 435 | return err 436 | } 437 | laddr := &net.UDPAddr{ 438 | IP: GetListenIP(), 439 | Port: 0, 440 | } 441 | for { 442 | c, err := net.DialUDP("udp", laddr, address) 443 | if err != nil { 444 | logError(err) 445 | return err 446 | } 447 | // Compose and send the multicast announcement 448 | msgBytes := encodeMulticastAnnounceBytes() 449 | _, err = c.Write(msgBytes) 450 | if err != nil { 451 | logError(err) 452 | return err 453 | } 454 | 455 | logfTrace("Sent announcement multicast from %v to %v", laddr, fullAddr) 456 | 457 | if GetMulticastAnnounceIntervalSeconds() > 0 { 458 | time.Sleep(time.Second * time.Duration(GetMulticastAnnounceIntervalSeconds())) 459 | } else { 460 | return nil 461 | } 462 | } 463 | } 464 | 465 | // The number of nodes to send a PINGREQ to when a PING times out. 466 | // Currently set to (lambda * log(node count)). 467 | func pingRequestCount() int { 468 | logn := math.Log(float64(knownNodes.length())) 469 | mult := (lambda * logn) + 0.5 470 | 471 | return int(mult) 472 | } 473 | 474 | func receiveMessageUDP(addr *net.UDPAddr, msgBytes []byte) error { 475 | msg, err := decodeMessage(addr.IP, msgBytes) 476 | if err != nil { 477 | return err 478 | } 479 | 480 | logfTrace("Got %v from %v code=%d", 481 | msg.verb, 482 | msg.sender.Address(), 483 | msg.senderHeartbeat) 484 | 485 | // Synchronize heartbeats 486 | if msg.senderHeartbeat > 0 && msg.senderHeartbeat-1 > currentHeartbeat { 487 | logfTrace("Heartbeat advanced from %d to %d", 488 | currentHeartbeat, 489 | msg.senderHeartbeat-1) 490 | 491 | currentHeartbeat = msg.senderHeartbeat - 1 492 | } 493 | 494 | // Update statuses of the sender and any members the message includes. 495 | updateStatusesFromMessage(msg) 496 | 497 | // If there are broadcast bytes in the message, handle them here. 498 | receiveBroadcast(msg.broadcast) 499 | 500 | // Handle the verb. 501 | switch msg.verb { 502 | case verbPing: 503 | err = receiveVerbPingUDP(msg) 504 | case verbAck: 505 | err = receiveVerbAckUDP(msg) 506 | case verbPingRequest: 507 | err = receiveVerbForwardUDP(msg) 508 | case verbNonForwardingPing: 509 | err = receiveVerbNonForwardPingUDP(msg) 510 | } 511 | 512 | if err != nil { 513 | return err 514 | } 515 | 516 | return nil 517 | } 518 | 519 | func receiveVerbAckUDP(msg message) error { 520 | key := msg.sender.Address() + ":" + strconv.FormatInt(int64(msg.senderHeartbeat), 10) 521 | 522 | pendingAcks.RLock() 523 | _, ok := pendingAcks.m[key] 524 | pendingAcks.RUnlock() 525 | 526 | if ok { 527 | msg.sender.Touch() 528 | 529 | pendingAcks.Lock() 530 | 531 | if pack, ok := pendingAcks.m[key]; ok { 532 | // If this is a response to a requested ping, respond to the 533 | // callback node 534 | if pack.callback != nil { 535 | go transmitVerbAckUDP(pack.callback, pack.callbackCode) 536 | } else { 537 | // Note the ping response time. 538 | notePingResponseTime(pack) 539 | } 540 | } 541 | 542 | delete(pendingAcks.m, key) 543 | pendingAcks.Unlock() 544 | } 545 | 546 | return nil 547 | } 548 | 549 | func notePingResponseTime(pack *pendingAck) { 550 | // Note the elapsed time 551 | elapsedMillis := pack.elapsed() 552 | 553 | pack.node.pingMillis = int(elapsedMillis) 554 | 555 | // For the purposes of timeout tolerance, we treat all pings less than 556 | // the ping lower bound as that lower bound. 557 | minMillis := uint32(GetMinPingTime()) 558 | if elapsedMillis < minMillis { 559 | elapsedMillis = minMillis 560 | } 561 | 562 | pingdata.add(elapsedMillis) 563 | 564 | mean, stddev := pingdata.data() 565 | sigmas := pingdata.nSigma(timeoutToleranceSigmas) 566 | 567 | logfTrace("Got ACK in %dms (mean=%.02f stddev=%.02f sigmas=%.02f)", 568 | elapsedMillis, 569 | mean, 570 | stddev, 571 | sigmas) 572 | } 573 | 574 | func receiveVerbForwardUDP(msg message) error { 575 | // We don't forward to a node that we don't know. 576 | 577 | if len(msg.members) >= 0 && 578 | msg.members[0].status == StatusForwardTo { 579 | 580 | member := msg.members[0] 581 | node := member.node 582 | code := member.heartbeat 583 | key := node.Address() + ":" + strconv.FormatInt(int64(code), 10) 584 | 585 | pack := pendingAck{ 586 | node: node, 587 | startTime: GetNowInMillis(), 588 | callback: msg.sender, 589 | callbackCode: code, 590 | packType: packNFP} 591 | 592 | pendingAcks.Lock() 593 | pendingAcks.m[key] = &pack 594 | pendingAcks.Unlock() 595 | 596 | return transmitVerbGenericUDP(node, nil, verbNonForwardingPing, code) 597 | } 598 | 599 | return nil 600 | } 601 | 602 | func receiveVerbPingUDP(msg message) error { 603 | return transmitVerbAckUDP(msg.sender, msg.senderHeartbeat) 604 | } 605 | 606 | func receiveVerbNonForwardPingUDP(msg message) error { 607 | return transmitVerbAckUDP(msg.sender, msg.senderHeartbeat) 608 | } 609 | 610 | func startTimeoutCheckLoop() { 611 | for { 612 | pendingAcks.Lock() 613 | for k, pack := range pendingAcks.m { 614 | elapsed := pack.elapsed() 615 | timeoutMillis := uint32(pingdata.nSigma(timeoutToleranceSigmas)) 616 | 617 | // Ping requests are expected to take quite a bit longer. 618 | // Just call it 2x for now. 619 | if pack.packType == packPingReq { 620 | timeoutMillis *= 2 621 | } 622 | 623 | // This pending ACK has taken longer than expected. Mark it as 624 | // timed out. 625 | if elapsed > timeoutMillis { 626 | switch pack.packType { 627 | case packPing: 628 | go doForwardOnTimeout(pack) 629 | case packPingReq: 630 | logDebug(k, "timed out after", timeoutMillis, "milliseconds (dropped PINGREQ)") 631 | 632 | if knownNodes.contains(pack.callback) { 633 | switch pack.callback.Status() { 634 | case StatusDead: 635 | break 636 | case StatusSuspected: 637 | updateNodeStatus(pack.callback, StatusDead, currentHeartbeat, thisHost) 638 | pack.callback.pingMillis = PingTimedOut 639 | default: 640 | updateNodeStatus(pack.callback, StatusSuspected, currentHeartbeat, thisHost) 641 | pack.callback.pingMillis = PingTimedOut 642 | } 643 | } 644 | case packNFP: 645 | logDebug(k, "timed out after", timeoutMillis, "milliseconds (dropped NFP)") 646 | 647 | if knownNodes.contains(pack.node) { 648 | switch pack.node.Status() { 649 | case StatusDead: 650 | break 651 | case StatusSuspected: 652 | updateNodeStatus(pack.node, StatusDead, currentHeartbeat, thisHost) 653 | pack.callback.pingMillis = PingTimedOut 654 | default: 655 | updateNodeStatus(pack.node, StatusSuspected, currentHeartbeat, thisHost) 656 | pack.callback.pingMillis = PingTimedOut 657 | } 658 | } 659 | } 660 | 661 | delete(pendingAcks.m, k) 662 | } 663 | } 664 | pendingAcks.Unlock() 665 | 666 | time.Sleep(time.Millisecond * 100) 667 | } 668 | } 669 | 670 | func transmitVerbGenericUDP(node *Node, forwardTo *Node, verb messageVerb, code uint32) error { 671 | // Transmit the ACK 672 | remoteAddr, err := net.ResolveUDPAddr("udp", node.Address()) 673 | 674 | c, err := net.DialUDP("udp", nil, remoteAddr) 675 | if err != nil { 676 | return err 677 | } 678 | defer c.Close() 679 | 680 | msg := newMessage(verb, thisHost, code) 681 | 682 | if forwardTo != nil { 683 | msg.addMember(forwardTo, StatusForwardTo, code, forwardTo.statusSource) 684 | } 685 | 686 | // Add members for update. 687 | nodes := getRandomUpdatedNodes(pingRequestCount(), node, thisHost) 688 | 689 | // No updates to distribute? Send out a few updates on other known nodes. 690 | if len(nodes) == 0 { 691 | nodes = knownNodes.getRandomNodes(pingRequestCount(), node, thisHost) 692 | } 693 | 694 | for _, n := range nodes { 695 | err = msg.addMember(n, n.status, n.heartbeat, n.statusSource) 696 | if err != nil { 697 | return err 698 | } 699 | 700 | n.emitCounter-- 701 | } 702 | 703 | // Emit counters for broadcasts can be less than 0. We transmit positive 704 | // numbers, and decrement all the others. At some value < 0, the broadcast 705 | // is removed from the map all together. 706 | broadcast := getBroadcastToEmit() 707 | if broadcast != nil { 708 | if broadcast.emitCounter > 0 { 709 | msg.addBroadcast(broadcast) 710 | } 711 | 712 | broadcast.emitCounter-- 713 | } 714 | 715 | _, err = c.Write(msg.encode()) 716 | if err != nil { 717 | return err 718 | } 719 | 720 | // Decrement the update counters on those nodes 721 | for _, m := range msg.members { 722 | m.node.emitCounter-- 723 | } 724 | 725 | logfTrace("Sent %v to %v", verb, node.Address()) 726 | 727 | return nil 728 | } 729 | 730 | func transmitVerbForwardUDP(node *Node, downstream *Node, code uint32) error { 731 | key := node.Address() + ":" + strconv.FormatInt(int64(code), 10) 732 | 733 | pack := pendingAck{ 734 | node: node, 735 | startTime: GetNowInMillis(), 736 | callback: downstream, 737 | packType: packPingReq} 738 | 739 | pendingAcks.Lock() 740 | pendingAcks.m[key] = &pack 741 | pendingAcks.Unlock() 742 | 743 | return transmitVerbGenericUDP(node, downstream, verbPingRequest, code) 744 | } 745 | 746 | func transmitVerbAckUDP(node *Node, code uint32) error { 747 | return transmitVerbGenericUDP(node, nil, verbAck, code) 748 | } 749 | 750 | func transmitVerbPingUDP(node *Node, code uint32) error { 751 | key := node.Address() + ":" + strconv.FormatInt(int64(code), 10) 752 | pack := pendingAck{ 753 | node: node, 754 | startTime: GetNowInMillis(), 755 | packType: packPing} 756 | 757 | pendingAcks.Lock() 758 | pendingAcks.m[key] = &pack 759 | pendingAcks.Unlock() 760 | 761 | return transmitVerbGenericUDP(node, nil, verbPing, code) 762 | } 763 | 764 | func updateStatusesFromMessage(msg message) { 765 | for _, m := range msg.members { 766 | // If the heartbeat in the message is less then the heartbeat 767 | // associated with the last known status, then we conclude that the 768 | // message is old and we drop it. 769 | if m.heartbeat < m.node.heartbeat { 770 | logfDebug("Message is old (%d vs %d): dropping", 771 | m.node.heartbeat, m.heartbeat) 772 | 773 | continue 774 | } 775 | 776 | switch m.status { 777 | case StatusForwardTo: 778 | // The FORWARD_TO status isn't useful here, so we ignore those. 779 | continue 780 | case StatusDead: 781 | // Don't tell ME I'm dead. 782 | if m.node.Address() != thisHost.Address() { 783 | updateNodeStatus(m.node, m.status, m.heartbeat, m.source) 784 | AddNode(m.node) 785 | } 786 | default: 787 | updateNodeStatus(m.node, m.status, m.heartbeat, m.source) 788 | AddNode(m.node) 789 | } 790 | } 791 | 792 | // Obviously, we know the sender is alive. Report it as such. 793 | if msg.senderHeartbeat > msg.sender.heartbeat { 794 | updateNodeStatus(msg.sender, StatusAlive, msg.senderHeartbeat, thisHost) 795 | } 796 | 797 | // Finally, if we don't know the sender we add it to the known hosts map. 798 | if !knownNodes.contains(msg.sender) { 799 | AddNode(msg.sender) 800 | } 801 | } 802 | 803 | // pendingAckType represents an expectation of a response to a previously 804 | // emitted PING, PINGREQ, or NFP. 805 | type pendingAck struct { 806 | startTime uint32 807 | node *Node 808 | callback *Node 809 | callbackCode uint32 810 | packType pendingAckType 811 | } 812 | 813 | func (a *pendingAck) elapsed() uint32 { 814 | return GetNowInMillis() - a.startTime 815 | } 816 | 817 | // pendingAckType represents the type of PING that a pendingAckType is waiting 818 | // for a response for: PING, PINGREQ, or NFP. 819 | type pendingAckType byte 820 | 821 | const ( 822 | packPing pendingAckType = iota 823 | packPingReq 824 | packNFP 825 | ) 826 | 827 | func (p pendingAckType) String() string { 828 | switch p { 829 | case packPing: 830 | return "PING" 831 | case packPingReq: 832 | return "PINGREQ" 833 | case packNFP: 834 | return "NFP" 835 | default: 836 | return "UNDEFINED" 837 | } 838 | } 839 | -------------------------------------------------------------------------------- /membership_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "os" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | func TestInitGlobalHostEnvironment(t *testing.T) { 27 | var err error 28 | 29 | err = os.Setenv(EnvVarListenIP, "0.0.0.0") 30 | defer os.Unsetenv(EnvVarListenIP) 31 | require.Nil(t, err) 32 | 33 | initGlobalHostEnvironment() 34 | 35 | require.Equal(t, thisHostAddress, thisHost.Address()) 36 | require.Equal(t, "0.0.0.0:9999", thisHost.Address()) 37 | require.Equal(t, uint16(9999), thisHost.Port()) 38 | } 39 | -------------------------------------------------------------------------------- /message.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "errors" 21 | "hash/adler32" 22 | "net" 23 | ) 24 | 25 | // Message contents 26 | // ---[ Base message (11 bytes)]--- 27 | // Bytes 00-03 Checksum (32-bit) 28 | // Bytes 04 Verb (one of {PING|ACK|PINGREQ|NFPING}) 29 | // Bytes 05-06 Sender response port 30 | // Bytes 07-10 Sender current heartbeat 31 | // ---[ Per member (23 bytes)]--- 32 | // Bytes 00 Member status byte 33 | // Bytes 01-16 Member host IP (01-04 for IPv4) 34 | // Bytes 17-18 Member host response port (05-06 for IPv4) 35 | // Bytes 19-22 Sender current heartbeat (07-10 for IPv4) 36 | // ---[ Per broadcast (1 allowed) (23+N bytes) ] 37 | // Bytes 00-15 Origin IP (00-03 for IPv4) 38 | // Bytes 16-17 Origin response port (04-05 for IPv4) 39 | // Bytes 18-21 Origin broadcast counter (06-09 for IPv4) 40 | // Bytes 22-23 Payload length (bytes) (10-11 for IPv4) 41 | // Bytes 24-NN Payload (12-NN for IPv4) 42 | 43 | type message struct { 44 | sender *Node 45 | senderHeartbeat uint32 46 | verb messageVerb 47 | members []*messageMember 48 | broadcast *Broadcast 49 | } 50 | 51 | // Represents a "member" of a message; i.e., a node that the sender knows 52 | // about, about which it wishes to notify the downstream recipient. 53 | type messageMember struct { 54 | // The source of the gossip about the member. 55 | source *Node 56 | 57 | // The last known heartbeat of node. 58 | heartbeat uint32 59 | 60 | // The subject of the gossip. 61 | node *Node 62 | 63 | // The status that the gossip is conveying. 64 | status NodeStatus 65 | } 66 | 67 | // Convenience function. Creates a new message instance. 68 | func newMessage(verb messageVerb, sender *Node, senderHeartbeat uint32) message { 69 | return message{ 70 | sender: sender, 71 | senderHeartbeat: senderHeartbeat, 72 | verb: verb, 73 | } 74 | } 75 | 76 | // Adds a broadcast to this message. Only one broadcast is allowed; subsequent 77 | // calls will replace an existing broadcast. 78 | func (m *message) addBroadcast(broadcast *Broadcast) { 79 | m.broadcast = broadcast 80 | } 81 | 82 | // Adds a member status update to this message. The maximum number of allowed 83 | // members is 2^6 - 1 = 63, though it is incredibly unlikely that this maximum 84 | // will be reached without an absurdly high lambda. There aren't yet many 85 | // 88 billion node clusters (assuming lambda of 2.5). 86 | func (m *message) addMember(node *Node, status NodeStatus, heartbeat uint32, gossipSource *Node) error { 87 | if m.members == nil { 88 | m.members = make([]*messageMember, 0, 32) 89 | } else if len(m.members) >= 63 { 90 | return errors.New("member list overflow") 91 | } 92 | 93 | messageMember := messageMember{ 94 | heartbeat: heartbeat, 95 | node: node, 96 | status: status, 97 | source: gossipSource, 98 | } 99 | 100 | m.members = append(m.members, &messageMember) 101 | 102 | return nil 103 | } 104 | 105 | // Message contents 106 | // ---[ Base message (12 bytes)]--- 107 | // Bytes 00-03 Checksum (32-bit) 108 | // Bytes 04 Verb (one of {PING|ACK|PINGREQ|NFPING}) 109 | // Bytes 05-06 Sender response port 110 | // Bytes 07-10 Sender ID Code 111 | // ---[ Per member (23 bytes, 17 bytes for IPv4)]--- 112 | // Bytes 00 Member status byte 113 | // Bytes 01-16 Member host IP (01-04 for IPv4) 114 | // Bytes 17-18 Member host response port (05-06 for IPv4) 115 | // Bytes 19-22 Member heartbeat (07-10 for IPv4) 116 | // Bytes 23-38 Gossip source IP (11-14 fit IPv4) 117 | // Bytes 39-40 Gossip source response port (15-16 for IPv4) 118 | 119 | func (m *message) encode() []byte { 120 | // Pre-calculate the message size. Each message prefix is 11 bytes. 121 | // Each member has a constant size of 9 bytes, plus 2 times the length of 122 | // the IP (4 for IPv4, 16 for IPv6). 123 | size := 11 + (len(m.members) * (9 + ipLen + ipLen)) 124 | 125 | if m.broadcast != nil { 126 | size += 8 + ipLen + len(m.broadcast.bytes) 127 | } 128 | 129 | bytes := make([]byte, size, size) 130 | 131 | // An index pointer (start at 4 to accommodate checksum) 132 | p := 4 133 | 134 | // Byte 00 135 | // Rightmost 2 bits: verb (one of {P|A|F|N}) 136 | // Leftmost 6 bits: number of members in payload 137 | verbByte := byte(len(m.members)) 138 | verbByte = (verbByte << 2) | byte(m.verb) 139 | p += encodeByte(verbByte, bytes, p) 140 | 141 | // Bytes 01-02 Sender response port 142 | p += encodeUint16(m.sender.port, bytes, p) 143 | 144 | // Bytes 03-06 ID Code 145 | p += encodeUint32(m.senderHeartbeat, bytes, p) 146 | 147 | // Each member data requires 23 bytes (11 for IPv4). 148 | for _, member := range m.members { 149 | mnode := member.node 150 | mstatus := member.status 151 | mcode := member.heartbeat 152 | snode := member.source 153 | 154 | // Byte p + 00 155 | bytes[p] = byte(mstatus) 156 | p++ 157 | 158 | var ipb net.IP 159 | 160 | // Member host IP 161 | // IPv4: Bytes (p + 01) to (p + 04) 162 | // IPv6: Bytes (p + 01) to (p + 16) 163 | if ipLen == net.IPv4len { 164 | ipb = mnode.ip.To4() 165 | } else if ipLen == net.IPv6len { 166 | ipb = mnode.ip.To16() 167 | } 168 | 169 | for i := 0; i < ipLen; i++ { 170 | bytes[p+i] = ipb[i] 171 | } 172 | p += ipLen 173 | 174 | // Member host response port 175 | // IPv4: Bytes (p + 05) to (p + 06) 176 | // IPv6: Bytes (p + 17) to (p + 18) 177 | p += encodeUint16(mnode.port, bytes, p) 178 | 179 | // Member heartbeat 180 | // IPv4: Bytes (p + 07) to (p + 10) 181 | // IPv6: Bytes (p + 19) to (p + 22) 182 | p += encodeUint32(mcode, bytes, p) 183 | 184 | if snode != nil { 185 | // Gossip source host IP 186 | // IPv4: Bytes (p + 11) to (p + 14) 187 | // IPv6: Bytes (p + 23) to (p + 39) 188 | if ipLen == net.IPv4len { 189 | ipb = snode.ip.To4() 190 | } else if ipLen == net.IPv6len { 191 | ipb = snode.ip.To16() 192 | } 193 | 194 | for i := 0; i < ipLen; i++ { 195 | bytes[p+i] = ipb[i] 196 | } 197 | 198 | p += ipLen 199 | 200 | // Gossip source host response port 201 | // IPv4: Bytes (p + 15) to (p + 16) 202 | // IPv6: Bytes (p + 40) to (p + 41) 203 | p += encodeUint16(snode.port, bytes, p) 204 | } else { 205 | p += ipLen + 2 206 | } 207 | } 208 | 209 | if m.broadcast != nil { 210 | bbytes := m.broadcast.encode() 211 | for i, v := range bbytes { 212 | bytes[p+i] = v 213 | } 214 | } 215 | 216 | checksum := adler32.Checksum(bytes[4:]) 217 | encodeUint32(checksum, bytes, 0) 218 | 219 | return bytes 220 | } 221 | 222 | // If members exist on this message, and that message has the "forward to" 223 | // status, this function returns it; otherwise it returns nil. 224 | func (m *message) getForwardTo() *messageMember { 225 | if len(m.members) > 0 && m.members[0].status == StatusForwardTo { 226 | return m.members[0] 227 | } 228 | 229 | return nil 230 | } 231 | 232 | // Parses the bytes received in a UDP message. 233 | // If the address:port from the message can't be associated with a known 234 | // (live) node, then an instance of message.sender will be created from 235 | // available data but not explicitly added to the known nodes. 236 | func decodeMessage(sourceIP net.IP, bytes []byte) (message, error) { 237 | var err error 238 | 239 | // An index pointer 240 | p := 0 241 | 242 | // Bytes 00-03 Checksum (32-bit) 243 | checksumStated, p := decodeUint32(bytes, p) 244 | checksumCalculated := adler32.Checksum(bytes[4:]) 245 | if checksumCalculated != checksumStated { 246 | return newMessage(255, nil, 0), 247 | errors.New("checksum failure from " + sourceIP.String()) 248 | } 249 | 250 | // Byte 04 251 | // Rightmost 2 bits: verb (one of {P|A|F|N}) 252 | // Leftmost 6 bits: number of members in payload 253 | v, p := decodeByte(bytes, p) 254 | verb := messageVerb(v & 0x03) 255 | 256 | memberCount := int(v >> 2) 257 | 258 | // Bytes 05-06 Sender response port 259 | senderPort, p := decodeUint16(bytes, p) 260 | 261 | // Bytes 07-10 Sender ID Code 262 | senderHeartbeat, p := decodeUint32(bytes, p) 263 | 264 | // Now that we have the IP and port, we can find the Node. 265 | sender := knownNodes.getByIP(sourceIP, senderPort) 266 | 267 | // We don't know this node, so create a new one! 268 | if sender == nil { 269 | sender, _ = CreateNodeByIP(sourceIP, senderPort) 270 | } 271 | 272 | // Now that we have the verb, node, and code, we can build the mesage 273 | m := newMessage(verb, sender, senderHeartbeat) 274 | 275 | memberLastIndex := p + (memberCount * (9 + ipLen + ipLen)) 276 | 277 | if len(bytes) > p { 278 | m.members = decodeMembers(memberCount, bytes[p:memberLastIndex]) 279 | } 280 | 281 | if len(bytes) > memberLastIndex { 282 | m.broadcast, err = decodeBroadcast(bytes[memberLastIndex:]) 283 | } 284 | 285 | return m, err 286 | } 287 | 288 | func decodeMembers(memberCount int, bytes []byte) []*messageMember { 289 | // Bytes 00 Member status byte 290 | // Bytes 01-16 Member host IP (01-04 for IPv4) 291 | // Bytes 17-18 Member host response port (05-06 for IPv4) 292 | // Bytes 19-22 Member heartbeat (07-10 for IPv4) 293 | 294 | members := make([]*messageMember, 0, 1) 295 | 296 | // An index pointer 297 | p := 0 298 | 299 | for p < len(bytes) { 300 | var mstatus NodeStatus 301 | var mip net.IP 302 | var mport uint16 303 | var mcode uint32 304 | var mnode *Node 305 | var sip net.IP 306 | var sport uint16 307 | var snode *Node 308 | 309 | // Byte 00 Member status byte 310 | mstatus = NodeStatus(bytes[p]) 311 | p++ 312 | 313 | if ipLen == net.IPv6len { 314 | // Bytes 01-16 member IP 315 | mip = make(net.IP, net.IPv6len) 316 | copy(mip, bytes[p:p+16]) 317 | } else { 318 | // Bytes 01-04 member IPv4 319 | mip = net.IPv4(bytes[p+0], bytes[p+1], bytes[p+2], bytes[p+3]) 320 | } 321 | p += ipLen 322 | 323 | // Bytes 17-18 member response port 324 | mport, p = decodeUint16(bytes, p) 325 | 326 | // Bytes 19-22 member heartbeat 327 | mcode, p = decodeUint32(bytes, p) 328 | 329 | if len(mip) > 0 { 330 | // Find the sender by the address associated with the message 331 | mnode = knownNodes.getByIP(mip, mport) 332 | 333 | // We still don't know this node, so create a new one! 334 | if mnode == nil { 335 | mnode, _ = CreateNodeByIP(mip, mport) 336 | } 337 | } 338 | 339 | if ipLen == net.IPv6len { 340 | // Bytes 01-16 member IP 341 | sip = make(net.IP, net.IPv6len) 342 | copy(sip, bytes[p:p+16]) 343 | } else { 344 | // Bytes 01-04 member IPv4 345 | sip = net.IPv4(bytes[p+0], bytes[p+1], bytes[p+2], bytes[p+3]) 346 | } 347 | p += ipLen 348 | 349 | // Bytes 17-18 member response port 350 | sport, p = decodeUint16(bytes, p) 351 | 352 | if len(sip) > 0 { 353 | // Find the sender by the address associated with the message 354 | snode = knownNodes.getByIP(sip, sport) 355 | 356 | // We still don't know this node, so create a new one! 357 | if snode == nil { 358 | snode, _ = CreateNodeByIP(sip, sport) 359 | } 360 | } 361 | 362 | member := messageMember{ 363 | heartbeat: mcode, 364 | node: mnode, 365 | source: snode, 366 | status: mstatus, 367 | } 368 | 369 | members = append(members, &member) 370 | } 371 | 372 | return members 373 | } 374 | -------------------------------------------------------------------------------- /messageVerb.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | type messageVerb byte 20 | 21 | const ( 22 | // VerbPing represents a simple ping. If this ping is not responded to with 23 | // an ack within a timeout period, the pinging host will attempt to ping 24 | // indirectly via one or more additional hosts with a ping request. 25 | verbPing messageVerb = iota 26 | 27 | // VerbAck represents a response to a ping request. 28 | verbAck 29 | 30 | // VerbPingRequest represents a request made by one host to another to ping 31 | // a third host whose live status is in question. 32 | verbPingRequest 33 | 34 | // VerbNonForwardingPing represents a ping in response to a ping request. 35 | // If the ping times out, the host does not follow up with a ping request 36 | // to any other hosts. 37 | verbNonForwardingPing 38 | ) 39 | 40 | func (v messageVerb) String() string { 41 | switch v { 42 | case verbPing: 43 | return "PING" 44 | case verbAck: 45 | return "ACK" 46 | case verbPingRequest: 47 | return "PINGREQ" 48 | case verbNonForwardingPing: 49 | return "NFPING" 50 | default: 51 | return "UNDEFINED" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /message_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "net" 21 | "reflect" 22 | "testing" 23 | ) 24 | 25 | // Identical but distinct instance from node1b 26 | var node1a = Node{ 27 | ip: net.IP([]byte{127, 0, 0, 1}), 28 | port: 1234, 29 | timestamp: 87878787, 30 | status: StatusAlive, 31 | emitCounter: 42, 32 | pingMillis: PingNoData} 33 | 34 | // Identical but distinct instance from node1a 35 | var node1b = Node{ 36 | ip: net.IP([]byte{127, 0, 0, 1}), 37 | port: 1234, 38 | timestamp: 87878787, 39 | status: StatusAlive, 40 | emitCounter: 42, 41 | pingMillis: PingNoData} 42 | 43 | // Different from node1a and node1b 44 | var node2 = Node{ 45 | ip: net.IP([]byte{127, 0, 0, 1}), 46 | port: 10001, 47 | timestamp: GetNowInMillis(), 48 | status: StatusAlive, 49 | emitCounter: 42, 50 | pingMillis: PingNoData} 51 | 52 | var message1a = message{ 53 | sender: &node1a, 54 | senderHeartbeat: 255, 55 | verb: verbPing} 56 | 57 | var message1b = message{ 58 | sender: &node1b, 59 | senderHeartbeat: 255, 60 | verb: verbPing} 61 | 62 | var message2 = message{ 63 | sender: &node2, 64 | senderHeartbeat: 23, 65 | verb: verbAck} 66 | 67 | // Does deep equality of two different but identical messages return true? 68 | func TestDeepEqualityTrue(t *testing.T) { 69 | if !reflect.DeepEqual(message1a, message1b) { 70 | t.Fail() 71 | } 72 | } 73 | 74 | // Does deep equality of two different messages return false? 75 | func TestDeepEqualityFalse(t *testing.T) { 76 | if reflect.DeepEqual(message1a, message2) { 77 | t.Fail() 78 | } 79 | } 80 | 81 | // Endode and decode a simple message without any members, and see if 82 | // the input/output match. 83 | func TestEncodeDecodeBasic(t *testing.T) { 84 | timestamp := uint32(87878787) 85 | 86 | sender := Node{ 87 | ip: net.IP([]byte{127, 0, 0, 1}), 88 | port: 1234, 89 | timestamp: timestamp, 90 | pingMillis: PingNoData, 91 | } 92 | 93 | message := message{ 94 | sender: &sender, 95 | senderHeartbeat: 255, 96 | verb: verbPing} 97 | 98 | ip := net.IP([]byte{127, 0, 0, 1}) 99 | bytes := message.encode() 100 | 101 | decoded, err := decodeMessage(ip, bytes) 102 | decoded.sender.timestamp = timestamp 103 | 104 | if err != nil { 105 | t.Error(err) 106 | } 107 | 108 | if !reflect.DeepEqual(message, decoded) { 109 | t.Error("Messages do not match:") 110 | 111 | t.Log(" Input:", message) 112 | t.Log("Output:", decoded) 113 | t.Log(" Input node:", message.sender) 114 | t.Log("Output node:", decoded.sender) 115 | } 116 | } 117 | 118 | // Endode and decode a simple IPv6 message without any members, and see if 119 | // the input/output match. 120 | func TestEncodeDecodeBasicIPv6(t *testing.T) { 121 | timestamp := uint32(87878787) 122 | 123 | sender := Node{ 124 | ip: net.IP{255, 254, 253, 252, 251, 250, 240, 230, 220, 210, 200, 10, 20, 30, 40, 50}, 125 | port: 1234, 126 | timestamp: timestamp, 127 | pingMillis: PingNoData, 128 | } 129 | 130 | message := message{ 131 | sender: &sender, 132 | senderHeartbeat: 255, 133 | verb: verbPing} 134 | 135 | ipLen = net.IPv6len // encode IPv6 136 | ip := net.IP{255, 254, 253, 252, 251, 250, 240, 230, 220, 210, 200, 10, 20, 30, 40, 50} 137 | bytes := message.encode() 138 | decoded, err := decodeMessage(ip, bytes) 139 | decoded.sender.timestamp = timestamp 140 | 141 | if err != nil { 142 | t.Error(err) 143 | } 144 | 145 | if !reflect.DeepEqual(message, decoded) { 146 | t.Error("Messages do not match:") 147 | 148 | t.Log(" Input:", message) 149 | t.Log("Output:", decoded) 150 | t.Log(" Input node:", message.sender) 151 | t.Log("Output node:", decoded.sender) 152 | } 153 | 154 | ipLen = net.IPv4len // reset to IPv4 155 | } 156 | 157 | // Endode and decode a simple message with one member, and see if 158 | // the input/output match. 159 | func TestEncodeDecode1Member(t *testing.T) { 160 | timestamp := uint32(87878787) 161 | 162 | sender := Node{ 163 | ip: net.IP([]byte{127, 0, 0, 1}), 164 | port: 1234, 165 | timestamp: timestamp, 166 | pingMillis: PingNoData, 167 | } 168 | 169 | member := Node{ 170 | ip: net.IP([]byte{127, 0, 0, 2}).To16(), 171 | port: 9000, 172 | timestamp: timestamp, 173 | pingMillis: PingNoData, 174 | } 175 | 176 | message := message{ 177 | sender: &sender, 178 | senderHeartbeat: 255, 179 | verb: verbPing} 180 | message.addMember(&member, StatusDead, 38, &member) 181 | 182 | if len(message.members) != 1 { 183 | t.Error("No member in the input members list!") 184 | } 185 | 186 | ip := net.IP([]byte{127, 0, 0, 1}) 187 | bytes := message.encode() 188 | if len(bytes) != 28 { 189 | t.Error("Encoded message length is invalid.") 190 | t.Log("Should be 28 but found: ", len(bytes)) 191 | } 192 | 193 | decoded, err := decodeMessage(ip, bytes) 194 | decoded.sender.timestamp = timestamp 195 | decoded.members[0].node.timestamp = timestamp 196 | decoded.members[0].source.timestamp = timestamp 197 | 198 | if err != nil { 199 | t.Error(err) 200 | } 201 | 202 | if len(decoded.members) != 1 { 203 | t.Error("No member in the output members list!") 204 | } 205 | 206 | if !reflect.DeepEqual(message, decoded) { 207 | t.Error("Messages do not match") 208 | 209 | t.Log(" Input:", message.members[0]) 210 | t.Log("Output:", decoded.members[0]) 211 | 212 | t.Log(" Input node:", message.members[0].node) 213 | t.Log("Output node:", decoded.members[0].node) 214 | 215 | t.Log(" Input source:", message.members[0].source) 216 | t.Log("Output source:", decoded.members[0].source) 217 | } 218 | } 219 | 220 | // Endode and decode a simple message with one ipv6 member, and see if 221 | // the input/output match. 222 | func TestEncodeDecode1MemberIPv6(t *testing.T) { 223 | timestamp := uint32(87878787) 224 | 225 | sender := Node{ 226 | ip: net.IP{255, 254, 253, 252, 251, 250, 240, 230, 220, 210, 200, 10, 20, 30, 40, 50}, 227 | port: 1234, 228 | timestamp: timestamp, 229 | pingMillis: PingNoData, 230 | } 231 | 232 | member := Node{ 233 | ip: net.IP{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160}, 234 | port: 9000, 235 | timestamp: timestamp, 236 | pingMillis: PingNoData, 237 | } 238 | 239 | message := message{ 240 | sender: &sender, 241 | senderHeartbeat: 255, 242 | verb: verbPing} 243 | message.addMember(&member, StatusDead, 38, &member) 244 | 245 | if len(message.members) != 1 { 246 | t.Error("No member in the input members list!") 247 | } 248 | 249 | ipLen = net.IPv6len // encode for IPv6 250 | ip := net.IP{255, 254, 253, 252, 251, 250, 240, 230, 220, 210, 200, 10, 20, 30, 40, 50} 251 | bytes := message.encode() 252 | if len(bytes) != 52 { 253 | t.Error("Encoded message length is invalid.") 254 | t.Log("Should be 52 but found: ", len(bytes)) 255 | } 256 | 257 | decoded, err := decodeMessage(ip, bytes) 258 | decoded.sender.timestamp = timestamp 259 | decoded.members[0].node.timestamp = timestamp 260 | decoded.members[0].source.timestamp = timestamp 261 | 262 | if err != nil { 263 | t.Error(err) 264 | } 265 | 266 | if len(decoded.members) != 1 { 267 | t.Error("No member in the output members list!") 268 | } 269 | 270 | if !reflect.DeepEqual(message, decoded) { 271 | t.Error("Messages do not match") 272 | 273 | t.Log(" Input:", message.members[0]) 274 | t.Log("Output:", decoded.members[0]) 275 | t.Log(" Input node:", message.members[0].node) 276 | t.Log("Output node:", decoded.members[0].node) 277 | t.Log(" Input source:", message.members[0].source) 278 | t.Log("Output source:", decoded.members[0].source) 279 | } 280 | 281 | ipLen = net.IPv4len // reset to IPv4 for next test 282 | } 283 | 284 | // Endode and decode a simple message with one member and message, and see if 285 | // the input/output match. 286 | func TestEncodeDecode1MemberBroadcast(t *testing.T) { 287 | timestamp := uint32(87878787) 288 | 289 | sender := Node{ 290 | ip: net.IP([]byte{127, 0, 0, 1}), 291 | port: 1234, 292 | timestamp: timestamp, 293 | pingMillis: PingNoData} 294 | 295 | member := Node{ 296 | ip: net.IP([]byte{127, 0, 0, 2}), 297 | port: 9000, 298 | timestamp: timestamp, 299 | pingMillis: PingNoData} 300 | 301 | message := message{ 302 | sender: &sender, 303 | senderHeartbeat: 255, 304 | verb: verbPing} 305 | message.addMember(&member, StatusDead, 38, &member) 306 | 307 | broadcast := Broadcast{ 308 | bytes: []byte("This is a message"), //len=17 309 | origin: &sender, 310 | index: 42} 311 | message.addBroadcast(&broadcast) 312 | 313 | if message.broadcast == nil { 314 | t.Error("Broadcast not set properly") 315 | } 316 | 317 | ip := net.IP([]byte{127, 0, 0, 1}) 318 | bytes := message.encode() 319 | if len(bytes) != 57 { 320 | t.Error("Encoded message length is invalid.") 321 | t.Log("Should be 57 but found: ", len(bytes)) 322 | } 323 | 324 | decoded, err := decodeMessage(ip, bytes) 325 | decoded.sender.timestamp = timestamp 326 | decoded.members[0].node.timestamp = timestamp 327 | decoded.members[0].source.timestamp = timestamp 328 | 329 | if err != nil { 330 | t.Error(err) 331 | } 332 | 333 | if decoded.broadcast == nil { 334 | t.Error("Broadcast not decoded") 335 | } 336 | 337 | message.broadcast.origin = nil 338 | decoded.broadcast.origin = nil 339 | 340 | if !reflect.DeepEqual(message.broadcast, decoded.broadcast) { 341 | t.Error("Broadcasts do not match:") 342 | t.Error(" Input bcast:", message.broadcast) 343 | t.Error("Output bcast:", decoded.broadcast) 344 | } 345 | } 346 | 347 | // Endode and decode a simple message with one ipv6 member and message, and see if 348 | // the input/output match. 349 | func TestEncodeDecode1MemberBroadcastIPv6(t *testing.T) { 350 | timestamp := uint32(87878787) 351 | 352 | sender := Node{ 353 | ip: net.IP{255, 254, 253, 252, 251, 250, 240, 230, 220, 210, 200, 10, 20, 30, 40, 50}, 354 | port: 1234, 355 | timestamp: timestamp, 356 | pingMillis: PingNoData} 357 | 358 | member := Node{ 359 | ip: net.IP{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160}, 360 | port: 9000, 361 | timestamp: timestamp, 362 | pingMillis: PingNoData} 363 | 364 | message := message{ 365 | sender: &sender, 366 | senderHeartbeat: 255, 367 | verb: verbPing} 368 | message.addMember(&member, StatusDead, 38, &member) 369 | 370 | broadcast := Broadcast{ 371 | bytes: []byte("This is a message"), 372 | origin: &sender, 373 | index: 42} 374 | message.addBroadcast(&broadcast) 375 | 376 | if message.broadcast == nil { 377 | t.Error("Broadcast not set properly") 378 | } 379 | 380 | ipLen = net.IPv6len // encode for IPv6 381 | ip := net.IP{255, 254, 253, 252, 251, 250, 240, 230, 220, 210, 200, 10, 20, 30, 40, 50} 382 | bytes := message.encode() 383 | if len(bytes) != 93 { 384 | t.Error("Encoded message length is invalid.") 385 | t.Log("Should be 93 but found: ", len(bytes)) 386 | } 387 | 388 | decoded, err := decodeMessage(ip, bytes) 389 | decoded.sender.timestamp = timestamp 390 | decoded.members[0].node.timestamp = timestamp 391 | decoded.members[0].source.timestamp = timestamp 392 | 393 | if err != nil { 394 | t.Error(err) 395 | } 396 | 397 | if decoded.broadcast == nil { 398 | t.Error("Broadcast not decoded") 399 | } 400 | 401 | message.broadcast.origin = nil 402 | decoded.broadcast.origin = nil 403 | 404 | if !reflect.DeepEqual(message.broadcast, decoded.broadcast) { 405 | t.Error("Broadcasts do not match:") 406 | t.Error(" Input bcast:", message.broadcast) 407 | t.Error("Output bcast:", decoded.broadcast) 408 | } 409 | 410 | ipLen = net.IPv4len 411 | } 412 | -------------------------------------------------------------------------------- /node.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "fmt" 21 | "net" 22 | "time" 23 | ) 24 | 25 | const ( 26 | // PingNoData is returned by n.PingMillis() to indicate that a node has 27 | // not yet been pinged, and therefore no ping data exists. 28 | PingNoData int = -1 29 | 30 | // PingTimedOut is returned by n.PingMillis() to indicate that a node's 31 | // last PING timed out. This is the typical value for dead nodes. 32 | PingTimedOut int = -2 33 | ) 34 | 35 | // Node represents a single node in the cluster and its status 36 | type Node struct { 37 | ip net.IP 38 | port uint16 39 | timestamp uint32 40 | address string 41 | pingMillis int 42 | status NodeStatus 43 | emitCounter int8 44 | heartbeat uint32 45 | statusSource *Node 46 | } 47 | 48 | // Address rReturns the address for this node in string format, which is simply 49 | // the node's local IP and listen port. This is used as a unique identifier 50 | // throughout the code base. 51 | func (n *Node) Address() string { 52 | if n.address == "" { 53 | n.address = nodeAddressString(n.ip, n.port) 54 | } 55 | 56 | return n.address 57 | } 58 | 59 | // Age returns the time since we last heard from this node, in milliseconds. 60 | func (n *Node) Age() uint32 { 61 | return GetNowInMillis() - n.timestamp 62 | } 63 | 64 | // EmitCounter returns the number of times remaining that current status 65 | // will be emitted by this node to other nodes. 66 | func (n *Node) EmitCounter() int8 { 67 | return n.emitCounter 68 | } 69 | 70 | // IP returns the IP associated with this node. 71 | func (n *Node) IP() net.IP { 72 | return n.ip 73 | } 74 | 75 | // PingMillis returns the milliseconds transpired between the most recent 76 | // PING to this node and its responded ACK. If this node has not yet been 77 | // pinged, this vaue will be PingNoData (-1). If this node's last PING timed 78 | // out, this value will be PingTimedOut (-2). 79 | func (n *Node) PingMillis() int { 80 | return n.pingMillis 81 | } 82 | 83 | // Port returns the port associated with this node. 84 | func (n *Node) Port() uint16 { 85 | return n.port 86 | } 87 | 88 | // Status returns this node's current status. 89 | func (n *Node) Status() NodeStatus { 90 | return n.status 91 | } 92 | 93 | // StatusSource returns a pointer to the node that originally stated this 94 | // node's Status; the source of the gossip. 95 | func (n *Node) StatusSource() *Node { 96 | return n.statusSource 97 | } 98 | 99 | // Timestamp returns the timestamp of this node's last ping or status update, 100 | // in milliseconds from the epoch 101 | func (n *Node) Timestamp() uint32 { 102 | return n.timestamp 103 | } 104 | 105 | // Touch updates the timestamp to the local time in milliseconds. 106 | func (n *Node) Touch() { 107 | n.timestamp = GetNowInMillis() 108 | } 109 | 110 | func nodeAddressString(ip net.IP, port uint16) string { 111 | if ip.To4() != nil { 112 | return fmt.Sprintf("%s:%d", ip.String(), port) 113 | } 114 | return fmt.Sprintf("[%s]:%d", ip.String(), port) 115 | } 116 | 117 | // GetNowInMillis returns the current local time in milliseconds since the 118 | // epoch. 119 | func GetNowInMillis() uint32 { 120 | return uint32(time.Now().UnixNano() / int64(time.Millisecond)) 121 | } 122 | -------------------------------------------------------------------------------- /nodeMap.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "math/rand" 21 | "net" 22 | "sync" 23 | ) 24 | 25 | type nodeMap struct { 26 | sync.RWMutex 27 | 28 | nodes map[string]*Node 29 | } 30 | 31 | func (m *nodeMap) init() { 32 | m.nodes = make(map[string]*Node) 33 | } 34 | 35 | // Adds a node. Returns key, value. 36 | // Updates node heartbeat in the process. 37 | // This is the method called by all Add* functions. 38 | func (m *nodeMap) add(node *Node) (string, *Node, error) { 39 | key := node.Address() 40 | 41 | m.Lock() 42 | m.nodes[node.Address()] = node 43 | m.Unlock() 44 | 45 | return key, node, nil 46 | } 47 | 48 | func (m *nodeMap) delete(node *Node) (string, *Node, error) { 49 | m.Lock() 50 | 51 | delete(m.nodes, node.Address()) 52 | 53 | m.Unlock() 54 | 55 | return node.Address(), node, nil 56 | } 57 | 58 | func (m *nodeMap) contains(node *Node) bool { 59 | return m.containsByAddress(node.Address()) 60 | } 61 | 62 | func (m *nodeMap) containsByAddress(address string) bool { 63 | m.RLock() 64 | 65 | _, ok := m.nodes[address] 66 | 67 | m.RUnlock() 68 | 69 | return ok 70 | } 71 | 72 | // Returns a pointer to the requested Node 73 | func (m *nodeMap) getByAddress(address string) *Node { 74 | m.RLock() 75 | node, _ := m.nodes[address] 76 | m.RUnlock() 77 | 78 | return node 79 | } 80 | 81 | // Returns a pointer to the requested Node. If port is 0, is uses the value 82 | // of GetListenPort(). If the Node cannot be found, this returns nil. 83 | func (m *nodeMap) getByIP(ip net.IP, port uint16) *Node { 84 | if port == 0 { 85 | port = uint16(GetListenPort()) 86 | } 87 | 88 | address := nodeAddressString(ip, port) 89 | 90 | return m.getByAddress(address) 91 | } 92 | 93 | // Returns a slice of Node[] of from 0 to len(nodes) nodes. 94 | // If size is < len(nodes), that many nodes are randomly chosen and 95 | // returned. 96 | func (m *nodeMap) getRandomNodes(size int, exclude ...*Node) []*Node { 97 | allNodes := m.values() 98 | 99 | if size == 0 { 100 | size = len(allNodes) 101 | } 102 | 103 | // First, shuffle the allNodes slice 104 | for i := range allNodes { 105 | j := rand.Intn(i + 1) 106 | allNodes[i], allNodes[j] = allNodes[j], allNodes[i] 107 | } 108 | 109 | // Copy the first size nodes that are not otherwise excluded 110 | filtered := make([]*Node, 0, len(allNodes)) 111 | 112 | // Horribly inefficient. Fix this later. 113 | 114 | var c int 115 | Outer: 116 | for _, n := range allNodes { 117 | // Is the node in the excluded list? 118 | for _, e := range exclude { 119 | if n.Address() == e.Address() { 120 | continue Outer 121 | } 122 | } 123 | 124 | // Now we can append it 125 | filtered = append(filtered, n) 126 | c++ 127 | 128 | if c >= size { 129 | break Outer 130 | } 131 | } 132 | 133 | return filtered 134 | } 135 | 136 | func (m *nodeMap) length() int { 137 | return len(m.nodes) 138 | } 139 | 140 | func (m *nodeMap) lengthWithStatus(status NodeStatus) int { 141 | m.RLock() 142 | 143 | i := 0 144 | for _, v := range m.nodes { 145 | if v.status == status { 146 | i++ 147 | } 148 | } 149 | 150 | m.RUnlock() 151 | 152 | return i 153 | } 154 | 155 | func (m *nodeMap) keys() []string { 156 | m.RLock() 157 | 158 | keys := make([]string, len(m.nodes)) 159 | 160 | i := 0 161 | for k := range m.nodes { 162 | keys[i] = k 163 | i++ 164 | } 165 | 166 | m.RUnlock() 167 | 168 | return keys 169 | } 170 | 171 | func (m *nodeMap) values() []*Node { 172 | m.RLock() 173 | 174 | values := make([]*Node, len(m.nodes)) 175 | 176 | i := 0 177 | for _, v := range m.nodes { 178 | values[i] = v 179 | i++ 180 | } 181 | 182 | m.RUnlock() 183 | 184 | return values 185 | } 186 | -------------------------------------------------------------------------------- /nodeStatus.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | // NodeStatus represents the believed status of a member node. 20 | type NodeStatus byte 21 | 22 | const ( 23 | // StatusUnknown is the default node status of newly-created nodes. 24 | StatusUnknown NodeStatus = iota 25 | 26 | // StatusAlive indicates that a node is alive and healthy. 27 | StatusAlive 28 | 29 | // StatusSuspected indicatates that a node is suspected of being dead. 30 | StatusSuspected 31 | 32 | // StatusDead indicatates that a node is dead and no longer healthy. 33 | StatusDead 34 | 35 | // StatusForwardTo is a pseudo status used by message to indicate 36 | // the target of a ping request. 37 | StatusForwardTo 38 | ) 39 | 40 | func (s NodeStatus) String() string { 41 | switch s { 42 | case StatusUnknown: 43 | return "UNKNOWN" 44 | case StatusAlive: 45 | return "ALIVE" 46 | case StatusDead: 47 | return "DEAD" 48 | case StatusSuspected: 49 | return "SUSPECTED" 50 | case StatusForwardTo: 51 | return "FORWARD_TO" 52 | default: 53 | return "UNDEFINED" 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /pingData.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "math" 21 | "sync" 22 | ) 23 | 24 | type pingData struct { 25 | sync.RWMutex 26 | 27 | // The ping data. Initialized with default values by NewPingData() 28 | pings []uint32 29 | 30 | // The index in pings where the next datapoint will be added 31 | pointer int 32 | 33 | // The last calulcated mean. Recalculated if updated is true 34 | lastMean float64 35 | 36 | // The last calulcated standard deviation. Recalculated if updated is true 37 | lastStddev float64 38 | 39 | // The modified flag. Set to true when a datapoint is added 40 | updated bool 41 | } 42 | 43 | func newPingData(initialAverage int, historyCount int) pingData { 44 | newPings := make([]uint32, historyCount, historyCount) 45 | 46 | for i := 0; i < historyCount; i++ { 47 | newPings[i] = uint32(initialAverage) 48 | } 49 | 50 | return pingData{pings: newPings, updated: true} 51 | } 52 | 53 | func (pd *pingData) add(datapoint uint32) { 54 | pd.Lock() 55 | 56 | pd.pings[pd.pointer] = datapoint 57 | 58 | // Advance the pointer 59 | pd.pointer++ 60 | pd.pointer %= len(pd.pings) 61 | 62 | pd.updated = true 63 | 64 | pd.Unlock() 65 | } 66 | 67 | // mean returns the simple mean (average) of the collected datapoints. 68 | func (pd *pingData) mean() float64 { 69 | pd.data() 70 | 71 | return pd.lastMean 72 | } 73 | 74 | // Returns the mean modified by the requested number of sigmas 75 | func (pd *pingData) nSigma(sigmas float64) float64 { 76 | mean, stddev := pd.data() 77 | 78 | return mean + (sigmas * stddev) 79 | } 80 | 81 | // stddev returns the standard deviation of the collected datapoints 82 | func (pd *pingData) stddev() float64 { 83 | pd.data() 84 | 85 | return pd.lastStddev 86 | } 87 | 88 | // Returns both mean and standard deviation 89 | func (pd *pingData) data() (float64, float64) { 90 | if pd.updated { 91 | pd.Lock() 92 | 93 | // Calculate the mean 94 | var accumulator float64 95 | for _, d := range pd.pings { 96 | accumulator += float64(d) 97 | } 98 | pd.lastMean = accumulator / float64(len(pd.pings)) 99 | 100 | // Subtract the mean and square the result; calculcate the mean 101 | accumulator = 0.0 // Reusing accumulator. 102 | for _, d := range pd.pings { 103 | diff := pd.lastMean - float64(d) 104 | accumulator += math.Pow(diff, 2.0) 105 | } 106 | squareDiffMean := accumulator / float64(len(pd.pings)) 107 | 108 | // Sqrt the square diffs mean and we have our stddev 109 | pd.lastStddev = math.Sqrt(squareDiffMean) 110 | 111 | pd.updated = false 112 | 113 | pd.Unlock() 114 | } 115 | 116 | return pd.lastMean, pd.lastStddev 117 | } 118 | -------------------------------------------------------------------------------- /properties.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "fmt" 21 | "net" 22 | "os" 23 | "regexp" 24 | "strconv" 25 | "strings" 26 | ) 27 | 28 | // Provides a series of methods and constants that revolve around the getting 29 | // (or programmatically setting/overriding) environmental properties, returning 30 | // default values if not set. 31 | 32 | const ( 33 | // EnvVarClusterName is the name of the environment variable the defines 34 | // the name of the cluster. Multicast messages from differently-named 35 | // instances are ignored. 36 | EnvVarClusterName = "SMUDGE_CLUSTER_NAME" 37 | 38 | // DefaultClusterName is the default name of the cluster for the purposes 39 | // of multicast announcements: multicast messages from differently-named 40 | // instances are ignored. 41 | DefaultClusterName string = "smudge" 42 | 43 | // EnvVarHeartbeatMillis is the name of the environment variable that 44 | // sets the heartbeat frequency (in millis). 45 | EnvVarHeartbeatMillis = "SMUDGE_HEARTBEAT_MILLIS" 46 | 47 | // DefaultHeartbeatMillis is the default heartbeat frequency (in millis). 48 | DefaultHeartbeatMillis int = 500 49 | 50 | // EnvVarInitialHosts is the name of the environment variable that sets 51 | // the initial known hosts. The value it sets should be a comma-delimitted 52 | // string of one or more IP:PORT pairs (port is optional if it matched the 53 | // value of SMUDGE_LISTEN_PORT). 54 | EnvVarInitialHosts = "SMUDGE_INITIAL_HOSTS" 55 | 56 | // DefaultInitialHosts default lists of initially known hosts. 57 | DefaultInitialHosts string = "" 58 | 59 | // EnvVarListenPort is the name of the environment variable that sets 60 | // the UDP listen port. 61 | EnvVarListenPort = "SMUDGE_LISTEN_PORT" 62 | 63 | // DefaultListenPort is the default UDP listen port. 64 | DefaultListenPort int = 9999 65 | 66 | // EnvVarListenIP is the name of the environment variable that sets 67 | // the listen IP. 68 | EnvVarListenIP = "SMUDGE_LISTEN_IP" 69 | 70 | // DefaultListenIP is the default listen IP. 71 | DefaultListenIP = "127.0.0.1" 72 | 73 | // EnvVarMaxBroadcastBytes is the name of the environment variable that 74 | // the maximum byte length for broadcast payloads. Note that increasing 75 | // this runs the risk of packet fragmentation and dropped messages. 76 | EnvVarMaxBroadcastBytes = "SMUDGE_MAX_BROADCAST_BYTES" 77 | 78 | // DefaultMaxBroadcastBytes is the default maximum byte length for 79 | // broadcast payloads. This is guided by the maximum safe UDP packet size 80 | // of 508 bytes, which must also contain status updates and additional 81 | // message overhead. 82 | DefaultMaxBroadcastBytes int = 256 83 | 84 | // EnvVarMulticastAddress is the name of the environment variable that 85 | // defines the multicast address that will be used. 86 | EnvVarMulticastAddress = "SMUDGE_MULTICAST_ADDRESS" 87 | 88 | // DefaultMulticastAddress is the default multicast address. Empty string 89 | // indicates 224.0.0.0 for IPv4 and [ff02::1] for IPv6. 90 | DefaultMulticastAddress string = "" 91 | 92 | // EnvVarMulticastEnabled is the name of the environment variable that 93 | // describes whether Smudge will attempt to announce its presence via 94 | // multicast on startup. 95 | EnvVarMulticastEnabled = "SMUDGE_MULTICAST_ENABLED" 96 | 97 | // DefaultMulticastEnabled is the default value for whether Smudge will 98 | // attempt to announce its presence via multicast on startup. 99 | DefaultMulticastEnabled string = "true" 100 | 101 | // EnvVarMulticastAnnounceIntervalSeconds is the name of the environment 102 | // variable that describes whether Smudge will attempt to re-announce its 103 | // presence via multicast every X seconds. 104 | EnvVarMulticastAnnounceIntervalSeconds = "SMUDGE_MULTICAST_ANNOUNCE_INTERVAL" 105 | 106 | // DefaultMulticastAnnounceIntervalSeconds is the default value for whether 107 | // Smudge will re-announce its presence via multicast 108 | DefaultMulticastAnnounceIntervalSeconds = 0 109 | 110 | // EnvVarMulticastPort is the name of the environment variable that 111 | // defines the multicast announcement listening port. 112 | EnvVarMulticastPort = "SMUDGE_MULTICAST_PORT" 113 | 114 | // DefaultMulticastPort is the default value for the multicast 115 | // listening port. 116 | DefaultMulticastPort int = 9998 117 | 118 | // EnvVarPingHistoryFrontload is the name of the environment variable that 119 | // defines the value (in milliseconds) used to pre-populate the ping 120 | // history buffer, which is used to dynamically calculate ping timeouts and 121 | // is gradually overwritten with real data over time. 122 | EnvVarPingHistoryFrontload = "SMUDGE_PING_HISTORY_FRONTLOAD" 123 | 124 | // DefaultPingHistoryFrontload is the default value (in milliseconds) used 125 | // to pre-populate the ping history buffer, which is used to dynamically 126 | // calculate ping timeouts and is gradually overwritten with real data 127 | // over time. 128 | DefaultPingHistoryFrontload = 200 129 | 130 | // EnvVarMinPingTime is the name of the environment variable that 131 | // defines the lower bound on recorded ping response times (in 132 | // milliseconds). This prevents the system instability and flapping that 133 | // can come from consistently small values. 134 | EnvVarMinPingTime = "SMUDGE_MIN_PING_TIME" 135 | 136 | // DefaultMinPingTime is default lower bound on recorded ping response 137 | // times (in milliseconds). This prevents the system instability and 138 | // flapping that can come from consistently small values. 139 | DefaultMinPingTime = 150 140 | ) 141 | 142 | var clusterName string 143 | 144 | var heartbeatMillis int 145 | 146 | var listenPort int 147 | 148 | var listenIP net.IP 149 | 150 | var initialHosts []string 151 | 152 | var maxBroadcastBytes int 153 | 154 | var minPingTime int 155 | 156 | var multicastEnabledString string 157 | 158 | var multicastEnabled = true 159 | 160 | var multicastAnnounceIntervalSeconds = 10 161 | 162 | var multicastPort int 163 | 164 | var multicastAddress string 165 | 166 | var pingHistoryFrontload int 167 | 168 | const stringListDelimitRegex = "\\s*((,\\s*)|(\\s+))" 169 | 170 | // GetClusterName gets the name of the cluster for the purposes of 171 | // multicast announcements: multicast messages from differently-named 172 | // instances are ignored. 173 | func GetClusterName() string { 174 | if clusterName == "" { 175 | clusterName = getStringVar(EnvVarClusterName, DefaultClusterName) 176 | } 177 | 178 | return clusterName 179 | } 180 | 181 | // GetHeartbeatMillis gets this host's heartbeat frequency in milliseconds. 182 | func GetHeartbeatMillis() int { 183 | if heartbeatMillis == 0 { 184 | heartbeatMillis = getIntVar(EnvVarHeartbeatMillis, DefaultHeartbeatMillis) 185 | } 186 | 187 | return heartbeatMillis 188 | } 189 | 190 | // GetInitialHosts returns the list of initially known hosts. 191 | func GetInitialHosts() []string { 192 | if initialHosts == nil { 193 | initialHosts = getStringArrayVar(EnvVarInitialHosts, DefaultInitialHosts) 194 | } 195 | 196 | return initialHosts 197 | } 198 | 199 | // GetListenPort returns the port that this host will listen on. 200 | func GetListenPort() int { 201 | if listenPort == 0 { 202 | listenPort = getIntVar(EnvVarListenPort, DefaultListenPort) 203 | } 204 | 205 | return listenPort 206 | } 207 | 208 | // GetListenIP returns the IP that this host will listen on. 209 | func GetListenIP() net.IP { 210 | if listenIP == nil { 211 | listenIP = net.ParseIP(getStringVar(EnvVarListenIP, DefaultListenIP)) 212 | } 213 | 214 | return listenIP 215 | } 216 | 217 | // GetMaxBroadcastBytes returns the maximum byte length for broadcast payloads. 218 | func GetMaxBroadcastBytes() int { 219 | if maxBroadcastBytes == 0 { 220 | maxBroadcastBytes = getIntVar(EnvVarMaxBroadcastBytes, DefaultMaxBroadcastBytes) 221 | } 222 | 223 | return maxBroadcastBytes 224 | } 225 | 226 | // GetMinPingTime returns the minimum ping response time in milliseconds. Ping 227 | // response times below this value are recorded as this minimum. 228 | func GetMinPingTime() int { 229 | if minPingTime == 0 { 230 | minPingTime = getIntVar(EnvVarMinPingTime, DefaultMinPingTime) 231 | } 232 | 233 | return minPingTime 234 | } 235 | 236 | // GetMulticastEnabled returns whether multicast announcements are enabled. 237 | func GetMulticastEnabled() bool { 238 | if multicastEnabledString == "" { 239 | multicastEnabledString = strings.ToLower(getStringVar(EnvVarMulticastEnabled, DefaultMulticastEnabled)) 240 | } 241 | 242 | multicastEnabled = len(multicastEnabledString) > 0 && []rune(multicastEnabledString)[0] == 't' 243 | return multicastEnabled 244 | } 245 | 246 | // GetMulticastAnnounceIntervalSeconds returns the amount of seconds to wait between 247 | // multicast announcements. 248 | func GetMulticastAnnounceIntervalSeconds() int { 249 | if multicastAnnounceIntervalSeconds == 0 { 250 | multicastAnnounceIntervalSeconds = getIntVar(EnvVarMulticastAnnounceIntervalSeconds, DefaultMulticastAnnounceIntervalSeconds) 251 | } 252 | return multicastAnnounceIntervalSeconds 253 | } 254 | 255 | // GetMulticastAddress returns the address the will be used for multicast 256 | // announcements. 257 | func GetMulticastAddress() string { 258 | if multicastAddress == "" { 259 | multicastAddress = getStringVar(EnvVarMulticastAddress, DefaultMulticastAddress) 260 | } 261 | 262 | return multicastAddress 263 | } 264 | 265 | // GetMulticastPort returns the defined multicast announcement listening port. 266 | func GetMulticastPort() int { 267 | if multicastPort == 0 { 268 | multicastPort = getIntVar(EnvVarMulticastPort, DefaultMulticastPort) 269 | } 270 | 271 | return multicastPort 272 | } 273 | 274 | // GetPingHistoryFrontload returns the value (in milliseconds) used to 275 | // pre-populate the ping history buffer, which is used to dynamically calculate 276 | // ping timeouts and is gradually overwritten with real data over time. 277 | func GetPingHistoryFrontload() int { 278 | if pingHistoryFrontload == 0 { 279 | pingHistoryFrontload = getIntVar(EnvVarPingHistoryFrontload, DefaultPingHistoryFrontload) 280 | } 281 | 282 | return pingHistoryFrontload 283 | } 284 | 285 | // SetClusterName sets the name of the cluster for the purposes of multicast 286 | // announcements: multicast messages from differently-named instances are 287 | // ignored. 288 | func SetClusterName(val string) { 289 | if val == "" { 290 | clusterName = DefaultClusterName 291 | } else { 292 | clusterName = val 293 | } 294 | } 295 | 296 | // SetHeartbeatMillis sets this nodes heartbeat frequency. Unlike 297 | // SetListenPort(), calling this function after Begin() has been called will 298 | // have an effect. 299 | func SetHeartbeatMillis(val int) { 300 | if val == 0 { 301 | heartbeatMillis = DefaultHeartbeatMillis 302 | } else { 303 | heartbeatMillis = val 304 | } 305 | } 306 | 307 | // SetListenPort sets the UDP port to listen on. It has no effect once 308 | // Begin() has been called. 309 | func SetListenPort(val int) { 310 | if val == 0 { 311 | listenPort = DefaultListenPort 312 | } else { 313 | listenPort = val 314 | } 315 | } 316 | 317 | // SetListenIP sets the IP to listen on. It has no effect once 318 | // Begin() has been called. 319 | func SetListenIP(val net.IP) { 320 | if len(AllNodes()) > 0 { 321 | logWarn("Do not call SetListenIP() after nodes have been added, it may cause unexpected behavior.") 322 | } 323 | 324 | if val == nil { 325 | listenIP = net.ParseIP(DefaultListenIP) 326 | } else { 327 | listenIP = val 328 | } 329 | } 330 | 331 | // SetMaxBroadcastBytes sets the maximum byte length for broadcast payloads. 332 | // Note that increasing this beyond the default of 256 runs the risk of packet 333 | // fragmentation and dropped messages. 334 | func SetMaxBroadcastBytes(val int) { 335 | if val == 0 { 336 | maxBroadcastBytes = DefaultMaxBroadcastBytes 337 | } else { 338 | maxBroadcastBytes = val 339 | } 340 | } 341 | 342 | // SetMinPingTime sets the minimum ping response time in milliseconds. Ping 343 | // response times below this value are recorded as this minimum. 344 | func SetMinPingTime(val int) { 345 | if val == 0 { 346 | minPingTime = DefaultMinPingTime 347 | } else { 348 | minPingTime = val 349 | } 350 | } 351 | 352 | // SetMulticastAddress sets the address that will be used for multicast 353 | // announcements. 354 | func SetMulticastAddress(val string) { 355 | if val == "" { 356 | multicastAddress = DefaultMulticastAddress 357 | } else { 358 | multicastAddress = val 359 | } 360 | } 361 | 362 | // SetMulticastEnabled sets whether multicast announcements are enabled. 363 | func SetMulticastEnabled(val bool) { 364 | multicastEnabledString = fmt.Sprintf("%v", val) 365 | } 366 | 367 | // SetMulticastAnnounceIntervalSeconds sets the number of seconds between multicast announcements 368 | func SetMulticastAnnounceIntervalSeconds(val int) { 369 | multicastAnnounceIntervalSeconds = val 370 | } 371 | 372 | // SetMulticastPort sets multicast announcement listening port. 373 | func SetMulticastPort(val int) { 374 | if val == 0 { 375 | multicastPort = DefaultMulticastPort 376 | } else { 377 | multicastPort = val 378 | } 379 | } 380 | 381 | // SetPingHistoryFrontload sets the value (in milliseconds) used to 382 | // pre-populate the ping history buffer, which is used to dynamically calculate 383 | // ping timeouts and is gradually overwritten with real data over time. 384 | // Setting this to 0 will restore the default value. 385 | func SetPingHistoryFrontload(val int) { 386 | if val == 0 { 387 | pingHistoryFrontload = DefaultPingHistoryFrontload 388 | } else { 389 | pingHistoryFrontload = val 390 | } 391 | } 392 | 393 | // Gets an environmental variable "key". If it does not exist, "defaultVal" is 394 | // returned; if it does, it attempts to convert to an integer, returning 395 | // "defaultVal" if it fails. 396 | func getIntVar(key string, defaultVal int) int { 397 | valueString := os.Getenv(key) 398 | valueInt := defaultVal 399 | 400 | if valueString != "" { 401 | i, err := strconv.Atoi(valueString) 402 | 403 | if err != nil { 404 | logfWarn("Failed to parse env property %s: %s is not "+ 405 | "an integer. Using default.", key, valueString) 406 | } else { 407 | valueInt = i 408 | } 409 | } 410 | 411 | return valueInt 412 | } 413 | 414 | // Gets an environmental variable "key". If it does not exist, "defaultVal" is 415 | // returned; if it does, it attempts to convert to a string slice, returning 416 | // "defaultVal" if it fails. 417 | func getStringArrayVar(key string, defaultVal string) []string { 418 | valueString := os.Getenv(key) 419 | 420 | if valueString == "" { 421 | valueString = defaultVal 422 | } 423 | 424 | valueSlice := splitDelimmitedString(valueString, stringListDelimitRegex) 425 | 426 | return valueSlice 427 | } 428 | 429 | // Gets an environmental variable "key". If it does not exist, "defaultVal" is 430 | // returned; if it does, it attempts to convert to a string, returning 431 | // "defaultVal" if it fails. 432 | func getStringVar(key string, defaultVal string) string { 433 | valueString := os.Getenv(key) 434 | 435 | if valueString == "" { 436 | valueString = defaultVal 437 | } 438 | 439 | return valueString 440 | } 441 | 442 | // Splits a string on a regular expression. 443 | func splitDelimmitedString(str string, regex string) []string { 444 | var result []string 445 | 446 | str = strings.TrimSpace(str) 447 | 448 | if str != "" { 449 | reg := regexp.MustCompile(regex) 450 | indices := reg.FindAllStringIndex(str, -1) 451 | 452 | result = make([]string, len(indices)+1) 453 | 454 | lastStart := 0 455 | for i, val := range indices { 456 | result[i] = str[lastStart:val[0]] 457 | lastStart = val[1] 458 | } 459 | 460 | result[len(indices)] = str[lastStart:] 461 | 462 | // Special case of single empty string 463 | if len(result) == 1 && result[0] == "" { 464 | result = make([]string, 0, 0) 465 | } 466 | } 467 | 468 | return result 469 | } 470 | -------------------------------------------------------------------------------- /properties_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | func TestSplitString0a(t *testing.T) { 24 | str := "" 25 | split := splitDelimmitedString(str, stringListDelimitRegex) 26 | 27 | if len(split) != 0 { 28 | t.Errorf("len=%d contents=%v\n", len(split), split) 29 | } 30 | } 31 | 32 | func TestSplitString0b(t *testing.T) { 33 | str := " " 34 | split := splitDelimmitedString(str, stringListDelimitRegex) 35 | 36 | if len(split) != 0 { 37 | t.Errorf("len=%d contents=%v\n", len(split), split) 38 | } 39 | } 40 | 41 | func TestSplitString1(t *testing.T) { 42 | str := "foo" 43 | split := splitDelimmitedString(str, stringListDelimitRegex) 44 | 45 | if len(split) != 1 || split[0] != "foo" { 46 | t.Errorf("len=%d contents=%v\n", len(split), split) 47 | } 48 | } 49 | 50 | func TestSplitString2a(t *testing.T) { 51 | str := "foo bar" 52 | split := splitDelimmitedString(str, stringListDelimitRegex) 53 | 54 | if len(split) != 2 || split[0] != "foo" { 55 | t.Errorf("len=%d contents=%v\n", len(split), split) 56 | } 57 | } 58 | 59 | func TestSplitString2b(t *testing.T) { 60 | str := "foo, bar" 61 | split := splitDelimmitedString(str, stringListDelimitRegex) 62 | 63 | if len(split) != 2 || split[0] != "foo" { 64 | t.Errorf("len=%d contents=%v\n", len(split), split) 65 | } 66 | } 67 | 68 | func TestSplitString2c(t *testing.T) { 69 | str := "foo bar" 70 | split := splitDelimmitedString(str, stringListDelimitRegex) 71 | 72 | if len(split) != 2 || split[0] != "foo" || split[1] != "bar" { 73 | t.Errorf("len=%d contents=%v\n", len(split), split) 74 | } 75 | } 76 | 77 | func TestSplitString2d(t *testing.T) { 78 | str := "localhost:10000,localhost:9999" 79 | split := splitDelimmitedString(str, stringListDelimitRegex) 80 | 81 | if len(split) != 2 || split[0] != "localhost:10000" { 82 | t.Errorf("len=%d contents=%v\n", len(split), split) 83 | } 84 | } 85 | 86 | func TestSplitString3a(t *testing.T) { 87 | str := "foo bar bat" 88 | split := splitDelimmitedString(str, stringListDelimitRegex) 89 | 90 | if len(split) != 3 || split[0] != "foo" { 91 | t.Errorf("len=%d contents=%v\n", len(split), split) 92 | } 93 | } 94 | 95 | func TestSplitString3b(t *testing.T) { 96 | str := "foo, bar, bat" 97 | split := splitDelimmitedString(str, stringListDelimitRegex) 98 | 99 | if len(split) != 3 || split[0] != "foo" { 100 | t.Errorf("len=%d contents=%v\n", len(split), split) 101 | } 102 | } 103 | 104 | func TestSplitString3c(t *testing.T) { 105 | str := "foo bar, bat" 106 | split := splitDelimmitedString(str, stringListDelimitRegex) 107 | 108 | if len(split) != 3 || split[0] != "foo" { 109 | t.Errorf("len=%d contents=%v\n", len(split), split) 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /registry.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "fmt" 21 | "net" 22 | "sort" 23 | "strconv" 24 | "sync" 25 | ) 26 | 27 | // All known nodes, living and dead. Dead nodes are pinged (far) less often, 28 | // and are eventually removed 29 | var knownNodes = nodeMap{} 30 | 31 | // All nodes that have been updated "recently", living and dead 32 | var updatedNodes = nodeMap{} 33 | 34 | var deadNodeRetries = struct { 35 | sync.RWMutex 36 | m map[string]*deadNodeCounter 37 | }{m: make(map[string]*deadNodeCounter)} 38 | 39 | const maxDeadNodeRetries = 10 40 | 41 | func init() { 42 | knownNodes.init() 43 | updatedNodes.init() 44 | } 45 | 46 | /****************************************************************************** 47 | * Exported functions (for public consumption) 48 | *****************************************************************************/ 49 | 50 | // AddNode can be used to explicitly add a node to the list of known live 51 | // nodes. Updates the node timestamp but DOES NOT implicitly update the node's 52 | // status; you need to do this explicitly. 53 | func AddNode(node *Node) (*Node, error) { 54 | if !knownNodes.contains(node) { 55 | if node.status == StatusUnknown { 56 | logWarn(node.Address(), 57 | "does not have a status! Setting to", 58 | StatusAlive) 59 | 60 | UpdateNodeStatus(node, StatusAlive, thisHost) 61 | } else if node.status == StatusForwardTo { 62 | panic("invalid status: " + StatusForwardTo.String()) 63 | } 64 | 65 | node.Touch() 66 | 67 | _, n, err := knownNodes.add(node) 68 | 69 | logfInfo("Adding host: %s (total=%d live=%d dead=%d)", 70 | node.Address(), 71 | knownNodes.length(), 72 | knownNodes.lengthWithStatus(StatusAlive), 73 | knownNodes.lengthWithStatus(StatusDead)) 74 | 75 | knownNodesModifiedFlag = true 76 | 77 | return n, err 78 | } 79 | 80 | return node, nil 81 | } 82 | 83 | // CreateNodeByAddress will create and return a new node when supplied with a 84 | // node address ("ip:port" string). This doesn't add the node to the list of 85 | // live nodes; use AddNode(). 86 | func CreateNodeByAddress(address string) (*Node, error) { 87 | ip, port, err := parseNodeAddress(address) 88 | 89 | if err == nil { 90 | return CreateNodeByIP(ip, port) 91 | } 92 | 93 | return nil, err 94 | } 95 | 96 | // CreateNodeByIP will create and return a new node when supplied with an 97 | // IP address and port number. This doesn't add the node to the list of live 98 | // nodes; use AddNode(). 99 | func CreateNodeByIP(ip net.IP, port uint16) (*Node, error) { 100 | node := Node{ 101 | ip: ip, 102 | port: port, 103 | timestamp: GetNowInMillis(), 104 | pingMillis: PingNoData, 105 | } 106 | 107 | return &node, nil 108 | } 109 | 110 | // GetLocalIP queries the host interface to determine the local IP address of this 111 | // machine. If a local IP address cannot be found, then nil is returned. Local IPv6 112 | // address takes presedence over a local IPv4 address. If the query to the underlying 113 | // OS fails, an error is returned. 114 | func GetLocalIP() (net.IP, error) { 115 | var ip net.IP 116 | 117 | addrs, err := net.InterfaceAddrs() 118 | if err != nil { 119 | return ip, err 120 | } 121 | 122 | for _, a := range addrs { 123 | if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { 124 | if ipnet.IP.To4() != nil { 125 | ip = ipnet.IP.To4() 126 | } else { 127 | ip = ipnet.IP 128 | break 129 | } 130 | } 131 | } 132 | 133 | return ip, nil 134 | } 135 | 136 | // AllNodes will return a list of all nodes known at the time of the request, 137 | // including nodes that have been marked as "dead" but haven't yet been 138 | // removed from the registry. 139 | func AllNodes() []*Node { 140 | return knownNodes.values() 141 | } 142 | 143 | // HealthyNodes will return a list of all nodes known at the time of the 144 | // request with a healthy status. 145 | func HealthyNodes() []*Node { 146 | values := knownNodes.values() 147 | filtered := make([]*Node, 0, len(values)) 148 | 149 | for _, v := range values { 150 | if v.Status() == StatusAlive { 151 | filtered = append(filtered, v) 152 | } 153 | } 154 | 155 | return filtered 156 | } 157 | 158 | // RemoveNode can be used to explicitly remove a node from the list of known 159 | // live nodes. Updates the node timestamp but DOES NOT implicitly update the 160 | // node's status; you need to do this explicitly. 161 | func RemoveNode(node *Node) (*Node, error) { 162 | if knownNodes.contains(node) { 163 | node.Touch() 164 | 165 | _, n, err := knownNodes.delete(node) 166 | 167 | logfInfo("Removing host: %s (total=%d live=%d dead=%d)", 168 | node.Address(), 169 | knownNodes.length(), 170 | knownNodes.lengthWithStatus(StatusAlive), 171 | knownNodes.lengthWithStatus(StatusDead)) 172 | 173 | knownNodesModifiedFlag = true 174 | 175 | return n, err 176 | } 177 | 178 | return node, nil 179 | } 180 | 181 | // UpdateNodeStatus assigns a new status for the specified node and adds it to 182 | // the list of recently updated nodes. If the status is StatusDead, then the 183 | // node will be moved from the live nodes list to the dead nodes list. 184 | func UpdateNodeStatus(node *Node, status NodeStatus, statusSource *Node) { 185 | updateNodeStatus(node, status, node.heartbeat, statusSource) 186 | } 187 | 188 | /****************************************************************************** 189 | * Private functions (for internal use only) 190 | *****************************************************************************/ 191 | 192 | func getRandomUpdatedNodes(size int, exclude ...*Node) []*Node { 193 | updatedNodesCopy := nodeMap{} 194 | updatedNodesCopy.init() 195 | 196 | // Prune nodes with emit counters of 0 (or less) from the map. Any 197 | // others we copy into a secondary nodemap. 198 | for _, n := range updatedNodes.values() { 199 | if n.emitCounter <= 0 { 200 | logDebug("Removing", n.Address(), "from recently updated list") 201 | updatedNodes.delete(n) 202 | } else { 203 | updatedNodesCopy.add(n) 204 | } 205 | } 206 | 207 | // Exclude the exclusions 208 | for _, ex := range exclude { 209 | updatedNodesCopy.delete(ex) 210 | } 211 | 212 | // Put the newest nodes on top. 213 | updatedNodesSlice := updatedNodesCopy.values() 214 | sort.Sort(byNodeEmitCounter(updatedNodesSlice)) 215 | 216 | // Grab and return the top N 217 | if size > len(updatedNodesSlice) { 218 | size = len(updatedNodesSlice) 219 | } 220 | 221 | return updatedNodesSlice[:size] 222 | } 223 | 224 | func parseNodeAddress(hostAndMaybePort string) (net.IP, uint16, error) { 225 | var host string 226 | var ip net.IP 227 | var port uint16 228 | var err error 229 | 230 | ip = net.ParseIP(hostAndMaybePort) 231 | port = uint16(GetListenPort()) 232 | 233 | host, sport, err := net.SplitHostPort(hostAndMaybePort) 234 | 235 | if err == nil { 236 | if sport != "" { 237 | p, e := strconv.ParseUint(sport, 10, 16) 238 | port = uint16(p) 239 | err = e 240 | } 241 | 242 | ip = net.ParseIP(host) 243 | } else { 244 | err = nil 245 | ip = net.ParseIP(hostAndMaybePort) 246 | port = uint16(GetListenPort()) 247 | 248 | if host == "" { 249 | host = hostAndMaybePort 250 | } 251 | } 252 | 253 | if ip == nil { 254 | ips, err := net.LookupIP(host) 255 | if err != nil { 256 | return ip, port, err 257 | } 258 | 259 | for _, i := range ips { 260 | if !i.IsLoopback() { 261 | if GetListenIP().To4() != nil && i.To4() != nil { 262 | ip = i 263 | break 264 | } else if GetListenIP().To4() == nil && i.To4() == nil { 265 | ip = i 266 | break 267 | } 268 | } 269 | } 270 | 271 | if ip == nil { 272 | err = fmt.Errorf("Could not parse the address of node %s", hostAndMaybePort) 273 | } 274 | } 275 | 276 | return ip, port, err 277 | } 278 | 279 | // UpdateNodeStatus assigns a new status for the specified node and adds it to 280 | // the list of recently updated nodes. If the status is StatusDead, then the 281 | // node will be moved from the live nodes list to the dead nodes list. 282 | func updateNodeStatus(node *Node, status NodeStatus, heartbeat uint32, statusSource *Node) { 283 | if node.status != status { 284 | if heartbeat < node.heartbeat { 285 | logfWarn("Decreasing known node heartbeat value from %d to %d", 286 | node.heartbeat, 287 | heartbeat) 288 | } 289 | 290 | node.timestamp = GetNowInMillis() 291 | node.status = status 292 | node.statusSource = statusSource 293 | node.emitCounter = int8(emitCount()) 294 | node.heartbeat = heartbeat 295 | 296 | // If this isn't in the recently updated list, add it. 297 | if !updatedNodes.contains(node) { 298 | updatedNodes.add(node) 299 | } 300 | 301 | if status != StatusDead { 302 | deadNodeRetries.Lock() 303 | delete(deadNodeRetries.m, node.Address()) 304 | deadNodeRetries.Unlock() 305 | } 306 | 307 | logfInfo("Updating host: %s to %s (total=%d live=%d dead=%d)", 308 | node.Address(), 309 | status, 310 | knownNodes.length(), 311 | knownNodes.lengthWithStatus(StatusAlive), 312 | knownNodes.lengthWithStatus(StatusDead)) 313 | 314 | doStatusUpdate(node, status) 315 | } 316 | } 317 | 318 | type deadNodeCounter struct { 319 | retry int 320 | retryCountdown int 321 | } 322 | 323 | // byNodeEmitCounter implements sort.Interface for []*Node based on 324 | // the emitCounter field. 325 | type byNodeEmitCounter []*Node 326 | 327 | func (a byNodeEmitCounter) Len() int { 328 | return len(a) 329 | } 330 | 331 | func (a byNodeEmitCounter) Swap(i, j int) { 332 | a[i], a[j] = a[j], a[i] 333 | } 334 | 335 | func (a byNodeEmitCounter) Less(i, j int) bool { 336 | return a[i].emitCounter > a[j].emitCounter 337 | } 338 | -------------------------------------------------------------------------------- /registry_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package smudge 18 | 19 | import ( 20 | "fmt" 21 | "net" 22 | "testing" 23 | 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | func TestAddNode(t *testing.T) { 28 | // Start empty 29 | require.Empty(t, knownNodes.nodes) 30 | 31 | n := testNode() 32 | n.status = StatusUnknown 33 | 34 | // Adding a nodes with an unknown status sets the status to ALIVE 35 | AddNode(n) 36 | require.Equal(t, StatusAlive, n.status) 37 | 38 | key := fmt.Sprintf("%s:%d", n.ip.String(), n.port) 39 | 40 | // Also, it adds exactly one node: this one. 41 | require.Equal(t, 1, len(knownNodes.values())) 42 | require.NotNil(t, knownNodes.nodes[key]) 43 | require.Equal(t, n, knownNodes.nodes[key]) 44 | 45 | // Adding again is a no-op 46 | AddNode(n) 47 | require.Equal(t, 1, len(knownNodes.values())) 48 | 49 | t.Log(n.status) 50 | } 51 | 52 | func TestCreateNodeByAddress(t *testing.T) { 53 | s := "127.0.0.1:1234" 54 | 55 | expected := &Node{ 56 | ip: net.IPv4(127, 0, 0, 1), 57 | port: 1234, 58 | } 59 | 60 | node, err := CreateNodeByAddress(s) 61 | require.Nil(t, err) 62 | require.Equal(t, expected.ip, node.ip) 63 | require.Equal(t, expected.port, node.port) 64 | } 65 | 66 | func TestIPv4(t *testing.T) { 67 | s := "127.0.0.1" 68 | ip, port, err := parseNodeAddress(s) 69 | 70 | if err != nil { 71 | t.Error("Error should be nil but was:", err) 72 | } 73 | 74 | if ip.String() != s { 75 | t.Errorf("Expected %s but found %s\n", s, ip.String()) 76 | } 77 | 78 | if port != 9999 { 79 | t.Errorf("Expected the port to be 9999 but found %d\n", port) 80 | } 81 | } 82 | 83 | func TestIPv4WithPort(t *testing.T) { 84 | s := "127.0.0.1:80" 85 | ip, port, err := parseNodeAddress(s) 86 | 87 | if err != nil { 88 | t.Error("Error should be nil but was:", err) 89 | } 90 | 91 | if ip.String() != "127.0.0.1" { 92 | t.Errorf("Expected %s but found %s\n", s, ip.String()) 93 | } 94 | 95 | if port != 80 { 96 | t.Errorf("Expected the port to be 80 but found %d\n", port) 97 | } 98 | } 99 | 100 | func TestIPv6(t *testing.T) { 101 | s := "fd02:6b8:b010:9020:1::2" 102 | ip, port, err := parseNodeAddress(s) 103 | 104 | if err != nil { 105 | t.Error("Error should be nil but was:", err) 106 | } 107 | 108 | if ip.String() != s { 109 | t.Errorf("Expected %s but found %s\n", s, ip.String()) 110 | } 111 | 112 | if port != 9999 { 113 | t.Errorf("Expected the port to be 9999 but found %d\n", port) 114 | } 115 | } 116 | 117 | func TestIPv6WithPort(t *testing.T) { 118 | s := "[fd02:6b8:b010:9020:1::2]:80" 119 | ip, port, err := parseNodeAddress(s) 120 | 121 | if err != nil { 122 | t.Error("Error should be nil but was:", err) 123 | } 124 | 125 | if ip.String() != "fd02:6b8:b010:9020:1::2" { 126 | t.Errorf("Expected fd02:6b8:b010:9020:1::2 but found %s\n", ip.String()) 127 | } 128 | 129 | if port != 80 { 130 | t.Errorf("Expected the port to be 80 but found %d\n", port) 131 | } 132 | } 133 | 134 | //func TestHostname(t *testing.T) { 135 | // s := "localhost" 136 | // ip, port, err := parseNodeAddress(s) 137 | // 138 | // if err != nil { 139 | // t.Error("Error should be nil but was:", err) 140 | // } 141 | // 142 | // if ip.String() != "127.0.0.1" { 143 | // t.Errorf("Expected %s but found %s\n", s, ip.String()) 144 | // } 145 | // 146 | // if port != 9999 { 147 | // t.Errorf("Expected the port to be 9999 but found %d\n", port) 148 | // } 149 | //} 150 | // 151 | //func TestHostnameWithPort(t *testing.T) { 152 | // s := "localhost:80" 153 | // ip, port, err := parseNodeAddress(s) 154 | // 155 | // if err != nil { 156 | // t.Error("Error should be nil but was:", err) 157 | // } 158 | // 159 | // if ip.String() != "127.0.0.1" { 160 | // t.Errorf("Expected %s but found %s\n", s, ip.String()) 161 | // } 162 | // 163 | // if port != 80 { 164 | // t.Errorf("Expected the port to be 80 but found %d\n", port) 165 | // } 166 | //} 167 | // 168 | //func TestHostnameErr(t *testing.T) { 169 | // SetListenIP(net.ParseIP("fd02:6b8:b010:9020:1::2")) 170 | // 171 | // s := "localhost" 172 | // _, _, err := parseNodeAddress(s) 173 | // 174 | // if err == nil { 175 | // t.Error("Error should not have been be nil") 176 | // } 177 | // 178 | // SetListenIP(net.ParseIP("127.0.0.1")) 179 | //} 180 | -------------------------------------------------------------------------------- /smudge/README.md: -------------------------------------------------------------------------------- 1 | This directory is contains a simple CLI tool used to test Smudge's member discovery and status dissemination functionality. 2 | -------------------------------------------------------------------------------- /smudge/smudge.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Smudge Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "fmt" 22 | "github.com/clockworksoul/smudge" 23 | "log" 24 | ) 25 | 26 | func main() { 27 | var nodeAddress string 28 | var heartbeatMillis int 29 | var listenPort int 30 | var err error 31 | 32 | flag.StringVar(&nodeAddress, "node", "", "Initial node") 33 | 34 | flag.IntVar(&listenPort, "port", 35 | int(smudge.GetListenPort()), 36 | "The bind port") 37 | 38 | flag.IntVar(&heartbeatMillis, "hbf", 39 | int(smudge.GetHeartbeatMillis()), 40 | "The heartbeat frequency in milliseconds") 41 | 42 | flag.Parse() 43 | 44 | ip, err := smudge.GetLocalIP() 45 | if err != nil { 46 | log.Fatal("Could not get local ip:", err) 47 | } 48 | 49 | smudge.SetLogThreshold(smudge.LogInfo) 50 | smudge.SetListenPort(listenPort) 51 | smudge.SetHeartbeatMillis(heartbeatMillis) 52 | smudge.SetListenIP(ip) 53 | 54 | if ip.To4() == nil { 55 | smudge.SetMaxBroadcastBytes(512) // 512 for IPv6 56 | } 57 | 58 | if nodeAddress != "" { 59 | node, err := smudge.CreateNodeByAddress(nodeAddress) 60 | 61 | if err == nil { 62 | smudge.AddNode(node) 63 | } else { 64 | fmt.Println(err) 65 | } 66 | } 67 | 68 | if err == nil { 69 | smudge.Begin() 70 | } else { 71 | fmt.Println(err) 72 | } 73 | } 74 | --------------------------------------------------------------------------------