├── .gitignore ├── .travis.yml ├── Cargo.toml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── Vagrantfile ├── setup.sh ├── simulation ├── .gitignore ├── Cargo.toml └── src │ ├── lib.rs │ └── net.rs ├── src ├── broadcast.rs ├── lib.rs ├── protocol.rs ├── result.rs ├── state.rs ├── stream.rs └── tag.rs └── tests ├── 3-node-cluster.rs ├── check-style.sh ├── compile.sh ├── mod.rs ├── proxy.rs └── simulations └── mod.rs /.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant 2 | # Compiled Object files 3 | *.slo 4 | *.lo 5 | *.o 6 | *.obj 7 | 8 | # Compiled Dynamic libraries 9 | *.so 10 | *.dylib 11 | *.dll 12 | 13 | # Compiled Static libraries 14 | *.lai 15 | *.la 16 | *.a 17 | *.lib 18 | 19 | # Executables 20 | *.exe 21 | *.out 22 | *.app 23 | *.rlib 24 | target 25 | Cargo.lock 26 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | 3 | script: make test 4 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | 3 | name = "gossip" 4 | version = "0.1.0" 5 | authors = [ "Daniel Fagnan " ] 6 | 7 | [dependencies.uuid] 8 | git = "https://github.com/rust-lang/uuid" 9 | 10 | [dependencies.msgpack] 11 | git = "https://github.com/thehydroimpulse/rust-msgpack" 12 | 13 | [dependencies.simulation] 14 | path = "simulation" 15 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rail44/rust 2 | MAINTAINER Daniel Fagnan 3 | 4 | RUN apt-get install wget 5 | RUN mkdir -p /usr/local/src 6 | RUN cd /usr/local/src && \ 7 | wget -O gossiper.tar.gz https://github.com/thehydroimpulse/gossiper/archive/master.tar.gz && \ 8 | tar zxvf gossiper.tar.gz && \ 9 | mv gossiper-master gossiper && \ 10 | cd gossiper 11 | 12 | RUN cd /usr/local/src/gossiper && cargo build 13 | CMD ./usr/local/src/gossiper/target/network 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Daniel Fagnan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | RUSTC ?= rustc 2 | RUSTC_FLAGS ?= 3 | CARGO ?= cargo 4 | 5 | SRC = $(shell find src -name '*.rs') 6 | 7 | all: libgossip 8 | 9 | libgossip: $(SRC) 10 | $(CARGO) build 11 | 12 | sh: tests/compile.sh 13 | chmod +x tests/compile.sh 14 | 15 | test: sh 16 | sh ./tests/check-style.sh 17 | ./tests/compile.sh 18 | $(CARGO) test 19 | 20 | clean: 21 | @rm -rf target 22 | 23 | .PHONY: clean test 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Gossiper 2 | 3 | [![Build Status](https://travis-ci.org/thehydroimpulse/gossiper.svg?branch=master)](https://travis-ci.org/thehydroimpulse/gossiper) [![Stories in Ready](https://badge.waffle.io/thehydroimpulse/gossip.rs.png?label=ready&title=Ready)](https://waffle.io/thehydroimpulse/gossip.rs) 4 | 5 | **Note**: This is a work-in-progress. It's not yet usable **at all!**. 6 | 7 | Gossip protocol written in Rust. 8 | 9 | ## Installing Gossip 10 | 11 | Gossip is a Cargo package. You can simply include Gossip as a dependency. 12 | 13 | ```toml 14 | # Cargo.toml 15 | [dependencies.gossip] 16 | git = "https://github.com/thehydroimpulse/gossip.rs" 17 | ``` 18 | 19 | ## Getting Started 20 | 21 | After adding Gossip as a dependency, you'll want to include the crate within your project: 22 | 23 | ```rust 24 | extern crate gossip; 25 | ``` 26 | 27 | Now you'll have access to the Gossip system. We'll simply start with a brief example 28 | of creating a single server that listens on a given address. By default, this actually 29 | doesn't include any transport mechanism, so it's purely in-memory. A TCP transport 30 | is shipped with Gossip which we'll get to later on. 31 | 32 | ```rust 33 | use gossip::{Node}; 34 | 35 | fn main() { 36 | // Create a new Node. 37 | let mut node = Node::new(); 38 | 39 | // Bind on a specific host/port. 40 | node.listen("localhost", 5999).unwrap(); 41 | 42 | // Join an existing cluster, given a peer. 43 | node.peer("localhost", 4555).unwrap(); 44 | 45 | // Loop through any broadcasts that we receive. 46 | for (broadcast, mut res) in node.incoming() { 47 | // ... 48 | // Send an OK broadcast back, acknowledging the initial 49 | // broadcast. 50 | res.ok(); 51 | } 52 | } 53 | ``` 54 | 55 | ## What's A Gossip Protocol? 56 | 57 | Wikipedia defines it as: 58 | 59 | > A gossip protocol is a style of computer-to-computer communication protocol inspired by the form of gossip seen in social networks. Modern distributed systems often use gossip protocols to solve problems that might be difficult to solve in other ways, either because the underlying network has an inconvenient structure, is extremely large, or because gossip solutions are the most efficient ones available. 60 | 61 | The concept goes like this: 62 | 63 | > You walk into work one morning and Sam (fictional) approaches you. He tells you a secret about Billy. Excited about knowing Billy's secret, you run over to the break room to tell John. At the same time, Sam, the one who first told you, also goes and tells Jimmy. In the afternoon, all of you get together in the meeting room discussing this secret. Then, Amy, who doesn't know it yet, walks in and everybody starts telling her. At first, nobody knows if she knows the secret, so you asked, in which she replied "No?" 64 | 65 | That's the basic workflow for gossip protocols, except, we're talking about computers and networks. This is how a network of computers can communicate without having a leader/master node. There are obvious trade-offs here. By achieving the no-leader semantics, you effectively have no control on how effective messages are getting across the network and who hasn't received them. That's the lack of consistency, yet you gain high-availability. It doesn't matter if nodes go down, there aren't any leaders, which means no quorum needs to be held, and no election processes need to happen. On top of that, any node is able to accept requests for information (i.e database queries). 66 | 67 | For many systems and tasks, this isn't desireable. There are situations where having a consistent cluster is much simpler and more effective. 68 | 69 | ## Why Rust? 70 | 71 | [Rust](http://www.rust-lang.org/) is Mozilla's new systems programming language that focuses on safety, concurrency and practicality. It doesn't have garbage collection — focusing on safety without sacrificing performance. 72 | 73 | While Rust is aimed at being a systems programming language, it has powerful features allowing a high-level of abstractions. With memory safety at it's core, you can have more guarantees about software. 74 | 75 | ## Papers / Research 76 | 77 | * **[Epidemic Broadcast Trees](http://www.gsd.inesc-id.pt/~jleitao/pdf/srds07-leitao.pdf)** (Original Plumtree design) 78 | * [Cassandra's Gossip Protocol](http://www.datastax.com/docs/0.8/cluster_architecture/gossip) 79 | * [How robust are gossip-based communication protocols](https://www.cs.utexas.edu/users/lorenzo/papers/p14-alvisi.pdf) 80 | * [Using Gossip Protocols For Failure Detection, Monitoring, Messaging And Other Good Things](http://highscalability.com/blog/2011/11/14/using-gossip-protocols-for-failure-detection-monitoring-mess.html) 81 | * [GEMS: Gossip-Enabled Monitoring Service for Scalable Heterogeneous Distributed Systems](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.160.2604) 82 | * [A Gossip-Style Failure Detection Service](http://www.cs.cornell.edu/home/rvr/papers/GossipFD.pdf) 83 | * [Controlled Epidemics: Riak's New Gossip Protocol and Metadata Store (Jordan West)](https://www.youtube.com/watch?v=s4cCUTPU8GI) 84 | * [Spanning Tree](https://en.wikipedia.org/wiki/Spanning_tree) 85 | 86 | ## Testing 87 | 88 | ``` 89 | make test 90 | ``` 91 | 92 | ## Building Gossip.rs 93 | 94 | ``` 95 | cargo build 96 | ``` 97 | 98 | ## License 99 | 100 | The MIT License (MIT) 101 | 102 | Copyright (c) 2014 Daniel Fagnan 103 | 104 | Permission is hereby granted, free of charge, to any person obtaining a copy 105 | of this software and associated documentation files (the "Software"), to deal 106 | in the Software without restriction, including without limitation the rights 107 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 108 | copies of the Software, and to permit persons to whom the Software is 109 | furnished to do so, subject to the following conditions: 110 | 111 | The above copyright notice and this permission notice shall be included in all 112 | copies or substantial portions of the Software. 113 | 114 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 115 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 116 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 117 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 118 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 119 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 120 | SOFTWARE. 121 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! 5 | VAGRANTFILE_API_VERSION = "2" 6 | 7 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 8 | # All Vagrant configuration is done here. The most common configuration 9 | # options are documented and commented below. For a complete reference, 10 | # please see the online documentation at vagrantup.com. 11 | 12 | # Every Vagrant virtual environment requires a box to build off of. 13 | config.vm.box = "hashicorp/precise32" 14 | 15 | config.vm.provision "shell", path: "setup.sh" 16 | 17 | # Disable automatic box update checking. If you disable this, then 18 | # boxes will only be checked for updates when the user runs 19 | # `vagrant box outdated`. This is not recommended. 20 | # config.vm.box_check_update = false 21 | 22 | # Create a forwarded port mapping which allows access to a specific port 23 | # within the machine from a port on the host machine. In the example below, 24 | # accessing "localhost:8080" will access port 80 on the guest machine. 25 | # config.vm.network "forwarded_port", guest: 80, host: 8080 26 | 27 | # Create a private network, which allows host-only access to the machine 28 | # using a specific IP. 29 | # config.vm.network "private_network", ip: "192.168.33.10" 30 | 31 | # Create a public network, which generally matched to bridged network. 32 | # Bridged networks make the machine appear as another physical device on 33 | # your network. 34 | # config.vm.network "public_network" 35 | 36 | # If true, then any SSH connections made will enable agent forwarding. 37 | # Default value: false 38 | # config.ssh.forward_agent = true 39 | 40 | # Share an additional folder to the guest VM. The first argument is 41 | # the path on the host to the actual folder. The second argument is 42 | # the path on the guest to mount the folder. And the optional third 43 | # argument is a set of non-required options. 44 | config.vm.synced_folder ".", "/gossiper" 45 | 46 | # Provider-specific configuration so you can fine-tune various 47 | # backing providers for Vagrant. These expose provider-specific options. 48 | # Example for VirtualBox: 49 | # 50 | # config.vm.provider "virtualbox" do |vb| 51 | # # Don't boot with headless mode 52 | # vb.gui = true 53 | # 54 | # # Use VBoxManage to customize the VM. For example to change memory: 55 | # vb.customize ["modifyvm", :id, "--memory", "1024"] 56 | # end 57 | # 58 | # View the documentation for the provider you're using for more 59 | # information on available options. 60 | 61 | # Enable provisioning with CFEngine. CFEngine Community packages are 62 | # automatically installed. For example, configure the host as a 63 | # policy server and optionally a policy file to run: 64 | # 65 | # config.vm.provision "cfengine" do |cf| 66 | # cf.am_policy_hub = true 67 | # # cf.run_file = "motd.cf" 68 | # end 69 | # 70 | # You can also configure and bootstrap a client to an existing 71 | # policy server: 72 | # 73 | # config.vm.provision "cfengine" do |cf| 74 | # cf.policy_server_address = "10.0.2.15" 75 | # end 76 | 77 | # Enable provisioning with Puppet stand alone. Puppet manifests 78 | # are contained in a directory path relative to this Vagrantfile. 79 | # You will need to create the manifests directory and a manifest in 80 | # the file default.pp in the manifests_path directory. 81 | # 82 | # config.vm.provision "puppet" do |puppet| 83 | # puppet.manifests_path = "manifests" 84 | # puppet.manifest_file = "default.pp" 85 | # end 86 | 87 | # Enable provisioning with chef solo, specifying a cookbooks path, roles 88 | # path, and data_bags path (all relative to this Vagrantfile), and adding 89 | # some recipes and/or roles. 90 | # 91 | # config.vm.provision "chef_solo" do |chef| 92 | # chef.cookbooks_path = "../my-recipes/cookbooks" 93 | # chef.roles_path = "../my-recipes/roles" 94 | # chef.data_bags_path = "../my-recipes/data_bags" 95 | # chef.add_recipe "mysql" 96 | # chef.add_role "web" 97 | # 98 | # # You may also specify custom JSON attributes: 99 | # chef.json = { mysql_password: "foo" } 100 | # end 101 | 102 | # Enable provisioning with chef server, specifying the chef server URL, 103 | # and the path to the validation key (relative to this Vagrantfile). 104 | # 105 | # The Opscode Platform uses HTTPS. Substitute your organization for 106 | # ORGNAME in the URL and validation key. 107 | # 108 | # If you have your own Chef Server, use the appropriate URL, which may be 109 | # HTTP instead of HTTPS depending on your configuration. Also change the 110 | # validation key to validation.pem. 111 | # 112 | # config.vm.provision "chef_client" do |chef| 113 | # chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME" 114 | # chef.validation_key_path = "ORGNAME-validator.pem" 115 | # end 116 | # 117 | # If you're using the Opscode platform, your validator client is 118 | # ORGNAME-validator, replacing ORGNAME with your organization name. 119 | # 120 | # If you have your own Chef Server, the default validation client name is 121 | # chef-validator, unless you changed the configuration. 122 | # 123 | # chef.validation_client_name = "ORGNAME-validator" 124 | end 125 | -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- 1 | echo "Initializing environment" 2 | echo "Installing Curl" 3 | if command -v curl >/dev/null 2>&1; then 4 | echo "Skipping curl installation" 5 | else 6 | apt-get -y install curl 7 | fi 8 | if command -v rustc >/dev/null 2>&1; then 9 | echo "Skipping Rust installation..." 10 | else 11 | echo "Installing Rust" 12 | curl -s https://static.rust-lang.org/rustup.sh | sudo sh 13 | fi 14 | -------------------------------------------------------------------------------- /simulation/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | -------------------------------------------------------------------------------- /simulation/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | 3 | name = "simulation" 4 | version = "0.0.1" 5 | authors = ["Daniel Fagnan "] 6 | -------------------------------------------------------------------------------- /simulation/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod net; 2 | 3 | -------------------------------------------------------------------------------- /simulation/src/net.rs: -------------------------------------------------------------------------------- 1 | //! Abstractions over networking. These are mostly 2 | //! just utility libraries used for simulation testing, like 3 | //! simulating a slow network, partitions, checking if a node 4 | //! is reachable, etc... 5 | //! 6 | //! This handles the testing at the network level, not the software 7 | //! level, which important to have. 8 | 9 | use std::io::process::Command; 10 | 11 | /// Slow down the network using "tc". 12 | #[cfg(target_os = "linux")] 13 | #[experimental] 14 | pub fn slow() { 15 | Command::new("tc") 16 | .arg("qdisc") 17 | .arg("add") 18 | .arg("dev") 19 | .arg("eth0") 20 | .arg("root") 21 | .arg("netem") 22 | .arg("delay") 23 | .arg("50ms") 24 | .arg("10ms") 25 | .arg("distribution") 26 | .arg("normal") 27 | .spawn(); 28 | } 29 | 30 | #[cfg(not(target_os = "linux"))] 31 | pub fn slow() { 32 | fail!("`slow` is only supported on Linux."); 33 | } 34 | 35 | /// Drop some packets, eh? 36 | #[cfg(target_os = "linux")] 37 | #[experimental] 38 | pub fn flaky() { 39 | Command::new("tc") 40 | .arg("qdisc") 41 | .arg("add") 42 | .arg("dev") 43 | .arg("eth0") 44 | .arg("root") 45 | .arg("netem") 46 | .arg("loss") 47 | .arg("20%") 48 | .arg("75%") 49 | .spawn(); 50 | } 51 | 52 | #[cfg(not(target_os = "linux"))] 53 | pub fn flaky() { 54 | fail!("`flaky` is only supported on Linux."); 55 | } 56 | 57 | /// Speed up the network to it's maximum. 58 | #[cfg(target_os = "linux")] 59 | #[experimental] 60 | pub fn fast() { 61 | Command::new("tc") 62 | .arg("qdisc") 63 | .arg("del") 64 | .arg("dev") 65 | .arg("eth0") 66 | .arg("root") 67 | .spawn(); 68 | } 69 | 70 | #[cfg(not(target_os = "linux"))] 71 | pub fn fast() { 72 | fail!("`fast` is only supported on Linux."); 73 | } 74 | 75 | #[experimental] 76 | pub fn is_reachable(ip: &str, port: u16) -> Result<(), &'static str> { 77 | match Command::new("ping") 78 | .arg("-w") 79 | .arg("1") 80 | .arg(format!("{}:{}", ip, port)) 81 | .spawn() { 82 | Ok(_) => Ok(()), 83 | Err(err) => Err("Node is unreachable") 84 | } 85 | } 86 | 87 | /// Use `iptables` to create a new network partition around 88 | /// a number of nodes in a given cluster. 89 | #[experimental] 90 | pub fn partition() { 91 | 92 | } 93 | 94 | /// Reset `iptables` back to it's proper configuration. 95 | #[experimental] 96 | pub fn heal() { 97 | 98 | } 99 | -------------------------------------------------------------------------------- /src/broadcast.rs: -------------------------------------------------------------------------------- 1 | //! A broadcast represents a single message being sent out. 2 | 3 | use uuid::Uuid; 4 | use std::collections::hashmap::HashSet; 5 | use std::io::MemReader; 6 | 7 | use result::{GossipResult, GossipError, UnknownError, io_err}; 8 | use stream::Stream; 9 | 10 | #[deriving(PartialEq, Show)] 11 | pub struct Version(u8); 12 | 13 | /// Broadcast represents a single bi-directional communication with two 14 | /// nodes within the cluster. The communication does **not** need to be 15 | /// bi-directional. Responses are completely optional. 16 | /// 17 | /// Each broadcast is tagged with a unique ID so that we may track 18 | /// which node has received a given broadcast. 19 | /// 20 | /// Each broadcast is sent in a custom binary format. This allows us to store 21 | /// the tag that associates the type of the broadcast in Rust code. 22 | /// 23 | /// Format: 24 | /// 25 | /// ```notrust 26 | /// bitdata RawBroadcast { 27 | /// RawBroadcast { 28 | /// version: u8, 29 | /// tag_size: u32, 30 | /// tag: &[u8], 31 | /// data_size: u32, 32 | /// data: &[u8] 33 | /// } 34 | /// } 35 | /// ``` 36 | /// 37 | /// This allows us to effectively parse the metadata we need then 38 | /// forward the decoding to the appropriate format's parser. We also have 39 | /// the ability to interoporate between different formats. As long as each 40 | /// Node has the ability to understand that format. 41 | pub struct Broadcast { 42 | /// A unique id for the broadcast. This allows the servers 43 | /// to talk about a unique broadcast in unison and also coordinate 44 | /// the response (if applicable). 45 | id: Uuid, 46 | version: Version, 47 | /// A tag represents the type of message it is without needing a physical type to decode it to. 48 | /// Since we may not always have that information. 49 | tag: String, 50 | /// The raw bytes of the full broadcast. 51 | reader: MemReader, 52 | /// A set of servers that have seen/committed the broadcast. 53 | committed: HashSet 54 | } 55 | 56 | impl Broadcast { 57 | /// Given a tag and message, create a new instance of Broadcast with 58 | /// a brand-new unique ID so that we can uniquely identify it. 59 | pub fn new(bytes: Vec) -> GossipResult { 60 | let mut reader = MemReader::new(bytes.clone()); 61 | let version = try!(reader.read_byte().map_err(io_err)); 62 | let tag = "foo".to_string(); 63 | Ok(Broadcast { 64 | id: Uuid::new_v4(), 65 | version: Version(version), 66 | tag: tag, 67 | reader: reader, 68 | committed: HashSet::new() 69 | }) 70 | } 71 | 72 | pub fn parse(&mut self) { 73 | 74 | } 75 | 76 | pub fn id(&self) -> Uuid { 77 | self.id 78 | } 79 | } 80 | 81 | #[cfg(test)] 82 | mod test { 83 | use super::*; 84 | use result::GossipResult; 85 | 86 | #[test] 87 | fn parse_broadcast() { 88 | let mut bytes = vec![1u8]; 89 | bytes = bytes.append(b"foo"); 90 | bytes = bytes.append(&[1u8,2,3]); 91 | let broadcast = Broadcast::new(bytes).unwrap(); 92 | assert_eq!(broadcast.tag, "foo".to_string()); 93 | let Version(ver) = broadcast.version; 94 | assert_eq!(ver, 1u8); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![feature(globs, macro_rules, unsafe_destructor, plugin_registrar, struct_inherit)] 2 | #![allow(missing_doc)] 3 | 4 | //! Gossip.rs is a gossip/epidemic protocol based on the 5 | //! paper "Epidemic Broadcast Trees" in which it introduced 6 | //! the novel idea of the Plumtree. 7 | //! 8 | //! Gossip provides a way for a group of nodes, otherwise 9 | //! known as a cluster, to communicate and achieve 10 | //! consensus. However, it's specifically meant to be a 11 | //! highly available system (AP); thus, it supports 12 | //! eventual consistency in face of partitions. 13 | //! 14 | //! This trade-off is fine for many distributed systems 15 | //! that need to be highly available and fault-tolerant. Things 16 | //! such as data processing, analytics, etc... are all great 17 | //! candidates. 18 | 19 | extern crate collections; 20 | extern crate uuid; 21 | extern crate rand; 22 | extern crate serialize; 23 | extern crate core; 24 | extern crate sync; 25 | extern crate time; 26 | extern crate msgpack; 27 | 28 | pub use result::{GossipResult, GossipError}; 29 | pub use protocol::{Node}; 30 | pub use stream::{Callback, SockAddr}; 31 | 32 | mod result; 33 | mod stream; 34 | mod tag; 35 | mod state; 36 | mod protocol; 37 | mod broadcast; 38 | -------------------------------------------------------------------------------- /src/protocol.rs: -------------------------------------------------------------------------------- 1 | use std::task::TaskBuilder; 2 | use std::io::{TcpListener, TcpStream, Acceptor, Listener}; 3 | use std::io::net::tcp::TcpAcceptor; 4 | use std::collections::{HashSet, HashMap}; 5 | 6 | use uuid::Uuid; 7 | use stream::{Stream, Response, SockAddr, Callback, BroadcastFrom}; 8 | use broadcast::Broadcast; 9 | use result::{GossipResult, GossipError, NotListening}; 10 | 11 | /// A health represents the current state of the cluster. This will be extremely useful 12 | /// to ping the health of a cluster and determine the high-level status of it. 13 | /// 14 | /// Green = Perfect state. 15 | /// Yellow = Nodes are failing, but the cluster is still operational. 16 | /// Red = Not good. Cluster might be completely dead. 17 | #[deriving(Show, PartialEq, Clone)] 18 | pub enum Health { 19 | Green, 20 | Yellow, 21 | Red 22 | } 23 | 24 | /// An iterator that receives new broadcasts and iterates over them. 25 | pub struct Incoming { 26 | node_tx: Sender, 27 | tx: Sender<(Broadcast, Stream)>, 28 | rx: Receiver<(Broadcast, Stream)>, 29 | listening: bool 30 | } 31 | 32 | impl Incoming { 33 | pub fn new(node_tx: Sender, 34 | sender: Sender>) -> Incoming { 35 | let (tx, rx) = channel(); 36 | 37 | sender.send(tx.clone()); 38 | 39 | Incoming { 40 | node_tx: node_tx, 41 | tx: tx, 42 | rx: rx, 43 | listening: true 44 | } 45 | } 46 | } 47 | 48 | impl Iterator for Incoming { 49 | 50 | /// TODO: Handle the response with the Drop trait. This means 51 | /// we'll need to have a reference to the `Stream`. 52 | fn next(&mut self) -> Option { 53 | if self.listening { 54 | let (broadcast, stream) = self.rx.recv(); 55 | let id = broadcast.id().clone(); 56 | Some((broadcast, Response::new(id, stream))) 57 | } else { 58 | None 59 | } 60 | } 61 | } 62 | 63 | /// A dedicated Rust task to manage a single `TcpStream`. Each 64 | /// stream is then associated to a given peer (although the peer 65 | /// isn't always set right away). 66 | /// 67 | /// This task then has channels to communicate with the main 68 | /// AcceptorTask which can communicate with other StreamTasks. 69 | struct StreamTask { 70 | peer: Option, 71 | pub stream: Stream 72 | } 73 | 74 | impl StreamTask { 75 | pub fn new(stream: Stream) -> StreamTask { 76 | StreamTask { 77 | peer: None, 78 | stream: stream 79 | } 80 | } 81 | 82 | pub fn incoming(&mut self) { 83 | } 84 | } 85 | 86 | 87 | /// A dedicated task to handle the incoming connections 88 | /// over a tcp socket (called streams). This task does 89 | /// not need to communicate with the streams themselves, that's 90 | /// done at another task that handles the core logic of the 91 | /// protocol. 92 | struct AcceptorTask { 93 | acceptor: TcpAcceptor, 94 | server_tx: Sender, 95 | tx: Sender, 96 | rx: Receiver 97 | } 98 | 99 | enum TaskMessage { 100 | StreamMsg(Stream), 101 | BroadcastMsg(Broadcast) 102 | } 103 | 104 | impl AcceptorTask { 105 | pub fn new(host: &str, port: u16, server_tx: Sender, 106 | inter_tx: Sender>) -> AcceptorTask { 107 | let listener = TcpListener::bind(host, port).unwrap(); 108 | let (tx, rx) = channel(); 109 | 110 | inter_tx.send(tx.clone()); 111 | 112 | AcceptorTask { 113 | acceptor: listener.listen().unwrap(), 114 | server_tx: server_tx, 115 | tx: tx, 116 | rx: rx 117 | } 118 | } 119 | 120 | pub fn run(&mut self) { 121 | for stream in self.acceptor.incoming() { 122 | match stream { 123 | Ok(s) => { 124 | // Handle the joining here... 125 | let stream_send = s.clone(); 126 | let server = self.server_tx.clone(); 127 | spawn(proc() { 128 | let stream = Stream::new(stream_send); 129 | server.send(StreamMsg(stream.clone())); 130 | StreamTask::new(stream).incoming(); 131 | }); 132 | }, 133 | Err(e) => println!("Error: {}", e) 134 | } 135 | } 136 | } 137 | } 138 | 139 | struct ServerTask { 140 | streams: HashMap, 141 | acceptor_tx: Sender, 142 | tx: Sender, 143 | rx: Receiver 144 | } 145 | 146 | impl ServerTask { 147 | pub fn new(host: String, port: u16) -> ServerTask { 148 | // Local channels that deal with broadcasts. 149 | let (tx, rx) = channel(); 150 | 151 | // Intermediate channels for the acceptor task. 152 | // We'll use this to retrieve the sender of the 153 | // acceptor task. 154 | let (acceptor_tx, acceptor_rx) = channel(); 155 | 156 | let server_tx = tx.clone(); 157 | spawn(proc() { 158 | AcceptorTask::new(host.as_slice(), port, server_tx, acceptor_tx).run(); 159 | }); 160 | 161 | ServerTask { 162 | streams: HashMap::new(), 163 | acceptor_tx: acceptor_rx.recv(), 164 | tx: tx, 165 | rx: rx 166 | } 167 | } 168 | 169 | pub fn broadcast(&self, peer: &Peer) -> GossipResult<()> { 170 | Ok(()) 171 | } 172 | 173 | pub fn run(&mut self) { 174 | for msg in self.rx.iter() { 175 | match msg { 176 | StreamMsg(stream) => { 177 | // self.streams.insert(peer, stream); 178 | }, 179 | BroadcastMsg(broadcast) => {} 180 | } 181 | } 182 | } 183 | } 184 | 185 | /// A peer describes a member within the cluster/network that 186 | /// is not the current one. 187 | #[deriving(Clone, Show, PartialEq, Hash, Eq)] 188 | pub struct Peer { 189 | id: Uuid, 190 | addr: SockAddr 191 | } 192 | 193 | impl Peer { 194 | pub fn new(id: Uuid, host: &str, port: u16) -> Peer { 195 | Peer { 196 | id: id, 197 | addr: SockAddr::new(host, port) 198 | } 199 | } 200 | 201 | pub fn empty() -> Peer { 202 | Peer { 203 | id: Uuid::new_v4(), 204 | addr: SockAddr::new("localhost", 3444) 205 | } 206 | } 207 | } 208 | 209 | /// A `Node` is a single member within the gossip protocol. Nodes that 210 | /// join together is called a cluster. This forms the distributed system 211 | /// in which the gossip protocol runs within. 212 | /// 213 | /// Each Node is an equal member in the cluster. That means there isn't a 214 | /// single leader. This has a significant trade-off and one must understand 215 | /// it before being able to use the system correctly. 216 | /// 217 | /// Node: Handle the state. 218 | /// Incoming: Handle incoming connections and broadcasts. 219 | pub struct Node { 220 | /// Each node generates their own unique Uuid (v4) to uniquely 221 | /// identify each other within the cluster. Instead of saying 222 | /// "I'm node A", you would say "I'm node 123e4567-e89b-12d3-a456-426655440000". 223 | id: Uuid, 224 | 225 | /// A set of other members within the cluster. This forms the basic 226 | /// information about each Node. This doesn't, however, contain connection 227 | /// information and what not. 228 | members: Vec, 229 | incoming_tx: Option>, 230 | tx: Sender<(Peer, Broadcast)>, 231 | rx: Receiver<(Peer, Broadcast)> 232 | } 233 | 234 | impl Node { 235 | /// Usage: 236 | /// 237 | /// ```rust 238 | /// use gossip::Node; 239 | /// let mut node = Node::new(); 240 | /// ``` 241 | pub fn new() -> Node { 242 | let (tx, rx) = channel(); 243 | 244 | Node { 245 | id: Uuid::new_v4(), 246 | members: Vec::new(), 247 | incoming_tx: None, 248 | tx: tx, 249 | rx: rx 250 | } 251 | } 252 | 253 | /// Initialize the Node to listen on the specified address/port 254 | /// combination. This will bootup the appropriate tasks to allow 255 | /// incoming connections and broadcasts. 256 | #[unimplemented] 257 | pub fn listen(&mut self, host: &str, port: u16) -> GossipResult<()> { 258 | let host = host.to_string(); 259 | 260 | spawn(proc() { 261 | ServerTask::new(host, port).run(); 262 | }); 263 | 264 | Ok(()) 265 | } 266 | 267 | /// Given a peer node, join it's existing cluster. Each node technically 268 | /// creates their own cluster automatically. Joining multiple nodes together 269 | /// is an explicit process. The peer node doesn't need to be the same one, 270 | /// but it's not a bad idea. 271 | #[unimplemented] 272 | pub fn join(&mut self, host: &str, port: u16) -> GossipResult<()> { 273 | Ok(()) 274 | } 275 | 276 | /// Shutdown all the running tasks that are listening to new broadcasts 277 | /// and incoming connections. This will send one last broadcast 278 | /// to the current cluster notifying all other nodes of the shutdown. 279 | /// 280 | /// Afterwhich tasks will shutdown and the node will be terminated. 281 | #[unimplemented] 282 | pub fn shutdown(&mut self) {} 283 | 284 | /// Create a new `Incoming` iterator that iterates over newly received 285 | /// broadcasts that the user can handle. 286 | /// 287 | /// Usage: 288 | /// 289 | /// ```notrust 290 | /// use gossip::Node; 291 | /// spawn(proc() { 292 | /// let mut node = Node::new(); 293 | /// 294 | /// node.listen("localhost", 4888).unwrap(); 295 | /// 296 | /// for (broadcast, mut res) in node.incoming() { 297 | /// println!("Broadcast ..."); 298 | /// // ... 299 | /// } 300 | /// }); 301 | /// ``` 302 | pub fn incoming(&mut self) -> Incoming { 303 | let (tx, rx) = channel(); 304 | let incoming = Incoming::new(self.tx.clone(), tx); 305 | 306 | self.incoming_tx = Some(rx.recv()); 307 | 308 | incoming 309 | } 310 | } 311 | 312 | #[cfg(test)] 313 | mod test { 314 | use super::*; 315 | 316 | #[test] 317 | fn empty_member_set() { 318 | let mut node = Node::new(); 319 | assert_eq!(node.members.len(), 0); 320 | } 321 | 322 | #[test] 323 | fn bind_listening() { 324 | let mut node = Node::new(); 325 | } 326 | } 327 | -------------------------------------------------------------------------------- /src/result.rs: -------------------------------------------------------------------------------- 1 | //! Custom Result and Error types. 2 | 3 | use std::io; 4 | use std::str::SendStr; 5 | 6 | pub type GossipResult = Result; 7 | 8 | /// Convert an IoError to a GossipError 9 | pub fn io_err(io: io::IoError) -> GossipError { 10 | GossipError { 11 | kind: IoError(io.clone()), 12 | desc: io.desc.into_maybe_owned() 13 | } 14 | } 15 | 16 | /// A gossip error represents an error that happens typically during any I/O. 17 | #[deriving(Show)] 18 | pub struct GossipError { 19 | kind: GossipErrorKind, 20 | desc: SendStr, 21 | } 22 | 23 | #[deriving(Show)] 24 | pub enum GossipErrorKind { 25 | NodeUnreachable, 26 | NotListening, 27 | UnknownError, 28 | IoError(io::IoError) 29 | } 30 | 31 | impl GossipError { 32 | pub fn new>(desc: T, kind: GossipErrorKind) -> GossipError { 33 | GossipError { 34 | kind: kind, 35 | desc: desc.into_maybe_owned(), 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/state.rs: -------------------------------------------------------------------------------- 1 | use std::collections::hashmap::HashSet; 2 | use protocol::{Health, Yellow}; 3 | use broadcast::Broadcast; 4 | 5 | pub struct State { 6 | eager: HashSet, 7 | lazy: HashSet, 8 | health: Health, 9 | broadcasts: Vec 10 | } 11 | 12 | impl State { 13 | /// Create a new default State that starts a new cluster in a 14 | /// Yellow state. 15 | pub fn new() -> State { 16 | State { 17 | eager: HashSet::new(), 18 | lazy: HashSet::new(), 19 | health: Yellow, 20 | broadcasts: Vec::new() 21 | } 22 | } 23 | } 24 | 25 | #[cfg(test)] 26 | mod tests { 27 | use super::*; 28 | use protocol::Yellow; 29 | 30 | #[test] 31 | fn default_state() { 32 | let s = State::new(); 33 | assert_eq!(s.eager.len(), 0); 34 | assert_eq!(s.lazy.len(), 0); 35 | assert_eq!(s.broadcasts.len(), 0); 36 | assert_eq!(s.health, Yellow); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/stream.rs: -------------------------------------------------------------------------------- 1 | use std::io::MemWriter; 2 | use std::io::{TcpListener, TcpStream, Acceptor, Listener}; 3 | use std::io::net::tcp::TcpAcceptor; 4 | use uuid::Uuid; 5 | 6 | use broadcast::Broadcast; 7 | use protocol::Peer; 8 | use result::{GossipResult, GossipError}; 9 | 10 | pub type Callback = (Broadcast, Response); 11 | pub type BroadcastFrom = (Peer, Broadcast); 12 | 13 | #[deriving(Clone)] 14 | pub struct Stream { 15 | stream: TcpStream, 16 | peer: Option 17 | } 18 | 19 | impl Stream { 20 | pub fn new(stream: TcpStream) -> Stream { 21 | Stream { 22 | stream: stream, 23 | peer: None 24 | } 25 | } 26 | } 27 | 28 | impl Iterator for Stream { 29 | fn next(&mut self) -> Option { 30 | None 31 | } 32 | } 33 | 34 | pub struct Response { 35 | id: Uuid, 36 | stream: Stream, 37 | wr: MemWriter 38 | } 39 | 40 | impl Response { 41 | pub fn new(id: Uuid, stream: Stream) -> Response { 42 | Response { 43 | id: id, 44 | stream: stream, 45 | wr: MemWriter::new() 46 | } 47 | } 48 | 49 | /// Acknowledge the incoming broadcast with a simple OK 50 | /// message back. Responses aren't always required, but it's 51 | /// often very useful to have a nice short way of saying 52 | /// "Got the message, it's all good!". 53 | /// 54 | /// This takes `self` as a value because we don't 55 | /// allow multiple responses. So the response will be moved and 56 | /// further responses won't be possible. 57 | pub fn ok(mut self) -> GossipResult<()> { 58 | write!(self.stream.stream, "{},OK", self.id); 59 | Ok(()) 60 | } 61 | } 62 | 63 | #[unsafe_destructor] 64 | impl Drop for Response { 65 | /// Handle the response on the drop call. 66 | fn drop(&mut self) { 67 | } 68 | } 69 | 70 | /// We work with an ip and port a lot. Let's make it easier 71 | /// and bundle these in a single record. 72 | #[deriving(Show, Clone, Eq, PartialEq, Hash)] 73 | pub struct SockAddr { 74 | /// Most of the Rust APIs now use a string for the ip 75 | /// instead of the IpSockAddr enum variants (v4, v6). 76 | pub ip: String, 77 | /// Standard port number. 78 | pub port: u16 79 | } 80 | 81 | impl SockAddr { 82 | /// Working with allocated strings are quite awkward. Slices 83 | /// are much easier to work with and allow things such as: 84 | /// 85 | /// ```rust 86 | /// use gossip::SockAddr; 87 | /// SockAddr::new("0.0.0.0", 8777); 88 | /// ``` 89 | /// 90 | /// Instead of: 91 | /// 92 | /// ```rust 93 | /// use gossip::SockAddr; 94 | /// SockAddr::new("0.0.0.0", 8777); 95 | /// ``` 96 | pub fn new(ip: &str, port: u16) -> SockAddr { 97 | SockAddr { 98 | ip: ip.to_string(), 99 | port: port 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/tag.rs: -------------------------------------------------------------------------------- 1 | pub trait Tag { 2 | fn get_tag(&self) -> &'static str; 3 | } 4 | -------------------------------------------------------------------------------- /tests/3-node-cluster.rs: -------------------------------------------------------------------------------- 1 | extern crate gossip; 2 | 3 | #[test] 4 | fn three_node_cluster() { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /tests/check-style.sh: -------------------------------------------------------------------------------- 1 | echo "checking for lines over 100 characters..." 2 | find src test -name '*.rs' | xargs grep '.\{101,\}' && exit 1 3 | echo "ok" 4 | -------------------------------------------------------------------------------- /tests/compile.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thehydroimpulse/gossiper/3e1945833b6359fc5d430c705bfaecdb538ab89b/tests/compile.sh -------------------------------------------------------------------------------- /tests/mod.rs: -------------------------------------------------------------------------------- 1 | extern crate gossip; 2 | extern crate simulation; 3 | 4 | pub mod proxy; 5 | pub mod simulations; 6 | -------------------------------------------------------------------------------- /tests/proxy.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thehydroimpulse/gossiper/3e1945833b6359fc5d430c705bfaecdb538ab89b/tests/proxy.rs -------------------------------------------------------------------------------- /tests/simulations/mod.rs: -------------------------------------------------------------------------------- 1 | 2 | --------------------------------------------------------------------------------