├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── README.md ├── app ├── app.go ├── app_test.go └── cache.go ├── client ├── client.go ├── client_test.go ├── kvclient │ └── kvclient.go └── stats.go ├── clientcli ├── cli │ └── cli.go └── main.go ├── config ├── addresses.go ├── addresses_test.go ├── client.go ├── client_test.go ├── server.go ├── server_test.go ├── workload.go └── workload_test.go ├── configfiles ├── README.md ├── delegated │ ├── client.conf │ └── server.conf ├── fpaxos │ ├── client.conf │ └── server.conf └── simple │ ├── client.conf │ └── server.conf ├── consensus ├── consensus.go ├── consensus_test.go ├── coordinator.go ├── interfacer.go ├── log.go ├── master.go ├── participant.go ├── quourm.go ├── recovery_coordinator.go ├── utils.go ├── window.go └── window_test.go ├── doc.go ├── docker-compose.yml ├── example.conf ├── example3.conf ├── gateway ├── main.go └── rest │ └── rest.go ├── ios.go ├── ios ├── example.conf ├── example3.conf ├── main.go └── server │ └── server.go ├── misc └── logo.png ├── msgs ├── failures.go ├── failures_test.go ├── formats.go ├── marshal.go ├── msgs.go ├── msgs_test.go ├── notifier.go └── storage.go ├── net ├── clients.go └── peers.go ├── scripts ├── coverage └── docker │ ├── deploy3servers │ ├── deploy3serversfailed │ ├── deploycluster │ ├── example3.conf │ └── stop3servers ├── services ├── dummy.go ├── services.go ├── store.go └── store_test.go ├── simulator ├── simulator.go └── simulator_test.go ├── storage ├── persist.go ├── persist_test.go ├── reader.go ├── restore.go ├── wal.go ├── wal_darwin.go ├── wal_linux.go ├── wal_test.go └── wal_unsupported.go ├── test ├── generator │ ├── generator.go │ └── generator_test.go ├── main.go └── workloads │ ├── balanced.conf │ ├── example.conf │ ├── read-heavy.conf │ └── write-heavy.conf ├── testset └── main.go └── tools ├── benchmark-disk └── main.go ├── generate-config └── main.go └── local-test └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | *.ipynb_checkpoints 27 | 28 | # project specific 29 | *.temp 30 | *.out 31 | *.html 32 | *.csv 33 | *.log 34 | results 35 | *.dat 36 | *.cover* 37 | server/server 38 | client/client 39 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | language: go 3 | go: 4 | - 1.6 5 | - 1.7.x 6 | - 1.8.x 7 | services: 8 | - docker 9 | install: 10 | - go get -t github.com/heidi-ann/ios/... 11 | - docker build -t ios . 12 | - go get golang.org/x/tools/cmd/cover 13 | - go get github.com/mattn/goveralls 14 | script: 15 | - go test -v ./... -logtostderr=true 16 | - PATH="$HOME/gopath/bin:$PATH" 17 | - scripts/coverage --coveralls 18 | - $GOPATH/bin/local-test 19 | - $GOPATH/src/github.com/heidi-ann/ios/scripts/docker/deploy3servers 20 | - $GOPATH/src/github.com/heidi-ann/ios/scripts/docker/stop3servers 21 | env: 22 | global: 23 | secure: cwwrH596UWLt1d8GHFb7BAt9BttJx8pIMwTglowQCsySjScC7Jqfofp/yYETC01Oe+cBYfhePopqkX++VpJI4dReWwPryoyXAoGsVWAa36Q/IW1BRb01o0awaGGP1BHfg7TwlgHlRgAOJJ+JRXfA0q42O+xrxFwVEiGrrFJkdyUiiVxrtORTGDSmJr63iFyCr+0SZmQbgo7eBeVQlaL9QpOlhfYrQY4C+lPUx+DhVVNo5iHIb1kWTBnXtuWHqX2DG7BnLR35HqIyvBgkrJf3SG+k4fbnIvOUdfbPeM0v5K9O4SRQHC1Iuj4ZniKJpyESJIrivERUfKo8uZWHEi3LSpib+zMF3Lr/NIN32Echi7p6Akfs7K8DwIXrkWgkiYvNYrwunA+4jr6BFYa6B/pKpAsbH5QY1K0d/s+ZR3TtASBN0bcUb5oWHBfCJO4xAyDzVw9b0n24g1oARWTpen+4/TzTaVDUleBZV8v6JNUoLGU0nWKG2gwyjvRJJf58OPdx3mmhuCq4+AzNuHKpZjjZrKFzihiVBRtOf/8vcwBXezu0SWCETRg/ykXTG4mJFc3iOHOrR6VaMs8YPUARS6raVXp0pkRUactvxLS/U6y29mCZuXiem1ejjKyCLsnnHdh+sswONeoRfGOifko0SKqOp4hhIffLp4Q92yDTLYdhCfA= 24 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang 2 | ADD . /go/src/github.com/heidi-ann/ios 3 | EXPOSE 8080 8090 4 | RUN go get github.com/heidi-ann/ios/... 5 | ENTRYPOINT ["ios","-listen-peers=8090","-listen-clients=8080"] 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016-2017 Heidi Howard 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Ios project logo](../master/misc/logo.png?raw=true) 2 | 3 | 4 | [![Build Status](https://travis-ci.org/heidi-ann/ios.svg?branch=master)](https://travis-ci.org/heidi-ann/ios) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/heidi-ann/ios)](https://goreportcard.com/report/github.com/heidi-ann/ios) 6 | [![GoDoc](https://godoc.org/github.com/heidi-ann/ios?status.svg)](https://godoc.org/github.com/heidi-ann/ios) 7 | [![Coverage Status](https://coveralls.io/repos/github/heidi-ann/ios/badge.svg?branch=master)](https://coveralls.io/github/heidi-ann/ios?branch=master) 8 | 9 | Welcome to Ios, a reliable distributed agreement service for cloud applications. Built upon a novel decentralised consensus protocol, Ios provides vital services for your cloud application such as distributed locking, consistent data structures and leader election as well as distributed configuration and coordination. 10 | 11 | *This repository is pre-alpha and under active development. APIs will be broken. This code has not been proven correct and is not ready for production deployment.* 12 | 13 | ## Getting Started 14 | These instructions will get you a simple Ios server and client up and running on your local machine. See deployment for notes on how to deploy Ios across a cluster. 15 | 16 | ### Prerequisites 17 | Ios is built in [Go](https://golang.org/) version 1.6.2 and currently supports Go version 1.6 and 1.7. The [Golang site](https://golang.org/) details how to install and setup Go on your local machine. Don't forget to add GOPATH to your .profile. 18 | 19 | ### Installation 20 | After installing Go, run: 21 | ``` 22 | go get github.com/heidi-ann/ios/... 23 | ``` 24 | This command will copy the Ios source code to $GOPATH/src/github.com/heidi-ann/ios and then fetch and build the following dependancies: 25 | * [glog](github.com/golang/glog) - logging library, in the style of glog for C++ 26 | * [gcfg](gopkg.in/gcfg.v1) - library for parsing git-config style config files 27 | It will then build and install Ios, the server and client binaries will be placed in $GOPATH/bin. 28 | 29 | ### Up & Running 30 | You can now start a simple 1 node Ios cluster as follows: 31 | ``` 32 | $GOPATH/bin/ios -id 0 33 | ``` 34 | This will start an Ios server providing a simple key-value store. The server is listening for clients on port 8080. 35 | 36 | You can now start an Ios client as follows: 37 | ``` 38 | $ $GOPATH/bin/clientcli 39 | Starting Ios client in interactive mode. 40 | 41 | The following commands are available: 42 | get [key]: to return the value of a given key 43 | exists [key]: to test if a given key is present 44 | update [key] [value]: to set the value of a given key, if key already exists then overwrite 45 | delete [key]: to remove a key value pair if present 46 | count: to return the number of keys 47 | print: to return all key value pairs 48 | 49 | Enter command: update A 1 50 | OK 51 | Enter command: get A 52 | 1 53 | ... 54 | ``` 55 | You can now enter commands for the key value store, followed by the enter key. These commands are being sent to the Ios server, executed and the result is returned to the user. 56 | 57 | The Ios server is using files called persistent_log_0.temp, persistent_snap_0.temp and persistent_data_0.temp to store Ios's persistent state. If these files are present when a server starts, it will restore its state from these files. You can try this by killing the server process and restarting it, it should carry on from where it left off. 58 | 59 | When you would like to start a fresh server instance, use ``rm persistent*.temp`` first to clear these files and then start the server again. 60 | 61 | ## Building in Docker 62 | 63 | Alternatively, you can build and run in Ios using [Docker](https://www.docker.com/). Make sure Docker is installed and running, then clone this repository and cd into it. 64 | 65 | Build an image named 'ios' using the following command 66 | 67 | ``` 68 | docker build -t ios . 69 | ``` 70 | You should now be able to run ```docker images``` and see the Ios image you just created. If you run, ```docker ps``` you will see that no docker containers are currently running, start a simple 1 node Ios cluster as follows: 71 | 72 | ``` 73 | docker run --name ios-server -d ios -id 0 74 | ``` 75 | Running ```docker ps``` we can now see that is Ios server is running. We can test this by communicate with it using an Ios command line client: 76 | ``` 77 | docker run --net=container:ios-server -it --name ios-client --entrypoint clientcli ios 78 | ``` 79 | 80 | 81 | Note that this will only use storage local to the container instance. If you want persistence/recoverability for instances you will need to store persistence logs on a mounted data volume 82 | 83 | ## Next steps 84 | 85 | In this section, we are going to take a closer look at what is going on underneath. We will then use this information to setup a 3 server Ios cluster on your local machine and automatically generate a workload to put it to the test. PS: you might want to start by opening up a few terminal windows. 86 | 87 | #### Server configuration 88 | The server we ran in previous section was using the default configuration file found in [example.conf](example.conf). The first section of this file lists the Ios servers in the cluster and how the peers can connect to them and the second section lists how the client can connect to them. The configuration file [example3.conf](example3.conf) shows what this looks like for 3 servers running on localhost. The same configuration file is used for all the servers, at run time they are each given an ID (starting from 0) and use this to know which ports to listen on. The rest of the configuration file options are documented at https://godoc.org/github.com/heidi-ann/ios/config. After removing the persistent storage, start 3 Ios servers in 3 separate terminal windows as follows: 89 | 90 | ``` 91 | $GOPATH/bin/ios -id [ID] -config $GOPATH/src/github.com/heidi-ann/ios/configfiles/simple/server3.conf -stderrthreshold=INFO 92 | ``` 93 | For ID 0, 1 and 2 94 | 95 | #### Client configuration 96 | 97 | Like the servers, the client we ran in the previous section was using the default configuration file found in [example.conf](example.conf). The first section lists the Ios servers in the cluster and how to connect to them. The configuration file [example3.conf](example3.conf) shows what this looks like for 3 servers currently running on localhost. 98 | 99 | We are run a client as before and interact with our 3 servers. 100 | ``` 101 | $GOPATH/bin/clientcli -config $GOPATH/src/github.com/heidi-ann/ios/client/example3.conf 102 | ``` 103 | 104 | You should be able to kill and restart the servers to test when the system is available to the client. Since the Ios cluster you have deployed is configured to use strict majority quorums then the system should be available whenever at least two servers are up. 105 | 106 | #### Workload configuration 107 | 108 | Typing requests into a terminal is, of course, slow and unrealistic. To help test the system, Ios provides test clients which can automatically generate a workload and measure system performance. To run a client in test mode use: 109 | ``` 110 | $GOPATH/bin/test -config $GOPATH/src/github.com/heidi-ann/ios/client/example3.conf -auto $GOPATH/src/github.com/heidi-ann/ios/test/workload.conf 111 | ``` 112 | This client will run the workload described in [test/workloads/example.conf](test/workloads/example.conf) and then terminate. It will write performance metrics into a file called latency.csv. Ios currently also support a REST API mode which listens for HTTP on port 12345. 113 | 114 | ## Contributing 115 | 116 | #### Debugging 117 | 118 | We use glog for logging. Adding `-logtostderr=true -v=1` when running executables prints the logging output. For more information, visit https://godoc.org/github.com/golang/glog. 119 | 120 | Likewise, the following commands work with the above example and are useful for debugging: 121 | ``` 122 | sudo tcpdump -i lo0 -nnAS "(src portrange 8080-8092 or dst portrange 8080-8092) and (length>0)" 123 | ``` 124 | ``` 125 | sudo strace -p $(pidof server) -T -e fsync -f 126 | sudo strace -p $(pidof server) -T -e trace=write -f 127 | ``` 128 | 129 | ### Benchmarking 130 | 131 | The benchmarking scripts for Ios can found here https://github.com/heidi-ann/consensus_eval 132 | 133 | ## License 134 | 135 | This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details 136 | -------------------------------------------------------------------------------- /app/app.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/golang/glog" 6 | "github.com/heidi-ann/ios/msgs" 7 | "github.com/heidi-ann/ios/services" 8 | ) 9 | 10 | // StateMachine abstracts over the services state machine and the cache which ensure exactly-once execution 11 | type StateMachine struct { 12 | Cache *Cache 13 | Store services.Service 14 | } 15 | 16 | // New creates a StateMachine with the given service application 17 | func New(appConfig string) *StateMachine { 18 | return &StateMachine{newCache(), services.StartService(appConfig)} 19 | } 20 | 21 | // Apply request will apply a request (or fetch the result of its application) and return the result 22 | func (s *StateMachine) Apply(req msgs.ClientRequest) msgs.ClientResponse { 23 | glog.V(1).Info("Request has been safely replicated by consensus algorithm", req) 24 | 25 | // check if request already applied 26 | if found, reply := s.Cache.check(req); found { 27 | glog.V(1).Info("Request found in cache and thus need not be applied", req) 28 | return reply 29 | } 30 | // apply request and cache 31 | reply := msgs.ClientResponse{ 32 | req.ClientID, req.RequestID, true, s.Store.Process(req.Request)} 33 | s.Cache.add(reply) 34 | return reply 35 | } 36 | 37 | // ApplyRead request will apply a read request and return the result. It will not cache the result 38 | func (s *StateMachine) ApplyRead(req msgs.ClientRequest) msgs.ClientResponse { 39 | glog.V(1).Info("Read request has been passed by consensus algorithm", req) 40 | return msgs.ClientResponse{ 41 | req.ClientID, req.RequestID, true, s.Store.Process(req.Request)} 42 | } 43 | 44 | // ApplyReads request will apply a slice of read requests and return the results. It will not cache the results. 45 | func (s *StateMachine) ApplyReads(reqs []msgs.ClientRequest) []msgs.ClientResponse { 46 | glog.V(1).Info("Read requests has been passed to state machine by consensus algorithm") 47 | responses := make([]msgs.ClientResponse, len(reqs)) 48 | for i := 0; i < len(reqs); i++ { 49 | responses[i] = msgs.ClientResponse{ 50 | reqs[i].ClientID, reqs[i].RequestID, true, s.Store.Process(reqs[i].Request)} 51 | } 52 | return responses 53 | } 54 | 55 | // Check request return true and the result of the request if the request has already been applied to the state machine 56 | func (s *StateMachine) Check(req msgs.ClientRequest) (bool, msgs.ClientResponse) { 57 | return s.Cache.check(req) 58 | } 59 | 60 | // MakeSnapshot serializes a state machine into bytes 61 | func (s *StateMachine) MakeSnapshot() ([]byte, error) { 62 | return json.Marshal(s) 63 | } 64 | 65 | // RestoreSnapshot deserializes bytes into a state machine 66 | func RestoreSnapshot(snap []byte, appConfig string) (*StateMachine, error) { 67 | sm := New(appConfig) 68 | err := json.Unmarshal(snap, sm) 69 | return sm, err 70 | } 71 | -------------------------------------------------------------------------------- /app/app_test.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "github.com/heidi-ann/ios/msgs" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestApply(t *testing.T) { 10 | assert := assert.New(t) 11 | sm := New("kv-store") 12 | 13 | request1 := msgs.ClientRequest{ 14 | ClientID: 1, 15 | RequestID: 1, 16 | ForceViewChange: false, 17 | ReadOnly: false, 18 | Request: "update A 1", 19 | } 20 | expectedResponse1 := msgs.ClientResponse{ 21 | ClientID: 1, 22 | RequestID: 1, 23 | Success: true, 24 | Response: "OK", 25 | } 26 | 27 | // check caching 28 | found, res := sm.Check(request1) 29 | assert.False(found, "Empty cache found result ", res) 30 | 31 | actualResponse := sm.Apply(request1) 32 | assert.Equal(actualResponse, expectedResponse1, "Unexpected response") 33 | 34 | found, res = sm.Check(request1) 35 | assert.True(found, "Unexpected cache miss for ", request1) 36 | assert.Equal(res, expectedResponse1, "Cache did not return expected result") 37 | 38 | actualResponseB := sm.Apply(request1) 39 | assert.Equal(actualResponseB, expectedResponse1, "Unexpected response") 40 | 41 | // check snapshotting 42 | snap, err := sm.MakeSnapshot() 43 | assert.Nil(err) 44 | smRestored, err := RestoreSnapshot(snap, "kv-store") 45 | assert.Nil(err) 46 | found, res = smRestored.Check(request1) 47 | assert.True(found, "Unexpected cache miss for ", request1) 48 | assert.Equal(res, expectedResponse1, "Cache did not return expected result") 49 | 50 | // check application 51 | request2 := msgs.ClientRequest{ 52 | ClientID: 1, 53 | RequestID: 2, 54 | ForceViewChange: false, 55 | ReadOnly: true, 56 | Request: "get A", 57 | } 58 | expectedResponse2 := msgs.ClientResponse{ 59 | ClientID: 1, 60 | RequestID: 2, 61 | Success: true, 62 | Response: "1", 63 | } 64 | 65 | found, res = sm.Check(request2) 66 | assert.False(found, "Empty cache found result ", res) 67 | 68 | actualResponse2 := sm.Apply(request2) 69 | assert.Equal(actualResponse2, expectedResponse2, "Unexpected response") 70 | 71 | found, res = sm.Check(request2) 72 | assert.True(found, "Unexpected cache miss for ", request2) 73 | assert.Equal(res, expectedResponse2, "Cache did not return expected result") 74 | 75 | } 76 | -------------------------------------------------------------------------------- /app/cache.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/golang/glog" 6 | "github.com/heidi-ann/ios/msgs" 7 | "strconv" 8 | "sync" 9 | ) 10 | 11 | // TODO: locking could be more fine grained for improved concurreny 12 | 13 | // Cache provides a simple key value store mapping client ID's to the last request sent to them. 14 | // It is safe for concurreny access 15 | type Cache struct { 16 | m map[int]msgs.ClientResponse 17 | sync.RWMutex 18 | } 19 | 20 | func newCache() *Cache { 21 | c := map[int]msgs.ClientResponse{} 22 | return &Cache{c, sync.RWMutex{}} 23 | } 24 | 25 | func (c *Cache) check(req msgs.ClientRequest) (bool, msgs.ClientResponse) { 26 | c.RLock() 27 | last := c.m[req.ClientID] 28 | c.RUnlock() 29 | if last.RequestID > req.RequestID { 30 | glog.Fatal("Request has already been applied to state machine and overwritten in request cache") 31 | } 32 | return req.RequestID == last.RequestID, last 33 | } 34 | 35 | func (c *Cache) add(res msgs.ClientResponse) { 36 | c.Lock() 37 | if c.m[res.ClientID].RequestID != 0 && c.m[res.ClientID].RequestID > res.RequestID { 38 | glog.Fatal("Requests must be added to request cache in order, expected ", c.m[res.ClientID].RequestID+1, 39 | " but received ", res.RequestID) 40 | } 41 | c.m[res.ClientID] = res 42 | c.Unlock() 43 | } 44 | 45 | // MarshalJSON marshals a cache into bytes 46 | // default JSON marshalling requires string map keys thus custom function is provided 47 | func (c *Cache) MarshalJSON() ([]byte, error) { 48 | c.Lock() 49 | // convert to string map 50 | strMap := map[string]msgs.ClientResponse{} 51 | for k, v := range c.m { 52 | strMap[strconv.Itoa(k)] = v 53 | } 54 | b, err := json.Marshal(strMap) 55 | if err != nil { 56 | glog.Warning("Unable to snapshot request cache: ", err) 57 | } 58 | c.Unlock() 59 | return b, err 60 | } 61 | 62 | // UnmarshalJSON unmarshals bytes into a cache 63 | func (c *Cache) UnmarshalJSON(snap []byte) error { 64 | var strMap map[string]msgs.ClientResponse 65 | err := json.Unmarshal(snap, &strMap) 66 | if err != nil { 67 | glog.Warning("Unable to restore from snapshot: ", err) 68 | return err 69 | } 70 | // convert to int map 71 | c.m = map[int]msgs.ClientResponse{} 72 | for k, v := range strMap { 73 | i, err := strconv.Atoi(k) 74 | if err != nil { 75 | glog.Warning("Unable to restore from snapshot: ", err) 76 | return err 77 | } 78 | c.m[i] = v 79 | } 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | // Package client provides Ios client side code for connecting to an Ios cluster 2 | package client 3 | 4 | import ( 5 | "bufio" 6 | "errors" 7 | "io" 8 | "math/rand" 9 | "net" 10 | "time" 11 | 12 | "github.com/golang/glog" 13 | "github.com/heidi-ann/ios/config" 14 | "github.com/heidi-ann/ios/msgs" 15 | ) 16 | 17 | //TODO: write requestID to disk 18 | 19 | // Client holds the data associated with a client 20 | type Client struct { 21 | id int // ID of client, must be unique 22 | requestID int // ID of current request, starting from 1 23 | stats *statsFile // handler for stats collection, maybe nil 24 | servers []config.NetAddress // address of Ios servers 25 | conn net.Conn 26 | rd *bufio.Reader 27 | timeout time.Duration 28 | backoff time.Duration // time to wait after trying n servers when client cannot connect to Ios cluster 29 | random bool // if enabled, client connects to servers at random instead of systematically 30 | beforeForce int // number of times a request should be submitted before client sets ForceViewChange, if -1 then will not set 31 | serverID int // ID of the server currently/last connected to 32 | } 33 | 34 | // connectRandom tries to connect to a server specified in addresses 35 | func connectRandom(addrs []config.NetAddress, backoff time.Duration) (net.Conn, int) { 36 | for { 37 | for tried := 0; tried < len(addrs); tried++ { 38 | id := rand.Intn(len(addrs)) 39 | glog.V(1).Info("Trying to connect to ", addrs[id].ToString()) 40 | conn, err := net.Dial("tcp", addrs[id].ToString()) 41 | // if successful 42 | if err == nil { 43 | glog.Infof("Connect established to %s", addrs[id].ToString()) 44 | return conn, id 45 | } 46 | time.Sleep(backoff) 47 | } 48 | } 49 | } 50 | 51 | // connectSystematic try to establish a connection with server ID hint 52 | // if unsuccessful, it tries to connect to other servers sytematically, waiting for backoff after trying each server 53 | // once successful, connect will return the net.Conn and the ID of server connected to 54 | // connectSystematic may never return if it cannot connect to any server 55 | func connectSystematic(addrs []config.NetAddress, hint int, backoff time.Duration) (net.Conn, int) { 56 | // reset invalid hint 57 | if len(addrs) >= hint { 58 | hint = 0 59 | } 60 | 61 | // first, try on to connect to the most likely leader 62 | glog.Info("Trying to connect to ", addrs[hint].ToString()) 63 | conn, err := net.Dial("tcp", addrs[hint].ToString()) 64 | // if successful 65 | if err == nil { 66 | glog.Infof("Connect established to %s", addrs[hint].ToString()) 67 | return conn, hint 68 | } 69 | glog.Warning(err) //if unsuccessful 70 | 71 | // if fails, try everyone else 72 | for { 73 | // TODO: start from hint instead of from ID:0 74 | for i := range addrs { 75 | glog.V(1).Info("Trying to connect to ", addrs[i].ToString()) 76 | conn, err = net.Dial("tcp", addrs[i].ToString()) 77 | 78 | // if successful 79 | if err == nil { 80 | glog.Infof("Connect established to %s", addrs[i].ToString()) 81 | return conn, i 82 | } 83 | 84 | //if unsuccessful 85 | glog.Warning(err) 86 | } 87 | time.Sleep(backoff) 88 | } 89 | } 90 | 91 | // dispatcher will send bytes and wait for reply, return bytes returned if succussful or error otherwise 92 | func dispatcher(b []byte, conn net.Conn, r *bufio.Reader, timeout time.Duration) ([]byte, error) { 93 | // check for nil connection 94 | if conn == nil { 95 | glog.Warning("connection missing") 96 | return nil, errors.New("Connection closed") 97 | } 98 | 99 | // setup channels for timeout implementation 100 | errCh := make(chan error, 1) 101 | replyCh := make(chan []byte, 1) 102 | 103 | go func() { 104 | // send request 105 | _, err := conn.Write(b) 106 | _, err = conn.Write([]byte("\n")) 107 | if err != nil && err != io.EOF { 108 | glog.Warning(err) 109 | errCh <- err 110 | } 111 | glog.V(1).Info("Sent") 112 | // read response 113 | reply, err := r.ReadBytes('\n') 114 | if err != nil && err != io.EOF { 115 | glog.Warning(err) 116 | errCh <- err 117 | } 118 | // success, return reply 119 | replyCh <- reply 120 | }() 121 | 122 | //handling outcomes 123 | select { 124 | case reply := <-replyCh: 125 | return reply, nil 126 | case err := <-errCh: 127 | return nil, err 128 | case <-time.After(timeout): 129 | return nil, errors.New("Timeout of " + timeout.String()) 130 | } 131 | } 132 | 133 | // StartClient creates an Ios client and tries to connect to an Ios cluster 134 | // If ID is -1 then a random one will be generated 135 | func StartClient(id int, statsFilename string, addrs []config.NetAddress, timeout time.Duration, backoff time.Duration, beforeForce int, random bool) (*Client, error) { 136 | 137 | // TODO: find a better way to handle required flags 138 | if id == -1 { 139 | rand.Seed(time.Now().UTC().UnixNano()) 140 | id = rand.Int() 141 | glog.V(1).Info("ID was not provided, ID ", id, " has been assigned") 142 | } 143 | 144 | glog.Info("Starting up client ", id) 145 | 146 | // set up stats collection 147 | var stats *statsFile 148 | var err error 149 | if statsFilename != "" { 150 | stats, err = openStatsFile(statsFilename) 151 | if err != nil { 152 | return nil, err 153 | } 154 | } 155 | 156 | // connecting to server 157 | var conn net.Conn 158 | var serverID int 159 | if random { 160 | glog.Info("Client trying to connect to servers randomly") 161 | conn, serverID = connectRandom(addrs, backoff) 162 | } else { 163 | glog.Info("Client trying to connect to servers systematically") 164 | conn, serverID = connectSystematic(addrs, 0, backoff) 165 | } 166 | glog.Info("Client is ready to start processing incoming requests") 167 | 168 | rd := bufio.NewReader(conn) 169 | return &Client{id, 1, stats, addrs, conn, rd, timeout, backoff, random, beforeForce, serverID}, nil 170 | } 171 | 172 | func (c *Client) SubmitRequest(text string, readonly bool) (string, error) { 173 | glog.V(1).Info("Request ", c.requestID, " is: ", text) 174 | 175 | // prepare request 176 | req := msgs.ClientRequest{ 177 | ClientID: c.id, 178 | RequestID: c.requestID, 179 | ForceViewChange: false, 180 | ReadOnly: readonly, 181 | Request: text} 182 | b, err := msgs.Marshal(req) 183 | if err != nil { 184 | glog.Warning(err) 185 | return "", err 186 | } 187 | glog.V(1).Info(string(b)) 188 | 189 | if c.stats != nil { 190 | c.stats.startRequest(c.requestID) 191 | } 192 | tries := 0 193 | var reply *msgs.ClientResponse 194 | 195 | // dispatch request until successful 196 | for { 197 | if c.beforeForce != -1 && tries > c.beforeForce { 198 | glog.Warning("Request ", c.requestID, " is being set to force view change") 199 | req.ForceViewChange = true 200 | b, err = msgs.Marshal(req) 201 | if err != nil { 202 | glog.Warning(err) 203 | return "", err 204 | } 205 | } 206 | 207 | replyBytes, err := dispatcher(b, c.conn, c.rd, c.timeout) 208 | if err == nil { 209 | //handle reply 210 | reply = new(msgs.ClientResponse) 211 | err = msgs.Unmarshal(replyBytes, reply) 212 | 213 | if err == nil && !reply.Success { 214 | err = errors.New("request marked by server as unsuccessful") 215 | } 216 | if err == nil && reply.Success { 217 | glog.V(1).Info("request was Successful", reply) 218 | break 219 | } 220 | } 221 | 222 | // continue if request failed 223 | glog.Warning("Request ", c.requestID, " failed due to: ", err) 224 | 225 | // close connection 226 | if c.conn != nil { 227 | err = c.conn.Close() 228 | if err != nil { 229 | glog.Warning(err) 230 | } 231 | } 232 | // establish a new connection 233 | if c.random { 234 | c.conn, c.serverID = connectRandom(c.servers, c.backoff) 235 | } else { 236 | // next try last serverID +1 mod n 237 | nextID := c.serverID + 1 238 | if nextID >= len(c.servers) { 239 | nextID = 0 240 | } 241 | c.conn, c.serverID = connectSystematic(c.servers, nextID, c.backoff) 242 | } 243 | c.rd = bufio.NewReader(c.conn) 244 | tries++ 245 | } 246 | 247 | //check reply is not nil 248 | if *reply == (msgs.ClientResponse{}) { 249 | return "", errors.New("Response is nil") 250 | } 251 | 252 | //check reply is as expected 253 | if reply.ClientID != c.id { 254 | return "", errors.New("Response received has wrong ClientID") 255 | } 256 | if reply.RequestID != c.requestID { 257 | return "", errors.New("Response received has wrong RequestID") 258 | } 259 | if !reply.Success { 260 | return "", errors.New("Response marked as unsuccessful but not retried") 261 | } 262 | 263 | if c.stats != nil { 264 | // write to latency to log 265 | err = c.stats.stopRequest(tries, readonly) 266 | } 267 | c.requestID++ 268 | return reply.Response, err 269 | } 270 | 271 | // StartClientFromConfigFile creates an Ios client 272 | // If ID is -1 then a random one will be generated 273 | func StartClientFromConfigFile(id int, statFile string, configFile string, addressFile string) (*Client, error) { 274 | conf, err := config.ParseClientConfig(configFile) 275 | if err != nil { 276 | return nil, err 277 | } 278 | if err := config.CheckConfig(conf); err != nil { 279 | return nil, err 280 | } 281 | addresses, err := config.ParseAddresses(addressFile) 282 | if err != nil { 283 | return nil, err 284 | } 285 | timeout := time.Millisecond * time.Duration(conf.Parameters.Timeout) 286 | backoff := time.Millisecond * time.Duration(conf.Parameters.Backoff) 287 | return StartClient(id, statFile, addresses.Clients, timeout, backoff, conf.Parameters.BeforeForce, conf.Parameters.ConnectRandom) 288 | } 289 | 290 | // StartClientFromConfig is the same as StartClientFromConfigFile but for config structs instead of files 291 | func StartClientFromConfig(id int, statFile string, conf config.Config, addresses []config.NetAddress) (*Client, error) { 292 | if err := config.CheckConfig(conf); err != nil { 293 | return nil, err 294 | } 295 | timeout := time.Millisecond * time.Duration(conf.Parameters.Timeout) 296 | backoff := time.Millisecond * time.Duration(conf.Parameters.Backoff) 297 | return StartClient(id, statFile, addresses, timeout, backoff, conf.Parameters.BeforeForce, conf.Parameters.ConnectRandom) 298 | } 299 | 300 | func (c *Client) StopClient() { 301 | glog.Info("Shutting down client ", c.id) 302 | // close stats file 303 | if c.stats != nil { 304 | c.stats.closeStatsFile() 305 | } 306 | // close connection 307 | if c.conn != nil { 308 | c.conn.Close() 309 | } 310 | } 311 | -------------------------------------------------------------------------------- /client/client_test.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | package client 4 | 5 | import ( 6 | "github.com/golang/glog" 7 | "github.com/heidi-ann/ios/config" 8 | "github.com/heidi-ann/ios/ios/server" 9 | "github.com/stretchr/testify/assert" 10 | "io/ioutil" 11 | "os" 12 | "testing" 13 | ) 14 | 15 | func TestStartClient(t *testing.T) { 16 | assert := assert.New(t) 17 | //Create temp directories 18 | dirClient, err := ioutil.TempDir("", "IosStartClientTests") 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | defer os.RemoveAll(dirClient) 23 | 24 | //start 3 node Ios cluster 25 | serverConfigFile := os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/ios/example3.conf" 26 | for id := 0; id <= 2; id++ { 27 | dirServer, err := ioutil.TempDir("", "IosStartClientTests") 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | defer os.RemoveAll(dirServer) 32 | go server.RunIos(id, config.ParseServerConfig(serverConfigFile), dirServer) 33 | } 34 | 35 | //start client 36 | clientConfigFile := os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/client/example3.conf" 37 | 38 | client := StartClientFromConfigFile(1, dirClient+"/latency.csv", clientConfigFile) 39 | 40 | //submit requests 41 | success, reply, err := client.SubmitRequest("update A 1") 42 | assert.Nil(err) 43 | assert.True(success, "Request not successful") 44 | assert.Equal("OK", reply, "Response not as expected") 45 | 46 | success, reply, err = client.SubmitRequest("get A") 47 | assert.Nil(err) 48 | assert.True(success, "Request not successful") 49 | assert.Equal("1", reply, "Response not as expected") 50 | 51 | client2 := StartClientFromConfigFile(2, dirClient+"/latency2.csv", clientConfigFile) 52 | 53 | //submit requests to new client 54 | success, reply, err = client2.SubmitRequest("get A") 55 | assert.Nil(err) 56 | assert.True(success, "Request not successful") 57 | assert.Equal("1", reply, "Response not as expected") 58 | 59 | success, reply, err = client2.SubmitRequest("update B 2") 60 | assert.Nil(err) 61 | assert.True(success, "Request not successful") 62 | assert.Equal("OK", reply, "Response not as expected") 63 | 64 | //check original client is still ok 65 | success, reply, err = client.SubmitRequest("get B") 66 | assert.Nil(err) 67 | assert.True(success, "Request not successful") 68 | assert.Equal("2", reply, "Response not as expected") 69 | 70 | //stopping client 71 | client.StopClient() 72 | client2.StopClient() 73 | 74 | } 75 | -------------------------------------------------------------------------------- /client/kvclient/kvclient.go: -------------------------------------------------------------------------------- 1 | // Package kvclient provides a client for interacting with a key-value type Ios cluster 2 | package kvclient 3 | 4 | import ( 5 | "strconv" 6 | "time" 7 | 8 | "fmt" 9 | 10 | "github.com/heidi-ann/ios/client" 11 | "github.com/heidi-ann/ios/config" 12 | ) 13 | 14 | type KvClient struct { 15 | iosClient *client.Client 16 | } 17 | 18 | // StartKvClient creates an Ios client and tries to connect to an Ios cluster 19 | // If ID is -1 then a random one will be generated 20 | func StartKvClient(id int, statFile string, addrs []config.NetAddress, timeout time.Duration, backoff time.Duration, beforeForce int, random bool) (*KvClient, error) { 21 | iosClient, err := client.StartClient(id, statFile, addrs, timeout, backoff, beforeForce, random) 22 | return &KvClient{iosClient}, err 23 | } 24 | 25 | func (kvc *KvClient) Update(key string, value string) error { 26 | _, err := kvc.iosClient.SubmitRequest(fmt.Sprintf("update %v %v", key, value), false) 27 | return err 28 | } 29 | 30 | func (kvc *KvClient) Get(key string) (string, error) { 31 | return kvc.iosClient.SubmitRequest(fmt.Sprintf("get %v", key), true) 32 | } 33 | 34 | func (kvc *KvClient) Delete(key string) error { 35 | _, err := kvc.iosClient.SubmitRequest(fmt.Sprintf("delete %v", key), false) 36 | return err 37 | } 38 | 39 | func (kvc *KvClient) Count() (int, error) { 40 | reply, err := kvc.iosClient.SubmitRequest("count", true) 41 | if err != nil { 42 | return 0, err 43 | } 44 | count, err := strconv.Atoi(reply) 45 | return count, err 46 | } 47 | 48 | func (kvc *KvClient) Print() (string, error) { 49 | return kvc.iosClient.SubmitRequest("print", true) 50 | } 51 | -------------------------------------------------------------------------------- /client/stats.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "encoding/csv" 5 | "github.com/golang/glog" 6 | "os" 7 | "strconv" 8 | "time" 9 | ) 10 | 11 | // statsFile handles writing basic request stats such as latency to a csv file 12 | type statsFile struct { 13 | w *csv.Writer 14 | startTime time.Time 15 | requestID int 16 | } 17 | 18 | func openStatsFile(filename string) (*statsFile, error) { 19 | glog.Info("Opening file: ", filename) 20 | file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0777) 21 | if err != nil { 22 | return nil, err 23 | } 24 | writer := csv.NewWriter(file) 25 | return &statsFile{writer, time.Now(), 1}, nil 26 | } 27 | 28 | func (sf *statsFile) startRequest(requestID int) { 29 | sf.requestID = requestID 30 | sf.startTime = time.Now() 31 | } 32 | 33 | func (sf *statsFile) stopRequest(tries int, readonly bool) error { 34 | latency := strconv.FormatInt(time.Since(sf.startTime).Nanoseconds(), 10) 35 | return sf.w.Write([]string{ 36 | strconv.FormatInt(sf.startTime.UnixNano(), 10), 37 | strconv.Itoa(sf.requestID), 38 | latency, 39 | strconv.Itoa(tries), 40 | strconv.FormatBool(readonly)}) 41 | } 42 | 43 | func (sf *statsFile) closeStatsFile() { 44 | sf.w.Flush() 45 | //TODO:close stats file 46 | } 47 | -------------------------------------------------------------------------------- /clientcli/cli/cli.go: -------------------------------------------------------------------------------- 1 | // Package cli provides a command line interface for interactive with Ios, useful for testing 2 | package cli 3 | 4 | import ( 5 | "bufio" 6 | "fmt" 7 | "github.com/golang/glog" 8 | "github.com/heidi-ann/ios/services" 9 | "os" 10 | "strings" 11 | ) 12 | 13 | type Interative bufio.Reader 14 | 15 | func CreateInteractiveTerminal(app string) *Interative { 16 | fmt.Printf("Starting Ios %s client in interactive mode.\n", app) 17 | fmt.Print(services.GetInteractiveText(app)) 18 | return (*Interative)(bufio.NewReader(os.Stdin)) 19 | 20 | } 21 | 22 | func (i *Interative) FetchTerminalInput() (string, bool, bool) { 23 | b := (*bufio.Reader)(i) 24 | for { 25 | fmt.Print("Enter command: ") 26 | text, err := b.ReadString('\n') 27 | if err != nil { 28 | glog.Fatal(err) 29 | } 30 | text = strings.Trim(text, "\n") 31 | text = strings.Trim(text, "\r") 32 | glog.V(1).Info("User entered", text) 33 | ok, read := services.Parse("kv-store", text) 34 | if ok { 35 | return text, read, true 36 | } else { 37 | fmt.Print("Invalid command\n") 38 | } 39 | } 40 | } 41 | 42 | func (_ *Interative) ReturnToTerminal(str string) { 43 | // , time time.Duration "request took ", time 44 | fmt.Print(str + "\n") 45 | } 46 | -------------------------------------------------------------------------------- /clientcli/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/golang/glog" 6 | "github.com/heidi-ann/ios/client" 7 | "github.com/heidi-ann/ios/clientcli/cli" 8 | "github.com/heidi-ann/ios/config" 9 | "os" 10 | "os/signal" 11 | "syscall" 12 | ) 13 | 14 | var configFile = flag.String("config", os.Getenv("GOPATH")+"/src/github.com/heidi-ann/ios/example.conf", "Client configuration file") 15 | var statFile = flag.String("stat", "latency.csv", "File to write stats to") 16 | var algorithmFile = flag.String("algorithm", os.Getenv("GOPATH")+"/src/github.com/heidi-ann/ios/configfiles/simple/client.conf", "Algorithm description file") // optional flag 17 | var id = flag.Int("id", -1, "ID of client (must be unique) or random number will be generated") 18 | 19 | func main() { 20 | // set up logging 21 | flag.Parse() 22 | defer glog.Flush() 23 | 24 | // always flush (whatever happens) 25 | sigs := make(chan os.Signal, 1) 26 | finish := make(chan bool, 1) 27 | signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) 28 | 29 | // parse config files 30 | conf, err := config.ParseClientConfig(*algorithmFile) 31 | if err != nil { 32 | glog.Fatal(err) 33 | } 34 | addresses, err := config.ParseAddresses(*configFile) 35 | if err != nil { 36 | glog.Fatal(err) 37 | } 38 | c, err := client.StartClientFromConfig(*id, *statFile, conf, addresses.Clients) 39 | if err != nil { 40 | glog.Fatal(err) 41 | } 42 | 43 | // setup API 44 | terminal := cli.CreateInteractiveTerminal(conf.Parameters.Application) 45 | 46 | go func() { 47 | for { 48 | // get next command 49 | text, read, ok := terminal.FetchTerminalInput() 50 | if !ok { 51 | finish <- true 52 | break 53 | } 54 | // pass to ios client 55 | reply, err := c.SubmitRequest(text, read) 56 | if err != nil { 57 | finish <- true 58 | break 59 | } 60 | // notify API of result 61 | terminal.ReturnToTerminal(reply) 62 | 63 | } 64 | }() 65 | 66 | select { 67 | case sig := <-sigs: 68 | glog.Warning("Termination due to: ", sig) 69 | case <-finish: 70 | glog.Info("No more commands") 71 | } 72 | c.StopClient() 73 | glog.Flush() 74 | 75 | } 76 | -------------------------------------------------------------------------------- /config/addresses.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "strconv" 6 | "strings" 7 | 8 | "github.com/golang/glog" 9 | "gopkg.in/gcfg.v1" 10 | ) 11 | 12 | // addressFile describes the addresses as parsed by the address file 13 | type addressFile struct { 14 | Servers struct { 15 | Address []string 16 | } 17 | } 18 | 19 | // NetAddress holds a network address 20 | type NetAddress struct { 21 | Address string 22 | Port int 23 | } 24 | 25 | func (n NetAddress) ToString() string { 26 | return n.Address + ":" + strconv.Itoa(n.Port) 27 | } 28 | 29 | // Addresses describes the network configuarion of an Ios cluster. 30 | type Addresses struct { 31 | Peers []NetAddress 32 | Clients []NetAddress 33 | } 34 | 35 | // ParseAddresses filename will parse the given file into an Addresses struct 36 | func ParseAddresses(filename string) (Addresses, error) { 37 | var config addressFile 38 | var addresses Addresses 39 | 40 | if err := gcfg.ReadFileInto(&config, filename); err != nil { 41 | glog.Warning("Unable to parse configuration file") 42 | return addresses, err 43 | } 44 | // checking configuation is sensible 45 | n := len(config.Servers.Address) 46 | if n == 0 { 47 | return addresses, errors.New("At least one server is required") 48 | } 49 | // parse into addresses 50 | addresses = Addresses{make([]NetAddress, n), make([]NetAddress, n)} 51 | 52 | for i, addr := range config.Servers.Address { 53 | address := strings.Split(addr, ":") 54 | if len(address) != 3 { 55 | return addresses, errors.New("Address must be of the form, ipv4:serverport:clientport e.g. 127.0.0.1:8090:8080 ") 56 | } 57 | // TODO: check format of IP address/domain name 58 | 59 | // parse peer port 60 | peerPort, err := strconv.Atoi(address[1]) 61 | if err != nil || peerPort < 0 || peerPort > 65535 { 62 | return addresses, errors.New("Address must be of the form, ipv4:serverport:clientport e.g. 127.0.0.1:8090:8080 ") 63 | } 64 | addresses.Peers[i] = NetAddress{address[0], peerPort} 65 | 66 | // parse client port 67 | clientPort, err := strconv.Atoi(address[2]) 68 | if err != nil || clientPort < 0 || clientPort > 65535 { 69 | return addresses, errors.New("Address must be of the form, ipv4:serverport:clientport e.g. 127.0.0.1:8090:8080 ") 70 | } 71 | addresses.Clients[i] = NetAddress{address[0], clientPort} 72 | 73 | } 74 | return addresses, nil 75 | } 76 | -------------------------------------------------------------------------------- /config/addresses_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | // TestParseAddresses calls ParseAddresses for the single server example config file 11 | func TestParseParseAddresses(t *testing.T) { 12 | conf, err := ParseAddresses(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/ios/example.conf") 13 | assert.Nil(t, err) 14 | 15 | assert.Equal(t, 1, len(conf.Clients)) 16 | assert.Equal(t, "127.0.0.1", conf.Clients[0].Address) 17 | assert.Equal(t, 8080, conf.Clients[0].Port) 18 | 19 | assert.Equal(t, 1, len(conf.Peers)) 20 | assert.Equal(t, "127.0.0.1", conf.Peers[0].Address) 21 | assert.Equal(t, 8090, conf.Peers[0].Port) 22 | } 23 | 24 | // TestParseAddresses3 calls ParseAddresses for the 3 server example config file 25 | func TestParseParseAddresses3(t *testing.T) { 26 | conf, err := ParseAddresses(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/ios/example3.conf") 27 | assert.Nil(t, err) 28 | 29 | assert.Equal(t, 3, len(conf.Clients)) 30 | for i := 0; i < 3; i++ { 31 | assert.Equal(t, "127.0.0.1", conf.Clients[i].Address) 32 | assert.Equal(t, 8080+i, conf.Clients[i].Port) 33 | } 34 | 35 | assert.Equal(t, 3, len(conf.Peers)) 36 | for i := 0; i < 3; i++ { 37 | assert.Equal(t, "127.0.0.1", conf.Peers[i].Address) 38 | assert.Equal(t, 8090+i, conf.Peers[i].Port) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /config/client.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "github.com/golang/glog" 6 | "gopkg.in/gcfg.v1" 7 | ) 8 | 9 | type Config struct { 10 | Parameters struct { 11 | Timeout int 12 | Backoff int 13 | ConnectRandom bool 14 | BeforeForce int 15 | Application string 16 | } 17 | } 18 | 19 | // CheckConfig checks wheather a given configuration is sensible 20 | func CheckConfig(config Config) error { 21 | if config.Parameters.Timeout <= 0 { 22 | return errors.New("Timeout must be >= 0") 23 | } 24 | if config.Parameters.Backoff <= 0 { 25 | return errors.New("Backoff must be >= 0") 26 | } 27 | if config.Parameters.BeforeForce < -1 { 28 | return errors.New("Backoff must be >= 0") 29 | } 30 | if config.Parameters.Application != "kv-store" && config.Parameters.Application != "dummy" { 31 | return errors.New("Application must be either kv-store or dummy") 32 | } 33 | return nil 34 | } 35 | 36 | // ParseConfig reads a client configuration from a file and returns a config struct 37 | // Callers usually then pass result to CheckConfig 38 | func ParseClientConfig(filename string) (Config, error) { 39 | var config Config 40 | err := gcfg.ReadFileInto(&config, filename) 41 | if err != nil { 42 | glog.Warning("Unable to parse client configuration file") 43 | return config, err 44 | } 45 | return config, nil 46 | } 47 | -------------------------------------------------------------------------------- /config/client_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | // TestParseClientConfig calls ParseConfig for the two example configuration files 11 | func TestParseClientConfig(t *testing.T) { 12 | conf, err := ParseClientConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/simple/client.conf") 13 | assert.Nil(t, err) 14 | assert.Nil(t, CheckConfig(conf)) 15 | conf, err = ParseClientConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/fpaxos/client.conf") 16 | assert.Nil(t, err) 17 | assert.Nil(t, CheckConfig(conf)) 18 | conf, err = ParseClientConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/delegated/client.conf") 19 | assert.Nil(t, err) 20 | assert.Nil(t, CheckConfig(conf)) 21 | } 22 | -------------------------------------------------------------------------------- /config/server.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/golang/glog" 7 | "gopkg.in/gcfg.v1" 8 | ) 9 | 10 | // ServerConfig describes the configuration options for an Ios server. 11 | // Example valid configuration files can be found in server/example.conf and server/example3.conf 12 | type ServerConfig struct { 13 | Algorithm struct { 14 | DelegateReplication int // how many replication coordinators to delegate to when master, -1 means use reverse delegation 15 | QuorumSystem string // which quorum system to use: either "strict majority", "non-strict majority", "all-in", "one-in" or "fixed:n" 16 | IndexExclusivity bool // if enabled, Ios will assign each index to at most one request 17 | ParticipantHandle bool // if enabled, non-master servers will handle/forward client requests, otherwise they will redirect clients 18 | ParticipantRead bool // if set then non-master servers can service reads after getting backing from a read quorum. "forward mode only" 19 | ImplicitWindowCommit bool // if uncommitted request is outside of current window then commit 20 | ExplicitCommit bool // if enabled, Ios coordinators will send commit messages to all after replication 21 | ThriftyQuorum bool // if enabled, coordinators will send writes to quorum only instead of broadcasting 22 | } 23 | Performance struct { 24 | Length int // max log size 25 | BatchInterval int // how often to batch process request in ms, 0 means no batching 26 | MaxBatch int // maximum requests in a batch, unused if BatchInterval=0 27 | WindowSize int // how many requests can the master have inflight at once 28 | SnapshotInterval int // how often to record state machine snapshots, 0 means snapshotting is disabled 29 | } 30 | Application struct { 31 | Name string // which application should Ios serve: either "kv-store" or "dummy" 32 | } 33 | Unsafe struct { 34 | DumpPersistentStorage bool // if enabled, then persistent storage is not written to a file, always set to false 35 | PersistenceMode string // mode of write ahead logging: either "none", "fsync" or "osync", "direct" or "dsync". The "none" option is unsafe. 36 | } 37 | } 38 | 39 | // CheckServerConfig checks wheather a given configuration is sensible 40 | func CheckServerConfig(config ServerConfig) error { 41 | if config.Performance.Length <= 0 { 42 | return errors.New("Log length must be at least 1") 43 | } 44 | if config.Performance.BatchInterval < 0 { 45 | return errors.New("Batch interval must be positive") 46 | } 47 | if config.Performance.MaxBatch < 0 { 48 | return errors.New("Max batch size must be positive") 49 | } 50 | if config.Algorithm.DelegateReplication < -1 { 51 | return errors.New("DelegateReplication must be within range, or -1 for reverse delegation") 52 | } 53 | if config.Performance.WindowSize <= 0 { 54 | return errors.New("Window Size must be greater than one") 55 | } 56 | if config.Application.Name != "kv-store" && config.Application.Name != "dummy" { 57 | return errors.New("Application must be either kv-store or dummy") 58 | } 59 | // TODO: check QuorumSystem 60 | return nil 61 | } 62 | 63 | // ParseServerConfig filename will parse the given file and return a ServerConfig struct containing its data 64 | // Callers usually then pass result to CheckServerConfig 65 | func ParseServerConfig(filename string) (ServerConfig, error) { 66 | var config ServerConfig 67 | err := gcfg.ReadFileInto(&config, filename) 68 | if err != nil { 69 | glog.Warning("Unable to parse server configuration file") 70 | return config, err 71 | } 72 | return config, nil 73 | } 74 | -------------------------------------------------------------------------------- /config/server_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestParseServerConfig(t *testing.T) { 11 | conf, err := ParseServerConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/delegated/server.conf") 12 | assert.Nil(t, err) 13 | assert.Nil(t, CheckServerConfig(conf)) 14 | 15 | conf, err = ParseServerConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/fpaxos/server.conf") 16 | assert.Nil(t, err) 17 | assert.Nil(t, CheckServerConfig(conf)) 18 | 19 | conf, err = ParseServerConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/simple/server.conf") 20 | assert.Nil(t, err) 21 | assert.Nil(t, CheckServerConfig(conf)) 22 | } 23 | 24 | // TestParseServerConfig calls ParseServerConfig for the two example configuration files 25 | func TestParseSingleServerConfig(t *testing.T) { 26 | conf, err := ParseServerConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/simple/server.conf") 27 | assert.Nil(t, err) 28 | assert.Nil(t, CheckServerConfig(conf)) 29 | 30 | assert.Equal(t, 0, conf.Algorithm.DelegateReplication) 31 | assert.Equal(t, "strict majority", conf.Algorithm.QuorumSystem) 32 | assert.False(t, conf.Algorithm.IndexExclusivity) 33 | assert.False(t, conf.Algorithm.ParticipantHandle) 34 | assert.False(t, conf.Algorithm.ParticipantRead) 35 | assert.False(t, conf.Algorithm.ParticipantRead) 36 | 37 | assert.Equal(t, 100000, conf.Performance.Length) 38 | assert.Equal(t, 0, conf.Performance.BatchInterval) 39 | assert.Equal(t, 10, conf.Performance.MaxBatch) 40 | assert.Equal(t, 1, conf.Performance.WindowSize) 41 | assert.Equal(t, 0, conf.Performance.SnapshotInterval) 42 | 43 | assert.Equal(t, "kv-store", conf.Application.Name) 44 | 45 | assert.False(t, conf.Unsafe.DumpPersistentStorage) 46 | assert.Equal(t, "fsync", conf.Unsafe.PersistenceMode) 47 | 48 | } 49 | -------------------------------------------------------------------------------- /config/workload.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/golang/glog" 7 | "gopkg.in/gcfg.v1" 8 | ) 9 | 10 | // ConfigAuto describes a client workload to be generated. 11 | type ConfigAuto struct { 12 | Reads int // percentage of read requests, the remaining requests are writes. 13 | Interval int // milliseconand delay between recieving a response and sending next request 14 | KeySize int // size of key for generated requests, unit is string characters 15 | ValueSize int // size of value for generated requests, unit is string characters 16 | Requests int // terminate after this number of requests have been completed 17 | Keys int // number of keys to operate on 18 | } 19 | 20 | // WorkloadConfig is a wrapper around ConfigAuto. 21 | type workloadConfig struct { 22 | Config ConfigAuto 23 | } 24 | 25 | // CheckWorkloadConfig verifies that a workload is sensible 26 | func CheckWorkloadConfig(config ConfigAuto) error { 27 | if config.Reads > 100 { 28 | return errors.New("Reads must be a percentage") 29 | } 30 | if config.Reads < 0 || config.Interval < 0 || config.KeySize < 0 || 31 | config.ValueSize < 0 || config.Requests < 0 || config.Keys < 0 { 32 | return errors.New("Workload parameter must be a postive integers") 33 | } 34 | return nil 35 | } 36 | 37 | // ParseWorkloadConfig filenames parses the given workload configation file. 38 | // Callers usually then pass result to CheckWorkloadConfig 39 | func ParseWorkloadConfig(filename string) (ConfigAuto, error) { 40 | var config workloadConfig 41 | if err := gcfg.ReadFileInto(&config, filename); err != nil { 42 | glog.Warning("Unable to parse workload config") 43 | return config.Config, err 44 | } 45 | return config.Config, nil 46 | } 47 | -------------------------------------------------------------------------------- /config/workload_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | // TestParseAuto calls ParseAuto for the example configuration file 11 | func TestParseAuto(t *testing.T) { 12 | files := []string{"example.conf", "balanced.conf", "read-heavy.conf", "write-heavy.conf"} 13 | for _, file := range files { 14 | workload, err := ParseWorkloadConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/test/workloads/" + file) 15 | assert.Nil(t, err) 16 | assert.Nil(t, CheckWorkloadConfig(workload)) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /configfiles/README.md: -------------------------------------------------------------------------------- 1 | This directory contains various sets of example configurations for Ios. Each set contains, server.conf, the config file for an Ios server 2 | and client.conf, the config file for a Ios client. 3 | -------------------------------------------------------------------------------- /configfiles/delegated/client.conf: -------------------------------------------------------------------------------- 1 | [parameters] 2 | Timeout=500 3 | Backoff=10 4 | ConnectRandom=false 5 | BeforeForce=3 6 | Application = kv-store 7 | -------------------------------------------------------------------------------- /configfiles/delegated/server.conf: -------------------------------------------------------------------------------- 1 | ;; example description of Ios server 2 | 3 | [algorithm] 4 | delegateReplication = 2 5 | quorumSystem = strict majority 6 | indexExclusivity = false 7 | participantHandle = false 8 | participantRead = false 9 | implicitWindowCommit = false 10 | ExplicitCommit=false 11 | ThriftyQuorum=true 12 | 13 | [performance] 14 | length = 100000 15 | batchInterval = 0 16 | maxBatch = 100 17 | windowSize = 10 18 | snapshotInterval = 0 19 | 20 | [application] 21 | name = kv-store 22 | 23 | [unsafe] 24 | dumpPersistentStorage = false 25 | persistenceMode = fsync 26 | -------------------------------------------------------------------------------- /configfiles/fpaxos/client.conf: -------------------------------------------------------------------------------- 1 | [parameters] 2 | Timeout=500 3 | Backoff=10 4 | ConnectRandom=true 5 | BeforeForce=3 6 | Application = kv-store 7 | -------------------------------------------------------------------------------- /configfiles/fpaxos/server.conf: -------------------------------------------------------------------------------- 1 | ;; example description of Ios server 2 | 3 | [algorithm] 4 | delegateReplication = -1 5 | quorumSystem = strict majority 6 | indexExclusivity = true 7 | participantHandle = true 8 | participantRead = true 9 | implicitWindowCommit = true 10 | ExplicitCommit=true 11 | ThriftyQuorum=false 12 | 13 | [performance] 14 | length = 100000 15 | batchInterval = 0 16 | maxBatch = 100 17 | windowSize = 1 18 | snapshotInterval = 1000 19 | 20 | [application] 21 | name = kv-store 22 | 23 | [unsafe] 24 | dumpPersistentStorage = false 25 | persistenceMode = fsync 26 | -------------------------------------------------------------------------------- /configfiles/simple/client.conf: -------------------------------------------------------------------------------- 1 | [parameters] 2 | Timeout=500 3 | Backoff=10 4 | ConnectRandom=false 5 | BeforeForce=3 6 | Application = kv-store 7 | -------------------------------------------------------------------------------- /configfiles/simple/server.conf: -------------------------------------------------------------------------------- 1 | ;; example description of Ios server 2 | 3 | [algorithm] 4 | delegateReplication = 0 5 | quorumSystem = strict majority 6 | indexExclusivity = false 7 | participantHandle = false 8 | participantRead = false 9 | implicitWindowCommit = false 10 | ExplicitCommit=true 11 | ThriftyQuorum=false 12 | 13 | [performance] 14 | length = 100000 15 | batchInterval = 0 16 | maxBatch = 10 17 | windowSize = 1 18 | snapshotInterval = 0 19 | 20 | [application] 21 | name = kv-store 22 | 23 | [unsafe] 24 | dumpPersistentStorage = false 25 | persistenceMode = fsync 26 | -------------------------------------------------------------------------------- /consensus/consensus.go: -------------------------------------------------------------------------------- 1 | //Package consensus implements the core of the Ios consensus algorithm. 2 | //Package consensus is pure and does not perform any of its own IO operations such as writing to disk or sending packets. 3 | //It is uses msgs.Io for this purpose. 4 | package consensus 5 | 6 | import ( 7 | "github.com/golang/glog" 8 | "github.com/heidi-ann/ios/app" 9 | "github.com/heidi-ann/ios/msgs" 10 | ) 11 | 12 | type ConfigAll struct { 13 | ID int // id of node 14 | N int // size of cluster (nodes numbered 0 to N-1) 15 | WindowSize int // how many requests can the master have inflight at once 16 | Quorum QuorumSys // which quorum system to use 17 | } 18 | 19 | type ConfigMaster struct { 20 | BatchInterval int // how often to batch process request in ms, 0 means no batching 21 | MaxBatch int // maximum requests in a batch, unused if BatchInterval=0 22 | DelegateReplication int // how many replication coordinators to delegate to when master 23 | IndexExclusivity bool // if enabled, Ios will assign each index to at most one request 24 | } 25 | 26 | type ConfigCoordinator struct { 27 | ExplicitCommit bool // if enabled, Ios coordinators will send commit messages to all after replication 28 | ThriftyQuorum bool // if enabled, Ios coordinations will send writes to only a quorum (instead of broadcast) 29 | } 30 | 31 | type ConfigParticipant struct { 32 | SnapshotInterval int // how often to record state machine snapshots, 0 means snapshotting is disabled 33 | ImplicitWindowCommit bool // if enabled, then commit pending out-of-window requests 34 | LogLength int // max log size 35 | } 36 | 37 | type ConfigInterfacer struct { 38 | ParticipantHandle bool // if enabled, non-master nodes can handle to client requests 39 | ParticipantRead bool // if enabled, non-master nodes can serve reads. To enable, ParticipantHandle must also be enabled 40 | } 41 | 42 | // Config describes the static configuration of the consensus algorithm 43 | type Config struct { 44 | All ConfigAll 45 | Master ConfigMaster 46 | Coordinator ConfigCoordinator 47 | Participant ConfigParticipant 48 | Interfacer ConfigInterfacer 49 | } 50 | 51 | // state describes the current state of the consensus algorithm 52 | type state struct { 53 | View int // local view number (persistent) 54 | Log *Log // log entries, index from 0 (persistent) 55 | CommitIndex int // index of the last entry applied to the state machine, -1 means no entries have been applied yet 56 | masterID int // ID of the current master, calculated from View 57 | LastSnapshot int // index of the last state machine snapshot 58 | StateMachine *app.StateMachine // ref to current state machine 59 | Failures *msgs.FailureNotifier // ref to failure notifier to subscribe to failure notification 60 | Storage msgs.Storage // ref to persistent storage 61 | } 62 | 63 | // noop is a explicitly empty request 64 | var noop = msgs.ClientRequest{-1, -1, false, false, "noop"} 65 | 66 | // Init runs a fresh instance of the consensus algorithm. 67 | // The caller is requried to process Io requests using msgs.Io 68 | // It will not return until the application is terminated. 69 | func Init(peerNet *msgs.PeerNet, clientNet *msgs.ClientNet, config Config, app *app.StateMachine, fail *msgs.FailureNotifier, storage msgs.Storage) { 70 | 71 | // setup 72 | glog.Infof("Starting node ID:%d of %d", config.All.ID, config.All.N) 73 | state := state{ 74 | View: 0, 75 | Log: NewLog(config.Participant.LogLength), 76 | CommitIndex: -1, 77 | masterID: 0, 78 | LastSnapshot: 0, 79 | StateMachine: app, 80 | Failures: fail, 81 | Storage: storage} 82 | 83 | // write initial term to persistent storage 84 | // TODO: if not master then we need not wait until view has been fsynced 85 | err := storage.PersistView(0) 86 | if err != nil { 87 | glog.Fatal(err) 88 | } 89 | 90 | // operator as normal node 91 | glog.Info("Starting participant module, ID ", config.All.ID) 92 | go runCoordinator(&state, peerNet, config.All, config.Coordinator) 93 | go monitorMaster(&state, peerNet, config.All, config.Master, true) 94 | go runClientHandler(&state, peerNet, clientNet, config.All, config.Interfacer) 95 | runParticipant(&state, peerNet, clientNet, config.All, config.Participant) 96 | 97 | } 98 | 99 | // Recover restores an instance of the consensus algorithm. 100 | // The caller is requried to process Io requests using msgs.Io 101 | // It will not return until the application is terminated. 102 | func Recover(peerNet *msgs.PeerNet, clientNet *msgs.ClientNet, config Config, view int, log *Log, app *app.StateMachine, snapshotIndex int, fail *msgs.FailureNotifier, storage msgs.Storage) { 103 | // setup 104 | glog.Infof("Restarting node %d of %d with recovered log of length %d", config.All.ID, config.All.N, log.LastIndex) 105 | 106 | // restore previous state 107 | state := state{ 108 | View: view, 109 | Log: log, 110 | CommitIndex: snapshotIndex, 111 | masterID: mod(view, config.All.N), 112 | LastSnapshot: snapshotIndex, 113 | StateMachine: app, 114 | Failures: fail, 115 | Storage: storage} 116 | 117 | // apply recovered requests to state machine 118 | for i := snapshotIndex + 1; i <= state.Log.LastIndex; i++ { 119 | if !state.Log.GetEntry(i).Committed { 120 | break 121 | } 122 | state.CommitIndex = i 123 | 124 | for _, request := range state.Log.GetEntry(i).Requests { 125 | if request != noop { 126 | reply := state.StateMachine.Apply(request) 127 | clientNet.OutgoingResponses <- msgs.Client{request, reply} 128 | } 129 | } 130 | } 131 | glog.Info("Recovered ", state.CommitIndex+1, " committed entries") 132 | 133 | // do not start leader without view change 134 | 135 | // operator as normal node 136 | glog.Info("Starting participant module, ID ", config.All.ID) 137 | go runCoordinator(&state, peerNet, config.All, config.Coordinator) 138 | go monitorMaster(&state, peerNet, config.All, config.Master, false) 139 | go runClientHandler(&state, peerNet, clientNet, config.All, config.Interfacer) 140 | runParticipant(&state, peerNet, clientNet, config.All, config.Participant) 141 | } 142 | -------------------------------------------------------------------------------- /consensus/consensus_test.go: -------------------------------------------------------------------------------- 1 | package consensus 2 | 3 | import ( 4 | "flag" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | "github.com/heidi-ann/ios/app" 10 | "github.com/heidi-ann/ios/msgs" 11 | 12 | "github.com/golang/glog" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestInit(t *testing.T) { 17 | flag.Parse() 18 | defer glog.Flush() 19 | 20 | // create a node in system of 3 nodes 21 | peerNet := msgs.MakePeerNet(10, 3) 22 | clientNet := msgs.MakeClientNet(10) 23 | store := app.New("kv-store") 24 | quorum, err := NewQuorum("strict majority", 3) 25 | assert.Nil(t, err) 26 | config := Config{ 27 | All: ConfigAll{ 28 | ID: 0, 29 | N: 3, 30 | WindowSize: 1, 31 | Quorum: quorum, 32 | }, 33 | Master: ConfigMaster{ 34 | BatchInterval: 0, 35 | MaxBatch: 1, 36 | DelegateReplication: 0, 37 | IndexExclusivity: true, 38 | }, 39 | Participant: ConfigParticipant{ 40 | SnapshotInterval: 1000, 41 | ImplicitWindowCommit: true, 42 | LogLength: 10000, 43 | }, 44 | Interfacer: ConfigInterfacer{ 45 | ParticipantHandle: true, 46 | ParticipantRead: true, 47 | }, 48 | } 49 | failure := msgs.NewFailureNotifier(3) 50 | storage := msgs.MakeExternalStorage() 51 | go Init(peerNet, clientNet, config, store, failure, storage) 52 | 53 | // TEST 1 - SIMPLE COMMIT 54 | 55 | // tell node to prepare update A 3 56 | request1 := []msgs.ClientRequest{{ 57 | ClientID: 2, 58 | RequestID: 0, 59 | ForceViewChange: false, 60 | Request: "update A 3"}} 61 | 62 | entries1 := []msgs.Entry{{ 63 | View: 0, 64 | Committed: false, 65 | Requests: request1}} 66 | 67 | prepare1 := msgs.PrepareRequest{ 68 | SenderID: 0, 69 | View: 0, 70 | StartIndex: 0, 71 | EndIndex: 1, 72 | Entries: entries1} 73 | 74 | prepare1Res := msgs.PrepareResponse{ 75 | SenderID: 0, 76 | Success: true} 77 | 78 | // check view update is persisted 79 | select { 80 | case viewUpdate := <-storage.ViewPersist: 81 | if viewUpdate != 0 { 82 | t.Error(viewUpdate) 83 | } 84 | storage.ViewPersistFsync <- viewUpdate 85 | case <-time.After(time.Second): 86 | t.Error("Participant not responding") 87 | } 88 | 89 | peerNet.Incoming.Requests.Prepare <- prepare1 90 | 91 | // check node tried to dispatch request correctly 92 | select { 93 | case logUpdate := <-storage.LogPersist: 94 | if !reflect.DeepEqual(logUpdate.Entries, entries1) { 95 | t.Error(logUpdate) 96 | } 97 | storage.LogPersistFsync <- logUpdate 98 | case <-time.After(time.Second): 99 | t.Error("Participant not responding") 100 | } 101 | 102 | // check node tried to dispatch request correctly 103 | select { 104 | case reply := <-peerNet.OutgoingUnicast[0].Responses.Prepare: 105 | if reply.Response != prepare1Res { 106 | t.Error(reply) 107 | } 108 | case <-time.After(time.Second): 109 | t.Error("Participant not responding") 110 | } 111 | 112 | // tell node to commit update A 3 113 | entries1[0].Committed = true 114 | commit1 := msgs.CommitRequest{ 115 | SenderID: 0, 116 | ResponseRequired: true, 117 | StartIndex: 0, 118 | EndIndex: 1, 119 | Entries: entries1} 120 | 121 | commit1Res := msgs.CommitResponse{ 122 | SenderID: 0, 123 | Success: true, 124 | CommitIndex: 0} 125 | 126 | peerNet.Incoming.Requests.Commit <- commit1 127 | 128 | // check node replies correctly 129 | select { 130 | case reply := <-peerNet.OutgoingUnicast[0].Responses.Commit: 131 | if reply.Response != commit1Res { 132 | t.Error(reply) 133 | } 134 | case <-time.After(time.Second): 135 | t.Error("Participant not responding") 136 | } 137 | 138 | // check if update A 3 was committed to state machine 139 | 140 | select { 141 | case reply := <-clientNet.OutgoingResponses: 142 | if reply.Request != request1[0] { 143 | t.Error(reply) 144 | } 145 | case <-time.After(time.Second): 146 | t.Error("Participant not responding") 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /consensus/coordinator.go: -------------------------------------------------------------------------------- 1 | package consensus 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | "github.com/heidi-ann/ios/msgs" 6 | "reflect" 7 | ) 8 | 9 | func doCoordination(view int, startIndex int, endIndex int, entries []msgs.Entry, peerNet *msgs.PeerNet, 10 | config ConfigAll, preparePhase bool, commitPhase bool, thrifty bool, master int) bool { 11 | // PHASE 2: prepare 12 | if preparePhase { 13 | 14 | // check that committed is not set 15 | for i := 0; i < endIndex-startIndex; i++ { 16 | entries[i].Committed = false 17 | } 18 | 19 | prepare := msgs.PrepareRequest{config.ID, view, startIndex, endIndex, entries} 20 | glog.V(1).Info("Starting prepare phase", prepare) 21 | if thrifty { 22 | // send to random quorum only 23 | for _, id := range config.Quorum.getReplicationQuourm(config.ID, config.N) { 24 | peerNet.OutgoingUnicast[id].Requests.Prepare <- prepare 25 | } 26 | // BUG: retry if not successful 27 | } else { 28 | // broadcasting 29 | peerNet.OutgoingBroadcast.Requests.Prepare <- prepare 30 | } 31 | 32 | // collect responses 33 | glog.V(1).Info("Waiting for ", config.Quorum.ReplicationSize, " prepare responses") 34 | for replied := make([]bool, config.N); !config.Quorum.checkReplicationQuorum(replied); { 35 | msg := <-peerNet.Incoming.Responses.Prepare 36 | // check msg replies to the msg we just sent 37 | if reflect.DeepEqual(msg.Request, prepare) { 38 | glog.V(1).Info("Received ", msg) 39 | if !msg.Response.Success { 40 | glog.Warning("Coordinator is stepping down") 41 | return false 42 | } 43 | replied[msg.Response.SenderID] = true 44 | glog.V(1).Info("Successful response received, waiting for more") 45 | } 46 | } 47 | } 48 | 49 | // set committed so requests will be applied to state machines 50 | for i := 0; i < endIndex-startIndex; i++ { 51 | entries[i].Committed = true 52 | } 53 | 54 | // PHASE 3: commit 55 | // dispatch commit requests to all 56 | // TODO: add configuration option to set response required 57 | commit := msgs.CommitRequest{config.ID, false, startIndex, endIndex, entries} 58 | 59 | if commitPhase { 60 | glog.V(1).Info("Starting commit phase", commit) 61 | peerNet.OutgoingBroadcast.Requests.Commit <- commit 62 | return true 63 | } 64 | 65 | peerNet.OutgoingUnicast[master].Requests.Commit <- commit 66 | return true 67 | // TODO: handle replies properly 68 | // go func() { 69 | // for replied := make([]bool, config.N); !config.Quorum.checkReplicationQuorum(replied); { 70 | // msg := <-peerNet.Incoming.Responses.Commit 71 | // // check msg replies to the msg we just sent 72 | // if reflect.DeepEqual(msg.Request, commit) { 73 | // glog.V(1).Info("Received ", msg) 74 | // replied[msg.Response.SenderID] = true 75 | // } 76 | // } 77 | // }() 78 | } 79 | 80 | // runCoordinator eturns true if successful 81 | func runCoordinator(state *state, peerNet *msgs.PeerNet, config ConfigAll, configCoordinator ConfigCoordinator) { 82 | 83 | for { 84 | glog.V(1).Info("Coordinator is ready to handle request") 85 | req := <-peerNet.Incoming.Requests.Coordinate 86 | success := doCoordination(req.View, req.StartIndex, req.EndIndex, req.Entries, peerNet, config, 87 | req.Prepare, configCoordinator.ExplicitCommit, configCoordinator.ThriftyQuorum, req.SenderID) 88 | // TODO: BUG: check view 89 | reply := msgs.CoordinateResponse{config.ID, success} 90 | peerNet.OutgoingUnicast[req.SenderID].Responses.Coordinate <- msgs.Coordinate{req, reply} 91 | glog.V(1).Info("Coordinator is finished handling request") 92 | // TOD0: handle failure 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /consensus/interfacer.go: -------------------------------------------------------------------------------- 1 | package consensus 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | "github.com/heidi-ann/ios/msgs" 6 | "reflect" 7 | "time" 8 | ) 9 | 10 | // flush empties a Client request channel and returns a slice containing the requests 11 | // if channel is empty, block until non-empty 12 | func flush(incoming chan msgs.ClientRequest) []msgs.ClientRequest { 13 | // BUG: will overflow if more than 100 requests are waiting 14 | requests := make([]msgs.ClientRequest, 100) 15 | index := 0 16 | // block on first request 17 | requests[index] = <-incoming 18 | glog.V(1).Info("Read only request added ", requests[index]) 19 | index++ 20 | // fetch more if present 21 | for { 22 | select { 23 | case requests[index] = <-incoming: 24 | glog.V(1).Info("Read only request added ", requests[index]) 25 | index++ 26 | default: 27 | glog.V(1).Info("No more request waiting") 28 | return requests[:index] 29 | } 30 | } 31 | } 32 | 33 | // replace readded Client requests to channel 34 | func replace(incoming chan msgs.ClientRequest, requests []msgs.ClientRequest) { 35 | for _, req := range requests { 36 | incoming <- req 37 | } 38 | } 39 | 40 | // runReader takes read only requests from the incoming channels and applies them to the state machine 41 | // non-terminating 42 | // Channels used to ensure only one instance of runReader at a time 43 | func runReader(state *state, peerNet *msgs.PeerNet, clientNet *msgs.ClientNet, config ConfigAll, incoming chan msgs.ClientRequest) { 44 | for { 45 | // wait for readonly request 46 | requests := flush(incoming) 47 | glog.V(1).Info(len(requests), " read-only request received") 48 | 49 | // dispatch check request 50 | check := msgs.CheckRequest{config.ID, requests} 51 | peerNet.OutgoingBroadcast.Requests.Check <- check 52 | 53 | // collect responses 54 | success := make(chan []msgs.ClientResponse) 55 | failure := make(chan bool) 56 | go func() { 57 | glog.V(1).Info("Waiting for ", config.Quorum.RecoverySize, " successful check responses") 58 | var reply []msgs.ClientResponse // holds reply associated with commitIndex 59 | commitIndex := -2 // holds greatest commit index seen 60 | replies := config.N //number of responses minus N, successful or otherwise 61 | successful := make([]bool, config.N) //holds positive responses 62 | 63 | for { 64 | msg := <-peerNet.Incoming.Responses.Check 65 | // check msg replies to the msg we just sent 66 | if reflect.DeepEqual(msg.Request, check) { 67 | glog.V(1).Info("Received ", msg) 68 | if msg.Response.Success { 69 | successful[msg.Response.SenderID] = true 70 | glog.V(1).Info("Successful response received") 71 | if msg.Response.CommitIndex > commitIndex { 72 | commitIndex = msg.Response.CommitIndex 73 | reply = msg.Response.Replies 74 | } 75 | if config.Quorum.checkRecoveryQuorum(successful) { 76 | success <- reply 77 | break 78 | } 79 | } 80 | replies-- 81 | if replies == 0 { 82 | failure <- true 83 | break 84 | } 85 | } 86 | } 87 | }() 88 | 89 | // timeout or complete 90 | select { 91 | case reply := <-success: 92 | // dispatch replies 93 | for i := 0; i < len(requests); i++ { 94 | clientNet.OutgoingResponses <- msgs.Client{requests[i], reply[i]} 95 | glog.V(1).Info("Finished handling read-only requests", requests[i]) 96 | } 97 | case <-failure: 98 | replace(incoming, requests) 99 | glog.Warning("Check unsuccessful, will try again ") 100 | case <-time.After(time.Millisecond * 20): 101 | replace(incoming, requests) 102 | glog.Warning("Check unsuccessful due to timeout, will try again") 103 | } 104 | 105 | } 106 | } 107 | 108 | func runClientHandler(state *state, peerNet *msgs.PeerNet, clientNet *msgs.ClientNet, config ConfigAll, configInterfacer ConfigInterfacer) { 109 | glog.Info("Starting client handler") 110 | 111 | //setup readonly Handling 112 | readOnly := make(chan msgs.ClientRequest, 100) 113 | go runReader(state, peerNet, clientNet, config, readOnly) 114 | 115 | for { 116 | // wait for request 117 | req := <-clientNet.IncomingRequests 118 | if req.ForceViewChange { 119 | glog.Warning("Forcing view change received with ", req) 120 | peerNet.OutgoingUnicast[config.ID].Requests.Forward <- msgs.ForwardRequest{config.ID, state.View, req} 121 | } else { 122 | if configInterfacer.ParticipantHandle { 123 | if req.ReadOnly && configInterfacer.ParticipantRead { 124 | glog.V(1).Info("Request recieved, handling read locally ", req) 125 | readOnly <- req 126 | } else { 127 | glog.V(1).Info("Request received, forwarding to ", state.masterID, req) 128 | peerNet.OutgoingUnicast[state.masterID].Requests.Forward <- msgs.ForwardRequest{config.ID, state.View, req} 129 | } 130 | } else { 131 | if config.ID == state.masterID { 132 | glog.V(1).Info("Request received by master server ", req) 133 | peerNet.OutgoingUnicast[state.masterID].Requests.Forward <- msgs.ForwardRequest{config.ID, state.View, req} 134 | } else { 135 | glog.Info("Request received by non-master server and redirect enabled", req) 136 | clientNet.OutgoingRequestsFailed <- req 137 | } 138 | } 139 | } 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /consensus/log.go: -------------------------------------------------------------------------------- 1 | package consensus 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | "github.com/heidi-ann/ios/msgs" 6 | "reflect" 7 | ) 8 | 9 | // Log holds the replication log used of Ios. 10 | // Only indexes between AbsoluteIndex and (AbsoluteIndex + maxLength -1) are accessible 11 | // Do not access LogEntries direct, always use AddEntry and AddEntries functions 12 | type Log struct { 13 | LogEntries []msgs.Entry // contents of log, indexed from 0 to maxLength - 1 14 | LastIndex int // greatest absolute index in log with entry, -1 means that the log has no entries 15 | AbsoluteIndex int // absolute index of first index in log 16 | maxLength int // maximum length of LogEntries 17 | } 18 | 19 | // check protocol invariant 20 | func (l *Log) checkInvariants(index int, nxtEntry msgs.Entry) { 21 | prevEntry := l.LogEntries[index-l.AbsoluteIndex] 22 | // if no entry, then no problem 23 | if reflect.DeepEqual(prevEntry, msgs.Entry{}) { 24 | return 25 | } 26 | // if committed, request never changes 27 | if prevEntry.Committed && !reflect.DeepEqual(prevEntry.Requests, nxtEntry.Requests) { 28 | glog.Fatal("Committed entry is being overwritten at ", prevEntry, nxtEntry, index) 29 | } 30 | // each index is allocated once per view 31 | if prevEntry.View == nxtEntry.View && !reflect.DeepEqual(prevEntry.Requests, nxtEntry.Requests) { 32 | glog.Fatal("Index ", index, " has been reallocated from ", prevEntry, " to ", nxtEntry) 33 | } 34 | // entries should only be overwritten by same or higher view 35 | if prevEntry.View > nxtEntry.View { 36 | glog.Fatal("Attempting to overwrite entry with lower view", prevEntry, nxtEntry, index) 37 | } 38 | } 39 | 40 | func NewLog(maxLength int) *Log { 41 | return &Log{make([]msgs.Entry, maxLength), -1, 0, maxLength} 42 | } 43 | 44 | func RestoreLog(maxLength int, startIndex int) *Log { 45 | return &Log{make([]msgs.Entry, maxLength), startIndex, startIndex + 1, maxLength} 46 | } 47 | 48 | func (l *Log) AddEntries(startIndex int, endIndex int, entries []msgs.Entry) { 49 | // check correct number of entries has been given 50 | if len(entries) != endIndex-startIndex { 51 | glog.Fatal("Wrong number of log entries provided") 52 | } 53 | // check indexes are accessible 54 | if startIndex < l.AbsoluteIndex { 55 | return 56 | } 57 | // check indexes are accessible 58 | if endIndex > l.AbsoluteIndex+l.maxLength-1 { 59 | glog.Fatal("Log index is too large, please snapshot log") 60 | } 61 | // add entries and check invariants 62 | for i := 0; i < len(entries); i++ { 63 | l.checkInvariants(startIndex+i, entries[i]) 64 | l.LogEntries[startIndex+i-l.AbsoluteIndex] = entries[i] 65 | } 66 | // update LastIndex 67 | if endIndex-1 > l.LastIndex { 68 | l.LastIndex = endIndex - 1 69 | } 70 | } 71 | 72 | func (l *Log) AddEntry(index int, entry msgs.Entry) { 73 | l.AddEntries(index, index+1, []msgs.Entry{entry}) 74 | } 75 | 76 | func (l *Log) GetEntries(startIndex int, endIndex int) []msgs.Entry { 77 | // check indexes are accessible 78 | if startIndex < l.AbsoluteIndex || endIndex > l.AbsoluteIndex+l.maxLength-1 { 79 | glog.Fatal("Trying to access log out of bounds") 80 | } 81 | // return no entries if range is incorrect 82 | if startIndex > endIndex { 83 | return []msgs.Entry{} 84 | } 85 | return l.LogEntries[startIndex-l.AbsoluteIndex : endIndex-l.AbsoluteIndex] 86 | } 87 | 88 | func (l *Log) GetEntriesFrom(startIndex int) []msgs.Entry { 89 | // check indexes are accessible 90 | if startIndex < l.AbsoluteIndex { 91 | glog.Fatal("Trying to access log out of bounds") 92 | } 93 | return l.LogEntries[startIndex-l.AbsoluteIndex : l.LastIndex-l.AbsoluteIndex] 94 | } 95 | 96 | func (l *Log) GetEntry(index int) msgs.Entry { 97 | // check indexes are accessible 98 | if index < l.AbsoluteIndex || index > l.AbsoluteIndex+l.maxLength-1 { 99 | glog.Fatal("Trying to access log out of bounds") 100 | } 101 | return l.LogEntries[index-l.AbsoluteIndex] 102 | } 103 | 104 | // ImplicitCommit will marked any uncommitted entries after commitIndex as committed 105 | // if they are out-of-window and of the same view 106 | func (l *Log) ImplicitCommit(windowSize int, commitIndex int) { 107 | view := l.GetEntry(l.LastIndex).View 108 | for i, entry := range l.GetEntries(commitIndex+1, l.LastIndex-windowSize) { 109 | if !entry.Committed && entry.View == view { 110 | entry.Committed = true 111 | l.AddEntry(i+l.AbsoluteIndex, entry) 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /consensus/master.go: -------------------------------------------------------------------------------- 1 | package consensus 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | "github.com/heidi-ann/ios/msgs" 6 | "time" 7 | ) 8 | 9 | func monitorMaster(s *state, peerNet *msgs.PeerNet, config ConfigAll, configMaster ConfigMaster, new bool) { 10 | 11 | // if initial master, start master goroutine 12 | if config.ID == 0 && new { 13 | glog.Info("Starting leader module") 14 | runMaster(0, -1, true, peerNet, config, configMaster, s) 15 | } 16 | 17 | // if only node, start master 18 | if config.N == 1 { 19 | s.View++ 20 | err := s.Storage.PersistView(s.View) 21 | if err != nil { 22 | glog.Fatal(err) 23 | } 24 | s.masterID = config.ID 25 | runMaster(s.View, s.CommitIndex, false, peerNet, config, configMaster, s) 26 | } 27 | 28 | for { 29 | select { 30 | case <-s.Failures.NotifyOnFailure(s.masterID): 31 | nextMaster := mod(s.View+1, config.N) 32 | glog.Warningf("Master (ID:%d,View:%d) failed, next up is ID:%d in View:%d", s.masterID, s.View, nextMaster, s.View+1) 33 | s.masterID = nextMaster 34 | s.View++ 35 | if nextMaster == config.ID { 36 | s.View++ 37 | glog.V(1).Info("Starting new master in view ", s.View, " at ", config.ID) 38 | err := s.Storage.PersistView(s.View) 39 | if err != nil { 40 | glog.Fatal(err) 41 | } 42 | s.masterID = nextMaster 43 | runMaster(s.View, s.CommitIndex, false, peerNet, config, configMaster, s) 44 | } 45 | 46 | case req := <-peerNet.Incoming.Requests.Forward: 47 | glog.Warning("Request received by non-master server ", req) 48 | if req.Request.ForceViewChange { 49 | glog.Warning("Forcing view change") 50 | s.View = next(s.View, config.ID, config.N) 51 | err := s.Storage.PersistView(s.View) 52 | if err != nil { 53 | glog.Fatal(err) 54 | } 55 | s.masterID = config.ID 56 | req.Request.ForceViewChange = false 57 | peerNet.Incoming.Requests.Forward <- req 58 | runMaster(s.View, s.CommitIndex, false, peerNet, config, configMaster, s) 59 | } 60 | } 61 | } 62 | } 63 | 64 | // runRecovery executes the recovery phase of leadership election, 65 | // Returns if it was successful and the previous view's end index 66 | func runRecovery(view int, commitIndex int, peerNet *msgs.PeerNet, config ConfigAll, indexExclusivity bool) (bool, int) { 67 | // dispatch new view requests 68 | req := msgs.NewViewRequest{config.ID, view} 69 | peerNet.OutgoingBroadcast.Requests.NewView <- req 70 | 71 | // collect responses 72 | glog.Info("Waiting for ", config.Quorum.RecoverySize, " new view responses") 73 | endIndex := commitIndex 74 | 75 | for replied := make([]bool, config.N); !config.Quorum.checkRecoveryQuorum(replied); { 76 | msg := <-peerNet.Incoming.Responses.NewView 77 | // check msg replies to the msg we just sent 78 | if msg.Request == req { 79 | res := msg.Response 80 | if msg.Response.View != view { 81 | glog.Warning("New view failed, stepping down from master") 82 | return false, 0 83 | } 84 | glog.V(1).Info("Received ", res) 85 | if res.Index > endIndex { 86 | endIndex = res.Index 87 | } 88 | replied[msg.Response.SenderID] = true 89 | glog.V(1).Info("Successful new view received, waiting for more") 90 | } 91 | } 92 | 93 | glog.Info("End index of the previous views is ", endIndex) 94 | startIndex := endIndex 95 | if indexExclusivity { 96 | startIndex += config.WindowSize 97 | } 98 | glog.Info("Start index of view ", view, " will be ", startIndex) 99 | 100 | if commitIndex+1 == startIndex { 101 | glog.Info("New master is up to date, No recovery coordination is required") 102 | return true, startIndex 103 | } 104 | 105 | // recover entries 106 | result := runRecoveryCoordinator(view, commitIndex+1, startIndex+1, peerNet, config) 107 | return result, startIndex 108 | } 109 | 110 | // runMaster implements the Master mode 111 | func runMaster(view int, commitIndex int, initial bool, peerNet *msgs.PeerNet, config ConfigAll, configMaster ConfigMaster, s *state) { 112 | // setup 113 | glog.Info("Starting up master in view ", view) 114 | glog.Info("Master is configured to delegate replication to ", configMaster.DelegateReplication) 115 | s.masterID = config.ID 116 | 117 | // determine next safe index 118 | startIndex := -1 119 | 120 | if !initial { 121 | var success bool 122 | success, startIndex = runRecovery(view, commitIndex, peerNet, config, configMaster.IndexExclusivity) 123 | if !success { 124 | glog.Warning("Recovery failed") 125 | return 126 | } 127 | } 128 | 129 | coordinator := config.ID 130 | 131 | // if delegation is enabled then store the first coordinator to ask 132 | if configMaster.DelegateReplication > 0 { 133 | coordinator = s.Failures.NextConnected(config.ID) 134 | } 135 | window := newReplicationWindow(startIndex, config.WindowSize) 136 | stepDown := false 137 | 138 | for { 139 | 140 | if stepDown { 141 | glog.Warning("Master stepping down due to coordinator step down") 142 | break 143 | } 144 | 145 | glog.V(1).Info("Ready to handle request") 146 | forwarded := <-peerNet.Incoming.Requests.Forward 147 | 148 | glog.V(1).Info("Request received: ", forwarded.Request, " Received from ", forwarded.SenderID) 149 | var reqs []msgs.ClientRequest 150 | 151 | //wait for window slot 152 | index := window.nextIndex() 153 | 154 | if configMaster.BatchInterval == 0 || configMaster.MaxBatch == 1 { 155 | glog.V(1).Info("No batching enabled") 156 | // handle client requests (1 at a time) 157 | // setup for holding requests 158 | reqsAll := make([]msgs.ClientRequest, configMaster.MaxBatch) 159 | reqsNum := 1 160 | reqsAll[0] = forwarded.Request 161 | exit := false 162 | 163 | if configMaster.MaxBatch == 1 { 164 | exit = true 165 | } 166 | 167 | for exit == false { 168 | select { 169 | case nextForwarded := <-peerNet.Incoming.Requests.Forward: 170 | reqsAll[reqsNum] = nextForwarded.Request 171 | glog.V(1).Info("Request ", reqsNum, " is : ", nextForwarded.Request) 172 | reqsNum = reqsNum + 1 173 | if reqsNum == configMaster.MaxBatch { 174 | exit = true 175 | break 176 | } 177 | default: 178 | exit = true 179 | break 180 | } 181 | } 182 | // this batch is ready 183 | glog.V(1).Info("Starting to replicate ", reqsNum, " requests") 184 | reqs = reqsAll[:reqsNum] 185 | } else { 186 | glog.V(1).Info("Ready to handle more requests. Batch every ", configMaster.BatchInterval, " milliseconds") 187 | // setup for holding requests 188 | reqsAll := make([]msgs.ClientRequest, configMaster.MaxBatch) 189 | reqsNum := 1 190 | reqsAll[0] = forwarded.Request 191 | 192 | exit := false 193 | for exit == false { 194 | select { 195 | case nextForwarded := <-peerNet.Incoming.Requests.Forward: 196 | reqsAll[reqsNum] = nextForwarded.Request 197 | glog.V(1).Info("Request ", reqsNum, " is : ", nextForwarded.Request) 198 | reqsNum = reqsNum + 1 199 | if reqsNum == configMaster.MaxBatch { 200 | exit = true 201 | break 202 | } 203 | case <-time.After(time.Millisecond * time.Duration(configMaster.BatchInterval)): 204 | exit = true 205 | break 206 | } 207 | } 208 | // this batch is ready 209 | glog.V(1).Info("Starting to replicate ", reqsNum, " requests") 210 | reqs = reqsAll[:reqsNum] 211 | } 212 | glog.V(1).Info("Request assigned index: ", index) 213 | 214 | // if reverse delegation is enabled then assign to node who forwarded request 215 | if configMaster.DelegateReplication == -1 { 216 | coordinator = forwarded.SenderID 217 | } 218 | 219 | // dispatch to coordinator 220 | entries := []msgs.Entry{{view, false, reqs}} 221 | coord := msgs.CoordinateRequest{config.ID, view, index, index + 1, true, entries} 222 | peerNet.OutgoingUnicast[coordinator].Requests.Coordinate <- coord 223 | // TODO: BUG: need to handle coordinator failure 224 | 225 | go func() { 226 | reply := <-peerNet.Incoming.Responses.Coordinate 227 | // TODO: check msg replies to the msg we just sent 228 | if !reply.Response.Success { 229 | glog.Warning("Commit unsuccessful") 230 | stepDown = true 231 | return 232 | } 233 | glog.V(1).Info("Finished replicating request: ", reqs) 234 | window.indexCompleted(reply.Request.StartIndex) 235 | }() 236 | 237 | // rotate coordinator is nessacary 238 | if configMaster.DelegateReplication > 1 { 239 | coordinator = s.Failures.NextConnected(coordinator) 240 | } 241 | } 242 | glog.Warning("Master stepping down") 243 | 244 | } 245 | -------------------------------------------------------------------------------- /consensus/participant.go: -------------------------------------------------------------------------------- 1 | package consensus 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | "github.com/heidi-ann/ios/msgs" 6 | ) 7 | 8 | // PROTOCOL BODY 9 | func runParticipant(state *state, peerNet *msgs.PeerNet, clientNet *msgs.ClientNet, config ConfigAll, configParticipant ConfigParticipant) { 10 | glog.V(1).Info("Ready for requests") 11 | for { 12 | 13 | // get request 14 | select { 15 | 16 | case req := <-peerNet.Incoming.Requests.Prepare: 17 | glog.V(1).Info("Prepare requests received at ", config.ID, ": ", req) 18 | // check view 19 | if req.View < state.View { 20 | glog.Warning("Sender ID:", req.SenderID, " is behind. Local view is ", state.View, ", sender's view was ", req.View) 21 | reply := msgs.PrepareResponse{config.ID, false} 22 | peerNet.OutgoingUnicast[req.SenderID].Responses.Prepare <- msgs.Prepare{req, reply} 23 | break 24 | } 25 | 26 | if req.View > state.View { 27 | glog.Warning("Participant is behind") 28 | state.View = req.View 29 | err := state.Storage.PersistView(state.View) 30 | if err != nil { 31 | glog.Fatal(err) 32 | } 33 | state.masterID = mod(state.View, config.N) 34 | } 35 | 36 | // add enties to the log (in-memory) 37 | state.Log.AddEntries(req.StartIndex, req.EndIndex, req.Entries) 38 | // add entries to the log (persistent Storage) 39 | logUpdate := msgs.LogUpdate{req.StartIndex, req.EndIndex, req.Entries} 40 | err := state.Storage.PersistLogUpdate(logUpdate) 41 | if err != nil { 42 | glog.Fatal(err) 43 | } 44 | 45 | // implicit commits from window_size 46 | if configParticipant.ImplicitWindowCommit { 47 | state.Log.ImplicitCommit(config.WindowSize, state.CommitIndex) 48 | // pass requests to state machine if ready 49 | for state.Log.GetEntry(state.CommitIndex + 1).Committed { 50 | for _, request := range state.Log.GetEntry(state.CommitIndex + 1).Requests { 51 | if request != noop { 52 | reply := state.StateMachine.Apply(request) 53 | clientNet.OutgoingResponses <- msgs.Client{request, reply} 54 | glog.V(1).Info("Request Committed: ", request) 55 | } 56 | } 57 | state.CommitIndex++ 58 | } 59 | } 60 | 61 | // reply to coordinator 62 | reply := msgs.PrepareResponse{config.ID, true} 63 | (peerNet.OutgoingUnicast[req.SenderID]).Responses.Prepare <- msgs.Prepare{req, reply} 64 | glog.V(1).Info("Response dispatched: ", reply) 65 | 66 | case req := <-peerNet.Incoming.Requests.Commit: 67 | glog.V(1).Info("Commit requests received at ", config.ID, ": ", req) 68 | 69 | // add enties to the log (in-memory) 70 | state.Log.AddEntries(req.StartIndex, req.EndIndex, req.Entries) 71 | if configParticipant.ImplicitWindowCommit { 72 | state.Log.ImplicitCommit(config.WindowSize, state.CommitIndex) 73 | } 74 | //peerNet.LogPersist <- msgs.LogUpdate{req.StartIndex, req.EndIndex, req.Entries, false} 75 | 76 | // pass requests to state machine if ready 77 | for state.Log.GetEntry(state.CommitIndex + 1).Committed { 78 | for _, request := range state.Log.GetEntry(state.CommitIndex + 1).Requests { 79 | if request != noop { 80 | reply := state.StateMachine.Apply(request) 81 | clientNet.OutgoingResponses <- msgs.Client{request, reply} 82 | glog.V(1).Info("Request Committed: ", request) 83 | } 84 | } 85 | state.CommitIndex++ 86 | } 87 | 88 | // if blocked on request out-of-window send a copy to solicit a commit 89 | if state.CommitIndex < state.Log.LastIndex-config.WindowSize && config.N > 1 { 90 | peerID := randPeer(config.N, config.ID) 91 | peerNet.OutgoingUnicast[peerID].Requests.Copy <- msgs.CopyRequest{config.ID, state.CommitIndex + 1} 92 | 93 | } 94 | 95 | // check if its time for another snapshot 96 | if configParticipant.SnapshotInterval != 0 && 97 | state.LastSnapshot+configParticipant.SnapshotInterval <= state.CommitIndex { 98 | snap, err := state.StateMachine.MakeSnapshot() 99 | if err != nil { 100 | glog.Fatal(err) 101 | } 102 | err = state.Storage.PersistSnapshot(state.CommitIndex, snap) 103 | if err != nil { 104 | glog.Fatal(err) 105 | } 106 | state.LastSnapshot = state.CommitIndex 107 | } 108 | 109 | // reply to coordinator if required 110 | if req.ResponseRequired { 111 | reply := msgs.CommitResponse{config.ID, true, state.CommitIndex} 112 | (peerNet.OutgoingUnicast[req.SenderID]).Responses.Commit <- msgs.Commit{req, reply} 113 | glog.V(1).Info("Commit response dispatched") 114 | } 115 | 116 | case req := <-peerNet.Incoming.Requests.NewView: 117 | glog.Info("New view requests received at ", config.ID, ": ", req) 118 | 119 | // check view 120 | if req.View < state.View { 121 | glog.Warning("Sender of NewView is behind, message view ", req.View, " local view is ", state.View) 122 | } 123 | 124 | if req.View > state.View { 125 | glog.Warning("Participant is behind") 126 | state.View = req.View 127 | err := state.Storage.PersistView(state.View) 128 | if err != nil { 129 | glog.Fatal(err) 130 | } 131 | state.masterID = mod(state.View, config.N) 132 | } 133 | 134 | reply := msgs.NewViewResponse{config.ID, state.View, state.Log.LastIndex} 135 | peerNet.OutgoingUnicast[req.SenderID].Responses.NewView <- msgs.NewView{req, reply} 136 | glog.Info("Response dispatched") 137 | 138 | case req := <-peerNet.Incoming.Requests.Query: 139 | glog.V(1).Info("Query requests received at ", config.ID, ": ", req) 140 | 141 | // check view 142 | if req.View < state.View { 143 | glog.Warning("Sender is behind") 144 | break 145 | 146 | } 147 | 148 | if req.View > state.View { 149 | glog.Warning("Participant is behind") 150 | state.View = req.View 151 | err := state.Storage.PersistView(state.View) 152 | if err != nil { 153 | glog.Fatal(err) 154 | } 155 | state.masterID = mod(state.View, config.N) 156 | } 157 | 158 | reply := msgs.QueryResponse{config.ID, state.View, state.Log.GetEntries(req.StartIndex, req.EndIndex)} 159 | peerNet.OutgoingUnicast[req.SenderID].Responses.Query <- msgs.Query{req, reply} 160 | 161 | case req := <-peerNet.Incoming.Requests.Copy: 162 | glog.V(1).Info("Copy requests received at ", config.ID, ": ", req) 163 | if state.CommitIndex > req.StartIndex { 164 | reply := msgs.CommitRequest{config.ID, false, req.StartIndex, state.CommitIndex, state.Log.GetEntries(req.StartIndex, state.CommitIndex)} 165 | peerNet.OutgoingUnicast[req.SenderID].Requests.Commit <- reply 166 | } 167 | 168 | case req := <-peerNet.Incoming.Requests.Check: 169 | glog.V(1).Info("Check requests received at ", config.ID) 170 | reply := msgs.CheckResponse{config.ID, 171 | state.CommitIndex == state.Log.LastIndex, 172 | state.CommitIndex, 173 | state.StateMachine.ApplyReads(req.Requests), 174 | } 175 | peerNet.OutgoingUnicast[req.SenderID].Responses.Check <- msgs.Check{req, reply} 176 | } 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /consensus/quourm.go: -------------------------------------------------------------------------------- 1 | package consensus 2 | 3 | import ( 4 | "errors" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | // QuorumSys offers basic suport for various of quorum systems 10 | // currently only "counting systems" are supported 11 | type QuorumSys struct { 12 | Name string 13 | RecoverySize int 14 | ReplicationSize int 15 | } 16 | 17 | func NewQuorum(configName string, n int) (QuorumSys, error) { 18 | var qs QuorumSys 19 | qs.Name = "counting" // currently only "counting systems" are supported 20 | qType := strings.Split(configName, ":") 21 | // note that golang / will truncate 22 | switch qType[0] { 23 | case "strict majority": 24 | qs.ReplicationSize = (n / 2) + 1 25 | qs.RecoverySize = (n / 2) + 1 26 | case "non-strict majority": 27 | qs.ReplicationSize = (n + 1) / 2 28 | qs.RecoverySize = (n / 2) + 1 29 | case "all-in": 30 | qs.ReplicationSize = n 31 | qs.RecoverySize = 1 32 | case "one-in": 33 | qs.ReplicationSize = 1 34 | qs.RecoverySize = n 35 | case "fixed": 36 | i, err := strconv.Atoi(qType[1]) 37 | if err != nil { 38 | return qs, errors.New("Quourm system is not recognised") 39 | } 40 | qs.ReplicationSize = i 41 | qs.RecoverySize = n + 1 - i 42 | default: 43 | return qs, errors.New("Quourm system is not recognised") 44 | } 45 | if qs.RecoverySize+qs.ReplicationSize <= n { 46 | return qs, errors.New("Unsafe quorum system has been chosen") 47 | } 48 | return qs, nil 49 | } 50 | 51 | func (q QuorumSys) checkReplicationQuorum(nodes []bool) bool { 52 | // count responses 53 | count := 0 54 | for _, node := range nodes { 55 | if node { 56 | count++ 57 | } 58 | } 59 | // check if responses are sufficient 60 | return count >= q.ReplicationSize 61 | } 62 | 63 | func (q QuorumSys) getReplicationQuourm(id int, n int) []int { 64 | quorum := make([]int, q.ReplicationSize) 65 | // TODO: consider replacing with random quorums 66 | for i := 0; i < q.ReplicationSize; i++ { 67 | quorum[i] = mod(i+id, n) 68 | } 69 | return quorum 70 | } 71 | 72 | func (q QuorumSys) checkRecoveryQuorum(nodes []bool) bool { 73 | // count responses 74 | count := 0 75 | for _, node := range nodes { 76 | if node { 77 | count++ 78 | } 79 | } 80 | // check if responses are sufficient 81 | return count >= q.RecoverySize 82 | } 83 | -------------------------------------------------------------------------------- /consensus/recovery_coordinator.go: -------------------------------------------------------------------------------- 1 | package consensus 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | "github.com/heidi-ann/ios/msgs" 6 | "reflect" 7 | ) 8 | 9 | // returns true if successful 10 | // start index is inclusive and end index is exclusive 11 | func runRecoveryCoordinator(view int, startIndex int, endIndex int, peerNet *msgs.PeerNet, config ConfigAll) bool { 12 | if startIndex == endIndex { 13 | return true 14 | } else if endIndex < startIndex { 15 | glog.Fatal("Invalid recovery range ", startIndex, endIndex) 16 | } 17 | glog.Info("Starting recovery for indexes ", startIndex, " to ", endIndex) 18 | 19 | // dispatch query to all 20 | query := msgs.QueryRequest{config.ID, view, startIndex, endIndex} 21 | peerNet.OutgoingBroadcast.Requests.Query <- query 22 | 23 | // collect responses 24 | noopEntry := msgs.Entry{0, false, []msgs.ClientRequest{noop}} 25 | candidates := make([]msgs.Entry, endIndex-startIndex) 26 | for i := 0; i < endIndex-startIndex; i++ { 27 | candidates[i] = noopEntry 28 | } 29 | 30 | //check only one response is received per sender, index= node ID 31 | 32 | for replied := make([]bool, config.N); !config.Quorum.checkRecoveryQuorum(replied); { 33 | msg := <-peerNet.Incoming.Responses.Query 34 | if msg.Request == query { 35 | 36 | // check this is not a duplicate 37 | if replied[msg.Response.SenderID] { 38 | glog.Warning("Response already received from ", msg.Response.SenderID) 39 | } else { 40 | // check view 41 | if msg.Response.View < view { 42 | glog.Fatal("Reply view is < current view, this should not have occurred") 43 | } 44 | 45 | if view < msg.Response.View { 46 | glog.Warning("Stepping down from recovery coordinator") 47 | return false 48 | } 49 | 50 | res := msg.Response 51 | replied[msg.Response.SenderID] = true 52 | 53 | for i := 0; i < endIndex-startIndex; i++ { 54 | if !reflect.DeepEqual(res.Entries[i], msgs.Entry{}) { 55 | // if committed, then done 56 | if res.Entries[i].Committed { 57 | candidates[i] = res.Entries[i] 58 | // TODO: add early exit here 59 | } 60 | 61 | // if first entry, then new candidate 62 | if reflect.DeepEqual(candidates[i], noopEntry) { 63 | candidates[i] = res.Entries[i] 64 | } 65 | 66 | // if higher view then candidate then new candidate 67 | if res.Entries[i].View > candidates[i].View { 68 | candidates[i] = res.Entries[i] 69 | } 70 | 71 | // if same view and different requests then panic! 72 | if res.Entries[i].View == candidates[i].View && !reflect.DeepEqual(res.Entries[i].Requests, candidates[i].Requests) { 73 | glog.Fatal("Same index has been issued more then once", res.Entries[i].Requests, candidates[i].Requests) 74 | } 75 | } else { 76 | glog.V(1).Info("Log entry at index ", i, " on node ID ", msg.Response.SenderID, " is missing") 77 | } 78 | } 79 | } 80 | } 81 | } 82 | glog.Info("New view phase is finished") 83 | 84 | // set the next view and marked as uncommitted 85 | // TODO: add shortcut to skip prepare phase is entries are already committed. 86 | for i := 0; i < endIndex-startIndex; i++ { 87 | candidates[i] = msgs.Entry{view, false, candidates[i].Requests} 88 | } 89 | 90 | coord := msgs.CoordinateRequest{config.ID, view, startIndex, endIndex, true, candidates} 91 | peerNet.OutgoingUnicast[config.ID].Requests.Coordinate <- coord 92 | <-peerNet.Incoming.Responses.Coordinate 93 | // TODO: check msg replies to the msg we just sent 94 | 95 | glog.Info("Recovery completed for indexes ", startIndex, " to ", endIndex) 96 | return true 97 | } 98 | -------------------------------------------------------------------------------- /consensus/utils.go: -------------------------------------------------------------------------------- 1 | package consensus 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | "math/rand" 6 | ) 7 | 8 | func mod(x int, y int) int { 9 | if x < y { 10 | return x 11 | } 12 | dif := x - y 13 | if dif < y { 14 | return dif 15 | } 16 | return mod(dif, y) 17 | } 18 | 19 | func next(view int, id int, n int) int { 20 | round := view / n 21 | return (round+1)*n + id 22 | } 23 | 24 | func randPeer(n int, id int) int { 25 | if n == 1 { 26 | glog.Fatal("No peers present") 27 | } 28 | c := rand.Intn(n) 29 | for c == id { 30 | c = rand.Intn(n) 31 | } 32 | return c 33 | } 34 | -------------------------------------------------------------------------------- /consensus/window.go: -------------------------------------------------------------------------------- 1 | package consensus 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/golang/glog" 7 | ) 8 | 9 | type rwindow struct { 10 | outstanding []bool // outstanding holds in progress request indexes 11 | ready chan int // indexes that can be allocated 12 | windowStart int // the last committed entry, window is from windowStart+1 to windowStart+windowSize 13 | windowSize int // limit on the size of the window 14 | sync.RWMutex // lock for concurrent access to outstanding 15 | } 16 | 17 | func newReplicationWindow(startIndex int, windowSize int) *rwindow { 18 | outstanding := make([]bool, windowSize) 19 | ready := make(chan int, windowSize) 20 | // preload ready with token which are ready 21 | for i := startIndex + 1; i <= startIndex+windowSize; i++ { 22 | ready <- i 23 | outstanding[i%windowSize] = true 24 | } 25 | return &rwindow{outstanding, ready, startIndex, windowSize, sync.RWMutex{}} 26 | } 27 | 28 | func (rw *rwindow) nextIndex() int { 29 | index := <-rw.ready 30 | glog.V(1).Info("Allocating index ", index) 31 | return index 32 | } 33 | 34 | func (rw *rwindow) indexCompleted(index int) { 35 | // remove from outstanding 36 | rw.Lock() 37 | glog.V(1).Info("marking index no longer outstanding: ", index%rw.windowSize) 38 | glog.V(1).Info("Window start is: ", rw.windowStart) 39 | rw.outstanding[index%rw.windowSize] = false 40 | 41 | // check if we can advance the windowStart 42 | // if so, indexes can be loaded into ready 43 | for !rw.outstanding[index%rw.windowSize] && (index%rw.windowSize == (rw.windowStart+1)%rw.windowSize) { 44 | glog.V(1).Info("Moving window") 45 | rw.windowStart++ 46 | glog.V(1).Info("marking index as outstanding: ", (rw.windowStart+rw.windowSize)%rw.windowSize) 47 | rw.outstanding[(rw.windowStart+rw.windowSize)%rw.windowSize] = true 48 | rw.ready <- rw.windowStart + rw.windowSize 49 | index++ 50 | } 51 | 52 | rw.Unlock() 53 | } 54 | -------------------------------------------------------------------------------- /consensus/window_test.go: -------------------------------------------------------------------------------- 1 | package consensus 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/golang/glog" 8 | ) 9 | 10 | func TestNextIndex(t *testing.T) { 11 | glog.Info("Starting window test 0") 12 | window := newReplicationWindow(-1, 1) 13 | index := window.nextIndex() 14 | if index != 0 { 15 | t.Error("ReplicationWindow giving wrong index") 16 | } 17 | window.indexCompleted(index) 18 | glog.Info("Starting window test 1") 19 | index2 := window.nextIndex() 20 | if index2 != 1 { 21 | t.Error("ReplicationWindow giving wrong index", index2) 22 | } 23 | } 24 | 25 | func TestOutOfOrder(t *testing.T) { 26 | glog.Info("Starting out of order window test") 27 | window := newReplicationWindow(-1, 5) 28 | 29 | // Get first 5 indices 30 | for i := 0; i < 5; i++ { 31 | index := window.nextIndex() 32 | if index != i { 33 | t.Error(fmt.Printf("Unexpected index at %v", i)) 34 | } 35 | } 36 | if window.windowStart != -1 { 37 | t.Error("Window has moved before any indices marked completed") 38 | } 39 | //Mark index 1 as completed, window should not have moved 40 | window.indexCompleted(1) 41 | if window.windowStart != -1 { 42 | t.Error("Window has moved before first index marked completed") 43 | } 44 | //Mark index 2 as completed, window should not have moved 45 | window.indexCompleted(2) 46 | if window.windowStart != -1 { 47 | t.Error("Window has moved before first index marked completed") 48 | } 49 | 50 | //Mark first index completed, window should move 3 51 | window.indexCompleted(0) 52 | if window.windowStart != 2 { 53 | t.Error(fmt.Printf("Window has not moved to expected position. Actual position: %v", window.windowStart)) 54 | } 55 | } 56 | 57 | func TestWrapAroundWindowSize(t *testing.T) { 58 | glog.Info("Starting enough requests to wrap around array") 59 | window := newReplicationWindow(-1, 5) 60 | 61 | for i := 0; i < 100; i++ { 62 | index := window.nextIndex() 63 | if index != i { 64 | t.Fatal(fmt.Printf("Unexpected index at %v", i)) 65 | } 66 | window.indexCompleted(i) 67 | if window.windowStart != i { 68 | t.Fatal(fmt.Printf("Window has not moved to expected position. Actual position: %v, Expected Position: %v", window.windowStart, i)) 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Welcome to Ios, a distributed and strongly consistent key-value store, 2 | // built upon a novel delegated and decentralised consensus protocol. 3 | 4 | // This repository is pre-alpha and under active development. APIs will be broken. 5 | 6 | package ios 7 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | networks: 3 | testnet: 4 | driver: bridge 5 | ipam: 6 | config: 7 | - subnet: 192.168.1.0/16 8 | services: 9 | node0: 10 | image: ios 11 | ports: 12 | - 8080:8080 13 | entrypoint: ["ios","-listen-peers=8090","-listen-clients=8080","-id=0","-config=src/github.com/heidi-ann/ios/scripts/docker/example3.conf","-logtostderr=true"] 14 | networks: 15 | testnet: 16 | ipv4_address: 192.168.1.1 17 | node1: 18 | image: ios 19 | ports: 20 | - 8081:8080 21 | entrypoint: ios -id 1 -listen-peers 8090 -listen-clients 8080 -config src/github.com/heidi-ann/ios/scripts/docker/example3.conf -logtostderr=true 22 | networks: 23 | testnet: 24 | ipv4_address: 192.168.1.2 25 | node2: 26 | image: ios 27 | ports: 28 | - 8082:8080 29 | entrypoint: ios -id 2 -listen-peers 8090 -listen-clients 8080 -config src/github.com/heidi-ann/ios/scripts/docker/example3.conf -logtostderr=true 30 | networks: 31 | testnet: 32 | ipv4_address: 192.168.1.3 33 | -------------------------------------------------------------------------------- /example.conf: -------------------------------------------------------------------------------- 1 | ; servers to try connecting to, in order 2 | [servers] 3 | address = 127.0.0.1:8090:8080 4 | -------------------------------------------------------------------------------- /example3.conf: -------------------------------------------------------------------------------- 1 | ; servers to try connecting to, in order 2 | [servers] 3 | address = 127.0.0.1:8090:8080 4 | address = 127.0.0.1:8091:8081 5 | address = 127.0.0.1:8092:8082 6 | -------------------------------------------------------------------------------- /gateway/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/golang/glog" 6 | "github.com/heidi-ann/ios/client" 7 | "github.com/heidi-ann/ios/gateway/rest" 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | ) 12 | 13 | var configFile = flag.String("config", os.Getenv("GOPATH")+"/src/github.com/heidi-ann/ios/example.conf", "Client configuration file") 14 | var statFile = flag.String("stat", "latency.csv", "File to write stats to") 15 | var algorithmFile = flag.String("algorithm", os.Getenv("GOPATH")+"/src/github.com/heidi-ann/ios/configfiles/simple/client.conf", "Algorithm description file") // optional flag 16 | var id = flag.Int("id", -1, "ID of client (must be unique) or random number will be generated") 17 | var port = flag.Int("port", 12345, "Port to listen for HTTP request on") 18 | 19 | func main() { 20 | // set up logging 21 | flag.Parse() 22 | defer glog.Flush() 23 | 24 | // always flush (whatever happens) 25 | sigs := make(chan os.Signal, 1) 26 | finish := make(chan bool, 1) 27 | signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) 28 | 29 | // parse config files 30 | 31 | c, err := client.StartClientFromConfigFile(*id, *statFile, *algorithmFile, *configFile) 32 | if err != nil { 33 | glog.Fatal(err) 34 | } 35 | 36 | // setup API 37 | ioapi := rest.Create(*port) 38 | 39 | go func() { 40 | for { 41 | // get next command 42 | text, read, ok := ioapi.Next() 43 | if !ok { 44 | finish <- true 45 | break 46 | } 47 | // pass to ios client 48 | reply, err := c.SubmitRequest(text, read) 49 | if err != nil { 50 | finish <- true 51 | break 52 | } 53 | // notify API of result 54 | ioapi.Return(reply) 55 | 56 | } 57 | }() 58 | 59 | select { 60 | case sig := <-sigs: 61 | glog.Warning("Termination due to: ", sig) 62 | case <-finish: 63 | glog.Info("No more commands") 64 | } 65 | c.StopClient() 66 | glog.Flush() 67 | 68 | } 69 | -------------------------------------------------------------------------------- /gateway/rest/rest.go: -------------------------------------------------------------------------------- 1 | // Package rest provides a REST API for client to interact with Ios clusters. 2 | package rest 3 | 4 | import ( 5 | "github.com/golang/glog" 6 | "io" 7 | "net/http" 8 | "strconv" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | // Rest is a placeholder 14 | type Rest struct{} 15 | 16 | type restrequest struct { 17 | Req string 18 | ReplyTo http.ResponseWriter 19 | } 20 | 21 | var waiting chan restrequest 22 | var outstanding chan restrequest 23 | 24 | func versionServer(w http.ResponseWriter, req *http.Request) { 25 | io.WriteString(w, "ios 0.1\n") 26 | } 27 | 28 | func closeServer(w http.ResponseWriter, req *http.Request) { 29 | close(waiting) 30 | io.WriteString(w, "Will do\n") 31 | } 32 | 33 | // main request handler 34 | func requestServer(w http.ResponseWriter, req *http.Request) { 35 | // NB: ResponseWriter needs to be used before this function exits 36 | glog.V(1).Info("Incoming GET request to", req.URL.String()) 37 | reqs := strings.Split(req.URL.String(), "/") 38 | reqNew := strings.Join(reqs[2:], " ") 39 | glog.V(1).Info("API request is:", reqNew) 40 | waiting <- restrequest{reqNew + "\n", w} 41 | 42 | //wait for response, else give up 43 | time.Sleep(time.Second) 44 | } 45 | 46 | func Create(port int) *Rest { 47 | portStr := ":" + strconv.Itoa(port) 48 | glog.V(1).Info("Setting up HTTP server on ", port) 49 | 50 | //setup HTTP server 51 | http.HandleFunc("/request/", requestServer) 52 | http.HandleFunc("/close", closeServer) 53 | http.HandleFunc("/version", versionServer) 54 | go func() { 55 | err := http.ListenAndServe(portStr, nil) 56 | if err != nil { 57 | glog.Fatal("ListenAndServe: ", err) 58 | } 59 | }() 60 | 61 | // setup channels 62 | waiting = make(chan restrequest, 10) 63 | outstanding = make(chan restrequest, 1) 64 | 65 | return &(Rest{}) 66 | 67 | } 68 | 69 | // Next returns the next request for the state machine or false if wishes to terminate 70 | func (r *Rest) Next() (string, bool, bool) { 71 | glog.V(1).Info("Waiting for next request") 72 | restreq, ok := <-waiting 73 | if !ok { 74 | return "", false, false 75 | } 76 | outstanding <- restreq 77 | glog.V(1).Info("Next request received: ", restreq.Req) 78 | // TODO: work out if request is read or write 79 | return restreq.Req, false, true 80 | } 81 | 82 | // Return provides the REST API with the response to the current outstanding request 83 | func (r *Rest) Return(str string) { 84 | glog.V(1).Info("Response received: ", str) 85 | restreq := <-outstanding 86 | io.WriteString(restreq.ReplyTo, str) 87 | glog.V(1).Info("Response sent") 88 | 89 | } 90 | -------------------------------------------------------------------------------- /ios.go: -------------------------------------------------------------------------------- 1 | package ios 2 | -------------------------------------------------------------------------------- /ios/example.conf: -------------------------------------------------------------------------------- 1 | ; servers to try connecting to, in order 2 | [servers] 3 | address = 127.0.0.1:8090:8080 4 | -------------------------------------------------------------------------------- /ios/example3.conf: -------------------------------------------------------------------------------- 1 | ; servers to try connecting to, in order 2 | [servers] 3 | address = 127.0.0.1:8090:8080 4 | address = 127.0.0.1:8091:8081 5 | address = 127.0.0.1:8092:8082 6 | -------------------------------------------------------------------------------- /ios/main.go: -------------------------------------------------------------------------------- 1 | // Package main is the entry point to directly run an Ios server as an executable 2 | package main 3 | 4 | import ( 5 | "flag" 6 | "github.com/golang/glog" 7 | "github.com/heidi-ann/ios/config" 8 | "github.com/heidi-ann/ios/ios/server" 9 | "os" 10 | "os/signal" 11 | "strconv" 12 | "syscall" 13 | ) 14 | 15 | // command line flags 16 | var id = flag.Int("id", -1, "server ID [REQUIRED]") // required flag 17 | var configFile = flag.String("config", os.Getenv("GOPATH")+"/src/github.com/heidi-ann/ios/example.conf", "Server configuration file") // optional flag 18 | var algorithmFile = flag.String("algorithm", os.Getenv("GOPATH")+"/src/github.com/heidi-ann/ios/configfiles/simple/server.conf", "Algorithm description file") // optional flag 19 | var diskPath = flag.String("disk", "persistent_id", "Path to directory to store persistent storage") // optional flag 20 | var peerPort = flag.Int("listen-peers", 0, "Overwrite the port specified in config file to listen for peers on") // optional flag 21 | var clientPort = flag.Int("listen-clients", 0, "Overwrite the port specified in config file to listen for clients on") // optional flag 22 | 23 | // entry point of server executable 24 | func main() { 25 | // set up logging 26 | flag.Parse() 27 | defer glog.Flush() 28 | 29 | // check go path is set 30 | if os.Getenv("GOPATH") == "" { 31 | glog.Fatal("GOPATH not set, please set GOPATH and try again") 32 | } 33 | 34 | // parse configuration files 35 | conf, err := config.ParseServerConfig(*algorithmFile) 36 | if err != nil { 37 | glog.Fatal(err) 38 | } 39 | addresses, err := config.ParseAddresses(*configFile) 40 | if err != nil { 41 | glog.Fatal(err) 42 | } 43 | if *id == -1 { 44 | glog.Fatal("ID is required") 45 | } 46 | // add ID to diskPath 47 | disk := *diskPath 48 | if disk == "persistent_id" { 49 | disk = "persistent_id" + strconv.Itoa(*id) 50 | } 51 | 52 | // overwrite ports if given 53 | if *peerPort != 0 { 54 | glog.Info("Peer port overwritten to ", *peerPort) 55 | addresses.Peers[*id].Port = *peerPort 56 | } 57 | 58 | if *clientPort != 0 { 59 | glog.Info("Client port overwritten to ", *clientPort) 60 | addresses.Clients[*id].Port = *clientPort 61 | } 62 | 63 | // logging 64 | glog.Info("Starting server ", *id) 65 | defer glog.Warning("Shutting down server ", *id) 66 | 67 | // start Ios server 68 | go func() { 69 | err := server.RunIos(*id, conf, addresses, disk) 70 | if err != nil { 71 | glog.Fatal(err) 72 | } 73 | }() 74 | 75 | // waiting for exit 76 | // always flush (whatever happens) 77 | sigs := make(chan os.Signal, 1) 78 | signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) 79 | sig := <-sigs 80 | glog.Flush() 81 | glog.Warning("Shutting down due to ", sig) 82 | } 83 | -------------------------------------------------------------------------------- /ios/server/server.go: -------------------------------------------------------------------------------- 1 | // Package server is the entry point to run an Ios server 2 | package server 3 | 4 | import ( 5 | "errors" 6 | 7 | "github.com/golang/glog" 8 | "github.com/heidi-ann/ios/config" 9 | "github.com/heidi-ann/ios/consensus" 10 | "github.com/heidi-ann/ios/msgs" 11 | "github.com/heidi-ann/ios/net" 12 | "github.com/heidi-ann/ios/storage" 13 | ) 14 | 15 | // RunIos id conf diskPath is the main entry point of Ios server 16 | // RunIos does not return until an error is detected during setup 17 | func RunIos(id int, conf config.ServerConfig, addresses config.Addresses, diskPath string) error { 18 | // check ID 19 | n := len(addresses.Peers) 20 | if id >= n { 21 | return errors.New("Invalid node ID") 22 | } 23 | 24 | // check config 25 | if err := config.CheckServerConfig(conf); err != nil { 26 | return err 27 | } 28 | 29 | // setup iO 30 | // TODO: remove this hardcoded limit on channel size 31 | peerNet := msgs.MakePeerNet(2000, n) 32 | clientNet := msgs.MakeClientNet(2000) 33 | 34 | // setup persistent storage 35 | found, view, log, index, state := storage.RestoreStorage( 36 | diskPath, conf.Performance.Length, conf.Application.Name) 37 | var store msgs.Storage 38 | if conf.Unsafe.DumpPersistentStorage { 39 | store = msgs.MakeDummyStorage() 40 | } else { 41 | var err error 42 | store, err = storage.MakeFileStorage(diskPath, conf.Unsafe.PersistenceMode) 43 | if err != nil { 44 | return err 45 | } 46 | } 47 | 48 | // setup peers & clients 49 | failureDetector := msgs.NewFailureNotifier(n) 50 | if err := net.SetupPeers(id, addresses.Peers, peerNet, failureDetector); err != nil { 51 | return err 52 | } 53 | if err := net.SetupClients(addresses.Clients[id].Port, state, clientNet); err != nil { 54 | return err 55 | } 56 | 57 | quorum, err := consensus.NewQuorum(conf.Algorithm.QuorumSystem, n) 58 | if err != nil { 59 | return err 60 | } 61 | // configure consensus algorithms 62 | configuration := consensus.Config{ 63 | All: consensus.ConfigAll{ 64 | ID: id, 65 | N: n, 66 | WindowSize: conf.Performance.WindowSize, 67 | Quorum: quorum, 68 | }, 69 | Master: consensus.ConfigMaster{ 70 | BatchInterval: conf.Performance.BatchInterval, 71 | MaxBatch: conf.Performance.MaxBatch, 72 | DelegateReplication: conf.Algorithm.DelegateReplication, 73 | IndexExclusivity: conf.Algorithm.IndexExclusivity, 74 | }, 75 | Coordinator: consensus.ConfigCoordinator{ 76 | ExplicitCommit: conf.Algorithm.ExplicitCommit, 77 | ThriftyQuorum: conf.Algorithm.ThriftyQuorum, 78 | }, 79 | Participant: consensus.ConfigParticipant{ 80 | SnapshotInterval: conf.Performance.SnapshotInterval, 81 | ImplicitWindowCommit: conf.Algorithm.ImplicitWindowCommit, 82 | LogLength: conf.Performance.Length, 83 | }, 84 | Interfacer: consensus.ConfigInterfacer{ 85 | ParticipantHandle: conf.Algorithm.ParticipantHandle, 86 | ParticipantRead: conf.Algorithm.ParticipantRead, 87 | }, 88 | } 89 | // setup consensus algorithm 90 | if !found { 91 | glog.Info("Starting fresh consensus instance") 92 | consensus.Init(peerNet, clientNet, configuration, state, failureDetector, store) 93 | } else { 94 | glog.Info("Restoring consensus instance") 95 | consensus.Recover(peerNet, clientNet, configuration, view, log, state, index, failureDetector, store) 96 | } 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /misc/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heidihoward/ios/fc1e718851fcfd91ba483436b9d7e89af97fd50f/misc/logo.png -------------------------------------------------------------------------------- /msgs/failures.go: -------------------------------------------------------------------------------- 1 | package msgs 2 | 3 | import ( 4 | "fmt" 5 | "github.com/golang/glog" 6 | "sync" 7 | ) 8 | 9 | type FailureNotifier struct { 10 | up []bool 11 | mutex sync.RWMutex 12 | subscribed [](chan bool) 13 | n int 14 | } 15 | 16 | func NewFailureNotifier(n int) *FailureNotifier { 17 | return &FailureNotifier{ 18 | make([]bool, n), 19 | sync.RWMutex{}, 20 | make([](chan bool), n), 21 | n} 22 | } 23 | 24 | func (f *FailureNotifier) NotifyOnFailure(id int) chan bool { 25 | note := make(chan bool, 1) 26 | f.subscribed[id] = note 27 | // if !f.up[id] { 28 | // note <- true 29 | // } 30 | return note 31 | } 32 | 33 | func (f *FailureNotifier) IsConnected(id int) bool { 34 | f.mutex.RLock() 35 | up := f.up[id] 36 | f.mutex.RUnlock() 37 | return up 38 | } 39 | 40 | // Return the ID of the next connected node 41 | func (f *FailureNotifier) NextConnected(id int) int { 42 | f.mutex.RLock() 43 | id++ 44 | if id == f.n { 45 | id = 0 46 | } 47 | for !f.up[id] { 48 | id++ 49 | if id == f.n { 50 | id = 0 51 | } 52 | } 53 | f.mutex.RUnlock() 54 | return id 55 | } 56 | 57 | func (f *FailureNotifier) NowConnected(id int) error { 58 | f.mutex.Lock() 59 | if f.up[id] { 60 | f.mutex.Unlock() 61 | return fmt.Errorf("Possible multiple connections to peer %d", id) 62 | } 63 | f.up[id] = true 64 | f.mutex.Unlock() 65 | return nil 66 | } 67 | 68 | func (f *FailureNotifier) NowDisconnected(id int) { 69 | f.mutex.Lock() 70 | if !f.up[id] { 71 | glog.Warning("Possible multiple connections to one peer") 72 | } 73 | f.up[id] = false 74 | f.subscribed[id] <- true 75 | f.mutex.Unlock() 76 | } 77 | -------------------------------------------------------------------------------- /msgs/failures_test.go: -------------------------------------------------------------------------------- 1 | package msgs 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestFailureNotifier(t *testing.T) { 10 | assert := assert.New(t) 11 | fn := NewFailureNotifier(5) 12 | 13 | for id := 0; id < 5; id++ { 14 | assert.False(fn.IsConnected(id), "Node should be initially disconnected") 15 | err := fn.NowConnected(id) 16 | assert.Nil(err, "Node could not connect") 17 | assert.True(fn.IsConnected(id), "Node should be connected") 18 | } 19 | 20 | // check on false failures 21 | select { 22 | case <-fn.NotifyOnFailure(3): 23 | t.Error("Unexpected failure") 24 | case <-time.After(100 * time.Millisecond): 25 | } 26 | 27 | wait := fn.NotifyOnFailure(3) 28 | fn.NowDisconnected(3) 29 | 30 | // check on false failures 31 | select { 32 | case <-wait: 33 | case <-time.After(100 * time.Millisecond): 34 | t.Error("Failure not reported") 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /msgs/formats.go: -------------------------------------------------------------------------------- 1 | package msgs 2 | 3 | // MESSAGE FORMATS 4 | 5 | // ClientRequest desribes a request. 6 | type ClientRequest struct { 7 | ClientID int 8 | RequestID int 9 | ForceViewChange bool 10 | ReadOnly bool 11 | Request string 12 | } 13 | 14 | // ClientResponse desribes a request response. 15 | type ClientResponse struct { 16 | ClientID int 17 | RequestID int 18 | Success bool 19 | Response string 20 | } 21 | 22 | // Client wraps a ClientRequest and ClientResponse. 23 | type Client struct { 24 | Request ClientRequest 25 | Response ClientResponse 26 | } 27 | 28 | // Entry describes a item stored in the replicated log. 29 | type Entry struct { 30 | View int 31 | Committed bool 32 | Requests []ClientRequest 33 | } 34 | 35 | // PrepareRequest describes a Prepare messages. 36 | type PrepareRequest struct { 37 | SenderID int 38 | View int 39 | StartIndex int // inclusive 40 | EndIndex int // exclusive 41 | Entries []Entry 42 | } 43 | 44 | // PrepareResponse describes a Prepare response messages. 45 | type PrepareResponse struct { 46 | SenderID int 47 | Success bool 48 | } 49 | 50 | // Prepare wraps a PrepareRequest and PrepareResponse. 51 | type Prepare struct { 52 | Request PrepareRequest 53 | Response PrepareResponse 54 | } 55 | 56 | // CommitRequest describes a Commit messages. 57 | type CommitRequest struct { 58 | SenderID int 59 | ResponseRequired bool 60 | StartIndex int 61 | EndIndex int 62 | Entries []Entry 63 | } 64 | 65 | // CommitResponse describes a Commit response messages. 66 | type CommitResponse struct { 67 | SenderID int 68 | Success bool 69 | CommitIndex int 70 | } 71 | 72 | // Commit wraps a CommitRequest and CommitResponse. 73 | type Commit struct { 74 | Request CommitRequest 75 | Response CommitResponse 76 | } 77 | 78 | type NewViewRequest struct { 79 | SenderID int 80 | View int 81 | } 82 | 83 | type NewViewResponse struct { 84 | SenderID int 85 | View int 86 | Index int 87 | } 88 | 89 | type NewView struct { 90 | Request NewViewRequest 91 | Response NewViewResponse 92 | } 93 | 94 | type QueryRequest struct { 95 | SenderID int 96 | View int 97 | StartIndex int // inclusive 98 | EndIndex int // exclusive 99 | } 100 | 101 | type QueryResponse struct { 102 | SenderID int 103 | View int 104 | Entries []Entry 105 | } 106 | 107 | type Query struct { 108 | Request QueryRequest 109 | Response QueryResponse 110 | } 111 | 112 | type CopyRequest struct { 113 | SenderID int 114 | StartIndex int // inclusive 115 | } 116 | 117 | // CopyResponse is not currently used 118 | type CopyResponse struct { 119 | SenderID int 120 | View int 121 | Success bool 122 | Entries []Entry 123 | } 124 | 125 | type Copy struct { 126 | Request CopyRequest 127 | Response CopyResponse 128 | } 129 | 130 | type CoordinateRequest struct { 131 | SenderID int 132 | View int 133 | StartIndex int //inclusive 134 | EndIndex int //exclusive 135 | Prepare bool 136 | Entries []Entry 137 | } 138 | 139 | type CoordinateResponse struct { 140 | SenderID int 141 | Success bool 142 | } 143 | 144 | type Coordinate struct { 145 | Request CoordinateRequest 146 | Response CoordinateResponse 147 | } 148 | 149 | type ForwardRequest struct { 150 | SenderID int 151 | View int 152 | Request ClientRequest 153 | } 154 | 155 | // Check is used to see apply a read without the master 156 | type CheckRequest struct { 157 | SenderID int 158 | Requests []ClientRequest 159 | } 160 | 161 | type CheckResponse struct { 162 | SenderID int 163 | Success bool 164 | CommitIndex int 165 | Replies []ClientResponse 166 | } 167 | 168 | type Check struct { 169 | Request CheckRequest 170 | Response CheckResponse 171 | } 172 | 173 | type LogUpdate struct { 174 | StartIndex int 175 | EndIndex int 176 | Entries []Entry 177 | } 178 | -------------------------------------------------------------------------------- /msgs/marshal.go: -------------------------------------------------------------------------------- 1 | package msgs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | "encoding/json" 8 | "github.com/golang/glog" 9 | ) 10 | 11 | // abstract the fact that JSON is used for marshalling 12 | 13 | func Marshal(v interface{}) ([]byte, error) { 14 | return json.Marshal(v) 15 | } 16 | 17 | func Unmarshal(data []byte, v interface{}) error { 18 | return json.Unmarshal(data, v) 19 | } 20 | 21 | func (msgch *ProtoMsgs) BytesToProtoMsg(b []byte) error { 22 | if len(b) == 0 { 23 | return errors.New("Empty message received") 24 | } 25 | glog.V(1).Info("Received ", string(b)) 26 | label := int(b[0]) 27 | switch label { 28 | case 1: 29 | var msg PrepareRequest 30 | err := Unmarshal(b[1:], &msg) 31 | if err != nil { 32 | return fmt.Errorf("Cannot unmarshal PrepareRequest due to: %v", err) 33 | } 34 | glog.V(1).Info("Unmarshalled ", msg) 35 | select { 36 | case msgch.Requests.Prepare <- msg: 37 | default: 38 | glog.Fatal("Buffer overflow, dropping message", msg) 39 | } 40 | case 2: 41 | var msg CommitRequest 42 | err := Unmarshal(b[1:], &msg) 43 | if err != nil { 44 | return fmt.Errorf("Cannot unmarshal CommitRequest due to: %v", err) 45 | } 46 | glog.V(1).Info("Unmarshalled ", msg) 47 | select { 48 | case msgch.Requests.Commit <- msg: 49 | default: 50 | glog.Fatal("Buffer overflow, dropping message", msg) 51 | } 52 | case 3: 53 | var msg Prepare 54 | err := Unmarshal(b[1:], &msg) 55 | if err != nil { 56 | return fmt.Errorf("Cannot unmarshal Prepare due to: %v", err) 57 | } 58 | glog.V(1).Info("Unmarshalled ", msg) 59 | select { 60 | case msgch.Responses.Prepare <- msg: 61 | default: 62 | glog.Fatal("Buffer overflow, dropping message", msg) 63 | } 64 | case 4: 65 | var msg Commit 66 | err := Unmarshal(b[1:], &msg) 67 | if err != nil { 68 | return fmt.Errorf("Cannot unmarshal Commit due to: %v", err) 69 | } 70 | glog.V(1).Info("Unmarshalled ", msg) 71 | select { 72 | case msgch.Responses.Commit <- msg: 73 | default: 74 | glog.Fatal("Buffer overflow, dropping message", msg) 75 | } 76 | case 5: 77 | var msg NewViewRequest 78 | err := Unmarshal(b[1:], &msg) 79 | if err != nil { 80 | return fmt.Errorf("Cannot unmarshal NewViewRequest due to: %v", err) 81 | } 82 | glog.V(1).Info("Unmarshalled ", msg) 83 | select { 84 | case msgch.Requests.NewView <- msg: 85 | default: 86 | glog.Fatal("Buffer overflow, dropping message", msg) 87 | } 88 | case 6: 89 | var msg NewView 90 | err := Unmarshal(b[1:], &msg) 91 | if err != nil { 92 | return fmt.Errorf("Cannot unmarshal NewView due to: %v", err) 93 | } 94 | glog.V(1).Info("Unmarshalled ", msg) 95 | select { 96 | case msgch.Responses.NewView <- msg: 97 | default: 98 | glog.Fatal("Buffer overflow, dropping message", msg) 99 | } 100 | case 7: 101 | var msg CoordinateRequest 102 | err := Unmarshal(b[1:], &msg) 103 | if err != nil { 104 | return fmt.Errorf("Cannot unmarshal CoordinateRequest due to: %v", err) 105 | } 106 | glog.V(1).Info("Unmarshalled ", msg) 107 | select { 108 | case msgch.Requests.Coordinate <- msg: 109 | default: 110 | glog.Fatal("Buffer overflow, dropping message", msg) 111 | } 112 | case 8: 113 | var msg Coordinate 114 | err := Unmarshal(b[1:], &msg) 115 | if err != nil { 116 | return fmt.Errorf("Cannot unmarshal Coordinate due to: %v", err) 117 | } 118 | glog.V(1).Info("Unmarshalled ", msg) 119 | select { 120 | case msgch.Responses.Coordinate <- msg: 121 | default: 122 | glog.Fatal("Buffer overflow, dropping message", msg) 123 | } 124 | case 9: 125 | var msg QueryRequest 126 | err := Unmarshal(b[1:], &msg) 127 | if err != nil { 128 | return fmt.Errorf("Cannot unmarshal QueryRequest due to: %v", err) 129 | } 130 | glog.V(1).Info("Unmarshalled ", msg) 131 | select { 132 | case msgch.Requests.Query <- msg: 133 | default: 134 | glog.Fatal("Buffer overflow, dropping message", msg) 135 | } 136 | case 0: 137 | var msg Query 138 | err := Unmarshal(b[1:], &msg) 139 | if err != nil { 140 | return fmt.Errorf("Cannot unmarshal Query due to: %v", err) 141 | } 142 | glog.V(1).Info("Unmarshalled ", msg) 143 | select { 144 | case msgch.Responses.Query <- msg: 145 | default: 146 | glog.Fatal("Buffer overflow, dropping message", msg) 147 | } 148 | case 15: 149 | var msg ForwardRequest 150 | err := Unmarshal(b[1:], &msg) 151 | if err != nil { 152 | return fmt.Errorf("Cannot unmarshal ForwardRequest due to: %v", err) 153 | } 154 | glog.V(1).Info("Unmarshalled ", msg) 155 | select { 156 | case msgch.Requests.Forward <- msg: 157 | default: 158 | glog.Fatal("Buffer overflow, dropping message ", msg) 159 | } 160 | case 11: 161 | var msg CopyRequest 162 | err := Unmarshal(b[1:], &msg) 163 | if err != nil { 164 | return fmt.Errorf("Cannot unmarshal CopyRequest due to: %v", err) 165 | } 166 | glog.V(1).Info("Unmarshalled ", msg) 167 | select { 168 | case msgch.Requests.Copy <- msg: 169 | default: 170 | glog.Fatal("Buffer overflow, dropping message ", msg) 171 | } 172 | case 12: 173 | var msg Copy 174 | err := Unmarshal(b[1:], &msg) 175 | if err != nil { 176 | return fmt.Errorf("Cannot unmarshal Copy due to: %v", err) 177 | } 178 | glog.V(1).Info("Unmarshalled ", msg) 179 | select { 180 | case msgch.Responses.Copy <- msg: 181 | default: 182 | glog.Fatal("Buffer overflow, dropping message", msg) 183 | } 184 | case 13: 185 | var msg CheckRequest 186 | err := Unmarshal(b[1:], &msg) 187 | if err != nil { 188 | return fmt.Errorf("Cannot unmarshal CheckRequest due to: %v", err) 189 | } 190 | glog.V(1).Info("Unmarshalled ", msg) 191 | select { 192 | case msgch.Requests.Check <- msg: 193 | default: 194 | glog.Fatal("Buffer overflow, dropping message", msg) 195 | } 196 | case 14: 197 | var msg Check 198 | err := Unmarshal(b[1:], &msg) 199 | if err != nil { 200 | return fmt.Errorf("Cannot unmarshal Check due to: %v", err) 201 | } 202 | glog.V(1).Info("Unmarshalled ", msg) 203 | select { 204 | case msgch.Responses.Check <- msg: 205 | default: 206 | glog.Fatal("Buffer overflow, dropping message", msg) 207 | } 208 | default: 209 | return fmt.Errorf("Cannot parse message label: %d", label) 210 | } 211 | return nil 212 | } 213 | 214 | // append a byte at the start of a byte array 215 | func appendr(x byte, xs []byte) []byte { 216 | // TODO: find a better way todo this 217 | ans := make([]byte, len(xs)+1) 218 | ans[0] = x 219 | for i := range xs { 220 | ans[i+1] = xs[i] 221 | } 222 | return ans 223 | } 224 | 225 | func (msgch *ProtoMsgs) ProtoMsgToBytes() ([]byte, error) { 226 | select { 227 | case msg := <-msgch.Requests.Prepare: 228 | glog.V(1).Info("Marshalling ", msg) 229 | b, err := Marshal(msg) 230 | snd := appendr(byte(1), b) 231 | return snd, err 232 | 233 | case msg := <-msgch.Requests.Commit: 234 | glog.V(1).Info("Marshalling ", msg) 235 | b, err := Marshal(msg) 236 | snd := appendr(byte(2), b) 237 | return snd, err 238 | 239 | case msg := <-msgch.Responses.Prepare: 240 | glog.V(1).Info("Marshalling ", msg) 241 | b, err := Marshal(msg) 242 | snd := appendr(byte(3), b) 243 | return snd, err 244 | 245 | case msg := <-msgch.Responses.Commit: 246 | glog.V(1).Info("Marshalling ", msg) 247 | b, err := Marshal(msg) 248 | snd := appendr(byte(4), b) 249 | return snd, err 250 | 251 | case msg := <-msgch.Requests.NewView: 252 | glog.V(1).Info("Marshalling ", msg) 253 | b, err := Marshal(msg) 254 | snd := appendr(byte(5), b) 255 | return snd, err 256 | 257 | case msg := <-msgch.Responses.NewView: 258 | glog.V(1).Info("Marshalling ", msg) 259 | b, err := Marshal(msg) 260 | snd := appendr(byte(6), b) 261 | return snd, err 262 | 263 | case msg := <-msgch.Requests.Coordinate: 264 | glog.V(1).Info("Marshalling ", msg) 265 | b, err := Marshal(msg) 266 | snd := appendr(byte(7), b) 267 | return snd, err 268 | 269 | case msg := <-msgch.Responses.Coordinate: 270 | glog.V(1).Info("Marshalling ", msg) 271 | b, err := Marshal(msg) 272 | snd := appendr(byte(8), b) 273 | return snd, err 274 | 275 | case msg := <-msgch.Requests.Query: 276 | glog.V(1).Info("Marshalling ", msg) 277 | b, err := Marshal(msg) 278 | snd := appendr(byte(9), b) 279 | return snd, err 280 | 281 | case msg := <-msgch.Responses.Query: 282 | glog.V(1).Info("Marshalling ", msg) 283 | b, err := Marshal(msg) 284 | snd := appendr(byte(0), b) 285 | return snd, err 286 | 287 | case msg := <-msgch.Requests.Copy: 288 | glog.V(1).Info("Marshalling ", msg) 289 | b, err := Marshal(msg) 290 | snd := appendr(byte(11), b) 291 | return snd, err 292 | 293 | case msg := <-msgch.Responses.Copy: 294 | glog.V(1).Info("Marshalling ", msg) 295 | b, err := Marshal(msg) 296 | snd := appendr(byte(12), b) 297 | return snd, err 298 | 299 | case msg := <-msgch.Requests.Check: 300 | glog.V(1).Info("Marshalling ", msg) 301 | b, err := Marshal(msg) 302 | snd := appendr(byte(13), b) 303 | return snd, err 304 | 305 | case msg := <-msgch.Responses.Check: 306 | glog.V(1).Info("Marshalling ", msg) 307 | b, err := Marshal(msg) 308 | snd := appendr(byte(14), b) 309 | return snd, err 310 | 311 | case msg := <-msgch.Requests.Forward: 312 | glog.V(1).Info("Marshalling ", msg) 313 | b, err := Marshal(msg) 314 | snd := appendr(byte(15), b) 315 | return snd, err 316 | 317 | } 318 | } 319 | -------------------------------------------------------------------------------- /msgs/msgs.go: -------------------------------------------------------------------------------- 1 | // Package msgs describes all I/O formatting 2 | package msgs 3 | 4 | import ( 5 | "github.com/golang/glog" 6 | ) 7 | 8 | // DATA STRUCTURES FOR ABSTRACTING MSG iO 9 | 10 | type Requests struct { 11 | Prepare chan PrepareRequest 12 | Commit chan CommitRequest 13 | NewView chan NewViewRequest 14 | Query chan QueryRequest 15 | Copy chan CopyRequest 16 | Coordinate chan CoordinateRequest 17 | Forward chan ForwardRequest 18 | Check chan CheckRequest 19 | } 20 | 21 | type Responses struct { 22 | Prepare chan Prepare 23 | Commit chan Commit 24 | NewView chan NewView 25 | Query chan Query 26 | Copy chan Copy 27 | Coordinate chan Coordinate 28 | Check chan Check 29 | } 30 | 31 | type ProtoMsgs struct { 32 | Requests Requests 33 | Responses Responses 34 | } 35 | 36 | type ClientNet struct { 37 | IncomingRequests chan ClientRequest 38 | OutgoingResponses chan Client 39 | OutgoingRequestsFailed chan ClientRequest 40 | } 41 | 42 | type PeerNet struct { 43 | Incoming ProtoMsgs 44 | OutgoingBroadcast ProtoMsgs 45 | OutgoingUnicast map[int]*ProtoMsgs 46 | } 47 | 48 | // broadcaster forwards messages on Outgoing broadcast channels to all outgoing unicast channels 49 | func broadcaster(broadcast *ProtoMsgs, unicast map[int]*ProtoMsgs) { 50 | glog.Info("Setting up broadcaster for ", len(unicast), " nodes") 51 | for { 52 | // TODO: find a more generic method 53 | select { 54 | // Requests 55 | case r := <-broadcast.Requests.Prepare: 56 | glog.V(1).Info("Broadcasting ", r) 57 | for id := range unicast { 58 | unicast[id].Requests.Prepare <- r 59 | } 60 | case r := <-broadcast.Requests.Commit: 61 | glog.V(1).Info("Broadcasting ", r) 62 | for id := range unicast { 63 | unicast[id].Requests.Commit <- r 64 | } 65 | case r := <-broadcast.Requests.NewView: 66 | glog.V(1).Info("Broadcasting ", r) 67 | for id := range unicast { 68 | unicast[id].Requests.NewView <- r 69 | } 70 | case r := <-broadcast.Requests.Query: 71 | glog.V(1).Info("Broadcasting ", r) 72 | for id := range unicast { 73 | unicast[id].Requests.Query <- r 74 | } 75 | case r := <-broadcast.Requests.Copy: 76 | glog.V(1).Info("Broadcasting ", r) 77 | for id := range unicast { 78 | unicast[id].Requests.Copy <- r 79 | } 80 | case r := <-broadcast.Requests.Coordinate: 81 | glog.V(1).Info("Broadcasting ", r) 82 | for id := range unicast { 83 | unicast[id].Requests.Coordinate <- r 84 | } 85 | case r := <-broadcast.Requests.Check: 86 | glog.V(1).Info("Broadcasting ", r) 87 | for id := range unicast { 88 | unicast[id].Requests.Check <- r 89 | } 90 | case r := <-broadcast.Requests.Forward: 91 | glog.V(1).Info("Broadcasting ", r) 92 | for id := range unicast { 93 | unicast[id].Requests.Forward <- r 94 | } 95 | // Responses 96 | case r := <-broadcast.Responses.Prepare: 97 | glog.V(1).Info("Broadcasting ", r) 98 | for id := range unicast { 99 | unicast[id].Responses.Prepare <- r 100 | } 101 | case r := <-broadcast.Responses.Commit: 102 | glog.V(1).Info("Broadcasting ", r) 103 | for id := range unicast { 104 | unicast[id].Responses.Commit <- r 105 | } 106 | case r := <-broadcast.Responses.NewView: 107 | glog.V(1).Info("Broadcasting ", r) 108 | for id := range unicast { 109 | unicast[id].Responses.NewView <- r 110 | } 111 | case r := <-broadcast.Responses.Query: 112 | glog.V(1).Info("Broadcasting ", r) 113 | for id := range unicast { 114 | unicast[id].Responses.Query <- r 115 | } 116 | case r := <-broadcast.Responses.Copy: 117 | glog.V(1).Info("Broadcasting ", r) 118 | for id := range unicast { 119 | unicast[id].Responses.Copy <- r 120 | } 121 | case r := <-broadcast.Responses.Coordinate: 122 | glog.V(1).Info("Broadcasting ", r) 123 | for id := range unicast { 124 | unicast[id].Responses.Coordinate <- r 125 | } 126 | case r := <-broadcast.Responses.Check: 127 | glog.V(1).Info("Broadcasting ", r) 128 | for id := range unicast { 129 | unicast[id].Responses.Check <- r 130 | } 131 | } 132 | } 133 | } 134 | 135 | // Forward mesaages from one ProtoMsgs to another 136 | func (to *ProtoMsgs) Forward(from *ProtoMsgs) { 137 | for { 138 | select { 139 | // Requests 140 | case r := <-from.Requests.Prepare: 141 | glog.V(1).Info("Forwarding ", r) 142 | to.Requests.Prepare <- r 143 | case r := <-from.Requests.Commit: 144 | glog.V(1).Info("Forwarding", r) 145 | to.Requests.Commit <- r 146 | case r := <-from.Requests.NewView: 147 | glog.V(1).Info("Forwarding", r) 148 | to.Requests.NewView <- r 149 | case r := <-from.Requests.Query: 150 | glog.V(1).Info("Forwarding", r) 151 | to.Requests.Query <- r 152 | case r := <-from.Requests.Copy: 153 | glog.V(1).Info("Forwarding", r) 154 | to.Requests.Copy <- r 155 | case r := <-from.Requests.Coordinate: 156 | glog.V(1).Info("Forwarding", r) 157 | to.Requests.Coordinate <- r 158 | case r := <-from.Requests.Check: 159 | glog.V(1).Info("Forwarding", r) 160 | to.Requests.Check <- r 161 | case r := <-from.Requests.Forward: 162 | glog.V(1).Info("Forwarding", r) 163 | to.Requests.Forward <- r 164 | // Responses 165 | case r := <-from.Responses.Prepare: 166 | glog.V(1).Info("Forwarding", r) 167 | to.Responses.Prepare <- r 168 | case r := <-from.Responses.Commit: 169 | glog.V(1).Info("Forwarding", r) 170 | to.Responses.Commit <- r 171 | case r := <-from.Responses.NewView: 172 | glog.V(1).Info("Forwarding", r) 173 | to.Responses.NewView <- r 174 | case r := <-from.Responses.Query: 175 | glog.V(1).Info("Forwarding", r) 176 | to.Responses.Query <- r 177 | case r := <-from.Responses.Copy: 178 | glog.V(1).Info("Forwarding", r) 179 | to.Responses.Copy <- r 180 | case r := <-from.Responses.Coordinate: 181 | glog.V(1).Info("Forwarding", r) 182 | to.Responses.Coordinate <- r 183 | case r := <-from.Responses.Check: 184 | glog.V(1).Info("Forwarding", r) 185 | to.Responses.Check <- r 186 | } 187 | } 188 | } 189 | 190 | // Forward mesaages from one ProtoMsgs to another 191 | func (from *ProtoMsgs) Discard() { 192 | for { 193 | select { 194 | // Requests 195 | case <-from.Requests.Prepare: 196 | case <-from.Requests.Commit: 197 | case <-from.Requests.NewView: 198 | case <-from.Requests.Query: 199 | case <-from.Requests.Copy: 200 | case <-from.Requests.Coordinate: 201 | case <-from.Requests.Check: 202 | case <-from.Requests.Forward: 203 | // Responses 204 | case <-from.Responses.Prepare: 205 | case <-from.Responses.Commit: 206 | case <-from.Responses.NewView: 207 | case <-from.Responses.Query: 208 | case <-from.Requests.Copy: 209 | case <-from.Responses.Coordinate: 210 | case <-from.Responses.Check: 211 | default: 212 | return 213 | } 214 | } 215 | } 216 | 217 | func MakeProtoMsgs(buf int) ProtoMsgs { 218 | return ProtoMsgs{ 219 | Requests{ 220 | make(chan PrepareRequest, buf), 221 | make(chan CommitRequest, buf), 222 | make(chan NewViewRequest, buf), 223 | make(chan QueryRequest, buf), 224 | make(chan CopyRequest, buf), 225 | make(chan CoordinateRequest, buf), 226 | make(chan ForwardRequest, buf), 227 | make(chan CheckRequest, buf)}, 228 | Responses{ 229 | make(chan Prepare, buf), 230 | make(chan Commit, buf), 231 | make(chan NewView, buf), 232 | make(chan Query, buf), 233 | make(chan Copy, buf), 234 | make(chan Coordinate, buf), 235 | make(chan Check, buf)}} 236 | } 237 | 238 | func MakeClientNet(buf int) *ClientNet { 239 | net := ClientNet{ 240 | IncomingRequests: make(chan ClientRequest, buf), 241 | OutgoingResponses: make(chan Client, buf), 242 | OutgoingRequestsFailed: make(chan ClientRequest, buf)} 243 | return &net 244 | } 245 | 246 | func MakePeerNet(buf int, n int) *PeerNet { 247 | net := PeerNet{ 248 | Incoming: MakeProtoMsgs(buf), 249 | OutgoingBroadcast: MakeProtoMsgs(buf), 250 | OutgoingUnicast: make(map[int]*ProtoMsgs)} 251 | 252 | for id := 0; id < n; id++ { 253 | protomsgs := MakeProtoMsgs(buf) 254 | net.OutgoingUnicast[id] = &protomsgs 255 | } 256 | 257 | go broadcaster(&net.OutgoingBroadcast, net.OutgoingUnicast) 258 | return &net 259 | } 260 | -------------------------------------------------------------------------------- /msgs/msgs_test.go: -------------------------------------------------------------------------------- 1 | package msgs 2 | 3 | import ( 4 | "flag" 5 | "github.com/golang/glog" 6 | "reflect" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestPeerNet(t *testing.T) { 12 | flag.Parse() 13 | defer glog.Flush() 14 | 15 | // SAMPLE MESSAGES 16 | 17 | request1 := []ClientRequest{{ 18 | ClientID: 2, 19 | RequestID: 0, 20 | Request: "update A 3"}} 21 | 22 | entries1 := []Entry{{ 23 | View: 0, 24 | Committed: false, 25 | Requests: request1}} 26 | 27 | prepare := PrepareRequest{ 28 | SenderID: 0, 29 | View: 0, 30 | StartIndex: 0, 31 | EndIndex: 1, 32 | Entries: entries1} 33 | 34 | prepareRes := PrepareResponse{ 35 | SenderID: 0, 36 | Success: true} 37 | 38 | prep := Prepare{ 39 | prepare, prepareRes} 40 | 41 | // create a node in system of 3 nodes 42 | nodes := 3 43 | peerNet := MakePeerNet(10, nodes) 44 | 45 | // TEST 46 | if len(peerNet.OutgoingUnicast) != nodes { 47 | t.Error("Wrong number of unicast channels created") 48 | } 49 | 50 | // TEST 51 | peerNet.Incoming.Requests.Prepare <- prepare 52 | 53 | select { 54 | case reply := <-peerNet.Incoming.Requests.Prepare: 55 | if !reflect.DeepEqual(reply, prepare) { 56 | t.Error(reply) 57 | } 58 | case <-time.After(time.Millisecond): 59 | t.Error("Channel not delivering messages as expected") 60 | } 61 | 62 | // TEST 63 | out := peerNet.OutgoingUnicast[0] 64 | (*out).Responses.Prepare <- prep 65 | select { 66 | case reply := <-(*out).Responses.Prepare: 67 | if reply.Response != prepareRes { 68 | t.Error(reply) 69 | } 70 | case <-time.After(time.Millisecond): 71 | t.Error("Channel not delivering messages as expected") 72 | } 73 | 74 | //TEST 75 | go broadcaster(&peerNet.OutgoingBroadcast, peerNet.OutgoingUnicast) 76 | peerNet.OutgoingBroadcast.Responses.Prepare <- prep 77 | 78 | for id := 0; id < nodes; id++ { 79 | // check each receives it 80 | select { 81 | case reply := <-peerNet.OutgoingUnicast[id].Responses.Prepare: 82 | if reply.Response != prepareRes { 83 | t.Error(reply) 84 | } 85 | case <-time.After(10 * time.Millisecond): 86 | t.Error("Nodes ", id, " didn't receive broadcasted message") 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /msgs/notifier.go: -------------------------------------------------------------------------------- 1 | package msgs 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type Notificator struct { 8 | subscribed map[ClientRequest](chan ClientResponse) 9 | mutex sync.RWMutex 10 | } 11 | 12 | func NewNotificator() *Notificator { 13 | return &Notificator{ 14 | make(map[ClientRequest](chan ClientResponse)), 15 | sync.RWMutex{}} 16 | } 17 | 18 | func (n *Notificator) Notify(request ClientRequest, response ClientResponse) { 19 | // if any handleRequests are waiting on this reply, then reply to them 20 | n.mutex.Lock() 21 | if n.subscribed[request] != nil { 22 | n.subscribed[request] <- response 23 | } 24 | n.mutex.Unlock() 25 | } 26 | 27 | // Blocking call 28 | func (n *Notificator) Subscribe(request ClientRequest) ClientResponse { 29 | n.mutex.Lock() 30 | if n.subscribed[request] == nil { 31 | n.subscribed[request] = make(chan ClientResponse) 32 | } 33 | response := n.subscribed[request] 34 | n.mutex.Unlock() 35 | return <-response 36 | } 37 | 38 | func (n *Notificator) IsSubscribed(request ClientRequest) bool { 39 | n.mutex.RLock() 40 | result := n.subscribed[request] != nil 41 | n.mutex.RUnlock() 42 | return result 43 | } 44 | -------------------------------------------------------------------------------- /msgs/storage.go: -------------------------------------------------------------------------------- 1 | package msgs 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | ) 6 | 7 | type Storage interface { 8 | PersistView(view int) error 9 | PersistLogUpdate(logUpdate LogUpdate) error 10 | PersistSnapshot(index int, snap []byte) error 11 | } 12 | 13 | type ExternalStorage struct { 14 | ViewPersist chan int 15 | ViewPersistFsync chan int 16 | LogPersist chan LogUpdate 17 | LogPersistFsync chan LogUpdate 18 | } 19 | 20 | func MakeExternalStorage() *ExternalStorage { 21 | buf := 10 22 | s := ExternalStorage{ 23 | ViewPersist: make(chan int, buf), 24 | ViewPersistFsync: make(chan int, buf), 25 | LogPersist: make(chan LogUpdate, buf), 26 | LogPersistFsync: make(chan LogUpdate, buf)} 27 | return &s 28 | } 29 | 30 | func (s *ExternalStorage) PersistView(view int) error { 31 | s.ViewPersist <- view 32 | <-s.ViewPersistFsync 33 | return nil 34 | } 35 | 36 | func (s *ExternalStorage) PersistLogUpdate(logUpdate LogUpdate) error { 37 | s.LogPersist <- logUpdate 38 | <-s.LogPersistFsync 39 | return nil 40 | } 41 | 42 | func (s *ExternalStorage) PersistSnapshot(index int, snap []byte) error { 43 | // TODO: complete stub 44 | return nil 45 | } 46 | 47 | type DummyStorage struct{} 48 | 49 | func MakeDummyStorage() *DummyStorage { 50 | return &DummyStorage{} 51 | } 52 | 53 | func (_ *DummyStorage) PersistView(view int) error { 54 | glog.V(1).Info("Updating view to ", view) 55 | return nil 56 | } 57 | 58 | func (_ *DummyStorage) PersistLogUpdate(log LogUpdate) error { 59 | glog.V(1).Info("Updating log with ", log) 60 | return nil 61 | } 62 | 63 | func (_ *DummyStorage) PersistSnapshot(index int, snap []byte) error { 64 | glog.V(1).Info("Updating snap with ", index, snap) 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /net/clients.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "bufio" 5 | "github.com/golang/glog" 6 | "github.com/heidi-ann/ios/app" 7 | "github.com/heidi-ann/ios/msgs" 8 | "io" 9 | "net" 10 | "strconv" 11 | ) 12 | 13 | type clientHandler struct { 14 | notify *msgs.Notificator 15 | app *app.StateMachine 16 | clientNet *msgs.ClientNet 17 | } 18 | 19 | func (ch *clientHandler) stateMachine() { 20 | for { 21 | var req msgs.ClientRequest 22 | var reply msgs.ClientResponse 23 | 24 | select { 25 | case response := <-ch.clientNet.OutgoingResponses: 26 | req = response.Request 27 | reply = response.Response 28 | case req = <-ch.clientNet.OutgoingRequestsFailed: 29 | glog.V(1).Info("Request could not been safely replicated by consensus algorithm", req) 30 | reply = msgs.ClientResponse{ 31 | req.ClientID, req.RequestID, false, ""} 32 | } 33 | 34 | // if any handleRequests are waiting on this reply, then reply to them 35 | ch.notify.Notify(req, reply) 36 | } 37 | } 38 | 39 | func (ch *clientHandler) handleRequest(req msgs.ClientRequest) msgs.ClientResponse { 40 | glog.V(1).Info("Handling ", req.Request) 41 | 42 | // check if already applied 43 | if found, res := ch.app.Check(req); found { 44 | glog.V(1).Info("Request found in cache") 45 | return res // FAST PASS 46 | } 47 | 48 | // CONSENESUS ALGORITHM HERE 49 | glog.V(1).Info("Passing request to consensus algorithm") 50 | ch.clientNet.IncomingRequests <- req 51 | 52 | if ch.notify.IsSubscribed(req) { 53 | glog.Warning("Client has multiple outstanding connections for the same request, usually not a good sign") 54 | } 55 | 56 | // wait for reply 57 | reply := ch.notify.Subscribe(req) 58 | 59 | // check reply is as expected 60 | if reply.ClientID != req.ClientID { 61 | glog.Fatal("ClientID is different ", reply.ClientID, req.ClientID) 62 | } 63 | if reply.RequestID != req.RequestID { 64 | glog.Fatal("RequestID is different", reply.RequestID, req.RequestID) 65 | } 66 | 67 | return reply 68 | } 69 | 70 | func (ch *clientHandler) handleConnection(cn net.Conn) { 71 | glog.Info("Incoming client connection from ", 72 | cn.RemoteAddr().String()) 73 | 74 | reader := bufio.NewReader(cn) 75 | writer := bufio.NewWriter(cn) 76 | 77 | for { 78 | 79 | // read request 80 | glog.V(1).Info("Ready for Reading") 81 | text, err := reader.ReadBytes(byte('\n')) 82 | if err != nil { 83 | if err == io.EOF { 84 | break 85 | } 86 | glog.Warning(err) 87 | break 88 | } 89 | glog.V(1).Info("--------------------New request----------------------") 90 | glog.V(1).Info("Request: ", string(text)) 91 | req := new(msgs.ClientRequest) 92 | err = msgs.Unmarshal(text, req) 93 | if err != nil { 94 | glog.Fatal(err) 95 | } 96 | 97 | // construct reply 98 | reply := ch.handleRequest(*req) 99 | b, err := msgs.Marshal(reply) 100 | if err != nil { 101 | glog.Fatal("error:", err) 102 | } 103 | glog.V(1).Info(string(b)) 104 | 105 | // send reply 106 | glog.V(1).Info("Sending ", string(b)) 107 | n, err := writer.Write(b) 108 | if err != nil { 109 | glog.Fatal(err) 110 | } 111 | _, err = writer.Write([]byte("\n")) 112 | if err != nil { 113 | glog.Fatal(err) 114 | } 115 | 116 | // tidy up 117 | err = writer.Flush() 118 | if err != nil { 119 | glog.Fatal(err) 120 | } 121 | glog.V(1).Info("Finished sending ", n, " bytes") 122 | 123 | } 124 | 125 | cn.Close() 126 | } 127 | 128 | // SetupClients listen for client on the given port, it forwards their requests to the consensus algorithm and 129 | // then applies them to the state machine 130 | // SetupClients returns when setup is completed, spawning goroutines to listen for clients. 131 | func SetupClients(port int, app *app.StateMachine, clientNet *msgs.ClientNet) error { 132 | ch := clientHandler{ 133 | notify: msgs.NewNotificator(), 134 | app: app, 135 | clientNet: clientNet} 136 | 137 | go ch.stateMachine() 138 | 139 | // set up client server 140 | glog.Info("Starting up client server on port ", port) 141 | listeningPort := ":" + strconv.Itoa(port) 142 | ln, err := net.Listen("tcp", listeningPort) 143 | if err != nil { 144 | glog.Warning("Unable to listen for clients", err) 145 | return err 146 | } 147 | 148 | // handle for incoming clients 149 | go func() { 150 | for { 151 | conn, err := ln.Accept() 152 | if err != nil { 153 | glog.Fatal(err) 154 | } 155 | go ch.handleConnection(conn) 156 | } 157 | }() 158 | 159 | return nil 160 | } 161 | -------------------------------------------------------------------------------- /net/peers.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "bufio" 5 | "net" 6 | "strconv" 7 | "strings" 8 | "time" 9 | 10 | "github.com/heidi-ann/ios/config" 11 | "github.com/heidi-ann/ios/msgs" 12 | 13 | "github.com/golang/glog" 14 | ) 15 | 16 | type peerHandler struct { 17 | id int 18 | peers []config.NetAddress 19 | failures *msgs.FailureNotifier 20 | net *msgs.PeerNet 21 | } 22 | 23 | // iterative through peers and check if there is a handler for each 24 | // try to create one if not, report failure if not possible 25 | func (ph *peerHandler) checkPeer() { 26 | for i := range ph.peers { 27 | if !ph.failures.IsConnected(i) { 28 | cn, err := net.Dial("tcp", ph.peers[i].ToString()) 29 | if err == nil { 30 | go ph.handlePeer(cn, true) 31 | } else { 32 | go ph.net.OutgoingUnicast[i].Discard() 33 | } 34 | } 35 | } 36 | } 37 | 38 | // handlePeer handles a peer connection until closed 39 | func (ph *peerHandler) handlePeer(cn net.Conn, init bool) { 40 | addr := cn.RemoteAddr().String() 41 | if init { 42 | glog.Info("Outgoing peer connection to ", addr) 43 | } else { 44 | glog.Info("Incoming peer connection from ", addr) 45 | } 46 | 47 | defer cn.Close() 48 | defer glog.Warningf("Connection closed from %s ", addr) 49 | 50 | // handle requests 51 | reader := bufio.NewReader(cn) 52 | writer := bufio.NewWriter(cn) 53 | 54 | // exchange peer ID's via handshake 55 | shake := strconv.Itoa(ph.id) + "\n" 56 | n, err := writer.WriteString(shake) 57 | if err != nil { 58 | glog.Warning(err) 59 | } 60 | if n != len(shake) { 61 | glog.Warning("Short write") 62 | } 63 | err = writer.Flush() 64 | if err != nil { 65 | glog.Warning(err) 66 | } 67 | text, err := reader.ReadString('\n') 68 | if err != nil { 69 | glog.Warning(err) 70 | } 71 | glog.V(1).Info("Received ", text) 72 | 73 | peerID, err := strconv.Atoi(strings.Trim(text, "\n")) 74 | if err != nil { 75 | glog.Warning(err) 76 | return 77 | } 78 | 79 | // check ID is expected 80 | if peerID < 0 || peerID >= len(ph.peers) || peerID == ph.id { 81 | glog.Warning("Unexpected peer ID ", peerID) 82 | return 83 | } 84 | 85 | // check IP address is as expected 86 | // TODO: allow dynamic changes of IP 87 | actualAddr := strings.Split(addr, ":")[0] 88 | if ph.peers[peerID].Address != actualAddr { 89 | glog.Warning("Peer ID ", peerID, " has connected from an unexpected address ", actualAddr, 90 | " expected ", ph.peers[peerID].Address) 91 | return 92 | } 93 | 94 | glog.Infof("Ready to handle traffic from peer %d at %s ", peerID, addr) 95 | err = ph.failures.NowConnected(peerID) 96 | if err != nil { 97 | glog.Warning(err) 98 | } 99 | 100 | closeErr := make(chan error) 101 | go func() { 102 | for { 103 | // read request 104 | glog.V(1).Infof("Ready for next message from %d", peerID) 105 | text, err := reader.ReadBytes(byte('\n')) 106 | if err != nil { 107 | glog.Warning(err) 108 | closeErr <- err 109 | break 110 | } 111 | glog.V(1).Infof("Read from peer %d: ", peerID, string(text)) 112 | err = ph.net.Incoming.BytesToProtoMsg(text) 113 | if err != nil { 114 | glog.Warning(err) 115 | } 116 | 117 | } 118 | }() 119 | 120 | go func() { 121 | for { 122 | // send reply 123 | glog.V(1).Infof("Ready to send message to %d", peerID) 124 | b, err := ph.net.OutgoingUnicast[peerID].ProtoMsgToBytes() 125 | if err != nil { 126 | glog.Fatal("Could not marshal message") 127 | } 128 | glog.V(1).Infof("Sending to %d: %s", peerID, string(b)) 129 | _, err = writer.Write(b) 130 | if err != nil { 131 | glog.Warning(err) 132 | // return packet for retry 133 | ph.net.OutgoingUnicast[peerID].BytesToProtoMsg(b) 134 | closeErr <- err 135 | break 136 | } 137 | _, err = writer.Write([]byte("\n")) 138 | if err != nil { 139 | glog.Warning(err) 140 | // return packet for retry 141 | ph.net.OutgoingUnicast[peerID].BytesToProtoMsg(b) 142 | closeErr <- err 143 | break 144 | } 145 | // TODO: BUG need to retry packet 146 | err = writer.Flush() 147 | if err != nil { 148 | glog.Warning(err) 149 | // return packet for retry 150 | ph.net.OutgoingUnicast[peerID].BytesToProtoMsg(b) 151 | closeErr <- err 152 | break 153 | } 154 | glog.V(1).Info("Sent") 155 | } 156 | }() 157 | 158 | // block until connection fails 159 | <-closeErr 160 | 161 | // tidy up 162 | glog.Warningf("No longer able to handle traffic from peer %d at %s ", peerID, addr) 163 | ph.failures.NowDisconnected(peerID) 164 | } 165 | 166 | // SetupPeers is an async function to handle/start peer connections 167 | // TODO: switch to sync function 168 | func SetupPeers(localId int, addresses []config.NetAddress, peerNet *msgs.PeerNet, fail *msgs.FailureNotifier) error { 169 | peerHandler := peerHandler{ 170 | id: localId, 171 | peers: addresses, 172 | failures: fail, 173 | net: peerNet, 174 | } 175 | 176 | //set up peer server 177 | glog.Info("Starting up peer server on ", addresses[peerHandler.id].Port) 178 | listeningPort := ":" + strconv.Itoa(addresses[peerHandler.id].Port) 179 | lnPeers, err := net.Listen("tcp", listeningPort) 180 | if err != nil { 181 | glog.Info("Unable to start listen for peers") 182 | return err 183 | } 184 | 185 | // handle local peer (without sending network traffic) 186 | peerHandler.failures.NowConnected(peerHandler.id) 187 | from := &(peerHandler.net.Incoming) 188 | go from.Forward(peerHandler.net.OutgoingUnicast[peerHandler.id]) 189 | 190 | // handle for incoming peers 191 | go func() { 192 | for { 193 | conn, err := lnPeers.Accept() 194 | if err != nil { 195 | glog.Fatal(err) 196 | } 197 | go (&peerHandler).handlePeer(conn, false) 198 | } 199 | }() 200 | 201 | // regularly check if all peers are connected and retry if not 202 | go func() { 203 | for { 204 | (&peerHandler).checkPeer() 205 | time.Sleep(500 * time.Millisecond) 206 | } 207 | }() 208 | 209 | return nil 210 | } 211 | -------------------------------------------------------------------------------- /scripts/coverage: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Generate test coverage statistics for Go packages. 3 | # 4 | # Works around the fact that `go test -coverprofile` currently does not work 5 | # with multiple packages, see https://code.google.com/p/go/issues/detail?id=6909 6 | # 7 | # Usage: script/coverage [--html|--coveralls] 8 | # 9 | # --html Additionally create HTML report and open it in browser 10 | # --coveralls Push coverage statistics to coveralls.io 11 | # 12 | 13 | set -e 14 | 15 | workdir=.cover 16 | profile="$workdir/cover.out" 17 | mode=count 18 | 19 | generate_cover_data() { 20 | rm -rf "$workdir" 21 | mkdir "$workdir" 22 | 23 | for pkg in "$@"; do 24 | f="$workdir/$(echo $pkg | tr / -).cover" 25 | go test -covermode="$mode" -coverprofile="$f" "$pkg" 26 | done 27 | 28 | echo "mode: $mode" >"$profile" 29 | grep -h -v "^mode:" "$workdir"/*.cover >>"$profile" 30 | } 31 | 32 | show_cover_report() { 33 | go tool cover -${1}="$profile" 34 | } 35 | 36 | push_to_coveralls() { 37 | echo "Pushing coverage statistics to coveralls.io" 38 | goveralls -coverprofile="$profile" 39 | } 40 | 41 | generate_cover_data $(go list ./...) 42 | show_cover_report func 43 | case "$1" in 44 | "") 45 | ;; 46 | --html) 47 | show_cover_report html ;; 48 | --coveralls) 49 | push_to_coveralls ;; 50 | *) 51 | echo >&2 "error: invalid option: $1"; exit 1 ;; 52 | esac 53 | -------------------------------------------------------------------------------- /scripts/docker/deploy3servers: -------------------------------------------------------------------------------- 1 | docker network create ios --driver=bridge --subnet=192.168.1.0/16 2 | docker run -d --ip=192.168.1.1 --net=ios --name node0 ios -id 0 -config src/github.com/heidi-ann/ios/scripts/docker/example3.conf 3 | docker run -d --ip=192.168.1.2 --net=ios --name node1 ios -id 1 -config src/github.com/heidi-ann/ios/scripts/docker/example3.conf 4 | docker run -d --ip=192.168.1.3 --net=ios --name node2 ios -id 2 -config src/github.com/heidi-ann/ios/scripts/docker/example3.conf 5 | docker run --net=ios --name client --entrypoint test ios -config src/github.com/heidi-ann/ios/scripts/docker/example3.conf --stderrthreshold=INFO 6 | -------------------------------------------------------------------------------- /scripts/docker/deploy3serversfailed: -------------------------------------------------------------------------------- 1 | docker network create ios --driver=bridge --subnet=192.168.1.0/16 2 | docker run -d --ip=192.168.1.1 --net=ios -p 8080:8080 -p 8090:8090 --name node0 ios -id 0 -config src/github.com/heidi-ann/ios/scripts/docker/server-example.conf -logtostderr=true 3 | docker run -d --ip=192.168.1.2 --net=ios -p 8081:8080 -p 8091:8090 --name node1 ios -id 1 -config src/github.com/heidi-ann/ios/scripts/docker/server-example.conf -logtostderr=true 4 | docker run -d --ip=192.168.1.3 --net=ios -p 8082:8080 -p 8092:8090 --name node2 ios -id 2 -config src/github.com/heidi-ann/ios/scripts/docker/server-example.conf -logtostderr=true 5 | docker run --net=ios -it --name test --entrypoint test ios -config src/github.com/heidi-ann/ios/scripts/docker/client-example.conf -logtostderr=true 6 | docker logs node0 7 | docker logs node1 8 | docker logs node2 9 | docker stop node0 10 | docker start test 11 | docker exec -it test test -config src/github.com/heidi-ann/ios/scripts/docker/client-example.conf -logtostderr=true 12 | docker logs node1 13 | docker logs node2 14 | docker exec -t test cat latency.csv 15 | -------------------------------------------------------------------------------- /scripts/docker/deploycluster: -------------------------------------------------------------------------------- 1 | docker-compose up & 2 | $GOPATH/bin/test -config=$GOPATH/src/github.com/heidi-ann/ios/example3.conf 3 | docker-compose down 4 | -------------------------------------------------------------------------------- /scripts/docker/example3.conf: -------------------------------------------------------------------------------- 1 | ; servers to try connecting to, in order 2 | [servers] 3 | address = 192.168.1.1:8090:8080 4 | address = 192.168.1.2:8091:8081 5 | address = 192.168.1.3:8092:8082 6 | -------------------------------------------------------------------------------- /scripts/docker/stop3servers: -------------------------------------------------------------------------------- 1 | docker stop node0 2 | docker stop node1 3 | docker stop node2 4 | docker rm node0 5 | docker rm node1 6 | docker rm node2 7 | docker rm client 8 | docker network rm ios 9 | -------------------------------------------------------------------------------- /services/dummy.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "encoding/json" 5 | ) 6 | 7 | type dummy struct { 8 | requests int 9 | } 10 | 11 | func newDummy() *dummy { 12 | return &dummy{0} 13 | } 14 | 15 | func (d *dummy) Process(request string) string { 16 | if request == "ping" { 17 | d.requests++ 18 | return "pong" 19 | } 20 | return "" 21 | } 22 | 23 | func (d *dummy) CheckFormat(req string) bool { 24 | switch req { 25 | case "ping": 26 | return true 27 | default: 28 | return false 29 | } 30 | } 31 | 32 | func (D *dummy) CheckRead(req string) bool { 33 | return false 34 | } 35 | 36 | func (d *dummy) MarshalJSON() ([]byte, error) { 37 | return json.Marshal(*d) 38 | } 39 | 40 | func (d *dummy) UnmarshalJSON(_ []byte) error { 41 | // TODO: finish placeholder 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /services/services.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | ) 6 | 7 | type Service interface { 8 | Process(req string) string 9 | CheckFormat(req string) bool 10 | CheckRead(req string) bool 11 | MarshalJSON() ([]byte, error) 12 | UnmarshalJSON(snap []byte) error 13 | } 14 | 15 | func StartService(config string) Service { 16 | var serv Service 17 | switch config { 18 | case "kv-store": 19 | serv = newStore() 20 | case "dummy": 21 | serv = newDummy() 22 | default: 23 | glog.Fatal("No valid service specified") 24 | } 25 | return serv 26 | } 27 | 28 | func GetInteractiveText(config string) string { 29 | var s string 30 | switch config { 31 | case "kv-store": 32 | s = 33 | `The following commands are available: 34 | get [key]: to return the value of a given key 35 | exists [key]: to test if a given key is present 36 | update [key] [value]: to set the value of a given key, if key already exists then overwrite 37 | delete [key]: to remove a key value pair if present 38 | count: to return the number of keys 39 | print: to return all key value pairs 40 | ` 41 | case "dummy": 42 | s = 43 | `The following commands are available: 44 | ping: ping dummy application 45 | ` 46 | } 47 | return s 48 | } 49 | 50 | func Parse(config string, request string) (bool, bool) { 51 | serv := StartService(config) 52 | if !serv.CheckFormat(request) { 53 | return false, false 54 | } 55 | return true, serv.CheckRead(request) 56 | } 57 | -------------------------------------------------------------------------------- /services/store.go: -------------------------------------------------------------------------------- 1 | // Package services provides a simple key value store 2 | // Not safe for concurreny access 3 | // TODO: replace map with https://github.com/orcaman/concurrent-map 4 | package services 5 | 6 | import ( 7 | "encoding/json" 8 | "github.com/golang/glog" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | type store map[string]string 14 | 15 | func newStore() *store { 16 | var s store 17 | s = map[string]string{} 18 | return &s 19 | } 20 | 21 | func (s *store) execute(req string) string { 22 | request := strings.Split(req, " ") 23 | 24 | switch request[0] { 25 | case "update": 26 | if len(request) != 3 { 27 | return "not recognised" 28 | } 29 | glog.V(1).Infof("Updating %s to %s", request[1], request[2]) 30 | (*s)[request[1]] = request[2] 31 | return "OK" 32 | case "exists": 33 | if len(request) != 2 { 34 | return "request not recognised" 35 | } 36 | glog.V(1).Infof("Checking if %s exists", request[1]) 37 | _, exists := (*s)[request[1]] 38 | return strconv.FormatBool(exists) 39 | case "get": 40 | if len(request) != 2 { 41 | return "request not recognised" 42 | } 43 | glog.V(1).Infof("Getting %s", request[1]) 44 | value, ok := (*s)[request[1]] 45 | if ok { 46 | return value 47 | } else { 48 | return "key not found" 49 | } 50 | case "delete": 51 | if len(request) != 2 { 52 | return "request not recognised" 53 | } 54 | glog.V(1).Infof("Deleting %s", request[1]) 55 | delete(*s, request[1]) 56 | return "OK" 57 | case "count": 58 | if len(request) != 1 { 59 | return "request not recognised" 60 | } 61 | glog.V(1).Infof("Counting size of key-value store") 62 | return strconv.Itoa(len(*s)) 63 | case "print": 64 | if len(request) != 1 { 65 | return "request not recognised" 66 | } 67 | glog.V(1).Infof("Printing key-value store") 68 | return s.print() 69 | default: 70 | return "request not recognised" 71 | } 72 | } 73 | 74 | func (s *store) Process(req string) string { 75 | reqs := strings.Split(strings.Trim(req, "\n"), "; ") 76 | var reply string 77 | 78 | for i := range reqs { 79 | if i == 0 { 80 | reply = s.execute(reqs[i]) 81 | } else { 82 | reply = reply + "; " + s.execute(reqs[i]) 83 | } 84 | } 85 | return reply 86 | } 87 | 88 | func (s *store) CheckFormat(req string) bool { 89 | // TODO: check all requests, not just first 90 | request := strings.Split(strings.Trim(req, "\n"), " ") 91 | switch request[0] { 92 | case "update": 93 | return len(request) == 3 94 | case "exists", "get", "delete": 95 | return len(request) == 2 96 | case "count", "print": 97 | return len(request) == 1 98 | default: 99 | return false 100 | } 101 | } 102 | 103 | func (s *store) CheckRead(req string) bool { 104 | // TODO: check all request, not just first 105 | request := strings.Split(strings.Trim(req, "\n"), " ") 106 | switch request[0] { 107 | case "exists", "get", "count", "print": 108 | return true 109 | default: 110 | return false 111 | } 112 | } 113 | 114 | func (s *store) print() string { 115 | str := "" 116 | for key, value := range *s { 117 | str += key + ", " + value + "\n" 118 | } 119 | return str 120 | } 121 | 122 | func (s *store) MarshalJSON() ([]byte, error) { 123 | return json.Marshal(*s) 124 | } 125 | 126 | func (s *store) UnmarshalJSON(snap []byte) error { 127 | // this seems like a strange approach but unmarshalling directly into store causes memory leak 128 | var sTemp map[string]string 129 | json.Unmarshal(snap, &sTemp) 130 | for key, value := range sTemp { 131 | (*s)[key] = value 132 | } 133 | return nil 134 | } 135 | -------------------------------------------------------------------------------- /services/store_test.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | 7 | "fmt" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestProcess(t *testing.T) { 13 | store := newStore() 14 | 15 | cases := []struct { 16 | req, res string 17 | }{ 18 | {"update A 3", "OK"}, 19 | {"get A", "3"}, 20 | {"print", "A, 3\n"}, 21 | {"count", "1"}, 22 | {"exists A", "true"}, 23 | {"delete A", "OK"}, 24 | {"count", "0"}, 25 | {"exists A", "false"}, 26 | {"get B", "key not found"}, 27 | } 28 | 29 | for _, c := range cases { 30 | got := store.Process(c.req) 31 | if got != c.res { 32 | t.Errorf("%s returned %s but %s was expected", c.req, got, c.res) 33 | } 34 | } 35 | } 36 | 37 | func TestCheckFormat(t *testing.T) { 38 | store := StartService("kv-store") 39 | 40 | cases := []struct { 41 | req string 42 | res bool 43 | }{ 44 | {"update foo bar", true}, 45 | {"get ;234", true}, 46 | {"print", true}, 47 | {"count", true}, 48 | {"exists $£$%", true}, 49 | {"delete _)(*", true}, 50 | {"get B F", false}, 51 | {"", false}, 52 | } 53 | for _, c := range cases { 54 | got := store.CheckFormat(c.req) 55 | if got != c.res { 56 | assert.Equal(t, c.res, got, fmt.Sprintf("Error for case: %s", c.req)) 57 | } 58 | } 59 | 60 | dummy := StartService("dummy") 61 | assert.True(t, dummy.CheckFormat("ping")) 62 | assert.False(t, dummy.CheckFormat("")) 63 | assert.False(t, dummy.CheckFormat("update A 1")) 64 | } 65 | 66 | func TestMarshalling(t *testing.T) { 67 | store := StartService("kv-store") 68 | // add data to store 69 | store.Process("update A 1") 70 | for i := 0; i < 100; i++ { 71 | store.Process(fmt.Sprintf("update %v %v", i, i)) 72 | } 73 | store.Process("delete 0") 74 | 75 | //marshal store 76 | json, err := store.MarshalJSON() 77 | if err != nil { 78 | t.Fatal("err marshalling store. Error: ", err.Error()) 79 | } 80 | //make more changes (should be overwritten when unmarshalling) 81 | store.Process("delete 1") 82 | 83 | //unmarshal and validate store 84 | err = store.UnmarshalJSON(json) 85 | if err != nil { 86 | t.Fatal("err unmarshalling store. Error: ", err.Error()) 87 | } 88 | 89 | assert.Equal(t, "1", store.Process("get A")) 90 | assert.Equal(t, "false", store.Process("exists 0")) 91 | assert.Equal(t, "100", store.Process("count")) 92 | for i := 1; i < 100; i++ { 93 | assert.Equal(t, strconv.Itoa(i), store.Process(fmt.Sprintf("get %v", i))) 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /simulator/simulator.go: -------------------------------------------------------------------------------- 1 | // Package simulator provides an interface with package consensus without I/O. 2 | package simulator 3 | 4 | import ( 5 | "github.com/heidi-ann/ios/app" 6 | "github.com/heidi-ann/ios/consensus" 7 | "github.com/heidi-ann/ios/msgs" 8 | ) 9 | 10 | func runSimulator(config consensus.Config) ([]*msgs.PeerNet, []*msgs.ClientNet, []*msgs.FailureNotifier) { 11 | nodes := config.All.N 12 | peerNets := make([]*msgs.PeerNet, nodes) 13 | clientNets := make([]*msgs.ClientNet, nodes) 14 | failures := make([]*msgs.FailureNotifier, nodes) 15 | // setup state 16 | for id := 0; id < nodes; id++ { 17 | app := app.New("kv-store") 18 | peerNet := msgs.MakePeerNet(10, nodes) 19 | clientNet := msgs.MakeClientNet(10) 20 | fail := msgs.NewFailureNotifier(nodes) 21 | storage := msgs.MakeDummyStorage() 22 | config.All.ID = id 23 | go consensus.Init(peerNet, clientNet, config, app, fail, storage) 24 | peerNets[id] = peerNet 25 | clientNets[id] = clientNet 26 | failures[id] = fail 27 | } 28 | 29 | // forward traffic 30 | for to := range peerNets { 31 | for from := range peerNets { 32 | go peerNets[to].Incoming.Forward(peerNets[from].OutgoingUnicast[to]) 33 | } 34 | } 35 | 36 | return peerNets, clientNets, failures 37 | } 38 | 39 | // // same as runSimulator except where the log in persistent storage is given 40 | // func runRecoverySimulator(nodes int, logs []*consensus.Log, views []int) []*msgs.Io { 41 | // ios := make([]*msgs.Io, nodes) 42 | // // setup state 43 | // for id := 0; id < nodes; id++ { 44 | // app := app.New("kv-store") 45 | // io := msgs.MakeIo(10, nodes) 46 | // failure := msgs.NewFailureNotifier(nodes) 47 | // storage := msgs.MakeDummyStorage() 48 | // conf := consensus.Config{ID: id, N: nodes, LogLength: 1000} 49 | // go consensus.Recover(io, conf, views[id], logs[id], app, -1, failure, storage) 50 | // ios[id] = io 51 | // } 52 | // 53 | // // forward traffic 54 | // for to := range ios { 55 | // for from := range ios { 56 | // go ios[to].Incoming.Forward(ios[from].OutgoingUnicast[to]) 57 | // } 58 | // } 59 | // 60 | // return ios 61 | // } 62 | -------------------------------------------------------------------------------- /simulator/simulator_test.go: -------------------------------------------------------------------------------- 1 | package simulator 2 | 3 | import ( 4 | "flag" 5 | "github.com/golang/glog" 6 | "github.com/heidi-ann/ios/app" 7 | "github.com/heidi-ann/ios/consensus" 8 | "github.com/heidi-ann/ios/msgs" 9 | "github.com/stretchr/testify/assert" 10 | "strconv" 11 | "testing" 12 | "time" 13 | ) 14 | 15 | func checkRequest(t *testing.T, req msgs.ClientRequest, reply msgs.ClientResponse, clientNets []*msgs.ClientNet, masterID int) { 16 | // send request direct to master 17 | clientNets[masterID].IncomingRequests <- req 18 | 19 | select { 20 | case response := <-(clientNets[masterID]).OutgoingResponses: 21 | assert.Equal(t, req, response.Request) 22 | assert.Equal(t, reply, response.Response) 23 | case <-time.After(time.Second): 24 | assert.Fail(t, "Participant not responding", strconv.Itoa(masterID)) 25 | } 26 | } 27 | 28 | func TestRunSimulator(t *testing.T) { 29 | flag.Parse() 30 | defer glog.Flush() 31 | 32 | quorum, err := consensus.NewQuorum("strict majority", 3) 33 | assert.Nil(t, err) 34 | // create a system of 3 nodes 35 | config := consensus.Config{ 36 | All: consensus.ConfigAll{ 37 | ID: 0, 38 | N: 3, 39 | WindowSize: 1, 40 | Quorum: quorum, 41 | }, 42 | Master: consensus.ConfigMaster{ 43 | BatchInterval: 0, 44 | MaxBatch: 1, 45 | DelegateReplication: 0, 46 | IndexExclusivity: true, 47 | }, 48 | Coordinator: consensus.ConfigCoordinator{ 49 | ExplicitCommit: true, 50 | ThriftyQuorum: false, 51 | }, 52 | Participant: consensus.ConfigParticipant{ 53 | SnapshotInterval: 1000, 54 | ImplicitWindowCommit: false, 55 | LogLength: 10000, 56 | }, 57 | Interfacer: consensus.ConfigInterfacer{ 58 | ParticipantHandle: false, 59 | ParticipantRead: false, 60 | }, 61 | } 62 | 63 | peerNets, clientNets, _ := runSimulator(config) 64 | app := app.New("kv-store") 65 | 66 | // check that 3 nodes were created 67 | if len(peerNets) != 3 { 68 | t.Error("Correct number of nodes not created") 69 | } 70 | 71 | // check that master can replicate a request when no failures occur 72 | request1 := msgs.ClientRequest{ 73 | ClientID: 200, 74 | RequestID: 1, 75 | ForceViewChange: false, 76 | ReadOnly: false, 77 | Request: "update A 3"} 78 | 79 | checkRequest(t, request1, app.Apply(request1), clientNets, 0) 80 | 81 | request2 := msgs.ClientRequest{ 82 | ClientID: 200, 83 | RequestID: 2, 84 | ForceViewChange: false, 85 | ReadOnly: false, 86 | Request: "get A"} 87 | 88 | checkRequest(t, request2, app.Apply(request2), clientNets, 0) 89 | 90 | request3 := msgs.ClientRequest{ 91 | ClientID: 400, 92 | RequestID: 1, 93 | ForceViewChange: false, 94 | ReadOnly: false, 95 | Request: "get C"} 96 | 97 | checkRequest(t, request3, app.Apply(request3), clientNets, 0) 98 | 99 | //check failure recovery by notifying node 1 that node 0 has failed 100 | // failures[1].NowConnected(0) 101 | // failures[1].NowDisconnected(0) 102 | 103 | request4 := msgs.ClientRequest{ 104 | ClientID: 400, 105 | RequestID: 2, 106 | ForceViewChange: false, 107 | ReadOnly: false, 108 | Request: "get A"} 109 | 110 | checkRequest(t, request4, app.Apply(request4), clientNets, 0) 111 | 112 | // check 2nd failure by notifying node 2 that node 1 has failed 113 | // failures[2].NowConnected(1) 114 | // failures[2].NowDisconnected(1) 115 | 116 | request5 := msgs.ClientRequest{ 117 | ClientID: 400, 118 | RequestID: 3, 119 | ForceViewChange: false, 120 | ReadOnly: false, 121 | Request: "update B 3"} 122 | 123 | checkRequest(t, request5, app.Apply(request5), clientNets, 0) 124 | 125 | request6 := msgs.ClientRequest{ 126 | ClientID: 400, 127 | RequestID: 4, 128 | ForceViewChange: false, 129 | ReadOnly: false, 130 | Request: "get B"} 131 | 132 | checkRequest(t, request6, app.Apply(request6), clientNets, 0) 133 | 134 | } 135 | -------------------------------------------------------------------------------- /storage/persist.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | "github.com/heidi-ann/ios/msgs" 6 | "os" 7 | "strconv" 8 | ) 9 | 10 | type FileStorage struct { 11 | viewFile wal 12 | logFile wal 13 | snapFile wal 14 | } 15 | 16 | // MakeFileStorage opens the files for persistent storage 17 | func MakeFileStorage(diskPath string, persistenceMode string) (*FileStorage, error) { 18 | // create disk path if needs be 19 | if _, err := os.Stat(diskPath); os.IsNotExist(err) { 20 | err = os.MkdirAll(diskPath, 0777) 21 | if err != nil { 22 | return nil, err 23 | } 24 | } 25 | 26 | dataFilename := diskPath + "/view.temp" 27 | viewFile, err := openWriteAheadFile(dataFilename, persistenceMode, 64) 28 | if err != nil { 29 | return nil, err 30 | } 31 | logFilename := diskPath + "/log.temp" 32 | logFile, err := openWriteAheadFile(logFilename, persistenceMode, 4*1000*1000) 33 | if err != nil { 34 | return nil, err 35 | } 36 | snapFilename := diskPath + "/snapshot.temp" 37 | snapFile, err := openWriteAheadFile(snapFilename, persistenceMode, 4*1000*1000) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | s := FileStorage{viewFile, logFile, snapFile} 43 | return &s, nil 44 | } 45 | 46 | // PersistView writes view update to persistent storage 47 | // Function returns when write+sync is completed 48 | // Error is returned if it was not possible to write 49 | func (fs *FileStorage) PersistView(view int) error { 50 | glog.Info("Updating view to ", view, " in persistent storage") 51 | v := []byte(strconv.Itoa(view)) 52 | return fs.viewFile.writeAhead(v) 53 | } 54 | 55 | // PersistLogUpdate writes the specified log update to persistent storage 56 | // Function returns when write+sync is completed 57 | // Error is returned if it was not possible to write 58 | func (fs *FileStorage) PersistLogUpdate(log msgs.LogUpdate) error { 59 | glog.V(1).Info("Updating log with ", log, " in persistent storage") 60 | b, err := msgs.Marshal(log) 61 | if err != nil { 62 | return err 63 | } 64 | // write to persistent storage 65 | if err := fs.logFile.writeAhead(b); err != nil { 66 | return err 67 | } 68 | return nil 69 | } 70 | 71 | // PersistSnapshot writes the provided snapshot to persistent storage 72 | // index should be the index of the last request applied to be state machine before snapshoting 73 | func (fs *FileStorage) PersistSnapshot(index int, bytes []byte) error { 74 | glog.Info("Saving request cache and state machine snapshot upto index", index, 75 | " of size ", len(bytes)) 76 | if err := fs.snapFile.writeAhead([]byte(strconv.Itoa(index))); err != nil { 77 | return err 78 | } 79 | if err := fs.snapFile.writeAhead(bytes); err != nil { 80 | return err 81 | } 82 | 83 | return nil 84 | // TODO: garbage collection - remove log entries/snapshots from before index 85 | } 86 | -------------------------------------------------------------------------------- /storage/persist_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "flag" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | "github.com/golang/glog" 10 | "github.com/heidi-ann/ios/app" 11 | "github.com/heidi-ann/ios/msgs" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func restoreStorageEmpty(t *testing.T, dir string) { 16 | assert := assert.New(t) 17 | found, view, log, index, state := RestoreStorage(dir, 1000, "kv-store") 18 | assert.False(found, "Unexpected persistent storage found") 19 | assert.Equal(0, view, "Unexpected view") 20 | assert.Equal(make([]msgs.Entry, 1000), log.LogEntries, "Unexpected log entries") 21 | assert.Equal(-1, index, "Unexpected index") 22 | assert.Equal(app.New("kv-store"), state, "Unexpected kv store") 23 | } 24 | 25 | func TestPersistentStorage(t *testing.T) { 26 | flag.Parse() 27 | defer glog.Flush() 28 | assert := assert.New(t) 29 | 30 | //Create temp directory 31 | dir, err := ioutil.TempDir("", "IosPersistentStorageTests") 32 | if err != nil { 33 | glog.Fatal(err) 34 | } 35 | defer os.RemoveAll(dir) // clean up 36 | 37 | // check recovery when no files exist 38 | restoreStorageEmpty(t, dir) 39 | 40 | //check file creation 41 | fs, err := MakeFileStorage(dir, "fsync") 42 | assert.Nil(err) 43 | restoreStorageEmpty(t, dir) 44 | 45 | //verify files were created in directory 46 | files, err := ioutil.ReadDir(dir) 47 | if err != nil { 48 | glog.Fatal(err) 49 | } 50 | 51 | filenames := make([]string, len(files)) 52 | for i, file := range files { 53 | filenames[i] = file.Name() 54 | } 55 | assert.EqualValues([]string{"log.temp", "snapshot.temp", "view.temp"}, filenames) 56 | 57 | //verfiy that view storage works 58 | viewFile := dir + "/view.temp" 59 | found, view, err := restoreView(viewFile) 60 | assert.Nil(err) 61 | assert.False(found, "Unexpected view found") 62 | for v := 0; v < 5; v++ { 63 | err = fs.PersistView(v) 64 | assert.Nil(err) 65 | found, view, err = restoreView(viewFile) 66 | assert.Nil(err) 67 | assert.True(found, "Missing view in ", viewFile) 68 | assert.Equal(v, view, "Incorrect view") 69 | } 70 | 71 | //verfiy that log storage works 72 | logFile := dir + "/log.temp" 73 | found, log, err := restoreLog(logFile, 100, -1) 74 | assert.Nil(err) 75 | assert.False(found, "Unexpected log found") 76 | 77 | req1 := msgs.ClientRequest{ 78 | ClientID: 1, 79 | RequestID: 1, 80 | ForceViewChange: false, 81 | Request: "update A 1"} 82 | 83 | req2 := msgs.ClientRequest{ 84 | ClientID: 1, 85 | RequestID: 2, 86 | ForceViewChange: false, 87 | Request: "update B 2"} 88 | 89 | entry1 := msgs.Entry{ 90 | View: 0, 91 | Committed: false, 92 | Requests: []msgs.ClientRequest{req1}} 93 | 94 | up1 := msgs.LogUpdate{ 95 | StartIndex: 0, 96 | EndIndex: 1, 97 | Entries: []msgs.Entry{entry1}} 98 | 99 | err = fs.PersistLogUpdate(up1) 100 | assert.Nil(err) 101 | found, log, err = restoreLog(logFile, 100, -1) 102 | assert.Nil(err) 103 | assert.True(found, "Log expected but is missing") 104 | assert.Equal(entry1, log.GetEntry(0), "Log entry not as expected") 105 | 106 | //verify that snapshot storage works 107 | snapFile := dir + "/snapshot.temp" 108 | found, index, actualSm, err := restoreSnapshot(snapFile, "kv-store") 109 | assert.Nil(err) 110 | assert.False(found, "Unexpected log found") 111 | assert.Equal(-1, index, "Unexpected index") 112 | assert.Equal(app.New("kv-store"), actualSm, "Unexpected kv store") 113 | 114 | sm := app.New("kv-store") 115 | sm.Apply(req1) 116 | snap, err := sm.MakeSnapshot() 117 | assert.Nil(err) 118 | err = fs.PersistSnapshot(0, snap) 119 | assert.Nil(err) 120 | found, index, actualSm, err = restoreSnapshot(snapFile, "kv-store") 121 | assert.Nil(err) 122 | assert.True(found, "Unexpected log missing") 123 | assert.Equal(0, index, "Unexpected index") 124 | assert.Equal(sm, actualSm, "Missing kv store") 125 | 126 | // try 2nd snapshot 127 | sm.Apply(req2) 128 | snap, err = sm.MakeSnapshot() 129 | assert.Nil(err) 130 | err = fs.PersistSnapshot(1, snap) 131 | assert.Nil(err) 132 | found, index, actualSm, err = restoreSnapshot(snapFile, "kv-store") 133 | assert.Nil(err) 134 | assert.True(found, "Unexpected log missing") 135 | assert.Equal(1, index, "Unexpected index") 136 | assert.Equal(sm, actualSm, "Missing kv store") 137 | 138 | } 139 | -------------------------------------------------------------------------------- /storage/reader.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "bufio" 5 | "os" 6 | 7 | "github.com/golang/glog" 8 | ) 9 | 10 | type fileReader struct { 11 | filename string 12 | rd *bufio.Reader 13 | fd *os.File 14 | } 15 | 16 | func openReader(filename string) (bool, *fileReader, error) { 17 | // check if file exists 18 | if _, err := os.Stat(filename); os.IsNotExist(err) { 19 | return false, nil, nil 20 | } 21 | 22 | // open file 23 | glog.Info("Opening file: ", filename) 24 | file, err := os.OpenFile(filename, os.O_RDONLY|os.O_CREATE, 0777) 25 | if err != nil { 26 | return false, nil, err 27 | } 28 | 29 | // create reader 30 | r := bufio.NewReader(file) 31 | return true, &fileReader{filename, r, file}, nil 32 | } 33 | 34 | func (r *fileReader) read() ([]byte, error) { 35 | return r.rd.ReadBytes(byte('\n')) 36 | } 37 | 38 | func (r *fileReader) closeReader() error { 39 | return r.fd.Close() 40 | } 41 | -------------------------------------------------------------------------------- /storage/restore.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "os" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/heidi-ann/ios/app" 11 | "github.com/heidi-ann/ios/consensus" 12 | "github.com/heidi-ann/ios/msgs" 13 | 14 | "github.com/golang/glog" 15 | ) 16 | 17 | // restoreLog looks for an existing log file and if found, recoveres an Ios log from it. 18 | func restoreLog(logFilename string, maxLength int, snapshotIndex int) (bool, *consensus.Log, error) { 19 | exists, logFile, err := openReader(logFilename) 20 | if err != nil { 21 | return false, nil, err 22 | } 23 | 24 | if !exists { 25 | return false, consensus.NewLog(maxLength), nil 26 | } 27 | 28 | found := false 29 | log := consensus.RestoreLog(maxLength, snapshotIndex) 30 | 31 | for { 32 | b, err := logFile.read() 33 | if err != nil { 34 | glog.V(1).Info("No more commands in persistent storage, ", log.LastIndex, " log entries were recovered") 35 | break 36 | } 37 | found = true 38 | var update msgs.LogUpdate 39 | err = msgs.Unmarshal(b, &update) 40 | if err != nil { 41 | return true, nil, errors.New("Log file corrupted") 42 | } 43 | // add enties to the log (in-memory) 44 | log.AddEntries(update.StartIndex, update.EndIndex, update.Entries) 45 | glog.V(1).Info("Adding from persistent storage :", update) 46 | } 47 | 48 | return found, log, logFile.closeReader() 49 | } 50 | 51 | // restoreView looks for an existing metadata file and if found, recoveres an Ios view from it. 52 | func restoreView(viewFilename string) (bool, int, error) { 53 | exists, viewFile, err := openReader(viewFilename) 54 | if err != nil { 55 | return false, 0, nil 56 | } 57 | 58 | if !exists { 59 | glog.Info("No view update file found") 60 | return false, 0, nil 61 | } 62 | 63 | found := 0 64 | view := 0 65 | 66 | for { 67 | b, err := viewFile.read() 68 | if err == io.EOF { 69 | break 70 | } 71 | if err != nil { 72 | return true, 0, errors.New("View storage corrupted") 73 | } 74 | view, err = strconv.Atoi(strings.Trim(string(b), "\n")) 75 | if err != nil { 76 | return true, 0, errors.New("View storage corrupted") 77 | } 78 | found++ 79 | } 80 | 81 | if found > 0 { 82 | glog.Info("No more view updates in persistent storage, most recent view is ", view) 83 | } else { 84 | glog.Info("No view updates found in persistent storage") 85 | } 86 | 87 | return found > 0, view, viewFile.closeReader() 88 | } 89 | 90 | func restoreSnapshot(snapFilename string, appConfig string) (bool, int, *app.StateMachine, error) { 91 | exists, snapFile, err := openReader(snapFilename) 92 | if err != nil { 93 | return false, -1, nil, err 94 | } 95 | 96 | if !exists { 97 | return false, -1, app.New(appConfig), nil 98 | } 99 | 100 | found := false 101 | currIndex := -1 102 | stateMachine := app.New(appConfig) 103 | 104 | for { 105 | // fetching index from snapshot file 106 | b, err := snapFile.read() 107 | if err == io.EOF { 108 | break 109 | } 110 | if err != nil { 111 | glog.Warning("Snapshot corrupted, ignoring snapshot", err) 112 | break 113 | } 114 | index, err := strconv.Atoi(strings.Trim(string(b), "\n")) 115 | if err != nil { 116 | glog.Warning("Snapshot corrupted, ignoring snapshot", err) 117 | break 118 | } 119 | // fetch state machine snapshot from shapshot file 120 | snapshot, err := snapFile.read() 121 | if err != nil { 122 | glog.Warning("Snapshot corrupted, ignoring snapshot", err) 123 | break 124 | } 125 | // update with latest snapshot, now that it is completed 126 | found = true 127 | currIndex = index 128 | stateMachine, err = app.RestoreSnapshot(snapshot, appConfig) 129 | if err != nil { 130 | glog.Warning("Snapshot corrupted, ignoring snapshot", err) 131 | break 132 | } 133 | 134 | } 135 | 136 | return found, currIndex, stateMachine, snapFile.closeReader() 137 | } 138 | 139 | func RestoreStorage(diskPath string, maxLength int, appConfig string) (bool, int, *consensus.Log, int, *app.StateMachine) { 140 | // check if disk directory exists 141 | if _, err := os.Stat(diskPath); os.IsNotExist(err) { 142 | return false, 0, consensus.NewLog(maxLength), -1, app.New(appConfig) 143 | } 144 | 145 | // check persistent storage for view 146 | dataFile := diskPath + "/view.temp" 147 | foundView, view, err := restoreView(dataFile) 148 | if err != nil { 149 | glog.Fatal(err) 150 | } 151 | 152 | // check persistent storage for snapshots 153 | snapFile := diskPath + "/snapshot.temp" 154 | foundSnapshot, index, state, err := restoreSnapshot(snapFile, appConfig) 155 | if err != nil { 156 | glog.Fatal(err) 157 | } 158 | 159 | // check persistent storage for commands 160 | logFile := diskPath + "/log.temp" 161 | foundLog, log, err := restoreLog(logFile, maxLength, index) 162 | if err != nil { 163 | glog.Fatal(err) 164 | } 165 | 166 | // check results are consistent 167 | if foundLog && !foundView { 168 | glog.Fatal("Log is present but view is not, this should not occur") 169 | } 170 | if foundSnapshot && !foundView && !foundLog { 171 | glog.Fatal("Snapshot is present but view/log is not, this should not occur") 172 | } 173 | 174 | return foundView, view, log, index, state 175 | } 176 | -------------------------------------------------------------------------------- /storage/wal.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | "time" 6 | ) 7 | 8 | func slowDiskWarning(startTime time.Time, n int) { 9 | delay := time.Since(startTime) 10 | if delay > time.Millisecond { 11 | glog.Info("Slow disk warning - ", n, " bytes written & synced to persistent log in ", delay.String()) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /storage/wal_darwin.go: -------------------------------------------------------------------------------- 1 | // +build darwin 2 | 3 | package storage 4 | 5 | import ( 6 | "errors" 7 | "os" 8 | "syscall" 9 | "time" 10 | 11 | "github.com/golang/glog" 12 | ) 13 | 14 | type wal struct { 15 | fd int 16 | mode string 17 | } 18 | 19 | // openWriteAheadFile will open the specified file for the purpose of write ahead logging 20 | // error returned if file cannot be opened or the persistence mode is not none or fsync 21 | func openWriteAheadFile(filename string, mode string, _ int) (wal, error) { 22 | var WAL wal 23 | var err error 24 | WAL.mode = mode 25 | 26 | switch mode { 27 | case "none", "fsync": 28 | WAL.fd, err = syscall.Open(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 29 | if err != nil { 30 | glog.Warning(err) 31 | return WAL, err 32 | } 33 | glog.Info("Opened file: ", filename) 34 | return WAL, nil 35 | default: 36 | return WAL, errors.New("PersistenceMode not recognised, only fsync and none are avalible on darwin") 37 | } 38 | } 39 | 40 | // writeAhead writes the specified bytes to the write ahead file 41 | // function returns once bytes are written using specified persistence mode 42 | // error is returned if bytes could not be written 43 | func (w wal) writeAhead(bytes []byte) error { 44 | startTime := time.Now() 45 | n, err := syscall.Write(w.fd, bytes) 46 | if err != nil || n != len(bytes) { 47 | glog.Warning(err) 48 | return err 49 | } 50 | n2, err := syscall.Write(w.fd, []byte("\n")) 51 | if err != nil { 52 | glog.Warning(err) 53 | return err 54 | } 55 | glog.V(1).Info(n+n2, " bytes written to persistent log in ", time.Since(startTime).String()) 56 | 57 | if w.mode == "fsync" || w.mode == "direct" { 58 | err = syscall.Fsync(w.fd) 59 | if err != nil { 60 | glog.Warning(err) 61 | return err 62 | } 63 | glog.V(1).Info(n+n2, " bytes synced to persistent log in ", time.Since(startTime).String()) 64 | } 65 | slowDiskWarning(startTime, n+n2) 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /storage/wal_linux.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | package storage 4 | 5 | import ( 6 | "errors" 7 | "os" 8 | "syscall" 9 | "time" 10 | 11 | "github.com/golang/glog" 12 | ) 13 | 14 | type wal struct { 15 | fd int 16 | mode string 17 | } 18 | 19 | func openWriteAheadFile(filename string, mode string, size int) (wal, error) { 20 | var WAL wal 21 | WAL.mode = mode 22 | var err error 23 | 24 | switch mode { 25 | case "osync": 26 | WAL.fd, err = syscall.Open(filename, syscall.O_SYNC|os.O_WRONLY|os.O_CREATE, 0666) 27 | case "dsync": 28 | WAL.fd, err = syscall.Open(filename, syscall.O_DSYNC|os.O_WRONLY|os.O_CREATE, 0666) 29 | case "direct": 30 | WAL.fd, err = syscall.Open(filename, syscall.O_DIRECT|os.O_WRONLY|os.O_CREATE, 0666) 31 | case "none", "fsync": 32 | WAL.fd, err = syscall.Open(filename, os.O_WRONLY|os.O_CREATE, 0666) 33 | default: 34 | return WAL, errors.New("PersistenceMode not reconised") 35 | } 36 | if err != nil { 37 | return WAL, err 38 | } 39 | glog.Info("Opened file: ", filename) 40 | start, err := syscall.Seek(WAL.fd, 0, 2) 41 | if err != nil { 42 | return WAL, err 43 | } 44 | 45 | glog.Info("Starting file pointer ", start) 46 | err = syscall.Fallocate(WAL.fd, 0, 0, int64(size)) 47 | if err != nil { 48 | return WAL, err 49 | } 50 | return WAL, nil 51 | } 52 | 53 | func (w wal) writeAhead(bytes []byte) error { 54 | startTime := time.Now() 55 | // TODO: remove hack 56 | n, err := syscall.Write(w.fd, bytes) 57 | if err != nil { 58 | return err 59 | } 60 | if n != len(bytes) { 61 | return errors.New("Short write") 62 | } 63 | delim := []byte("\n") 64 | n2, err := syscall.Write(w.fd, delim) 65 | if err != nil { 66 | return err 67 | } 68 | if n2 != len(delim) { 69 | return errors.New("Short write") 70 | } 71 | glog.V(1).Info(n+n2, " bytes written to persistent log in ", time.Since(startTime).String()) 72 | if w.mode == "fsync" || w.mode == "direct" { 73 | err = syscall.Fdatasync(w.fd) 74 | if err != nil { 75 | return err 76 | } 77 | glog.V(1).Info(n+n2, " bytes synced to persistent log in ", time.Since(startTime).String()) 78 | } 79 | slowDiskWarning(startTime, n+n2) 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /storage/wal_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | "github.com/stretchr/testify/assert" 6 | "io/ioutil" 7 | "math/rand" 8 | "os" 9 | "testing" 10 | ) 11 | 12 | func TestOpenWriteAheadFile(t *testing.T) { 13 | assert := assert.New(t) 14 | 15 | //Create temp directory 16 | dir, err := ioutil.TempDir("", "IosWALTests") 17 | if err != nil { 18 | glog.Fatal(err) 19 | } 20 | defer os.RemoveAll(dir) // clean up 21 | 22 | //create file 23 | testFile := dir + "/test.temp" 24 | wal, err := openWriteAheadFile(testFile, "fsync", 2560) 25 | assert.Nil(err) 26 | actualBytes, err := ioutil.ReadFile(testFile) 27 | assert.Nil(err) 28 | // TODO; check file is expected size (maybe system dependant) 29 | glog.Info("File created of size ", len(actualBytes)) 30 | // TODO: check actualBytes are empty 31 | 32 | //verfiy that write ahead logging works 33 | start := 0 34 | history := make([]byte, 0, 1000) 35 | for size := 1; size < 100; size += 10 { 36 | expectedBytes := make([]byte, size) 37 | rand.Read(expectedBytes) 38 | err = wal.writeAhead(expectedBytes) 39 | assert.Nil(err) 40 | actualBytes, err = ioutil.ReadFile(testFile) 41 | assert.Nil(err) 42 | assert.Equal(expectedBytes, actualBytes[start:start+size], "Bytes read are not same as written") 43 | assert.Equal([]byte{0xa}, actualBytes[start+size:start+size+1], "Delim missing from end of write") 44 | assert.Equal(history[:start], actualBytes[:start], "Past writes have been corrupted") 45 | // TODO: check that rest of file is empty 46 | // update state for next run 47 | history = append(history, expectedBytes...) 48 | history = append(history, 0xa) 49 | start = start + size + 1 50 | } 51 | 52 | //verfiy that write ahead logging continues after failures 53 | openWriteAheadFile(testFile, "fsync", 2560) 54 | actualBytesf, err := ioutil.ReadFile(testFile) 55 | assert.Nil(err) 56 | glog.Info("File now of size ", len(actualBytes)) 57 | assert.Equal(history, actualBytesf[:start], "File has not recovered") 58 | 59 | // continue logging 60 | // for size := 1; size < 100; size += 10 { 61 | // expectedBytes := make([]byte, size) 62 | // rand.Read(expectedBytes) 63 | // err = walf.writeAhead(expectedBytes) 64 | // actualBytes, err = ioutil.ReadFile(testFile) 65 | // assert.Nil(err) 66 | // assert.Equal(expectedBytes, actualBytes[start:start+size], "Bytes read are not same as written") 67 | // assert.Equal([]byte{0xa}, actualBytes[start+size:start+size+1], "Delim missing from end of write") 68 | // assert.Equal(history[:start], actualBytes[:start], "Past writes have been corrupted") 69 | // // update state for next run 70 | // history = append(history, expectedBytes...) 71 | // history = append(history, 0xa) 72 | // start = start + size + 1 73 | // } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /storage/wal_unsupported.go: -------------------------------------------------------------------------------- 1 | // +build !darwin,!linux 2 | 3 | package storage 4 | 5 | import ( 6 | "github.com/golang/glog" 7 | "os" 8 | "time" 9 | ) 10 | 11 | type wal struct { 12 | file *os.File 13 | mode string 14 | } 15 | 16 | func openWriteAheadFile(filename string, mode string, _ int) (wal, error) { 17 | var file *os.File 18 | switch mode { 19 | case "none", "fsync": 20 | file, err = os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 21 | if err != nil { 22 | glog.Fatal(err) 23 | } 24 | default: 25 | glog.Fatal("PersistenceMode not recognised, only fsync and none are avalible on darwin") 26 | } 27 | return wal{file, mode}, nil 28 | } 29 | 30 | func (w wal) writeAhead(bytes []byte) error { 31 | // write bytes 32 | startTime := time.Now() 33 | n, err := w.file.Write(bytes) 34 | if err != nil { 35 | glog.Fatal(err) 36 | } 37 | n2, err := w.file.Write([]byte("\n")) 38 | if err != nil { 39 | glog.Fatal(err) 40 | } 41 | glog.V(1).Info(n+n2, " bytes written to persistent log in ", time.Since(startTime).String()) 42 | // sync if needed 43 | if w.mode == "fsync" { 44 | err = w.file.Sync() 45 | if err != nil { 46 | glog.Fatal(err) 47 | } 48 | glog.V(1).Info(n+n2, " bytes synced to persistent log in ", time.Since(startTime).String()) 49 | } 50 | slowDiskWarning(startTime, n+n2) 51 | return nil 52 | } 53 | -------------------------------------------------------------------------------- /test/generator/generator.go: -------------------------------------------------------------------------------- 1 | // Package generator provides test clients for benchmarking Ios's key value store performance. 2 | // Currently, generator only generates get and updates requests. 3 | package generator 4 | 5 | import ( 6 | "fmt" 7 | "github.com/golang/glog" 8 | "github.com/heidi-ann/ios/config" 9 | "github.com/heidi-ann/ios/services" 10 | "math/rand" 11 | "time" 12 | ) 13 | 14 | // Generator generates workloads for the key value store application. 15 | type Generator struct { 16 | config config.ConfigAuto // workload configuration. 17 | keys []string // key value store keys for the workload to operate on 18 | store services.Service // a local kv store to check consistency of responses 19 | request string // current outstanding request 20 | consistencyCheck bool // is consistency checking enabled 21 | } 22 | 23 | const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" 24 | 25 | // randStringBytes n generates a random alphanumeric string of n bytes. 26 | func randStringBytes(n int) string { 27 | b := make([]byte, n) 28 | for i := range b { 29 | b[i] = letterBytes[rand.Intn(len(letterBytes))] 30 | } 31 | return string(b) 32 | } 33 | 34 | // Generate creates a workload generator with the specific configuration. 35 | func Generate(config config.ConfigAuto, consistencyCheck bool) *Generator { 36 | keys := make([]string, config.Keys) 37 | for i := range keys { 38 | keys[i] = randStringBytes(config.KeySize) 39 | } 40 | return &Generator{config, keys, services.StartService("kv-store"), "", consistencyCheck} 41 | } 42 | 43 | // Next return the next request in the workload or false if no more are available. 44 | func (g *Generator) Next() (string, bool, bool) { 45 | //handle termination after n requests 46 | if g.config.Requests == 0 { 47 | return "", false, false 48 | } 49 | g.config.Requests-- 50 | 51 | delay := 0 52 | if g.config.Interval > 0 { 53 | delay = rand.Intn(g.config.Interval) 54 | } 55 | time.Sleep(time.Duration(delay) * time.Millisecond) 56 | 57 | // generate key 58 | key := g.keys[rand.Intn(g.config.Keys)] 59 | glog.V(1).Info("Key is ", key) 60 | 61 | if rand.Intn(100) < g.config.Reads { 62 | g.request = fmt.Sprintf("get %s", key) 63 | return g.request, true, true 64 | } 65 | value := randStringBytes(g.config.ValueSize) 66 | g.request = fmt.Sprintf("update %s %s", key, value) 67 | return g.request, false, true 68 | } 69 | 70 | // Return notifies the generator of Ios's response so it can check consistency 71 | func (g *Generator) Return(response string) { 72 | if g.consistencyCheck { 73 | expected := g.store.Process(g.request) 74 | if expected != response { 75 | glog.Fatal("Unexpected response ", response, " expected ", expected) 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /test/generator/generator_test.go: -------------------------------------------------------------------------------- 1 | package generator 2 | 3 | import ( 4 | "github.com/heidi-ann/ios/config" 5 | "github.com/heidi-ann/ios/services" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func checkFormatSize(t *testing.T, req string, keySize int, valueSize int) { 11 | request := strings.Split(strings.Trim(req, "\n"), " ") 12 | 13 | // check key size 14 | if len(request[1]) != keySize { 15 | t.Errorf("Incorrect key length: '%s'", req) 16 | } 17 | // check value size 18 | if request[0] == "update" && len(request[2]) != valueSize { 19 | t.Errorf("Incorrect key length: '%s'", req) 20 | } 21 | } 22 | 23 | func checkGenerateConfig(t *testing.T, conf config.ConfigAuto) { 24 | kv := services.StartService("kv-store") 25 | gen := Generate(conf, true) 26 | 27 | for i := 0; i < conf.Requests; i++ { 28 | str, _, ok := gen.Next() 29 | // check for early termination 30 | if !ok { 31 | if conf.Requests != i { 32 | t.Errorf("Generator terminated a request %d, should terminate at %d", 33 | i, conf.Requests) 34 | } 35 | break 36 | } 37 | // check format 38 | checkFormatSize(t, str, conf.KeySize, conf.ValueSize) 39 | if !kv.CheckFormat(str) { 40 | t.Errorf("incorrect format for kv store") 41 | } 42 | } 43 | // check for late termination 44 | _, _, ok := gen.Next() 45 | if ok { 46 | t.Errorf("Generator not terminated at %d", conf.Requests+1) 47 | } 48 | 49 | } 50 | 51 | // TestGenerates check that the generator is producing valid key value store commands 52 | func TestGenerates(t *testing.T) { 53 | conf := config.ConfigAuto{ 54 | Reads: 50, 55 | Interval: 0, 56 | KeySize: 8, 57 | ValueSize: 8, 58 | Requests: 20, 59 | Keys: 1, 60 | } 61 | checkGenerateConfig(t, conf) 62 | 63 | } 64 | -------------------------------------------------------------------------------- /test/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/golang/glog" 6 | "github.com/heidi-ann/ios/client" 7 | "github.com/heidi-ann/ios/config" 8 | "github.com/heidi-ann/ios/test/generator" 9 | "os" 10 | "os/signal" 11 | "syscall" 12 | ) 13 | 14 | var configFile = flag.String("config", os.Getenv("GOPATH")+"/src/github.com/heidi-ann/ios/example.conf", "Client configuration file") 15 | var autoFile = flag.String("auto", os.Getenv("GOPATH")+"/src/github.com/heidi-ann/ios/test/workloads/example.conf", "Configure file for workload") 16 | var algorithmFile = flag.String("algorithm", os.Getenv("GOPATH")+"/src/github.com/heidi-ann/ios/configfiles/simple/client.conf", "Algorithm description file") // optional flag 17 | var statFile = flag.String("stat", "latency.csv", "File to write stats to") 18 | var consistencyCheck = flag.Bool("check", false, "Enable consistency checking (use with only one client)") 19 | var id = flag.Int("id", -1, "ID of client (must be unique) or random number will be generated") 20 | 21 | func main() { 22 | // set up logging 23 | flag.Parse() 24 | defer glog.Flush() 25 | 26 | // always flush (whatever happens) 27 | sigs := make(chan os.Signal, 1) 28 | finish := make(chan bool, 1) 29 | signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) 30 | 31 | // parse config files 32 | c, err := client.StartClientFromConfigFile(*id, *statFile, *algorithmFile, *configFile) 33 | if err != nil { 34 | glog.Fatal(err) 35 | } 36 | 37 | // setup API 38 | workload, err := config.ParseWorkloadConfig(*autoFile) 39 | if err != nil { 40 | glog.Fatal(err) 41 | } 42 | if err = config.CheckWorkloadConfig(workload); err != nil { 43 | glog.Fatal(err) 44 | } 45 | ioapi := generator.Generate(workload, *consistencyCheck) 46 | 47 | go func() { 48 | for { 49 | // get next command 50 | text, read, ok := ioapi.Next() 51 | if !ok { 52 | finish <- true 53 | break 54 | } 55 | // pass to ios client 56 | reply, err := c.SubmitRequest(text, read) 57 | if err != nil { 58 | finish <- true 59 | break 60 | } 61 | // notify API of result 62 | ioapi.Return(reply) 63 | 64 | } 65 | }() 66 | 67 | select { 68 | case sig := <-sigs: 69 | glog.Warning("Termination due to: ", sig) 70 | case <-finish: 71 | glog.Info("No more commands") 72 | } 73 | c.StopClient() 74 | glog.Flush() 75 | 76 | } 77 | -------------------------------------------------------------------------------- /test/workloads/balanced.conf: -------------------------------------------------------------------------------- 1 | ; example workload configuration 2 | [config] 3 | reads = 50 4 | interval = 0 5 | keys = 10 6 | keySize = 1 7 | valueSize = 8 8 | requests = 1000 9 | -------------------------------------------------------------------------------- /test/workloads/example.conf: -------------------------------------------------------------------------------- 1 | ; example workload configuration 2 | [config] 3 | reads = 50 4 | interval = 0 5 | keys = 10 6 | keySize = 1 7 | valueSize = 8 8 | requests = 1000 9 | -------------------------------------------------------------------------------- /test/workloads/read-heavy.conf: -------------------------------------------------------------------------------- 1 | ; example workload configuration 2 | [config] 3 | reads = 90 4 | interval = 0 5 | keys = 10 6 | keySize = 1 7 | valueSize = 8 8 | requests = 1000 9 | -------------------------------------------------------------------------------- /test/workloads/write-heavy.conf: -------------------------------------------------------------------------------- 1 | ; example workload configuration 2 | [config] 3 | reads = 10 4 | interval = 0 5 | keys = 10 6 | keySize = 1 7 | valueSize = 8 8 | requests = 1000 9 | -------------------------------------------------------------------------------- /testset/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "flag" 6 | "os" 7 | "strconv" 8 | "time" 9 | 10 | "github.com/golang/glog" 11 | "github.com/heidi-ann/ios/client" 12 | "github.com/heidi-ann/ios/config" 13 | "github.com/heidi-ann/ios/test/generator" 14 | ) 15 | 16 | var configFile = flag.String("config", os.Getenv("GOPATH")+"/src/github.com/heidi-ann/ios/example.conf", "Client configuration file") 17 | var autoFile = flag.String("auto", os.Getenv("GOPATH")+"/src/github.com/heidi-ann/ios/test/workloads/example.conf", "Configure file for workload") 18 | var algorithmFile = flag.String("algorithm", os.Getenv("GOPATH")+"/src/github.com/heidi-ann/ios/configfiles/simple/client.conf", "Algorithm description file") // optional flag 19 | var resultsFile = flag.String("results", "results.csv", "File to write results to") 20 | var clientsMin = flag.Int("clients-min", 1, "Min number of clients to create (inclusive)") 21 | var clientsMax = flag.Int("clients-max", 10, "Max number of clients to create (inclusive)") 22 | var clientsStep = flag.Int("clients-step", 1, "Step in number of clients to create") 23 | 24 | // runClient returns when workload is finished or SubmitRequest fails 25 | func runClient(id int, clientConfig config.Config, addresses []config.NetAddress, workload config.ConfigAuto) (requests int, bytes int) { 26 | requests = 0 27 | bytes = 0 28 | c, err := client.StartClientFromConfig(-1, "", clientConfig, addresses) 29 | if err != nil { 30 | glog.Fatal(err) 31 | } 32 | ioapi := generator.Generate(workload, false) 33 | if err != nil { 34 | glog.Fatal(err) 35 | } 36 | for { 37 | // get next command 38 | text, read, ok := ioapi.Next() 39 | if !ok { 40 | break 41 | } 42 | // pass to ios client 43 | reply, err := c.SubmitRequest(text, read) 44 | if err != nil { 45 | break 46 | } 47 | // notify API of result 48 | ioapi.Return(reply) 49 | requests++ 50 | bytes += len(text) 51 | } 52 | c.StopClient() 53 | return 54 | } 55 | 56 | func main() { 57 | // set up logging 58 | flag.Parse() 59 | defer glog.Flush() 60 | 61 | // set up results file 62 | file, err := os.OpenFile(*resultsFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0777) 63 | if err != nil { 64 | glog.Fatal(err) 65 | } 66 | writer := csv.NewWriter(file) 67 | writer.Write([]string{ 68 | "clients", 69 | "total time [ms]", 70 | "throughput [req/sec]", 71 | "throughput [Kbps]", 72 | }) 73 | 74 | // parse config files (instead of requiring each client to do it) 75 | finished := make(chan bool) 76 | conf, err := config.ParseClientConfig(*algorithmFile) 77 | if err != nil { 78 | glog.Fatal(err) 79 | } 80 | addresses, err := config.ParseAddresses(*configFile) 81 | if err != nil { 82 | glog.Fatal(err) 83 | } 84 | workloadConfig, err := config.ParseWorkloadConfig(*autoFile) 85 | if err != nil { 86 | glog.Fatal(err) 87 | } 88 | 89 | for clients := *clientsMin; clients <= *clientsMax; clients += *clientsStep { 90 | startTime := time.Now() 91 | requestsCompleted := 0 92 | bytesCommitted := 0 93 | remaining := clients 94 | for id := 0; id < clients; id++ { 95 | go func(id int) { 96 | requests, bytes := runClient(id, conf, addresses.Clients, workloadConfig) 97 | requestsCompleted += requests 98 | bytesCommitted += bytes 99 | remaining-- 100 | if remaining == 0 { 101 | finished <- true 102 | } 103 | }(id) 104 | } 105 | 106 | // wait for workload to finish 107 | <-finished 108 | totalTime := time.Since(startTime).Seconds() // time in secs 109 | requestThroughput := float64(requestsCompleted) / totalTime // throughput in req/sec 110 | byteThroughput := float64(8*bytesCommitted/1000) / totalTime // throughput in Kbps 111 | writer.Write([]string{ 112 | strconv.Itoa(clients), 113 | strconv.FormatFloat(totalTime*1000000, 'f', 0, 64), 114 | strconv.FormatFloat(requestThroughput, 'f', 0, 64), 115 | strconv.FormatFloat(byteThroughput, 'f', 0, 64), 116 | }) 117 | writer.Flush() 118 | glog.Info("Client set terminating after completing ", requestsCompleted, " requests at ", requestThroughput, " [reqs/sec]") 119 | } 120 | 121 | // finish up 122 | glog.Flush() 123 | writer.Flush() 124 | } 125 | -------------------------------------------------------------------------------- /tools/benchmark-disk/main.go: -------------------------------------------------------------------------------- 1 | //+build ignore 2 | 3 | package main 4 | 5 | import ( 6 | "errors" 7 | "flag" 8 | "fmt" 9 | "math/rand" 10 | "os" 11 | "syscall" 12 | "time" 13 | ) 14 | 15 | func benchmarkDisk(filename string, size int, count int) { 16 | startTime := time.Now() 17 | bytes := make([]byte, size) 18 | rand.Read(bytes) 19 | fd, err := syscall.Open(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) 20 | if err != nil { 21 | panic(err) 22 | } 23 | err = syscall.Fallocate(fd, 0, 0, int64(size*count)) 24 | if err != nil { 25 | panic(err) 26 | } 27 | for i := 0; i < count; i++ { 28 | n, err := syscall.Write(fd, bytes) 29 | if n != size { 30 | panic(errors.New("Short write")) 31 | } 32 | if err != nil { 33 | panic(err) 34 | } 35 | err = syscall.Fdatasync(fd) 36 | if err != nil { 37 | panic(err) 38 | } 39 | } 40 | fmt.Printf("%s\n", time.Since(startTime).String()) 41 | syscall.Close(fd) 42 | } 43 | 44 | func main() { 45 | size := flag.Int("size", 1, "number of bytes to append to file") 46 | count := flag.Int("count", 1000, "number of appends") 47 | file := flag.String("file", "bench.dat", "file to write to") 48 | flag.Parse() 49 | benchmarkDisk(*file, *size, *count) 50 | 51 | } 52 | -------------------------------------------------------------------------------- /tools/generate-config/main.go: -------------------------------------------------------------------------------- 1 | // writes the configurate file for deploying Ios clusters locally 2 | package main 3 | 4 | import ( 5 | "bufio" 6 | "flag" 7 | "github.com/golang/glog" 8 | "os" 9 | "strconv" 10 | ) 11 | 12 | func writeConfig(servers int, filename string, peerStart int, clientStart int) { 13 | file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0666) 14 | if err != nil { 15 | glog.Fatal(err) 16 | } 17 | w := bufio.NewWriter(file) 18 | 19 | w.WriteString("[servers]\n") 20 | for id := 0; id < servers; id++ { 21 | peerPort := strconv.Itoa(peerStart + id) 22 | clientPort := strconv.Itoa(clientStart + id) 23 | w.WriteString("address = 127.0.0.1:" + peerPort + ":" + clientPort + "\n") 24 | } 25 | w.Flush() 26 | file.Close() 27 | } 28 | 29 | func main() { 30 | servers := flag.Int("servers", 1, "number of servers to run locally") 31 | filename := flag.String("file", "local.conf", "file to write configuration to") 32 | peerStart := flag.Int("peer-port", 9080, "port from which to allocate peer ports") 33 | clientStart := flag.Int("client-port", 8080, "port from which to allocate client ports") 34 | flag.Parse() 35 | defer glog.Flush() 36 | 37 | writeConfig(*servers, *filename, *peerStart, *clientStart) 38 | } 39 | -------------------------------------------------------------------------------- /tools/local-test/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "io/ioutil" 6 | "os" 7 | "os/exec" 8 | "syscall" 9 | "time" 10 | 11 | "github.com/golang/glog" 12 | ) 13 | 14 | func main() { 15 | // set up logging 16 | flag.Parse() 17 | defer glog.Flush() 18 | 19 | // check go path is set 20 | if os.Getenv("GOPATH") == "" { 21 | glog.Fatal("GOPATH not set, please set GOPATH and try again") 22 | } 23 | 24 | for _, algorithm := range []string{"delegated", "simple", "fpaxos"} { 25 | serverFile := os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/" + algorithm + "/server.conf" 26 | clientFile := os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/" + algorithm + "/client.conf" 27 | statsFile := "latency_" + algorithm + ".csv" 28 | 29 | //Create temp directory 30 | dir, err := ioutil.TempDir("", "IosTests_"+algorithm) 31 | if err != nil { 32 | glog.Fatal(err) 33 | } 34 | defer os.RemoveAll(dir) // clean up 35 | 36 | // start server 37 | server := exec.Command(os.Getenv("GOPATH")+"/bin/ios", "-id", "0", "-algorithm", serverFile, "-disk", dir, "-stderrthreshold", "WARNING") 38 | server.Stderr = os.Stderr 39 | err = server.Start() 40 | if err != nil { 41 | glog.Fatal("Error starting server process. Error: ", err.Error()) 42 | } 43 | 44 | time.Sleep(1 * time.Second) 45 | 46 | //start client 47 | client := exec.Command(os.Getenv("GOPATH")+"/bin/test", "-stat", statsFile, "-algorithm", clientFile, "-check", "true", "-stderrthreshold", "WARNING") 48 | client.Stderr = os.Stderr 49 | 50 | err = client.Start() 51 | if err != nil { 52 | glog.Fatal("Error starting server process. Error: ", err.Error()) 53 | } 54 | 55 | // wait for workload to finish 56 | timer := time.AfterFunc(10*time.Second, func() { 57 | client.Process.Kill() 58 | server.Process.Kill() 59 | glog.Fatal("Client did not complete in time") 60 | }) 61 | err = client.Wait() 62 | if err != nil { 63 | glog.Fatal("Client did not exit cleanly. Error: ", err.Error()) 64 | } 65 | timer.Stop() 66 | 67 | //close server 68 | err = server.Process.Signal(syscall.SIGKILL) 69 | if err != nil { 70 | glog.Fatal("Error killing server process. Error: ", err.Error()) 71 | } 72 | 73 | // restart server 74 | server = exec.Command(os.Getenv("GOPATH")+"/bin/ios", "-id", "0", "-algorithm", serverFile, "-disk", dir, "-stderrthreshold", "WARNING") 75 | server.Stderr = os.Stderr 76 | err = server.Start() 77 | if err != nil { 78 | glog.Fatal("Error starting server process. Error: ", err.Error()) 79 | 80 | } 81 | time.Sleep(1 * time.Second) 82 | 83 | //start client 84 | client = exec.Command(os.Getenv("GOPATH")+"/bin/test", "-stat", statsFile, "-algorithm", clientFile, "-stderrthreshold", "WARNING") 85 | client.Stderr = os.Stderr 86 | 87 | err = client.Start() 88 | if err != nil { 89 | glog.Fatal("Error starting server process. Error: ", err.Error()) 90 | } 91 | 92 | // wait for workload to finish 93 | timer = time.AfterFunc(10*time.Second, func() { 94 | client.Process.Kill() 95 | server.Process.Kill() 96 | glog.Fatal("Client did not complete in time") 97 | }) 98 | err = client.Wait() 99 | if err != nil { 100 | glog.Fatal("Client did not exit cleanly. Error: ", err.Error()) 101 | } 102 | timer.Stop() 103 | 104 | //close server 105 | err = server.Process.Signal(syscall.SIGKILL) 106 | if err != nil { 107 | glog.Fatal("Error killing server process. Error: ", err.Error()) 108 | } 109 | } 110 | } 111 | --------------------------------------------------------------------------------