├── .gitignore ├── .travis.yml ├── AUTHORS ├── Dockerfile ├── LICENSE.md ├── README.md ├── api ├── api.go └── api_test.go ├── bftkv.go ├── cmd ├── bftkv │ └── main.go └── bftrw │ ├── bftrw.go │ ├── mkca.sh │ └── sign.sh ├── crypto ├── auth │ ├── auth.go │ ├── auth_test.go │ └── gen │ │ ├── dhdump.go │ │ └── gen.sh ├── cert │ └── cert.go ├── crypto.go ├── pgp │ └── crypto_pgp.go ├── sss │ ├── sss.go │ └── sss_test.go └── threshold │ ├── dsa │ ├── dsa.go │ ├── dsa_core.go │ ├── dsa_test.go │ ├── test.pkcs8 │ └── test_utils │ │ └── test_utils.go │ ├── ecdsa │ ├── ecdsa.go │ └── ecdsa_test.go │ ├── rsa │ ├── rsa.go │ ├── rsa_test.go │ └── test.pkcs8 │ └── threhold.go ├── docs ├── bftkv.pdf ├── design.md ├── http_api.md ├── images │ ├── clique.png │ ├── failureDetectionInBFTKV.png │ ├── failureDetectionInQuorum.png │ ├── quorumAccept.png │ ├── quorumNodeStates.png │ ├── revokeOnRead.gif │ ├── trustGraph.png │ └── write.gif ├── notes.md ├── tests.md └── tex │ ├── abstract.tex │ ├── ack.tex │ ├── algo.tex │ ├── analysis.tex │ ├── application.tex │ ├── background.tex │ ├── bftkv.tex │ ├── bftkv_eabs.tex │ ├── conclusion.tex │ ├── intro.tex │ ├── method.tex │ ├── protocol.tex │ ├── ref.tex │ └── tpa.tex ├── go.mod ├── go.sum ├── node ├── graph │ ├── graph.go │ └── graph_test.go └── node.go ├── packet └── packet.go ├── protocol ├── client.go ├── dist_test.go ├── mal_test.go ├── malclient_test.go ├── malserver_test.go ├── malstorage_test.go ├── protocol.go ├── revoke_test.go ├── roaming_test.go ├── rw_test.go ├── server.go ├── server_test.go └── test_utils │ └── test_utils.go ├── quorum ├── quorum.go └── wotqs │ └── wotqs.go ├── scripts ├── check_gpg.sh ├── clique.sh ├── gen.sh ├── run.sh ├── setup.sh ├── sign.sh ├── test.go ├── test.sh └── trust.sh ├── storage ├── leveldb │ └── leveldb.go ├── plain │ └── plain.go └── storage.go ├── transport ├── http-visual │ └── http-visual.go ├── http │ ├── http.go │ └── malhttp.go ├── maltransport.go └── transport.go └── visual ├── README.md ├── css └── style.css ├── index.html └── js ├── cytoscape-dagre.js ├── cytoscape.js ├── dagre.min.js └── displayGraph.js /.gitignore: -------------------------------------------------------------------------------- 1 | scripts/run/ 2 | scripts/run_*/ 3 | *~ 4 | run/ 5 | .DS_Store 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.12.14 5 | - 1.13.5 6 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Contributors to the project include the following authors: 2 | You are welome to add your name to this list if you have made a meaningful contribution. 3 | 4 | Ryuji Ishiguro 5 | Ercan Ozturk 6 | Neeki Hushyar 7 | Juan A. Garay 8 | Edward Bortnikov 9 | Idit Keidar 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang 2 | 3 | ADD . /go/src/github.com/yahoo/bftkv 4 | 5 | WORKDIR /go/src/github.com/yahoo/bftkv/scripts/run_docker 6 | 7 | ENTRYPOINT ../run.sh 8 | 9 | EXPOSE 5601-5606 10 | EXPOSE 5701-5710 11 | EXPOSE 5801-5810 12 | 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bftkv 2 | 3 | BFTKV is a distributed key-value storage which is tolerant to Byzantine fault. See [Abstract](docs/design.md) for details. 4 | 5 | [![GoDoc](https://godoc.org/github.com/yahoo/bftkv?status.svg)](https://godoc.org/github.com/yahoo/bftkv) 6 | 7 | ### Additional documents: 8 | [Design Document (restricted)](https://docs.google.com/document/d/14xYeGR291UKba1pimO9DnFNvfZ3ceTYsfbWZbgpWkTQ/edit?usp=sharing) 9 | 10 | [Paper (draft)](docs/bftkv.pdf) 11 | 12 | [HTTP-API](docs/http_api.md) 13 | 14 | [Implementation Notes](docs/notes.md) 15 | 16 | [Test Notes](docs/tests.md) 17 | 18 | ## Setup 19 | 1. Install [Go 1.13](https://golang.org/doc/install). 20 | 2. `go get -u github.com/yahoo/bftkv` 21 | 3. Install [GnuPG 2.x](https://www.gnupg.org/download/index.en.html) 22 | 4. Install [Docker](https://www.docker.com) (if you want to run BFTKV in a Docker container) 23 | 5. Run `setup.sh` in scripts (`setup.sh -host bftkv` for Docker) 24 | 6. If bftkv runs with KeyTransparency, run `$GOPATH/src/github.com/google/keytranspreancy/scripts/gen_bftkv_keys.sh` 25 | 26 | ## Build 27 | ``` 28 | # change to the directory where the source code is checked out 29 | cd bftkv 30 | go install -v github.com/yahoo/bftkv/cmd/bftkv 31 | ``` 32 | 33 | ## Parameters 34 | A list of parameters that can be supplied to bftkv is given below: 35 | 36 | 
37 | Flag     Purpose                               Default
38 | -home    Path to PGP home directory,           ~/.gnupg
39 | -sec     Secret key ring path,                 $home/secring.gpg
40 | -pub     Public key ring path,                 $home/pubring.gpg
41 | -rev     Revocation list path,                 $home/revocation.gpg
42 | -db      Database path,                        db
43 | -api     Http api address,                     localhost:5792
44 | -ws      Web socket port,                      5001
45 |
46 | 47 | ## Run Options 48 | 1. Run a node `bftkv -home gnupg.key` 49 | 50 | 2. Run a BFTKV cluster `cd scripts/run; ../run.sh` 51 | 52 | 3. Run a BFTKV cluster in Docker 53 | 54 | ``` 55 | docker build -t bftkv . 56 | docker run -d bftkv 57 | ``` 58 | 59 | ## Visualization 60 | BFTKV includes a visualization tool (located in `visual/`) for observing the current system state. The tool can display 61 | 62 | * Trust graphs for the servers 63 | * Read, write and sign requests sent to the servers 64 | * Revoked and inaccessible servers 65 | 66 | To show the graph, run `run.sh` and `open visual/index.html`. 67 | 68 | ### Write in Action 69 | Write 70 | 71 | ### Revoke on Read in Action 72 | Revoke on Read 73 | 74 | ## License 75 | Copyright 2017, Yahoo Holdings Inc. 76 | 77 | Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 78 | -------------------------------------------------------------------------------- /api/api.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package api 5 | 6 | import ( 7 | gocrypto "crypto" 8 | "os" 9 | 10 | "github.com/yahoo/bftkv" 11 | "github.com/yahoo/bftkv/crypto" 12 | "github.com/yahoo/bftkv/crypto/pgp" 13 | "github.com/yahoo/bftkv/node" 14 | "github.com/yahoo/bftkv/node/graph" 15 | "github.com/yahoo/bftkv/packet" 16 | "github.com/yahoo/bftkv/protocol" 17 | "github.com/yahoo/bftkv/quorum" 18 | "github.com/yahoo/bftkv/quorum/wotqs" 19 | "github.com/yahoo/bftkv/transport" 20 | transport_http "github.com/yahoo/bftkv/transport/http" 21 | ) 22 | 23 | type API struct { 24 | path string 25 | client *protocol.Client 26 | g *graph.Graph 27 | crypt *crypto.Crypto 28 | qs quorum.QuorumSystem 29 | tr transport.Transport 30 | } 31 | 32 | func OpenClient(path string) (*API, error) { 33 | api := &API{ 34 | path: path, 35 | client: nil, 36 | g: graph.New(), 37 | crypt: pgp.New(), 38 | } 39 | if _, err := api.readCerts(path+"/pubring.gpg", false, true); err != nil { 40 | return nil, err 41 | } 42 | if _, err := api.readCerts(path+"/secring.gpg", true, true); err != nil { 43 | return nil, err 44 | } 45 | 46 | // make a quorum system from the graph 47 | api.qs = wotqs.New(api.g) 48 | api.tr = transport_http.New(api.crypt) 49 | api.client = protocol.NewClient(node.SelfNode(api.g), api.qs, api.tr, api.crypt) 50 | if err := api.client.Joining(); err != nil { 51 | return nil, err 52 | } 53 | return api, nil 54 | } 55 | 56 | func (api *API) CloseClient() { 57 | api.client.Leaving() 58 | } 59 | 60 | func (api *API) signPeers(certs []string) error { 61 | for _, cert := range certs { 62 | if peers, err := api.readCerts(cert+"/pubring.gpg", false, false); err == nil { 63 | // make an edge: self -> cert 64 | if len(peers) > 0 { 65 | if err := api.crypt.Certificate.Sign(peers[0]); err == nil { 66 | api.g.AddNodes(peers[0:1]) 67 | } 68 | } 69 | } 70 | } 71 | return nil 72 | } 73 | 74 | func (api *API) Register(certs []string, password string) error { 75 | // support PGP certs only 76 | if err := api.signPeers(certs); err != nil { 77 | return err 78 | } 79 | // read them into the graph 80 | if err := api.client.Joining(); err != nil { // re-joining so the client can construct the full graph 81 | return err 82 | } 83 | // need to re-sign as Joining has overwritten some nodes 84 | if err := api.signPeers(certs); err != nil { 85 | return err 86 | } 87 | // now the quorum system should work even if it is not completed yet 88 | 89 | self := node.SelfNode(api.g) 90 | variable := []byte(self.UId()) 91 | 92 | proof, _, err := api.client.Authenticate(variable, []byte(password)) 93 | if err != nil { 94 | return err 95 | } 96 | 97 | // do "register" and get back signed certs 98 | t := uint64(1) // make sure it's no longer temporary 99 | cert, err := self.SerializeSelf() 100 | if err != nil { 101 | return err 102 | } 103 | tbs, err := packet.Serialize(variable, cert, t) 104 | if err != nil { 105 | return err 106 | } 107 | sig, err := api.crypt.Signature.Sign(tbs) 108 | if err != nil { 109 | return err 110 | } 111 | pkt, err := packet.Serialize(variable, cert, t, sig, proof) // use the value as the PGP cert 112 | if err != nil { 113 | return err 114 | } 115 | q := api.qs.ChooseQuorum(quorum.AUTH | quorum.PEER) 116 | var sigs []node.Node 117 | var succ []node.Node 118 | api.tr.Multicast(transport.Register, q.Nodes(), pkt, func(res *transport.MulticastResponse) bool { 119 | if res.Err == nil { 120 | certs, err := api.crypt.Certificate.Parse(res.Data) 121 | if err == nil { 122 | sigs = append(sigs, certs...) 123 | succ = append(succ, res.Peer) 124 | } 125 | } 126 | return false // collect signatures as many as possible 127 | }) 128 | 129 | // accumulate all signatures into the self cert 130 | if !q.IsSufficient(succ) { 131 | return bftkv.ErrAuthenticationFailure 132 | } 133 | selfNode := api.crypt.Keyring.GetCertById(self.Id()) 134 | for _, sig := range sigs { 135 | api.crypt.Certificate.Merge(selfNode, sig) 136 | } 137 | 138 | // re-set the self node 139 | nodes := []node.Node{selfNode} 140 | api.g.AddNodes(nodes) 141 | // renew the self cert 142 | api.crypt.Keyring.Remove(nodes) 143 | if err := api.crypt.Keyring.Register(nodes, false, true); err != nil { 144 | return err 145 | } 146 | return nil 147 | } 148 | 149 | func (api *API) Write(variable []byte, value []byte, password string) (err error) { 150 | var proof *packet.SignaturePacket 151 | if password != "" { 152 | var key []byte 153 | proof, key, err = api.client.Authenticate(variable, []byte(password)) 154 | if err != nil { 155 | return err 156 | } 157 | value, err = api.crypt.DataEncryption.Encrypt(key, value) 158 | if err != nil { 159 | return err 160 | } 161 | } 162 | return api.client.Write(variable, value, proof) 163 | } 164 | 165 | func (api *API) Read(variable []byte, password string) (value []byte, err error) { 166 | var proof *packet.SignaturePacket 167 | var key []byte 168 | if password != "" { 169 | proof, key, err = api.client.Authenticate(variable, []byte(password)) 170 | if err != nil { 171 | return nil, err 172 | } 173 | } 174 | value, err = api.client.Read(variable, proof) 175 | if err != nil { 176 | return nil, err 177 | } 178 | if key != nil && value != nil && len(value) > 0 { 179 | value, err = api.crypt.DataEncryption.Decrypt(key, value) 180 | if err != nil { 181 | return nil, err 182 | } 183 | } 184 | return value, nil 185 | } 186 | 187 | func (api *API) UpdateCert() error { 188 | path := api.path + "/pubring.gpg" 189 | err := os.Rename(path, path+"~") 190 | if err != nil { 191 | return err 192 | } 193 | f, err := os.Create(path) 194 | if err != nil { 195 | return err 196 | } 197 | err = api.g.SerializeNodes(f) 198 | f.Close() 199 | if err != nil { 200 | os.Rename(path+"~", path) 201 | } 202 | return err 203 | } 204 | 205 | func (api *API) readCerts(path string, sec bool, self bool) ([]node.Node, error) { 206 | f, err := os.Open(path) 207 | if err != nil { 208 | return nil, err 209 | } 210 | defer f.Close() 211 | certs, err := api.crypt.Certificate.ParseStream(f) 212 | if err != nil { 213 | return nil, err 214 | } 215 | if self { 216 | if sec { 217 | api.g.SetSelfNodes(certs) 218 | } else { 219 | api.g.AddNodes(certs) 220 | } 221 | if err := api.crypt.Keyring.Register(certs, sec, self); err != nil { 222 | return nil, err 223 | } 224 | } 225 | return certs, nil 226 | } 227 | 228 | func (api *API) Distribute(caname string, key interface{}) error { 229 | return api.client.Distribute(caname, key) 230 | } 231 | 232 | func (api *API) Sign(caname string, tbs []byte, algo crypto.ThresholdAlgo, dgst gocrypto.Hash) (sig []byte, err error) { 233 | return api.client.DistSign(caname, tbs, algo, dgst) 234 | } 235 | 236 | func (api *API) UId() string { 237 | self := node.SelfNode(api.g) 238 | return self.UId() 239 | } 240 | -------------------------------------------------------------------------------- /api/api_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package api 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "os" 10 | "testing" 11 | 12 | "github.com/yahoo/bftkv/node" 13 | "github.com/yahoo/bftkv/node/graph" 14 | "github.com/yahoo/bftkv/protocol/test_utils" 15 | "github.com/yahoo/bftkv/quorum" 16 | ) 17 | 18 | var ( 19 | testValue = []byte("testvalue") 20 | ) 21 | 22 | const password = "1234" 23 | 24 | var certlist = []string{ 25 | test_utils.KeyPath + "/a01", 26 | test_utils.KeyPath + "/a02", 27 | test_utils.KeyPath + "/a03", 28 | test_utils.KeyPath + "/a04", 29 | test_utils.KeyPath + "/a05", 30 | test_utils.KeyPath + "/a06", 31 | test_utils.KeyPath + "/a07", 32 | test_utils.KeyPath + "/a08", 33 | test_utils.KeyPath + "/a09", 34 | test_utils.KeyPath + "/a10", 35 | test_utils.KeyPath + "/rw01", 36 | test_utils.KeyPath + "/rw02", 37 | test_utils.KeyPath + "/rw03", 38 | test_utils.KeyPath + "/rw04", 39 | test_utils.KeyPath + "/rw05", 40 | test_utils.KeyPath + "/rw06", 41 | } 42 | 43 | const ( 44 | preRegisteredKey = "u01" 45 | virginKey = "test1" 46 | ) 47 | 48 | func testRW(t *testing.T, clientKey string, variable string) { 49 | client, err := OpenClient(test_utils.KeyPath + "/" + clientKey) 50 | if err != nil { 51 | t.Fatal(err) 52 | } 53 | defer client.CloseClient() 54 | 55 | // plain read/write 56 | plainVariable := []byte(variable) 57 | if err := client.Write(plainVariable, testValue, ""); err != nil { 58 | t.Fatal(err) 59 | } 60 | val, err := client.Read(plainVariable, "") 61 | if err != nil { 62 | t.Fatal(err) 63 | } 64 | if !bytes.Equal(val, testValue) { 65 | t.Error("value mismatch") 66 | } 67 | t.Logf("plain read/write done. move on to auth read/write\n") 68 | 69 | // authorized read/write 70 | authVariable := []byte(variable + "_auth") 71 | if err := client.Write(authVariable, testValue, password); err != nil { 72 | t.Fatal(err) 73 | } 74 | val, err = client.Read(authVariable, password) 75 | if err != nil { 76 | t.Fatal(err) 77 | } 78 | if !bytes.Equal(val, testValue) { 79 | t.Error("auth read/write mistmatch") 80 | } 81 | 82 | // auth read with a wrong password 83 | val, err = client.Read(authVariable, "123") 84 | if err == nil { 85 | t.Error("auth succeeded with a wrong key!?") 86 | } 87 | t.Logf("auth read/write done.") 88 | } 89 | 90 | func testRegistration(t *testing.T, clientKey string) { 91 | client, err := OpenClient(test_utils.KeyPath + "/" + clientKey) // should be a virgin key 92 | if err != nil { 93 | t.Fatal(err) 94 | } 95 | defer client.CloseClient() 96 | 97 | if err := client.Register(certlist, password); err != nil { 98 | t.Fatal(err) 99 | } 100 | 101 | // test if the self cert has the signature from the quorum 102 | signers := client.crypt.Certificate.Signers(node.SelfNode(client.g)) 103 | sm := make(map[uint64]node.Node) 104 | for _, sig := range signers { 105 | sm[sig.Id()] = sig 106 | } 107 | q := client.qs.ChooseQuorum(quorum.AUTH) 108 | for _, qm := range q.Nodes() { 109 | if _, ok := sm[qm.Id()]; !ok { 110 | t.Logf("couldn't find the signature of %s\n", qm.Name()) 111 | } 112 | } 113 | if err := client.UpdateCert(); err != nil { // update the pubring 114 | t.Error(err) 115 | } 116 | 117 | // double-check if the password auth went through 118 | if _, _, err := client.client.Authenticate([]byte(client.UId()), []byte(password)); err != nil { 119 | t.Error(err) 120 | } 121 | } 122 | 123 | func TestAPI(t *testing.T) { 124 | t.Skip("skip failing test - FIXME") 125 | servers := test_utils.RunServers(t, "a", "rw") 126 | defer test_utils.StopServers(servers) 127 | 128 | testRW(t, preRegisteredKey, "testkey") // use the pre-registered key 129 | if t.Failed() { 130 | return 131 | } 132 | testRegistration(t, virginKey) // register the virgin key 133 | if !t.Failed() { 134 | defer func() { 135 | path := test_utils.KeyPath + "/" + virginKey + "/pubring.gpg" 136 | os.Rename(path+"~", path) 137 | }() 138 | testRW(t, virginKey, "virginkey") // to check the registration has finished in success 139 | } 140 | } 141 | 142 | func dump(g *graph.Graph, prompt string) { 143 | fmt.Printf(">>> %s <<<\n", prompt) 144 | for _, v := range g.Vertices { 145 | n := v.Instance 146 | if n == nil { 147 | continue 148 | } 149 | instance := "" 150 | if n.Instance() == nil { 151 | instance = "nil" 152 | } 153 | fmt.Printf(" %s [%x] %s\n", n.Name(), n.Id(), instance) 154 | for _, e := range v.Edges { 155 | if e.Instance != nil { 156 | fmt.Printf(" %s\n", e.Instance.Name()) 157 | } else { 158 | fmt.Printf(" ---\n") 159 | } 160 | } 161 | } 162 | } 163 | 164 | func (api *API) isSelf(v *graph.Vertex) bool { 165 | for _, s := range api.g.Self { 166 | if v == s { 167 | return true 168 | } 169 | } 170 | return false 171 | } 172 | 173 | func (api *API) dump2(prompt string) { 174 | fmt.Printf(">>> %s <<<\n", prompt) 175 | for _, v := range api.g.Self { 176 | if v.Instance == nil { 177 | continue 178 | } 179 | fmt.Printf("self: %s (%x)\n", v.Instance.Name(), v.Instance) 180 | for _, s := range api.crypt.Certificate.Signers(v.Instance) { 181 | fmt.Printf(" %s\n", s.Name()) 182 | } 183 | } 184 | for _, v := range api.g.Vertices { 185 | if v.Instance == nil || api.isSelf(v) { 186 | continue 187 | } 188 | fmt.Printf("peer: %s (%x)\n", v.Instance.Name(), v.Instance) 189 | for _, s := range api.crypt.Certificate.Signers(v.Instance) { 190 | fmt.Printf(" %s\n", s.Name()) 191 | } 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /bftkv.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package bftkv 5 | 6 | import ( 7 | "errors" 8 | "sync" 9 | ) 10 | 11 | var ( 12 | ErrInsufficientNumberOfQuorum = NewError("insufficient number of quorum") 13 | ErrInsufficientNumberOfResponses = NewError("insufficient number of responses") 14 | ErrInsufficientNumberOfValidResponses = NewError("insufficient number of valid responses") 15 | ErrInvalidQuorumCertificate = NewError("invalid quorum certficate") 16 | ErrInvalidTimestamp = NewError("invalid timestamp") 17 | ErrInvalidSignRequest = NewError("invalid signature request") 18 | ErrPermissionDenied = NewError("permission denied") 19 | ErrBadTimestamp = NewError("bad timestamp") 20 | ErrEquivocation = NewError("equivocation error") 21 | ErrInvalidVariable = NewError("invalid variable") 22 | ErrUnknownCommand = NewError("unknown command") 23 | ErrMalformedRequest = NewError("malformed request") 24 | ErrNoMoreWrite = NewError("no more write") 25 | ErrAuthenticationFailure = NewError("authentication failure") 26 | ErrExist = NewError("already exist") 27 | ErrInvalidUserID = NewError("invalid user ID") 28 | ErrInvalidResponse = NewError("invalid response") 29 | ) 30 | 31 | var errMap = make(map[string]error) 32 | var mutex sync.Mutex 33 | 34 | func NewError(s string) error { 35 | err := errors.New(s) 36 | errMap[s] = err 37 | return err 38 | } 39 | 40 | func ErrorFromString(s string) error { 41 | mutex.Lock() 42 | err, ok := errMap[s] 43 | if !ok { 44 | err = NewError(s) 45 | } 46 | mutex.Unlock() 47 | return err 48 | } 49 | -------------------------------------------------------------------------------- /cmd/bftkv/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package main 5 | 6 | import ( 7 | "context" 8 | "flag" 9 | "fmt" 10 | "io/ioutil" 11 | "log" 12 | "net/http" 13 | "os" 14 | "os/signal" 15 | "strconv" 16 | "strings" 17 | "syscall" 18 | "time" 19 | 20 | "net/http/pprof" 21 | 22 | "github.com/yahoo/bftkv/crypto" 23 | "github.com/yahoo/bftkv/crypto/pgp" 24 | "github.com/yahoo/bftkv/node" 25 | "github.com/yahoo/bftkv/node/graph" 26 | "github.com/yahoo/bftkv/protocol" 27 | "github.com/yahoo/bftkv/quorum/wotqs" 28 | "github.com/yahoo/bftkv/storage" 29 | storage_leveldb "github.com/yahoo/bftkv/storage/leveldb" 30 | storage_plain "github.com/yahoo/bftkv/storage/plain" 31 | "github.com/yahoo/bftkv/transport" 32 | transport_http "github.com/yahoo/bftkv/transport/http" 33 | transport_http_visual "github.com/yahoo/bftkv/transport/http-visual" 34 | ) 35 | 36 | func main() { 37 | defaultPath := os.Getenv("HOME") + "/.gnupg/" 38 | pathp := flag.String("home", defaultPath, "path to home") 39 | secringp := flag.String("sec", "", "secret key ring") 40 | pubringp := flag.String("pub", "", "public key ring") 41 | revocationp := flag.String("rev", "", "revocation list") 42 | dbPathp := flag.String("db", "db", "database path") 43 | ldbPathp := flag.String("ldb", "", "level db path") 44 | apiPortp := flag.Int("api", 0, "http api address") 45 | wsPortp := flag.Int("ws", 0, "web socket port") 46 | 47 | flag.Parse() 48 | path := *pathp 49 | secring := *secringp 50 | if secring == "" { 51 | secring = path + "/secring.gpg" 52 | } 53 | pubring := *pubringp 54 | if pubring == "" { 55 | pubring = path + "/pubring.gpg" 56 | } 57 | 58 | revocation := *revocationp 59 | if revocation == "" { 60 | revocation = path + "/revocation.gpg" 61 | } 62 | 63 | wsPort := *wsPortp 64 | 65 | // crypt package 66 | crypt := pgp.New() 67 | 68 | // construct a graph from pgp keys 69 | g := graph.New() 70 | readCerts(g, crypt, pubring, false) 71 | readCerts(g, crypt, secring, true) 72 | readRevocationList(g, crypt, revocation) 73 | 74 | // make a quorum system from the graph 75 | qs := wotqs.New(g) 76 | 77 | var tr transport.Transport 78 | if wsPort == 0 { 79 | tr = transport_http.New(crypt) 80 | } else { 81 | // create a HTTP visual transport layer with PGP security 82 | tr = transport_http_visual.New(crypt, g, qs, strconv.Itoa(wsPort)) 83 | } 84 | 85 | // create a storage for the server 86 | var storage storage.Storage 87 | if *ldbPathp != "" { 88 | storage = storage_leveldb.New(*dbPathp) 89 | } else { 90 | storage = storage_plain.New(*dbPathp) 91 | } 92 | 93 | // create BFTKV client and server, and start the server 94 | bftClient := protocol.NewClient(node.SelfNode(g), qs, tr, crypt) 95 | bftServer := protocol.NewServer(node.SelfNode(g), qs, tr, crypt, storage) 96 | 97 | if err := bftServer.Start(); err != nil { 98 | log.Fatal(err) 99 | } 100 | 101 | var apiServer *apiService 102 | if *apiPortp != 0 { 103 | // start HTTP API 104 | apiServer = &apiService{bftClient: bftClient, g: g} 105 | apiServer.Start(*apiPortp) 106 | } 107 | 108 | // wait for a signal 109 | ch := make(chan os.Signal, 1) 110 | signal.Notify(ch, syscall.SIGINT, syscall.SIGABRT, syscall.SIGQUIT, syscall.SIGTERM) 111 | <-ch 112 | 113 | // stop servers 114 | if apiServer != nil { 115 | apiServer.Stop() 116 | } 117 | bftServer.Stop() 118 | 119 | // save pubring and revocation list 120 | // writeCerts(g, crypt, pubring) // don't write it back for now, as all nodes leave when this program terminates 121 | // writeRevocationList(g, crypt, revocation) 122 | } 123 | 124 | func readCerts(g *graph.Graph, crypt *crypto.Crypto, path string, sec bool) { 125 | f, err := os.Open(path) 126 | if err != nil { 127 | return 128 | } 129 | certs, err := crypt.Certificate.ParseStream(f) 130 | if err != nil { 131 | f.Close() 132 | log.Fatal(err) 133 | } 134 | if sec { 135 | g.SetSelfNodes(certs) 136 | } else { 137 | g.AddNodes(certs) 138 | } 139 | crypt.Keyring.Register(certs, sec, true) 140 | f.Close() 141 | } 142 | 143 | func readRevocationList(g *graph.Graph, crypt *crypto.Crypto, path string) { 144 | f, err := os.Open(path) 145 | if err != nil { 146 | return 147 | } 148 | certs, err := crypt.Certificate.ParseStream(f) 149 | if err == nil { 150 | g.RevokeNodes(certs) 151 | } 152 | f.Close() 153 | } 154 | 155 | func writeCerts(g *graph.Graph, crypt *crypto.Crypto, path string) { 156 | log.Printf("writing back certs: %s\n", path) 157 | os.Rename(path, path+"~") 158 | f, err := os.Create(path) 159 | if err != nil { 160 | log.Print(err) 161 | } else { 162 | err := g.SerializeNodes(f) 163 | f.Close() 164 | if err != nil { 165 | log.Print(err) 166 | } 167 | } 168 | } 169 | 170 | func writeRevocationList(g *graph.Graph, crypt *crypto.Crypto, path string) { 171 | log.Printf("saving the revocation list: %s\n", path) 172 | os.Rename(path, path+"~") 173 | f, err := os.Create(path) 174 | if err != nil { 175 | log.Print(err) 176 | } else { 177 | err := g.SerializeRevokedNodes(f) 178 | f.Close() 179 | if err != nil { 180 | log.Print(err) 181 | } 182 | } 183 | } 184 | 185 | // 186 | // HTTP API 187 | // 188 | 189 | type apiService struct { 190 | bftClient *protocol.Client 191 | g *graph.Graph 192 | httpServer *http.Server 193 | } 194 | 195 | func (s *apiService) Start(port int) { 196 | s.httpServer = &http.Server{ 197 | Addr: ":" + strconv.Itoa(port), 198 | Handler: s, 199 | } 200 | go s.httpServer.ListenAndServe() 201 | } 202 | 203 | func (s *apiService) Stop() { 204 | ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) 205 | s.httpServer.Shutdown(ctx) 206 | s.httpServer.Close() 207 | } 208 | 209 | func (s *apiService) ServeHTTP(w http.ResponseWriter, r *http.Request) { 210 | w.Header().Set("Access-Control-Allow-Origin", "*") 211 | if r.Method == http.MethodGet { 212 | if err := r.ParseForm(); err != nil { 213 | http.Error(w, err.Error(), http.StatusBadRequest) 214 | return 215 | } 216 | } 217 | path := strings.ToLower(r.URL.Path) 218 | var res []byte 219 | var err error 220 | a := strings.Split(path, "/") 221 | switch a[1] { 222 | case "read": 223 | err = s.bftClient.Joining() 224 | if err == nil { 225 | res, err = s.bftClient.Read([]byte(a[2]), nil) 226 | s.bftClient.Leaving() 227 | } 228 | case "write": 229 | fallthrough 230 | case "writeonce": 231 | body, err2 := ioutil.ReadAll(r.Body) 232 | if err2 != nil { 233 | http.Error(w, err2.Error(), http.StatusInternalServerError) 234 | return 235 | } 236 | r.Body.Close() 237 | err = s.bftClient.Joining() 238 | if err == nil { 239 | if a[1] == "writeonce" { 240 | err = s.bftClient.WriteOnce([]byte(a[2]), body, nil) 241 | } else { 242 | err = s.bftClient.Write([]byte(a[2]), body, nil) 243 | } 244 | s.bftClient.Leaving() 245 | } 246 | case "joining": 247 | err = s.bftClient.Joining() 248 | case "leaving": 249 | err = s.bftClient.Leaving() 250 | case "show": 251 | s.dump() 252 | case "debug": 253 | pprof.Profile(w, r) 254 | return 255 | default: 256 | fmt.Printf("unknown path: %s\n", a[1]) 257 | return 258 | } 259 | if err == nil { 260 | fmt.Printf("\"%s\" success\n", a[1]) 261 | } else { 262 | fmt.Printf("\"%s\" error: %s\n", a[1], err) 263 | http.Error(w, err.Error(), http.StatusInternalServerError) 264 | return 265 | } 266 | w.Write(res) 267 | } 268 | 269 | func (s *apiService) dump() { 270 | fmt.Printf(">>> %s <<<\n", s.g.Self[0].Instance.Name()) 271 | for _, v := range s.g.Vertices { 272 | n := v.Instance 273 | if n == nil { 274 | continue 275 | } 276 | instance := "" 277 | if n.Instance() == nil { 278 | instance = "nil" 279 | } 280 | fmt.Printf(" %s [%x] %s\n", n.Name(), n.Id(), instance) 281 | for _, e := range v.Edges { 282 | if e.Instance != nil { 283 | fmt.Printf(" %s\n", e.Instance.Name()) 284 | } else { 285 | fmt.Printf(" ---\n") 286 | } 287 | } 288 | } 289 | } 290 | -------------------------------------------------------------------------------- /cmd/bftrw/mkca.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NAME=$1 4 | if [ "$NAME" == "" ]; then NAME="ca"; fi 5 | openssl req -x509 -newkey RSA:2048 -subj '/CN=testca.example.com' -nodes -keyout $NAME.pkcs8 -out $NAME.x509 2> /dev/null 6 | openssl x509 -pubkey -in $NAME.x509 -noout -out $NAME.pub 7 | -------------------------------------------------------------------------------- /cmd/bftrw/sign.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ $# -lt 2 ]; then echo "usage: sign caname cert"; exit 1; fi 4 | 5 | TMP=tmp.x509 6 | # make up a dummy CA keys 7 | ./mkca.sh dummy 8 | openssl x509 -CA dummy.x509 -CAkey dummy.pkcs8 -set_serial 1 -clrext -out $TMP -in $2 2> /dev/null 9 | rm -f dummy.* 10 | ./bftrw sign $1 $TMP 11 | ret=$? 12 | rm -f $TMP 13 | exit $ret 14 | -------------------------------------------------------------------------------- /crypto/auth/auth_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package auth 5 | 6 | import ( 7 | "bytes" 8 | "crypto/rand" 9 | "fmt" 10 | "math/big" 11 | "testing" 12 | ) 13 | 14 | func testAuth(t *testing.T, password []byte, proof []byte) { 15 | k := 7 16 | n := 10 17 | 18 | params, err := GeneratePartialAuthenticationParams(password, n, k) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | var ss []*AuthServer 24 | for _, p := range params { 25 | s, err := NewServer(p, proof) 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | ss = append(ss, s) 30 | } 31 | c := NewClient(password, n, k) 32 | 33 | X, err := c.generateX() 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | 38 | var Xis map[uint64][]byte 39 | for i := 0; i < n; i++ { 40 | Yi, err := ss[i].makeYi(X) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | Xis, err = c.processYi(Yi, uint64(i)) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | if Xis != nil { 50 | break 51 | } 52 | } 53 | if Xis == nil { 54 | t.Error("not enough responses for Xi") 55 | return 56 | } 57 | 58 | var Nis map[uint64][]byte 59 | for id, Xi := range Xis { 60 | Bi, err := ss[id].makeBi(Xi) 61 | if err != nil { 62 | t.Fatal(err) 63 | } 64 | Nis, err = c.processBi(Bi, id) 65 | if err != nil { 66 | t.Fatal(err) 67 | } 68 | if Nis != nil { 69 | break 70 | } 71 | } 72 | if Nis == nil { 73 | t.Error("not enough responses for Ni") 74 | return 75 | } 76 | 77 | var Pis map[uint64][]byte 78 | for id, Ni := range Nis { 79 | Zi, err := ss[id].makeZi(Ni) 80 | if err != nil { 81 | t.Fatal(err) 82 | } 83 | Pis, err = c.processZi(Zi, id) 84 | if err != nil { 85 | t.Fatal(err) 86 | } 87 | if Pis != nil { 88 | break 89 | } 90 | } 91 | if Pis == nil || len(Pis) < k { 92 | t.Error("not enough response for Pi") 93 | return 94 | } 95 | 96 | for id, Pi := range Pis { 97 | if !bytes.Equal(Pi, proof) { 98 | t.Errorf("%d: decryption failed", id) 99 | } 100 | } 101 | } 102 | 103 | func TestAuth(t *testing.T) { 104 | var password [8]byte 105 | var plainData [16]byte 106 | for ntests := 10; ntests > 0; ntests-- { 107 | rand.Read(password[:]) 108 | rand.Read(plainData[:]) 109 | testAuth(t, password[:], plainData[:]) 110 | if t.Failed() { 111 | break 112 | } 113 | } 114 | } 115 | 116 | type SharedSecret struct { 117 | x int 118 | y *big.Int 119 | } 120 | 121 | func TestSSS(t *testing.T) { 122 | n := 6 123 | k := 4 124 | q := big.NewInt(1237) 125 | poly := make([]*big.Int, k) 126 | poly[0] = big.NewInt(1234) 127 | poly[1] = big.NewInt(166) 128 | poly[2] = big.NewInt(94) 129 | poly[3] = big.NewInt(666) 130 | var sss []*SharedSecret 131 | for i := 1; i <= n; i++ { 132 | x0 := big.NewInt(int64(i)) 133 | x := new(big.Int).Set(x0) 134 | f := new(big.Int).Set(poly[0]) 135 | for j := 1; j < k; j++ { 136 | f.Mod(f.Add(f, new(big.Int).Mul(poly[j], x)), q) 137 | x.Mul(x, x0) 138 | } 139 | sss = append(sss, &SharedSecret{i, f}) 140 | fmt.Printf("(%d, %d)\n", i, f) 141 | } 142 | 143 | samples := []int{1, 3, 4, 5} 144 | var results []*SharedSecret 145 | for _, i := range samples { 146 | results = append(results, sss[i]) 147 | } 148 | s := big.NewInt(0) 149 | for _, res := range results { 150 | l := lagrange(res.x, results, q) 151 | s.Mod(s.Add(s, new(big.Int).Mod(new(big.Int).Mul(l, res.y), q)), q) 152 | fmt.Printf("l = %d\n", l) 153 | } 154 | fmt.Printf("%d\n", s) 155 | } 156 | 157 | func lagrange(x int, sss []*SharedSecret, q *big.Int) *big.Int { 158 | a := big.NewInt(1) 159 | b := big.NewInt(1) 160 | xj := big.NewInt(int64(x)) 161 | for _, ss := range sss { 162 | if ss.x == x { 163 | continue 164 | } 165 | xm := big.NewInt(int64(ss.x)) 166 | a.Mod(a.Mul(a, xm), q) 167 | b.Mod(b.Mul(b, xm.Sub(xm, xj)), q) 168 | } 169 | a.Mod(a.Mul(a, b.ModInverse(b, q)), q) 170 | return a 171 | } 172 | -------------------------------------------------------------------------------- /crypto/auth/gen/dhdump.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package main 5 | 6 | import ( 7 | "bufio" 8 | "encoding/asn1" 9 | "encoding/pem" 10 | "fmt" 11 | "io/ioutil" 12 | "math/big" 13 | "os" 14 | ) 15 | 16 | func main() { 17 | // read up all data from stdin 18 | r := bufio.NewReader(os.Stdin) 19 | b, err := ioutil.ReadAll(r) 20 | if err != nil { 21 | fmt.Errorf("%s\n", err) 22 | os.Exit(1) 23 | } 24 | 25 | pem, _ := pem.Decode(b) 26 | // the asn1/der must be a SEQUENCE of INTEGER 27 | var val []*big.Int 28 | if _, err := asn1.Unmarshal(pem.Bytes, &val); err != nil { 29 | fmt.Errorf("%s\n", err) 30 | os.Exit(1) 31 | } 32 | p := val[0].Bytes() 33 | for i := 0; i < len(p); i++ { 34 | fmt.Printf("0x%02x, ", p[i]) 35 | } 36 | fmt.Printf("\n") 37 | } 38 | -------------------------------------------------------------------------------- /crypto/auth/gen/gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | openssl gendh 2048 | ./dhdump 4 | -------------------------------------------------------------------------------- /crypto/cert/cert.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package cert 5 | 6 | type CertificateInstance interface { 7 | Id() uint64 8 | Name() string 9 | Address() string 10 | UId() string 11 | Signers() []uint64 12 | Serialize() ([]byte, error) 13 | Instance() interface{} 14 | SetActive(active bool) 15 | Active() bool 16 | } 17 | -------------------------------------------------------------------------------- /crypto/crypto.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package crypto 5 | 6 | import ( 7 | gocrypto "crypto" 8 | "io" 9 | 10 | "github.com/yahoo/bftkv" 11 | "github.com/yahoo/bftkv/node" 12 | "github.com/yahoo/bftkv/packet" 13 | "github.com/yahoo/bftkv/quorum" 14 | ) 15 | 16 | var ( 17 | ErrCertificateNotFound = bftkv.NewError("crypto: certifiate not found") 18 | ErrKeyNotFound = bftkv.NewError("crypto: key not found") 19 | ErrInvalidTransportSecurityData = bftkv.NewError("crypto: invalid transport security data") 20 | ErrInsufficientNumberOfSignatures = bftkv.NewError("crypto: insufficient number of signatures") 21 | ErrInvalidSignature = bftkv.NewError("crypto: invalid signature") 22 | ErrSigningFailed = bftkv.NewError("crypto: failed to sign") 23 | ErrEncryptionFailed = bftkv.NewError("crypto: failed to encrypt") 24 | ErrDecryptionFailed = bftkv.NewError("crypto: failed to decrypt") 25 | ErrInsufficientNumberOfSecrets = bftkv.NewError("crypto: insufficient number of secrets") 26 | ErrInvalidInput = bftkv.NewError("crypto: invalid input") 27 | ErrNoAuthenticationData = bftkv.NewError("crypto: no authentication data") 28 | ErrAuthTooManyAttempts = bftkv.NewError("crypto: too many attempts") 29 | ErrUnsupported = bftkv.NewError("crypto: unsupported algorithm") 30 | ErrInsufficientNumberOfThresholdSignatures = bftkv.NewError("crypto: insufficient number of threshold signatures") 31 | ErrShareNotFound = bftkv.NewError("crypto: share not found") 32 | ErrContinue = bftkv.NewError("crypto: continue") // not an error but to tell the client the threshold process continues 33 | ) 34 | 35 | type Keyring interface { 36 | Register(nodes []node.Node, priv bool, self bool) error 37 | Remove(node []node.Node) 38 | GetCertById(id uint64) node.Node 39 | GetKeyring() []node.Node 40 | } 41 | 42 | type Certificate interface { 43 | Parse(pkt []byte) ([]node.Node, error) 44 | ParseStream(r io.Reader) ([]node.Node, error) 45 | Signers(signee node.Node) []node.Node 46 | Sign(signee node.Node) error 47 | Merge(cert node.Node, sub node.Node) error 48 | } 49 | 50 | type Signature interface { 51 | // signature includes the signers (IDs or certificates) 52 | Verify(tbs []byte, sig *packet.SignaturePacket) error 53 | VerifyWithCertificate(tbs []byte, sig *packet.SignaturePacket, cert node.Node) error 54 | Sign(tbs []byte) (*packet.SignaturePacket, error) 55 | Signers(sig *packet.SignaturePacket) []node.Node // return only nodes that have been verified 56 | Issuer(sig *packet.SignaturePacket) node.Node 57 | Certs(sig *packet.SignaturePacket) ([]node.Node, error) 58 | } 59 | 60 | type Message interface { 61 | Encrypt(peers []node.Node, plain []byte, nonce []byte) (cipher []byte, err error) 62 | EncryptStream(out io.Writer, peerId uint64, nonce []byte) (in io.WriteCloser, err error) 63 | Decrypt(body io.Reader) (plain []byte, nonce []byte, peer node.Node, err error) 64 | } 65 | 66 | type CollectiveSignature interface { 67 | Verify(tbs []byte, ss *packet.SignaturePacket, q quorum.Quorum) error 68 | Sign(tbs []byte) (partialSignature *packet.SignaturePacket, err error) 69 | Combine(ss *packet.SignaturePacket, s *packet.SignaturePacket, q quorum.Quorum) bool 70 | Signers(ss *packet.SignaturePacket) []node.Node 71 | } 72 | 73 | type DataEncryption interface { 74 | Encrypt(key []byte, plain []byte) ([]byte, error) 75 | Decrypt(key []byte, cipher []byte) ([]byte, error) 76 | } 77 | 78 | type RNG interface { 79 | Initialize(seed []byte) 80 | Generate(n int) []byte 81 | } 82 | 83 | type ThresholdAlgo byte 84 | 85 | const ( 86 | TH_UNKNOWN ThresholdAlgo = iota 87 | TH_RSA 88 | TH_DSA 89 | TH_ECDSA 90 | ) 91 | 92 | type Threshold interface { 93 | Distribute(key interface{}, nodes []node.Node, k int) ([][]byte, ThresholdAlgo, error) 94 | Sign(sec []byte, req []byte, peerId, selfId uint64) ([]byte, error) 95 | NewProcess(tbs []byte, algo ThresholdAlgo, hash gocrypto.Hash) (ThresholdProcess, error) 96 | } 97 | 98 | type ThresholdProcess interface { 99 | MakeRequest() ([]node.Node, []byte, error) 100 | ProcessResponse(res []byte, peer node.Node) ([]byte, error) 101 | } 102 | 103 | type Crypto struct { 104 | Keyring Keyring 105 | Certificate Certificate 106 | Signature Signature 107 | Message Message 108 | CollectiveSignature CollectiveSignature 109 | DataEncryption DataEncryption 110 | RNG RNG 111 | } 112 | -------------------------------------------------------------------------------- /crypto/sss/sss.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package sss 5 | 6 | import ( 7 | "crypto/rand" 8 | "math/big" 9 | ) 10 | 11 | type Coordinate struct { 12 | X int 13 | Y *big.Int 14 | } 15 | 16 | type SSSProcess struct { 17 | n, k int 18 | m *big.Int 19 | res []*Coordinate 20 | S []byte 21 | } 22 | 23 | func Distribute(secret *big.Int, n, k int, m *big.Int) ([]*Coordinate, error) { 24 | // generate a (k-1)-degree polynomial (on m) 25 | poly := make([]*big.Int, k) 26 | poly[0] = secret 27 | for i := 1; i < k; i++ { 28 | coeff, err := rand.Int(rand.Reader, m) 29 | if err != nil { 30 | return nil, err 31 | } 32 | poly[i] = coeff 33 | } 34 | t := new(big.Int) 35 | res := make([]*Coordinate, n) 36 | for i := 0; i < n; i++ { 37 | x0 := big.NewInt(int64(i + 1)) 38 | x := new(big.Int).Set(x0) 39 | f := new(big.Int).Set(poly[0]) 40 | for j := 1; j < k; j++ { 41 | f.Mod(f.Add(f, t.Mul(poly[j], x)), m) 42 | x.Mul(x, x0) 43 | } 44 | res[i] = &Coordinate{i + 1, f} 45 | } 46 | return res, nil 47 | } 48 | 49 | func NewProcess(secrets []*Coordinate, n, k int, m *big.Int) (*SSSProcess, error) { 50 | p := &SSSProcess{ 51 | n: n, 52 | k: k, 53 | m: m, 54 | res: nil, 55 | S: nil, 56 | } 57 | for _, secret := range secrets { 58 | S, err := p.ProcessResponse(secret) 59 | if err != nil { 60 | return nil, err 61 | } 62 | if S != nil { 63 | break 64 | } 65 | } 66 | return p, nil 67 | } 68 | 69 | func (p *SSSProcess) ProcessResponse(coordinate *Coordinate) ([]byte, error) { 70 | if p.S != nil { 71 | return p.S, nil 72 | } 73 | p.res = append(p.res, coordinate) 74 | if len(p.res) == p.k { 75 | S := p.calculateSecret() 76 | p.S = S.Bytes() 77 | } 78 | return p.S, nil 79 | } 80 | 81 | func (p *SSSProcess) calculateSecret() *big.Int { 82 | var xs []int 83 | for _, r := range p.res { 84 | xs = append(xs, r.X) 85 | } 86 | S := big.NewInt(0) 87 | for _, r := range p.res { 88 | l := Lagrange(r.X, xs, p.m) 89 | S.Mod(S.Add(S, l.Mul(l, r.Y)), p.m) 90 | } 91 | return S 92 | } 93 | 94 | func Lagrange(x int, results []int, m *big.Int) *big.Int { 95 | a := big.NewInt(1) 96 | b := big.NewInt(1) 97 | xj := big.NewInt(int64(x)) 98 | for _, res := range results { 99 | if res == x { 100 | continue 101 | } 102 | xm := big.NewInt(int64(res)) 103 | a.Mul(a, xm) 104 | b.Mul(b, xm.Sub(xm, xj)) 105 | } 106 | return a.Mod(a.Mul(a, b.ModInverse(b, m)), m) 107 | } 108 | -------------------------------------------------------------------------------- /crypto/sss/sss_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package sss 5 | 6 | import ( 7 | "bytes" 8 | "math/big" 9 | "math/rand" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | var ( 15 | pb = []byte{ 16 | 0xb0, 0xa6, 0x7d, 0x9f, 0x5c, 0xeb, 0xc0, 0xff, 0xe8, 17 | 0x16, 0x90, 0xe7, 0xb2, 0x67, 0x0a, 0xb0, 0x5f, 0x9f, 18 | 0xa4, 0xc2, 0xe7, 0x36, 0x39, 0xf6, 0x60, 0xc0, 0x40, 19 | 0x8a, 0x2d, 0x9a, 0x4a, 0x8b, 0x45, 0x4a, 0x98, 0x93, 20 | 0xfd, 0x7d, 0x4e, 0x8f, 0xa3, 0x99, 0xcf, 0xc9, 0xc9, 21 | 0xba, 0x05, 0xb0, 0x80, 0xf9, 0x03, 0xe3, 0x3b, 0xcd, 22 | 0xcb, 0xef, 0xae, 0xd4, 0x09, 0x15, 0xe5, 0x1d, 0x46, 23 | 0xf5, 0x8d, 0x1a, 0x5b, 0xd2, 0x04, 0xdb, 0x20, 0xfa, 24 | 0x3f, 0xe9, 0xdb, 0x71, 0xf0, 0xb8, 0xe0, 0xaa, 0x87, 25 | 0xb5, 0x77, 0x14, 0x06, 0xf2, 0x5f, 0xad, 0x59, 0xe7, 26 | 0xf1, 0x0f, 0xe5, 0x25, 0x56, 0x44, 0x75, 0x88, 0x72, 27 | 0xea, 0x2d, 0xec, 0x1f, 0x6d, 0xcd, 0x11, 0xbe, 0x90, 28 | 0x5d, 0xe5, 0x9a, 0x04, 0x4f, 0x6c, 0x2e, 0xa3, 0x98, 29 | 0x2b, 0x22, 0x35, 0xac, 0xc9, 0x02, 0x1a, 0x19, 0x6f, 30 | 0xc4, 0xce, 0x0b, 0x19, 0xf6, 0xb3, 0x12, 0xee, 0x9c, 31 | 0xfc, 0x59, 0x97, 0xdc, 0x5f, 0x7c, 0xe2, 0xf3, 0x86, 32 | 0x13, 0x12, 0x94, 0xa5, 0x6b, 0xa9, 0x3a, 0x41, 0xa3, 33 | 0xb6, 0x0e, 0x27, 0xe0, 0x39, 0x56, 0x03, 0x9f, 0x51, 34 | 0xae, 0x73, 0xb8, 0x9c, 0x79, 0x5c, 0x5a, 0xe7, 0xd8, 35 | 0x41, 0xe9, 0xb4, 0x55, 0xc3, 0x73, 0x41, 0xc0, 0x52, 36 | 0x40, 0x4e, 0x8f, 0xe9, 0xfe, 0x4f, 0x0d, 0x52, 0xbc, 37 | 0x16, 0x2a, 0x41, 0xf1, 0xee, 0xb9, 0xef, 0x29, 0x2c, 38 | 0x66, 0xa9, 0xd6, 0xa6, 0x19, 0xaa, 0x54, 0x88, 0x07, 39 | 0xeb, 0x11, 0x87, 0xee, 0x22, 0xbd, 0x62, 0xe2, 0x0e, 40 | 0x26, 0xc3, 0xc0, 0x8c, 0x22, 0xec, 0xef, 0x12, 0xd3, 41 | 0xb2, 0x30, 0x4a, 0x01, 0x0e, 0xd1, 0xf5, 0x0a, 0x68, 42 | 0xe0, 0x26, 0x1a, 0xfe, 0x1a, 0x0b, 0xdd, 0xdf, 0x7a, 43 | 0xb8, 0xa6, 0x17, 0x74, 0xd3, 0xaf, 0x3f, 0x1c, 0xce, 44 | 0x2b, 0x95, 0xda, 0xd3, 45 | } 46 | secret = []byte("secret") 47 | ) 48 | 49 | func TestSSS(t *testing.T) { 50 | n, k := 10, 7 51 | m := new(big.Int).SetBytes(pb) 52 | s := new(big.Int).SetBytes(secret) 53 | secrets, err := Distribute(s, n, k, m) 54 | if err != nil { 55 | t.Fatal(err) 56 | } 57 | p, err := NewProcess(nil, n, k, m) 58 | if err != nil { 59 | t.Fatal(err) 60 | } 61 | rand.Seed(time.Now().Unix()) 62 | var S []byte 63 | for _, i := range rand.Perm(n) { 64 | S, err = p.ProcessResponse(secrets[i]) 65 | if err != nil { 66 | t.Fatal(err) 67 | } 68 | if S != nil { 69 | break 70 | } 71 | } 72 | if !bytes.Equal(secret, S) { 73 | t.Fatal("couldn't recover the secret") 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /crypto/threshold/dsa/dsa.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package dsa 5 | 6 | import ( 7 | "bytes" 8 | godsa "crypto/dsa" 9 | "math/big" 10 | 11 | "github.com/yahoo/bftkv/crypto" 12 | "github.com/yahoo/bftkv/crypto/sss" 13 | "github.com/yahoo/bftkv/packet" 14 | ) 15 | 16 | type dsaGroupOperations struct { 17 | params *godsa.Parameters 18 | } 19 | 20 | type dsaGroup struct { 21 | } 22 | 23 | func New(crypt *crypto.Crypto) crypto.Threshold { 24 | return NewWithGroup(crypt, &dsaGroup{}, crypto.TH_DSA) 25 | } 26 | 27 | func (g *dsaGroupOperations) CalculatePartialR(ai *big.Int) []byte { 28 | // ri = g^ai mod p 29 | ri := new(big.Int).Exp(g.params.G, ai, g.params.P) 30 | return ri.Bytes() 31 | } 32 | 33 | func (g *dsaGroupOperations) CalculateR(rs []*PartialR) *big.Int { 34 | // r = \Pi (ri)^li mod p mod q 35 | var xs []int 36 | for _, ri := range rs { 37 | xs = append(xs, ri.X) 38 | } 39 | r := big.NewInt(1) // r = g^a mod p 40 | v := big.NewInt(0) // v = a*k mod q 41 | t := new(big.Int) 42 | for _, ri := range rs { 43 | l := sss.Lagrange(ri.X, xs, g.params.Q) 44 | t.Exp(new(big.Int).SetBytes(ri.Ri), l, g.params.P) 45 | r.Mod(r.Mul(r, t), g.params.P) 46 | t.Mod(t.Mul(ri.Vi, l), g.params.Q) 47 | v.Mod(v.Add(v, t), g.params.Q) 48 | } 49 | v.ModInverse(v, g.params.Q) // v = v^-1 mod q 50 | r.Exp(r, v, g.params.P) // r = r^v mod q 51 | return r.Mod(r, g.params.Q) 52 | } 53 | 54 | func (g *dsaGroupOperations) SubGroupOrder() *big.Int { 55 | return g.params.Q 56 | } 57 | 58 | func (g *dsaGroupOperations) Serialize(bp *bytes.Buffer) error { 59 | if err := packet.WriteBigInt(bp, g.params.P); err != nil { 60 | return err 61 | } 62 | if err := packet.WriteBigInt(bp, g.params.Q); err != nil { 63 | return err 64 | } 65 | if err := packet.WriteBigInt(bp, g.params.G); err != nil { 66 | return err 67 | } 68 | return nil 69 | } 70 | func (g *dsaGroupOperations) OS2I(os []byte) *big.Int { 71 | orderSize := (g.params.Q.BitLen() + 7) / 8 72 | return new(big.Int).SetBytes(os[:orderSize]) 73 | } 74 | 75 | func (g *dsaGroup) ParseKey(key interface{}) (GroupOperations, *big.Int) { 76 | priv := key.(*godsa.PrivateKey) 77 | return &dsaGroupOperations{&priv.Parameters}, priv.X 78 | } 79 | 80 | func (g *dsaGroup) ParseParams(r *bytes.Reader) (group GroupOperations, err error) { 81 | var params godsa.Parameters 82 | params.P, err = packet.ReadBigInt(r) 83 | if err != nil { 84 | return nil, err 85 | } 86 | params.Q, err = packet.ReadBigInt(r) 87 | if err != nil { 88 | return nil, err 89 | } 90 | params.G, err = packet.ReadBigInt(r) 91 | if err != nil { 92 | return nil, err 93 | } 94 | return &dsaGroupOperations{¶ms}, nil 95 | } 96 | -------------------------------------------------------------------------------- /crypto/threshold/dsa/test.pkcs8: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIBSgIBADCCASsGByqGSM44BAEwggEeAoGBAIeq50CQzwwfmccKa1AD4CdR09J4 3 | TKBdNDkjKLjjSl7c+MxPIhClVkCcJxENL2J+FmW4H7OuUHz8M/74LF9aQRcNEkqR 4 | GPpy+9MBurk9fudZOS/Y4PEhKpGlaJSBRi1qxKnzPp639rrdYbmrZkHMgGH6Jr2F 5 | YFBNa6ZhcQsepTgdAhUAj1EDn4kpeXUG3Iy6y2QtC2FPxBMCgYAYZ8sNqKmNZimD 6 | IlFireK6OabId2qUyXl1bcDMfHXUdUV7MKqFvMXEBfTCS55xpzVG9xSM5P9iaE1v 7 | F0wuLah67wNVdOsN3tqKqTQkagVRY6JN1mLXNauOL14ikF8rqdkc+hWpCqHpfAG0 8 | OTVXCdQ9yqoWlE1SrjAETMjUR/jgVwQWAhRpeb2CtR5UjL2dv8G+HMal9CQYKw== 9 | -----END PRIVATE KEY----- 10 | -------------------------------------------------------------------------------- /crypto/threshold/dsa/test_utils/test_utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package test_utils 5 | 6 | import ( 7 | "io/ioutil" 8 | "os" 9 | "strings" 10 | 11 | "github.com/yahoo/bftkv/crypto" 12 | "github.com/yahoo/bftkv/crypto/pgp" 13 | "github.com/yahoo/bftkv/node" 14 | "github.com/yahoo/bftkv/node/graph" 15 | ) 16 | 17 | const ( 18 | scriptPath = "../../../scripts" 19 | keyPath = scriptPath + "/run/keys" 20 | clientKey = keyPath + "/u01" 21 | ) 22 | 23 | type Server struct { 24 | Self node.SelfNode 25 | Th crypto.Threshold 26 | } 27 | 28 | func NewServers(newf func(crypt *crypto.Crypto) crypto.Threshold, prefixes ...string) (map[uint64]*Server, error) { 29 | files, err := ioutil.ReadDir(keyPath) 30 | if err != nil { 31 | return nil, err 32 | } 33 | servers := make(map[uint64]*Server) 34 | for _, f := range files { 35 | for _, prefix := range prefixes { 36 | if strings.HasPrefix(f.Name(), prefix) { 37 | path := keyPath + "/" + f.Name() 38 | g := graph.New() 39 | crypt := pgp.New() 40 | if err := readPeers(g, crypt, files, prefixes); err != nil { 41 | return nil, err 42 | } 43 | if err := readCerts(g, crypt, path+"/secring.gpg", true); err != nil { 44 | return nil, err 45 | } 46 | servers[g.Id()] = &Server{ 47 | Self: node.SelfNode(g), 48 | Th: newf(crypt), 49 | } 50 | } 51 | } 52 | } 53 | return servers, nil 54 | } 55 | 56 | func readPeers(g *graph.Graph, crypt *crypto.Crypto, files []os.FileInfo, prefixes []string) error { 57 | for _, f := range files { 58 | for _, prefix := range prefixes { 59 | if strings.HasPrefix(f.Name(), prefix) { 60 | path := keyPath + "/" + f.Name() 61 | if err := readCerts(g, crypt, path+"/pubring.gpg", false); err != nil { 62 | return err 63 | } 64 | } 65 | } 66 | } 67 | return nil 68 | } 69 | 70 | func NewClient(crypt *crypto.Crypto, path string) (node.Node, error) { 71 | g := graph.New() 72 | if err := readCerts(g, crypt, path+"/pubring.gpg", false); err != nil { 73 | return nil, err 74 | } 75 | if err := readCerts(g, crypt, path+"/secring.gpg", true); err != nil { 76 | return nil, err 77 | } 78 | return node.Node(g), nil 79 | } 80 | 81 | func readCerts(g *graph.Graph, crypt *crypto.Crypto, path string, sec bool) error { 82 | f, err := os.Open(path) 83 | if err != nil { 84 | return err 85 | } 86 | defer f.Close() 87 | certs, err := crypt.Certificate.ParseStream(f) 88 | if err != nil { 89 | return err 90 | } 91 | if sec { 92 | g.SetSelfNodes(certs) 93 | } else { 94 | g.AddNodes(certs) 95 | } 96 | crypt.Keyring.Register(certs, sec, true) 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /crypto/threshold/ecdsa/ecdsa.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package ecdsa 5 | 6 | import ( 7 | "bytes" 8 | goecdsa "crypto/ecdsa" 9 | "crypto/elliptic" 10 | "encoding/binary" 11 | "math/big" 12 | 13 | "github.com/yahoo/bftkv/crypto" 14 | "github.com/yahoo/bftkv/crypto/sss" 15 | "github.com/yahoo/bftkv/crypto/threshold/dsa" 16 | "github.com/yahoo/bftkv/packet" 17 | ) 18 | 19 | type ecdsaGroup struct { 20 | } 21 | 22 | type ecdsaGroupOperations struct { 23 | curve elliptic.Curve 24 | params *elliptic.CurveParams 25 | } 26 | 27 | func New(crypt *crypto.Crypto) crypto.Threshold { 28 | return dsa.NewWithGroup(crypt, &ecdsaGroup{}, crypto.TH_ECDSA) 29 | } 30 | 31 | func (g *ecdsaGroupOperations) CalculatePartialR(ai *big.Int) []byte { 32 | x, y := g.curve.ScalarBaseMult(ai.Bytes()) 33 | return elliptic.Marshal(g.curve, x, y) 34 | } 35 | 36 | func (g *ecdsaGroupOperations) CalculateR(rs []*dsa.PartialR) *big.Int { 37 | var xs []int 38 | for _, ri := range rs { 39 | xs = append(xs, ri.X) 40 | } 41 | var x, y *big.Int 42 | v := big.NewInt(0) // v = a*k mod q 43 | t := new(big.Int) 44 | for _, ri := range rs { 45 | l := sss.Lagrange(ri.X, xs, g.params.N) 46 | x1, y1 := elliptic.Unmarshal(g.curve, ri.Ri) 47 | x1, y1 = g.curve.ScalarMult(x1, y1, l.Bytes()) 48 | if x == nil { 49 | x, y = x1, y1 50 | } else { 51 | x, y = g.curve.Add(x, y, x1, y1) // P += li*ai*P 52 | } 53 | t.Mod(t.Mul(ri.Vi, l), g.params.N) 54 | v.Mod(v.Add(v, t), g.params.N) 55 | } 56 | v.ModInverse(v, g.params.N) 57 | x, y = g.curve.ScalarMult(x, y, v.Bytes()) 58 | return x.Mod(x, g.params.N) 59 | } 60 | 61 | func (g *ecdsaGroupOperations) SubGroupOrder() *big.Int { 62 | return g.params.N 63 | } 64 | 65 | func (g *ecdsaGroupOperations) Serialize(bp *bytes.Buffer) error { 66 | if err := packet.WriteBigInt(bp, g.params.P); err != nil { 67 | return err 68 | } 69 | if err := packet.WriteBigInt(bp, g.params.N); err != nil { 70 | return err 71 | } 72 | if err := packet.WriteBigInt(bp, g.params.B); err != nil { 73 | return err 74 | } 75 | if err := packet.WriteBigInt(bp, g.params.Gx); err != nil { 76 | return err 77 | } 78 | if err := packet.WriteBigInt(bp, g.params.Gy); err != nil { 79 | return err 80 | } 81 | if err := binary.Write(bp, binary.BigEndian, uint32(g.params.BitSize)); err != nil { 82 | return err 83 | } 84 | // ignore string 85 | return nil 86 | } 87 | 88 | func (g *ecdsaGroupOperations) OS2I(os []byte) *big.Int { 89 | // copied from https://golang.org/src/crypto/ecdsa/ecdsa.go 90 | orderSize := (g.params.N.BitLen() + 7) / 8 91 | os = os[:orderSize] 92 | ret := new(big.Int).SetBytes(os) 93 | excess := len(os)*8 - g.params.N.BitLen() 94 | if excess > 0 { 95 | ret.Rsh(ret, uint(excess)) 96 | } 97 | return ret 98 | } 99 | 100 | func (g *ecdsaGroup) ParseKey(key interface{}) (dsa.GroupOperations, *big.Int) { 101 | priv := key.(*goecdsa.PrivateKey) 102 | return &ecdsaGroupOperations{priv.Curve, priv.Curve.Params()}, priv.D 103 | } 104 | 105 | func (g *ecdsaGroup) ParseParams(r *bytes.Reader) (group dsa.GroupOperations, err error) { 106 | var params elliptic.CurveParams 107 | if params.P, err = packet.ReadBigInt(r); err == nil { 108 | if params.N, err = packet.ReadBigInt(r); err == nil { 109 | if params.B, err = packet.ReadBigInt(r); err == nil { 110 | if params.Gx, err = packet.ReadBigInt(r); err == nil { 111 | if params.Gy, err = packet.ReadBigInt(r); err == nil { 112 | var sz uint32 113 | if err = binary.Read(r, binary.BigEndian, &sz); err == nil { 114 | params.BitSize = int(sz) 115 | group = &ecdsaGroupOperations{¶ms, ¶ms} 116 | } 117 | } 118 | } 119 | } 120 | } 121 | } 122 | return 123 | } 124 | -------------------------------------------------------------------------------- /crypto/threshold/ecdsa/ecdsa_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package ecdsa 5 | 6 | import ( 7 | gocrypto "crypto" 8 | goecdsa "crypto/ecdsa" 9 | "crypto/elliptic" 10 | crand "crypto/rand" 11 | "math/big" 12 | "math/rand" 13 | "testing" 14 | "time" 15 | 16 | "github.com/yahoo/bftkv/crypto" 17 | "github.com/yahoo/bftkv/crypto/pgp" 18 | "github.com/yahoo/bftkv/crypto/sss" 19 | "github.com/yahoo/bftkv/crypto/threshold/dsa" 20 | "github.com/yahoo/bftkv/crypto/threshold/dsa/test_utils" 21 | "github.com/yahoo/bftkv/node" 22 | ) 23 | 24 | const ( 25 | k = 4 26 | n = 10 27 | ) 28 | 29 | const ( 30 | scriptPath = "../../../scripts" 31 | keyPath = scriptPath + "/run/keys" 32 | clientKey = keyPath + "/u01" 33 | testTBS = "tbs..." 34 | ) 35 | 36 | func TestMul(t *testing.T) { 37 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 38 | priv, err := goecdsa.GenerateKey(elliptic.P256(), crand.Reader) 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | group, _ := (&ecdsaGroup{}).ParseKey(priv) 43 | q := group.SubGroupOrder() 44 | f0 := new(big.Int).Rand(r, q) 45 | f, err := sss.Distribute(f0, n, k, q) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | var rs []*dsa.PartialR 50 | for _, i := range r.Perm(n) { 51 | ri := group.CalculatePartialR(f[i].Y) 52 | rs = append(rs, &dsa.PartialR{ 53 | X: f[i].X, 54 | Ri: ri, 55 | Vi: f[i].Y, 56 | }) 57 | if len(rs) >= k { 58 | break 59 | } 60 | } 61 | 62 | var xs []int 63 | for _, ri := range rs { 64 | xs = append(xs, ri.X) 65 | } 66 | g := group.(*ecdsaGroupOperations) 67 | var x, y *big.Int 68 | v := big.NewInt(0) 69 | for _, ri := range rs { 70 | l := sss.Lagrange(ri.X, xs, g.params.N) 71 | x1, y1 := elliptic.Unmarshal(g.curve, ri.Ri) 72 | x1, y1 = g.curve.ScalarMult(x1, y1, l.Bytes()) 73 | if x == nil { 74 | x, y = x1, y1 75 | } else { 76 | x, y = g.curve.Add(x, y, x1, y1) // P += li*ai*P 77 | } 78 | 79 | t := new(big.Int).Mul(ri.Vi, l) 80 | t.Mod(t, g.params.N) 81 | v.Mod(v.Add(v, t), g.params.N) 82 | } 83 | 84 | if v.Cmp(f0) != 0 { 85 | t.Fatalf("mismatch: %x, %x", v, f0) 86 | } 87 | 88 | x0, y0 := g.curve.ScalarBaseMult(f0.Bytes()) 89 | if x.Cmp(x0) != 0 || y.Cmp(y0) != 0 { 90 | t.Fatalf("mismatch: (%x, %x) != (%x, %x)", x, y, x0, y0) 91 | } 92 | } 93 | 94 | type server struct { 95 | self node.SelfNode 96 | crypt *crypto.Crypto 97 | th crypto.Threshold 98 | } 99 | 100 | func TestThreshold(t *testing.T) { 101 | t.Skip("skip failing test - FIXME") 102 | // construct ECDSA parameters 103 | priv, err := goecdsa.GenerateKey(elliptic.P256(), crand.Reader) 104 | if err != nil { 105 | t.Fatal(err) 106 | } 107 | 108 | orderSize := (priv.Curve.Params().N.BitLen() + 7) / 8 109 | 110 | servers, err := test_utils.NewServers(New, "a") 111 | if err != nil { 112 | t.Fatal(err) 113 | } 114 | 115 | crypt := pgp.New() 116 | client := New(nil) 117 | self, err := test_utils.NewClient(crypt, clientKey) 118 | if err != nil { 119 | t.Fatal(err) 120 | } 121 | 122 | var peers []node.Node 123 | for _, server := range servers { 124 | peers = append(peers, server.Self) 125 | } 126 | params, algo, err := client.Distribute(priv, peers, k) 127 | if err != nil { 128 | t.Fatal(err) 129 | } 130 | shares := make(map[uint64][]byte) 131 | for i, peer := range peers { 132 | shares[peer.Id()] = params[i] 133 | } 134 | 135 | proc, err := client.NewProcess([]byte(testTBS), algo, gocrypto.SHA256) 136 | if err != nil { 137 | t.Fatal(err) 138 | } 139 | for retry := 3; retry > 0; retry-- { 140 | nodes, req, err := proc.MakeRequest() 141 | if err != nil { 142 | t.Fatal(err) 143 | } 144 | if nodes == nil || len(nodes) == 0 { 145 | t.Fatal(crypto.ErrInsufficientNumberOfThresholdSignatures) 146 | } 147 | for _, nd := range nodes { 148 | serverId := nd.Id() 149 | psig, err := servers[serverId].Th.Sign(shares[serverId], req, self.Id(), serverId) // don't be confused with self/peer IDs: peer=client(self) 150 | if err != nil { 151 | t.Fatal(err) 152 | } 153 | sig, err := proc.ProcessResponse(psig, nd) 154 | if err == crypto.ErrContinue { 155 | break 156 | } else if err != nil { 157 | t.Fatal(err) 158 | } 159 | if sig != nil { 160 | n := len(sig) / 2 161 | if n != orderSize { 162 | t.Fatalf("unmatch order size: %d, %d", n, orderSize) 163 | } 164 | r := new(big.Int).SetBytes(sig[:n]) 165 | s := new(big.Int).SetBytes(sig[n:]) 166 | h := gocrypto.SHA256.New() 167 | h.Write([]byte(testTBS)) 168 | dgst := h.Sum(nil)[:orderSize] 169 | if !goecdsa.Verify(&priv.PublicKey, dgst, r, s) { 170 | t.Fatal("verifiation failed") 171 | } 172 | return 173 | } 174 | } 175 | } 176 | t.Fatal(crypto.ErrInsufficientNumberOfThresholdSignatures) 177 | } 178 | -------------------------------------------------------------------------------- /crypto/threshold/rsa/test.pkcs8: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDidHFXkX64rpIZ 3 | eE38EfkZb+DEH5YkjFm7eBJgzEaPuGWO7eSElJKRZYonfVWwaW6/GMuUF7tJO8bu 4 | Tc8QSCIyiuW+TkO2JuEVkLETaKZ/id0Wz6nO5VjCHcuEUVu6uhcgKfjP4Q21hKjC 5 | xWL1VQ1GhSKKIBjQbVz1/9Ai4aFAltTV7tOqEsQh3VBMfb3Io+5LWwPHru/o33uQ 6 | ofoF8qrUWqNKBq0YRPdT3gm+D9pKOHrA9TiMXxvHEb7xGWCoAnDGfTfJFsbZoxzT 7 | Gi1OpzqLYIEsfxklmm/SKvxjAf6p/mJ3lPkHGuePSKLQSkibmDU7k69U26aVvE2a 8 | 54TyoPwjAgMBAAECggEAf44dZbjjiz1xt/0G2Ex7ZnOZRH+SP+X6iVlZ6MJHoJ66 9 | PecMCA3z/QgX6KR1htfsdKGP0kwHx3lvVT3ou4AWGygXvrkw9MIzmMUWqRsSZdaO 10 | /VFdkdenxjeoMlDtfWFis1BRAS8AXNVrhcJPOmd9AdLuIKVxtL1VVo+QeoVjIhCE 11 | UFSXH8Y6y0JGzZvVHth9bXBq3d87Sp+yVss+uwZqNe3rWe1o2tZ2JzQ752BBI2Oq 12 | 6qgpeWm97N2E9+LtQJCsguD/9Jo+GCKRsTjhfA9ucVbIEryucsVJrmBzWiSLmNSz 13 | MiKNV/rx7eBESw+9gt0TEbRah33aXx0D+2SQLJZvIQKBgQD4v2z2EPVA6jS0zElM 14 | xzBhDGAllWahZh85GZimQ75BBzqEbR8OSOwJNlnHXl03sRn1sOUIuMx5pIpw5VHG 15 | 1XW4dl4A5rZUcU6eqsMdEfOrdVS0EB0XmiVWHZVTVw5F0P7CuhXKtE8nkVzSshWl 16 | rGk5MlPwkN7vKOG9kl/6DmHbEwKBgQDpDqFFORmORm3q190ngyncMV4vz4eNJ30M 17 | HPgwAfEi/LRfMQ+S0ycFJC2n9xPZ634ip6whQJnba2ILiblnfiq8qERrx3YSZhs2 18 | TppFZmwLE24bVt26uthXiCxEMGU4OXB+1Bzz0Tytfa43aQoSxLls5wB0w83hJT+t 19 | 1Fya3lHssQKBgQDMtUzInGAZkiTZ/Zz9MJfh5jth8rfJb6+WYDg97FYwUfCYdu0t 20 | 5llJo47zAfeZE4iXp4J1URAFs6GzEyXfimAc9FycRUIb8DuRXFrCG7n6is6weOpY 21 | OCUt3566PVSjpEFko5u3e/gASiKnyqMB+weFaKOnwLGXBH/ycPnHYfqqPQKBgGtF 22 | LZukFYw0BKHEa+mk6J6OJpERD44778Wki+Pk8O2urQLnnQhyohIuvckC46M5Tlx7 23 | GrJPfsHM4lr3MFxfaSJevOdy7ni3gKz1bvKkmvRaJCL3T6WEHNHlPqLAN8ayQYJO 24 | 9WiQFUKxh3+/nEBZRhPHG6GjAl8v+uhyx5EM5V5xAoGAaKD72UDvg0Ht2XJ24jly 25 | wufjR1Tefir1ZbdvXuhKUzOrNXhWtuwSXBt9pihluQMkoOKZndoapVY8Lxi2itK7 26 | pS7v4oER/7EqqaL0xPbKTiDv6Oh/EX9ormzJksQ5Y7+4pEZ4qB5XNc4JHr5d90zC 27 | OAW0oTeG+O1MaI4G3BiBanI= 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /crypto/threshold/threhold.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package threshold 5 | 6 | import ( 7 | gocrypto "crypto" 8 | godsa "crypto/dsa" 9 | goecdsa "crypto/ecdsa" 10 | gorsa "crypto/rsa" 11 | 12 | "github.com/yahoo/bftkv/crypto" 13 | "github.com/yahoo/bftkv/crypto/threshold/dsa" 14 | "github.com/yahoo/bftkv/crypto/threshold/ecdsa" 15 | "github.com/yahoo/bftkv/crypto/threshold/rsa" 16 | "github.com/yahoo/bftkv/node" 17 | ) 18 | 19 | type ThresholdInstance struct { 20 | rsa crypto.Threshold 21 | dsa crypto.Threshold 22 | ecdsa crypto.Threshold 23 | } 24 | 25 | func New(crypt *crypto.Crypto) crypto.Threshold { 26 | return &ThresholdInstance{ 27 | rsa: rsa.New(crypt), 28 | dsa: dsa.New(crypt), 29 | ecdsa: ecdsa.New(crypt), 30 | } 31 | } 32 | 33 | func (instance *ThresholdInstance) Distribute(key interface{}, nodes []node.Node, k int) (shares [][]byte, algo crypto.ThresholdAlgo, err error) { 34 | var th crypto.Threshold 35 | switch key.(type) { 36 | case *gorsa.PrivateKey: 37 | th = instance.rsa 38 | case *godsa.PrivateKey: 39 | th = instance.dsa 40 | case *goecdsa.PrivateKey: 41 | th = instance.ecdsa 42 | default: 43 | return nil, algo, crypto.ErrUnsupported 44 | } 45 | return th.Distribute(key, nodes, k) 46 | } 47 | 48 | func (instance *ThresholdInstance) Sign(aux []byte, m []byte, peerId, selfId uint64) (sig []byte, err error) { 49 | algo, params := ParseParams(aux) 50 | var th crypto.Threshold 51 | switch algo { 52 | case crypto.TH_RSA: 53 | th = instance.rsa 54 | case crypto.TH_DSA: 55 | th = instance.dsa 56 | case crypto.TH_ECDSA: 57 | th = instance.ecdsa 58 | default: 59 | return nil, crypto.ErrUnsupported 60 | } 61 | return th.Sign(params, m, peerId, selfId) 62 | } 63 | 64 | func (instance *ThresholdInstance) NewProcess(tbs []byte, algo crypto.ThresholdAlgo, hash gocrypto.Hash) (crypto.ThresholdProcess, error) { 65 | var th crypto.Threshold 66 | switch algo { 67 | case crypto.TH_RSA: 68 | th = instance.rsa 69 | case crypto.TH_DSA: 70 | th = instance.dsa 71 | case crypto.TH_ECDSA: 72 | th = instance.ecdsa 73 | default: 74 | return nil, crypto.ErrUnsupported 75 | } 76 | return th.NewProcess(tbs, algo, hash) 77 | } 78 | 79 | func ParseParams(aux []byte) (algo crypto.ThresholdAlgo, data []byte) { 80 | return crypto.ThresholdAlgo(aux[0]), aux[1:] 81 | } 82 | 83 | func SerializeParams(algo crypto.ThresholdAlgo, data []byte) []byte { 84 | aux := make([]byte, len(data)+1) 85 | aux[0] = byte(algo) 86 | copy(aux[1:], data) 87 | return aux 88 | } 89 | -------------------------------------------------------------------------------- /docs/bftkv.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/bftkv/cd85d4b30a769562d8d408a53f827eda1655af0e/docs/bftkv.pdf -------------------------------------------------------------------------------- /docs/design.md: -------------------------------------------------------------------------------- 1 | # BFTKV 2 | Reliable data storage is one of the fundamental problems. 3 | BFTKV uses [b-masking quorum](http://www.cs.utexas.edu/users/lorenzo/corsi/cs380d/papers/bquorum-dc.pdf) based `read`/`write` operations to 4 | ensure [Byzantine fault-tolerance](https://dl.acm.org/citation.cfm?id=357176) and GPG's [Web of Trust](https://www.gnupg.org/gph/en/manual/x547.html) 5 | mechanism to build trust relationships between entities. Trust relationships are used to build quorums. 6 | 7 | Moreover, BFTKV provides the following guarantees: 8 | 9 | * Value corresponding to the key is up to date and not forged 10 | * Entities trying to deceive users will be revoked immediately 11 | * Entities can join and leave the system dynamically 12 | * Communications between entities are encrypted using public keys 13 | 14 | # Design 15 | BFTKV leverages integration of three concepts to provide a Byzantine fault tolerant distributed key-value storage: 16 | 17 | * Byzantine Quorum Systems 18 | * Web of Trust 19 | * Quorum Certificate 20 | 21 | In this document, we will first describe PGP's Web of Trust mechanism and then build the other concepts on top of it. 22 | 23 | ## Background 24 | ### Web of Trust 25 | Web of Trust is a way of building trust between entities without a central authority, unlike Public Key Infrastructure (PKI). 26 | Trust is established by signing public keys (implies the signer trusts the owner of the signed public key). A Web of Trust 27 | is created by exchanging signed keys between entities. 28 | 29 | Trust relationships can be represented with a graph, such as this: 30 | 31 | ![Trust Graph](images/trustGraph.png) 32 | 33 | The graph can be transcribed as: 34 | ``` 35 | Alice trusts Bob and Erin 36 | Bob trusts Erin and Dave 37 | Carol trusts Bob and Erin 38 | Erin trusts Alice, Carol and Dave 39 | ``` 40 | 41 | Web of Trust mechanism plays a huge role in BFTKV's quorum selection mechanism. 42 | 43 | ### Byzantine Quorum Systems 44 | In a network system, servers might be inaccessible or return wrong/not up to date data. Byzantine failure refers to 45 | both of these failures and the naming is based on [The Byzantine General's Problem](https://dl.acm.org/citation.cfm?id=357176). 46 | 47 | BFTKV uses `b-masking quorum`s to tolerate Byzantine failures where `b` is the number of failure nodes. 48 | `b-masking quorum`s are due to Malkhi and Reiter ([Paper](http://www.cs.utexas.edu/users/lorenzo/corsi/cs380d/papers/bquorum-dc.pdf)). 49 | 50 | ### Quorum Certificate 51 | Castro and Liskov ([Paper](http://pmg.lcs.mit.edu/papers/osdi99.pdf)) introduced 52 | a Byzantine fault tolerant replication mechanism that expects `f+1` responses 53 | from the servers to verify the data, where `f` is the number of faulty nodes. We use the Web of Trust mechanism to specify 54 | the nodes that their responses will be accepted by a quorum member. 55 | 56 | ![Data Acceptance Mechanism](images/quorumAccept.png) 57 | 58 | The following parts of this document deals with how previously discussed concepts are 59 | used in BFTKV. 60 | 61 | ## Implementation 62 | ### Quorum Selection 63 | Quorum selection is based on the trust graph built using the Web of Trust mechanism. BFTKV, usually, chooses the maximal cliques that are 64 | `L` hops away from the clients. For example, 65 | 66 | ![Example Clique Graph](images/clique.png) 67 | 68 | Client1 has two cliques: Clique 1 and Clique 2 where `L=1` 69 | Client2 has two cliques: Clique 2 and Clique 3 where `L=1` 70 | 71 | ### Write 72 | The `Write` procedure saves a value associated with a key in the system. A high-level pseudocode for the procedure: 73 | 74 | 1. Choose a quorum. 75 | 2. Get times for the key from quorum members. 76 | 3. Pick a new time that is higher than the maximum time returned by the quorum members. 77 | 4. Request and gather signatures from quorum members for new value for the key with the new timestamp. 78 | 5. Choose another quorum that includes the first quorum. 79 | 6. Write the key, value and signature set to the new quorum members. 80 | 81 | ### Read 82 | The `read` procedure reads a value associated with a key in the system. A high level pseudocode for the procedure: 83 | 84 | 1. Choose a quorum. 85 | 2. Collect values associated with the key 86 | 3. Revoke signers who signed different values with the same timestamp. 87 | 4. Return value having signatures more than the number of faulty nodes and has the maximum timestamp. 88 | 89 | ### Design Decisions and Security Analysis 90 | In this section, we will go over somewhat unclear points in the chapters we discussed `read`/`write` operations. 91 | 92 | #### Write 93 | 94 | 1. The quorum `Q` chosen here should have the property `|Q| >= 3b + 1` where `b` is the number of faulty nodes. This is required for Byzantine fault 95 | tolerance since `f` nodes may be inaccessible and `f` nodes may be returning a previous value for the key. The remaining honest `f + 1` nodes will keep the 96 | system in a safe condition. As `Q`, BFTKV uses a maximal clique that a client is connected to in the trust graph. 97 | 2. Number of timestamps should be greater than or equal to `2b + 1` since we will tolerate `b` inaccessible nodes. 98 | 3. - 99 | 4. The number of signatures `m` should be greater than `b + (n - b) / 2`. Please see the security analysis for details. 100 | 5. All nodes may be chosen which is BFTKV's current strategy. 101 | 6. Before writing, each server verifies the signature, checks the number of valid signatures gathered from quorum members and accepts write if the number 102 | is greater than `b + (n - b) / 2`. Moreover, every server makes sure that they haven't signed this key with the same timestamp before. 103 | 7. `write` operation succeeds if the received acks from server is greater than `2f + 1`. 104 | 105 | 106 | #### Read 107 | 108 | 1. BFTKV chooses a random quorum `Q` that has the property `|Q| > b + (n - b) / 2`. 109 | 2. Collect pairs up to `2f + 1`. The reason is the same with `write` operation second explanation. 110 | 3. A server should not sign the same key, same timestamp and a different value. This is equivocation attack. 111 | It is very important that servers revoke these nodes at this phase for the system to survive. 112 | 4. To return a value for the key, the value should have at least `b + 1` signatures, which guarantees that the value is valid, and have the maximum timestamp. 113 | 114 | ### Security Analysis 115 | **Equivocation Attack:** An adversary can try to create two different views of a quorum by trying to store different values for a key in half of the quorum and 116 | another value in the other half (Half is the best option from an attacker's perspective if he wants to succeed). With the help of the `b` faulty nodes, the basic check for 117 | `b + 1` signatures will succeed. However, if the quorum size is chosen carefully, this can be prevented. 118 | 119 | Consider the node states below (`n` = the number of nodes in the quorum, `b` = faulty nodes in the quorum): 120 | 121 | ![Quorum Node States](images/quorumNodeStates.png) 122 | 123 | Maximum number of signatures an attacker can get is `b + (n - b) / 2`. To make sure that the majority has the correct value `n - b > b + (n - b) / 2` should hold. Therefore `n` 124 | should be greater than `3b`. 125 | 126 | ### Detecting Equivocation on Read 127 | Let `Fp` define the failure probability of an adversary (i.e., he won't be detected); `H1` and `H2` honest node sets, `F` the set of faulty nodes and `N = H1 U H2 U F`. 128 | Then the nodes chosen from the quorum `Q` should be either from `H1 U F` or `H2 U F` to prevent detection. This probability is 129 | 130 | ``` 131 | Fp ~= 1 - ((|F| + |N|) / 2|N|)^|Q| 132 | ``` 133 | 134 | In a reqular quorum system after if the number of faulty nodes exceed `n/3` trust to data drops down to 0. BFTKV can keep the adversary's failure probability 135 | close to 1 for more than `f` failing nodes. Below two graphs represent this: 136 | 137 |

138 | Failure Detection in a Quorum System 139 | Failure Detection in BFTKV 140 |

141 | -------------------------------------------------------------------------------- /docs/http_api.md: -------------------------------------------------------------------------------- 1 | # HTTP API 2 | BFTKV node exposes two functions via HTTP API (for debugging purpose only): 3 | 4 | 1. `/read/key`, `GET` request returns the value associated with the `key` 5 | 6 | **Example**: 7 | 8 | ``` 9 | curl http://httpApiAddress/read/test // assuming the system stores <"test", "this is a test">, returns "this is a test" 10 | ``` 11 | 12 | 2. `/write/key`, `POST` request stores the data carried by the request as a key-value pair (i.e. <"key", data>). 13 | 14 | **Example**: 15 | 16 | ``` 17 | curl http://httpApiAddress/write/test -d "this is a test" // the system will store <"test", "this is a test"> 18 | ``` 19 | 20 | Please note that `httpApiAddress` can be provided on start or the default value will be used by each BFTKV node. 21 | 22 | The returned value has no proof of correctness. DO NOT use HTTP API for any applications. 23 | 24 | # Errors 25 | In addition to successful results listed above, BFTKV might return an error. Possible errors and causes are listed below: 26 | 27 | 
28 | Error                                             Cause
29 | "insufficient number of quorum"                   There are not enough responses from servers to make a quorum
30 | "insufficient number of signatures"               The number of signatures for the  is below threshold
31 | "insufficient number of responses"                The number of responses from servers is below threshold
32 | "insufficient number of valid responses"          The number of valid responses from servers is below threshold
33 | "invalid timestamp"                               Timestamp is MaxUint64
34 | "invalid signature request"                       Timestamp difference is more than maxTimestampDiff or less than 0
35 | "permission denied"                               TOFU policy error
36 | "bad timestamp"                                   Timestamp difference is more than maxTimestampDiff or less than previous
37 | "equivocation error"                              Same servers signed same key and timestamp with a different value
38 | "unknown command"                                 Unrecognized command
39 |
40 | 41 | -------------------------------------------------------------------------------- /docs/images/clique.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/bftkv/cd85d4b30a769562d8d408a53f827eda1655af0e/docs/images/clique.png -------------------------------------------------------------------------------- /docs/images/failureDetectionInBFTKV.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/bftkv/cd85d4b30a769562d8d408a53f827eda1655af0e/docs/images/failureDetectionInBFTKV.png -------------------------------------------------------------------------------- /docs/images/failureDetectionInQuorum.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/bftkv/cd85d4b30a769562d8d408a53f827eda1655af0e/docs/images/failureDetectionInQuorum.png -------------------------------------------------------------------------------- /docs/images/quorumAccept.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/bftkv/cd85d4b30a769562d8d408a53f827eda1655af0e/docs/images/quorumAccept.png -------------------------------------------------------------------------------- /docs/images/quorumNodeStates.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/bftkv/cd85d4b30a769562d8d408a53f827eda1655af0e/docs/images/quorumNodeStates.png -------------------------------------------------------------------------------- /docs/images/revokeOnRead.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/bftkv/cd85d4b30a769562d8d408a53f827eda1655af0e/docs/images/revokeOnRead.gif -------------------------------------------------------------------------------- /docs/images/trustGraph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/bftkv/cd85d4b30a769562d8d408a53f827eda1655af0e/docs/images/trustGraph.png -------------------------------------------------------------------------------- /docs/images/write.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/bftkv/cd85d4b30a769562d8d408a53f827eda1655af0e/docs/images/write.gif -------------------------------------------------------------------------------- /docs/notes.md: -------------------------------------------------------------------------------- 1 | # Implementation Notes 2 | We define (go) interface for: 3 | * Quorum system 4 | * Transport layer (including transport security) 5 | * Node (certificate) 6 | * Crypto package 7 | * signature scheme 8 | * message encryption / signature 9 | * collective signature 10 | * Keyring 11 | * RNG 12 | * Storage (DB) 13 | 14 | We implement (as of now): 15 | * Quorum system with WoT 16 | * Transport with HTTP 17 | * Transport security with PGP message encryption / signature 18 | * Certificate with PGP key 19 | * Trust graph to manage nodes 20 | * All crypto functions with PGP 21 | * PGP keyring to store certs, private key and revocation list 22 | * PGP key for certificate 23 | * PGP signatures in PGP key to construct the graph 24 | * PGP encryption / signature for transport security 25 | * PGP signature to sign 26 | * PGP signature for collective signature 27 | * PGP User ID for the URL 28 | * Storage with (plain) Unix file to store the value with the filename: "variable (hex string).timestamp", or with leveldb ("github.com/syndtr/goleveldb/leveldb") 29 | 30 | In golang, we use "golang.org/x/crypto/openpgp" for the PGP operations. Except PGP and leveldb, we use the standard library only. 31 | 32 | All messages are encrypted with the PGP key of recipients. With the PGP encryption scheme, a message is signed by the the sender and then encrypted only once with all recipient's keys. The same PGP packet is sent out to each recipient. 33 | 34 | The collective signature is just a series of the PGP signature packet in the current implementation. The reason why the interface defines the collective signature separated from the signature scheme is because of a possibility of replacing it with a threshold signature scheme in the future. 35 | 36 | ### Concurrent access to the local storage 37 | The keyring and revocation list can be updated during the operations on memory only. When the process terminates both keyring and revocation list will be stored in the local storage (not using the storage interface). The value (with the signatures) will be stored in the storage (using the storage interface) on the spot. We assume the db is not shared among multiple processes. The db can be accessed concurrently from multiple threads. 38 | 39 | ## PGP Key 40 | The following PGP packets are used in the system. 41 | ### Public-Key Packet 42 | Must include the primary public key. The key is used to verify the signature to make the trust graph within the quorum system. 43 | ### Signature Packet 44 | Represents trust incoming edges from signers. Must include the self-signed signature as specified in OpenPGP. 45 | ### Sub key packets 46 | Inside the signature packet, at least one encryption key has to be included. The system uses the key for transport-security as well as encrypting messages. 47 | ### User ID Packet 48 | Must include a unique ID. For end users, the ID must be an email address. For servers, the ID can be an URL, UUID or PGP fingerprint. 49 | ### User Attribute Packet 50 | With subtype = 101, 51 | Can include any data necessary for "email address proof", e.g., DKIM, SAML, OpenID. 52 | ### Revocation List Packet 53 | A list of PIDs the node no longer trusts while it trusted before (therefore it signed the PGP certs). 54 | -------------------------------------------------------------------------------- /docs/tests.md: -------------------------------------------------------------------------------- 1 | # Tests 2 | * All tests: `go test -v` 3 | * Note: When all tests are run, -r must be small enough that the number of open files does not exceed the maximum limit as allowed by os. The default value, 10, is small enough to keep the number of open files << 1024 (default Linux/Unix open file maximum) when the tests run concurrently. 4 | * Server test: `go test -run=TestServer -v` 5 | * TestServer: Creates servers and a client. Client writes to and then reads value from servers. 6 | * TOFU tests: `go test -run=TOFU -v` 7 | * A client writes to x for the first time: - expected: write successful 8 | * The same client overwrites with - expected: write successful 9 | * A client with a different id but same uid as the previous client overwrites with - expected: write successful 10 | * A client with neither the id or uid of the previous client, attempts to overwrite with - expected: invalid quorum certificate 11 | * Revoke tests: `go test -run=Revoke -v` 12 | * TestRevokeNone: Generates a map of times, values and signatures corresponding to a given key. Each signature signed only one value given some . Each writer wrote only one value given some 13 | * TestRevokeMaliciousClientColludingServer: Generates a map of times, values and signatures corresponding to a given key. Some signers/writers, signed/wrote more than one value given some . These signers/writers will be revoked. 14 | * Run malicious client/server test: `go test -run=Collusion -v` 15 | * TestMaliciousCollusion: A malicious client writes and to colluding servers. The colluding servers signed both values. An honest client will be able to identify and revoke the malicious writers/signers (assuming n >= 3f + 1). 16 | * Read/Write timing tests: `go test -run=Many -v` 17 | * TestManyWrites: Prints average time of write. Optional parameter -r specifies number of writes to take the average of. Default is 100. Writes are sequential. 18 | * TestManyReads: Prints average time of read. Optional parameter -r specifies number of reads to take average of. Default is 100. Reads are sequential. 19 | * Read/Write concurrency tests: `go test -run=Concurrent` 20 | * TestManyClientsConcurrentReads: Optional parameter -r specifies number of concurrent reads by each client. Default is 10. Note, this number can only be as high as maximum number of open files/number of clients as allowed by os. 21 | * TestManyClientsConcurrentWrites: Optional parameter -r specifies number of concurrent writes by each client. Default is 10. Note, this number can only be as high as maximum number of open files/number of clients as allowed by os. 22 | * Conflict test: `go test -run=Conflict -v` 23 | * TestConflict: Has n different clients concurrently write a value to the same key. Expected: n-1 invalid signature requests and 1 successful write. Invalid signature requests are returned by signers who cannot sign some value because a different client has already written to that key. 24 | -------------------------------------------------------------------------------- /docs/tex/abstract.tex: -------------------------------------------------------------------------------- 1 | \begin{abstract} 2 | \noindent 3 | Most distributed key-value stores are tolerant to only benign 4 | failure, which makes it difficult to run the system in an insecure 5 | environment (i.e., the Internet) as a single point of failure could 6 | compromise the whole system. Such systems not only are vulnerable to 7 | malicious attacks, but also need to rely on centralized authorities to 8 | protect data integrity. 9 | We developed a distributed key-value store that is tolerant 10 | to Byzantine faults, keeping it in mind to run the system over the 11 | public Internet without any central 12 | authorities. While data integrity is secured among distributed 13 | nodes, anyone can join / leave the network freely and can see all 14 | transactions. 15 | The system provides not only a robust key-value store but also 16 | authentication, encryption and signing features with a threshold 17 | cryptosystem. Integrating encryption and signature schemes with 18 | a Byzantine fault-tolerant protocol is much more robust than using 19 | separated KMS and PKI with an ordinary distributed key-value store, 20 | as it maximizes a benefit of distributed systems which fit very well 21 | with threshold cryptography. There is no single point of failure in 22 | the system. 23 | The system by itself is useful as a secure key-value store, but those 24 | properties such as Byzantine fault-tolerance, transparency, 25 | distributed global storage and threshold cryptosystem can make the 26 | system an ideal building block for blockchain technologies. 27 | \end{abstract} 28 | -------------------------------------------------------------------------------- /docs/tex/ack.tex: -------------------------------------------------------------------------------- 1 | \section*{Acknowledgements} 2 | We would like to thank Prof.~Idit Keidar for her expert advices about 3 | the quorum systems. In fact, the idea of separating the KV quorum 4 | system from the auth quorum system first appeared in her email 5 | messages. Also, Dr.~Edward Bortnikov and Prof.~Juan A.~Garay helped us 6 | move the project forward in many ways. 7 | -------------------------------------------------------------------------------- /docs/tex/algo.tex: -------------------------------------------------------------------------------- 1 | \section{Algorithms} 2 | % 3 | % algorithms 4 | % 5 | \SetAlCapNameFnt{\sf} 6 | 7 | \begin{algorithm} 8 | \label{GetQC} 9 | \caption{GetQC} 10 | \KwIn{$G$: the graph, $s$: the start node, $L$: the maximum distance} 11 | \KwOut{$QC$: quorum cliques} 12 | \SetKwData{queue}{queue} 13 | \SetKwFunction{FindMaximalClique}{FindMaximalClique} 14 | \queue $\leftarrow {(s, 0)}$\; 15 | $QC = \{\}$\; 16 | \Repeat{\queue $= \emptyset$} 17 | { 18 | $(v, d) \leftarrow$ \queue\; 19 | \If{$d \ge L$}{break} 20 | $QC = QC\; \cup$ \FindMaximalClique($G, v$)\; 21 | \ForEach{$n \in v.adj$} 22 | { 23 | \queue $\leftarrow (n, d + 1)$ if $n$ has not been visited\; 24 | } 25 | } 26 | Check if $\forall C_1, C_2 \in QC, C_1 \cap C_2 = \emptyset$\; 27 | \Return{$QC$} 28 | \end{algorithm} 29 | 30 | \begin{algorithm} 31 | \caption{FindMaximalClique} 32 | \KwIn{$G$: the graph, $s$: the start node} 33 | \KwOut{$C$: a clique} 34 | $C \leftarrow \{s\}$\; 35 | \ForEach{$v \in G.V$} 36 | { 37 | $C = C \cup \{v\} \;\text{\bf if } (c_i, v) \in G.E \wedge (v, c_i) \in G.E \text{ for } \forall c_i \in C$\; 38 | } 39 | \If{$|C| < 4$}{\Return{$\bot$}} 40 | \Return{$C$} 41 | \end{algorithm} 42 | 43 | \begin{algorithm} 44 | \label{VerifyCollectiveSignatures} 45 | \caption{VerifyCollectiveSignatures} 46 | \SetKwFunction{GetQC}{GetQC} 47 | \SetKwFunction{Verify}{Verify} 48 | \KwIn{$G$: the graph, $S = \{S_i | S_i = Sign_{Q_i}(\langle x, t, v, s_C \rangle)\}$} 49 | $QC = $ \GetQC($G, self$)\; 50 | \ForEach{$C \in QC$} 51 | { 52 | $cnt = 0$\; 53 | \ForEach{$s \in S \cap C$} 54 | { 55 | \If{\Verify($s$)} 56 | { 57 | $cnt++$\; 58 | } 59 | } 60 | \If{$cnt \leq 2f$} 61 | { 62 | \Return{false} 63 | } 64 | } 65 | \Return{true} 66 | \end{algorithm} 67 | 68 | \begin{algorithm} 69 | \label{CheckQuorumCert} 70 | \caption{CheckQuorumCert} 71 | \SetKwFunction{FindMaximalClique}{FindMaximalClique} 72 | \KwIn{$G$: the graph, $Cert$: quorum certificate} 73 | $C = $ \FindMaximalClique($self$)\; 74 | $cnt = 0$\; 75 | \ForEach{$c \in Cert$} 76 | { 77 | \If{$c.Signer \in C$ {\bf and} \Verify(c.Signature)} 78 | { 79 | $cnt$++\; 80 | } 81 | } 82 | \Return{$cnt > (2 |C|) / 3$} 83 | \end{algorithm} 84 | 85 | \begin{algorithm} 86 | \label{CheckEquivocation} 87 | \caption{CheckEquivocation} 88 | \SetKwData{Storage}{Storage} 89 | \SetKwFunction{Revoke}{Revoke} 90 | \KwIn{$req = \langle x, t, v, s_C, S \rangle$} 91 | $z = $ \Storage$[x, t]$\; 92 | \If{$z \neq \bot$ {\bf and} $req.v \neq z.v$} 93 | { 94 | \Revoke($req.S \cap z.S$)\; 95 | } 96 | \end{algorithm} 97 | 98 | \begin{algorithm} 99 | \label{CheckTOFU} 100 | \caption{CheckTOFU} 101 | \SetKwData{Storage}{Storage} 102 | \KwIn{$req = \langle x, t, v, s_C, S \rangle$} 103 | Verify $req.s_C$ with quorum certificate\; 104 | \eIf{\Storage$[x, 0] = \bot$} 105 | { 106 | \Storage$[x, t] = req$\; 107 | }{ 108 | $last = $ \Storage$[x, t-1]$\; 109 | \eIf{$last.s_C.cert.ID$ = $req.s_C.cert.ID$} 110 | { 111 | \Storage$[x, t] = req$\; 112 | }{ 113 | Error\; 114 | } 115 | } 116 | \end{algorithm} 117 | 118 | \ifdefined\ABSTRACT 119 | \else 120 | 121 | \begin{algorithm} 122 | \label{Join} 123 | \caption{Join} 124 | \SetKwFunction{Send}{Send} 125 | \SetKwFunction{Receive}{Receive} 126 | \KwIn{Cert} 127 | $G.V = Cert.sigs[*].cert$\; 128 | $peers = G.V$\; 129 | \For{$peers = \bot$} 130 | { 131 | $newPeers = \{\}$\; 132 | \For{$peer \in peers$} 133 | { 134 | \Send($peer.addr, Cert$)\; 135 | $certs = $ \Receive()\; 136 | $newPeers = newPeers \cup (certs \setminus G.V$)\; 137 | $G.V = G.V \cup certs$\; 138 | } 139 | $peers = newPeers$\; 140 | } 141 | \end{algorithm} 142 | 143 | \begin{algorithm} 144 | \label{Register} 145 | \caption{Register} 146 | \SetKwData{Storage}{Storage} 147 | \SetKwFunction{FindMaximalClique}{FindMaximalClique} 148 | \SetKwFunction{Sign}{Sign} 149 | \KwIn{$req$: a client certificate, $proof$: the proof of the password 150 | authentication} 151 | $z = $ \Storage$[req.x]$\; 152 | \If{$z \neq \bot$ {\bf and} $req.s_C.ID = z.s_C.ID$} 153 | { 154 | $clique = $ \FindMaximalClique($self$)\; 155 | \If{$proof \subseteq clique$ {\bf and} $|proof| \ge 156 | |clique|\cdot2/3$} 157 | { 158 | \Sign($req$)\; 159 | } 160 | } 161 | \end{algorithm} 162 | 163 | \fi 164 | -------------------------------------------------------------------------------- /docs/tex/analysis.tex: -------------------------------------------------------------------------------- 1 | \section{Security Analysis} 2 | We look into attacks against the fundamental property: 3 | $READ(Q_1,x) = READ(Q_2,x)$ for $\forall Q_1, Q_2 \in QS$, which is 4 | known as equivocation. The best that attackers can do is divide a 5 | clique into two sets ($\mathcal{H}_1$ and $\mathcal{H}_2$) and ask 6 | each set to sign $\langle x,t,v \rangle$ and $\langle x,t,v' \rangle$ 7 | separately. Then do the {\sf write} protocol for the target nodes with 8 | collected signature sets $S$ and $S'$. Honest servers will refuse the 9 | request because it does not satisfy the basic $b$-masking quorum 10 | condition: $|S| \geq b+1$. But with $b+1$ colluding nodes 11 | ($\mathcal{F}$), the attack will succeed. 12 | 13 | \ifdefined\ABSTRACT 14 | \else 15 | \newcommand{\slice}[4]{ 16 | \pgfmathparse{0.5*#1+0.5*#2} 17 | \let\midangle\pgfmathresult 18 | 19 | % slice 20 | \draw[thick,fill=black!10] (0,0) -- (#1:1) arc (#1:#2:1) -- cycle; 21 | 22 | % outer label 23 | \node[label=\midangle:#4] at (\midangle:1) {}; 24 | 25 | % inner label 26 | \pgfmathparse{min((#2-#1-10)/110*(-0.3),0)} 27 | \let\temp\pgfmathresult 28 | \pgfmathparse{max(\temp,-0.5) + 0.8} 29 | \let\innerpos\pgfmathresult 30 | \node at (\midangle:\innerpos) {#3}; 31 | } 32 | 33 | \begin{tikzpicture}[scale=1.5] 34 | 35 | \newcounter{a} 36 | \newcounter{b} 37 | \foreach \p/\t/\l in {25//,40/$\mathcal{H}_1$/Honest nodes, 38 | 20/$\mathcal{F}$/Faulty nodes, 40/$\mathcal{H}_2$/Honest nodes} 39 | { 40 | \setcounter{a}{\value{b}} 41 | \addtocounter{b}{\p} 42 | \slice{\thea/100*360} 43 | {\theb/100*360} 44 | {\t}{\l} 45 | } 46 | 47 | \end{tikzpicture} 48 | \fi 49 | 50 | The maximum number of signatures dishonest clients can get is 51 | $b+(n-b)/2$. Therefore, to overcome the attack we need: 52 | \[ n-b > b+(n-b)/2 \Rightarrow n > 3b. \] 53 | 54 | \subsubsection*{Detecting equivocation on {\sf read}} 55 | Even if the number of faulty nodes exceeds the above threshold, the 56 | system can detect malicious actions with the following probability: 57 | \begin{align*} 58 | D_p &= Pr[Q \cap \mathcal{H}_1 \neq \emptyset \wedge Q \cap 59 | \mathcal{H}_2 \neq \emptyset] \\ 60 | & = 1 - Pr[Q \subseteq \mathcal{F} \cup \mathcal{H}_1] \\ 61 | & = 1 - ((n + f) / 2n)^{|Q|} 62 | \end{align*} 63 | when $f > b$ and $|Q| < (f + n)/2$, where $f = |\mathcal{F}|$ is the 64 | number of the faulty nodes, assuming the sizes of $\mathcal{H}_1$ and 65 | $\mathcal{H}_2$ are the same. 66 | 67 | In the case of $f \le b$ malicious messages will never go through 68 | therefore it does not make sense to talk about the detection rate on 69 | {\sf read}, however, it is possible to detect malicious messages on 70 | {\sf write} if such actions are taken by clients. 71 | When the number of faulty nodes exceeds the threshold, i.e., $f > b$, 72 | it will be possible that the (honest) client cannot detect all 73 | malicious actions. Since the minimum quorum size is $(n-1)/3$ it does 74 | not need to consider the case where $|Q| < n/3$. Also, if the size 75 | exceeds $(f+n)/2$ any quorum always includes at least one node from 76 | each $\mathcal{H}_i$ which makes the detection rate 100\%. 77 | 78 | \ifdefined\ABSTRACT 79 | \else 80 | \begin{tikzpicture}[xscale=6,yscale=6] 81 | \draw [<->] (0,0.9) -- (0,0) -- (1.0,0); 82 | \node [below right] at (1,0) {$|Q|$}; 83 | \node [left] at (0,{1-pow(17/300,0.7)}) {$1$}; 84 | \node [left] at (0,0) {$0$}; 85 | \node [below] at (0.3,0) {$n/3$}; 86 | \node [below] at (0.7,0) {$(f+n)/2$}; 87 | \draw[dashed, domain=0:0.3] plot (\x, {1-pow(17/300,\x)}); 88 | \draw[thick, domain=0.3:0.7] plot (\x, {1-pow(17/300,\x)}); 89 | \draw[dashed, domain=0.7:1.0] plot (\x, {1-pow(17/300,0.7)}); 90 | \end{tikzpicture} 91 | \fi 92 | 93 | For example, assuming we choose a quorum $|Q| = 3b+1$ out of $n = 4b+1$, 94 | which is the default setup of the kv quorum system, the detection rate 95 | is 100\% up to $f = 2b$ failure nodes. 96 | 97 | \ifdefined\ABSTRACT 98 | \else 99 | \begin{tikzpicture}[xscale=6,yscale=6] 100 | \draw [<->] (0,1.05) -- (0,0) -- (1.05,0); 101 | \node [left] at (0,1) {$1$}; 102 | \node [left] at (0,0) {$0$}; 103 | \node [below] at (0.3,0) {$n/3$}; 104 | \node [below] at (0.7,0) {$2n/3$}; 105 | \node [below] at (1,0) {$n$}; 106 | \node [right] at (1.05,0) {$f$}; 107 | \draw[thick, smooth] (0,1) to (0.6,1) to [out=0,in=90] (1,0); 108 | \end{tikzpicture} 109 | \fi 110 | -------------------------------------------------------------------------------- /docs/tex/application.tex: -------------------------------------------------------------------------------- 1 | \section{Applications} 2 | We discuss some potential appliations of BFTKV. 3 | 4 | \subsection{Decentralized PKI with DKMS} 5 | 6 | User authentication is a long-standing problem for end-to-end 7 | systems. Even if we have semantically secure cryptographic protocols 8 | to exchange data between users, if it was with a wrong one, the whole 9 | security system would not make sense. On the other hand, once we have 10 | a robust user authentication scheme, we can build up many kinds of 11 | security systems on top of that, such as PGP and Signal. Our goal is 12 | to construct an infrastructure to exchange public keys that represent 13 | users' identities. Exchanging public keys can be done in person, 14 | using a QR code, confirming the fingerprint of public keys, etc. Those 15 | methods seem to be relevant for some situations, such as sending 16 | money. Also, public key infrastructures using central authorities, 17 | such as X.509 which is based on chain of trust, are widely used. A PKI 18 | like X.509, however, still have a problem when issuing a certificate 19 | to each end user. CAs issue certificates to corporates, organizations, 20 | and individuals based on trustworthiness of requesters but for end 21 | users whose authenticity is not easy to be proven, we have the same 22 | basic issues. From end users' point of view, blindly relying on a 23 | central authority based on its authenticity is no longer secure and 24 | contradict the end-to-end philosophy. 25 | 26 | Our proposed PKI does not ``strongly'' rely on central authorities, 27 | yet it does not require to exchange public keys in person. Here are 28 | the high level system requirements: 29 | \begin{itemize} 30 | \item Scalability -- the system can grow without affecting the current running services 31 | \item Transparency -- anyone can monitor every system activity 32 | \item Quantifiability -- security and efficiency can be formally analyzed 33 | \item Robustness -- the system has to recover from erroneous situations by itself 34 | \item Privacy -- the system should not reveal unnecessary information about users 35 | \item Non-interactivity -- a client may not be able to interact peers 36 | before sending a message. This particular requirement makes it 37 | difficult to design a system that guarantees the ``what I saw is what 38 | you see'' concept. SMTP, for example, is not a mutual explicit 39 | authentication protocol. When an email is encrypted then sent out, if 40 | it is encrypted with a wrong key, it will be too late -- someone in 41 | the middle could read the email when the recipient receives the email 42 | and notice that the encrypted email is not actually for her. 43 | \end{itemize} 44 | 45 | \subsection{Consensus Mechanism for Blockchain Technologies} 46 | Key-value stores can be suitable to store transactions. Without 47 | knowing the key it will be difficult to access the value -- in this 48 | sense it can be less transparent than other ledgers such as hash 49 | chain. To keep the transactions we can simply use the hash value of 50 | the whole transaction data. As every transaction should have a public 51 | key (``address'') and transaction ID in the sense of bitcoin 52 | Tx, the hash value should be unique. We write the value with 53 | the WRITE ONCE permission. 54 | -------------------------------------------------------------------------------- /docs/tex/background.tex: -------------------------------------------------------------------------------- 1 | \section{Background} 2 | This section describes some existing technologies used in the system. 3 | 4 | \subsection{Quorum Systems} 5 | A variety of quorum systems have been used to manage replicated data / 6 | storage in distributed systems. We briefly describe the original 7 | quorum systems and its extension called Byzantine quorum 8 | systems. Later, we construct a byzantine quorum system based on Web of 9 | Trust. 10 | The system defines two operations, {\sf read} and {\sf write}, between 11 | a client and a set of servers called a quorum. A quorum system ($QS 12 | \subseteq 2^U$) is a subset of the powerset of all servers ($U$), and 13 | it satisfies a property: 14 | \[ 15 | \forall Q_1, Q_2 \in QS, Q_1 \cap Q_2 \neq \emptyset 16 | \] 17 | \hfill (intersection property)\\ 18 | 19 | With this property, it is guaranteed that the client always retrieves 20 | the latest value from at least one server. 21 | 22 | Melhi and Reiter \cite{Delhi:1} extend the property to handle 23 | Byzantine failure: 24 | \begin{align*} 25 | \forall Q_1, Q_2 \in QS, \forall B \in BF, Q_1 \cap Q_2 \nsubseteq B 26 | \end{align*} 27 | where $BF \subseteq 2^U$ and $\cup_{B \in BF} B$ is all Byzantine 28 | failure nodes. 29 | Especially when $|Q_1 \cap Q_2| \geq 2b+1$, $QS$ is called a $b$-masking 30 | quorum system. When the client can be dishonet, signed messages are no 31 | longer trustworthy therefore we rely on the quorum system to avoid 32 | equivocation (aka double spending in blockchain terms). 33 | 34 | \subsection{Web of Trust} 35 | A web of trust is a directed graph $G = (V, E)$, where $V$ is a set of 36 | nodes (each of which is a pair of unique ID and public key) and $E$ is a 37 | set of trust relationship: when $(v_1, v_2) \in E$, $v1$ {\em trusts} 38 | $v_2$, 39 | i.e., the certificate of $v2$ includes a signature over its public key 40 | with the private key of $v_1$. WoT was introduced by PGP to 41 | authenticate certificates of peers without relying on central 42 | authorities. We use the same mechanism to authenticate not only 43 | end-users' certificates but quorum members' as well. 44 | 45 | \subsection{Threshold Cryptosystems} 46 | Threshold cryptosystems play important roles in this system. We use it 47 | not just for fault-tolerance but for improving security. 48 | 49 | Shamir's Shared Secret (SSS) \cite{shamir} is a major tool to 50 | construct threshold cryptographic schemes. The system uses SSS for 51 | both password authentication and DSA / ECDSA signature 52 | schemes. Especially for the latter, the system uses a threshold 53 | siganture scheme introduced by Gennaro et al.\ \cite{Gennaro}. For the 54 | threshold password authentication, the system uses a simple DH key 55 | exchange protocol with a similar setting to \cite{ford}. 56 | See ~\ref{auth} for the details. 57 | SSS uses a $t-1$ degree random polynomial on $\mathbb{Z}_q$ 58 | \[ 59 | f(x) = \sum_{i=0}^{t-1}a_ix^i \bmod q 60 | \] 61 | Each {\em share} is $(i, f(i))_{i = 1..n}$. To reconstruct the shared 62 | secret $f(0)$, calculate lagrange interpolate from $t$ responses out 63 | of $n$ 64 | \[ 65 | \lambda_j = \prod_{l \in \mathcal{T} \setminus \{j\}} 66 | i_l / (i_l - i_j) \bmod q 67 | \] 68 | then we get 69 | \[ 70 | f(0) = \sum_{j \in \mathcal{T}} f(j)\lambda_j 71 | \] 72 | where $\mathcal{T}$ is a subset of $\{1..n\}$ and $|\mathcal{T}| = 73 | t$. 74 | 75 | To construct a $(t, n)$ threshold scheme, we follow the quorum 76 | threshold, i.e., $n = |Q|, t = n-b$. 77 | 78 | For RSA signatures, it may seem straightforward to apply SSS to the 79 | RSA signing process such as: 80 | \[ S = M^{f(0)} = \prod_{i \in \mathcal{T}} M^{f(i)\lambda_i} \bmod N \] 81 | however, since to calculate $\lambda_i$ we need the multiplicative 82 | inverse in $\varphi(N)$ which must be kept as secret as the private 83 | key, we cannot simply appy SSS to RSA signatures. Shoup \cite{shoup} 84 | solved this problem by getting rid of multiplicative inverse all 85 | together but it needs a special construction of the RSA parameters, 86 | which makes it difficult to apply existing keys to the 87 | method. Therefore we use a simple key hierachy to address this 88 | issue. See \ref{thrsa} for the details. 89 | 90 | For DSA and ECDSA, SSS has preferable properties due to its linearity: 91 | \begin{align*} 92 | f(0) + g(0) &= \sum_{i \in \mathcal{T}} (f(i) + g(i)) \lambda_i \\ 93 | f(0) \cdot g(0) &= \sum_{i \in \mathcal{T'}} (f(i) \cdot g(i)) \lambda_i 94 | \end{align*} 95 | where 96 | \begin{align*} 97 | f(0) &= \sum_{i \in \mathcal{T}} f(i) \lambda_i \\ 98 | g(0) &= \sum_{i \in \mathcal{T}} g(i) \lambda_i 99 | \end{align*} 100 | $\mathcal{T'}$ is a subset of $\{1..n\}$ and $|\mathcal{T'}| = 101 | 2t$. These properties make it possible to calculate the DSA / ECDSA 102 | signature $(r, s)$ which invovles additions and multiplications of 103 | shared variables, such that: 104 | \begin{align*} 105 | r &= g^{k^{-1}} \bmod p \bmod q\\ 106 | s &= k(x + mr) \bmod q 107 | \end{align*} 108 | from partial signatures $(r_i, v_i, s_i)$: 109 | \begin{align*} 110 | r_i &= g^{a_i} \bmod p \\ 111 | v_i &= k_ia_i \bmod q \\ 112 | s_i &= k_i(x_i + mr) \bmod q 113 | \end{align*} 114 | where $x_i$ is the share of the private key for each node. $(a_i, k_i)$ is 115 | generated randomly by each node. See \cite{Gennaro} for how to 116 | construct $r$ from $(r_i, v_i)$. 117 | -------------------------------------------------------------------------------- /docs/tex/bftkv.tex: -------------------------------------------------------------------------------- 1 | %\documentclass[10pt,fleqn]{article} 2 | %\documentclass[letterpaper,10pt,fleqn]{article} 3 | \documentclass[twoside,twocolumn,10pt,fleqn]{article} 4 | \usepackage[margin=1in,columnsep=20pt]{geometry} 5 | \usepackage{amsmath} 6 | \usepackage{amsthm} 7 | \usepackage{amsfonts} 8 | \usepackage{amssymb} 9 | \renewcommand{\theequation}{\arabic{equation}} 10 | \usepackage[ruled,noline]{algorithm2e} 11 | \usepackage{tikz} 12 | \usepackage{hyperref} 13 | \newtheorem{lemma}{Lemma} 14 | 15 | \title{A Byzantine Fault-tolerant KV Store for\\Decentralized PKI and Blockchain} 16 | \author{% 17 | \textsc{Ryuji Ishiguro}\\% 18 | } 19 | \date{\today} 20 | 21 | \usepackage{titling} % Customizing the title section 22 | \usepackage{abstract} % Allows abstract customization 23 | \renewcommand{\maketitlehookd}{% 24 | \begin{center} 25 | \url{https://github.com/yahoo/bftkv} 26 | \end{center} 27 | \input{abstract} 28 | } 29 | 30 | \begin{document} 31 | 32 | \maketitle 33 | 34 | \input{intro} 35 | 36 | \input{background} 37 | 38 | \input{method} 39 | 40 | \input{protocol} 41 | 42 | \input{analysis} 43 | 44 | \input{application} 45 | 46 | \input{conclusion} 47 | 48 | \input{ref} 49 | 50 | \input{ack} 51 | 52 | \appendix 53 | \input{tpa} 54 | 55 | \input{algo} 56 | 57 | \end{document} 58 | -------------------------------------------------------------------------------- /docs/tex/bftkv_eabs.tex: -------------------------------------------------------------------------------- 1 | %\documentclass[10pt,fleqn]{article} 2 | %\documentclass[letterpaper,10pt,fleqn]{article} 3 | \documentclass[twoside,twocolumn,10pt,fleqn]{article} 4 | \usepackage[margin=1in,columnsep=20pt]{geometry} 5 | \usepackage{amsmath} 6 | \usepackage{amsthm} 7 | \usepackage{amsfonts} 8 | \usepackage{amssymb} 9 | \renewcommand{\theequation}{\arabic{equation}} 10 | \usepackage[ruled,noline]{algorithm2e} 11 | \usepackage{tikz} 12 | \usepackage{hyperref} 13 | 14 | \newcommand*{\ABSTRACT}{} 15 | 16 | \title{A Byzantine Fault-tolerant KV Store for\\Decentralized PKI and 17 | Blockchain\\(Extended Abstract)\footnotemark[1]} 18 | 19 | \author{% 20 | \textsc{Ryuji Ishiguro}\\ 21 | \normalsize \href{mailto:rishiguro@oath.com}{rishiguro@oath.com}\\ 22 | \normalsize \url{https://github.com/yahoo/bftkv}\\ 23 | } 24 | \date{\today} 25 | 26 | \usepackage{titling} % Customizing the title section 27 | \usepackage{abstract} % Allows abstract customization 28 | \renewcommand{\maketitlehookd}{% 29 | % \begin{center} 30 | % \url{https://github.com/yahoo/bftkv} 31 | % \end{center} 32 | \input{abstract} 33 | } 34 | 35 | \begin{document} 36 | 37 | \maketitle 38 | 39 | \footnotetext[1]{The full paper is available from the github 40 | repository. \url{https://github.com/yahoo/bftkv/tree/master/docs/bftkv.pdf}} 41 | 42 | \input{intro} 43 | 44 | \input{method} 45 | 46 | \input{analysis} 47 | 48 | \input{conclusion} 49 | 50 | \input{ref} 51 | 52 | \input{ack} 53 | 54 | \appendix 55 | \newpage 56 | \input{algo} 57 | 58 | \end{document} 59 | -------------------------------------------------------------------------------- /docs/tex/conclusion.tex: -------------------------------------------------------------------------------- 1 | \ifdefined\ABSTRACT 2 | \section{Conclusions} 3 | \else 4 | \section{Conclusions and Future Work} 5 | \fi 6 | We developed an efficient and secure distributed key-value store which 7 | can be a solution for a long standing problem: How to authenticate 8 | public keys without relying on central authorities. Here is a quote 9 | from ``The Sesame Algorithm'' \cite{signal} which is a part of the 10 | Signal protocol: 11 | \begin{quote} 12 | ``Sesame relies on users to authenticate identity public keys with their 13 | correspondents. For example, a pair of users might compare public key 14 | fingerprints for all their devices manually, or by scanning a QR 15 | code. The details of authentication methods are outside the scope of 16 | this document. 17 | 18 | If authentication is not performed, users receive no cryptographic 19 | guarantee as to who they are communicating with.'' 20 | \end{quote} 21 | BFTKV is designed to be the last piece of this kind of real end-to-end 22 | encryption system. 23 | 24 | We also integrated unique threshold password authentication and 25 | distributed signature schemes with a Byzantine fault-tolerant 26 | key-value store to solve nasty problems like unauthorized enrollment, 27 | recovering from a key-loss situation and supporting multiple devices 28 | under the same ID, which most permissioned blockchain systems will 29 | suffer when they actually try to deploy a system. 30 | 31 | \ifdefined\ABSTRACT 32 | \else 33 | \subsection*{Future Work} 34 | Scalability of bitcoin blockchain is excellent -- you can add as many 35 | nodes as you want and it will not affect the whole system, but the 36 | latency and throughput of the whole process is terrible -- it takes 37 | one hour to settle a transaction. 38 | On the other hand, BFT type of blockchain settles transactions in a 39 | range from sub milliseconds to a few seconds, but scaling out is 40 | difficult. Specifically, with quorum systems we have theoretical lower 41 | bounds such as $n = 3f + 1$ and we cannot do anything about it. 42 | 43 | BFTKV can separate the role of the node into two parts: signing and 44 | reading / writing, corresponding to the auth quorum and kv quorum for 45 | load balancing. 46 | Once a transaction gets a collective signature, the transaction 47 | can be stored in any way. The only concern is how to know a 48 | transaction is latest. Each transaction has the timestamp and we can 49 | easily get the latest $\langle x, t, v \rangle$ corresponding to a $x$ 50 | in a node but we do not know if other nodes might have a newer 51 | value. BFTKV addresses this issue by a simple quorum system, i.e., as 52 | long as $\forall Q_1, Q_2 \in QS, |Q_1 \cap Q_2| > f$ holds, at least 53 | one node in a quorum has the latest value and the client will choose 54 | that one. This method is simple but it is difficult to scale out. We 55 | need to collect data from at least $f+1$ nodes, which depends on the 56 | total number of servers. 57 | 58 | Another concern is the number of collective signatures. Increasing the 59 | number of servers does not help improve the throughput at all, because 60 | every available server in quorum cliques needs to get invovled in 61 | the signing process. Also it will increase the number of the 62 | collective signatures so that each client and kv node needs to verify 63 | more signatures. 64 | One of the ideas to address this issue is to use a threshold signature 65 | to combine the set of collective signatures so that each node verifies 66 | the signature only once. But we need a dealer for the threshold 67 | signature scheme and it will contradict the decentralized concept and 68 | make the system less flexible. 69 | \fi 70 | -------------------------------------------------------------------------------- /docs/tex/intro.tex: -------------------------------------------------------------------------------- 1 | \section{Introduction} 2 | How to reach a consensus with Byzantine type failure is the main 3 | problem of blockchain technologies. Bitcoin blockchain uses PoW (Proof 4 | of Work) with incentives \cite{bitcoin}. The majority of public 5 | blockchain (or permission-less blockchain) systems have the same type 6 | of consensus mechanism. Another way to establish a consensus is to use 7 | BFT (Byzantine Fault-Tolerant) protocols, which is a major mechanism 8 | for local/enterprise blockchain (or permissioned system). Some PoS 9 | (Proof of Stake) type blockchain technologies use BFT as well to avoid 10 | the mining process. 11 | 12 | The proposed system is based on the Byzantine quorum system introduced 13 | by Malhi and Reiter \cite{Delhi:1}, and adopts a BFT protocol proposed 14 | by the same authors \cite{Delhi:2}. We construct a $b$-masking quorum 15 | system based on the WoT (Web of Trust) graph. We also introduce quorum 16 | certificates combined with a threshold authentication scheme to 17 | protect data from unauthorized mutation. Unlike a centralized PKI such 18 | as X.509, quorum certificates are verified collectively with other 19 | quorum members that are chosen with a dynamic graph constructed 20 | independently at each node. With the strong authentication scheme 21 | backed by a threshold cryptosystem and flexible certificate mechanism, 22 | the system allows anyone to join / leave the network freely without 23 | any authorization process while it is immune from sybil attacks. 24 | 25 | Another key aspect of blockchain technologies is the global 26 | distributed ledger. Bitcoin blockchain uses hash chain -- transactions 27 | in a specific period are all hashed together with the hash value of 28 | the previous block. All bitcoin nodes maintain the same view of the 29 | hash chain. 30 | The proposed system uses a distributed key-value store with the TOFU 31 | (Trust on First Use) policy, that is, the first use of a key locks out 32 | others from mutating the value associated with the key. Only the 33 | user who has written the key-value first will posses the right to 34 | update the value. Also it supports WRITE ONCE permission which 35 | guarantees that once a value is written it will never be modified in 36 | any way. This special permission will be preferable for applications 37 | like the global ledger which must be tamper-proof. 38 | The system does not guarantee absolute consistency. Also the order of 39 | transactions for different keys does not matter. The system does not 40 | even provide eventual consistency among individual nodes. Instead, it 41 | has a property such that: $READ(Q_1, x) = READ(Q_2, x), \forall Q_1, 42 | Q_2 \in QS$, that is, the consistency is established collectively with 43 | a quorum system ($QS$). 44 | 45 | Smart contracts play an important role in some blockchain technologies 46 | such as Ethereum and Hyperledger Fabric. As the proposed system is a simple 47 | key-value store, it does not provide a platform for smart contracts at 48 | the moment. Also, the system itself does not provide any incentive 49 | mechanism to discourage nodes to do malicious actions, or to encourage 50 | to participate the consensus process. The system has a robust 51 | revocation scheme but it does not answer for a question like why would 52 | one want to run a node? Although all transactions keep a proof of good 53 | or bad actions for each node therefore it will be straightforward to 54 | map it to economic incenstives and penalties, we defer fintech 55 | discussions to an application layer. 56 | -------------------------------------------------------------------------------- /docs/tex/protocol.tex: -------------------------------------------------------------------------------- 1 | \section{Protocols} 2 | \label{Protocols} 3 | The system uses Phalanx \cite{Delhi:2} for the underlying {\sf 4 | read/write} operations. To maintain a graph for the quorum system, we 5 | extend the protocols with {\sf join/leave}. Also {\sf register} 6 | protocol is used to get quorum certificates with the threshold 7 | password authentication. 8 | 9 | {\em Notations}: 10 | $\langle x, t, v, s_C, S \rangle$ is an ordered set that denotes the protocol 11 | data. $s_C$ and $S$ can be omitted. 12 | 13 | \begin{tabular}{cl} 14 | $x$ & \begin{minipage}[t]{0.8\columnwidth}% 15 | is the variable and an arbitrary length octet 16 | string. The variable will be the ``key'' of the key-value store.% 17 | \end{minipage}\\ 18 | $t$ & \begin{minipage}[t]{0.8\columnwidth}% 19 | is the timestamp and a 64-bit non-negative 20 | integer. $2^{64}-1$ is a special value to denote that the variable is 21 | no longer able to be updated.% 22 | \end{minipage}\\ 23 | $v$ & \begin{minipage}[t]{0.8\columnwidth}% 24 | is the value and an arbitrary length octet string.% 25 | \end{minipage}\\ 26 | $s_C$ & \begin{minipage}[t]{0.8\columnwidth}% 27 | is an object of the signature and quorum certificate 28 | of a client $C$. $s_C.sig$ is the signature over $\langle x, t, v 29 | \rangle$ signed by the private key of $C$. $s_C.cert$ is the quorum 30 | certificate of $C$. $s_C.cert.ID$ is the unique ID to identify each 31 | node.% 32 | \end{minipage}\\ 33 | $S$ & \begin{minipage}[t]{0.8\columnwidth}% 34 | is an unordered set of the signature object. Each $S_i 35 | \in S$ has the 36 | same structure of the $s_C$. The signers must be quorum members.% 37 | \end{minipage}\\ 38 | \end{tabular} 39 | 40 | \subsection{Read / Write} 41 | \label{rw} 42 | {\sf read/write} protocols are done between a client ($C$) and a 43 | quorum ($Q$). A quorum is chosen from a quorum system ($QS \subseteq 44 | 2^U$) which is a subset of the powerset of all nodes ($U$). To write 45 | a value $v$ into a variable $x$, we follow the write protocol 46 | specified in Phalanx.\\ 47 | 48 | [ {\sf write} ] 49 | \begin{align*} 50 | C :& \text{ choose a quorum from quorum cliques} \\ 51 | & Q \in QC \\ 52 | C \rightarrow Q :& \text{ \sf get timestamp}(x) \\ 53 | C \leftarrow Q_i :& \text{ } t_i \\ 54 | C :& \text{ collect } 2b+1 \text{ timestamps: } \\ 55 | & \text{ } \{t_i\}_{i \in \mathcal{T}}, |\mathcal{T}|=2b+1, 56 | \mathcal{T} \subseteq Q \\ 57 | C :& \text{ choose the maximum timestamp } \\ 58 | & \text{ } t = max(t_i) + 1 \\ 59 | C \rightarrow Q :& \text{ \sf get signature}(\langle x, t, v, s_C \rangle),\\ 60 | & \text{ where } s_C = (Sign_C(\langle x, t, v \rangle), Cert_C) \\ 61 | Q_i :& \text{ verify $s_C$ with $C$'s quorum certificates} \\ 62 | Q_i :& \text{ check the TOFU policy} \\ 63 | C \leftarrow Q_i :& \text{ } S_i = \{\langle x, t, v, s_C 64 | \rangle\}_{Q_i \in Q} \\ 65 | C :& \text{ collect signatures from $|\mathcal{T}|$ members} \\ 66 | & \text{ } S = \{S_i\}_{i \in \mathcal{T}} \\ 67 | C :& \text{ choose a quorum from } Q' = U \setminus QC \\ 68 | C \rightarrow Q' :& \text{ \sf write}(\langle x, t, v, s_C, S \rangle) \\ 69 | Q'_i :& \text{ verify the signature set $S$} \\ 70 | Q'_i :& \text{ do the equivocation check} \\ 71 | Q'_i :& \text{ \sf store}(\langle x, t, v, S_C, S \rangle) 72 | \end{align*} 73 | 74 | [ {\sf read} ] 75 | \setcounter{equation}{0} 76 | \begin{align*} 77 | C :&\; \text{choose a quorum } Q \in U \setminus QC \\ 78 | C \rightarrow Q :&\; \text{\sf read}(x) \\ 79 | C \leftarrow Q_i :&\; M_i = \langle x, t, v, s_C, S \rangle \\ 80 | C :&\; \text{collect } f + 1 \text{ responses} \\ 81 | C :&\; \text{verify the signature set $S$} \\ 82 | C :&\; \text{do the equivocation check} \\ 83 | C :&\; \text{choose the latest timestamp } \\ 84 | &\; t_L = max(M_i.t) \\ 85 | C :&\; Q' = \{Q_i \subset Q | M_i.t < t_L\} \\ 86 | C \rightarrow Q' :&\; \text{\sf write back}(\langle x, t_L, v, s_C, S \rangle) 87 | \end{align*} 88 | 89 | 90 | \subsection{Join / Leave} 91 | Any node can join / leave anytime by sending its quorum certificate to 92 | nodes it trusts. The node received the request verifies the 93 | certificate and adds it to the local graph which will be returned to 94 | the caller node. To leave the network, a node broadcasts its quorum 95 | certificate to the quorum it belongs to. 96 | The node that has received the graph constructs its own local graph 97 | from the graphs, then sends the join request to nodes that have not 98 | been connected. 99 | See algorithm {\sf Join} \ref{Join}. 100 | 101 | \subsection{Register} 102 | \label{register} 103 | Each node generates its own public/private key pair, then sends the 104 | public key part to a quorum to get a quorum certificate. Each quorum 105 | member keeps all certificate issued to clients along with a partial 106 | password secret. If it finds a certificate request that has the same 107 | ID as one of the stored certificates it will sign it only if the 108 | password authentication is done. The client will get a ``proof'' of 109 | the authentication when it is finished. 110 | See algorithm {\sf Register} \ref{Register} 111 | 112 | \subsection{Authenticate} 113 | \label{authenticate} 114 | {\sf auth} protocol is done in advance of {\sf read / write / 115 | register} to get a proof and each subsequence protocol verifies it 116 | before processing the request. The follwing diagram is for a quick 117 | review of $\mathcal{TPA}$ \ref{auth}. 118 | 119 | \begin{align*} 120 | C &: \pi \leftarrow H(pw), \; 121 | g_{\pi} \leftarrow \pi^2, \; 122 | a \xleftarrow{\mathcal{R}} \mathbb{Z}_q \\ 123 | C \rightarrow Q &: X := g_{\pi}^a \bmod p \\ 124 | C \leftarrow Q_i &: Y_i := X^{y_i} \bmod p, u_i \\ 125 | C &: G_S \leftarrow \textstyle \prod_{j \in \mathcal{T}}Y_i^{\lambda_j} \bmod p, \\ 126 | & \; s_i \leftarrow H(pw, u_i), \; 127 | a' \xleftarrow{\mathcal{R}} \mathbb{Z}_q \\ 128 | C \rightarrow Q_i &: X_i := G_S^{s_ia'} \bmod p \\ 129 | Q_i &: K_i \leftarrow X_i^{b_i} \bmod p, \; 130 | b_i \xleftarrow{\mathcal{R}} \mathbb{Z}_q \\ 131 | C \leftarrow Q_i &: B_i := v_i^{b_i} \bmod p \\ 132 | C &: K_i \leftarrow B_i^{aa'} \bmod p \\ 133 | C \rightarrow Q_i &: N_i := \text{\em MAC}(K_i, X_i || B_i) \\ 134 | C \leftarrow Q_i &: Z_i := E_{K_i}(P_i, N_i) \\ 135 | C &: (P_i, N_i) = D_{K_i}(Z_i) 136 | \end{align*} 137 | At the end of the protocol, $C$ obtains the proof $P = \{P_i\}_{i \in 138 | \mathcal{T}}$. 139 | 140 | The servers keep track of the client ID, the number of attempts and 141 | the time. If the process does not go through next time the first 142 | response from each server will be slowed down to mitigate online 143 | dictionary attacks. 144 | -------------------------------------------------------------------------------- /docs/tex/ref.tex: -------------------------------------------------------------------------------- 1 | \begin{thebibliography}{99} 2 | 3 | \bibitem{bitcoin} 4 | Nakamoto, S. (2008) Bitcoin: A Peer-to-Peer Electronic Cash System 5 | 6 | \bibitem{Delhi:1} 7 | Malhi, D. and Reiter, M. (1998) Byzantine Quorum Systems 8 | 9 | \bibitem{Delhi:2} 10 | Malhi, D. and Reiter, M. (1998) Secure and Scalable Replication in Phalanx 11 | 12 | \bibitem{Gennaro} 13 | Gennaro, R., Jarecki, S., Krawczyk, H. and Rabin, T. (1999) Robust 14 | Threshold DSS Signatures 15 | 16 | \bibitem{ford} 17 | Ford, W. and Kaliski, B. (2000) Server-Assisted Generation of a 18 | Strong Secret from a Password, Proc. 9\textsuperscript{th} International 19 | Workshop on Enabling Technologies: Infrastructure for Collaborative 20 | Enterprise, IEEE, June 14-16, 2000 21 | 22 | \ifdefined\ABSTRACT 23 | \else 24 | \bibitem{shoup} 25 | Shoup, V. Practical Threshold Signatures 26 | \fi 27 | 28 | \bibitem{shamir} 29 | Shamir, A. (1979) How to Share a Secret 30 | 31 | \bibitem{garay} 32 | Garay, J. A., Gennaro, R., Jutlab, C., Rabin, T. (2000) Secure 33 | distributed storage and retrieval 34 | 35 | \bibitem{rabin} 36 | Rabin, T. (1998) A Simplified Approach to Threshold and Proactive RSA 37 | 38 | \bibitem{signal} 39 | Marlinspike, M. and Perrin, T. (2017) The Sesame Algorithm: Session Management for Asynchronous Message Encryption 40 | 41 | \ifdefined\ABSTRACT 42 | \else 43 | \bibitem{boneh} 44 | Boneh, D. The Decision Diffie-Hellman Problem 45 | \fi 46 | 47 | \end{thebibliography} 48 | -------------------------------------------------------------------------------- /docs/tex/tpa.tex: -------------------------------------------------------------------------------- 1 | \section{Security Analysis for $\mathcal{TPA}$} 2 | \label{tpa} 3 | \subsection{Correctness} 4 | Assume each $\{Y_i\}_{\{i \in \mathcal{T}\}}$ is correctly calculated, 5 | $G_S$ will be: 6 | \[ 7 | G_S = \prod_{j \in \mathcal{T}}Y_i^{\lambda_j} = g_{\pi}^{a \sum_{j 8 | \in \mathcal{T}} f(j) \lambda_j} = g_{\pi}^{aS} \pmod p 9 | \] 10 | and by raising $s_i = H(pw, u_i)$ to $G_S$, we get: 11 | \[ 12 | X_i = (G_S^{s_i})^{a'} = (g_{\pi}^{Ss_i})^{aa'} \pmod p 13 | \] 14 | which must be the same as $v_i^{aa'} \bmod p$ for each $i \in 15 | \mathcal{T}$ iff the correct password ($g_{\pi}$) is given at the 16 | step 1. With each $B_i = v_i^{b_i} \bmod p$, the client and servers 17 | share DH keys $K_i = g_{\pi}^{Ss_iaa'b_i} \bmod p$. 18 | 19 | \begin{lemma} 20 | \label{tpa1} 21 | From $\{Y_i\}_{i \in \mathcal{L}}$ where $\mathcal{L} \subset \{1..n\}$ and $|\mathcal{L}| < 22 | t$, correct $G_S$ cannot be obtained. 23 | \end{lemma} 24 | 25 | \begin{proof} 26 | $S$ cannot be reconstructed from $|\mathcal{L}|$ servers, from the SSS 27 | property, thus $g_{\pi}^{aS}$ cannot be obtained whatever $a$ is. 28 | \end{proof} 29 | 30 | \subsection{Protocol Analysis} 31 | We look into the $\mathcal{TPA}$ protocols by dividing it into two 32 | parts: the first roundtrip (step 1 and 2) is to recover the shared 33 | secret $S$, and the second roundtrip (step 3 and 4) is for key 34 | exchange. 35 | 36 | $g_{\pi} = \pi^2 \bmod p$, where $\pi$ is a secure hash value over the 37 | password, is a safe generator in $\mathbb{Z}^*_q$ with high 38 | probability. Raise a random exponent $a$ to mask the 39 | generator. Servers blindly raise its own share $y_i$ to whatever $X = 40 | g_{\pi}^a$ is. As long as $p$ is a safe prime $X^{y_i} \bmod p$ does 41 | not reveal $y_i$ whatever $X$ is. 42 | 43 | The second roundtrip can be seen as the Elgamal encryption with a 44 | generator $G_S^{s_i}$ where $s_i := H(pw, u_i)$. $u_i$ is a salt 45 | generated uniquely for each server. The encrypted data is sent in the 46 | next roundtrip. Another random exponent $a'$ is applied to make it 47 | difficult to do offline dictionary attacks over $s_i$. 48 | 49 | The last roundtrip (step 4 and 5) is to check if both parties have 50 | shared a random session key and to obtain a proof. The client receives 51 | the encrypted data iff the MAC is verified at the server, otherwise it 52 | will receive $\bot$. 53 | 54 | \subsection{Reduction to Elgamal Encryption} 55 | Since $\mathbb{Z}_p^*$ is a cyclic group, there exist $g \in 56 | \mathbb{Z}_p^*$ and $x \in \mathbb{Z}_p$ such that $g_{\pi} = g^x 57 | \pmod p$ therefore in the step 3, $X_i = G_S^{s_ia'}$ can be 58 | seen as a $DH$ term: $g^{x'}$ where $x' = xSs_iaa' \bmod q$. The 59 | session key on the server side is the same as the standard Elgamal 60 | encryption, i.e., $(g^{x'})^{b_i}$. 61 | At the step 4, the server returns $B_i = v_i^{b_i}$ where $v_i = 62 | g^{x'/aa'} \pmod p$. For any $aa' \in \mathbb{Z}_q$ there exist a 63 | multiplicative inverse in $\mathbb{Z}_q$ such that $v_i^{aa'} = g^{x'} 64 | \pmod p$. Therefore we get the standard Elgamal key such as $B_i^{aa'} 65 | = (g^{x'})^{b_i} \pmod p$ on the client side. 66 | 67 | \subsection{Resistance to offline dictionary attacks} 68 | Assume we have $|\mathcal{L}| < t$ compromised servers and try to 69 | reconstruct $g_{\pi}^S$ from $\{y_i\}_{i \in \mathcal{L}}$. 70 | From the Lemma ~\ref{tpa1} we need at least one $Y_i$, $i \notin 71 | \mathcal{L}$, and unless the server $i$ is compromised the only 72 | way to get $Y_i$ is to issue the protocol $X = g_{\pi}^a \bmod 73 | p$ in the step 1. 74 | Unless the attacker knows $\pi$ all he can do is to guess the 75 | password $\pi'$ and to keep asking the server to calculate $Y'_i = 76 | g_{\pi'}^a \bmod p$. If the guess is wrong $Y'_i$ will not 77 | calculate $g_{\pi}^S$ correctly with the compromised data $\{f(j)\}_{j 78 | \in \mathcal{L}}$. 79 | Unless the attacker can calculate $S$ from $g^S \bmod p$ with a 80 | fabricated $g$, he cannot calculate $g_{\pi'}^S \bmod p$ in 81 | order to brute force for $g_{\pi'} \in \{g^e | e = 1..p-1\}$. 82 | 83 | Next, we look into dictionary attacks for {\em share} that is 84 | revealed from compromised servers, i.e., 85 | \begin{align*} 86 | y_i &= f(i) \\ 87 | v_i &= g_{\pi}^{Ss_i} \bmod p \\ 88 | \end{align*} 89 | for $i \in \mathcal{L}$. We also get $G_S = g_{\pi}^{aS} \bmod p$ from 90 | the step 2. $B_i$ will not contribute anything to the attack as they already 91 | have $v_i$. 92 | To brute force on $v_i$ there are two possible ways: 93 | \begin{enumerate} 94 | \item $\exists S \in \mathbb{Z}_q$, check if 95 | $v_i \stackrel{\text{\tiny ?}}{=} (g_{\pi}^{s_i})^S \pmod p$ 96 | for each $g_{\pi}^{s_i}$ \\ 97 | \item $\exists g_{\pi} \in \mathbb{Z}^*_q$, check if 98 | $v_i \stackrel{\text{\tiny ?}}{=} (g_{\pi}^S)^{s_i} \pmod p$ 99 | for each $s_i$ 100 | \end{enumerate} 101 | For the former case we need $S$ and for the latter case we need 102 | $g_{\pi}^S$. From the protocol 1, we can obtain either $g^S \bmod p$ 103 | for $\forall g \in \mathbb{Z}$ or $g_{\pi}^{aS} \bmod p$. Obtaining 104 | $S$ is simply the {\sf DLog} problem. $g_{\pi}^S \bmod p$ can be 105 | obtained iff $g$ happens to be $g_{\pi}$ or a legitimate client 106 | chooses $a = 1$ which must be excluded. 107 | If $g_{\pi}^S$ is obtained, the dictionary attack is possible on 108 | $s_i$, however, it is difficult to obtain $g_{\pi}^S$ from $G_S$ which 109 | is calcualted by a legitimate client as $a$ has been randomly 110 | chosen. Attackers can choose $a = 1$ at the step 1 and they can obtain 111 | $g_{\pi'}^S$ but they need to start over the protocol to get the 112 | correct $g_{\pi}^S$. 113 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/yahoo/bftkv 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf 7 | github.com/syndtr/goleveldb v1.0.0 8 | golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 9 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 10 | ) 11 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 2 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 3 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 4 | github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws= 5 | github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 6 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 7 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 8 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 9 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 10 | github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= 11 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 12 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 13 | golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 h1:sKJQZMuxjOAR/Uo2LBfU90onWEf1dF4C+0hPJCc9Mpc= 14 | golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 15 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= 16 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 17 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= 18 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 19 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 20 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 21 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 22 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 23 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 24 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 25 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 26 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 27 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 28 | -------------------------------------------------------------------------------- /node/graph/graph_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package graph 5 | 6 | import ( 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "os" 11 | "sort" 12 | "strings" 13 | "testing" 14 | 15 | "github.com/yahoo/bftkv/crypto" 16 | "github.com/yahoo/bftkv/crypto/pgp" 17 | "github.com/yahoo/bftkv/node" 18 | ) 19 | 20 | const ( 21 | maxDistance = -1 22 | keyPath = "../../scripts/run/keys" 23 | ) 24 | 25 | type Keyring struct { 26 | pubrngs [][]byte 27 | secrng []byte 28 | } 29 | 30 | type VertexDistance struct { 31 | v *Vertex 32 | d int 33 | } 34 | 35 | func distance(v1, v2 *Vertex) int { 36 | m := make(map[uint64]bool) 37 | q := make([]VertexDistance, 0) 38 | q = append(q, VertexDistance{v1, 0}) 39 | m[v1.Instance.Id()] = true 40 | for len(q) > 0 { 41 | vd := q[0] 42 | q = q[1:] 43 | if vd.v == v2 { 44 | return vd.d 45 | } 46 | for id, e := range vd.v.Edges { 47 | if _, ok := m[id]; !ok { 48 | q = append(q, VertexDistance{e, vd.d + 1}) 49 | } 50 | } 51 | } 52 | return -1 53 | } 54 | 55 | func constructGraph() (*Graph, error) { 56 | g := New() 57 | crypt := pgp.New() 58 | 59 | files, err := ioutil.ReadDir(keyPath) 60 | if err != nil { 61 | return nil, err 62 | } 63 | for _, f := range files { 64 | if strings.HasPrefix(f.Name(), ".") { 65 | continue 66 | } 67 | path := keyPath + "/" + f.Name() 68 | readCerts(g, crypt, path+"/pubring.gpg", false) 69 | readCerts(g, crypt, path+"/secring.gpg", true) 70 | } 71 | return g, nil 72 | } 73 | 74 | func checkDistance(t *testing.T, g *Graph, nodes []node.Node, start uint64) { 75 | sv := g.Vertices[start] 76 | if sv.Instance == nil { 77 | t.Fatalf("%X: null instance!", start) 78 | } 79 | di := make([]int, len(nodes)) 80 | for i, n := range nodes { 81 | id := n.Id() 82 | v := g.Vertices[id] 83 | if v.Instance == nil { 84 | t.Fatalf("%X: null instance!", id) 85 | } 86 | di[i] = distance(sv, v) 87 | if di[i] < 0 { 88 | t.Errorf("not connected: %X, %X\n", sv.Instance.Id(), v.Instance.Id()) 89 | } 90 | } 91 | if !sort.SliceIsSorted(di, func(i, j int) bool { 92 | return di[i] < di[j] 93 | }) { 94 | t.Errorf("[%X] not sorted: %v", start, di) 95 | } 96 | } 97 | 98 | func checkUniqueness(t *testing.T, nodes []node.Node) { 99 | for i, n1 := range nodes { 100 | for _, n2 := range nodes[i+1:] { 101 | if n1 == n2 { 102 | t.Errorf("duplicated: %s[%X]", n1.Name(), n1.Id()) 103 | } 104 | } 105 | } 106 | } 107 | 108 | func TestBFS(t *testing.T) { 109 | t.Skip("skip failing test - FIXME") 110 | g, err := constructGraph() 111 | if err != nil { 112 | t.Fatal(err) 113 | } 114 | 115 | for start, v := range g.Vertices { 116 | if v.Instance == nil { 117 | continue 118 | } 119 | for n := 1; n < maxDistance; n++ { 120 | nodes := g.GetReachableNodes(start, n) 121 | // vvv print the result vvv 122 | var ids []string 123 | for _, nn := range nodes { 124 | ids = append(ids, nn.Name()) 125 | } 126 | t.Log(ids) 127 | // ^^^ 128 | checkDistance(t, g, nodes, start) 129 | checkUniqueness(t, nodes) 130 | } 131 | } 132 | } 133 | 134 | func checkEdges(v1, v2 *Vertex) bool { 135 | if _, ok := v1.Edges[v2.Instance.Id()]; !ok { 136 | return false 137 | } 138 | if _, ok := v2.Edges[v1.Instance.Id()]; !ok { 139 | return false 140 | } 141 | return true 142 | } 143 | 144 | func checkClique(g *Graph, clique []node.Node) bool { 145 | for i, c := range clique { 146 | v := g.Vertices[c.Id()] 147 | for _, cc := range clique[i+1:] { 148 | vv := g.Vertices[cc.Id()] 149 | if !checkEdges(v, vv) { 150 | return false 151 | } 152 | } 153 | } 154 | return true 155 | } 156 | 157 | func isMember(v *Vertex, clique []node.Node) bool { 158 | for _, n := range clique { 159 | if n == v.Instance { 160 | return true 161 | } 162 | } 163 | return false 164 | } 165 | 166 | func checkMaximal(g *Graph, clique []node.Node) bool { 167 | for _, v := range g.Vertices { 168 | if isMember(v, clique) { 169 | continue 170 | } 171 | if v.Instance == nil { 172 | continue 173 | } 174 | if checkClique(g, append(clique, v.Instance)) { 175 | return false 176 | } 177 | } 178 | return true 179 | } 180 | 181 | func TestClieque(t *testing.T) { 182 | t.Skip("skip failing test - FIXME)") 183 | g, err := constructGraph() 184 | if err != nil { 185 | t.Fatal(err) 186 | } 187 | 188 | for start, v := range g.Vertices { 189 | if v.Instance == nil { 190 | continue 191 | } 192 | for n := 1; n < maxDistance; n++ { 193 | cliques := g.GetCliques(start, n) 194 | for _, clique := range cliques { 195 | // vvv print the resut vvv 196 | var ids []string 197 | for _, nn := range clique.Nodes { 198 | ids = append(ids, nn.Name()) 199 | } 200 | t.Log(ids) 201 | // ^^^ 202 | if !checkClique(g, clique.Nodes) { 203 | t.Errorf("not a clique: %s", v.Instance.Name()) 204 | } 205 | if !checkMaximal(g, clique.Nodes) { 206 | t.Errorf("not maximal: %s", v.Instance.Name()) 207 | } 208 | checkUniqueness(t, clique.Nodes) 209 | } 210 | } 211 | } 212 | } 213 | 214 | func readCerts(g *Graph, crypt *crypto.Crypto, path string, sec bool) { 215 | f, err := os.Open(path) 216 | if err != nil { 217 | log.Fatal(err) 218 | return 219 | } 220 | certs, err := crypt.Certificate.ParseStream(f) 221 | if err != nil { 222 | f.Close() 223 | log.Fatal(err) 224 | } 225 | if sec { 226 | g.SetSelfNodes(certs) 227 | } else { 228 | g.AddNodes(certs) 229 | } 230 | crypt.Keyring.Register(certs, sec, true) 231 | f.Close() 232 | } 233 | 234 | func dump(g *Graph, prompt string) { 235 | fmt.Printf(">>> %s <<<\n", prompt) 236 | for _, v := range g.Vertices { 237 | n := v.Instance 238 | if n == nil { 239 | continue 240 | } 241 | instance := "" 242 | if n.Instance() == nil { 243 | instance = "nil" 244 | } 245 | fmt.Printf(" %s [%x] %s\n", n.Name(), n.Id(), instance) 246 | for _, e := range v.Edges { 247 | if e.Instance != nil { 248 | fmt.Printf(" %s\n", e.Instance.Name()) 249 | } else { 250 | fmt.Printf(" ---\n") 251 | } 252 | } 253 | } 254 | } 255 | -------------------------------------------------------------------------------- /node/node.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package node 5 | 6 | import ( 7 | "io" 8 | 9 | "github.com/yahoo/bftkv/crypto/cert" 10 | ) 11 | 12 | type Node interface { 13 | cert.CertificateInstance 14 | } 15 | 16 | type SelfNode interface { 17 | Node 18 | 19 | SerializeSelf() ([]byte, error) 20 | AddPeers(nodes []Node) []Node 21 | GetPeers() []Node 22 | RemovePeers(nodes []Node) 23 | Revoke(node Node) 24 | 25 | SerializeNodes(w io.Writer) error 26 | SerializeRevokedNodes(w io.Writer) error 27 | } 28 | -------------------------------------------------------------------------------- /packet/packet.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package packet 5 | 6 | import ( 7 | "bytes" 8 | "encoding/binary" 9 | "io" 10 | "math/big" 11 | ) 12 | 13 | const ( 14 | SignatureTypeNil = 0 15 | SignatureTypePGP = 1 16 | // add threshold signature, etc... 17 | 18 | SignatureTypePasswordAuthProof = 256 19 | ) 20 | 21 | // 22 | // signature format 23 | // 24 | 25 | type SignaturePacket struct { 26 | Type byte 27 | Version uint32 28 | Completed bool 29 | Data []byte 30 | Cert []byte // optional 31 | } 32 | 33 | // packetizer 34 | 35 | func Serialize(args ...interface{}) ([]byte, error) { 36 | var buf bytes.Buffer 37 | for i, arg := range args { 38 | var err error 39 | switch i { 40 | case 0, 1, 5: // variable and value in []byte, and auth (optional) 41 | if arg == nil { 42 | err = WriteChunk(&buf, []byte{}) // empty slice 43 | } else { 44 | err = WriteChunk(&buf, arg.([]byte)) 45 | } 46 | case 2: // timestamp 47 | err = binary.Write(&buf, binary.BigEndian, arg.(uint64)) 48 | case 3, 4: // *signature 49 | if arg == nil { 50 | err = writeSignature(&buf, nil) 51 | } else { 52 | err = writeSignature(&buf, arg.(*SignaturePacket)) 53 | } 54 | } 55 | if err != nil { 56 | return nil, err 57 | } 58 | } 59 | return buf.Bytes(), nil 60 | } 61 | 62 | func Parse(pkt []byte) (variable []byte, value []byte, t uint64, sig *SignaturePacket, ss *SignaturePacket, auth []byte, err error) { 63 | r := bytes.NewReader(pkt) 64 | // variable 65 | variable, err = ReadChunk(r) 66 | if err != nil { 67 | return 68 | } 69 | // value 70 | value, err = ReadChunk(r) 71 | if err != nil { 72 | if err == io.EOF { 73 | err = nil 74 | value = nil 75 | } 76 | return 77 | } 78 | // timestamp 79 | err = binary.Read(r, binary.BigEndian, &t) 80 | if err != nil { 81 | if err == io.EOF { 82 | err = nil 83 | t = 0 84 | } 85 | return 86 | } 87 | // signature 88 | sig, err = readSignature(r) 89 | if err != nil { 90 | if err == io.EOF { 91 | err = nil 92 | sig = nil 93 | } 94 | return 95 | } 96 | // collective signature 97 | ss, err = readSignature(r) 98 | if err != nil { 99 | if err == io.EOF { 100 | err = nil 101 | ss = nil 102 | } 103 | return 104 | } 105 | // auth 106 | auth, err = ReadChunk(r) 107 | if err != nil { 108 | if err == io.EOF { 109 | err = nil 110 | auth = nil 111 | } 112 | return 113 | } 114 | return 115 | } 116 | 117 | func WriteChunk(buf *bytes.Buffer, chunk []byte) error { 118 | l := len(chunk) 119 | if err := binary.Write(buf, binary.BigEndian, uint64(l)); err != nil { 120 | return err 121 | } 122 | _, err := buf.Write(chunk) 123 | return err 124 | } 125 | 126 | func ReadChunk(r *bytes.Reader) ([]byte, error) { 127 | var l uint64 128 | if err := binary.Read(r, binary.BigEndian, &l); err != nil { 129 | return nil, err 130 | } 131 | if l == 0 { 132 | return nil, nil 133 | } 134 | chunk := make([]byte, l) 135 | _, err := io.ReadFull(r, chunk) 136 | if err != nil { 137 | return nil, err 138 | } 139 | return chunk, nil 140 | } 141 | 142 | func seek2tbs(r *bytes.Reader) (int64, error) { 143 | // skip the variable 144 | var l int64 145 | binary.Read(r, binary.BigEndian, &l) 146 | r.Seek(l, io.SeekCurrent) 147 | // skip value 148 | binary.Read(r, binary.BigEndian, &l) 149 | r.Seek(l, io.SeekCurrent) 150 | // skip timestamp 151 | var t uint64 152 | binary.Read(r, binary.BigEndian, &t) 153 | return r.Seek(0, io.SeekCurrent) 154 | } 155 | 156 | func TBS(pkt []byte) ([]byte, error) { 157 | r := bytes.NewReader(pkt) 158 | offset, err := seek2tbs(r) 159 | if err != nil { 160 | return nil, err 161 | } 162 | chunk := make([]byte, offset) 163 | r.Seek(0, io.SeekStart) 164 | if _, err := io.ReadFull(r, chunk); err != nil { 165 | return nil, err 166 | } 167 | return chunk, nil 168 | } 169 | 170 | func TBSS(pkt []byte) ([]byte, error) { 171 | r := bytes.NewReader(pkt) 172 | if _, err := seek2tbs(r); err != nil { 173 | return nil, err 174 | } 175 | // skip signature 176 | if _, err := readSignature(r); err != nil { 177 | return nil, err 178 | } 179 | offset, err := r.Seek(0, io.SeekCurrent) 180 | if err != nil { 181 | return nil, err 182 | } 183 | chunk := make([]byte, offset) 184 | r.Seek(0, io.SeekStart) 185 | _, err = io.ReadFull(r, chunk) 186 | if err != nil { 187 | return nil, err 188 | } 189 | return chunk, nil 190 | } 191 | 192 | func writeSignature(buf *bytes.Buffer, sig *SignaturePacket) error { 193 | if sig == nil { 194 | sig = &SignaturePacket{} 195 | } 196 | if _, err := buf.Write([]byte{sig.Type}); err != nil { 197 | return err 198 | } 199 | if err := binary.Write(buf, binary.BigEndian, sig.Version); err != nil { 200 | return err 201 | } 202 | if err := binary.Write(buf, binary.BigEndian, sig.Completed); err != nil { 203 | return err 204 | } 205 | if err := WriteChunk(buf, sig.Data); err != nil { 206 | return err 207 | } 208 | if err := WriteChunk(buf, sig.Cert); err != nil { 209 | return err 210 | } 211 | return nil 212 | } 213 | 214 | func readSignature(r *bytes.Reader) (sig *SignaturePacket, err error) { 215 | sig = &SignaturePacket{} 216 | if sig.Type, err = r.ReadByte(); err != nil { 217 | return nil, err 218 | } 219 | if err = binary.Read(r, binary.BigEndian, &sig.Version); err != nil { 220 | return nil, err 221 | } 222 | if err = binary.Read(r, binary.BigEndian, &sig.Completed); err != nil { 223 | return nil, err 224 | } 225 | if sig.Data, err = ReadChunk(r); err != nil { 226 | return nil, err 227 | } 228 | if sig.Cert, err = ReadChunk(r); err != nil { 229 | return nil, err 230 | } 231 | if sig.Type == SignatureTypeNil { 232 | sig = nil 233 | } 234 | return sig, nil 235 | } 236 | 237 | func ParseSignature(pkt []byte) (*SignaturePacket, error) { 238 | r := bytes.NewReader(pkt) 239 | return readSignature(r) 240 | } 241 | 242 | func SerializeSignature(sig *SignaturePacket) ([]byte, error) { 243 | var buf bytes.Buffer 244 | if err := writeSignature(&buf, sig); err != nil { 245 | return nil, err 246 | } 247 | return buf.Bytes(), nil 248 | } 249 | 250 | func ParseAuthenticationRequest(pkt []byte) (phase int, variable []byte, adata []byte, err error) { 251 | r := bytes.NewReader(pkt) 252 | b, err := r.ReadByte() 253 | if err != nil { 254 | return 255 | } 256 | phase = int(b) 257 | if variable, err = ReadChunk(r); err != nil { 258 | return 259 | } 260 | if adata, err = ReadChunk(r); err != nil { 261 | return 262 | } 263 | return 264 | } 265 | 266 | func SerializeAuthenticationRequest(phase int, variable []byte, adata []byte) ([]byte, error) { 267 | var buf bytes.Buffer 268 | if _, err := buf.Write([]byte{byte(phase)}); err != nil { 269 | return nil, err 270 | } 271 | if err := WriteChunk(&buf, variable); err != nil { 272 | return nil, err 273 | } 274 | if err := WriteChunk(&buf, adata); err != nil { 275 | return nil, err 276 | } 277 | return buf.Bytes(), nil 278 | } 279 | 280 | func ReadBigInt(r *bytes.Reader) (*big.Int, error) { 281 | c, err := ReadChunk(r) 282 | if err != nil { 283 | return nil, err 284 | } 285 | return new(big.Int).SetBytes(c), nil 286 | } 287 | 288 | func WriteBigInt(buf *bytes.Buffer, b *big.Int) error { 289 | var c []byte 290 | if b != nil { 291 | c = b.Bytes() 292 | } 293 | return WriteChunk(buf, c) 294 | } 295 | -------------------------------------------------------------------------------- /protocol/dist_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package protocol 5 | 6 | import ( 7 | "bytes" 8 | gocrypto "crypto" 9 | "crypto/dsa" 10 | "crypto/ecdsa" 11 | "crypto/elliptic" 12 | "crypto/rand" 13 | "crypto/rsa" 14 | "crypto/x509" 15 | "encoding/pem" 16 | "io/ioutil" 17 | "math/big" 18 | "testing" 19 | 20 | "github.com/yahoo/bftkv/crypto" 21 | ) 22 | 23 | const ( 24 | rsakey = "../crypto/threshold/rsa/test.pkcs8" 25 | dsakey = "../crypto/threshold/dsa/test.pkcs8" 26 | testData = "TBS" 27 | ) 28 | 29 | func TestDist(t *testing.T) { 30 | t.Skip("skip failing test - FIXME") 31 | servers := runServers(t, "a", "rw") 32 | defer stopServers(servers) 33 | 34 | c := newClient(keyPath + "/u01") 35 | c.Joining() 36 | 37 | doTest(t, c, "testRSA", crypto.TH_RSA) 38 | doTest(t, c, "testDSA", crypto.TH_DSA) 39 | doTest(t, c, "testECDSA", crypto.TH_ECDSA) 40 | } 41 | 42 | func doTest(t *testing.T, c *Client, caname string, algo crypto.ThresholdAlgo) { 43 | var key interface{} 44 | var err error 45 | switch algo { 46 | case crypto.TH_RSA: 47 | key, err = rsa.GenerateKey(rand.Reader, 2048) 48 | case crypto.TH_DSA: 49 | var dsaPriv dsa.PrivateKey 50 | if err = dsa.GenerateParameters(&dsaPriv.Parameters, rand.Reader, dsa.L1024N160); err == nil { 51 | if err = dsa.GenerateKey(&dsaPriv, rand.Reader); err == nil { 52 | key = &dsaPriv 53 | } 54 | } 55 | case crypto.TH_ECDSA: 56 | key, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) 57 | } 58 | if err != nil { 59 | t.Fatal(err) 60 | } 61 | if err := c.Distribute(caname, key); err != nil { 62 | t.Fatal(err) 63 | } 64 | 65 | sig, err := c.DistSign(caname, []byte(testData), algo, gocrypto.SHA256) 66 | if err != nil { 67 | t.Fatal(err) 68 | } 69 | 70 | // calculate the standard sig 71 | h := gocrypto.SHA256.New() 72 | h.Write([]byte(testData)) 73 | hashed := h.Sum(nil) 74 | switch algo { 75 | case crypto.TH_RSA: 76 | want, err := rsa.SignPKCS1v15(nil, key.(*rsa.PrivateKey), gocrypto.SHA256, hashed) 77 | if err != nil { 78 | t.Fatal(err) 79 | } 80 | if !bytes.Equal(sig, want) { 81 | t.Fatal("sig mismatch") 82 | } 83 | default: 84 | // simply divide the sig into two 85 | n := len(sig) 86 | r := new(big.Int).SetBytes(sig[:n/2]) 87 | s := new(big.Int).SetBytes(sig[n/2:]) 88 | h := gocrypto.SHA256.New() 89 | h.Write([]byte(testData)) 90 | dgst := h.Sum(nil) 91 | switch algo { 92 | case crypto.TH_DSA: 93 | priv := key.(*dsa.PrivateKey) 94 | orderSize := (priv.Q.BitLen() + 7) / 8 95 | if !dsa.Verify(&priv.PublicKey, dgst[:orderSize], r, s) { 96 | t.Fatal("dsa fail") 97 | } 98 | case crypto.TH_ECDSA: 99 | priv := key.(*ecdsa.PrivateKey) 100 | if !ecdsa.Verify(&priv.PublicKey, dgst, r, s) { 101 | t.Fatal("ecdsa fail") 102 | } 103 | } 104 | } 105 | } 106 | 107 | func readPKCS8(path string) (algo crypto.ThresholdAlgo, key interface{}, err error) { 108 | data, err := ioutil.ReadFile(path) 109 | if err != nil { 110 | return 111 | } 112 | block, _ := pem.Decode(data) 113 | var der []byte 114 | if block != nil { 115 | if block.Type != "PRIVATE KEY" { 116 | return 117 | } 118 | der = block.Bytes 119 | } else { // not PEM, assume the data is DER 120 | der = data 121 | } 122 | key, err = x509.ParsePKCS8PrivateKey(der) 123 | if err != nil { 124 | return 125 | } 126 | switch key.(type) { 127 | case *rsa.PrivateKey: 128 | algo = crypto.TH_RSA 129 | case *dsa.PrivateKey: 130 | algo = crypto.TH_DSA 131 | case *ecdsa.PrivateKey: 132 | algo = crypto.TH_ECDSA 133 | default: 134 | return 135 | } 136 | return 137 | } 138 | -------------------------------------------------------------------------------- /protocol/mal_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package protocol 5 | 6 | import ( 7 | "fmt" 8 | "io/ioutil" 9 | "strings" 10 | "testing" 11 | "time" 12 | 13 | "github.com/yahoo/bftkv/crypto/pgp" 14 | "github.com/yahoo/bftkv/node" 15 | "github.com/yahoo/bftkv/node/graph" 16 | "github.com/yahoo/bftkv/quorum/wotqs" 17 | "github.com/yahoo/bftkv/transport" 18 | transport_http "github.com/yahoo/bftkv/transport/http" 19 | ) 20 | 21 | var _ = fmt.Println 22 | 23 | func TestMaliciousCollusion(t *testing.T) { 24 | t.Skip("skip failing test - FIXME") 25 | 26 | // malicious client writes and to colluding servers 27 | // read is attempted for key x => insufficient responses expected 28 | sMal = []string{"http://localhost:5705", "http://localhost:5708", "http://localhost:5709", "http://localhost:5706", "http://localhost:5707", "http://localhost:5704", "http://localhost:5703"} 29 | rwMal = []string{"http://localhost:5602", "http://localhost:5603", "http://localhost:5604", "http://localhost:5605", "http://localhost:5606", "http://localhost:5607", "http://localhost:5608"} 30 | mal = append(sMal, rwMal...) // used in malserver_test.go 31 | files, err := ioutil.ReadDir(keyPath) 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | 36 | // create test servers 37 | var servers []*MalServer 38 | for _, f := range files { 39 | if strings.HasPrefix(f.Name(), "a") || strings.HasPrefix(f.Name(), "rw") { 40 | s := newMalServer(keyPath+"/"+f.Name(), dbPrefix+f.Name()) 41 | if err := s.Start(); err != nil { 42 | t.Fatal(err) 43 | } 44 | servers = append(servers, s) 45 | } 46 | } 47 | 48 | defer func() { 49 | for _, s := range servers { 50 | s.Stop() 51 | } 52 | }() 53 | 54 | // create a client 55 | c := newClient(keyPath + "/" + clientKey) 56 | c.Joining() 57 | defer c.Leaving() 58 | 59 | key := []byte(testKey) 60 | value := []byte("honesttestvalue") 61 | if err := c.WriteMal(key, value); err != nil { 62 | t.Fatal(err) 63 | } 64 | 65 | res, err := c.Read(key, nil) 66 | time.Sleep(time.Second * 3) // sleep to leave time for revoke check 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | fmt.Printf("Key: %s, Value: %s\n", string(key), string(res)) 71 | } 72 | 73 | // to test tofu --- run 'go test -run=TOFU -v' 74 | 75 | func TestTOFU(t *testing.T) { 76 | t.Skip("skip failing test - FIXME") 77 | 78 | // tests must run in following order 79 | 80 | //unique time stamp is key 81 | uts := time.Now().String() 82 | 83 | // exp: successful --- the client will write for the first time 84 | // exp: successful --- the client will wrtie 85 | servers := runServers(t, "a", "rw") 86 | c1 := newClient(keyPath + "/u01") 87 | c1.Joining() 88 | defer c1.Leaving() 89 | c1.checkTofu(uts, t, "original write successful") 90 | c1.checkTofu(uts, t, "self overwrite successful") 91 | stopServers(servers) 92 | c1.Leaving() 93 | 94 | // exp: permission denied --- diff user id 95 | servers = runServers(t, "a", "rw") 96 | c2 := newClient(keyPath + "/u02") 97 | c2.Joining() 98 | defer c2.Leaving() 99 | c2.checkTofu(uts, t, "trusted entity overwrite successful (same UId)") 100 | stopServers(servers) 101 | c2.Leaving() 102 | 103 | // exp: permission denied --- diff servers will sign for c01 than u1 104 | servers = runServers(t, "a", "rw") 105 | c3 := newClient(keyPath + "/u04") 106 | c3.Joining() 107 | defer c3.Leaving() 108 | c3.checkTofu(uts, t, "untrusted entity overwrite successful - expected error") 109 | stopServers(servers) 110 | c3.Leaving() 111 | } 112 | 113 | func (c *Client) checkTofu(key string, t *testing.T, exp string) { 114 | if err := c.Write([]byte(key), []byte(testValue), nil); err != nil { 115 | t.Log(err) 116 | } else { 117 | t.Log(exp) 118 | } 119 | } 120 | 121 | func newMalClient(path string) *Client { 122 | crypt := pgp.New() 123 | g := graph.New() 124 | readCerts(g, crypt, path+"/pubring.gpg", false) 125 | readCerts(g, crypt, path+"/secring.gpg", true) 126 | qs := wotqs.New(g) 127 | var tr transport.Transport 128 | tr = transport_http.New(crypt) 129 | return NewClient(node.SelfNode(g), qs, tr, crypt) 130 | } 131 | 132 | func newMalServer(path string, dbPath string) *MalServer { 133 | crypt := pgp.New() 134 | g := graph.New() 135 | readCerts(g, crypt, path+"/pubring.gpg", false) 136 | readCerts(g, crypt, path+"/secring.gpg", true) 137 | qs := wotqs.New(g) 138 | var tr transport.Transport 139 | tr = transport_http.MalNew(crypt) 140 | storage := MalStorageNew(dbPath) 141 | return NewMalServer(node.SelfNode(g), qs, tr, crypt, storage) 142 | } 143 | -------------------------------------------------------------------------------- /protocol/malclient_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package protocol 5 | 6 | import ( 7 | "encoding/binary" 8 | "fmt" 9 | "math" 10 | "strings" 11 | 12 | "github.com/yahoo/bftkv" 13 | "github.com/yahoo/bftkv/node" 14 | "github.com/yahoo/bftkv/node/graph" 15 | "github.com/yahoo/bftkv/packet" 16 | "github.com/yahoo/bftkv/quorum" 17 | "github.com/yahoo/bftkv/transport" 18 | ) 19 | 20 | var sMal []string 21 | var rwMal []string 22 | var mal []string 23 | 24 | type nodeGroup struct { 25 | honest1 []node.Node 26 | honest2 []node.Node 27 | mal_nodes []node.Node 28 | } 29 | 30 | func getCliques(g *graph.Graph) ([]node.Node, []node.Node) { 31 | cliques := g.GetCliques(g.GetSelfId(), -1) 32 | // choose cliques that have only sufficient number of nodes 33 | var signers []node.Node 34 | var rw []node.Node 35 | for _, clique := range cliques { 36 | n := len(clique.Nodes) 37 | if n == 0 { 38 | continue 39 | } 40 | if len(clique.Nodes) > 1 { 41 | for _, i := range clique.Nodes { 42 | signers = append(signers, i) 43 | } 44 | } else { 45 | rw = append(rw, clique.Nodes[0]) 46 | } 47 | } 48 | return signers, rw 49 | } 50 | 51 | func (c *Client) getNodeGroups() (nodeGroup, nodeGroup) { 52 | // the following block splits up each clique into two different groups 53 | // and isolates the malicious nodes further 54 | s, rw := getCliques(c.self.(*graph.Graph)) 55 | return getGroup(s, sMal), getGroup(rw, rwMal) 56 | } 57 | 58 | func getGroup(nodes []node.Node, mal_array []string) nodeGroup { 59 | var group nodeGroup 60 | ctr := true 61 | for _, client := range nodes { 62 | flag := true 63 | for _, malicious := range mal_array { 64 | if strings.Compare(malicious, client.Address()) == 0 { 65 | group.mal_nodes = append(group.mal_nodes, client) 66 | flag = false 67 | break 68 | } 69 | } 70 | if flag { 71 | if ctr { 72 | group.honest1 = append(group.honest1, client) 73 | ctr = false 74 | } else { 75 | group.honest2 = append(group.honest2, client) 76 | ctr = true 77 | } 78 | } 79 | } 80 | return group 81 | } 82 | 83 | func (c *Client) WriteMal(variable []byte, value []byte) error { 84 | // writes two different values for the same variable at the same time to colluding servers 85 | // note colluding servers (mal) defined in mal_test.go 86 | quorum := c.qs.ChooseQuorum(quorum.AUTH) 87 | maxt := uint64(0) 88 | s, rw := c.getNodeGroups() 89 | group_a := append(s.honest1, s.mal_nodes...) 90 | group_b := append(s.honest2, s.mal_nodes...) 91 | group_c := append(rw.honest1, rw.mal_nodes...) 92 | group_d := append(rw.honest2, rw.mal_nodes...) 93 | 94 | var actives, failure []node.Node 95 | c.tr.Multicast(transport.Time, quorum.Nodes(), variable, func(res *transport.MulticastResponse) bool { 96 | if res.Err == nil && len(res.Data) > 0 && len(res.Data) <= 8 { 97 | t := binary.BigEndian.Uint64(res.Data) 98 | if t > maxt { 99 | maxt = t 100 | } 101 | actives = append(actives, res.Peer) 102 | return quorum.IsThreshold(actives) 103 | } else { 104 | failure = append(failure, res.Peer) 105 | return quorum.Reject(failure) 106 | } 107 | }) 108 | if !quorum.IsThreshold(actives) { 109 | return bftkv.ErrInsufficientNumberOfQuorum 110 | } 111 | 112 | if maxt == math.MaxUint64 { 113 | return bftkv.ErrInvalidTimestamp 114 | } 115 | maxt++ 116 | 117 | err1 := c.signAndWrite(group_a, group_c, value, variable, maxt, quorum) 118 | if err1 != nil { 119 | return err1 120 | } 121 | err2 := c.signAndWrite(group_b, group_d, []byte("second value"), variable, maxt, quorum) 122 | if err2 != nil { 123 | return err2 124 | } 125 | 126 | return nil 127 | } 128 | 129 | func (c *Client) signAndWrite(s_group []node.Node, rw_group []node.Node, value []byte, variable []byte, maxt uint64, quorum quorum.Quorum) error { 130 | s := make([]node.Node, len(s_group)) 131 | copy(s, s_group) 132 | 133 | rw := make([]node.Node, len(rw_group)) 134 | copy(rw, rw_group) 135 | 136 | fmt.Println("Client: WriteMal - writing: ", string(value)) 137 | 138 | // self-sign over 139 | tbs, err := packet.Serialize(variable, value, maxt) 140 | if err != nil { 141 | return err 142 | } 143 | sig, err := c.crypt.Signature.Sign(tbs) 144 | if err != nil { 145 | return err 146 | } 147 | tbss, err := packet.Serialize(variable, value, maxt, sig) 148 | if err != nil { 149 | return err 150 | } 151 | ss, err := c.crypt.CollectiveSignature.Sign(tbss) // the first one is self-signed 152 | if err != nil { 153 | return err 154 | } 155 | pkt, err := packet.Serialize(variable, value, maxt, sig, ss) 156 | if err != nil { 157 | return err 158 | } 159 | 160 | var failure []node.Node 161 | var errs []error 162 | c.tr.Multicast(transport.Sign, s, pkt, func(res *transport.MulticastResponse) bool { 163 | if res.Err == nil { 164 | s, err := packet.ParseSignature(res.Data) 165 | if err == nil { 166 | return c.crypt.CollectiveSignature.Combine(ss, s, quorum) // whatever the response is 167 | } 168 | errs = append(errs, err) 169 | } else { 170 | errs = append(errs, res.Err) 171 | } 172 | failure = append(failure, res.Peer) 173 | return quorum.Reject(failure) 174 | }) 175 | if err := c.crypt.CollectiveSignature.Verify(tbss, ss, quorum); err != nil { 176 | return majorityError(errs, err) 177 | } 178 | 179 | pkt, err = packet.Serialize(variable, value, maxt, sig, ss) 180 | if err != nil { 181 | return err 182 | } 183 | 184 | c.tr.Multicast(transport.Write, rw, pkt, func(res *transport.MulticastResponse) bool { 185 | return false 186 | }) 187 | 188 | return nil 189 | } 190 | -------------------------------------------------------------------------------- /protocol/malserver_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package protocol 5 | 6 | import ( 7 | "io" 8 | "log" 9 | "net/url" 10 | "strings" 11 | 12 | "github.com/yahoo/bftkv" 13 | "github.com/yahoo/bftkv/crypto" 14 | "github.com/yahoo/bftkv/node" 15 | "github.com/yahoo/bftkv/packet" 16 | "github.com/yahoo/bftkv/quorum" 17 | "github.com/yahoo/bftkv/storage" 18 | "github.com/yahoo/bftkv/transport" 19 | ) 20 | 21 | // need to duplicate malserver instance of revokesigners 22 | 23 | type MalServer struct { 24 | Server 25 | malst MalStorage 26 | } 27 | 28 | func NewMalServer(self node.SelfNode, qs quorum.QuorumSystem, tr transport.Transport, crypt *crypto.Crypto, st MalStorage) *MalServer { 29 | return &MalServer{ 30 | Server: Server{ 31 | Protocol: Protocol{ 32 | self: self, 33 | qs: qs, 34 | tr: tr, 35 | crypt: crypt, 36 | }, 37 | st: st, 38 | }, 39 | malst: st, 40 | } 41 | } 42 | 43 | func (s *MalServer) Start() error { 44 | // start the server first 45 | if addr := s.self.Address(); addr != "" { 46 | if u, err := url.Parse(addr); err == nil { 47 | addr = ":" + u.Port() 48 | } 49 | s.tr.Start(s, addr) 50 | log.Printf("Server @ %s running\n", addr) 51 | } 52 | return nil 53 | } 54 | 55 | func (s *MalServer) signResult(req []byte, peer node.Node) ([]byte, error) { 56 | for _, url := range mal { 57 | if strings.Compare(s.self.Address(), url) == 0 { 58 | return s.malSign(req, peer) 59 | } 60 | } 61 | return s.sign(req, peer) 62 | } 63 | 64 | func (s *MalServer) malSign(req []byte, peer node.Node) ([]byte, error) { 65 | _, _, _, _, _, _, err := packet.Parse(req) 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | tbss, err := packet.TBSS(req) 71 | if err != nil { 72 | return nil, err 73 | } 74 | /* 75 | if err := s.crypt.Signature.Verify(tv, sig); err != nil { 76 | return nil, err 77 | } 78 | */ 79 | ss, err := s.crypt.CollectiveSignature.Sign(tbss) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | pkt, err := packet.SerializeSignature(ss) 85 | if err != nil { 86 | return nil, err 87 | } 88 | return pkt, nil 89 | } 90 | 91 | func (s *MalServer) writeResult(req []byte, peer node.Node) ([]byte, error) { 92 | for _, url := range mal { 93 | if strings.Compare(s.self.Address(), url) == 0 { 94 | return s.malWrite(req, peer) 95 | } 96 | } 97 | return s.write(req, peer) 98 | } 99 | 100 | func (s *MalServer) malWrite(req []byte, peer node.Node) ([]byte, error) { 101 | variable, _, t, _, _, _, err := packet.Parse(req) 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | _, err1 := packet.TBS(req) 107 | if err1 != nil { 108 | return nil, err1 109 | } 110 | 111 | if err := s.malst.MalWrite(variable, t, req); err != nil { 112 | return nil, err 113 | } 114 | return nil, nil 115 | } 116 | 117 | func (s *MalServer) readResult(req []byte, peer node.Node) ([]byte, error) { 118 | for _, url := range mal { 119 | if strings.Compare(s.self.Address(), url) == 0 { 120 | return s.malRead(req, peer) 121 | } 122 | } 123 | return s.read(req, peer) 124 | } 125 | 126 | func (s *MalServer) malRead(req []byte, peer node.Node) ([]byte, error) { 127 | variable := req 128 | tvs, err := s.malst.MalRead(variable, 0) 129 | if err != nil && err != storage.ErrNotFound { 130 | // honest read intended 131 | return s.read(req, peer) 132 | } 133 | 134 | if tvs != nil { 135 | _, _, _, _, ss, _, err := packet.Parse(tvs) 136 | if err != nil { 137 | return nil, err 138 | } 139 | if ss == nil || !ss.Completed { 140 | return nil, nil 141 | } 142 | } 143 | return tvs, nil 144 | } 145 | 146 | func (s *MalServer) Handler(cmd int, r io.Reader, w io.Writer) error { 147 | req, nonce, peer, err := s.crypt.Message.Decrypt(r) 148 | if err != nil { 149 | if cmd != transport.Join || req == nil { 150 | log.Printf("server [%s]: transport security error: %s\n", s.self.Name(), err) 151 | return err 152 | } 153 | } 154 | 155 | var res []byte 156 | switch cmd { 157 | case transport.Join: 158 | res, err = s.join(req, peer) 159 | case transport.Leave: 160 | res, err = s.leave(req, peer) 161 | case transport.Time: 162 | res, err = s.time(req, peer) 163 | case transport.Read: 164 | res, err = s.readResult(req, peer) 165 | case transport.Write: 166 | res, err = s.writeResult(req, peer) 167 | case transport.Sign: 168 | res, err = s.signResult(req, peer) 169 | case transport.Revoke: 170 | res, err = s.revoke(req, peer) 171 | default: 172 | err = bftkv.ErrUnknownCommand 173 | } 174 | if err != nil { 175 | log.Printf("server[%s]: error: %s\n", s.self.Name(), err) 176 | return err 177 | // should not call Close() 178 | } 179 | var peers []node.Node 180 | if peer == nil { 181 | peers = s.crypt.Keyring.GetKeyring() 182 | if peers == nil { 183 | return crypto.ErrCertificateNotFound 184 | } 185 | } else { 186 | peers = []node.Node{peer} 187 | } 188 | cipher, err := s.crypt.Message.Encrypt(peers, res, nonce) 189 | if err != nil { 190 | return err 191 | } 192 | _, err = w.Write(cipher) 193 | return err 194 | } 195 | -------------------------------------------------------------------------------- /protocol/malstorage_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package protocol 5 | 6 | import ( 7 | "encoding/hex" 8 | "github.com/yahoo/bftkv/storage" 9 | storage_plain "github.com/yahoo/bftkv/storage/plain" 10 | "io/ioutil" 11 | _ "math/rand" 12 | "os" 13 | "strconv" 14 | "strings" 15 | "sync" 16 | "time" 17 | ) 18 | 19 | type MalStorage interface { 20 | storage.Storage 21 | MalRead(variable []byte, t uint64) (value []byte, err error) 22 | MalWrite(variable []byte, t uint64, value []byte) error 23 | } 24 | 25 | type malPlain struct { 26 | path string 27 | mutex sync.Mutex 28 | plain storage.Storage 29 | } 30 | 31 | func MalStorageNew(path string) MalStorage { 32 | return &malPlain{ 33 | path: path, 34 | plain: storage_plain.New(path), 35 | } 36 | } 37 | 38 | func (p *malPlain) getMalMaxT(fname string) (uint64, error) { 39 | p.mutex.Lock() 40 | stats, err := ioutil.ReadDir(p.path + "/mal") 41 | p.mutex.Unlock() 42 | if err != nil { 43 | return 0, err 44 | } 45 | fname += "." 46 | maxn := uint64(0) 47 | for _, st := range stats { 48 | if strings.HasPrefix(st.Name(), fname) { 49 | n, err := strconv.ParseUint(st.Name()[len(fname):], 10, 64) 50 | if err == nil && n > maxn { 51 | maxn = n 52 | } 53 | } 54 | } 55 | return maxn, nil 56 | } 57 | 58 | func (p *malPlain) constructMalPath(variable []byte, t uint64) (string, error) { 59 | fname := hex.EncodeToString(variable) 60 | if t == 0 { 61 | // read the latest one 62 | maxt, err := p.getMalMaxT(fname) 63 | if err != nil { 64 | return "", err 65 | } 66 | t = maxt 67 | } 68 | fname += "." + strconv.FormatUint(t, 10) 69 | return p.path + "/mal/" + fname, nil 70 | } 71 | 72 | func (p *malPlain) MalRead(variable []byte, t uint64) (value []byte, err error) { 73 | path, err := p.constructMalPath(variable, t) 74 | if err != nil { 75 | return nil, err 76 | } 77 | responses, err := ioutil.ReadDir(path) 78 | if err != nil { 79 | return value, storage.ErrNotFound 80 | } 81 | //rand.Seed(time.Now().UTC().UnixNano()) 82 | //index := rand.Intn(len(responses)) //cannot randomize index for now 83 | index := 0 84 | value, err = ioutil.ReadFile(path + "/" + responses[index].Name()) 85 | if err != nil && os.IsNotExist(err) { 86 | err = storage.ErrNotFound 87 | value = nil 88 | } 89 | return value, err 90 | } 91 | 92 | func (p *malPlain) MalWrite(variable []byte, t uint64, value []byte) error { 93 | path, err := p.constructMalPath(variable, t) 94 | if err != nil { 95 | return err 96 | } 97 | if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { 98 | if err := os.MkdirAll(path, 0777); err != nil { 99 | return err 100 | } 101 | } 102 | return ioutil.WriteFile(path+"/"+strconv.FormatInt(unixMilli(time.Now()), 10), value, 0644) 103 | } 104 | 105 | func unixMilli(t time.Time) int64 { 106 | return t.Round(time.Millisecond).UnixNano() / (int64(time.Millisecond) / int64(time.Nanosecond)) 107 | } 108 | 109 | func (p *malPlain) Read(variable []byte, t uint64) (value []byte, err error) { 110 | return p.plain.Read(variable, t) 111 | } 112 | 113 | func (p *malPlain) Write(variable []byte, t uint64, value []byte) error { 114 | return p.plain.Write(variable, t, value) 115 | } 116 | -------------------------------------------------------------------------------- /protocol/protocol.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package protocol 5 | 6 | import ( 7 | "github.com/yahoo/bftkv/crypto" 8 | "github.com/yahoo/bftkv/node" 9 | "github.com/yahoo/bftkv/quorum" 10 | "github.com/yahoo/bftkv/transport" 11 | ) 12 | 13 | type Protocol struct { 14 | self node.SelfNode 15 | qs quorum.QuorumSystem 16 | tr transport.Transport 17 | crypt *crypto.Crypto 18 | threshold crypto.Threshold 19 | } 20 | 21 | func (p *Protocol) Joining() error { 22 | m := make(map[uint64]bool) 23 | pkt, err := p.self.SerializeSelf() 24 | if err != nil { 25 | return err 26 | } 27 | for { 28 | peers := make([]node.Node, 0) 29 | for _, n := range p.self.GetPeers() { 30 | if _, ok := m[n.Id()]; !ok { 31 | peers = append(peers, n) 32 | m[n.Id()] = true 33 | } 34 | } 35 | if len(peers) == 0 { 36 | break 37 | } 38 | p.tr.Multicast(transport.Join, peers, pkt, func(res *transport.MulticastResponse) bool { 39 | if res.Data != nil { // ignore res.Err as it might be bacause the peer certificate hasn't been registered yet 40 | nodes, err := p.crypt.Certificate.Parse(res.Data) 41 | if err == nil { 42 | nodes = p.self.AddPeers(nodes) 43 | if p.crypt.Keyring.Register(nodes, false, false) != nil { 44 | p.self.RemovePeers(nodes) 45 | } 46 | } 47 | } 48 | return false // go through all nodes 49 | }) 50 | } 51 | return nil 52 | } 53 | 54 | func (p *Protocol) Leaving() error { 55 | pkt, err := p.self.SerializeSelf() 56 | if err == nil { 57 | p.tr.Multicast(transport.Leave, p.self.GetPeers(), pkt, nil) 58 | } 59 | return err 60 | } 61 | -------------------------------------------------------------------------------- /protocol/revoke_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package protocol 5 | 6 | import ( 7 | "fmt" 8 | "testing" 9 | 10 | "github.com/yahoo/bftkv/crypto/pgp" 11 | "github.com/yahoo/bftkv/node" 12 | "github.com/yahoo/bftkv/node/graph" 13 | "github.com/yahoo/bftkv/packet" 14 | "github.com/yahoo/bftkv/quorum/wotqs" 15 | ) 16 | 17 | func getClient(client_loc string) *Client { 18 | path := keyPath + "/" + client_loc 19 | crypt := pgp.New() 20 | g := graph.New() 21 | pubring := path + "/pubring.gpg" 22 | secring := path + "/secring.gpg" 23 | readCerts(g, crypt, pubring, false) 24 | readCerts(g, crypt, secring, true) 25 | 26 | qs := wotqs.New(g) 27 | return NewClient(node.SelfNode(g), qs, nil, crypt) 28 | } 29 | 30 | type info struct { 31 | m map[uint64]map[string][]*signedValue 32 | signers []string 33 | value string 34 | } 35 | 36 | func TestRevokeNone(t *testing.T) { 37 | t.Skip("skip failing test - FIXME") 38 | 39 | // all clients/servers are honest -> none should be revoked 40 | m := make(map[uint64]map[string][]*signedValue) 41 | client := "a01" 42 | c := getClient(client) 43 | clients := []string{"a02", "a03", "a04", "a05"} 44 | c.forgeMap(info{m, clients, "test1"}) 45 | c.revokeTest(m) 46 | } 47 | 48 | func TestRevokeMaliciousClientColludingServer(t *testing.T) { 49 | t.Skip("skip failing test - FIXME") 50 | 51 | // Malicious clients a01, a02, a03 write two values at time 1 to colluding servers 52 | // a03, a04, a05 which sign both values -> writers/signers a01-a05 should be revoked 53 | m := make(map[uint64]map[string][]*signedValue) 54 | clients := []string{"a01", "a02", "a03", "a04", "a05", "a06", "a07"} 55 | var c *Client 56 | for i, client := range clients { 57 | c = getClient(client) 58 | signers := []string{"a02", "a03", "a04", "a05", "a06", "a09", "a10"} 59 | c.forgeMap(info{m, signers, "test1"}) 60 | if i < 3 { 61 | c.forgeMap(info{m, signers[1:4], "test2"}) 62 | } 63 | } 64 | c.revokeTest(m) 65 | } 66 | 67 | func (c *Client) forgeMap(i info) { 68 | // adding values at time 1 69 | _, ok := i.m[1] 70 | if !ok { 71 | i.m[1] = make(map[string][]*signedValue) 72 | } 73 | sig := c.getWriterSig() 74 | ss := c.getSignerSigs(i.signers) 75 | i.m[1][i.value] = append(i.m[1][i.value], &signedValue{c.self, sig, ss, nil}) 76 | } 77 | 78 | func (c *Client) getWriterSig() *packet.SignaturePacket { 79 | // returns client's signatures 80 | req := []byte("test request writer") 81 | ss, err := c.crypt.CollectiveSignature.Sign(req) 82 | if err != nil { 83 | fmt.Println(err.Error()) 84 | return nil 85 | } 86 | return ss 87 | } 88 | 89 | func (c *Client) getSignerSigs(signers []string) *packet.SignaturePacket { 90 | // returns signatures of all nodes in clients 91 | req := []byte("test request signer") 92 | 93 | var ss packet.SignaturePacket 94 | for _, s := range signers { 95 | signer := getClient(s) 96 | s, err := signer.crypt.CollectiveSignature.Sign(req) 97 | if err != nil { 98 | fmt.Println(err.Error()) 99 | return nil 100 | } 101 | ss.Data = append(ss.Data, s.Data...) 102 | } 103 | return &ss 104 | } 105 | 106 | func (c *Client) revokeTest(m map[uint64]map[string][]*signedValue) { 107 | // if two different values at the same timestamp have some signers 108 | // in common ->> those signers should be revokes 109 | revoked := make([]uint64, 0) 110 | for t, vl := range m { 111 | if t == 0 { 112 | // temp solution 113 | continue 114 | } 115 | dup_map := make(map[string][]int) 116 | round := 0 117 | same_writer := make(map[uint64][]int) 118 | for _, l := range vl { 119 | for _, signedVal := range l { 120 | writer := c.crypt.CollectiveSignature.Signers(signedVal.sig)[0] 121 | if v, exists := same_writer[writer.Id()]; exists { 122 | writer_revoked: 123 | for _, r := range v { 124 | if r != round { 125 | for _, j := range revoked { 126 | if j == writer.Id() { 127 | break writer_revoked 128 | } 129 | } 130 | revoked = c.doRevoke(writer, revoked, "writer") 131 | } 132 | } 133 | } else { 134 | same_writer[writer.Id()] = append(same_writer[writer.Id()], round) 135 | } 136 | nodes := c.crypt.CollectiveSignature.Signers(signedVal.ss) 137 | prev_revoked: 138 | for _, i := range nodes { 139 | address := i.Address() 140 | if v, exists := dup_map[address]; exists { 141 | for _, iter_num := range v { 142 | if iter_num != round { 143 | // signer signed another value too 144 | for _, j := range revoked { 145 | if j == i.Id() { 146 | continue prev_revoked 147 | } 148 | } 149 | revoked = c.doRevoke(i, revoked, "signer") 150 | } 151 | } 152 | } else { 153 | dup_map[address] = append(dup_map[address], round) 154 | } 155 | } 156 | } 157 | round += 1 158 | } 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /protocol/roaming_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package protocol 5 | 6 | import ( 7 | "testing" 8 | ) 9 | 10 | var ( 11 | variable = []byte("authtestkey") 12 | cred = []byte("1234") 13 | ) 14 | 15 | func TestAuth(t *testing.T) { 16 | t.Skip("skip failing test - FIXME") 17 | 18 | servers := runServers(t, "a", "b", "rw") 19 | defer stopServers(servers) 20 | 21 | // create a client 22 | c := newClient(keyPath + "/u01") 23 | c.Joining() 24 | 25 | _, _, err := c.Authenticate(variable, cred) 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /protocol/rw_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package protocol 5 | 6 | import ( 7 | "bytes" 8 | "flag" 9 | "fmt" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | var rounds int 15 | 16 | func init() { 17 | // optional flag 18 | flag.IntVar(&rounds, "r", 10, "number of reads/writes") 19 | } 20 | 21 | func TestConflict(t *testing.T) { 22 | t.Skip("skip failing test - FIXME") 23 | 24 | // make writes for new value from several different clients concurrently 25 | // expecting invalid signature request twice and ultimately successful read 26 | servers := runServers(t, "a", "rw") 27 | defer stopServers(servers) 28 | var clients []*Client 29 | startManyClients([]string{"u01", "u02", "u03"}, &clients) 30 | 31 | ch := make(chan int, len(clients)) 32 | k := time.Now().String() 33 | var expecting string 34 | for _, client := range clients { 35 | go func(client *Client) { 36 | err := client.Write([]byte(k), []byte(client.self.Name()), nil) 37 | if err != nil { 38 | t.Log(err) 39 | } else { 40 | expecting = client.self.Name() 41 | t.Log("Winner: ", client.self.Name()) 42 | } 43 | ch <- 1 44 | }(client) 45 | } 46 | for i := 0; i < len(clients); i++ { 47 | <-ch 48 | } 49 | 50 | c4 := newClient(keyPath + "/u04") 51 | c4.Joining() 52 | defer c4.Leaving() 53 | res, err := c4.Read([]byte(k), nil) 54 | if err != nil { 55 | t.Log(err) 56 | } 57 | 58 | if !bytes.Equal(res, []byte(expecting)) { 59 | error := fmt.Sprintf("Expected: %s, Received: %s", expecting, string(res)) 60 | t.Log(error) 61 | } 62 | c4.Leaving() 63 | } 64 | 65 | func TestManyWrites(t *testing.T) { 66 | t.Skip("skip failing test - FIXME") 67 | 68 | // prints average time of writes 69 | servers := runServers(t, "a", "rw") 70 | defer stopServers(servers) 71 | c := newClient(keyPath + "/u01") 72 | c.Joining() 73 | defer c.Leaving() 74 | start := time.Now() 75 | for n := 0; n < rounds; n++ { 76 | err := c.Write([]byte("abc"), []byte("def"), nil) 77 | if err != nil { 78 | t.Log(err) 79 | } 80 | } 81 | duration := time.Since(start) 82 | fmt.Printf("Avg write: %.6f seconds\n", duration.Seconds()/float64(rounds)) 83 | c.Leaving() 84 | } 85 | 86 | func TestManyReads(t *testing.T) { 87 | t.Skip("skip failing test - FIXME") 88 | 89 | // prints average time of reads 90 | servers := runServers(t, "a", "rw") 91 | defer stopServers(servers) 92 | c := newClient(keyPath + "/u01") 93 | c.Joining() 94 | defer c.Leaving() 95 | err := c.Write([]byte("ghi"), []byte("jkl"), nil) 96 | if err != nil { 97 | t.Log(err) 98 | } 99 | start := time.Now() 100 | for n := 0; n < rounds; n++ { 101 | _, err := c.Read([]byte("ghi"), nil) 102 | if err != nil { 103 | t.Log(err) 104 | } 105 | } 106 | duration := time.Since(start) 107 | fmt.Printf("Avg read: %.6f seconds\n", duration.Seconds()/float64(rounds)) 108 | c.Leaving() 109 | } 110 | 111 | func TestManyClientsConcurrentReads(t *testing.T) { 112 | t.Skip("skip failing test - FIXME") 113 | 114 | // concurrent reads by different clients to the same quorum for the same 115 | servers := runServers(t, "a", "rw") 116 | defer stopServers(servers) 117 | c1 := newClient(keyPath + "/u01") 118 | c1.Joining() 119 | defer c1.Leaving() 120 | err := c1.Write([]byte("mno"), []byte("pqr"), nil) 121 | if err != nil { 122 | t.Log(err) 123 | } 124 | clients := []*Client{c1} 125 | startManyClients([]string{"u01"}, &clients) 126 | num_clients := len(clients) 127 | ch := make(chan int, num_clients) 128 | for _, client := range clients { 129 | go func(c *Client) { 130 | _, err := c.Read([]byte("mno"), nil) 131 | if err != nil { 132 | t.Log(err) 133 | } 134 | ch <- 1 135 | }(client) 136 | } 137 | for i := 0; i < num_clients; i++ { 138 | <-ch 139 | } 140 | c1.Leaving() 141 | } 142 | 143 | func startManyClients(c_paths []string, clients *[]*Client) { 144 | for _, c_path := range c_paths { 145 | c := newClient(keyPath + "/" + c_path) 146 | c.Joining() 147 | defer c.Leaving() 148 | *clients = append(*clients, c) 149 | } 150 | } 151 | 152 | func TestManyClientsConcurrentWrites(t *testing.T) { 153 | t.Skip("skip failing test - FIXME") 154 | 155 | // multiple different clients will write to different keys multiple times concurrently 156 | // no maximum - crashes when exceeding max # of open files 157 | servers := runServers(t, "a", "rw") 158 | defer stopServers(servers) 159 | var clients []*Client 160 | startManyClients([]string{"u01"}, &clients) 161 | 162 | ch_len := len(clients) 163 | ch := make(chan int, ch_len) 164 | for uid, c := range clients { 165 | go func(c *Client, uid int) { 166 | // byte array uid will always be unique 167 | err := c.Write([]byte{byte(uid)}, []byte("dummyval"), nil) 168 | if err != nil { 169 | t.Log(err) 170 | } 171 | ch <- 1 172 | }(c, uid) 173 | } 174 | for i := 0; i < ch_len; i++ { 175 | <-ch 176 | } 177 | for _, c := range clients { 178 | c.Leaving() 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /protocol/server_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package protocol 5 | 6 | import ( 7 | "bytes" 8 | "io/ioutil" 9 | "log" 10 | "os" 11 | "strings" 12 | "testing" 13 | 14 | "github.com/yahoo/bftkv/crypto" 15 | "github.com/yahoo/bftkv/crypto/pgp" 16 | "github.com/yahoo/bftkv/node" 17 | "github.com/yahoo/bftkv/node/graph" 18 | "github.com/yahoo/bftkv/quorum/wotqs" 19 | storage_plain "github.com/yahoo/bftkv/storage/plain" 20 | "github.com/yahoo/bftkv/transport" 21 | transport_http "github.com/yahoo/bftkv/transport/http" 22 | ) 23 | 24 | const ( 25 | scriptPath = "../scripts" // any way to specify the absolute path? 26 | keyPath = scriptPath + "/run/keys" 27 | serverKeyPrefix = "a" 28 | clientKey = "u01" 29 | dbPrefix = scriptPath + "/run/db." 30 | testKey = "test" 31 | testValue = "test" 32 | ) 33 | 34 | func TestServer(t *testing.T) { 35 | t.Skip("skip failing test - FIXME") 36 | 37 | servers := runServers(t, "a", "rw") 38 | 39 | defer func(servers []*Server) { 40 | stopServers(servers) 41 | }(servers) 42 | 43 | // create a client 44 | c := newClient(keyPath + "/" + clientKey) 45 | c.Joining() 46 | 47 | key := []byte(testKey) 48 | value := []byte(testValue) 49 | if err := c.Write(key, value, nil); err != nil { 50 | t.Fatal(err) 51 | } 52 | res, err := c.Read(key, nil) 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | if !bytes.Equal(res, value) { 57 | t.Errorf("Got %v\n", res) 58 | } 59 | } 60 | 61 | func newServer(path string, dbPath string) *Server { 62 | crypt := pgp.New() 63 | g := graph.New() 64 | readCerts(g, crypt, path+"/pubring.gpg", false) 65 | readCerts(g, crypt, path+"/secring.gpg", true) 66 | qs := wotqs.New(g) 67 | var tr transport.Transport 68 | tr = transport_http.New(crypt) 69 | storage := storage_plain.New(dbPath) 70 | return NewServer(node.SelfNode(g), qs, tr, crypt, storage) 71 | } 72 | 73 | func newClient(path string) *Client { 74 | crypt := pgp.New() 75 | g := graph.New() 76 | readCerts(g, crypt, path+"/pubring.gpg", false) 77 | readCerts(g, crypt, path+"/secring.gpg", true) 78 | qs := wotqs.New(g) 79 | var tr transport.Transport 80 | tr = transport_http.New(crypt) 81 | return NewClient(node.SelfNode(g), qs, tr, crypt) 82 | } 83 | 84 | func runServers(t *testing.T, prefixes ...string) []*Server { 85 | files, err := ioutil.ReadDir(keyPath) 86 | if err != nil { 87 | t.Fatal(err) 88 | } 89 | var servers []*Server 90 | for _, f := range files { 91 | for _, prefix := range prefixes { 92 | if strings.HasPrefix(f.Name(), prefix) { 93 | s := newServer(keyPath+"/"+f.Name(), dbPrefix+f.Name()) 94 | if err := s.Start(); err != nil { 95 | t.Fatal(err) 96 | } 97 | servers = append(servers, s) 98 | } 99 | } 100 | } 101 | 102 | return servers 103 | } 104 | 105 | func stopServers(servers []*Server) { 106 | for _, s := range servers { 107 | s.Stop() 108 | } 109 | } 110 | 111 | func readCerts(g *graph.Graph, crypt *crypto.Crypto, path string, sec bool) { 112 | f, err := os.Open(path) 113 | if err != nil { 114 | log.Fatal(err) 115 | return 116 | } 117 | certs, err := crypt.Certificate.ParseStream(f) 118 | if err != nil { 119 | f.Close() 120 | log.Fatal(err) 121 | } 122 | if sec { 123 | g.SetSelfNodes(certs) 124 | } else { 125 | g.AddNodes(certs) 126 | } 127 | crypt.Keyring.Register(certs, sec, true) 128 | f.Close() 129 | } 130 | -------------------------------------------------------------------------------- /protocol/test_utils/test_utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package test_utils 5 | 6 | import ( 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/yahoo/bftkv/crypto" 14 | "github.com/yahoo/bftkv/crypto/pgp" 15 | "github.com/yahoo/bftkv/node" 16 | "github.com/yahoo/bftkv/node/graph" 17 | "github.com/yahoo/bftkv/protocol" 18 | "github.com/yahoo/bftkv/quorum/wotqs" 19 | storage_plain "github.com/yahoo/bftkv/storage/plain" 20 | "github.com/yahoo/bftkv/transport" 21 | transport_http "github.com/yahoo/bftkv/transport/http" 22 | ) 23 | 24 | const ( 25 | scriptPath = "../scripts" // any way to specify the absolute path? 26 | KeyPath = scriptPath + "/run/keys" 27 | serverKeyPrefix = "a" 28 | ClientKey = "u01" 29 | dbPrefix = scriptPath + "/run/db." 30 | testKey = "test" 31 | testValue = "test" 32 | ) 33 | 34 | func NewServer(path string, dbPath string) *protocol.Server { 35 | crypt := pgp.New() 36 | g := graph.New() 37 | readCerts(g, crypt, path+"/pubring.gpg", false) 38 | readCerts(g, crypt, path+"/secring.gpg", true) 39 | qs := wotqs.New(g) 40 | var tr transport.Transport 41 | tr = transport_http.New(crypt) 42 | storage := storage_plain.New(dbPath) 43 | return protocol.NewServer(node.SelfNode(g), qs, tr, crypt, storage) 44 | } 45 | 46 | func NewClient(path string) *protocol.Client { 47 | crypt := pgp.New() 48 | g := graph.New() 49 | readCerts(g, crypt, path+"/pubring.gpg", false) 50 | readCerts(g, crypt, path+"/secring.gpg", true) 51 | qs := wotqs.New(g) 52 | var tr transport.Transport 53 | tr = transport_http.New(crypt) 54 | return protocol.NewClient(node.SelfNode(g), qs, tr, crypt) 55 | } 56 | 57 | func RunServers(t *testing.T, prefixes ...string) []*protocol.Server { 58 | files, err := ioutil.ReadDir(KeyPath) 59 | if err != nil { 60 | t.Fatal(err) 61 | } 62 | var servers []*protocol.Server 63 | for _, f := range files { 64 | for _, prefix := range prefixes { 65 | if strings.HasPrefix(f.Name(), prefix) { 66 | s := NewServer(KeyPath+"/"+f.Name(), dbPrefix+f.Name()) 67 | if err := s.Start(); err != nil { 68 | t.Fatal(err) 69 | } 70 | servers = append(servers, s) 71 | } 72 | } 73 | } 74 | 75 | return servers 76 | } 77 | 78 | func StopServers(servers []*protocol.Server) { 79 | for _, s := range servers { 80 | s.Stop() 81 | } 82 | } 83 | 84 | func readCerts(g *graph.Graph, crypt *crypto.Crypto, path string, sec bool) { 85 | f, err := os.Open(path) 86 | if err != nil { 87 | log.Fatal(err) 88 | return 89 | } 90 | certs, err := crypt.Certificate.ParseStream(f) 91 | if err != nil { 92 | f.Close() 93 | log.Fatal(err) 94 | } 95 | if sec { 96 | g.SetSelfNodes(certs) 97 | } else { 98 | g.AddNodes(certs) 99 | } 100 | crypt.Keyring.Register(certs, sec, true) 101 | f.Close() 102 | } 103 | -------------------------------------------------------------------------------- /quorum/quorum.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package quorum 5 | 6 | import ( 7 | "github.com/yahoo/bftkv/node" 8 | ) 9 | 10 | const ( 11 | READ = 0x01 12 | WRITE = 0x02 13 | AUTH = 0x04 14 | CERT = 0x08 15 | PEER = 0x10 16 | ) 17 | 18 | type Quorum interface { 19 | Nodes() []node.Node 20 | IsQuorum(nodes []node.Node) bool 21 | IsThreshold(nodes []node.Node) bool 22 | IsSufficient(nodes []node.Node) bool 23 | Reject(nodes []node.Node) bool 24 | GetThreshold() int 25 | } 26 | 27 | type QuorumSystem interface { 28 | ChooseQuorum(rw int) Quorum 29 | } 30 | -------------------------------------------------------------------------------- /quorum/wotqs/wotqs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package wotqs 5 | 6 | import ( 7 | "github.com/yahoo/bftkv/node" 8 | "github.com/yahoo/bftkv/node/graph" 9 | "github.com/yahoo/bftkv/quorum" 10 | ) 11 | 12 | type wot struct { 13 | g *graph.Graph 14 | } 15 | 16 | type qc struct { // quarum clique 17 | nodes []node.Node 18 | f int 19 | min int 20 | threshold int 21 | suff int 22 | } 23 | 24 | type wotq struct { 25 | qcs []qc 26 | } 27 | 28 | func howmany(a, b int) int { 29 | return (a + b - 1) / b 30 | } 31 | 32 | func New(g *graph.Graph) quorum.QuorumSystem { 33 | return &wot{g: g} 34 | } 35 | 36 | func (qs *wot) newQC(clique graph.Clique, rw int) *qc { 37 | var nodes []node.Node 38 | if (rw & quorum.PEER) != 0 { // exclude the self node 39 | self := qs.g.GetSelfId() 40 | for _, n := range clique.Nodes { 41 | if n.Id() != self { 42 | nodes = append(nodes, n) 43 | } 44 | } 45 | } else { 46 | nodes = clique.Nodes 47 | } 48 | n := len(nodes) 49 | if n == 0 { 50 | return nil 51 | } 52 | if rw == quorum.WRITE { 53 | return &qc{nodes, 0, 0, 0, 0} 54 | } 55 | f := (n - 1) / 3 56 | if f >= 1 { 57 | min := 3*f + 1 58 | threshold := 2*f + 1 59 | suff := f + (n-f)/2 + 1 60 | if (rw & (quorum.CERT | quorum.READ)) != 0 { 61 | threshold = f + 1 62 | } 63 | if clique.Weight <= n-suff { 64 | suff = 0 65 | } 66 | return &qc{nodes, f, min, threshold, suff} 67 | } else { 68 | return nil 69 | } 70 | } 71 | 72 | func (qs *wot) complement(u []node.Node, c []qc, e []qc, rw int) []qc { 73 | var nodes []node.Node 74 | for _, n1 := range u { 75 | found := false 76 | next: 77 | for _, qc := range c { 78 | for _, n2 := range qc.nodes { 79 | if n1.Id() == n2.Id() { 80 | found = true 81 | break next 82 | } 83 | } 84 | } 85 | if !found { 86 | nodes = append(nodes, n1) 87 | } 88 | } 89 | if q := qs.newQC(graph.Clique{nodes, 0}, rw); q != nil { 90 | e = append(e, *q) 91 | } 92 | return e 93 | } 94 | 95 | func (qs *wot) getQuorumFrom(rw int, s uint64, distance int) *wotq { 96 | q := &wotq{} 97 | cliques := qs.g.GetCliques(s, distance) 98 | for _, c := range cliques { 99 | if qc := qs.newQC(c, rw|quorum.AUTH); qc != nil { 100 | q.qcs = append(q.qcs, *qc) 101 | } 102 | } 103 | if (rw & (quorum.READ | quorum.WRITE)) != 0 { 104 | qcs := q.qcs 105 | if (rw & quorum.AUTH) == 0 { 106 | qcs = nil 107 | } 108 | qcs = qs.complement(qs.g.GetReachableNodes(s, distance), q.qcs, qcs, quorum.READ) // R = {Vi} - {Ci} 109 | if (rw & quorum.WRITE) != 0 { 110 | qcs = qs.complement(qs.g.GetPeers(), append(q.qcs, qcs...), qcs, quorum.WRITE) // W = U - {Ci} + R 111 | } 112 | q.qcs = qcs 113 | } 114 | return q 115 | } 116 | 117 | func (qs *wot) ChooseQuorum(rw int) quorum.Quorum { 118 | var distance int 119 | if (rw & quorum.CERT) != 0 { 120 | distance = 0 121 | } else if (rw & quorum.AUTH) != 0 { 122 | distance = 1 123 | } else { 124 | distance = 2 125 | } 126 | return qs.getQuorumFrom(rw, qs.g.GetSelfId(), distance) 127 | } 128 | 129 | // 130 | // quorum 131 | // 132 | func (q *wotq) Nodes() []node.Node { 133 | var r []node.Node 134 | for _, qc := range q.qcs { 135 | for _, n := range qc.nodes { 136 | if n.Active() && n.Address() != "" { 137 | r = append(r, n) 138 | } 139 | } 140 | } 141 | return r 142 | } 143 | 144 | func (q *wotq) IsQuorum(nodes []node.Node) bool { 145 | // |nodes| >= 3f+1 146 | if len(q.qcs) == 0 { 147 | return false 148 | } 149 | for _, qc := range q.qcs { 150 | if qc.f > 0 && len(intersection(nodes, qc.nodes)) < qc.min { 151 | return false 152 | } 153 | } 154 | return true 155 | } 156 | 157 | func (q *wotq) IsThreshold(nodes []node.Node) bool { 158 | if len(q.qcs) == 0 { 159 | return false 160 | } 161 | for _, qc := range q.qcs { 162 | if qc.threshold > 0 && len(intersection(nodes, qc.nodes)) < qc.threshold { 163 | return false 164 | } 165 | } 166 | return true 167 | } 168 | 169 | func (q *wotq) IsSufficient(nodes []node.Node) bool { 170 | for _, qc := range q.qcs { 171 | if qc.suff > 0 && len(intersection(nodes, qc.nodes)) >= qc.suff { 172 | return true 173 | } 174 | } 175 | return false 176 | } 177 | 178 | func (q *wotq) Reject(nodes []node.Node) bool { 179 | for _, qc := range q.qcs { 180 | if qc.f == 0 || len(intersection(nodes, qc.nodes)) <= qc.f { 181 | return false 182 | } 183 | } 184 | return true 185 | } 186 | 187 | func (q *wotq) GetThreshold() int { 188 | th := 0 189 | for _, qc := range q.qcs { 190 | th += qc.threshold 191 | } 192 | return th 193 | } 194 | 195 | func intersection(s1, s2 []node.Node) []node.Node { 196 | var ret []node.Node 197 | for _, n1 := range s1 { 198 | for _, n2 := range s2 { 199 | if n1.Id() == n2.Id() { 200 | ret = append(ret, n1) 201 | break 202 | } 203 | } 204 | } 205 | return ret 206 | } 207 | -------------------------------------------------------------------------------- /scripts/check_gpg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2017, Yahoo Holdings Inc. 4 | # Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 5 | 6 | if [ -x "$(command -v gpg2)" ]; then 7 | GPG=gpg2 8 | elif [ -x "$(command -v gpg)" ]; then 9 | gpg --version | head -1 | grep ' 2\.' >/dev/null 2>&1 10 | if [ $? -ne 0 ]; then echo "gpg needs to be version 2.x"; exit 1; fi 11 | GPG=gpg 12 | else 13 | echo "gpg: not found" exit 1 14 | fi 15 | -------------------------------------------------------------------------------- /scripts/clique.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2017, Yahoo Holdings Inc. 4 | # Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 5 | 6 | . ${CWD}check_gpg.sh 7 | 8 | for i in "$@"; do 9 | av=() 10 | for j in "$@"; do 11 | if [ $j != $i ]; then av=(${av[@]} $j); fi 12 | done 13 | trust.sh -t signee $i ${av[@]} 14 | done 15 | 16 | for i in "$@"; do 17 | av=() 18 | for j in "$@"; do 19 | if [ $j != $i ]; then 20 | $GPG --homedir .$j --export `basename $j` | $GPG --homedir .$i --batch --no-tty --fast-import > /dev/null 2>&1 21 | fi 22 | done 23 | $GPG --homedir .$i --batch --no-tty --export > $i/pubring.gpg 24 | done 25 | -------------------------------------------------------------------------------- /scripts/gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2017, Yahoo Holdings Inc. 4 | # Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 5 | 6 | . ${CWD}check_gpg.sh 7 | 8 | url="" 9 | addr="" 10 | port=0 11 | if [ "$1" == "-url" ]; then 12 | shift; url=$1; shift; 13 | addr=`expr $url : '\(.*\):[0-9]*$' \| $url` 14 | port=`expr $url : '.*:\([0-9]*\)$' \| 5601` 15 | fi 16 | 17 | uid="" 18 | if [ "$1" == "-uid" ]; then shift; uid=" <$1>"; shift; fi 19 | 20 | for i in "$@"; do 21 | rm -fr $i .$i 22 | mkdir -p $i .$i 23 | chmod 700 $i .$i 24 | url="" 25 | if [ "$addr" != "" ]; then url=" ($addr:$port)"; port=`expr $port + 1`; fi 26 | $GPG --homedir .$i --batch --passphrase "" --quick-gen-key "$i$url$uid" default default never 27 | $GPG --homedir .$i --export-secret-key > $i/secring.gpg 28 | $GPG --homedir .$i --export > $i/pubring.gpg 29 | done 30 | -------------------------------------------------------------------------------- /scripts/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017, Yahoo Holdings Inc. 4 | # Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 5 | 6 | WS_ADDR=5001 7 | if [ "$1" == "-novisual" ]; then shift; WS_ADDR=0; fi 8 | 9 | if [ "$GOPATH" == "" ]; then export GOPATH=~/go; fi 10 | 11 | APP=$GOPATH/src/github.com/yahoo/bftkv 12 | MAIN=$APP/cmd/main.go 13 | AOUT=./bftkv 14 | 15 | go build -o $AOUT $MAIN 16 | if [ $? -ne 0 ]; then echo "build failed"; exit; fi 17 | 18 | FAILURE_NODES=() 19 | 20 | function is_failure { 21 | for n in ${FAILURE_NODES[@]}; do if [ "$n" == "$1" ]; then return 0; fi; done 22 | return 1 23 | } 24 | 25 | 26 | API_ADDR=6001 27 | i=1 28 | JOBS="" 29 | for key in keys/{u*,a*,b*,rw*}; do 30 | GPGHOME=`basename $key` 31 | if is_failure $GPGHOME; then continue; fi 32 | DB=db.$GPGHOME 33 | mkdir -p $DB 34 | $AOUT -home $key -api $API_ADDR -ws $WS_ADDR -db $DB & 35 | JOBS="$JOBS %$i" 36 | API_ADDR=`expr $API_ADDR + 1` 37 | if [ $WS_ADDR -ne 0 ]; then WS_ADDR=`expr $WS_ADDR + 1`; fi 38 | i=`expr $i + 1` 39 | done 40 | 41 | trap "kill $JOBS; exit" SIGINT SIGTERM 42 | 43 | while true; do sleep 60; done 44 | -------------------------------------------------------------------------------- /scripts/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2017, Yahoo Holdings Inc. 4 | # Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 5 | 6 | export CWD=`pwd`/ 7 | export PATH=$CWD:$PATH 8 | 9 | HOST=localhost 10 | if [ "$1" == "-host" ]; then shift; HOST=$1; shift; fi 11 | 12 | WD="$1" 13 | if [ "$WD" == "" ]; then WD="run"; fi 14 | mkdir -p $WD/keys 15 | cd $WD/keys 16 | 17 | gen.sh -uid "foo@example.com" u01 u02 u03 u04 18 | gen.sh -uid "bar@example.com" u11 19 | gen.sh -url http://$HOST:5601 rw01 rw02 rw03 rw04 rw05 rw06 20 | gen.sh -url http://$HOST:5701 a01 a02 a03 a04 a05 a06 a07 a08 a09 a10 21 | gen.sh -url http://$HOST:5801 b01 b02 b03 b04 b05 b06 b07 b08 b09 b10 22 | 23 | clique.sh a* 24 | clique.sh b* 25 | 26 | trust.sh -t signer rw01 a* b* 27 | trust.sh -t signer rw02 a* b* 28 | trust.sh -t signer rw03 a* b* 29 | trust.sh -t signer rw04 a* b* 30 | trust.sh -t signer rw05 a* b* 31 | trust.sh -t signer rw06 a* b* 32 | 33 | trust.sh -t signer u01 a0[1-6] rw* 34 | trust.sh -t signer u02 a0[1-6] rw* 35 | trust.sh -t signer u03 a0[1-6] rw* 36 | trust.sh -t signer u04 a0[1-6] rw* 37 | trust.sh -t signer u11 b0[1-6] rw* 38 | 39 | # for client certs (do not sign u04 for TOFU testing) 40 | trust.sh -t signee a07 u01 u02 u03 41 | trust.sh -t signee a08 u01 u02 u03 42 | trust.sh -t signee a09 u01 u02 u03 43 | trust.sh -t signee a10 u01 u02 u03 44 | trust.sh -t signee b07 u11 45 | trust.sh -t signee b08 u11 46 | trust.sh -t signee b09 u11 47 | trust.sh -t signee b10 u11 48 | 49 | # for registration test 50 | gen.sh -uid "test1@example.com" test1 51 | -------------------------------------------------------------------------------- /scripts/sign.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017, Yahoo Holdings Inc. 4 | # Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 5 | 6 | . ${CWD}check_gpg.sh 7 | 8 | output= 9 | if [ "$1" == "-o" ]; then shift; output="-o $1"; shift; fi 10 | 11 | signer=$1 12 | if [ "$signer" == "" ]; then echo "$0 signer key..."; exit; fi 13 | shift 14 | 15 | function e { 16 | echo $* 1>&2 17 | } 18 | 19 | function sign { 20 | keyid=`$GPG --homedir .$signer --batch --no-tty --fast-import $1 2>&1 | grep 'gpg: key' | sed 's/.*key \([A-Z0-9]*\):.*/\1/' | head -1` 21 | if [ $? -ne 0 ]; then e "$0: import failed"; return 1; fi 22 | if [ "$keyid" == "" ]; then e "$0: invalid keyid"; return 1; fi 23 | $GPG --homedir .$signer --batch --sign-key --yes $keyid > /dev/null 2>&1 24 | if [ $? -ne 0 ]; then e "$0: sign-key failed"; return 1; fi 25 | $GPG $output --yes --homedir .$signer --export $keyid 26 | if [ $? -ne 0 ]; then e "$0: export failed"; return 1; fi 27 | } 28 | 29 | cp -pf .$signer/pubring.kbx{,.bak} 30 | sign $* 31 | RET=$? 32 | mv -f .$signer/pubring.kbx{.bak,} 33 | exit $RET 34 | -------------------------------------------------------------------------------- /scripts/test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package main 5 | 6 | import ( 7 | "bytes" 8 | "flag" 9 | "log" 10 | "os" 11 | "strconv" 12 | 13 | "github.com/yahoo/bftkv/crypto" 14 | "github.com/yahoo/bftkv/crypto/pgp" 15 | "github.com/yahoo/bftkv/node" 16 | "github.com/yahoo/bftkv/node/graph" 17 | "github.com/yahoo/bftkv/protocol" 18 | "github.com/yahoo/bftkv/quorum/wotqs" 19 | "github.com/yahoo/bftkv/transport" 20 | transport_http "github.com/yahoo/bftkv/transport/http" 21 | ) 22 | 23 | func main() { 24 | defaultPath := os.Getenv("HOME") + "/.gnupg/" 25 | pathp := flag.String("home", defaultPath, "path to home") 26 | keyp := flag.String("key", "key", "key") 27 | np := flag.Int("n", 1, "num keys") 28 | mp := flag.Int("m", 1, "num values") 29 | readonlyp := flag.Bool("ro", false, "readonly") 30 | 31 | flag.Parse() 32 | key := *keyp 33 | n := *np 34 | m := *mp 35 | 36 | c := newClient(*pathp) 37 | c.Joining() 38 | 39 | value := []byte("value1") 40 | for ; n > 0; n-- { 41 | key := []byte(key + strconv.Itoa(n)) 42 | for ; m > 0; m-- { 43 | if !*readonlyp { 44 | value = []byte("value" + strconv.Itoa(m)) 45 | if err := c.Write(key, value, nil); err != nil { 46 | log.Fatal(err) 47 | } 48 | } 49 | res, err := c.Read(key, nil) 50 | if err != nil { 51 | log.Fatal(err) 52 | } 53 | if !bytes.Equal(res, value) { 54 | log.Printf("Got %s, expected %s\n", res, value) 55 | } 56 | } 57 | } 58 | } 59 | 60 | func newClient(path string) *protocol.Client { 61 | crypt := pgp.New() 62 | g := graph.New() 63 | readCerts(g, crypt, path+"/pubring.gpg", false) 64 | readCerts(g, crypt, path+"/secring.gpg", true) 65 | qs := wotqs.New(g) 66 | var tr transport.Transport 67 | tr = transport_http.New(crypt) 68 | return protocol.NewClient(node.SelfNode(g), qs, tr, crypt) 69 | } 70 | 71 | func readCerts(g *graph.Graph, crypt *crypto.Crypto, path string, sec bool) { 72 | f, err := os.Open(path) 73 | if err != nil { 74 | return 75 | } 76 | certs, err := crypt.Certificate.ParseStream(f) 77 | if err != nil { 78 | f.Close() 79 | log.Fatal(err) 80 | } 81 | if sec { 82 | g.SetSelfNodes(certs) 83 | } else { 84 | g.AddNodes(certs) 85 | } 86 | crypt.Keyring.Register(certs, sec, true) 87 | f.Close() 88 | } 89 | -------------------------------------------------------------------------------- /scripts/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017, Yahoo Holdings Inc. 4 | # Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 5 | 6 | cd run 7 | 8 | SRC=../test.go 9 | AOUT=./stresstest 10 | 11 | go build -o $AOUT $SRC 12 | if [ $? -ne 0 ]; then echo "build failed"; exit; fi 13 | 14 | i=1 15 | for k in gnupg.c*; do 16 | $AOUT -home $k -key "key2_$i" -n 10 -m 10 & 17 | i=`expr $i + 1` 18 | done 19 | -------------------------------------------------------------------------------- /scripts/trust.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2017, Yahoo Holdings Inc. 4 | # Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 5 | 6 | . ${CWD}check_gpg.sh 7 | 8 | trust=both 9 | if [ "$1" == "-t" ]; then shift; trust=$1; shift; fi 10 | 11 | signer=$1 12 | shift 13 | 14 | if [ "$signer" == "" ]; then echo "$0 signer signee..."; exit; fi 15 | 16 | for i in "$@"; do 17 | id=`basename $i` 18 | $GPG --homedir .$i --export $id | sign.sh -o /tmp/$id.gpg $signer 19 | if [ $? -ne 0 ]; then echo "$0: failed to sign $i" 1>&2; fi 20 | done 21 | 22 | for i in "$@"; do 23 | id=`basename $i` 24 | if [ "$trust" == "both" ] || [ "$trust" == "signer" ]; then 25 | $GPG --homedir .$signer --batch --no-tty --fast-import /tmp/$id.gpg > /dev/null 2>&1 26 | if [ $? -ne 0 ]; then echo "$0: failed to import $i"; fi 27 | fi 28 | if [ "$trust" == "both" ] || [ "$trust" == "signee" ]; then 29 | $GPG --homedir .$i --batch --no-tty --fast-import /tmp/$id.gpg > /dev/null 2>&1 30 | if [ $? -ne 0 ]; then echo "$0: failed to import $i"; fi 31 | $GPG --homedir .$i --export > $i/pubring.gpg 32 | fi 33 | rm -f /tmp/$id.gpg 34 | done 35 | 36 | if [ "$trust" == "both" ] || [ "$trust" == "signer" ]; then 37 | $GPG --homedir .$signer --batch --no-tty --export > $signer/pubring.gpg 38 | fi 39 | -------------------------------------------------------------------------------- /storage/leveldb/leveldb.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package leveldb 5 | 6 | import ( 7 | "bytes" 8 | "encoding/binary" 9 | "log" 10 | 11 | "github.com/syndtr/goleveldb/leveldb" 12 | "github.com/syndtr/goleveldb/leveldb/opt" 13 | "github.com/syndtr/goleveldb/leveldb/util" 14 | 15 | "github.com/yahoo/bftkv/storage" 16 | ) 17 | 18 | type ldb struct { 19 | db *leveldb.DB 20 | } 21 | 22 | func New(path string) storage.Storage { 23 | db, err := leveldb.OpenFile(path, nil) 24 | if err != nil { 25 | log.Panic(err) 26 | } 27 | return &ldb{db: db} 28 | } 29 | 30 | func (db *ldb) Read(variable []byte, t uint64) (value []byte, err error) { 31 | if t == 0 { 32 | iter := db.db.NewIterator(util.BytesPrefix(variable), nil) 33 | if iter.Last() { 34 | value = iter.Value() 35 | } else { 36 | err = storage.ErrNotFound 37 | } 38 | iter.Release() 39 | return 40 | } else { 41 | var buf bytes.Buffer 42 | buf.Write(variable) 43 | binary.Write(&buf, binary.BigEndian, t) 44 | return db.db.Get(buf.Bytes(), nil) 45 | } 46 | } 47 | 48 | func (db *ldb) Write(variable []byte, t uint64, value []byte) error { 49 | var buf bytes.Buffer 50 | buf.Write(variable) 51 | binary.Write(&buf, binary.BigEndian, t) 52 | return db.db.Put(buf.Bytes(), value, &opt.WriteOptions{Sync: true}) 53 | } 54 | -------------------------------------------------------------------------------- /storage/plain/plain.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package plain 5 | 6 | import ( 7 | "encoding/hex" 8 | "io/ioutil" 9 | "os" 10 | "strconv" 11 | "strings" 12 | "sync" 13 | 14 | "github.com/yahoo/bftkv/storage" 15 | ) 16 | 17 | type plain struct { 18 | path string 19 | mutex sync.Mutex 20 | } 21 | 22 | func New(path string) storage.Storage { 23 | return &plain{ 24 | path: path, 25 | } 26 | } 27 | 28 | func (p *plain) getMaxT(fname string) (uint64, error) { 29 | p.mutex.Lock() 30 | stats, err := ioutil.ReadDir(p.path) 31 | p.mutex.Unlock() 32 | if err != nil { 33 | return 0, err 34 | } 35 | fname += "." 36 | maxn := uint64(0) 37 | for _, st := range stats { 38 | if strings.HasPrefix(st.Name(), fname) { 39 | n, err := strconv.ParseUint(st.Name()[len(fname):], 10, 64) 40 | if err == nil && n > maxn { 41 | maxn = n 42 | } 43 | } 44 | } 45 | return maxn, nil 46 | } 47 | 48 | func (p *plain) constructPath(variable []byte, t uint64) (string, error) { 49 | fname := hex.EncodeToString(variable) 50 | if t == 0 { 51 | // read the latest one 52 | maxt, err := p.getMaxT(fname) 53 | if err != nil { 54 | return "", err 55 | } 56 | t = maxt 57 | } 58 | fname += "." + strconv.FormatUint(t, 10) 59 | return p.path + "/" + fname, nil 60 | } 61 | 62 | func (p *plain) Read(variable []byte, t uint64) (value []byte, err error) { 63 | path, err := p.constructPath(variable, t) 64 | if err == nil { 65 | p.mutex.Lock() 66 | value, err = ioutil.ReadFile(path) 67 | p.mutex.Unlock() 68 | } 69 | if err != nil && os.IsNotExist(err) { 70 | err = storage.ErrNotFound 71 | value = nil 72 | } 73 | return value, err 74 | } 75 | 76 | func (p *plain) Write(variable []byte, t uint64, value []byte) error { 77 | if _, err := os.Stat(p.path); err != nil && os.IsNotExist(err) { 78 | if err := os.MkdirAll(p.path, 0777); err != nil { 79 | return err 80 | } 81 | } 82 | path, err := p.constructPath(variable, t) 83 | if err != nil { 84 | return err 85 | } 86 | p.mutex.Lock() 87 | err = ioutil.WriteFile(path, value, 0644) 88 | p.mutex.Unlock() 89 | return err 90 | } 91 | -------------------------------------------------------------------------------- /storage/storage.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package storage 5 | 6 | import ( 7 | "github.com/yahoo/bftkv" 8 | ) 9 | 10 | var ( 11 | ErrNotFound = bftkv.NewError("storage: not found") 12 | ) 13 | 14 | type Storage interface { 15 | Read(variable []byte, t uint64) (value []byte, err error) 16 | Write(variable []byte, t uint64, value []byte) error 17 | } 18 | -------------------------------------------------------------------------------- /transport/http-visual/http-visual.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package http_visual 5 | 6 | import ( 7 | "encoding/json" 8 | "fmt" 9 | "log" 10 | "net/http" 11 | "strings" 12 | 13 | "github.com/yahoo/bftkv/crypto" 14 | "github.com/yahoo/bftkv/node/graph" 15 | "github.com/yahoo/bftkv/quorum" 16 | "github.com/yahoo/bftkv/transport" 17 | transport_http "github.com/yahoo/bftkv/transport/http" 18 | "golang.org/x/net/websocket" 19 | ) 20 | 21 | type TrHTTPVisual struct { 22 | transport_http.TrHTTP 23 | graph *graph.Graph 24 | qs quorum.QuorumSystem 25 | wss []*websocket.Conn 26 | } 27 | 28 | type Edge struct { 29 | Source string 30 | Destination string 31 | } 32 | 33 | type VisualGraph struct { 34 | Names map[uint64]string 35 | Edges []Edge 36 | Revoked []string 37 | } 38 | 39 | type message struct { 40 | Message string `json:"message"` 41 | } 42 | 43 | func New(security *crypto.Crypto, graph *graph.Graph, qs quorum.QuorumSystem, wsAddress string) transport.Transport { 44 | h := transport_http.New(security) 45 | hv := &TrHTTPVisual{*(h.(*transport_http.TrHTTP)), graph, qs, nil} 46 | // create listener to send and receive data from js 47 | http.Handle("/"+wsAddress, websocket.Handler(hv.HandleConnection)) 48 | go func() { 49 | err := http.ListenAndServe(":"+wsAddress, nil) 50 | if err != nil { 51 | log.Println("Websocket already open and serving.") 52 | } 53 | }() 54 | return hv 55 | } 56 | 57 | func (hVisual *TrHTTPVisual) HandleConnection(ws *websocket.Conn) { 58 | log.Printf("Socket open: %s, %s ", ws.LocalAddr(), ws.RemoteAddr()) 59 | // send the node id of the server 60 | websocket.Message.Send(ws, fmt.Sprintf("{\"actionType\": \"id\", \"id\": \"%v\"}", hVisual.graph.Id())) 61 | for { 62 | hVisual.wss = append(hVisual.wss, ws) 63 | var m message 64 | if err := websocket.JSON.Receive(ws, &m); err != nil { 65 | if fmt.Sprint(err) == "EOF" { 66 | log.Println("Socket connection terminated.") 67 | } 68 | break 69 | } 70 | 71 | log.Printf("Received message: %s\n", m) 72 | 73 | switch m.Message { 74 | case "graph": 75 | graphString := hVisual.graphToJSONString() 76 | websocket.Message.Send(ws, graphString) 77 | case "trustGraph": 78 | graphString := hVisual.graphToJSONString() 79 | websocket.Message.Send(ws, fmt.Sprintf("{\"actionType\": \"trustGraph\", \"graph\": %v}", graphString)) 80 | } 81 | } 82 | 83 | } 84 | 85 | func (hVisual *TrHTTPVisual) graphToJSONString() string { 86 | var edges []Edge 87 | names := make(map[uint64]string) 88 | for nodeId, node := range hVisual.graph.Vertices { 89 | if node.Instance != nil { 90 | names[node.Instance.Id()] = node.Instance.Name() 91 | } 92 | for destinationId, _ := range node.Edges { 93 | edges = append(edges, Edge{fmt.Sprintf("%v", nodeId), fmt.Sprintf("%v", destinationId)}) 94 | } 95 | } 96 | 97 | var revokedList []string 98 | 99 | for id, node := range hVisual.graph.Revoked { 100 | name := "" 101 | if node != nil { 102 | name = node.Name() 103 | } 104 | names[id] = name // add also nodes in the revoked list to names 105 | revokedList = append(revokedList, fmt.Sprintf("%v", id)) // fill the revoked list with ids 106 | } 107 | 108 | vg := VisualGraph{names, edges, revokedList} 109 | k, err := json.Marshal(vg) 110 | 111 | if err != nil { 112 | //log.Println(err) 113 | return "" 114 | } 115 | return string(k) 116 | } 117 | 118 | func (hv *TrHTTPVisual) ServeHTTP(w http.ResponseWriter, r *http.Request) { 119 | hv.TrHTTP.ServeHTTP(w, r) 120 | path := strings.ToLower(r.URL.Path) 121 | if !strings.HasPrefix(path, transport.Prefix) { 122 | return 123 | } 124 | actionType := path[len(transport.Prefix):] 125 | if hv.wss != nil { 126 | for _, ws := range hv.wss { 127 | if actionType == "read" || actionType == "sign" || actionType == "write" || actionType == "malwrite" || actionType == "malsign" { // for now, just inform the graph on these 128 | websocket.Message.Send(ws, fmt.Sprintf("{\"actionType\": \"%s\", \"to\": \"%s\"}", actionType, hv.graph.Name())) 129 | } else if actionType == "notify" { 130 | // decrypt the packet 131 | crypt := &crypto.Crypto{} 132 | req, _, _, err := crypt.Message.Decrypt(r.Body) 133 | 134 | if err != nil { 135 | log.Printf("server [%s]: transport security error: %s\n", hv.Server.Addr, err) 136 | } 137 | 138 | // get the list of revoked nodes 139 | nodes, err := crypt.Certificate.Parse(req) 140 | 141 | if err != nil { 142 | log.Printf("server [%s]: certificate parse error: %s\n", hv.Server.Addr, err) 143 | } 144 | 145 | var revoked []uint64 146 | for _, node := range nodes { 147 | revoked = append(revoked, node.Id()) 148 | } 149 | 150 | // inform the visualization on the revoked nodes 151 | hv.RevokeVisual(revoked) 152 | } 153 | } 154 | } 155 | } 156 | 157 | func (hv *TrHTTPVisual) RevokeVisual(ids []uint64) { 158 | for _, ws := range hv.wss { 159 | for _, id := range ids { 160 | websocket.Message.Send(ws, fmt.Sprintf("{\"actionType\": \"revoke\", \"id\": \"%v\"}", id)) 161 | } 162 | } 163 | } 164 | 165 | // If visualization is active, ServeHTTP here 166 | func (hv *TrHTTPVisual) Start(o transport.TransportServer, addr string) { 167 | hv.Server = &http.Server{ 168 | Addr: addr, 169 | Handler: hv, 170 | } 171 | hv.O = o 172 | go hv.Server.ListenAndServe() 173 | } 174 | -------------------------------------------------------------------------------- /transport/http/http.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package http 5 | 6 | import ( 7 | "context" 8 | "io" 9 | "log" 10 | "net" 11 | "net/http" 12 | "strings" 13 | "time" 14 | 15 | "github.com/yahoo/bftkv" 16 | "github.com/yahoo/bftkv/crypto" 17 | "github.com/yahoo/bftkv/node" 18 | "github.com/yahoo/bftkv/transport" 19 | ) 20 | 21 | type TrHTTP struct { 22 | client *http.Client 23 | Server *http.Server 24 | O transport.TransportServer 25 | security *crypto.Crypto 26 | } 27 | 28 | const ( 29 | NonceSize = 8 30 | DIAL_TIMEOUT = 5 31 | IDLE_TIMEOUT = 10 32 | RESPONSE_TIMEOUT = 10 33 | ) 34 | 35 | func New(security *crypto.Crypto) transport.Transport { 36 | h := &TrHTTP{security: security} 37 | 38 | // client 39 | tr := &http.Transport{ 40 | Dial: func(network, addr string) (net.Conn, error) { 41 | return net.DialTimeout(network, addr, time.Duration(DIAL_TIMEOUT)*time.Second) 42 | }, 43 | MaxIdleConns: 1, // for testing -- running multiple servers in one process process may exceeds the limit of #sockets 44 | IdleConnTimeout: time.Duration(IDLE_TIMEOUT) * time.Second, 45 | ResponseHeaderTimeout: time.Duration(RESPONSE_TIMEOUT) * time.Second, 46 | } 47 | h.client = &http.Client{ 48 | Transport: tr, 49 | } 50 | return h 51 | } 52 | 53 | func (h *TrHTTP) Post(addr string, msg io.Reader) (io.ReadCloser, error) { 54 | res, err := h.client.Post(addr, "application/octet-stream", msg) 55 | if err != nil { 56 | return nil, err 57 | } 58 | if res.StatusCode != 200 { 59 | err := transport.ErrServerError 60 | if res.StatusCode == http.StatusInternalServerError { 61 | errs := res.Header.Get("X-error") 62 | if errs != "" { 63 | err = bftkv.ErrorFromString(errs) 64 | } 65 | } 66 | return nil, err 67 | } 68 | return res.Body, nil 69 | } 70 | 71 | func (h *TrHTTP) Multicast(path int, peers []node.Node, data []byte, cb func(res *transport.MulticastResponse) bool) { 72 | transport.Multicast(h, path, peers, [][]byte{data}, cb) 73 | } 74 | 75 | func (h *TrHTTP) MulticastM(path int, peers []node.Node, mdata [][]byte, cb func(res *transport.MulticastResponse) bool) { 76 | transport.Multicast(h, path, peers, mdata, cb) 77 | } 78 | 79 | func (h *TrHTTP) Start(o transport.TransportServer, addr string) { 80 | h.Server = &http.Server{ 81 | Addr: addr, 82 | Handler: h, 83 | } 84 | h.O = o 85 | go h.Server.ListenAndServe() 86 | } 87 | 88 | func (h *TrHTTP) Stop() { 89 | if h.Server == nil { 90 | return 91 | } 92 | ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) 93 | h.Server.Shutdown(ctx) 94 | h.Server.Close() 95 | } 96 | 97 | func (h *TrHTTP) ServeHTTP(w http.ResponseWriter, r *http.Request) { 98 | if r.Method == http.MethodGet { 99 | if err := r.ParseForm(); err != nil { 100 | http.Error(w, err.Error(), http.StatusBadRequest) 101 | return 102 | } 103 | } 104 | path := strings.ToLower(r.URL.Path) 105 | if !strings.HasPrefix(path, transport.Prefix) { 106 | log.Printf("http: %s not found\n", path) 107 | http.NotFound(w, r) 108 | return 109 | } 110 | var cmd int 111 | switch path[len(transport.Prefix):] { 112 | case "join": 113 | cmd = transport.Join 114 | case "leave": 115 | cmd = transport.Leave 116 | case "time": 117 | cmd = transport.Time 118 | case "read": 119 | cmd = transport.Read 120 | case "write": 121 | cmd = transport.Write 122 | case "sign": 123 | cmd = transport.Sign 124 | case "auth": 125 | cmd = transport.Auth 126 | case "setauth": 127 | cmd = transport.SetAuth 128 | case "distribute": 129 | cmd = transport.Distribute 130 | case "distsign": 131 | cmd = transport.DistSign 132 | case "register": 133 | cmd = transport.Register 134 | case "revoke": 135 | cmd = transport.Revoke 136 | case "notify": 137 | cmd = transport.Notify 138 | default: 139 | http.NotFound(w, r) 140 | log.Printf("http: %s not found\n", path) 141 | return 142 | } 143 | err := h.O.Handler(cmd, r.Body, w) 144 | if err != nil { 145 | w.Header().Add("X-error", err.Error()) 146 | http.Error(w, "Internal Server Error", http.StatusInternalServerError) 147 | return 148 | } 149 | } 150 | 151 | func (h *TrHTTP) GenerateRandom() []byte { 152 | return h.security.RNG.Generate(NonceSize) 153 | } 154 | 155 | func (h *TrHTTP) Encrypt(peers []node.Node, plain []byte, nonce []byte) (cipher []byte, err error) { 156 | return h.security.Message.Encrypt(peers, plain, nonce) 157 | } 158 | 159 | func (h *TrHTTP) Decrypt(r io.Reader) (plain []byte, nonce []byte, peer node.Node, err error) { 160 | return h.security.Message.Decrypt(r) 161 | } 162 | -------------------------------------------------------------------------------- /transport/http/malhttp.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package http 5 | 6 | import ( 7 | "net" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/yahoo/bftkv/crypto" 12 | _ "github.com/yahoo/bftkv/node" 13 | "github.com/yahoo/bftkv/transport" 14 | ) 15 | 16 | type MalTrHTTP struct { 17 | TrHTTP 18 | O transport.MalTransportServer 19 | } 20 | 21 | func MalNew(security *crypto.Crypto) transport.Transport { 22 | h := &MalTrHTTP{ 23 | TrHTTP: TrHTTP{ 24 | security: security, 25 | }, 26 | } 27 | 28 | // client 29 | tr := &http.Transport{ 30 | Dial: func(network, addr string) (net.Conn, error) { 31 | return net.DialTimeout(network, addr, time.Duration(DIAL_TIMEOUT)*time.Second) 32 | }, 33 | MaxIdleConns: 5, // for testing -- running multiple servers in one process process may exceeds the limit of #sockets 34 | IdleConnTimeout: time.Duration(IDLE_TIMEOUT) * time.Second, 35 | ResponseHeaderTimeout: time.Duration(RESPONSE_TIMEOUT) * time.Second, 36 | } 37 | h.client = &http.Client{ 38 | Transport: tr, 39 | } 40 | return h 41 | } 42 | -------------------------------------------------------------------------------- /transport/maltransport.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package transport 5 | 6 | import ( 7 | "io" 8 | ) 9 | 10 | type MalTransportServer interface { 11 | MalHandler(cmd int, r io.Reader, w io.Writer) error 12 | } 13 | -------------------------------------------------------------------------------- /transport/transport.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | package transport 5 | 6 | import ( 7 | "bytes" 8 | "io" 9 | 10 | "github.com/yahoo/bftkv" 11 | "github.com/yahoo/bftkv/node" 12 | ) 13 | 14 | const ( 15 | Join = iota 16 | Leave 17 | Time 18 | Read 19 | Write 20 | Sign 21 | Auth 22 | SetAuth 23 | Distribute 24 | DistSign 25 | Register 26 | Revoke 27 | Notify 28 | ) 29 | 30 | // notification 31 | const ( 32 | NetworkConfigChanged = 0 33 | ) 34 | 35 | const Prefix = "/bftkv/v1/" 36 | 37 | var ( 38 | ErrTransportSecurity = bftkv.NewError("transport: transport security error") 39 | ErrTransportNonceMismatch = bftkv.NewError("transport: nonce mismatch") 40 | ErrServerError = bftkv.NewError("transport: server error") 41 | ErrNoAddress = bftkv.NewError("transport: no address") 42 | ) 43 | 44 | type MulticastResponse struct { 45 | Peer node.Node 46 | Data []byte 47 | Err error 48 | } 49 | 50 | type TransportServer interface { 51 | Handler(cmd int, r io.Reader, w io.Writer) error 52 | } 53 | 54 | type Transport interface { 55 | Multicast(path int, peers []node.Node, data []byte, cb func(res *MulticastResponse) bool) 56 | MulticastM(path int, peers []node.Node, mdata [][]byte, cb func(res *MulticastResponse) bool) 57 | Start(o TransportServer, addr string) 58 | Stop() 59 | 60 | // internal use 61 | Post(addr string, msg io.Reader) (res io.ReadCloser, err error) 62 | GenerateRandom() []byte 63 | Encrypt(peers []node.Node, plain []byte, nonce []byte) (cipher []byte, err error) 64 | Decrypt(r io.Reader) (plain []byte, nonce []byte, peer node.Node, err error) 65 | } 66 | 67 | func Multicast(tr Transport, path int, peers []node.Node, mdata [][]byte, cb func(res *MulticastResponse) bool) { 68 | cmd := "" 69 | switch path { 70 | case Join: 71 | cmd = "join" 72 | case Leave: 73 | cmd = "leave" 74 | case Time: 75 | cmd = "time" 76 | case Read: 77 | cmd = "read" 78 | case Write: 79 | cmd = "write" 80 | case Sign: 81 | cmd = "sign" 82 | case Auth: 83 | cmd = "auth" 84 | case SetAuth: 85 | cmd = "setauth" 86 | case Distribute: 87 | cmd = "distribute" 88 | case DistSign: 89 | cmd = "distsign" 90 | case Register: 91 | cmd = "register" 92 | case Revoke: 93 | cmd = "revoke" 94 | case Notify: 95 | cmd = "notify" 96 | } 97 | ch := make(chan (*MulticastResponse), len(peers)) 98 | var cipher []byte 99 | var nonce []byte 100 | var err error 101 | for i, peer := range peers { 102 | if i < len(mdata) { 103 | nonce = tr.GenerateRandom() 104 | cipher, err = tr.Encrypt(peers[i:i+len(peers)-len(mdata)+1], mdata[i], nonce) 105 | if err != nil { 106 | ch <- &MulticastResponse{peer, nil, err} 107 | continue 108 | } 109 | } 110 | go func(peer node.Node, cipher []byte, nonce []byte) { 111 | if peer.Address() == "" { 112 | ch <- &MulticastResponse{peer, nil, ErrNoAddress} 113 | return 114 | } 115 | var plain []byte 116 | r, err := tr.Post(peer.Address()+Prefix+cmd, bytes.NewReader(cipher)) 117 | if err == nil { 118 | var t []byte 119 | plain, t, _, err = tr.Decrypt(r) 120 | r.Close() 121 | if err == nil && !bytes.Equal(t, nonce) { 122 | err = ErrTransportNonceMismatch 123 | plain = nil 124 | } 125 | } 126 | ch <- &MulticastResponse{peer, plain, err} // Node is always available 127 | }(peer, cipher, nonce) 128 | } 129 | for i := 0; i < len(peers); i++ { 130 | mr := <-ch 131 | if cb != nil { 132 | if cb(mr) { 133 | break // should cancel the remaining Post requests? 134 | } 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /visual/README.md: -------------------------------------------------------------------------------- 1 | # BFTKV-visual 2 | -------------------------------------------------------------------------------- /visual/css/style.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Holdings Inc. 3 | * Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 4 | */ 5 | 6 | #graph { 7 | height: 80%; 8 | width: 100%; 9 | position: absolute; 10 | left: 0; 11 | top: 100; 12 | } 13 | 14 | .button { 15 | border: none; 16 | color: white; 17 | padding: 10px 25px; 18 | text-align: center; 19 | text-decoration: none; 20 | display: inline-block; 21 | font-size: 16px; 22 | } 23 | 24 | .greenBG { 25 | background-color: green; 26 | } 27 | 28 | 29 | .orangeBG { 30 | background-color: orange; 31 | } 32 | 33 | .yellowBG { 34 | background-color: yellow; 35 | } 36 | 37 | .redBG { 38 | background-color: red; 39 | } 40 | 41 | .magentaBG { 42 | background-color: magenta; 43 | } 44 | 45 | .pinkBG { 46 | background-color: #DB7093; 47 | } 48 | 49 | input[type=text] { 50 | width: 15%; 51 | padding: 12px 30px; 52 | margin: 8px 0; 53 | box-sizing: border-box; 54 | font-size: 15px; 55 | } 56 | 57 | input[type="checkbox"] { 58 | font-size: 50px; 59 | } 60 | -------------------------------------------------------------------------------- /visual/js/displayGraph.js: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Yahoo Holdings Inc. 2 | // Licensed under the terms of the Apache license. See LICENSE file in project root for terms. 3 | 4 | var cy = null; 5 | 6 | function createNodes(edges, names) { 7 | console.log("creating nodes") 8 | nodes = [] 9 | for (var key in names) { 10 | nodes.push({data: {id: key, name: names[key]}}) 11 | } 12 | for(var i = 0; i < edges.length; i++) { 13 | if (!isInNodeArray(nodes, edges[i].Source)) { 14 | nodes.push({data: {id: edges[i].Source + "", name: names[edges[i].Source + ""]}}) 15 | } 16 | if (!isInNodeArray(nodes, edges[i].Destination)) { 17 | nodes.push({data: {id: edges[i].Destination + "", name: names[edges[i].Destination + ""]}}) 18 | } 19 | } 20 | 21 | console.log("Nodes: ") 22 | console.log(nodes) 23 | return nodes 24 | } 25 | 26 | function isInNodeArray(nodeArray, id) { 27 | //console.log("Questioned id: " + id) 28 | for(var i = 0; i < nodeArray.length; i++) { 29 | var node = nodeArray[i] 30 | 31 | //console.log(node) 32 | if (node.data.id == id) { 33 | return true 34 | } 35 | } 36 | return false 37 | } 38 | 39 | 40 | function createEdges(edges) { 41 | graphEdges = [] 42 | console.log("Creating graph edges. ") 43 | for (var counter = 0; counter < edges.length; counter++) { 44 | var source = edges[counter].Source; 45 | var destination = edges[counter].Destination; 46 | 47 | // edge id = edge.source + edge.destination 48 | graphEdges.push({data: {id: source + destination, weight: 1, source, target: destination}}) 49 | } 50 | 51 | // sort edges so that the graph will look the same 52 | graphEdges.sort(function(edge1, edge2) { 53 | return edge1.data.id.localeCompare(edge2.data.id) 54 | }) 55 | 56 | return graphEdges 57 | } 58 | 59 | function buildGraph(data) { 60 | console.log("Data:") 61 | console.log(data) 62 | var nodes = createNodes(data.Edges, data.Names) // send the graph data too since we don't have the names for some nodes in the edges 63 | var edges = createEdges(data.Edges) 64 | 65 | // create the graph 66 | cy = cytoscape({container: document.getElementById("graph"), elements: {"nodes": nodes, "edges": edges}, 67 | layout : { 68 | name: "dagre", 69 | roots: "#8E44AD", 70 | padding: 10 71 | }, 72 | style : cytoscape.stylesheet().selector('node').css({'content': 'data(name)', 73 | 'text-valign': 'center', 74 | 'color': 'white', 75 | 'width': 80, 76 | "height": 80, 77 | 'text-outline-width': 2, 78 | 'background-color': '#8E44AD', 79 | 'text-outline-color': '#999' 80 | }).selector("edge") 81 | .css({ 82 | 'curve-style': 'bezier', 83 | 'target-arrow-shape': 'triangle', 84 | 'width': 4, 85 | 'line-color': '#104FEB', 86 | 'target-arrow-color': '#104FEB'}) 87 | }) 88 | markRevoked(data.Revoked) 89 | } 90 | 91 | function markRevoked(revokedArr) { 92 | if(revokedArr == null) { 93 | console.log("No nodes revoked."); 94 | return; 95 | } 96 | for (var i = 0; i < revokedArr.length; i++) { 97 | cy.filter('node[name="' + names[revokedArr[i]] + '"]').style({'background-color': 'red'}) 98 | // clear edges going to and from the revoked node 99 | cy.filter('edge[target="' + revokedArr[i] + '"]').remove(); 100 | cy.filter('edge[source="' + revokedArr[i] + '"]').remove(); 101 | } 102 | } 103 | 104 | --------------------------------------------------------------------------------