├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── cluster ├── doc.go ├── httpcluster │ ├── doc.go │ └── peer.go └── peer.go ├── cmd └── caspaxos-http │ ├── core_test.go │ └── main.go ├── extension ├── acceptor.go ├── cluster.go ├── doc.go ├── operator_node.go ├── proposer.go └── user_node.go ├── httpapi ├── core_test.go ├── doc.go ├── helpers.go ├── http_acceptor.go ├── http_operator_node.go ├── http_proposer.go └── http_user_node.go ├── internal └── eventsource │ ├── LICENSE │ ├── README.md │ ├── bench_test.go │ ├── decoder.go │ ├── decoder_test.go │ ├── encoder.go │ ├── encoder_test.go │ ├── eventsource.go │ ├── eventsource_test.go │ ├── example_test.go │ ├── handler.go │ ├── handler_test.go │ └── identity_test.go └── protocol ├── age.go ├── age_test.go ├── ballot.go ├── ballot_test.go ├── basic_test.go ├── doc.go ├── memory_acceptor.go ├── memory_acceptor_test.go ├── memory_proposer.go ├── memory_proposer_test.go ├── operations.go └── operations_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | script: go test -race -v ./... 4 | 5 | go: 6 | - 1.9.x 7 | - 1.10.x 8 | - tip 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /cluster/doc.go: -------------------------------------------------------------------------------- 1 | // Package cluster implements a gossip-based cluster based on HashiCorp memberlist. 2 | package cluster 3 | -------------------------------------------------------------------------------- /cluster/httpcluster/doc.go: -------------------------------------------------------------------------------- 1 | // Package httpcluster enhances a plain cluster peer with extension.Cluster 2 | // behavior. It assumes peers in the cluster serve httpapi APIs. 3 | package httpcluster 4 | -------------------------------------------------------------------------------- /cluster/httpcluster/peer.go: -------------------------------------------------------------------------------- 1 | package httpcluster 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/url" 7 | "strings" 8 | 9 | "github.com/peterbourgon/caspaxos/cluster" 10 | "github.com/peterbourgon/caspaxos/extension" 11 | "github.com/peterbourgon/caspaxos/httpapi" 12 | ) 13 | 14 | // The types of peers in the cluster. 15 | const ( 16 | PeerTypeAcceptor = "acceptor" 17 | PeerTypeProposer = "proposer" 18 | PeerTypeOperatorNode = "operator-node" 19 | PeerTypeUserNode = "user-node" 20 | ) 21 | 22 | // Peer wraps a plain cluster.Peer and implements extension.Cluster. 23 | // We assume API host:ports in the cluster map to httpapi servers. 24 | type Peer struct { 25 | *cluster.Peer 26 | } 27 | 28 | var _ extension.Cluster = (*Peer)(nil) 29 | 30 | // Acceptors implements extension.Cluster. 31 | func (p Peer) Acceptors(ctx context.Context) ([]extension.Acceptor, error) { 32 | var ( 33 | hostports = p.Query(func(peerType string) bool { return strings.Contains(peerType, PeerTypeAcceptor) }) 34 | acceptors = make([]extension.Acceptor, len(hostports)) 35 | ) 36 | for i := range hostports { 37 | u, _ := url.Parse(fmt.Sprintf("http://%s", hostports[i])) // TODO(pb): scheme 38 | acceptors[i] = httpapi.AcceptorClient{URL: u} // TODO(pb): HTTP client 39 | } 40 | return acceptors, nil 41 | } 42 | 43 | // Proposers implements extension.Cluster. 44 | func (p Peer) Proposers(ctx context.Context) ([]extension.Proposer, error) { 45 | var ( 46 | hostports = p.Query(func(peerType string) bool { return strings.Contains(peerType, PeerTypeProposer) }) 47 | proposers = make([]extension.Proposer, len(hostports)) 48 | ) 49 | for i := range hostports { 50 | u, _ := url.Parse(fmt.Sprintf("http://%s", hostports[i])) // TODO(pb): scheme 51 | proposers[i] = httpapi.ProposerClient{URL: u} // TODO(pb): HTTP client 52 | } 53 | return proposers, nil 54 | } 55 | 56 | // OperatorNodes implements extension.Cluster. 57 | func (p Peer) OperatorNodes(ctx context.Context) ([]extension.OperatorNode, error) { 58 | var ( 59 | hostports = p.Query(func(peerType string) bool { return strings.Contains(peerType, PeerTypeOperatorNode) }) 60 | operators = make([]extension.OperatorNode, len(hostports)) 61 | ) 62 | for i := range hostports { 63 | u, _ := url.Parse(fmt.Sprintf("http://%s", hostports[i])) // TODO(pb): scheme 64 | operators[i] = httpapi.OperatorNodeClient{URL: u} // TODO(pb): HTTP client 65 | } 66 | return operators, nil 67 | } 68 | 69 | // UserNodes implements extension.Cluster. 70 | func (p Peer) UserNodes(ctx context.Context) ([]extension.UserNode, error) { 71 | var ( 72 | hostports = p.Query(func(peerType string) bool { return strings.Contains(peerType, PeerTypeUserNode) }) 73 | users = make([]extension.UserNode, len(hostports)) 74 | ) 75 | for i := range hostports { 76 | u, _ := url.Parse(fmt.Sprintf("http://%s", hostports[i])) // TODO(pb): scheme 77 | users[i] = httpapi.UserNodeClient{URL: u} // TODO(pb): HTTP client 78 | } 79 | return users, nil 80 | } 81 | -------------------------------------------------------------------------------- /cluster/peer.go: -------------------------------------------------------------------------------- 1 | package cluster 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net" 8 | "strings" 9 | "sync" 10 | "time" 11 | 12 | "github.com/go-kit/kit/log" 13 | "github.com/go-kit/kit/log/level" 14 | "github.com/hashicorp/memberlist" 15 | "github.com/pborman/uuid" 16 | ) 17 | 18 | // Config for our node in the cluster. 19 | // All fields are mandatory unless otherwise noted. 20 | type Config struct { 21 | ClusterHost string 22 | ClusterPort int 23 | AdvertiseHost string // optional; by default, ClusterHost is used 24 | AdvertisePort int // optional; by default, ClusterPort is used 25 | APIHost string 26 | APIPort int 27 | Type string 28 | InitialPeers []string 29 | Logger log.Logger 30 | } 31 | 32 | // Peer models a node in the cluster. 33 | type Peer struct { 34 | ml *memberlist.Memberlist 35 | d *delegate 36 | } 37 | 38 | // NewPeer joins the cluster. Don't forget to leave it, at some point. 39 | func NewPeer(config Config) (*Peer, error) { 40 | var ( 41 | d = newDelegate(config.Logger) 42 | c = memberlist.DefaultLANConfig() 43 | ) 44 | { 45 | c.Name = uuid.New() 46 | c.BindAddr = config.ClusterHost 47 | c.BindPort = config.ClusterPort 48 | c.AdvertiseAddr = config.AdvertiseHost 49 | c.AdvertisePort = config.AdvertisePort 50 | c.LogOutput = ioutil.Discard 51 | c.Delegate = d 52 | c.Events = d 53 | } 54 | 55 | ml, err := memberlist.Create(c) 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | d.initialize(c.Name, config.Type, config.APIHost, config.APIPort, ml.NumMembers) 61 | n, _ := ml.Join(config.InitialPeers) 62 | level.Debug(config.Logger).Log("Join", n) 63 | 64 | return &Peer{ 65 | ml: ml, 66 | d: d, 67 | }, nil 68 | } 69 | 70 | // Leave the cluster, waiting up to timeout for a confirmation. 71 | func (p *Peer) Leave(timeout time.Duration) error { 72 | return p.ml.Leave(timeout) 73 | } 74 | 75 | // Query for other peers in the cluster whose type passes the function f. 76 | // Results are returned as matching API host:port pairs. 77 | func (p *Peer) Query(f func(peerType string) bool) (apiHostPorts []string) { 78 | for _, v := range p.d.query(f) { 79 | apiHostPort := net.JoinHostPort(v.APIHost, fmt.Sprint(v.APIPort)) 80 | apiHostPorts = append(apiHostPorts, apiHostPort) 81 | } 82 | return apiHostPorts 83 | } 84 | 85 | // 86 | // 87 | // 88 | 89 | type delegate struct { 90 | mtx sync.RWMutex 91 | bcast *memberlist.TransmitLimitedQueue 92 | data map[string]peerInfo 93 | logger log.Logger 94 | } 95 | 96 | type peerInfo struct { 97 | Type string `json:"type"` 98 | APIHost string `json:"api_host"` 99 | APIPort int `json:"api_port"` 100 | } 101 | 102 | func newDelegate(logger log.Logger) *delegate { 103 | return &delegate{ 104 | data: map[string]peerInfo{}, 105 | logger: logger, 106 | } 107 | } 108 | 109 | func (d *delegate) initialize(id string, peerType string, apiHost string, apiPort int, numNodes func() int) { 110 | d.mtx.Lock() 111 | defer d.mtx.Unlock() 112 | d.bcast = &memberlist.TransmitLimitedQueue{ 113 | NumNodes: numNodes, 114 | RetransmitMult: 3, 115 | } 116 | d.data[id] = peerInfo{peerType, apiHost, apiPort} 117 | } 118 | 119 | func (d *delegate) query(f func(peerType string) bool) map[string]peerInfo { 120 | res := map[string]peerInfo{} 121 | for k, info := range d.state() { 122 | if f(info.Type) { 123 | res[k] = info 124 | } 125 | } 126 | return res 127 | } 128 | 129 | func (d *delegate) state() map[string]peerInfo { 130 | d.mtx.RLock() 131 | defer d.mtx.RUnlock() 132 | res := map[string]peerInfo{} 133 | for k, v := range d.data { 134 | res[k] = v 135 | } 136 | return res 137 | } 138 | 139 | // NodeMeta is used to retrieve meta-data about the current node 140 | // when broadcasting an alive message. It's length is limited to 141 | // the given byte size. This metadata is available in the Node structure. 142 | // Implements memberlist.Delegate. 143 | func (d *delegate) NodeMeta(limit int) []byte { 144 | d.mtx.RLock() 145 | defer d.mtx.RUnlock() 146 | return []byte{} // no metadata 147 | } 148 | 149 | // NotifyMsg is called when a user-data message is received. 150 | // Care should be taken that this method does not block, since doing 151 | // so would block the entire UDP packet receive loop. Additionally, the byte 152 | // slice may be modified after the call returns, so it should be copied if needed. 153 | // Implements memberlist.Delegate. 154 | func (d *delegate) NotifyMsg(b []byte) { 155 | if len(b) == 0 { 156 | return 157 | } 158 | var data map[string]peerInfo 159 | if err := json.Unmarshal(b, &data); err != nil { 160 | level.Error(d.logger).Log("method", "NotifyMsg", "b", strings.TrimSpace(string(b)), "err", err) 161 | return 162 | } 163 | d.mtx.Lock() 164 | defer d.mtx.Unlock() 165 | for k, v := range data { 166 | // Removing data is handled by NotifyLeave 167 | d.data[k] = v 168 | } 169 | } 170 | 171 | // GetBroadcasts is called when user data messages can be broadcast. 172 | // It can return a list of buffers to send. Each buffer should assume an 173 | // overhead as provided with a limit on the total byte size allowed. 174 | // The total byte size of the resulting data to send must not exceed 175 | // the limit. Care should be taken that this method does not block, 176 | // since doing so would block the entire UDP packet receive loop. 177 | // Implements memberlist.Delegate. 178 | func (d *delegate) GetBroadcasts(overhead, limit int) [][]byte { 179 | d.mtx.RLock() 180 | defer d.mtx.RUnlock() 181 | if d.bcast == nil { 182 | panic("GetBroadcast before init") 183 | } 184 | return d.bcast.GetBroadcasts(overhead, limit) 185 | } 186 | 187 | // LocalState is used for a TCP Push/Pull. This is sent to 188 | // the remote side in addition to the membership information. Any 189 | // data can be sent here. See MergeRemoteState as well. The `join` 190 | // boolean indicates this is for a join instead of a push/pull. 191 | // Implements memberlist.Delegate. 192 | func (d *delegate) LocalState(join bool) []byte { 193 | d.mtx.RLock() 194 | defer d.mtx.RUnlock() 195 | buf, _ := json.Marshal(d.data) 196 | return buf 197 | } 198 | 199 | // MergeRemoteState is invoked after a TCP Push/Pull. This is the 200 | // state received from the remote side and is the result of the 201 | // remote side's LocalState call. The 'join' 202 | // boolean indicates this is for a join instead of a push/pull. 203 | // Implements memberlist.Delegate. 204 | func (d *delegate) MergeRemoteState(buf []byte, join bool) { 205 | if len(buf) == 0 { 206 | level.Debug(d.logger).Log("method", "MergeRemoteState", "join", join, "buf_sz", 0) 207 | return 208 | } 209 | var data map[string]peerInfo 210 | if err := json.Unmarshal(buf, &data); err != nil { 211 | level.Error(d.logger).Log("method", "MergeRemoteState", "err", err) 212 | return 213 | } 214 | d.mtx.Lock() 215 | defer d.mtx.Unlock() 216 | for k, v := range data { 217 | d.data[k] = v 218 | } 219 | } 220 | 221 | // NotifyJoin is invoked when a node is detected to have joined. 222 | // The Node argument must not be modified. 223 | // Implements memberlist.EventDelegate. 224 | func (d *delegate) NotifyJoin(n *memberlist.Node) { 225 | level.Debug(d.logger).Log("received", "NotifyJoin", "node", n.Name, "addr", fmt.Sprintf("%s:%d", n.Addr, n.Port)) 226 | } 227 | 228 | // NotifyUpdate is invoked when a node is detected to have updated, usually 229 | // involving the meta data. The Node argument must not be modified. 230 | // Implements memberlist.EventDelegate. 231 | func (d *delegate) NotifyUpdate(n *memberlist.Node) { 232 | level.Debug(d.logger).Log("received", "NotifyUpdate", "node", n.Name, "addr", fmt.Sprintf("%s:%d", n.Addr, n.Port)) 233 | } 234 | 235 | // NotifyLeave is invoked when a node is detected to have left. 236 | // The Node argument must not be modified. 237 | // Implements memberlist.EventDelegate. 238 | func (d *delegate) NotifyLeave(n *memberlist.Node) { 239 | level.Debug(d.logger).Log("received", "NotifyLeave", "node", n.Name, "addr", fmt.Sprintf("%s:%d", n.Addr, n.Port)) 240 | d.mtx.Lock() 241 | defer d.mtx.Unlock() 242 | delete(d.data, n.Name) 243 | } 244 | -------------------------------------------------------------------------------- /cmd/caspaxos-http/core_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "net" 9 | "net/http" 10 | "net/url" 11 | "testing" 12 | "time" 13 | 14 | "github.com/go-kit/kit/log" 15 | 16 | "github.com/peterbourgon/caspaxos/cluster" 17 | "github.com/peterbourgon/caspaxos/cluster/httpcluster" 18 | "github.com/peterbourgon/caspaxos/extension" 19 | "github.com/peterbourgon/caspaxos/httpapi" 20 | "github.com/peterbourgon/caspaxos/protocol" 21 | ) 22 | 23 | func TestOperations(t *testing.T) { 24 | ctx := context.Background() 25 | c, teardown := makeCluster(ctx, t) 26 | defer teardown() 27 | 28 | t.Logf("Fetching cluster state") 29 | { 30 | s, err := c.O1.ClusterState(context.Background()) 31 | if err != nil { 32 | t.Fatalf("ClusterState: %v", err) 33 | } 34 | t.Logf("%d Acceptors: %+v", len(s.Acceptors), s.Acceptors) 35 | t.Logf("%d Proposers: %+v", len(s.Proposers), s.Proposers) 36 | } 37 | 38 | t.Logf("Growing cluster with each acceptor") 39 | for i, a := range []extension.Acceptor{c.A1, c.A2, c.A3} { 40 | if err := c.O1.GrowCluster(ctx, a); err != nil { 41 | t.Fatalf("GrowCluster(a%d): %v", i+1, err) 42 | } 43 | } 44 | 45 | t.Logf("Listing preparers and accepters for each proposer") 46 | for i, p := range []extension.Proposer{c.P1, c.P2} { 47 | s, err := p.ListPreparers() 48 | if err != nil { 49 | t.Fatalf("p%d ListPreparers: %v", i+1, err) 50 | } 51 | t.Logf("p%d ListPreparers: %v", i+1, s) 52 | s, err = p.ListAccepters() 53 | if err != nil { 54 | t.Fatalf("p%d ListAccepters: %v", i+1, err) 55 | } 56 | t.Logf("p%d ListAccepters: %v", i+1, s) 57 | } 58 | 59 | t.Logf("Dumping cluster state") 60 | { 61 | s, _ := c.O1.ClusterState(context.Background()) 62 | buf, _ := json.MarshalIndent(s, "", " ") 63 | fmt.Printf("%s\n", buf) 64 | } 65 | 66 | t.Logf("Proposing a change") 67 | { 68 | version, value, err := c.U1.CAS(ctx, "foo", 0, []byte("val0")) 69 | if want, have := error(nil), err; want != have { 70 | t.Fatalf("first CAS: want error %v, have %v", want, have) 71 | } 72 | if want, have := uint64(1), version; want != have { 73 | t.Fatalf("first CAS: want version %d, have %d", want, have) 74 | } 75 | if want, have := "val0", string(value); want != have { 76 | t.Fatalf("first CAS: want value %q, have %q", want, have) 77 | } 78 | } 79 | 80 | t.Logf("Doing a read") 81 | { 82 | version, value, err := c.U2.Read(ctx, "foo") 83 | if want, have := error(nil), err; want != have { 84 | t.Fatalf("first Read: want error %v, have %v", want, have) 85 | } 86 | if want, have := uint64(1), version; want != have { 87 | t.Fatalf("first Read: want version %d, have %d", want, have) 88 | } 89 | if want, have := "val0", string(value); want != have { 90 | t.Fatalf("first Read: want value %q, have %q", want, have) 91 | } 92 | } 93 | 94 | t.Logf("Proposing another change") 95 | { 96 | version, value, err := c.U2.CAS(ctx, "foo", 1, []byte("val1")) 97 | if want, have := error(nil), err; want != have { 98 | t.Fatalf("second CAS: want error %v, have %v", want, have) 99 | } 100 | if want, have := uint64(2), version; want != have { 101 | t.Fatalf("second CAS: want version %d, have %d", want, have) 102 | } 103 | if want, have := "val1", string(value); want != have { 104 | t.Fatalf("second CAS: want value %q, have %q", want, have) 105 | } 106 | } 107 | 108 | t.Logf("Proposing a change with a bad version") 109 | { 110 | version, value, err := c.U1.CAS(ctx, "foo", 1, []byte("BAD_VALUE")) 111 | if want, have := error(nil), err; want != have { 112 | t.Fatalf("bad CAS: want error %v, have %v", want, have) 113 | } 114 | if want, have := uint64(2), version; want != have { 115 | t.Fatalf("bad CAS: want version %d, have %d", want, have) 116 | } 117 | if want, have := "val1", string(value); want != have { 118 | t.Fatalf("bad CAS: want value %q, have %q", want, have) 119 | } 120 | } 121 | 122 | t.Logf("Doing a CAS delete") 123 | { 124 | version, value, err := c.U2.CAS(ctx, "foo", 2, []byte{}) 125 | if want, have := error(nil), err; want != have { 126 | t.Fatalf("CAS delete: want error %v, have %v", want, have) 127 | } 128 | if want, have := uint64(3), version; want != have { 129 | t.Fatalf("CAS delete: want version %d, have %d", want, have) 130 | } 131 | if want, have := "", string(value); want != have { 132 | t.Fatalf("CAS delete: want value %q, have %q", want, have) 133 | } 134 | } 135 | 136 | t.Logf("Verifying CAS delete performed GC") 137 | { 138 | for i, reader := range []interface { 139 | Read(context.Context, string) (uint64, []byte, error) 140 | }{ 141 | c.P1, c.P2, c.U1, c.U2, 142 | } { 143 | version, value, err := reader.Read(ctx, "foo") 144 | if want, have := error(nil), err; want != have { 145 | t.Fatalf("verify GC %d: want error %v, have %v", i+1, want, have) 146 | } 147 | if want, have := uint64(0), version; want != have { 148 | t.Fatalf("verify GC %d: want version %d, have %d", i+1, want, have) 149 | } 150 | if want, have := []byte(nil), value; !bytes.Equal(want, have) { 151 | t.Fatalf("verify GC %d: want value %v, have %v", i+1, want, have) 152 | } 153 | } 154 | } 155 | 156 | t.Logf("Making another write for Watch") 157 | { 158 | version, value, err := c.U2.CAS(ctx, "foo", 0, []byte("hello")) 159 | if err != nil { 160 | t.Fatalf("watch CAS: %v", err) 161 | } 162 | if want, have := uint64(1), version; want != have { 163 | t.Fatalf("watch CAS: want version %d, have %d", want, have) 164 | } 165 | if want, have := "hello", string(value); want != have { 166 | t.Fatalf("watch CAS: want value %q, have %q", want, have) 167 | } 168 | } 169 | 170 | t.Logf("Starting Watch on user node") 171 | var ( 172 | values = make(chan []byte, 100) 173 | wctx, wcancel = context.WithCancel(ctx) 174 | done = make(chan error) 175 | ) 176 | { 177 | go func() { 178 | done <- c.U1.Watch(wctx, "foo", values) 179 | }() 180 | } 181 | 182 | t.Logf("Checking initial Watch value") 183 | { 184 | select { 185 | case err := <-done: 186 | t.Fatalf("first recv: Watch error: %v", err) 187 | case value := <-values: 188 | if want, have := "hello", string(value); want != have { 189 | t.Fatalf("first recv: want %q, have %q", want, have) 190 | } 191 | case <-time.After(time.Second): 192 | t.Fatal("first recv: timeout") 193 | } 194 | } 195 | 196 | t.Logf("Making write for Watch") 197 | { 198 | version, value, err := c.U2.CAS(ctx, "foo", 1, []byte("world")) 199 | if err != nil { 200 | t.Fatalf("second CAS: %v", err) 201 | } 202 | if want, have := uint64(2), version; want != have { 203 | t.Fatalf("second CAS: want version %d, have %d", want, have) 204 | } 205 | if want, have := "world", string(value); want != have { 206 | t.Fatalf("second CAS: want value %q, have %q", want, have) 207 | } 208 | } 209 | 210 | t.Logf("Checking second Watch value") 211 | { 212 | select { 213 | case err := <-done: 214 | t.Fatalf("second recv: Watch error: %v", err) 215 | case value := <-values: 216 | if want, have := "world", string(value); want != have { 217 | t.Fatalf("second recv: want %q, have %q", want, have) 218 | } 219 | case <-time.After(time.Second): 220 | t.Fatal("second recv: timeout") 221 | } 222 | } 223 | 224 | t.Logf("Canceling the Watch and waiting for it to finish") 225 | { 226 | wcancel() 227 | <-done 228 | } 229 | } 230 | 231 | type testWriter struct{ t *testing.T } 232 | 233 | func (tw testWriter) Write(p []byte) (int, error) { 234 | tw.t.Logf("%s", string(p)) 235 | return len(p), nil 236 | } 237 | 238 | type testCluster struct { 239 | A1 extension.Acceptor 240 | A2 extension.Acceptor 241 | A3 extension.Acceptor 242 | P1 extension.Proposer 243 | P2 extension.Proposer 244 | O1 extension.OperatorNode 245 | U1 extension.UserNode 246 | U2 extension.UserNode 247 | } 248 | 249 | func makeCluster(ctx context.Context, t *testing.T) (result testCluster, teardown func()) { 250 | var ( 251 | logger = log.NewLogfmtLogger(testWriter{t}) 252 | stack = []func(){} // for teardown 253 | ) 254 | 255 | { 256 | a1Acceptor := protocol.NewMemoryAcceptor("a1", log.With(logger, "acceptor_core", 1)) 257 | a1Handler := httpapi.NewAcceptorServer(a1Acceptor) 258 | a1Server := http.Server{Addr: "127.0.0.1:10011", Handler: a1Handler} 259 | a1Done := make(chan struct{}) 260 | go func() { a1Server.ListenAndServe(); close(a1Done) }() 261 | stack = append(stack, func() { a1Server.Close(); <-a1Done }) 262 | a1Peer, _ := cluster.NewPeer(cluster.Config{ 263 | APIHost: "127.0.0.1", 264 | APIPort: 10011, 265 | ClusterHost: "127.0.0.1", 266 | ClusterPort: 10012, 267 | Type: httpcluster.PeerTypeAcceptor, 268 | InitialPeers: []string{}, 269 | Logger: log.With(logger, "acceptor_peer", 1), 270 | }) 271 | stack = append(stack, func() { a1Peer.Leave(time.Second) }) 272 | result.A1 = httpapi.AcceptorClient{URL: mustParseURL(t, "http://127.0.0.1:10011")} 273 | waitForListener(t, "127.0.0.1:10011") 274 | } 275 | 276 | { 277 | a2Acceptor := protocol.NewMemoryAcceptor("a2", log.With(logger, "acceptor_core", 2)) 278 | a2Handler := httpapi.NewAcceptorServer(a2Acceptor) 279 | a2Server := http.Server{Addr: "127.0.0.1:10021", Handler: a2Handler} 280 | a2Done := make(chan struct{}) 281 | go func() { a2Server.ListenAndServe(); close(a2Done) }() 282 | stack = append(stack, func() { a2Server.Close(); <-a2Done }) 283 | a2Peer, _ := cluster.NewPeer(cluster.Config{ 284 | APIHost: "127.0.0.1", 285 | APIPort: 10021, 286 | ClusterHost: "127.0.0.1", 287 | ClusterPort: 10022, 288 | Type: httpcluster.PeerTypeAcceptor, 289 | InitialPeers: []string{"127.0.0.1:10012"}, 290 | Logger: log.With(logger, "acceptor_peer", 2), 291 | }) 292 | stack = append(stack, func() { a2Peer.Leave(time.Second) }) 293 | result.A2 = httpapi.AcceptorClient{URL: mustParseURL(t, "http://127.0.0.1:10021")} 294 | waitForListener(t, "127.0.0.1:10021") 295 | } 296 | 297 | { 298 | a3Acceptor := protocol.NewMemoryAcceptor("a3", log.With(logger, "acceptor_core", 3)) 299 | a3Handler := httpapi.NewAcceptorServer(a3Acceptor) 300 | a3Server := http.Server{Addr: "127.0.0.1:10031", Handler: a3Handler} 301 | a3Done := make(chan struct{}) 302 | go func() { a3Server.ListenAndServe(); close(a3Done) }() 303 | stack = append(stack, func() { a3Server.Close(); <-a3Done }) 304 | a3Peer, _ := cluster.NewPeer(cluster.Config{ 305 | APIHost: "127.0.0.1", 306 | APIPort: 10031, 307 | ClusterHost: "127.0.0.1", 308 | ClusterPort: 10032, 309 | Type: httpcluster.PeerTypeAcceptor, 310 | InitialPeers: []string{"127.0.0.1:10012", "127.0.0.1:10022"}, 311 | Logger: log.With(logger, "acceptor_peer", 3), 312 | }) 313 | stack = append(stack, func() { a3Peer.Leave(time.Second) }) 314 | result.A3 = httpapi.AcceptorClient{URL: mustParseURL(t, "http://127.0.0.1:10031")} 315 | waitForListener(t, "127.0.0.1:10031") 316 | } 317 | 318 | { 319 | p1Proposer := protocol.NewMemoryProposer("p1", log.With(logger, "proposer_core", 1)) 320 | p1Handler := httpapi.NewProposerServer(p1Proposer) 321 | p1Server := http.Server{Addr: "127.0.0.1:10041", Handler: p1Handler} 322 | p1Done := make(chan struct{}) 323 | go func() { p1Server.ListenAndServe(); close(p1Done) }() 324 | stack = append(stack, func() { p1Server.Close(); <-p1Done }) 325 | p1Peer, _ := cluster.NewPeer(cluster.Config{ 326 | APIHost: "127.0.0.1", 327 | APIPort: 10041, 328 | ClusterHost: "127.0.0.1", 329 | ClusterPort: 10042, 330 | Type: httpcluster.PeerTypeProposer, 331 | InitialPeers: []string{"127.0.0.1:10012", "127.0.0.1:10022", "127.0.0.1:10032"}, 332 | Logger: log.With(logger, "proposer_peer", 1), 333 | }) 334 | stack = append(stack, func() { p1Peer.Leave(time.Second) }) 335 | result.P1 = httpapi.ProposerClient{URL: mustParseURL(t, "http://127.0.0.1:10041")} 336 | waitForListener(t, "127.0.0.1:10041") 337 | } 338 | 339 | { 340 | p2Proposer := protocol.NewMemoryProposer("p2", log.With(logger, "proposer_core", 2)) 341 | p2Handler := httpapi.NewProposerServer(p2Proposer) 342 | p2Server := http.Server{Addr: "127.0.0.1:10051", Handler: p2Handler} 343 | p2Done := make(chan struct{}) 344 | go func() { p2Server.ListenAndServe(); close(p2Done) }() 345 | stack = append(stack, func() { p2Server.Close(); <-p2Done }) 346 | p2Peer, _ := cluster.NewPeer(cluster.Config{ 347 | APIHost: "127.0.0.1", 348 | APIPort: 10051, 349 | ClusterHost: "127.0.0.1", 350 | ClusterPort: 10052, 351 | Type: httpcluster.PeerTypeProposer, 352 | InitialPeers: []string{"127.0.0.1:10012", "127.0.0.1:10022", "127.0.0.1:10032", "127.0.0.1:10042"}, 353 | Logger: log.With(logger, "proposer_peer", 2), 354 | }) 355 | stack = append(stack, func() { p2Peer.Leave(time.Second) }) 356 | result.P2 = httpapi.ProposerClient{URL: mustParseURL(t, "http://127.0.0.1:10051")} 357 | waitForListener(t, "127.0.0.1:10051") 358 | } 359 | 360 | { 361 | // Construct the peer first, because the ClusterOperator needs it to 362 | // have access to extension.Cluster functionality. We have a little race 363 | // where we're advertising ourself in the cluster before our API is 364 | // listening. That's not really a problem. 365 | o1Peer, _ := cluster.NewPeer(cluster.Config{ 366 | APIHost: "127.0.0.1", 367 | APIPort: 10061, 368 | ClusterHost: "127.0.0.1", 369 | ClusterPort: 10062, 370 | Type: httpcluster.PeerTypeOperatorNode, 371 | InitialPeers: []string{"127.0.0.1:10012", "127.0.0.1:10022", "127.0.0.1:10032", "127.0.0.1:10042", "127.0.0.1:10052"}, 372 | Logger: log.With(logger, "operator_peer", 1), 373 | }) 374 | o1Cluster := httpcluster.Peer{Peer: o1Peer} 375 | o1OperatorNode := extension.NewClusterOperator("o1", o1Cluster, log.With(logger, "operator_core", 1)) 376 | o1Handler := httpapi.NewOperatorNodeServer(o1OperatorNode) 377 | o1Server := http.Server{Addr: "127.0.0.1:10061", Handler: o1Handler} 378 | o1Done := make(chan struct{}) 379 | go func() { o1Server.ListenAndServe(); close(o1Done) }() 380 | stack = append(stack, func() { o1Server.Close(); <-o1Done }) 381 | result.O1 = httpapi.OperatorNodeClient{URL: mustParseURL(t, "http://127.0.0.1:10061")} 382 | waitForListener(t, "127.0.0.1:10061") 383 | } 384 | 385 | { 386 | // Similarly, peer first, for access to the cluster. 387 | u1Peer, _ := cluster.NewPeer(cluster.Config{ 388 | APIHost: "127.0.0.1", 389 | APIPort: 10071, 390 | ClusterHost: "127.0.0.1", 391 | ClusterPort: 10072, 392 | Type: httpcluster.PeerTypeUserNode, 393 | InitialPeers: []string{"127.0.0.1:10012", "127.0.0.1:10022", "127.0.0.1:10032", "127.0.0.1:10042", "127.0.0.1:10052", "127.0.0.1:10062"}, 394 | Logger: log.With(logger, "user_peer", 1), 395 | }) 396 | u1Cluster := httpcluster.Peer{Peer: u1Peer} 397 | u1UserNode := extension.NewClusterUser("u1", u1Cluster, log.With(logger, "user_core", 1)) 398 | u1Handler := httpapi.NewUserNodeServer(u1UserNode) 399 | u1Server := http.Server{Addr: "127.0.0.1:10071", Handler: u1Handler} 400 | u1Done := make(chan struct{}) 401 | go func() { u1Server.ListenAndServe(); close(u1Done) }() 402 | stack = append(stack, func() { u1Server.Close(); <-u1Done }) 403 | result.U1 = httpapi.UserNodeClient{URL: mustParseURL(t, "http://127.0.0.1:10071")} 404 | waitForListener(t, "127.0.0.1:10071") 405 | } 406 | 407 | { 408 | u2Peer, _ := cluster.NewPeer(cluster.Config{ 409 | APIHost: "127.0.0.1", 410 | APIPort: 10081, 411 | ClusterHost: "127.0.0.1", 412 | ClusterPort: 10082, 413 | Type: httpcluster.PeerTypeUserNode, 414 | InitialPeers: []string{"127.0.0.1:10012", "127.0.0.1:10022", "127.0.0.1:10032", "127.0.0.1:10042", "127.0.0.1:10052", "127.0.0.1:10062", "127.0.0.1:10072"}, 415 | Logger: log.With(logger, "user_peer", 2), 416 | }) 417 | u2Cluster := httpcluster.Peer{Peer: u2Peer} 418 | u2UserNode := extension.NewClusterUser("u2", u2Cluster, log.With(logger, "user_core", 2)) 419 | u2Handler := httpapi.NewUserNodeServer(u2UserNode) 420 | u2Server := http.Server{Addr: "127.0.0.1:10081", Handler: u2Handler} 421 | u2Done := make(chan struct{}) 422 | go func() { u2Server.ListenAndServe(); close(u2Done) }() 423 | stack = append(stack, func() { u2Server.Close(); <-u2Done }) 424 | result.U2 = httpapi.UserNodeClient{URL: mustParseURL(t, "http://127.0.0.1:10081")} 425 | waitForListener(t, "127.0.0.1:10071") 426 | } 427 | 428 | teardown = func() { 429 | for i := len(stack) - 1; i >= 0; i-- { 430 | stack[i]() 431 | } 432 | } 433 | 434 | return result, teardown 435 | } 436 | 437 | func mustParseURL(t *testing.T, s string) *url.URL { 438 | t.Helper() 439 | u, err := url.Parse(s) 440 | if err != nil { 441 | t.Fatal(err) 442 | } 443 | return u 444 | } 445 | 446 | func waitForListener(t *testing.T, addr string) { 447 | var ( 448 | deadline = time.Now().Add(10 * time.Second) 449 | backoff = 25 * time.Millisecond 450 | ) 451 | for time.Now().Before(deadline) { 452 | conn, err := net.Dial("tcp", addr) 453 | if err == nil { 454 | conn.Close() 455 | return 456 | } 457 | t.Logf("waitForListener(%s): %v", addr, err) 458 | backoff *= 2 459 | if backoff > time.Second { 460 | backoff = time.Second 461 | } 462 | time.Sleep(backoff) 463 | } 464 | t.Fatalf("%s never came up", addr) 465 | } 466 | -------------------------------------------------------------------------------- /cmd/caspaxos-http/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | func main() { 4 | // TODO(pb) 5 | } 6 | -------------------------------------------------------------------------------- /extension/acceptor.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/peterbourgon/caspaxos/protocol" 7 | ) 8 | 9 | // Acceptor extends the core protocol acceptor. 10 | type Acceptor interface { 11 | protocol.Addresser 12 | protocol.Preparer 13 | protocol.Accepter 14 | protocol.RejectRemover 15 | ValueWatcher 16 | } 17 | 18 | // ValueWatcher wraps protocol.StateWatcher. Implementations must perform 19 | // (Version, Value) tuple deserialization of the received states to convert 20 | // them to values before sending them on. 21 | type ValueWatcher interface { 22 | Watch(ctx context.Context, key string, values chan<- []byte) error 23 | } 24 | -------------------------------------------------------------------------------- /extension/cluster.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import "context" 4 | 5 | // Cluster models the methods that operator and user nodes need from a cluster. 6 | type Cluster interface { 7 | Acceptors(context.Context) ([]Acceptor, error) 8 | Proposers(context.Context) ([]Proposer, error) 9 | OperatorNodes(context.Context) ([]OperatorNode, error) 10 | UserNodes(context.Context) ([]UserNode, error) 11 | } 12 | -------------------------------------------------------------------------------- /extension/doc.go: -------------------------------------------------------------------------------- 1 | // Package extension contains helper types, beyond the core CASPaxos protocol. 2 | // It's meant to be a bridge or adapter, towards a usable data system. 3 | package extension 4 | -------------------------------------------------------------------------------- /extension/operator_node.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/go-kit/kit/log" 7 | 8 | "github.com/peterbourgon/caspaxos/protocol" 9 | ) 10 | 11 | // OperatorNode models the functionality that a cluster admin needs. 12 | type OperatorNode interface { 13 | protocol.Addresser 14 | ClusterState(ctx context.Context) (ClusterState, error) 15 | GrowCluster(ctx context.Context, target protocol.Acceptor) error 16 | ShrinkCluster(ctx context.Context, target protocol.Acceptor) error 17 | GarbageCollect(ctx context.Context, key string) error 18 | } 19 | 20 | // ClusterState captures the current state of the cluster. 21 | type ClusterState struct { 22 | Acceptors []string `json:"acceptors"` 23 | Proposers []ProposerState `json:"proposers"` 24 | OperatorNodes []string `json:"operator_nodes"` 25 | UserNodes []string `json:"user_nodes"` 26 | } 27 | 28 | // ProposerState captures the current state of a proposer in the cluster. 29 | type ProposerState struct { 30 | Address string `json:"address"` 31 | Preparers []string `json:"preparers"` 32 | Accepters []string `json:"accepters"` 33 | } 34 | 35 | // 36 | // 37 | // 38 | 39 | // ClusterOperator provides OperatorNode methods over a cluster. 40 | type ClusterOperator struct { 41 | address string 42 | cluster Cluster 43 | logger log.Logger 44 | } 45 | 46 | var _ OperatorNode = (*ClusterOperator)(nil) 47 | 48 | // NewClusterOperator returns a usable ClusterOperator, which is an 49 | // implementation of the operator node interface that wraps the passed cluster. 50 | // It should be uniquely identified by address. 51 | func NewClusterOperator(address string, c Cluster, logger log.Logger) *ClusterOperator { 52 | return &ClusterOperator{ 53 | address: address, 54 | cluster: c, 55 | logger: logger, 56 | } 57 | } 58 | 59 | // Address implements OperatorNode. 60 | func (op ClusterOperator) Address() string { 61 | return op.address 62 | } 63 | 64 | // GrowCluster implements OperatorNode. 65 | func (op ClusterOperator) GrowCluster(ctx context.Context, target protocol.Acceptor) error { 66 | proposers, err := op.cluster.Proposers(ctx) 67 | if err != nil { 68 | return err 69 | } 70 | confChangers := make([]protocol.ConfigurationChanger, len(proposers)) 71 | for i := range proposers { 72 | confChangers[i] = proposers[i] 73 | } 74 | return protocol.GrowCluster(ctx, target, confChangers) 75 | } 76 | 77 | // ShrinkCluster implements OperatorNode. 78 | func (op ClusterOperator) ShrinkCluster(ctx context.Context, target protocol.Acceptor) error { 79 | proposers, err := op.cluster.Proposers(ctx) 80 | if err != nil { 81 | return err 82 | } 83 | confChangers := make([]protocol.ConfigurationChanger, len(proposers)) 84 | for i := range proposers { 85 | confChangers[i] = proposers[i] 86 | } 87 | return protocol.ShrinkCluster(ctx, target, confChangers) 88 | } 89 | 90 | // GarbageCollect implements OperatorNode. 91 | func (op ClusterOperator) GarbageCollect(ctx context.Context, key string) error { 92 | proposers, err := op.cluster.Proposers(ctx) 93 | if err != nil { 94 | return err 95 | } 96 | acceptors, err := op.cluster.Acceptors(ctx) 97 | if err != nil { 98 | return err 99 | } 100 | fastForwarders := make([]protocol.FastForwarder, len(proposers)) 101 | for i := range proposers { 102 | fastForwarders[i] = proposers[i] 103 | } 104 | rejectRemovers := make([]protocol.RejectRemover, len(acceptors)) 105 | for i := range acceptors { 106 | rejectRemovers[i] = acceptors[i] 107 | } 108 | return protocol.GarbageCollect(ctx, key, fastForwarders, rejectRemovers, op.logger) 109 | } 110 | 111 | // ClusterState implements OperatorNode. 112 | func (op ClusterOperator) ClusterState(ctx context.Context) (s ClusterState, err error) { 113 | acceptors, err := op.cluster.Acceptors(ctx) 114 | if err != nil { 115 | return s, err 116 | } 117 | for _, a := range acceptors { 118 | s.Acceptors = append(s.Acceptors, a.Address()) 119 | } 120 | s.Acceptors = denil(s.Acceptors) 121 | 122 | proposers, err := op.cluster.Proposers(ctx) 123 | if err != nil { 124 | return s, err 125 | } 126 | for _, p := range proposers { 127 | preparers, err := p.ListPreparers() 128 | if err != nil { 129 | return s, err 130 | } 131 | accepters, err := p.ListAccepters() 132 | if err != nil { 133 | return s, err 134 | } 135 | s.Proposers = append(s.Proposers, ProposerState{ 136 | Address: p.Address(), 137 | Preparers: denil(preparers), 138 | Accepters: denil(accepters), 139 | }) 140 | } 141 | 142 | operatorNodes, err := op.cluster.OperatorNodes(ctx) 143 | if err != nil { 144 | return s, err 145 | } 146 | for _, operatorNode := range operatorNodes { 147 | s.OperatorNodes = append(s.OperatorNodes, operatorNode.Address()) 148 | } 149 | s.OperatorNodes = denil(s.OperatorNodes) 150 | 151 | userNodes, err := op.cluster.UserNodes(ctx) 152 | if err != nil { 153 | return s, err 154 | } 155 | for _, userNode := range userNodes { 156 | s.UserNodes = append(s.UserNodes, userNode.Address()) 157 | } 158 | s.UserNodes = denil(s.UserNodes) 159 | 160 | return s, nil 161 | } 162 | 163 | func denil(s []string) []string { 164 | if s == nil { 165 | s = []string{} 166 | } 167 | return s 168 | } 169 | -------------------------------------------------------------------------------- /extension/proposer.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/peterbourgon/caspaxos/protocol" 8 | ) 9 | 10 | // Proposer models serializable methods of a proposer. Notably, the Watch method 11 | // has the same signature but returns values instead of states; and the Propose 12 | // method is dropped in favor of Read and CAS. The CAS protocol is taken from 13 | // the paper, with a (Version, Value) tuple. 14 | type Proposer interface { 15 | protocol.Addresser 16 | Read(ctx context.Context, key string) (version uint64, value []byte, err error) 17 | CAS(ctx context.Context, key string, currentVersion uint64, nextValue []byte) (version uint64, value []byte, err error) 18 | protocol.ConfigurationChanger 19 | protocol.FastForwarder 20 | protocol.AcceptorLister 21 | } 22 | 23 | // CASError indicates a conflict during CAS, likely a version conflict. 24 | type CASError struct{ Err error } 25 | 26 | // Error implements the error interface. 27 | func (e CASError) Error() string { 28 | return fmt.Sprintf("CAS error: %v", e.Err) 29 | } 30 | -------------------------------------------------------------------------------- /extension/user_node.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "math/rand" 7 | 8 | "github.com/go-kit/kit/log" 9 | "github.com/pkg/errors" 10 | 11 | "github.com/peterbourgon/caspaxos/protocol" 12 | ) 13 | 14 | // UserNode models the user API supported by the cluster. 15 | type UserNode interface { 16 | protocol.Addresser 17 | Read(ctx context.Context, key string) (version uint64, value []byte, err error) 18 | CAS(ctx context.Context, key string, currentVersion uint64, nextValue []byte) (version uint64, value []byte, err error) 19 | Watch(ctx context.Context, key string, values chan<- []byte) error // TODO(pb): correct API? 20 | } 21 | 22 | // ClusterUser provides UserNode methods over a cluster. 23 | type ClusterUser struct { 24 | address string 25 | cluster Cluster 26 | logger log.Logger 27 | } 28 | 29 | var _ UserNode = (*ClusterUser)(nil) 30 | 31 | // NewClusterUser returns a usable ClusterUser, which is an implementation of 32 | // the user node interface that wraps the passed cluster. It should be uniquely 33 | // identified by address. 34 | func NewClusterUser(address string, c Cluster, logger log.Logger) *ClusterUser { 35 | return &ClusterUser{ 36 | address: address, 37 | cluster: c, 38 | logger: logger, 39 | } 40 | } 41 | 42 | // Address implements UserNode. 43 | func (u ClusterUser) Address() string { 44 | return u.address 45 | } 46 | 47 | // Read implements UserNode. 48 | func (u ClusterUser) Read(ctx context.Context, key string) (version uint64, value []byte, err error) { 49 | proposers, err := u.cluster.Proposers(ctx) 50 | if err != nil { 51 | return version, value, err 52 | } 53 | if len(proposers) <= 0 { 54 | return version, value, errors.New("no proposers in cluster") 55 | } 56 | 57 | proposer := proposers[rand.Intn(len(proposers))] 58 | return proposer.Read(ctx, key) 59 | } 60 | 61 | // CAS implements UserNode. 62 | func (u ClusterUser) CAS(ctx context.Context, key string, currentVersion uint64, nextValue []byte) (version uint64, value []byte, err error) { 63 | proposers, err := u.cluster.Proposers(ctx) 64 | if err != nil { 65 | return version, value, err 66 | } 67 | if len(proposers) <= 0 { 68 | return version, value, errors.New("no proposers in cluster") 69 | } 70 | 71 | proposer := proposers[rand.Intn(len(proposers))] 72 | version, value, err = proposer.CAS(ctx, key, currentVersion, nextValue) 73 | if err != nil { 74 | return version, value, err 75 | } 76 | 77 | var ( 78 | worked = bytes.Equal(value, nextValue) 79 | delete = worked && len(value) <= 0 80 | ) 81 | if !worked || !delete { 82 | return version, value, err // regular exit 83 | } 84 | 85 | // The CAS worked, and it was a delete. 86 | // Assume we want to trigger a GC. TODO(pb): maybe move to explicit method? 87 | // Do the GC synchronously. TODO(pb): probably do this in the background? 88 | operators, err := u.cluster.OperatorNodes(ctx) 89 | if err != nil { 90 | return version, value, errors.Wrap(err, "GC failed") 91 | } 92 | if len(operators) <= 0 { 93 | return version, value, errors.New("GC failed: no operators in cluster") 94 | } 95 | operator := operators[rand.Intn(len(operators))] 96 | if err := operator.GarbageCollect(ctx, key); err != nil { 97 | return version, value, errors.Wrap(err, "GC failed") 98 | } 99 | 100 | // All good. 101 | return version, value, nil 102 | } 103 | 104 | // Watch implements UserNode. 105 | func (u ClusterUser) Watch(ctx context.Context, key string, values chan<- []byte) error { 106 | acceptors, err := u.cluster.Acceptors(ctx) 107 | if err != nil { 108 | return err 109 | } 110 | if len(acceptors) <= 0 { 111 | return errors.New("no acceptors in cluster") 112 | } 113 | acceptor := acceptors[rand.Intn(len(acceptors))] 114 | return acceptor.Watch(ctx, key, values) 115 | } 116 | -------------------------------------------------------------------------------- /httpapi/core_test.go: -------------------------------------------------------------------------------- 1 | package httpapi 2 | 3 | import ( 4 | "context" 5 | "math/rand" 6 | "net/http/httptest" 7 | "net/url" 8 | "testing" 9 | 10 | "github.com/go-kit/kit/log" 11 | 12 | "github.com/peterbourgon/caspaxos/extension" 13 | "github.com/peterbourgon/caspaxos/protocol" 14 | ) 15 | 16 | // TODO(pb): these tests are redundant with cmd/caspaxos-http tests; remove 17 | 18 | func TestProposersAndAcceptors(t *testing.T) { 19 | // Build the cluster. 20 | var ( 21 | logger = log.NewLogfmtLogger(testWriter{t}) 22 | a1 = protocol.NewMemoryAcceptor("1", log.With(logger, "a", 1)) 23 | a2 = protocol.NewMemoryAcceptor("2", log.With(logger, "a", 2)) 24 | a3 = protocol.NewMemoryAcceptor("3", log.With(logger, "a", 3)) 25 | p1 = protocol.NewMemoryProposer("1", log.With(logger, "p", 1)) 26 | p2 = protocol.NewMemoryProposer("2", log.With(logger, "p", 2)) 27 | p3 = protocol.NewMemoryProposer("3", log.With(logger, "p", 3)) 28 | ) 29 | 30 | // Wrap with HTTP adapters. 31 | var ( 32 | ha1 = NewAcceptorServer(a1) 33 | ha2 = NewAcceptorServer(a2) 34 | ha3 = NewAcceptorServer(a3) 35 | hp1 = NewProposerServer(p1) 36 | hp2 = NewProposerServer(p2) 37 | hp3 = NewProposerServer(p3) 38 | ) 39 | 40 | // Mount the HTTP adapters in servers. 41 | var ( 42 | as1 = httptest.NewServer(ha1) 43 | as2 = httptest.NewServer(ha2) 44 | as3 = httptest.NewServer(ha3) 45 | ps1 = httptest.NewServer(hp1) 46 | ps2 = httptest.NewServer(hp2) 47 | ps3 = httptest.NewServer(hp3) 48 | ) 49 | defer func() { 50 | as1.Close() 51 | as2.Close() 52 | as3.Close() 53 | ps1.Close() 54 | ps2.Close() 55 | ps3.Close() 56 | }() 57 | 58 | // Wrap with HTTP clients. 59 | var ( 60 | ac1 = AcceptorClient{URL: mustParseURL(t, as1.URL)} 61 | ac2 = AcceptorClient{URL: mustParseURL(t, as2.URL)} 62 | ac3 = AcceptorClient{URL: mustParseURL(t, as3.URL)} 63 | pc1 = ProposerClient{URL: mustParseURL(t, ps1.URL)} 64 | pc2 = ProposerClient{URL: mustParseURL(t, ps2.URL)} 65 | pc3 = ProposerClient{URL: mustParseURL(t, ps3.URL)} 66 | ) 67 | 68 | // HTTP client interface to HTTP server wrappers for proposers. 69 | var ( 70 | confChangers = []protocol.ConfigurationChanger{pc1, pc2, pc3} 71 | extProposers = []extension.Proposer{pc1, pc2, pc3} 72 | ) 73 | 74 | // Initialize. 75 | ctx := context.Background() 76 | if err := protocol.GrowCluster(ctx, ac1, confChangers); err != nil { 77 | t.Fatalf("first GrowCluster: %v", err) 78 | } 79 | if err := protocol.GrowCluster(ctx, ac2, confChangers); err != nil { 80 | t.Fatalf("second GrowCluster: %v", err) 81 | } 82 | if err := protocol.GrowCluster(ctx, ac3, confChangers); err != nil { 83 | t.Fatalf("third GrowCluster: %v", err) 84 | } 85 | 86 | // Do a CAS write. 87 | key := "foo" 88 | p := extProposers[rand.Intn(len(extProposers))] 89 | version, value, err := p.CAS(ctx, key, 0, []byte("val0")) 90 | if want, have := error(nil), err; want != have { 91 | t.Fatalf("CAS(%s): want error %v, have %v", key, want, have) 92 | } 93 | if want, have := uint64(1), version; want != have { 94 | t.Fatalf("CAS(%s): want version %d, have %d", key, want, have) 95 | } 96 | if want, have := "val0", string(value); want != have { 97 | t.Fatalf("CAS(%s): want value %q, have %q", key, want, have) 98 | } 99 | 100 | // Do some reads. 101 | for i, p := range extProposers { 102 | version, value, err := p.Read(ctx, key) 103 | if want, have := error(nil), err; want != have { 104 | t.Fatalf("Read(%s) %d: want error %v, have %v", key, i+1, want, have) 105 | } 106 | if want, have := uint64(1), version; want != have { 107 | t.Fatalf("Read(%s) %d: want version %d, have %d", key, i+1, want, have) 108 | } 109 | if want, have := "val0", string(value); want != have { 110 | t.Fatalf("Read(%s) %d: want value %q, have %q", key, i+1, want, have) 111 | } 112 | } 113 | 114 | // Make another write. 115 | p = extProposers[rand.Intn(len(extProposers))] 116 | version, value, err = p.CAS(ctx, key, 1, []byte("val1")) 117 | if want, have := error(nil), err; want != have { 118 | t.Fatalf("CAS(%s): want error %v, have %v", key, want, have) 119 | } 120 | if want, have := uint64(2), version; want != have { 121 | t.Fatalf("CAS(%s): want version %d, have %d", key, want, have) 122 | } 123 | if want, have := "val1", string(value); want != have { 124 | t.Fatalf("CAS(%s): want value %q, have %q", key, want, have) 125 | } 126 | } 127 | 128 | type testWriter struct{ t *testing.T } 129 | 130 | func (tw testWriter) Write(p []byte) (int, error) { 131 | tw.t.Logf("%s", string(p)) 132 | return len(p), nil 133 | } 134 | 135 | func mustParseURL(t *testing.T, s string) *url.URL { 136 | t.Helper() 137 | u, err := url.Parse(s) 138 | if err != nil { 139 | t.Fatal(err) 140 | } 141 | return u 142 | } 143 | -------------------------------------------------------------------------------- /httpapi/doc.go: -------------------------------------------------------------------------------- 1 | // Package httpapi provides HTTP interfaces to CASPaxos domain objects. 2 | // Generally it wraps types from package extension rather than package protocol. 3 | package httpapi 4 | -------------------------------------------------------------------------------- /httpapi/helpers.go: -------------------------------------------------------------------------------- 1 | package httpapi 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "regexp" 9 | "strconv" 10 | 11 | "github.com/peterbourgon/caspaxos/protocol" 12 | ) 13 | 14 | func setAge(h http.Header, age protocol.Age) { 15 | h.Set(ageHeaderKey, age.String()) 16 | } 17 | 18 | func setAges(h http.Header, ages []protocol.Age) { 19 | for _, age := range ages { 20 | h.Add(ageHeaderKey, age.String()) 21 | } 22 | } 23 | 24 | func setBallot(h http.Header, b protocol.Ballot) { 25 | h.Set(ballotHeaderKey, b.String()) 26 | } 27 | 28 | func getAge(h http.Header) protocol.Age { 29 | counter, id := getHeaderPair(h.Get(ageHeaderKey), ageRe) 30 | return protocol.Age{Counter: counter, ID: id} 31 | } 32 | 33 | func getAges(h http.Header) (ages []protocol.Age) { 34 | for _, s := range h[ageHeaderKey] { 35 | counter, id := getHeaderPair(s, ageRe) 36 | ages = append(ages, protocol.Age{Counter: counter, ID: id}) 37 | } 38 | return ages 39 | } 40 | 41 | func getBallot(h http.Header) protocol.Ballot { 42 | s := h.Get(ballotHeaderKey) 43 | if s == "ø" { 44 | return protocol.Ballot{} 45 | } 46 | counter, id := getHeaderPair(s, ballotRe) 47 | return protocol.Ballot{Counter: counter, ID: id} 48 | } 49 | 50 | func getHeaderPair(s string, re *regexp.Regexp) (uint64, string) { 51 | matches := re.FindAllStringSubmatch(s, 1) 52 | if len(matches) != 1 { 53 | panic("bad input: '" + s + "' (regex: " + re.String() + ")") 54 | } 55 | if len(matches[0]) != 3 { 56 | panic("bad input: '" + s + "' (regex: " + re.String() + ")") 57 | } 58 | counterstr, id := matches[0][1], matches[0][2] 59 | counter, _ := strconv.ParseUint(counterstr, 10, 64) 60 | return counter, id 61 | } 62 | 63 | func makeVersionValue(version uint64, value []byte) (state []byte) { 64 | state = make([]byte, 8+len(value)) 65 | binary.LittleEndian.PutUint64(state[:8], version) 66 | copy(state[8:], value) 67 | return state 68 | } 69 | 70 | func parseVersionValue(state []byte) (version uint64, value []byte, err error) { 71 | if len(state) == 0 { 72 | return version, value, nil 73 | } 74 | if len(state) < 8 { 75 | return version, value, errors.New("state slice is too small") 76 | } 77 | version = binary.LittleEndian.Uint64(state[:8]) 78 | value = state[8:] 79 | return version, value, nil 80 | } 81 | 82 | func setVersion(h http.Header, version uint64) { 83 | h.Set(versionHeaderKey, fmt.Sprint(version)) 84 | } 85 | 86 | func getVersion(h http.Header) uint64 { 87 | version, _ := strconv.ParseUint(h.Get(versionHeaderKey), 10, 64) 88 | return version 89 | } 90 | -------------------------------------------------------------------------------- /httpapi/http_acceptor.go: -------------------------------------------------------------------------------- 1 | package httpapi 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "net/url" 10 | "regexp" 11 | "strings" 12 | "time" 13 | 14 | "github.com/gorilla/mux" 15 | "github.com/pkg/errors" 16 | 17 | "github.com/peterbourgon/caspaxos/extension" 18 | "github.com/peterbourgon/caspaxos/internal/eventsource" 19 | "github.com/peterbourgon/caspaxos/protocol" 20 | ) 21 | 22 | // AcceptorServer wraps a core protocol.Acceptor, but provides the 23 | // extension.Acceptor methodset over HTTP. 24 | type AcceptorServer struct { 25 | acceptor protocol.Acceptor 26 | *mux.Router 27 | } 28 | 29 | var _ http.Handler = (*AcceptorServer)(nil) 30 | 31 | const ( 32 | ageHeaderKey = "X-Caspaxos-Age" 33 | ageHeaderRegex = "^([0-9]+):(.*)$" 34 | ballotHeaderKey = "X-Caspaxos-Ballot" 35 | ballotHeaderRegex = "^([0-9]+)/(.*)$" 36 | versionHeaderKey = "X-Caspaxos-Ext-Version" 37 | ) 38 | 39 | var ( 40 | ageRe = regexp.MustCompile(ageHeaderRegex) 41 | ballotRe = regexp.MustCompile(ballotHeaderRegex) 42 | ) 43 | 44 | // NewAcceptorServer returns a usable AcceptorServer wrapping the passed acceptor. 45 | func NewAcceptorServer(acceptor protocol.Acceptor) *AcceptorServer { 46 | as := &AcceptorServer{ 47 | acceptor: acceptor, 48 | } 49 | r := mux.NewRouter() 50 | { 51 | r.StrictSlash(true) 52 | r.Methods("POST").Path("/prepare/{key}").HeadersRegexp(ageHeaderKey, ageHeaderRegex, ballotHeaderKey, ballotHeaderRegex).HandlerFunc(as.handlePrepare) 53 | r.Methods("POST").Path("/accept/{key}").HeadersRegexp(ageHeaderKey, ageHeaderRegex, ballotHeaderKey, ballotHeaderRegex).HandlerFunc(as.handleAccept) 54 | r.Methods("POST").Path("/reject-by-age").HeadersRegexp(ageHeaderKey, ageHeaderRegex).HandlerFunc(as.handleRejectByAge) 55 | r.Methods("POST").Path("/remove-if-equal/{key}").HandlerFunc(as.handleRemoveIfEqual) 56 | r.Methods("POST").Path("/watch/{key}").HandlerFunc(as.handleWatch) 57 | } 58 | as.Router = r 59 | return as 60 | } 61 | 62 | // Prepare(ctx context.Context, key string, age Age, b Ballot) (value []byte, current Ballot, err error) 63 | func (as *AcceptorServer) handlePrepare(w http.ResponseWriter, r *http.Request) { 64 | var ( 65 | key, _ = url.PathUnescape(mux.Vars(r)["key"]) 66 | age = getAge(r.Header) 67 | b = getBallot(r.Header) 68 | ) 69 | 70 | state, ballot, err := as.acceptor.Prepare(r.Context(), key, age, b) 71 | setBallot(w.Header(), ballot) 72 | if err != nil { 73 | http.Error(w, err.Error(), http.StatusInternalServerError) 74 | return 75 | } 76 | w.Write(state) 77 | } 78 | 79 | // Accept(ctx context.Context, key string, age Age, b Ballot, value []byte) error 80 | func (as *AcceptorServer) handleAccept(w http.ResponseWriter, r *http.Request) { 81 | var ( 82 | key, _ = url.PathUnescape(mux.Vars(r)["key"]) 83 | age = getAge(r.Header) 84 | b = getBallot(r.Header) 85 | ) 86 | 87 | state, err := ioutil.ReadAll(r.Body) 88 | if err != nil { 89 | http.Error(w, err.Error(), http.StatusBadRequest) 90 | return 91 | } 92 | 93 | if err := as.acceptor.Accept(r.Context(), key, age, b, state); err != nil { 94 | http.Error(w, err.Error(), http.StatusInternalServerError) 95 | return 96 | } 97 | 98 | fmt.Fprintln(w, "OK") 99 | } 100 | 101 | // RejectByAge(ctx context.Context, ages []Age) error 102 | func (as *AcceptorServer) handleRejectByAge(w http.ResponseWriter, r *http.Request) { 103 | ages := getAges(r.Header) 104 | if err := as.acceptor.RejectByAge(r.Context(), ages); err != nil { 105 | http.Error(w, err.Error(), http.StatusInternalServerError) 106 | return 107 | } 108 | 109 | fmt.Fprintln(w, "OK") 110 | } 111 | 112 | // RemoveIfEqual(ctx context.Context, key string, state []byte) error 113 | func (as *AcceptorServer) handleRemoveIfEqual(w http.ResponseWriter, r *http.Request) { 114 | key, _ := url.PathUnescape(mux.Vars(r)["key"]) 115 | 116 | state, err := ioutil.ReadAll(r.Body) 117 | if err != nil { 118 | http.Error(w, err.Error(), http.StatusBadRequest) 119 | return 120 | } 121 | 122 | if err := as.acceptor.RemoveIfEqual(r.Context(), key, state); err != nil { 123 | http.Error(w, err.Error(), http.StatusInternalServerError) 124 | return 125 | } 126 | 127 | fmt.Fprintln(w, "OK") 128 | } 129 | 130 | // Watch(ctx context.Context, key string, values chan<- []byte) error 131 | func (as *AcceptorServer) handleWatch(w http.ResponseWriter, r *http.Request) { 132 | var ( 133 | key, _ = url.PathUnescape(mux.Vars(r)["key"]) 134 | states = make(chan []byte) 135 | errs = make(chan error) 136 | enc = eventsource.NewEncoder(w) 137 | ctx, cancel = context.WithCancel(r.Context()) 138 | ) 139 | 140 | go func() { 141 | errs <- as.acceptor.Watch(ctx, key, states) 142 | }() 143 | 144 | // via eventsource.Handler.ServeHTTP 145 | w.Header().Set("Cache-Control", "no-cache") 146 | w.Header().Set("Vary", "Accept") 147 | w.Header().Set("Content-Type", "text/event-stream") 148 | w.WriteHeader(http.StatusOK) 149 | 150 | for { 151 | select { 152 | case state := <-states: 153 | _, value, err := parseVersionValue(state) 154 | if err != nil { 155 | cancel() // kill the watcher goroutine 156 | <-errs // wait for it to return 157 | return // done 158 | } 159 | if err := enc.Encode(eventsource.Event{Data: value}); err != nil { 160 | cancel() 161 | <-errs 162 | return 163 | } 164 | 165 | case <-errs: 166 | cancel() // no-op for linter 167 | return // the watcher goroutine is dead 168 | } 169 | } 170 | } 171 | 172 | // 173 | // 174 | // 175 | 176 | // AcceptorClient implements the extension.Acceptor interface by making calls to 177 | // a remote AcceptorServer. 178 | type AcceptorClient struct { 179 | // HTTPClient to make requests. Optional. 180 | // If nil, http.DefaultClient is used. 181 | Client interface { 182 | Do(*http.Request) (*http.Response, error) 183 | } 184 | 185 | // URL of the remote AcceptorServer. 186 | // Only scheme and host are used. 187 | URL *url.URL 188 | 189 | // WatchRetry is the time between reconnect attempts 190 | // if a Watch session goes bad. Default of 1 second. 191 | WatchRetry time.Duration 192 | } 193 | 194 | var _ extension.Acceptor = (*AcceptorClient)(nil) 195 | 196 | // Address implements extension.Acceptor. 197 | func (ac AcceptorClient) Address() string { 198 | return ac.URL.String() 199 | } 200 | 201 | // Prepare implements extension.Acceptor. 202 | func (ac AcceptorClient) Prepare(ctx context.Context, key string, age protocol.Age, b protocol.Ballot) (value []byte, current protocol.Ballot, err error) { 203 | u := *ac.URL 204 | u.Path = fmt.Sprintf("/prepare/%s", url.PathEscape(key)) 205 | req, _ := http.NewRequest("POST", u.String(), nil) 206 | req = req.WithContext(ctx) 207 | setAge(req.Header, age) 208 | setBallot(req.Header, b) 209 | 210 | resp, err := ac.httpClient().Do(req) 211 | if err != nil { 212 | return value, current, err 213 | } 214 | 215 | current = getBallot(resp.Header) 216 | 217 | if resp.StatusCode != http.StatusOK { 218 | buf, _ := ioutil.ReadAll(resp.Body) 219 | return value, current, errors.New(strings.TrimSpace(string(buf))) 220 | } 221 | 222 | value, err = ioutil.ReadAll(resp.Body) 223 | if err != nil { 224 | return value, current, err 225 | } 226 | 227 | return value, current, nil 228 | } 229 | 230 | // Accept implements extension.Acceptor. 231 | func (ac AcceptorClient) Accept(ctx context.Context, key string, age protocol.Age, b protocol.Ballot, value []byte) error { 232 | u := *ac.URL 233 | u.Path = fmt.Sprintf("/accept/%s", url.PathEscape(key)) 234 | req, _ := http.NewRequest("POST", u.String(), bytes.NewReader(value)) 235 | req = req.WithContext(ctx) 236 | setAge(req.Header, age) 237 | setBallot(req.Header, b) 238 | 239 | resp, err := ac.httpClient().Do(req) 240 | if err != nil { 241 | return err 242 | } 243 | if resp.StatusCode != http.StatusOK { 244 | buf, _ := ioutil.ReadAll(resp.Body) 245 | return errors.New(strings.TrimSpace(string(buf))) 246 | } 247 | 248 | return nil 249 | } 250 | 251 | // RejectByAge implements extension.Acceptor. 252 | func (ac AcceptorClient) RejectByAge(ctx context.Context, ages []protocol.Age) error { 253 | u := *ac.URL 254 | u.Path = "/reject-by-age" 255 | req, _ := http.NewRequest("POST", u.String(), nil) 256 | req = req.WithContext(ctx) 257 | setAges(req.Header, ages) 258 | 259 | resp, err := ac.httpClient().Do(req) 260 | if err != nil { 261 | return err 262 | } 263 | if resp.StatusCode != http.StatusOK { 264 | buf, _ := ioutil.ReadAll(resp.Body) 265 | return errors.New(strings.TrimSpace(string(buf))) 266 | } 267 | 268 | return nil 269 | } 270 | 271 | // RemoveIfEqual implements extension.Acceptor. 272 | func (ac AcceptorClient) RemoveIfEqual(ctx context.Context, key string, state []byte) error { 273 | u := *ac.URL 274 | u.Path = fmt.Sprintf("/remove-if-equal/%s", url.PathEscape(key)) 275 | req, _ := http.NewRequest("POST", u.String(), bytes.NewReader(state)) 276 | req = req.WithContext(ctx) 277 | 278 | resp, err := ac.httpClient().Do(req) 279 | if err != nil { 280 | return err 281 | } 282 | if resp.StatusCode != http.StatusOK { 283 | buf, _ := ioutil.ReadAll(resp.Body) 284 | return errors.New(strings.TrimSpace(string(buf))) 285 | } 286 | 287 | return nil 288 | } 289 | 290 | // Watch implements extension.Acceptor. 291 | func (ac AcceptorClient) Watch(ctx context.Context, key string, values chan<- []byte) error { 292 | u := *ac.URL 293 | u.Path = fmt.Sprintf("/watch/%s", url.PathEscape(key)) 294 | req, _ := http.NewRequest("POST", u.String(), nil) 295 | req = req.WithContext(ctx) 296 | 297 | retry := ac.WatchRetry 298 | if retry <= 0 { 299 | retry = time.Second 300 | } 301 | 302 | s := eventsource.New(req, retry) // TODO(pb): this uses DefaultClient 303 | defer s.Close() 304 | 305 | for { 306 | ev, err := s.Read() 307 | if err != nil { 308 | return errors.Wrap(err, "remote acceptor EventSource error") 309 | } 310 | values <- ev.Data 311 | } 312 | } 313 | 314 | func (ac AcceptorClient) httpClient() interface { 315 | Do(*http.Request) (*http.Response, error) 316 | } { 317 | client := ac.Client 318 | if client == nil { 319 | client = http.DefaultClient 320 | } 321 | return client 322 | } 323 | -------------------------------------------------------------------------------- /httpapi/http_operator_node.go: -------------------------------------------------------------------------------- 1 | package httpapi 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io/ioutil" 9 | "net/http" 10 | "net/url" 11 | "strings" 12 | 13 | "github.com/gorilla/mux" 14 | "github.com/peterbourgon/caspaxos/extension" 15 | "github.com/peterbourgon/caspaxos/protocol" 16 | ) 17 | 18 | // OperatorNodeServer provides an HTTP interface to an operator node. 19 | type OperatorNodeServer struct { 20 | operatorNode extension.OperatorNode 21 | *mux.Router 22 | } 23 | 24 | var _ http.Handler = (*OperatorNodeServer)(nil) 25 | 26 | // NewOperatorNodeServer returns a usable OperatorNodeServer wrapping the passed 27 | // operator node. 28 | func NewOperatorNodeServer(on extension.OperatorNode) *OperatorNodeServer { 29 | ons := &OperatorNodeServer{ 30 | operatorNode: on, 31 | } 32 | r := mux.NewRouter() 33 | { 34 | r.StrictSlash(true) 35 | r.Methods("GET").Path("/cluster-state").HandlerFunc(ons.handleClusterState) 36 | r.Methods("POST").Path("/grow-cluster").HandlerFunc(ons.handleGrowCluster) 37 | r.Methods("POST").Path("/shrink-cluster").HandlerFunc(ons.handleShrinkCluster) 38 | r.Methods("POST").Path("/garbage-collect/{key}").HandlerFunc(ons.handleGarbageCollect) 39 | } 40 | ons.Router = r 41 | return ons 42 | } 43 | 44 | func (ons *OperatorNodeServer) handleClusterState(w http.ResponseWriter, r *http.Request) { 45 | s, err := ons.operatorNode.ClusterState(r.Context()) 46 | if err != nil { 47 | http.Error(w, err.Error(), http.StatusInternalServerError) 48 | return 49 | } 50 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 51 | json.NewEncoder(w).Encode(s) 52 | } 53 | 54 | func (ons *OperatorNodeServer) handleGrowCluster(w http.ResponseWriter, r *http.Request) { 55 | ons.handleClusterChange(w, r, ons.operatorNode.GrowCluster) 56 | } 57 | 58 | func (ons *OperatorNodeServer) handleShrinkCluster(w http.ResponseWriter, r *http.Request) { 59 | ons.handleClusterChange(w, r, ons.operatorNode.ShrinkCluster) 60 | } 61 | 62 | func (ons *OperatorNodeServer) handleClusterChange(w http.ResponseWriter, r *http.Request, method func(context.Context, protocol.Acceptor) error) { 63 | buf, _ := ioutil.ReadAll(r.Body) 64 | u, err := url.Parse(string(buf)) 65 | if err != nil { 66 | http.Error(w, err.Error(), http.StatusBadRequest) 67 | return 68 | } 69 | target := AcceptorClient{URL: u} // TODO(pb): HTTP client 70 | if err := method(r.Context(), target); err != nil { 71 | http.Error(w, err.Error(), http.StatusInternalServerError) 72 | return 73 | } 74 | fmt.Fprintln(w, "OK") 75 | } 76 | 77 | func (ons *OperatorNodeServer) handleGarbageCollect(w http.ResponseWriter, r *http.Request) { 78 | key, _ := url.PathUnescape(mux.Vars(r)["key"]) 79 | if err := ons.operatorNode.GarbageCollect(r.Context(), key); err != nil { 80 | http.Error(w, err.Error(), http.StatusInternalServerError) 81 | } 82 | fmt.Fprintln(w, "OK") 83 | } 84 | 85 | // 86 | // 87 | // 88 | 89 | // OperatorNodeClient implements the extension.OperatorNode interface by making 90 | // calls to a remote OperatorNodeServer. 91 | type OperatorNodeClient struct { 92 | // HTTPClient to make requests. Optional. 93 | // If nil, http.DefaultClient is used. 94 | Client interface { 95 | Do(*http.Request) (*http.Response, error) 96 | } 97 | 98 | // URL of the remote AcceptorServer. 99 | // Only scheme and host are used. 100 | URL *url.URL 101 | } 102 | 103 | var _ extension.OperatorNode = (*OperatorNodeClient)(nil) 104 | 105 | // Address implements OperatorNode. 106 | func (onc OperatorNodeClient) Address() string { 107 | return onc.URL.String() 108 | } 109 | 110 | // ClusterState implements OperatorNode. 111 | func (onc OperatorNodeClient) ClusterState(ctx context.Context) (s extension.ClusterState, err error) { 112 | u := *onc.URL 113 | u.Path = "/cluster-state" 114 | req, _ := http.NewRequest("GET", u.String(), nil) 115 | req = req.WithContext(ctx) 116 | 117 | resp, err := onc.httpClient().Do(req) 118 | if err != nil { 119 | return s, err 120 | } 121 | 122 | if resp.StatusCode != http.StatusOK { 123 | buf, _ := ioutil.ReadAll(resp.Body) 124 | return s, errors.New(strings.TrimSpace(string(buf))) 125 | } 126 | 127 | err = json.NewDecoder(resp.Body).Decode(&s) 128 | return s, err 129 | } 130 | 131 | // GrowCluster implements OperatorNode. 132 | func (onc OperatorNodeClient) GrowCluster(ctx context.Context, target protocol.Acceptor) error { 133 | return onc.configChange(ctx, target, "/grow-cluster") 134 | } 135 | 136 | // ShrinkCluster implements OperatorNode. 137 | func (onc OperatorNodeClient) ShrinkCluster(ctx context.Context, target protocol.Acceptor) error { 138 | return onc.configChange(ctx, target, "/shrink-cluster") 139 | } 140 | 141 | func (onc OperatorNodeClient) configChange(ctx context.Context, target protocol.Acceptor, path string) error { 142 | u := *onc.URL 143 | u.Path = path 144 | req, _ := http.NewRequest("POST", u.String(), strings.NewReader(target.Address())) 145 | req = req.WithContext(ctx) 146 | 147 | resp, err := onc.httpClient().Do(req) 148 | if err != nil { 149 | return err 150 | } 151 | 152 | if resp.StatusCode != http.StatusOK { 153 | buf, _ := ioutil.ReadAll(resp.Body) 154 | return errors.New(strings.TrimSpace(string(buf))) 155 | } 156 | 157 | return nil 158 | } 159 | 160 | // GarbageCollect implements OperatorNode. 161 | func (onc OperatorNodeClient) GarbageCollect(ctx context.Context, key string) error { 162 | u := *onc.URL 163 | u.Path = fmt.Sprintf("/garbage-collect/%s", url.PathEscape(key)) 164 | req, _ := http.NewRequest("POST", u.String(), nil) 165 | req = req.WithContext(ctx) 166 | 167 | resp, err := onc.httpClient().Do(req) 168 | if err != nil { 169 | return err 170 | } 171 | 172 | if resp.StatusCode != http.StatusOK { 173 | buf, _ := ioutil.ReadAll(resp.Body) 174 | return errors.New(strings.TrimSpace(string(buf))) 175 | } 176 | 177 | return nil 178 | } 179 | 180 | func (onc OperatorNodeClient) httpClient() interface { 181 | Do(*http.Request) (*http.Response, error) 182 | } { 183 | client := onc.Client 184 | if client == nil { 185 | client = http.DefaultClient 186 | } 187 | return client 188 | } 189 | -------------------------------------------------------------------------------- /httpapi/http_proposer.go: -------------------------------------------------------------------------------- 1 | package httpapi 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "errors" 8 | "fmt" 9 | "io/ioutil" 10 | "net/http" 11 | "net/url" 12 | "strings" 13 | 14 | "github.com/gorilla/mux" 15 | 16 | "github.com/peterbourgon/caspaxos/extension" 17 | "github.com/peterbourgon/caspaxos/protocol" 18 | ) 19 | 20 | // ProposerServer wraps a core protocol.Proposer, but 21 | // provides the extension.Proposer methodset over HTTP. 22 | type ProposerServer struct { 23 | proposer protocol.Proposer 24 | *mux.Router 25 | } 26 | 27 | var _ http.Handler = (*ProposerServer)(nil) 28 | 29 | // NewProposerServer returns a usable ProposerServer wrapping the passed proposer. 30 | func NewProposerServer(proposer protocol.Proposer) *ProposerServer { 31 | ps := &ProposerServer{ 32 | proposer: proposer, 33 | } 34 | r := mux.NewRouter() 35 | { 36 | r.StrictSlash(true) 37 | r.Methods("POST").Path("/read/{key}").HandlerFunc(ps.handleRead) 38 | r.Methods("POST").Path("/cas/{key}").HandlerFunc(ps.handleCAS) 39 | r.Methods("POST").Path("/identity-read/{key}").HandlerFunc(ps.handleIdentityRead) 40 | r.Methods("POST").Path("/add-accepter").HandlerFunc(ps.handleAddAccepter) 41 | r.Methods("POST").Path("/add-preparer").HandlerFunc(ps.handleAddPreparer) 42 | r.Methods("POST").Path("/remove-preparer").HandlerFunc(ps.handleRemovePreparer) 43 | r.Methods("POST").Path("/remove-accepter").HandlerFunc(ps.handleRemoveAccepter) 44 | r.Methods("POST").Path("/full-identity-read/{key}").HandlerFunc(ps.handleFullIdentityRead) 45 | r.Methods("POST").Path("/fast-forward-increment/{key}").HandlerFunc(ps.handleFastForwardIncrement) 46 | r.Methods("GET").Path("/list-preparers").HandlerFunc(ps.handleListPreparers) 47 | r.Methods("GET").Path("/list-accepters").HandlerFunc(ps.handleListAccepters) 48 | r.Methods("POST").Path("/watch/{key}").HandlerFunc(ps.handleWatch) 49 | } 50 | ps.Router = r 51 | return ps 52 | } 53 | 54 | func (ps *ProposerServer) handleRead(w http.ResponseWriter, r *http.Request) { 55 | var ( 56 | key, _ = url.PathUnescape(mux.Vars(r)["key"]) 57 | read = func(x []byte) []byte { return x } 58 | ) 59 | 60 | state, b, err := ps.proposer.Propose(r.Context(), key, read) 61 | if err != nil { 62 | http.Error(w, err.Error(), http.StatusInternalServerError) 63 | return 64 | } 65 | 66 | version, value, err := parseVersionValue(state) 67 | if err != nil { 68 | http.Error(w, err.Error(), http.StatusInternalServerError) 69 | return 70 | } 71 | 72 | setBallot(w.Header(), b) 73 | setVersion(w.Header(), version) 74 | w.Write(value) 75 | } 76 | 77 | func (ps *ProposerServer) handleCAS(w http.ResponseWriter, r *http.Request) { 78 | var ( 79 | key, _ = url.PathUnescape(mux.Vars(r)["key"]) 80 | currentVersion = getVersion(r.Header) 81 | nextValue, _ = ioutil.ReadAll(r.Body) 82 | ) 83 | 84 | cas := func(x []byte) []byte { 85 | if version, _, err := parseVersionValue(x); err == nil && version == currentVersion { 86 | return makeVersionValue(version+1, nextValue) 87 | } 88 | return x 89 | } 90 | 91 | state, _, err := ps.proposer.Propose(r.Context(), key, cas) 92 | if _, ok := err.(protocol.ConflictError); ok { 93 | http.Error(w, err.Error(), http.StatusPreconditionFailed) // ConflictError -> 412 (CASError) 94 | return 95 | } 96 | if err != nil { 97 | http.Error(w, err.Error(), http.StatusInternalServerError) 98 | return 99 | } 100 | 101 | version, value, err := parseVersionValue(state) 102 | if err != nil { 103 | http.Error(w, err.Error(), http.StatusInternalServerError) 104 | return 105 | } 106 | 107 | setVersion(w.Header(), version) 108 | w.Write(value) 109 | } 110 | 111 | func (ps *ProposerServer) handleIdentityRead(w http.ResponseWriter, r *http.Request) { 112 | ps.handleRead(w, r) 113 | } 114 | 115 | // AddAccepter(target Acceptor) error 116 | func (ps *ProposerServer) handleAddAccepter(w http.ResponseWriter, r *http.Request) { 117 | buf, _ := ioutil.ReadAll(r.Body) 118 | u, err := url.Parse(string(buf)) 119 | if err != nil { 120 | http.Error(w, err.Error(), http.StatusBadRequest) 121 | return 122 | } 123 | target := AcceptorClient{URL: u} 124 | if err := ps.proposer.AddAccepter(target); err != nil { 125 | http.Error(w, err.Error(), http.StatusInternalServerError) 126 | return 127 | } 128 | fmt.Fprintln(w, "OK") 129 | } 130 | 131 | // AddPreparer(target Acceptor) error 132 | func (ps *ProposerServer) handleAddPreparer(w http.ResponseWriter, r *http.Request) { 133 | buf, _ := ioutil.ReadAll(r.Body) 134 | u, err := url.Parse(string(buf)) 135 | if err != nil { 136 | http.Error(w, err.Error(), http.StatusBadRequest) 137 | return 138 | } 139 | target := AcceptorClient{URL: u} 140 | if err := ps.proposer.AddPreparer(target); err != nil { 141 | http.Error(w, err.Error(), http.StatusInternalServerError) 142 | return 143 | } 144 | fmt.Fprintln(w, "OK") 145 | } 146 | 147 | // RemovePreparer(target Acceptor) error 148 | func (ps *ProposerServer) handleRemovePreparer(w http.ResponseWriter, r *http.Request) { 149 | buf, _ := ioutil.ReadAll(r.Body) 150 | u, err := url.Parse(string(buf)) 151 | if err != nil { 152 | http.Error(w, err.Error(), http.StatusBadRequest) 153 | return 154 | } 155 | target := AcceptorClient{URL: u} 156 | if err := ps.proposer.RemovePreparer(target); err != nil { 157 | http.Error(w, err.Error(), http.StatusInternalServerError) 158 | return 159 | } 160 | fmt.Fprintln(w, "OK") 161 | } 162 | 163 | // RemoveAccepter(target Acceptor) error 164 | func (ps *ProposerServer) handleRemoveAccepter(w http.ResponseWriter, r *http.Request) { 165 | buf, _ := ioutil.ReadAll(r.Body) 166 | u, err := url.Parse(string(buf)) 167 | if err != nil { 168 | http.Error(w, err.Error(), http.StatusBadRequest) 169 | return 170 | } 171 | target := AcceptorClient{URL: u} 172 | if err := ps.proposer.RemoveAccepter(target); err != nil { 173 | http.Error(w, err.Error(), http.StatusInternalServerError) 174 | return 175 | } 176 | fmt.Fprintln(w, "OK") 177 | } 178 | 179 | // FullIdentityRead(ctx context.Context, key string) (state []byte, err error) 180 | func (ps *ProposerServer) handleFullIdentityRead(w http.ResponseWriter, r *http.Request) { 181 | key, _ := url.PathUnescape(mux.Vars(r)["key"]) 182 | state, b, err := ps.proposer.FullIdentityRead(r.Context(), key) 183 | setBallot(w.Header(), b) 184 | if err != nil { 185 | http.Error(w, err.Error(), http.StatusInternalServerError) 186 | return 187 | } 188 | w.Write(state) 189 | } 190 | 191 | // FastForwardIncrement(ctx context.Context, key string, tombstone Ballot) (Age, error) 192 | func (ps *ProposerServer) handleFastForwardIncrement(w http.ResponseWriter, r *http.Request) { 193 | var ( 194 | key, _ = url.PathUnescape(mux.Vars(r)["key"]) 195 | tombstone = getBallot(r.Header) 196 | ) 197 | age, err := ps.proposer.FastForwardIncrement(r.Context(), key, tombstone) 198 | if err != nil { 199 | http.Error(w, err.Error(), http.StatusInternalServerError) 200 | return 201 | } 202 | setAge(w.Header(), age) 203 | fmt.Fprintln(w, "OK") 204 | } 205 | 206 | func (ps *ProposerServer) handleListPreparers(w http.ResponseWriter, r *http.Request) { 207 | addrs, err := ps.proposer.ListPreparers() 208 | if err != nil { 209 | http.Error(w, err.Error(), http.StatusInternalServerError) 210 | return 211 | } 212 | for _, addr := range addrs { 213 | fmt.Fprintln(w, addr) 214 | } 215 | } 216 | 217 | func (ps *ProposerServer) handleListAccepters(w http.ResponseWriter, r *http.Request) { 218 | addrs, err := ps.proposer.ListAccepters() 219 | if err != nil { 220 | http.Error(w, err.Error(), http.StatusInternalServerError) 221 | return 222 | } 223 | for _, addr := range addrs { 224 | fmt.Fprintln(w, addr) 225 | } 226 | } 227 | 228 | func (ps *ProposerServer) handleWatch(w http.ResponseWriter, r *http.Request) { 229 | http.Error(w, "ProposerServer handleWatch not yet implemented", http.StatusNotImplemented) 230 | } 231 | 232 | // 233 | // 234 | // 235 | 236 | // ProposerClient implements the extension.Proposer interface by making calls to 237 | // a remote ProposerServer. 238 | type ProposerClient struct { 239 | // HTTPClient to make requests. Optional. 240 | // If nil, http.DefaultClient is used. 241 | Client interface { 242 | Do(*http.Request) (*http.Response, error) 243 | } 244 | 245 | // URL of the remote ProposerServer. 246 | // Only scheme and host are used. 247 | URL *url.URL 248 | } 249 | 250 | var _ extension.Proposer = (*ProposerClient)(nil) 251 | 252 | // Address implements extension.Proposer. 253 | func (pc ProposerClient) Address() string { 254 | return pc.URL.String() 255 | } 256 | 257 | // Read implements extension.Proposer. 258 | func (pc ProposerClient) Read(ctx context.Context, key string) (version uint64, value []byte, err error) { 259 | u := *pc.URL 260 | u.Path = fmt.Sprintf("/read/%s", url.PathEscape(key)) 261 | req, _ := http.NewRequest("POST", u.String(), nil) 262 | req = req.WithContext(ctx) 263 | resp, err := pc.httpClient().Do(req) 264 | if err != nil { 265 | return 0, nil, err 266 | } 267 | 268 | if resp.StatusCode != http.StatusOK { 269 | buf, _ := ioutil.ReadAll(resp.Body) 270 | return 0, nil, errors.New(strings.TrimSpace(string(buf))) 271 | } 272 | 273 | version = getVersion(resp.Header) 274 | value, _ = ioutil.ReadAll(resp.Body) 275 | return version, value, nil 276 | } 277 | 278 | // CAS implements extension.Proposer. 279 | func (pc ProposerClient) CAS(ctx context.Context, key string, currentVersion uint64, nextValue []byte) (version uint64, value []byte, err error) { 280 | u := *pc.URL 281 | u.Path = fmt.Sprintf("/cas/%s", url.PathEscape(key)) 282 | req, _ := http.NewRequest("POST", u.String(), bytes.NewReader(nextValue)) 283 | setVersion(req.Header, currentVersion) 284 | req = req.WithContext(ctx) 285 | resp, err := pc.httpClient().Do(req) 286 | if err != nil { 287 | return version, value, err 288 | } 289 | 290 | switch { 291 | case resp.StatusCode == http.StatusPreconditionFailed: // 412 -> CASError 292 | buf, _ := ioutil.ReadAll(resp.Body) 293 | return version, value, extension.CASError{Err: errors.New(strings.TrimSpace(string(buf)))} 294 | case resp.StatusCode != http.StatusOK: 295 | buf, _ := ioutil.ReadAll(resp.Body) 296 | return version, value, errors.New(strings.TrimSpace(string(buf))) 297 | } 298 | 299 | version = getVersion(resp.Header) 300 | value, _ = ioutil.ReadAll(resp.Body) 301 | return version, value, nil 302 | } 303 | 304 | // IdentityRead implements extension.Proposer. 305 | func (pc ProposerClient) IdentityRead(ctx context.Context, key string) error { 306 | _, _, err := pc.readVia(ctx, fmt.Sprintf("/identity-read/%s", url.PathEscape(key))) 307 | return err 308 | } 309 | 310 | // AddAccepter implements extension.Proposer. 311 | func (pc ProposerClient) AddAccepter(target protocol.Acceptor) error { 312 | return pc.postString("/add-accepter", target.Address()) 313 | } 314 | 315 | // AddPreparer implements extension.Proposer. 316 | func (pc ProposerClient) AddPreparer(target protocol.Acceptor) error { 317 | return pc.postString("/add-preparer", target.Address()) 318 | } 319 | 320 | // RemovePreparer implements extension.Proposer. 321 | func (pc ProposerClient) RemovePreparer(target protocol.Acceptor) error { 322 | return pc.postString("/remove-preparer", target.Address()) 323 | } 324 | 325 | // RemoveAccepter implements extension.Proposer. 326 | func (pc ProposerClient) RemoveAccepter(target protocol.Acceptor) error { 327 | return pc.postString("/remove-accepter", target.Address()) 328 | } 329 | 330 | // FullIdentityRead implements extension.Proposer. 331 | func (pc ProposerClient) FullIdentityRead(ctx context.Context, key string) (state []byte, b protocol.Ballot, err error) { 332 | return pc.readVia(ctx, fmt.Sprintf("/full-identity-read/%s", url.PathEscape(key))) 333 | } 334 | 335 | // FastForwardIncrement implements extension.Proposer. 336 | func (pc ProposerClient) FastForwardIncrement(ctx context.Context, key string, tombstone protocol.Ballot) (protocol.Age, error) { 337 | u := *pc.URL 338 | u.Path = fmt.Sprintf("/fast-forward-increment/%s", url.PathEscape(key)) 339 | req, _ := http.NewRequest("POST", u.String(), nil) 340 | setBallot(req.Header, tombstone) 341 | resp, err := pc.httpClient().Do(req) 342 | if err != nil { 343 | return protocol.Age{}, err 344 | } 345 | if resp.StatusCode != http.StatusOK { 346 | buf, _ := ioutil.ReadAll(resp.Body) 347 | return protocol.Age{}, errors.New(strings.TrimSpace(string(buf))) 348 | } 349 | return getAge(resp.Header), nil 350 | } 351 | 352 | // ListPreparers implements extension.Proposer. 353 | func (pc ProposerClient) ListPreparers() ([]string, error) { 354 | u := *pc.URL 355 | u.Path = "/list-preparers" 356 | req, _ := http.NewRequest("GET", u.String(), nil) 357 | resp, err := pc.httpClient().Do(req) 358 | if err != nil { 359 | return nil, err 360 | } 361 | if resp.StatusCode != http.StatusOK { 362 | buf, _ := ioutil.ReadAll(resp.Body) 363 | return nil, errors.New(strings.TrimSpace(string(buf))) 364 | } 365 | var ( 366 | addrs []string 367 | s = bufio.NewScanner(resp.Body) 368 | ) 369 | for s.Scan() { 370 | addrs = append(addrs, s.Text()) 371 | } 372 | return addrs, nil 373 | } 374 | 375 | // ListAccepters implements extension.Proposer. 376 | func (pc ProposerClient) ListAccepters() ([]string, error) { 377 | u := *pc.URL 378 | u.Path = "/list-accepters" 379 | req, _ := http.NewRequest("GET", u.String(), nil) 380 | resp, err := pc.httpClient().Do(req) 381 | if err != nil { 382 | return nil, err 383 | } 384 | if resp.StatusCode != http.StatusOK { 385 | buf, _ := ioutil.ReadAll(resp.Body) 386 | return nil, errors.New(strings.TrimSpace(string(buf))) 387 | } 388 | var ( 389 | addrs []string 390 | s = bufio.NewScanner(resp.Body) 391 | ) 392 | for s.Scan() { 393 | addrs = append(addrs, s.Text()) 394 | } 395 | return addrs, nil 396 | } 397 | 398 | func (pc ProposerClient) readVia(ctx context.Context, fullPath string) (state []byte, b protocol.Ballot, err error) { 399 | u := *pc.URL 400 | u.Path = fullPath 401 | req, _ := http.NewRequest("POST", u.String(), nil) 402 | if ctx != nil { 403 | req = req.WithContext(ctx) 404 | } 405 | resp, err := pc.httpClient().Do(req) 406 | if err != nil { 407 | return state, b, err 408 | } 409 | b = getBallot(resp.Header) 410 | buf, _ := ioutil.ReadAll(resp.Body) 411 | if resp.StatusCode != http.StatusOK { 412 | return state, b, errors.New(strings.TrimSpace(string(buf))) 413 | } 414 | state = buf 415 | return state, b, nil 416 | } 417 | 418 | func (pc ProposerClient) postString(path, body string) error { 419 | u := *pc.URL 420 | u.Path = path 421 | req, _ := http.NewRequest("POST", u.String(), strings.NewReader(body)) 422 | resp, err := pc.httpClient().Do(req) 423 | if err != nil { 424 | return err 425 | } 426 | if resp.StatusCode != http.StatusOK { 427 | buf, _ := ioutil.ReadAll(resp.Body) 428 | return errors.New(strings.TrimSpace(string(buf))) 429 | } 430 | return nil 431 | } 432 | 433 | func (pc ProposerClient) httpClient() interface { 434 | Do(*http.Request) (*http.Response, error) 435 | } { 436 | client := pc.Client 437 | if client == nil { 438 | client = http.DefaultClient 439 | } 440 | return client 441 | } 442 | -------------------------------------------------------------------------------- /httpapi/http_user_node.go: -------------------------------------------------------------------------------- 1 | package httpapi 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "net/url" 10 | "strings" 11 | "time" 12 | 13 | "github.com/gorilla/mux" 14 | "github.com/pkg/errors" 15 | 16 | "github.com/peterbourgon/caspaxos/extension" 17 | "github.com/peterbourgon/caspaxos/internal/eventsource" 18 | ) 19 | 20 | // UserNodeServer provides an HTTP interface to an user node. 21 | type UserNodeServer struct { 22 | userNode extension.UserNode 23 | *mux.Router 24 | } 25 | 26 | var _ http.Handler = (*UserNodeServer)(nil) 27 | 28 | // NewUserNodeServer returns a usable UserNodeServer wrapping the passed 29 | // user node. 30 | func NewUserNodeServer(un extension.UserNode) *UserNodeServer { 31 | uns := &UserNodeServer{ 32 | userNode: un, 33 | } 34 | r := mux.NewRouter() 35 | { 36 | r.StrictSlash(true) 37 | r.Methods("POST").Path("/read/{key}").HandlerFunc(uns.handleRead) 38 | r.Methods("POST").Path("/cas/{key}").HandlerFunc(uns.handleCAS) 39 | r.Methods("POST").Path("/watch/{key}").HandlerFunc(uns.handleWatch) 40 | } 41 | uns.Router = r 42 | return uns 43 | } 44 | 45 | func (uns *UserNodeServer) handleRead(w http.ResponseWriter, r *http.Request) { 46 | key, _ := url.PathUnescape(mux.Vars(r)["key"]) 47 | version, value, err := uns.userNode.Read(r.Context(), key) 48 | if err != nil { 49 | http.Error(w, err.Error(), http.StatusInternalServerError) 50 | return 51 | } 52 | 53 | setVersion(w.Header(), version) 54 | w.Write(value) 55 | } 56 | 57 | func (uns *UserNodeServer) handleCAS(w http.ResponseWriter, r *http.Request) { 58 | var ( 59 | key, _ = url.PathUnescape(mux.Vars(r)["key"]) 60 | currentVersion = getVersion(r.Header) 61 | nextValue, _ = ioutil.ReadAll(r.Body) 62 | ) 63 | 64 | version, value, err := uns.userNode.CAS(r.Context(), key, currentVersion, nextValue) 65 | if _, ok := err.(extension.CASError); ok { 66 | http.Error(w, err.Error(), http.StatusPreconditionFailed) // CASError -> 412 67 | return 68 | } 69 | if err != nil { 70 | http.Error(w, err.Error(), http.StatusInternalServerError) 71 | return 72 | } 73 | 74 | setVersion(w.Header(), version) 75 | w.Write(value) 76 | } 77 | 78 | // Watch(ctx context.Context, key string, values chan<- []byte) error 79 | func (uns *UserNodeServer) handleWatch(w http.ResponseWriter, r *http.Request) { 80 | var ( 81 | key, _ = url.PathUnescape(mux.Vars(r)["key"]) 82 | values = make(chan []byte) 83 | errs = make(chan error) 84 | enc = eventsource.NewEncoder(w) 85 | ctx, cancel = context.WithCancel(r.Context()) 86 | ) 87 | 88 | go func() { 89 | errs <- uns.userNode.Watch(ctx, key, values) 90 | }() 91 | 92 | // via eventsource.Handler.ServeHTTP 93 | w.Header().Set("Cache-Control", "no-cache") 94 | w.Header().Set("Vary", "Accept") 95 | w.Header().Set("Content-Type", "text/event-stream") 96 | w.WriteHeader(http.StatusOK) 97 | 98 | for { 99 | select { 100 | case value := <-values: 101 | if err := enc.Encode(eventsource.Event{Data: value}); err != nil { 102 | cancel() 103 | <-errs 104 | return 105 | } 106 | 107 | case <-errs: 108 | cancel() // no-op for linter 109 | return // the watcher goroutine is dead 110 | } 111 | } 112 | 113 | } 114 | 115 | // 116 | // 117 | // 118 | 119 | // UserNodeClient implements the extension.UserNode interface by making 120 | // calls to a remote UserNodeServer. 121 | type UserNodeClient struct { 122 | // HTTPClient to make requests. Optional. 123 | // If nil, http.DefaultClient is used. 124 | Client interface { 125 | Do(*http.Request) (*http.Response, error) 126 | } 127 | 128 | // URL of the remote AcceptorServer. 129 | // Only scheme and host are used. 130 | URL *url.URL 131 | 132 | // WatchRetry is the time between reconnect attempts 133 | // if a Watch session goes bad. Default of 1 second. 134 | WatchRetry time.Duration 135 | } 136 | 137 | var _ extension.UserNode = (*UserNodeClient)(nil) 138 | 139 | // Address implements UserNode. 140 | func (unc UserNodeClient) Address() string { 141 | return unc.URL.String() 142 | } 143 | 144 | // Read implements UserNode. 145 | func (unc UserNodeClient) Read(ctx context.Context, key string) (version uint64, value []byte, err error) { 146 | u := *unc.URL 147 | u.Path = fmt.Sprintf("/read/%s", url.PathEscape(key)) 148 | req, _ := http.NewRequest("POST", u.String(), nil) 149 | req = req.WithContext(ctx) 150 | resp, err := unc.httpClient().Do(req) 151 | if err != nil { 152 | return 0, nil, err 153 | } 154 | 155 | if resp.StatusCode != http.StatusOK { 156 | buf, _ := ioutil.ReadAll(resp.Body) 157 | return 0, nil, errors.New(strings.TrimSpace(string(buf))) 158 | } 159 | 160 | version = getVersion(resp.Header) 161 | value, _ = ioutil.ReadAll(resp.Body) 162 | return version, value, nil 163 | } 164 | 165 | // CAS implements UserNode. 166 | func (unc UserNodeClient) CAS(ctx context.Context, key string, currentVersion uint64, nextValue []byte) (version uint64, value []byte, err error) { 167 | u := *unc.URL 168 | u.Path = fmt.Sprintf("/cas/%s", url.PathEscape(key)) 169 | req, _ := http.NewRequest("POST", u.String(), bytes.NewReader(nextValue)) 170 | setVersion(req.Header, currentVersion) 171 | req = req.WithContext(ctx) 172 | resp, err := unc.httpClient().Do(req) 173 | if err != nil { 174 | return version, value, err 175 | } 176 | 177 | switch { 178 | case resp.StatusCode == http.StatusPreconditionFailed: // 412 -> CASError 179 | buf, _ := ioutil.ReadAll(resp.Body) 180 | return version, value, extension.CASError{Err: errors.New(strings.TrimSpace(string(buf)))} 181 | case resp.StatusCode != http.StatusOK: 182 | buf, _ := ioutil.ReadAll(resp.Body) 183 | return version, value, errors.New(strings.TrimSpace(string(buf))) 184 | } 185 | 186 | version = getVersion(resp.Header) 187 | value, _ = ioutil.ReadAll(resp.Body) 188 | return version, value, nil 189 | 190 | } 191 | 192 | // Watch implements UserNode. 193 | func (unc UserNodeClient) Watch(ctx context.Context, key string, values chan<- []byte) error { 194 | u := *unc.URL 195 | u.Path = fmt.Sprintf("/watch/%s", url.PathEscape(key)) 196 | req, _ := http.NewRequest("POST", u.String(), nil) 197 | req = req.WithContext(ctx) 198 | 199 | retry := unc.WatchRetry 200 | if retry <= 0 { 201 | retry = time.Second 202 | } 203 | 204 | s := eventsource.New(req, retry) // TODO(pb): this uses DefaultClient 205 | defer s.Close() 206 | 207 | for { 208 | ev, err := s.Read() 209 | if err != nil { 210 | return errors.Wrap(err, "remote user node EventSource error") 211 | } 212 | values <- ev.Data 213 | } 214 | } 215 | 216 | func (unc UserNodeClient) httpClient() interface { 217 | Do(*http.Request) (*http.Response, error) 218 | } { 219 | client := unc.Client 220 | if client == nil { 221 | client = http.DefaultClient 222 | } 223 | return client 224 | } 225 | -------------------------------------------------------------------------------- /internal/eventsource/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2013 Bernerd Schaefer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /internal/eventsource/README.md: -------------------------------------------------------------------------------- 1 | ## eventsource 2 | 3 | `eventsource` provides the building blocks for consuming and building 4 | [EventSource][spec] services. 5 | 6 | ### Installing 7 | 8 | $ go get github.com/bernerdschaefer/eventsource 9 | 10 | ### Importing 11 | 12 | ```go 13 | import "github.com/bernerdschaefer/eventsource" 14 | ``` 15 | 16 | ### Docs 17 | 18 | See [godoc][godoc] for pretty documentation or: 19 | 20 | # in the eventsource package directory 21 | $ go doc 22 | 23 | [spec]: http://www.w3.org/TR/eventsource/ 24 | [godoc]: http://godoc.org/github.com/bernerdschaefer/eventsource 25 | -------------------------------------------------------------------------------- /internal/eventsource/bench_test.go: -------------------------------------------------------------------------------- 1 | package eventsource 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "io/ioutil" 7 | "strconv" 8 | "testing" 9 | ) 10 | 11 | var benchmarkData []byte 12 | var benchmarkEvents []Event 13 | 14 | func initBenchmarkData() { 15 | var buf bytes.Buffer 16 | e := NewEncoder(&buf) 17 | 18 | for i := int64(0); i < 1000; i++ { 19 | event := Event{ 20 | Data: strconv.AppendInt(nil, i, 10), 21 | } 22 | 23 | benchmarkEvents = append(benchmarkEvents, event) 24 | e.Encode(event) 25 | } 26 | 27 | benchmarkData = buf.Bytes() 28 | } 29 | 30 | func BenchmarkDecoder(b *testing.B) { 31 | if benchmarkData == nil { 32 | b.StopTimer() 33 | initBenchmarkData() 34 | b.StartTimer() 35 | } 36 | var buf bytes.Buffer 37 | dec := NewDecoder(&buf) 38 | for i := 0; i < b.N; i++ { 39 | buf.Write(benchmarkData) 40 | 41 | var err error 42 | for err != io.EOF { 43 | var event Event 44 | err = dec.Decode(&event) 45 | 46 | if err != nil && err != io.EOF { 47 | b.Fatal("Decode:", err) 48 | } 49 | } 50 | } 51 | b.SetBytes(int64(len(benchmarkData))) 52 | } 53 | 54 | func BenchmarkEncoder(b *testing.B) { 55 | if benchmarkData == nil { 56 | b.StopTimer() 57 | initBenchmarkData() 58 | b.StartTimer() 59 | } 60 | enc := NewEncoder(ioutil.Discard) 61 | for i := 0; i < b.N; i++ { 62 | for _, e := range benchmarkEvents { 63 | if err := enc.Encode(e); err != nil { 64 | b.Fatal("Encode:", err) 65 | } 66 | } 67 | } 68 | b.SetBytes(int64(len(benchmarkData))) 69 | } 70 | -------------------------------------------------------------------------------- /internal/eventsource/decoder.go: -------------------------------------------------------------------------------- 1 | package eventsource 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "io" 7 | "unicode/utf8" 8 | ) 9 | 10 | // A Decoder reads and decodes EventSource events from an input stream. 11 | type Decoder struct { 12 | r *bufio.Reader 13 | 14 | checkedBOM bool 15 | } 16 | 17 | // NewDecoder returns a new decoder that reads from r. 18 | func NewDecoder(r io.Reader) *Decoder { 19 | return &Decoder{r: bufio.NewReader(r)} 20 | } 21 | 22 | func (d *Decoder) checkBOM() { 23 | r, _, err := d.r.ReadRune() 24 | 25 | if err != nil { 26 | // let other other callers handle this 27 | return 28 | } 29 | 30 | if r != 0xFEFF { // utf8 byte order mark 31 | d.r.UnreadRune() 32 | } 33 | 34 | d.checkedBOM = true 35 | } 36 | 37 | // ReadField reads a single line from the stream and parses it as a field. A 38 | // complete event is signalled by an empty key and value. The returned error 39 | // may either be an error from the stream, or an ErrInvalidEncoding if the 40 | // value is not valid UTF-8. 41 | func (d *Decoder) ReadField() (field string, value []byte, err error) { 42 | if !d.checkedBOM { 43 | d.checkBOM() 44 | } 45 | 46 | var buf []byte 47 | 48 | for { 49 | line, isPrefix, err := d.r.ReadLine() 50 | 51 | if err != nil { 52 | return "", nil, err 53 | } 54 | 55 | buf = append(buf, line...) 56 | 57 | if !isPrefix { 58 | break 59 | } 60 | } 61 | 62 | if len(buf) == 0 { 63 | return "", nil, nil 64 | } 65 | 66 | parts := bytes.SplitN(buf, []byte{':'}, 2) 67 | field = string(parts[0]) 68 | 69 | if len(parts) == 2 { 70 | value = parts[1] 71 | } 72 | 73 | // §7. If value starts with a U+0020 SPACE character, remove it from value. 74 | if len(value) > 0 && value[0] == ' ' { 75 | value = value[1:] 76 | } 77 | 78 | if !utf8.ValidString(field) || !utf8.Valid(value) { 79 | err = ErrInvalidEncoding 80 | } 81 | 82 | return 83 | } 84 | 85 | // Decode reads the next event from its input and stores it in the provided 86 | // Event pointer. 87 | func (d *Decoder) Decode(e *Event) error { 88 | var wroteData bool 89 | 90 | // set default event type 91 | e.Type = "message" 92 | 93 | for { 94 | field, value, err := d.ReadField() 95 | 96 | if err != nil { 97 | return err 98 | } 99 | 100 | if len(field) == 0 && len(value) == 0 { 101 | break 102 | } 103 | 104 | switch field { 105 | case "id": 106 | e.ID = string(value) 107 | if len(e.ID) == 0 { 108 | e.ResetID = true 109 | } 110 | case "retry": 111 | e.Retry = string(value) 112 | case "event": 113 | e.Type = string(value) 114 | case "data": 115 | if wroteData { 116 | e.Data = append(e.Data, '\n') 117 | } else { 118 | wroteData = true 119 | } 120 | e.Data = append(e.Data, value...) 121 | } 122 | } 123 | 124 | return nil 125 | } 126 | -------------------------------------------------------------------------------- /internal/eventsource/decoder_test.go: -------------------------------------------------------------------------------- 1 | package eventsource 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func longLine() string { 10 | buf := make([]byte, 4096) 11 | for i := 0; i < len(buf); i++ { 12 | buf[i] = 'a' 13 | } 14 | 15 | return string(buf) 16 | } 17 | 18 | func TestDecoderReadField(t *testing.T) { 19 | table := []struct { 20 | in string 21 | field string 22 | value []byte 23 | err error 24 | }{ 25 | {"\n", "", nil, nil}, 26 | {"id", "id", nil, nil}, 27 | {"id:", "id", nil, nil}, 28 | {"id:1", "id", []byte("1"), nil}, 29 | {"id: 1", "id", []byte("1"), nil}, 30 | {"data: " + longLine(), "data", []byte(longLine()), nil}, 31 | {"\xFF\xFE\xFD", "\xFF\xFE\xFD", nil, ErrInvalidEncoding}, 32 | {"data: \xFF\xFE\xFD", "data", []byte("\xFF\xFE\xFD"), ErrInvalidEncoding}, 33 | } 34 | 35 | for i, tt := range table { 36 | dec := NewDecoder(bytes.NewBufferString(tt.in)) 37 | 38 | field, value, err := dec.ReadField() 39 | 40 | if err != tt.err { 41 | t.Errorf("%d. expected err=%q, got %q", i, tt.err, err) 42 | continue 43 | } 44 | 45 | if exp, got := tt.field, field; exp != got { 46 | t.Errorf("%d. expected field=%q, got %q", i, exp, got) 47 | } 48 | 49 | if exp, got := tt.value, value; !bytes.Equal(exp, got) { 50 | t.Errorf("%d. expected value=%q, got %q", i, exp, got) 51 | } 52 | } 53 | } 54 | 55 | func TestDecoderDecode(t *testing.T) { 56 | table := []struct { 57 | in string 58 | out Event 59 | }{ 60 | {"event: type\ndata\n\n", Event{Type: "type"}}, 61 | {"id: 123\ndata\n\n", Event{Type: "message", ID: "123"}}, 62 | {"retry: 10000\ndata\n\n", Event{Type: "message", Retry: "10000"}}, 63 | {"data: data\n\n", Event{Type: "message", Data: []byte("data")}}, 64 | {"id\ndata\n\n", Event{Type: "message", ResetID: true}}, 65 | } 66 | 67 | for i, tt := range table { 68 | dec := NewDecoder(bytes.NewBufferString(tt.in)) 69 | 70 | var event Event 71 | if err := dec.Decode(&event); err != nil { 72 | t.Errorf("%d. %s", i, err) 73 | continue 74 | } 75 | 76 | if !reflect.DeepEqual(event, tt.out) { 77 | t.Errorf("%d. expected %#v, got %#v", i, tt.out, event) 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /internal/eventsource/encoder.go: -------------------------------------------------------------------------------- 1 | package eventsource 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "unicode/utf8" 8 | ) 9 | 10 | // The FlushWriter interface groups basic Write and Flush methods. 11 | type FlushWriter interface { 12 | io.Writer 13 | Flush() 14 | } 15 | 16 | // Adds a noop Flush method to a normal io.Writer. 17 | type noopFlusher struct { 18 | io.Writer 19 | } 20 | 21 | func (noopFlusher) Flush() {} 22 | 23 | // Encoder writes EventSource events to an output stream. 24 | type Encoder struct { 25 | w FlushWriter 26 | } 27 | 28 | // NewEncoder returns a new encoder that writes to w. 29 | func NewEncoder(w io.Writer) *Encoder { 30 | if w, ok := w.(FlushWriter); ok { 31 | return &Encoder{w} 32 | } 33 | 34 | return &Encoder{noopFlusher{w}} 35 | } 36 | 37 | // Flush sends an empty line to signal event is complete, and flushes the 38 | // writer. 39 | func (e *Encoder) Flush() error { 40 | _, err := e.w.Write([]byte{'\n'}) 41 | e.w.Flush() 42 | return err 43 | } 44 | 45 | // WriteField writes an event field to the connection. If the provided value 46 | // contains newlines, multiple fields will be emitted. If the returned error is 47 | // not nil, it will be either ErrInvalidEncoding or an error from the 48 | // connection. 49 | func (e *Encoder) WriteField(field string, value []byte) error { 50 | if !utf8.ValidString(field) || !utf8.Valid(value) { 51 | return ErrInvalidEncoding 52 | } 53 | 54 | for _, line := range bytes.Split(value, []byte{'\n'}) { 55 | if len(line) > 0 && line[len(line)-1] == '\r' { 56 | line = line[:len(line)-1] 57 | } 58 | 59 | if err := e.writeField(field, line); err != nil { 60 | return err 61 | } 62 | } 63 | 64 | return nil 65 | } 66 | 67 | func (e *Encoder) writeField(field string, value []byte) (err error) { 68 | if len(value) == 0 { 69 | _, err = fmt.Fprintf(e.w, "%s\n", field) 70 | } else { 71 | _, err = fmt.Fprintf(e.w, "%s: %s\n", field, value) 72 | } 73 | 74 | return 75 | } 76 | 77 | // Encode writes an event to the connection. 78 | func (e *Encoder) Encode(event Event) error { 79 | if event.ResetID || len(event.ID) > 0 { 80 | if err := e.WriteField("id", []byte(event.ID)); err != nil { 81 | return err 82 | } 83 | } 84 | 85 | if len(event.Retry) > 0 { 86 | if err := e.WriteField("retry", []byte(event.Retry)); err != nil { 87 | return err 88 | } 89 | } 90 | 91 | if len(event.Type) > 0 { 92 | if err := e.WriteField("event", []byte(event.Type)); err != nil { 93 | return err 94 | } 95 | } 96 | 97 | if err := e.WriteField("data", event.Data); err != nil { 98 | return err 99 | } 100 | 101 | return e.Flush() 102 | } 103 | -------------------------------------------------------------------------------- /internal/eventsource/encoder_test.go: -------------------------------------------------------------------------------- 1 | package eventsource 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "testing" 7 | ) 8 | 9 | type testFlusher struct { 10 | in bytes.Buffer 11 | out bytes.Buffer 12 | } 13 | 14 | func (f *testFlusher) Write(data []byte) (int, error) { 15 | return f.in.Write(data) 16 | } 17 | 18 | func (f *testFlusher) Flush() { 19 | io.Copy(&f.out, &f.in) 20 | } 21 | 22 | func TestEncoderFlush(t *testing.T) { 23 | buf := &testFlusher{} 24 | enc := NewEncoder(buf) 25 | enc.WriteField("data", []byte("data")) 26 | enc.Flush() 27 | 28 | if buf.out.String() != "data: data\n\n" { 29 | t.Fatal("Encoder.Flush did not flush underlying writer") 30 | } 31 | } 32 | 33 | func TestWriteField(t *testing.T) { 34 | table := []struct { 35 | field string 36 | value []byte 37 | out string 38 | error 39 | }{ 40 | {"data", []byte("data"), "data: data\n", nil}, 41 | {"data", nil, "data\n", nil}, 42 | {"\xFF\xFE\xFD", nil, "", ErrInvalidEncoding}, 43 | {"data", []byte("\xFF\xFE\xFD"), "", ErrInvalidEncoding}, 44 | {"data", []byte("a\nb\nc\n"), "data: a\ndata: b\ndata: c\ndata\n", nil}, 45 | {"data", []byte("a\r\nb\r\nc"), "data: a\ndata: b\ndata: c\n", nil}, 46 | } 47 | 48 | for i, tt := range table { 49 | buf := new(bytes.Buffer) 50 | 51 | err := NewEncoder(buf).WriteField(tt.field, tt.value) 52 | 53 | if tt.error != nil && err == tt.error { 54 | continue 55 | } 56 | 57 | if tt.error != err { 58 | t.Errorf("%d. expected err=%q, got %q", i, tt.error, err) 59 | continue 60 | } 61 | 62 | if buf.String() != tt.out { 63 | t.Errorf("%d. expected %q, got %q", i, tt.out, buf.String()) 64 | } 65 | } 66 | } 67 | 68 | func TestEncoderEncode(t *testing.T) { 69 | table := []struct { 70 | Event 71 | expected string 72 | }{ 73 | {Event{Type: "type"}, "event: type\ndata\n\n"}, 74 | {Event{ID: "123"}, "id: 123\ndata\n\n"}, 75 | {Event{Retry: "10000"}, "retry: 10000\ndata\n\n"}, 76 | {Event{Data: []byte("data")}, "data: data\n\n"}, 77 | {Event{ResetID: true}, "id\ndata\n\n"}, 78 | } 79 | 80 | for i, tt := range table { 81 | buf := new(bytes.Buffer) 82 | 83 | if err := NewEncoder(buf).Encode(tt.Event); err != nil { 84 | t.Errorf("%d. write error: %q", i, err) 85 | continue 86 | } 87 | 88 | if buf.String() != tt.expected { 89 | t.Errorf("%d. expected %q, got %q", i, tt.expected, buf.String()) 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /internal/eventsource/eventsource.go: -------------------------------------------------------------------------------- 1 | // Package eventsource provides the building blocks for consuming and building 2 | // EventSource services. 3 | package eventsource 4 | 5 | import ( 6 | "context" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "mime" 11 | "net/http" 12 | "strconv" 13 | "time" 14 | ) 15 | 16 | var ( 17 | // ErrClosed signals that the event source has been closed and will not be 18 | // reopened. 19 | ErrClosed = errors.New("closed") 20 | 21 | // ErrInvalidEncoding is returned by Encoder and Decoder when invalid UTF-8 22 | // event data is encountered. 23 | ErrInvalidEncoding = errors.New("invalid UTF-8 sequence") 24 | ) 25 | 26 | // An Event is a message can be written to an event stream and read from an 27 | // event source. 28 | type Event struct { 29 | Type string 30 | ID string 31 | Retry string 32 | Data []byte 33 | ResetID bool 34 | } 35 | 36 | // An EventSource consumes server sent events over HTTP with automatic 37 | // recovery. 38 | type EventSource struct { 39 | retry time.Duration 40 | request *http.Request 41 | err error 42 | r io.ReadCloser 43 | dec *Decoder 44 | lastEventID string 45 | } 46 | 47 | // New prepares an EventSource. The connection is automatically managed, using 48 | // req to connect, and retrying from recoverable errors after waiting the 49 | // provided retry duration. 50 | func New(req *http.Request, retry time.Duration) *EventSource { 51 | req.Header.Set("Accept", "text/event-stream") 52 | req.Header.Set("Cache-Control", "no-cache") 53 | 54 | return &EventSource{ 55 | retry: retry, 56 | request: req, 57 | } 58 | } 59 | 60 | // Close the source. Any further calls to Read() will return ErrClosed. 61 | func (es *EventSource) Close() { 62 | if es.r != nil { 63 | es.r.Close() 64 | } 65 | es.err = ErrClosed 66 | } 67 | 68 | // Connect to an event source, validate the response, and gracefully handle 69 | // reconnects. 70 | func (es *EventSource) connect() { 71 | for es.err == nil { 72 | if es.r != nil { 73 | es.r.Close() 74 | <-time.After(es.retry) 75 | } 76 | 77 | es.request.Header.Set("Last-Event-Id", es.lastEventID) 78 | 79 | resp, err := http.DefaultClient.Do(es.request) 80 | 81 | if err != nil { 82 | continue // reconnect 83 | } 84 | 85 | if resp.StatusCode >= 500 { 86 | // assumed to be temporary, try reconnecting 87 | resp.Body.Close() 88 | } else if resp.StatusCode == 204 { 89 | resp.Body.Close() 90 | es.err = ErrClosed 91 | } else if resp.StatusCode != 200 { 92 | resp.Body.Close() 93 | es.err = fmt.Errorf("endpoint returned unrecoverable status %q", resp.Status) 94 | } else { 95 | mediatype, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type")) 96 | 97 | if mediatype != "text/event-stream" { 98 | resp.Body.Close() 99 | es.err = fmt.Errorf("invalid content type %q", resp.Header.Get("Content-Type")) 100 | } else { 101 | es.r = resp.Body 102 | es.dec = NewDecoder(es.r) 103 | return 104 | } 105 | } 106 | } 107 | } 108 | 109 | // Read an event from EventSource. If an error is returned, the EventSource 110 | // will not reconnect, and any further call to Read() will return the same 111 | // error. 112 | func (es *EventSource) Read() (Event, error) { 113 | if es.r == nil { 114 | es.connect() 115 | } 116 | 117 | for es.err == nil { 118 | var e Event 119 | 120 | err := es.dec.Decode(&e) 121 | 122 | if err == ErrInvalidEncoding { 123 | continue 124 | } 125 | 126 | // peterbourgon: context cancelation is terminal 127 | if err == context.Canceled { 128 | return Event{}, err 129 | } 130 | 131 | if err != nil { 132 | es.connect() 133 | continue 134 | } 135 | 136 | // peterbourgon: empty data should be OK 137 | //if len(e.Data) == 0 { 138 | // continue 139 | //} 140 | 141 | if len(e.ID) > 0 || e.ResetID { 142 | es.lastEventID = e.ID 143 | } 144 | 145 | if len(e.Retry) > 0 { 146 | if retry, err := strconv.Atoi(e.Retry); err == nil { 147 | es.retry = time.Duration(retry) * time.Millisecond 148 | } 149 | } 150 | 151 | return e, nil 152 | } 153 | 154 | return Event{}, es.err 155 | } 156 | -------------------------------------------------------------------------------- /internal/eventsource/eventsource_test.go: -------------------------------------------------------------------------------- 1 | package eventsource 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/http" 7 | "net/http/httptest" 8 | "reflect" 9 | "strconv" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | type responseWriter interface { 15 | http.ResponseWriter 16 | http.Flusher 17 | http.CloseNotifier 18 | } 19 | 20 | func testServer(f func(responseWriter, *http.Request)) *httptest.Server { 21 | return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 22 | f(w.(responseWriter), r) 23 | })) 24 | } 25 | 26 | func request(url string) *http.Request { 27 | req, _ := http.NewRequest("GET", url, nil) 28 | return req 29 | } 30 | 31 | func TestEventSourceHeaders(t *testing.T) { 32 | headers := make(chan http.Header) 33 | server := testServer(func(w responseWriter, r *http.Request) { 34 | headers <- r.Header 35 | }) 36 | defer server.Close() 37 | 38 | es := New(request(server.URL), -1) 39 | go es.connect() 40 | 41 | h := <-headers 42 | 43 | if h.Get("Accept") != "text/event-stream" { 44 | t.Errorf("expected accept header = %q, got %q", "text/event-stream", h.Get("Accept")) 45 | } 46 | 47 | if h.Get("Cache-Control") != "no-cache" { 48 | t.Errorf("expected cache control header = %q, got %q", "no-cache", h.Get("Cache-Control")) 49 | } 50 | } 51 | 52 | func TestEventSource204(t *testing.T) { 53 | server := testServer(func(w responseWriter, r *http.Request) { 54 | w.WriteHeader(204) 55 | }) 56 | defer server.Close() 57 | 58 | es := New(request(server.URL), -1) 59 | 60 | es.connect() 61 | 62 | if es.err == nil { 63 | t.Fatal("event source did not close on 204") 64 | } 65 | } 66 | 67 | func TestEventSource(t *testing.T) { 68 | server := testServer(func(w responseWriter, r *http.Request) { 69 | w.WriteHeader(200) 70 | }) 71 | defer server.Close() 72 | 73 | es := New(request(server.URL), time.Millisecond) 74 | 75 | es.connect() 76 | 77 | if es.err == nil { 78 | t.Fatal("event source did not close on 200 with no content type") 79 | } 80 | } 81 | 82 | func TestEventSourceEmphemeral500(t *testing.T) { 83 | fail := true 84 | 85 | server := testServer(func(w responseWriter, r *http.Request) { 86 | if fail { 87 | w.WriteHeader(500) 88 | } else { 89 | w.Header().Set("Content-Type", "text/event-stream") 90 | w.WriteHeader(200) 91 | } 92 | 93 | fail = !fail 94 | }) 95 | defer server.Close() 96 | 97 | es := New(request(server.URL), time.Millisecond) 98 | 99 | es.connect() 100 | 101 | if es.err != nil { 102 | t.Fatalf("event source did not reconnect on 500; got %q", es.err) 103 | } 104 | } 105 | 106 | func TestEventSourceRead(t *testing.T) { 107 | fail := make(chan struct{}) 108 | more := make(chan bool, 1) 109 | server := testServer(func(w responseWriter, r *http.Request) { 110 | select { 111 | case <-fail: 112 | w.WriteHeader(204) 113 | return 114 | default: 115 | } 116 | w.Header().Set("Content-Type", "text/event-stream") 117 | w.WriteHeader(200) 118 | 119 | var id int 120 | 121 | if lastID := r.Header.Get("Last-Event-Id"); lastID != "" { 122 | if i, err := strconv.ParseInt(lastID, 10, 64); err == nil { 123 | id = int(i) + 1 124 | } 125 | } 126 | 127 | for { 128 | if !<-more { 129 | break 130 | } 131 | fmt.Fprintf(w, "id: %d\ndata: message %d\n\n", id, id) 132 | w.Flush() 133 | id++ 134 | } 135 | }) 136 | defer server.Close() 137 | defer close(more) 138 | 139 | es := New(request(server.URL), -1) 140 | more <- true 141 | 142 | event, err := es.Read() 143 | if err != nil { 144 | t.Fatal(err) 145 | } 146 | 147 | if event.ID != "0" { 148 | t.Fatalf("expected id = 0, got %s", event.ID) 149 | } 150 | 151 | if event.Type != "message" { 152 | t.Fatalf("expected event = message, got %s", event.Type) 153 | } 154 | 155 | if !bytes.Equal([]byte("message 0"), event.Data) { 156 | t.Fatalf("expected data = message 0, got %s", event.Data) 157 | } 158 | 159 | more <- true 160 | event, err = es.Read() 161 | if err != nil { 162 | t.Fatal(err) 163 | } 164 | 165 | if event.ID != "1" { 166 | t.Fatalf("expected id = 1, got %s", event.ID) 167 | } 168 | 169 | if event.Type != "message" { 170 | t.Fatalf("expected event = message, got %s", event.Type) 171 | } 172 | 173 | if !bytes.Equal([]byte("message 1"), event.Data) { 174 | t.Fatalf("expected data = message 1, got %s", event.Data) 175 | } 176 | 177 | // stop handler 178 | more <- false 179 | // start handler 180 | more <- true 181 | event, err = es.Read() 182 | if err != nil { 183 | t.Fatal(err) 184 | } 185 | 186 | if event.ID != "2" { 187 | t.Fatalf("expected id = 2, got %s", event.ID) 188 | } 189 | 190 | if event.Type != "message" { 191 | t.Fatalf("expected event = message, got %s", event.Type) 192 | } 193 | 194 | if !bytes.Equal([]byte("message 2"), event.Data) { 195 | t.Fatalf("expected data = message 2, got %s", event.Data) 196 | } 197 | 198 | more <- false 199 | close(fail) 200 | 201 | if _, err := es.Read(); err == nil { 202 | t.Fatal("expected fatal err") 203 | } 204 | } 205 | 206 | func TestEventSourceChangeRetry(t *testing.T) { 207 | server := testServer(func(w responseWriter, r *http.Request) { 208 | w.Header().Set("Content-Type", "text/event-stream") 209 | w.WriteHeader(200) 210 | 211 | NewEncoder(w).Encode(Event{ 212 | Retry: "10000", 213 | Data: []byte("foo"), 214 | }) 215 | }) 216 | 217 | defer server.Close() 218 | 219 | es := New(request(server.URL), -1) 220 | 221 | event, err := es.Read() 222 | 223 | if err != nil { 224 | t.Fatal(err) 225 | } 226 | 227 | if event.Retry != "10000" { 228 | t.Error("event retry not set") 229 | } 230 | 231 | if es.retry != (10 * time.Second) { 232 | t.Fatal("expected retry to be updated, but wasn't") 233 | } 234 | } 235 | 236 | func TestEventSourceBOM(t *testing.T) { 237 | server := testServer(func(w responseWriter, r *http.Request) { 238 | w.Header().Set("Content-Type", "text/event-stream") 239 | w.WriteHeader(200) 240 | 241 | w.Write([]byte("\xEF\xBB\xBF")) 242 | NewEncoder(w).Encode(Event{Type: "custom", Data: []byte("foo")}) 243 | }) 244 | defer server.Close() 245 | 246 | es := New(request(server.URL), -1) 247 | 248 | event, err := es.Read() 249 | 250 | if err != nil { 251 | t.Fatal(err) 252 | } 253 | 254 | if !reflect.DeepEqual(event, Event{Type: "custom", Data: []byte("foo")}) { 255 | t.Fatal("message was unsuccessfully decoded with BOM") 256 | } 257 | } 258 | -------------------------------------------------------------------------------- /internal/eventsource/example_test.go: -------------------------------------------------------------------------------- 1 | package eventsource_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/bernerdschaefer/eventsource" 6 | "io" 7 | "log" 8 | "net/http" 9 | "os" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | func ExampleHandler() { 15 | http.Handle("/events", eventsource.Handler(func(lastID string, e *eventsource.Encoder, stop <-chan bool) { 16 | for { 17 | select { 18 | case <-time.After(200 * time.Millisecond): 19 | e.Encode(eventsource.Event{Data: []byte("tick")}) 20 | case <-stop: 21 | return 22 | } 23 | } 24 | })) 25 | } 26 | 27 | func ExampleHandler_ServeHTTP() { 28 | es := eventsource.Handler(func(lastID string, e *eventsource.Encoder, stop <-chan bool) { 29 | for { 30 | select { 31 | case <-time.After(200 * time.Millisecond): 32 | e.Encode(eventsource.Event{Data: []byte("tick")}) 33 | case <-stop: 34 | return 35 | } 36 | } 37 | }) 38 | 39 | http.HandleFunc("/events", func(w http.ResponseWriter, r *http.Request) { 40 | if r.Header.Get("Authorization") == "" { 41 | w.WriteHeader(http.StatusUnauthorized) 42 | return 43 | } 44 | 45 | es.ServeHTTP(w, r) 46 | }) 47 | } 48 | 49 | func ExampleEncoder() { 50 | enc := eventsource.NewEncoder(os.Stdout) 51 | 52 | events := []eventsource.Event{ 53 | {ID: "1", Data: []byte("data")}, 54 | {ResetID: true, Data: []byte("id reset")}, 55 | {Type: "add", Data: []byte("1")}, 56 | } 57 | 58 | for _, event := range events { 59 | if err := enc.Encode(event); err != nil { 60 | log.Fatal(err) 61 | } 62 | } 63 | 64 | if err := enc.WriteField("", []byte("heartbeat")); err != nil { 65 | log.Fatal(err) 66 | } 67 | 68 | if err := enc.Flush(); err != nil { 69 | log.Fatal(err) 70 | } 71 | 72 | // Output: 73 | // id: 1 74 | // data: data 75 | // 76 | // id 77 | // data: id reset 78 | // 79 | // event: add 80 | // data: 1 81 | // 82 | // : heartbeat 83 | // 84 | } 85 | 86 | func ExampleDecoder() { 87 | stream := strings.NewReader(`id: 1 88 | event: add 89 | data: 123 90 | 91 | id: 2 92 | event: remove 93 | data: 321 94 | 95 | id: 3 96 | event: add 97 | data: 123 98 | 99 | `) 100 | dec := eventsource.NewDecoder(stream) 101 | 102 | for { 103 | var event eventsource.Event 104 | err := dec.Decode(&event) 105 | 106 | if err == io.EOF { 107 | break 108 | } else if err != nil { 109 | log.Fatal(err) 110 | } 111 | 112 | fmt.Printf("%s. %s %s\n", event.ID, event.Type, event.Data) 113 | } 114 | 115 | // Output: 116 | // 1. add 123 117 | // 2. remove 321 118 | // 3. add 123 119 | } 120 | 121 | func ExampleNew() { 122 | req, _ := http.NewRequest("GET", "http://localhost:9090/events", nil) 123 | req.SetBasicAuth("user", "pass") 124 | 125 | es := eventsource.New(req, 3*time.Second) 126 | 127 | for { 128 | event, err := es.Read() 129 | 130 | if err != nil { 131 | log.Fatal(err) 132 | } 133 | 134 | log.Printf("%s. %s %s\n", event.ID, event.Type, event.Data) 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /internal/eventsource/handler.go: -------------------------------------------------------------------------------- 1 | package eventsource 2 | 3 | import ( 4 | "mime" 5 | "net/http" 6 | "strings" 7 | ) 8 | 9 | // Handler is an adapter for ordinary functions to act as an HTTP handler for 10 | // event sources. It receives the ID of the last event processed by the client, 11 | // and Encoder to deliver messages, and a channel to be notified if the client 12 | // connection is closed. 13 | type Handler func(lastId string, encoder *Encoder, stop <-chan bool) 14 | 15 | func (h Handler) acceptable(accept string) bool { 16 | if accept == "" { 17 | // The absense of an Accept header is equivalent to "*/*". 18 | // https://tools.ietf.org/html/rfc2296#section-4.2.2 19 | return true 20 | } 21 | 22 | for _, a := range strings.Split(accept, ",") { 23 | mediatype, _, err := mime.ParseMediaType(a) 24 | if err != nil { 25 | continue 26 | } 27 | 28 | if mediatype == "text/event-stream" || mediatype == "text/*" || mediatype == "*/*" { 29 | return true 30 | } 31 | } 32 | 33 | return false 34 | } 35 | 36 | // ServeHTTP calls h with an Encoder and a close notification channel. It 37 | // performs Content-Type negotiation. 38 | func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 39 | w.Header().Set("Cache-Control", "no-cache") 40 | w.Header().Set("Vary", "Accept") 41 | 42 | if !h.acceptable(r.Header.Get("Accept")) { 43 | w.WriteHeader(http.StatusNotAcceptable) 44 | return 45 | } 46 | 47 | w.Header().Set("Content-Type", "text/event-stream") 48 | w.WriteHeader(http.StatusOK) 49 | 50 | var stop <-chan bool 51 | 52 | if notifier, ok := w.(http.CloseNotifier); ok { 53 | stop = notifier.CloseNotify() 54 | } 55 | 56 | h(r.Header.Get("Last-Event-Id"), NewEncoder(w), stop) 57 | } 58 | -------------------------------------------------------------------------------- /internal/eventsource/handler_test.go: -------------------------------------------------------------------------------- 1 | package eventsource 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "reflect" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | var emptyHandler Handler = func(id string, e *Encoder, s <-chan bool) {} 12 | 13 | type testCloseNotifier struct { 14 | closed chan bool 15 | http.ResponseWriter 16 | } 17 | 18 | func (n testCloseNotifier) Close() { 19 | n.closed <- true 20 | } 21 | 22 | func (n testCloseNotifier) CloseNotify() <-chan bool { 23 | return n.closed 24 | } 25 | 26 | func TestHandlerAcceptable(t *testing.T) { 27 | table := []struct { 28 | accept string 29 | result bool 30 | }{ 31 | {"", true}, 32 | {"text/event-stream", true}, 33 | {"text/*", true}, 34 | {"*/*", true}, 35 | {"text/event-stream; q=1.0", true}, 36 | {"text/*; q=1.0", true}, 37 | {"*/*; q=1.0", true}, 38 | {"text/html; q=1.0, text/*; q=0.8", true}, 39 | {"text/html; q=1.0, image/gif; q=0.6, image/jpeg; q=0.6", false}, 40 | } 41 | 42 | for i, tt := range table { 43 | if exp, got := tt.result, emptyHandler.acceptable(tt.accept); exp != got { 44 | t.Errorf("%d. expected acceptable(%q) == %t, got %t", i, tt.accept, exp, got) 45 | } 46 | } 47 | } 48 | 49 | func TestHandlerValidatesAcceptHeader(t *testing.T) { 50 | w, r := httptest.NewRecorder(), &http.Request{Header: map[string][]string{ 51 | "Accept": []string{"text/html"}, 52 | }} 53 | emptyHandler.ServeHTTP(w, r) 54 | 55 | if w.Code != http.StatusNotAcceptable { 56 | t.Fatal("handler did not set 406 status") 57 | } 58 | } 59 | 60 | func TestHandlerSetsContentType(t *testing.T) { 61 | w, r := httptest.NewRecorder(), &http.Request{Header: map[string][]string{ 62 | "Accept": []string{"text/event-stream"}, 63 | }} 64 | emptyHandler.ServeHTTP(w, r) 65 | 66 | if w.HeaderMap.Get("Content-Type") != "text/event-stream" { 67 | t.Fatal("handler did not set appropriate content type") 68 | } 69 | 70 | if w.Code != http.StatusOK { 71 | t.Fatal("handler did not set 200 status") 72 | } 73 | } 74 | 75 | func TestHandlerEncode(t *testing.T) { 76 | handler := func(lastID string, enc *Encoder, stop <-chan bool) { 77 | enc.Encode(Event{Data: []byte("hello")}) 78 | } 79 | 80 | w, r := httptest.NewRecorder(), &http.Request{Header: map[string][]string{ 81 | "Accept": []string{"text/event-stream"}, 82 | }} 83 | 84 | Handler(handler).ServeHTTP(w, r) 85 | 86 | var event Event 87 | NewDecoder(w.Body).Decode(&event) 88 | 89 | if !reflect.DeepEqual(event, Event{Type: "message", Data: []byte("hello")}) { 90 | t.Error("unexpected handler output") 91 | } 92 | } 93 | 94 | func TestHandlerCloseNotify(t *testing.T) { 95 | done := make(chan bool, 1) 96 | handler := func(lastID string, enc *Encoder, stop <-chan bool) { 97 | <-stop 98 | done <- true 99 | } 100 | 101 | w, r := httptest.NewRecorder(), &http.Request{Header: map[string][]string{ 102 | "Accept": []string{"text/event-stream"}, 103 | }} 104 | closer := testCloseNotifier{make(chan bool, 1), w} 105 | go Handler(handler).ServeHTTP(closer, r) 106 | 107 | closer.Close() 108 | select { 109 | case <-done: 110 | case <-time.After(time.Millisecond): 111 | t.Error("handler was not notified of close") 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /internal/eventsource/identity_test.go: -------------------------------------------------------------------------------- 1 | package eventsource 2 | 3 | import ( 4 | "io" 5 | "reflect" 6 | "strconv" 7 | "testing" 8 | ) 9 | 10 | func randomEvents() []Event { 11 | events := make([]Event, 1000) 12 | 13 | for i := 0; i < 1000; i++ { 14 | event := Event{Type: "custom"} 15 | 16 | if i%3 == 0 { 17 | event.Type = "other" 18 | } 19 | 20 | if i%5 == 0 { 21 | event.Retry = strconv.FormatInt(int64(i*1000), 10) 22 | } 23 | 24 | if i%10 == 0 { 25 | event.ResetID = true 26 | } else { 27 | event.ID = strconv.FormatInt(int64(i), 10) 28 | } 29 | 30 | if i%20 != 0 { 31 | event.Data = []byte(strconv.FormatInt(int64(i), 10)) 32 | } 33 | 34 | events[i] = event 35 | } 36 | 37 | return events 38 | } 39 | 40 | func TestEncodeDecodeIdentity(t *testing.T) { 41 | r, w := io.Pipe() 42 | d, e := NewDecoder(r), NewEncoder(w) 43 | 44 | in := randomEvents() 45 | 46 | go func() { 47 | for _, event := range in { 48 | if err := e.Encode(event); err != nil { 49 | t.Fatal(err) 50 | } 51 | } 52 | 53 | w.Close() 54 | }() 55 | 56 | out := make([]Event, 0, len(in)) 57 | 58 | for { 59 | var event Event 60 | 61 | if d.Decode(&event) != nil { 62 | break 63 | } 64 | 65 | out = append(out, event) 66 | } 67 | 68 | if !reflect.DeepEqual(in, out) { 69 | t.Fatal("output does not match input") 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /protocol/age.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import "fmt" 4 | 5 | // Age of a proposer, incremented by GC process. 6 | // 7 | // rystsov: "In theory we can use ballot numbers as age, but in practice if we 8 | // use caching (1 RTT optimization) we don't want to do it, because then a 9 | // deletion of a single key will invalidate all caches, which isn't a nice 10 | // behavior. Acceptors should keep the age of proposers. It can be zero in the 11 | // beginning, and the GC process is responsible for updating it. Proposers 12 | // should include their age as part of each message they send." 13 | type Age struct { 14 | Counter uint64 15 | ID string 16 | } 17 | 18 | func (a *Age) inc() Age { 19 | a.Counter++ 20 | return *a // copy 21 | } 22 | 23 | // youngerThan uses the language from the paper. 24 | func (a Age) youngerThan(other Age) bool { 25 | return a.Counter < other.Counter // ignoring ID 26 | } 27 | 28 | func (a Age) String() string { 29 | return fmt.Sprintf("%d:%s", a.Counter, a.ID) 30 | } 31 | -------------------------------------------------------------------------------- /protocol/age_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import "testing" 4 | 5 | func TestZeroAgeAlwaysLoses(t *testing.T) { 6 | // We rely on this property in a few places. 7 | for _, input := range []Age{ 8 | {Counter: 0, ID: "a"}, 9 | {Counter: 1, ID: ""}, 10 | {Counter: 1, ID: "b"}, 11 | {Counter: 2, ID: "a"}, 12 | {Counter: 2, ID: "b"}, 13 | } { 14 | t.Run(input.String(), func(t *testing.T) { 15 | var zero Age 16 | if input.youngerThan(zero) { 17 | t.Fatal("this age isn't younger than the zero age") 18 | } 19 | }) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /protocol/ballot.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import "fmt" 4 | 5 | // Ballot models a ballot number, which are maintained by proposers and cached 6 | // by acceptors. 7 | // 8 | // From the paper: "It's convenient to use tuples as ballot numbers. To generate 9 | // it a proposer combines its numerical ID with a local increasing counter: 10 | // (counter, ID)." 11 | type Ballot struct { 12 | Counter uint64 13 | ID string 14 | } 15 | 16 | func (b *Ballot) inc() Ballot { 17 | b.Counter++ 18 | return *b // copy 19 | } 20 | 21 | func (b *Ballot) isZero() bool { 22 | return b.Counter == 0 && b.ID == "" 23 | } 24 | 25 | // From the paper: "To compare ballot tuples, we should compare the first 26 | // component of the tuples and use ID only as a tiebreaker." 27 | func (b *Ballot) greaterThan(other Ballot) bool { 28 | if b.Counter == other.Counter { 29 | return b.ID > other.ID 30 | } 31 | return b.Counter > other.Counter 32 | } 33 | 34 | func (b Ballot) String() string { 35 | if b.isZero() { 36 | return "ø" 37 | } 38 | return fmt.Sprintf("%d/%s", b.Counter, b.ID) 39 | } 40 | -------------------------------------------------------------------------------- /protocol/ballot_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestZeroBallotString(t *testing.T) { 8 | var zero Ballot 9 | if want, have := "ø", zero.String(); want != have { 10 | t.Errorf("want %q, have %q", want, have) 11 | } 12 | } 13 | 14 | func TestZeroBallotAlwaysLoses(t *testing.T) { 15 | // We rely on this property in a few places. 16 | for _, input := range []Ballot{ 17 | {Counter: 0, ID: "a"}, 18 | {Counter: 1, ID: ""}, 19 | {Counter: 1, ID: "b"}, 20 | {Counter: 2, ID: "a"}, 21 | {Counter: 2, ID: "b"}, 22 | } { 23 | t.Run(input.String(), func(t *testing.T) { 24 | var zero Ballot 25 | if zero.greaterThan(input) { 26 | t.Fatal("this ballot isn't greater than the zero ballot") 27 | } 28 | }) 29 | } 30 | } 31 | 32 | func TestBallotIncrement(t *testing.T) { 33 | var ( 34 | orig Ballot 35 | prev = orig.Counter 36 | next = orig.inc() 37 | ) 38 | if want, have := (prev + 1), next.Counter; want != have { 39 | t.Fatalf("returned ballot number: want %d, have %d", want, have) 40 | } 41 | if want, have := (prev + 1), orig.Counter; want != have { 42 | t.Fatalf("persistent ballot number: want %d, have %d", want, have) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /protocol/basic_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "math/rand" 8 | "sync" 9 | "testing" 10 | 11 | "github.com/go-kit/kit/log" 12 | ) 13 | 14 | func TestReadUnwrittenKey(t *testing.T) { 15 | // Build the cluster. 16 | var ( 17 | logger = log.NewLogfmtLogger(testWriter{t}) 18 | a1 = NewMemoryAcceptor("1", log.With(logger, "a", 1)) 19 | a2 = NewMemoryAcceptor("2", log.With(logger, "a", 2)) 20 | a3 = NewMemoryAcceptor("3", log.With(logger, "a", 3)) 21 | p1 = NewMemoryProposer("1", log.With(logger, "p", 1), a1, a2, a3) 22 | p2 = NewMemoryProposer("2", log.With(logger, "p", 2), a1, a2, a3) 23 | p3 = NewMemoryProposer("3", log.With(logger, "p", 3), a1, a2, a3) 24 | ctx = context.Background() 25 | ) 26 | 27 | // Reads on an unwritten key should succeed, with a nil state. 28 | read := func(x []byte) []byte { return x } 29 | 30 | // Test all proposers. 31 | for _, p := range []*MemoryProposer{p1, p2, p3} { 32 | state, _, err := p2.Propose(ctx, "any key", read) 33 | if err != nil { 34 | t.Fatalf("%s: read failed: %v", p.ballot.ID, err) 35 | } 36 | if want, have := []byte(nil), state; !bytes.Equal(want, have) { 37 | t.Fatalf("%s: read: want %q, have %q", p.ballot.ID, want, have) 38 | } 39 | } 40 | } 41 | 42 | func TestInitializeOnlyOnce(t *testing.T) { 43 | // Build the cluster. 44 | var ( 45 | logger = log.NewLogfmtLogger(testWriter{t}) 46 | a1 = NewMemoryAcceptor("1", log.With(logger, "a", 1)) 47 | a2 = NewMemoryAcceptor("2", log.With(logger, "a", 2)) 48 | a3 = NewMemoryAcceptor("3", log.With(logger, "a", 3)) 49 | p1 = NewMemoryProposer("1", log.With(logger, "p", 1), a1, a2, a3) 50 | p2 = NewMemoryProposer("2", log.With(logger, "p", 2), a1, a2, a3) 51 | p3 = NewMemoryProposer("3", log.With(logger, "p", 3), a1, a2, a3) 52 | ctx = context.Background() 53 | ) 54 | 55 | // Model an Initialize-Only-Once distributed register 56 | // (i.e. Synod) with an idempotent change function. 57 | var ( 58 | key, val0 = "k", "val0" 59 | initialize = changeFuncInitializeOnlyOnce(val0) 60 | differentInit = changeFuncInitializeOnlyOnce("alternate value") 61 | stillDifferentInit = changeFuncInitializeOnlyOnce("abcdefghijklmno") 62 | ) 63 | 64 | // The first proposal should work. 65 | have, _, err := p1.Propose(ctx, key, initialize) 66 | if err != nil { 67 | t.Fatal(err) 68 | } 69 | if want, have := "val0", string(have); want != have { 70 | t.Errorf("want %q, have %q", want, have) 71 | } 72 | 73 | // If we make a read from anywhere, we should see the right thing. 74 | have, _, err = p2.Propose(ctx, key, changeFuncRead) 75 | if err != nil { 76 | t.Fatal(err) 77 | } 78 | if want, have := "val0", string(have); want != have { 79 | t.Errorf("want %q, have %q", want, have) 80 | } 81 | have, _, err = p3.Propose(ctx, key, changeFuncRead) 82 | if err != nil { 83 | t.Fatal(err) 84 | } 85 | if want, have := "val0", string(have); want != have { 86 | t.Errorf("want %q, have %q", want, have) 87 | } 88 | 89 | // Subsequent proposals should succeed but leave the value un-altered. 90 | have, _, err = p2.Propose(ctx, key, differentInit) 91 | if err != nil { 92 | t.Fatal(err) 93 | } 94 | if want, have := "val0", string(have); want != have { 95 | t.Errorf("want %q, have %q", want, have) 96 | } 97 | have, _, err = p3.Propose(ctx, key, stillDifferentInit) 98 | if err != nil { 99 | t.Fatal(err) 100 | } 101 | if want, have := "val0", string(have); want != have { 102 | t.Errorf("want %q, have %q", want, have) 103 | } 104 | } 105 | 106 | func TestFastForward(t *testing.T) { 107 | // Build the cluster. 108 | var ( 109 | logger = log.NewLogfmtLogger(testWriter{t}) 110 | a1 = NewMemoryAcceptor("1", log.With(logger, "a", 1)) 111 | a2 = NewMemoryAcceptor("2", log.With(logger, "a", 2)) 112 | a3 = NewMemoryAcceptor("3", log.With(logger, "a", 3)) 113 | p1 = NewMemoryProposer("1", log.With(logger, "p", 1), a1, a2, a3) 114 | p2 = NewMemoryProposer("2", log.With(logger, "p", 2), a1, a2, a3) 115 | p3 = NewMemoryProposer("3", log.With(logger, "p", 3), a1, a2, a3) 116 | ctx = context.Background() 117 | ) 118 | 119 | // Set an initial value, and increment the ballot a few times. 120 | const key, val0 = "k", "asdf" 121 | p1.Propose(ctx, key, changeFuncInitializeOnlyOnce(val0)) // ballot is incremented 122 | p1.Propose(ctx, key, changeFuncRead) // incremented again 123 | p1.Propose(ctx, key, changeFuncRead) // and again 124 | 125 | // Make sure reads thru other proposers succeed. 126 | for name, p := range map[string]*MemoryProposer{ 127 | "p2": p2, "p3": p3, 128 | } { 129 | if val, _, err := p.Propose(ctx, key, changeFuncRead); err != nil { 130 | t.Errorf("%s second read: got unexpected error: %v", name, err) 131 | } else if want, have := val0, string(val); want != have { 132 | t.Errorf("%s second read: want %q, have %q", name, want, have) 133 | } 134 | } 135 | } 136 | 137 | func TestMultiKeyReads(t *testing.T) { 138 | // Build the cluster. 139 | var ( 140 | logger = log.NewLogfmtLogger(testWriter{t}) 141 | a1 = NewMemoryAcceptor("1", log.With(logger, "a", 1)) 142 | a2 = NewMemoryAcceptor("2", log.With(logger, "a", 2)) 143 | a3 = NewMemoryAcceptor("3", log.With(logger, "a", 3)) 144 | p1 = NewMemoryProposer("1", log.With(logger, "p", 1), a1, a2, a3) 145 | p2 = NewMemoryProposer("2", log.With(logger, "p", 2), a1, a2, a3) 146 | p3 = NewMemoryProposer("3", log.With(logger, "p", 3), a1, a2, a3) 147 | ctx = context.Background() 148 | ) 149 | 150 | // If we write a key through one proposer... 151 | p1.Propose(ctx, "k1", changeFuncInitializeOnlyOnce("v1")) 152 | 153 | // And a different key through another proposer... 154 | p2.Propose(ctx, "k2", changeFuncInitializeOnlyOnce("v2")) 155 | 156 | // Reads should work through a still-different proposer. 157 | if val, _, err := p3.Propose(ctx, "k1", changeFuncRead); err != nil { 158 | t.Errorf("read k1 via p3: %v", err) 159 | } else if want, have := "v1", string(val); want != have { 160 | t.Errorf("read k1 via p3: want %q, have %q", want, have) 161 | } 162 | if val, _, err := p3.Propose(ctx, "k2", changeFuncRead); err != nil { 163 | t.Errorf("read k2 via p3: %v", err) 164 | } else if want, have := "v2", string(val); want != have { 165 | t.Errorf("read k2 via p3: want %q, have %q", want, have) 166 | } 167 | } 168 | 169 | func TestConcurrentCASWrites(t *testing.T) { 170 | // Build the cluster. 171 | var ( 172 | logger = log.NewLogfmtLogger(testWriter{t}) 173 | a1 = NewMemoryAcceptor("1", log.With(logger, "a", 1)) 174 | a2 = NewMemoryAcceptor("2", log.With(logger, "a", 2)) 175 | a3 = NewMemoryAcceptor("3", log.With(logger, "a", 3)) 176 | p1 = NewMemoryProposer("1", log.With(logger, "p", 1), a1, a2, a3) 177 | p2 = NewMemoryProposer("2", log.With(logger, "p", 2), a1, a2, a3) 178 | p3 = NewMemoryProposer("3", log.With(logger, "p", 3), a1, a2, a3) 179 | ctx = context.Background() 180 | ) 181 | 182 | // Define a function to pick a random proposer. 183 | randomProposer := func() Proposer { 184 | return []Proposer{p1, p2, p3}[rand.Intn(3)] 185 | } 186 | 187 | // Define a function to generate a bunch of values for a key. 188 | valuesFor := func(key string, n int) [][]byte { 189 | values := make([][]byte, n) 190 | for i := 0; i < n; i++ { 191 | values[i] = []byte(fmt.Sprintf("%s%d", key, i+1)) 192 | } 193 | return values 194 | } 195 | 196 | // Set up some keys, and a sequence of values that we want for each. 197 | mutations := map[string][][]byte{ 198 | "a": valuesFor("a", 997), 199 | "b": valuesFor("b", 998), 200 | "c": valuesFor("c", 999), 201 | "d": valuesFor("d", 1000), 202 | } 203 | 204 | // The compare-and-swap change function. 205 | cas := func(prev, next []byte) ChangeFunc { 206 | return func(current []byte) []byte { 207 | if (current == nil && prev == nil) || (bytes.Compare(current, prev) == 0) { 208 | return next // good 209 | } 210 | return current // bad 211 | } 212 | } 213 | 214 | // Define a worker function to CAS-write all the values in order. 215 | // Each proposal will go to a random proposer, to keep us honest. 216 | worker := func(key string, values [][]byte) { 217 | var prev []byte // initially no value is set 218 | for i, next := range values { 219 | var ( 220 | p = randomProposer() 221 | f = cas(prev, next) 222 | ) 223 | have, _, err := p.Propose(ctx, key, f) 224 | if err != nil { 225 | t.Errorf("%s worker: step %d (%s -> %s): %v", key, i+1, prettyPrint(prev), prettyPrint(next), err) 226 | return 227 | } 228 | if want, have := string(next), string(have); want != have { 229 | t.Errorf("%s worker: step %d (%s -> %s): want %s, have %s", key, i+1, prettyPrint(prev), prettyPrint(next), want, have) 230 | return 231 | } 232 | prev = have 233 | } 234 | } 235 | 236 | // Launch a CAS writer per key. 237 | // They'll do all their writes concurrently. 238 | var wg sync.WaitGroup 239 | for key, values := range mutations { 240 | wg.Add(1) 241 | go func(key string, values [][]byte) { 242 | defer wg.Done() 243 | worker(key, values) 244 | }(key, values) 245 | } 246 | wg.Wait() 247 | 248 | // This is kind of needless, but verify the final state with a read. 249 | for key, values := range mutations { 250 | var ( 251 | final = values[len(values)-1] 252 | have, _, _ = randomProposer().Propose(ctx, key, changeFuncRead) 253 | ) 254 | if want, have := string(final), string(have); want != have { 255 | t.Errorf("%s: final state: want %s, have %s", key, want, have) 256 | } 257 | } 258 | } 259 | 260 | func changeFuncInitializeOnlyOnce(s string) ChangeFunc { 261 | return func(x []byte) []byte { 262 | if x == nil { 263 | return []byte(s) 264 | } 265 | return x 266 | } 267 | } 268 | 269 | func changeFuncCompareAndSwap(prev, next string) ChangeFunc { 270 | return func(x []byte) []byte { 271 | if bytes.Equal(x, []byte(prev)) { 272 | return []byte(next) 273 | } 274 | return x 275 | } 276 | } 277 | 278 | func changeFuncRead(x []byte) []byte { 279 | return x 280 | } 281 | 282 | type prettyPrint []byte 283 | 284 | func (pp prettyPrint) String() string { 285 | if pp == nil { 286 | return "Ø" 287 | } 288 | return string(pp) 289 | } 290 | 291 | type testWriter struct{ t *testing.T } 292 | 293 | func (tw testWriter) Write(p []byte) (int, error) { 294 | tw.t.Logf("%s", string(p)) 295 | return len(p), nil 296 | } 297 | -------------------------------------------------------------------------------- /protocol/doc.go: -------------------------------------------------------------------------------- 1 | // Package protocol implements the core CASPaxos types and behaviors. 2 | package protocol 3 | -------------------------------------------------------------------------------- /protocol/memory_acceptor.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "sync" 9 | 10 | "github.com/go-kit/kit/log" 11 | "github.com/go-kit/kit/log/level" 12 | ) 13 | 14 | // ErrNotEqual indicates the tombstone value sent as part of a delete request 15 | // doesn't correspond to the stored value we have for that key. 16 | var ErrNotEqual = errors.New("not equal") 17 | 18 | // MemoryAcceptor persists data in-memory. 19 | type MemoryAcceptor struct { 20 | mtx sync.Mutex 21 | addr string 22 | ages map[string]Age 23 | values map[string]acceptedValue 24 | watchers watchers 25 | logger log.Logger 26 | } 27 | 28 | // An accepted value is associated with a key in an acceptor. 29 | // In this way, one acceptor can manage many key-value pairs. 30 | type acceptedValue struct { 31 | promise Ballot 32 | accepted Ballot 33 | value []byte 34 | } 35 | 36 | // The zero ballot can be used to clear promises. 37 | var zeroballot Ballot 38 | 39 | // NewMemoryAcceptor returns a usable in-memory acceptor. 40 | // Useful primarily for testing. 41 | func NewMemoryAcceptor(addr string, logger log.Logger) *MemoryAcceptor { 42 | return &MemoryAcceptor{ 43 | addr: addr, 44 | ages: map[string]Age{}, 45 | values: map[string]acceptedValue{}, 46 | watchers: watchers{}, 47 | logger: logger, 48 | } 49 | } 50 | 51 | // Address implements Addresser. 52 | func (a *MemoryAcceptor) Address() string { 53 | return a.addr 54 | } 55 | 56 | // Prepare implements the first-phase responsibilities of an acceptor. 57 | func (a *MemoryAcceptor) Prepare(ctx context.Context, key string, age Age, b Ballot) (value []byte, current Ballot, err error) { 58 | defer func() { 59 | level.Debug(a.logger).Log( 60 | "method", "Prepare", "key", key, "age", age, "B", b, 61 | "success", err == nil, "return_ballot", current, "err", err, 62 | ) 63 | }() 64 | 65 | a.mtx.Lock() 66 | defer a.mtx.Unlock() 67 | 68 | // From the GC section of the paper: "Acceptors [should] reject messages 69 | // from proposers if [the incoming age] is younger than the corresponding 70 | // age [that was previously accepted]." 71 | if incoming, existing := age, a.ages[age.ID]; incoming.youngerThan(existing) { 72 | return nil, zeroballot, AgeError{Incoming: incoming, Existing: existing} 73 | } 74 | 75 | // Select the promise/accepted/value tuple for this key. 76 | // A zero value is useful. 77 | av := a.values[key] 78 | 79 | // rystsov: "If a promise isn't empty during the prepare phase, we should 80 | // compare the proposed ballot number against the promise, and update the 81 | // promise if the promise is less." 82 | // 83 | // Here, we exploit the fact that a zero-value ballot number is less than 84 | // any non-zero-value ballot number. 85 | if av.promise.greaterThan(b) { 86 | return av.value, av.promise, ConflictError{Proposed: b, Existing: av.promise, ExistingKind: "promise"} 87 | } 88 | 89 | // Similarly, return a conflict if we already saw a greater ballot number. 90 | if av.accepted.greaterThan(b) { 91 | return av.value, av.accepted, ConflictError{Proposed: b, Existing: av.accepted, ExistingKind: "accepted"} 92 | } 93 | 94 | // If everything is satisfied, from the paper: "persist the ballot number as 95 | // a promise." 96 | av.promise = b 97 | a.values[key] = av 98 | 99 | // From the paper: "and return a confirmation either with an empty value (if 100 | // it hasn't accepted any value yet) or with a tuple of an accepted value 101 | // and its ballot number." 102 | // 103 | // Note: if the acceptor hasn't accepted any value yet, the value is nil, 104 | // which we take to mean "an empty value". The receiver should interpret 105 | // value == nil as an empty value and ignore the returned ballot, which will 106 | // be zero. 107 | return av.value, av.accepted, nil 108 | } 109 | 110 | // Accept implements the second-phase responsibilities of an acceptor. 111 | func (a *MemoryAcceptor) Accept(ctx context.Context, key string, age Age, b Ballot, value []byte) (err error) { 112 | defer func() { 113 | level.Debug(a.logger).Log( 114 | "method", "Accept", "key", key, "age", age, "B", b, 115 | "success", err == nil, "err", err, 116 | ) 117 | }() 118 | 119 | a.mtx.Lock() 120 | defer a.mtx.Unlock() 121 | 122 | // From the GC section of the paper: "Acceptors [should] reject messages 123 | // from proposers if [the incoming age] is younger than the corresponding 124 | // age [that was previously accepted]." 125 | if incoming, existing := age, a.ages[age.ID]; incoming.youngerThan(existing) { 126 | return AgeError{Incoming: incoming, Existing: existing} 127 | } 128 | 129 | // Select the promise/accepted/value tuple for this key. 130 | // A zero value is useful. 131 | av := a.values[key] 132 | 133 | // Return a conflict if it already saw a greater ballot number, either in 134 | // the promise or in the actual ballot number. 135 | // 136 | // rystsov: "During the accept phase, it's not necessary for the promise to 137 | // be equal to the passed ballot number. The promise simply cannot be 138 | // larger. The promise may even be empty; in this case, the request's ballot 139 | // number should be greater than the accepted ballot number." 140 | if av.promise.greaterThan(b) { 141 | return ConflictError{Proposed: b, Existing: av.promise, ExistingKind: "promise"} 142 | } 143 | 144 | // Similarly. 145 | if av.accepted.greaterThan(b) { 146 | return ConflictError{Proposed: b, Existing: av.accepted, ExistingKind: "accepted"} 147 | } 148 | 149 | // If everything is satisfied, from the paper: "Erase the promise, mark the 150 | // received tuple as the accepted value." 151 | av.promise, av.accepted, av.value = zeroballot, b, value 152 | a.values[key] = av 153 | 154 | // Extension: broadcast the state change to any watchers. 155 | // TODO(pb): be careful about deadlock on this one 156 | a.watchers.broadcast(key, value) 157 | 158 | // From the paper: "Return a confirmation." 159 | return nil 160 | } 161 | 162 | // RejectByAge implements part of the garbage collection responsibilities of an 163 | // acceptor. It updates the minimum age expected for the provided set of 164 | // proposers. 165 | func (a *MemoryAcceptor) RejectByAge(ctx context.Context, ages []Age) (err error) { 166 | defer func() { 167 | level.Debug(a.logger).Log( 168 | "method", "RejectByAge", "n", len(ages), 169 | "success", err == nil, "err", err, 170 | ) 171 | }() 172 | 173 | a.mtx.Lock() 174 | defer a.mtx.Unlock() 175 | 176 | for _, age := range ages { 177 | target := a.ages[age.ID] 178 | target.Counter = age.Counter 179 | a.ages[age.ID] = target 180 | } 181 | 182 | return nil 183 | } 184 | 185 | // RemoveIfEqual implements part of the garbage collection responsibilities of 186 | // an acceptor. It removes the key/value pair identified by key, if the stored 187 | // value is equal to the tombstone's value. 188 | func (a *MemoryAcceptor) RemoveIfEqual(ctx context.Context, key string, state []byte) (err error) { 189 | defer func() { 190 | level.Debug(a.logger).Log( 191 | "method", "RemoveIfEqual", "key", key, 192 | "success", err == nil, "err", err, 193 | ) 194 | }() 195 | 196 | a.mtx.Lock() 197 | defer a.mtx.Unlock() 198 | 199 | // If the key is already deleted, we don't need to do anything. 200 | av, ok := a.values[key] 201 | if !ok { 202 | return nil // great, no work to do 203 | } 204 | 205 | // If the states don't match, that's an error. 206 | if !bytes.Equal(av.value, state) { 207 | return ErrNotEqual 208 | } 209 | 210 | // Otherwise, we delete the thing. 211 | delete(a.values, key) 212 | 213 | // Extension: broadcast the state change to any watchers. 214 | // TODO(pb): be careful about deadlock on this one 215 | a.watchers.broadcast(key, nil) 216 | 217 | // Done. 218 | return nil 219 | } 220 | 221 | // Watch for changes to key, sending current states along the passed chan. 222 | func (a *MemoryAcceptor) Watch(ctx context.Context, key string, states chan<- []byte) (err error) { 223 | defer func() { 224 | level.Debug(a.logger).Log( 225 | "method", "Watch", "key", key, 226 | "success", err == nil, "err", err, 227 | ) 228 | }() 229 | 230 | func() { 231 | a.mtx.Lock() 232 | defer a.mtx.Unlock() 233 | a.watchers.subscribe(key, states) 234 | s := a.values[key] // zero value is fine 235 | states <- s.value // send initial value, which can be nil 236 | }() 237 | 238 | defer func() { 239 | a.mtx.Lock() 240 | defer a.mtx.Unlock() 241 | a.watchers.unsubscribe(key, states) 242 | }() 243 | 244 | <-ctx.Done() // block until canceled 245 | return ctx.Err() 246 | } 247 | 248 | func (a *MemoryAcceptor) dumpValue(key string) []byte { 249 | a.mtx.Lock() 250 | defer a.mtx.Unlock() 251 | av, ok := a.values[key] 252 | if !ok { 253 | return nil 254 | } 255 | dst := make([]byte, len(av.value)) 256 | copy(dst, av.value) 257 | return dst 258 | } 259 | 260 | // ConflictError is returned by acceptors when there's a ballot conflict. 261 | type ConflictError struct { 262 | Proposed Ballot 263 | Existing Ballot 264 | ExistingKind string // e.g. promise or accepted 265 | } 266 | 267 | func (ce ConflictError) Error() string { 268 | return fmt.Sprintf("conflict: proposed ballot %s isn't greater than existing '%s' ballot %s", ce.Proposed, ce.ExistingKind, ce.Existing) 269 | } 270 | 271 | // AgeError is returned by acceptors when there's an age conflict. 272 | type AgeError struct { 273 | Incoming Age 274 | Existing Age 275 | } 276 | 277 | func (ae AgeError) Error() string { 278 | return fmt.Sprintf("conflict: incoming age %s is younger than existing age %s", ae.Incoming, ae.Existing) 279 | } 280 | 281 | // 282 | // 283 | // 284 | 285 | type watchers map[string]map[chan<- []byte]bool 286 | 287 | func (w watchers) subscribe(key string, c chan<- []byte) { 288 | chans, ok := w[key] 289 | if !ok { 290 | chans = map[chan<- []byte]bool{} 291 | } 292 | chans[c] = true 293 | w[key] = chans 294 | } 295 | 296 | func (w watchers) unsubscribe(key string, c chan<- []byte) { 297 | chans, ok := w[key] 298 | if ok { 299 | delete(chans, c) 300 | } 301 | if len(chans) <= 0 { 302 | delete(w, key) 303 | } 304 | } 305 | 306 | func (w watchers) broadcast(key string, state []byte) { 307 | chans, ok := w[key] 308 | if !ok { 309 | return 310 | } 311 | for c := range chans { 312 | c <- state 313 | } 314 | } 315 | -------------------------------------------------------------------------------- /protocol/memory_acceptor_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | var _ Acceptor = (*MemoryAcceptor)(nil) 4 | -------------------------------------------------------------------------------- /protocol/memory_proposer.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "math/rand" 7 | "sort" 8 | "sync" 9 | 10 | "github.com/go-kit/kit/log" 11 | "github.com/go-kit/kit/log/level" 12 | ) 13 | 14 | // ChangeFunc models client change proposals. 15 | type ChangeFunc func(current []byte) (new []byte) 16 | 17 | var ( 18 | // ErrPrepareFailed indicates a failure during the first "prepare" phase. 19 | ErrPrepareFailed = errors.New("not enough confirmations during prepare phase; proposer ballot was fast-forwarded") 20 | 21 | // ErrAcceptFailed indicates a failure during the first "accept" phase. 22 | ErrAcceptFailed = errors.New("not enough confirmations during accept phase") 23 | 24 | // ErrDuplicate indicates the same acceptor was added twice. 25 | ErrDuplicate = errors.New("duplicate") 26 | 27 | // ErrNotFound indicates an attempt to remove a non-present acceptor. 28 | ErrNotFound = errors.New("not found") 29 | ) 30 | 31 | // MemoryProposer (from the paper) "performs the initialization by communicating 32 | // with acceptors, and keep minimal state needed to generate unique increasing 33 | // update IDs (ballot numbers)." 34 | type MemoryProposer struct { 35 | mtx sync.Mutex 36 | id string 37 | age Age 38 | ballot Ballot 39 | preparers map[string]Preparer 40 | accepters map[string]Accepter 41 | watchers map[string]StateWatcher 42 | logger log.Logger 43 | } 44 | 45 | // NewMemoryProposer returns a usable proposer uniquely identified by id. 46 | // It communicates with the initial set of acceptors. 47 | func NewMemoryProposer(id string, logger log.Logger, initial ...Acceptor) *MemoryProposer { 48 | p := &MemoryProposer{ 49 | id: id, 50 | age: Age{Counter: 0, ID: id}, 51 | ballot: Ballot{Counter: 0, ID: id}, 52 | preparers: map[string]Preparer{}, 53 | accepters: map[string]Accepter{}, 54 | watchers: map[string]StateWatcher{}, 55 | logger: logger, 56 | } 57 | for _, target := range initial { 58 | p.preparers[target.Address()] = target 59 | p.accepters[target.Address()] = target 60 | p.watchers[target.Address()] = target 61 | } 62 | return p 63 | } 64 | 65 | // Address of this proposer, which we interpret as the ID. 66 | func (p *MemoryProposer) Address() string { 67 | return p.id 68 | } 69 | 70 | // Propose a change from a client into the cluster. 71 | func (p *MemoryProposer) Propose(ctx context.Context, key string, f ChangeFunc) (state []byte, b Ballot, err error) { 72 | p.mtx.Lock() 73 | defer p.mtx.Unlock() 74 | 75 | state, b, err = p.propose(ctx, key, f, regularQuorum) 76 | if err == ErrPrepareFailed { 77 | // allow a single retry, to hide fast-forwards 78 | state, b, err = p.propose(ctx, key, f, regularQuorum) 79 | } 80 | 81 | return state, b, err 82 | } 83 | 84 | // quorumFunc declares how many of n nodes are required. 85 | type quorumFunc func(n int) int 86 | 87 | var ( 88 | regularQuorum = func(n int) int { return (n / 2) + 1 } // i.e. F+1 89 | fullQuorum = func(n int) int { return n } // i.e. 2F+1 90 | ) 91 | 92 | func (p *MemoryProposer) propose(ctx context.Context, key string, f ChangeFunc, qf quorumFunc) (state []byte, b Ballot, err error) { 93 | // From the paper: "A client submits the change function to a proposer. The 94 | // proposer generates a ballot number B, by incrementing the current ballot 95 | // number's counter." 96 | // 97 | // Note that this ballot number increment must be tracked in our state. 98 | // rystsov: "I proved correctness for the case when each *attempt* has a 99 | // unique ballot number. [Otherwise] I would bet that linearizability may be 100 | // violated." 101 | b = p.ballot.inc() 102 | 103 | // Set up a logger, for debugging. 104 | logger := level.Debug(log.With(p.logger, "method", "Propose", "key", key, "B", b)) 105 | 106 | // If prepare is successful, we'll have an accepted current state. 107 | var currentState []byte 108 | 109 | // Prepare phase. 110 | { 111 | // Set up a sub-logger for this phase. 112 | logger := log.With(logger, "phase", "prepare") 113 | 114 | // We collect prepare results into this channel. 115 | type result struct { 116 | addr string 117 | value []byte 118 | ballot Ballot 119 | err error 120 | } 121 | results := make(chan result, len(p.preparers)) 122 | 123 | // Broadcast the prepare requests to the preparers. 124 | // (Preparers are just acceptors, serving their first role.) 125 | logger.Log("broadcast_to", len(p.preparers)) 126 | for addr, target := range p.preparers { 127 | go func(addr string, target Preparer) { 128 | value, ballot, err := target.Prepare(ctx, key, p.age, b) 129 | results <- result{addr, value, ballot, err} 130 | }(addr, target) 131 | } 132 | 133 | // From the paper: "The proposer waits for F+1 confirmations. If they 134 | // all contain the empty value, then the proposer defines the current 135 | // state as nil; otherwise, it picks the value of the tuple with the 136 | // highest ballot number." 137 | var ( 138 | quorum = qf(len(p.preparers)) 139 | biggestConfirm Ballot 140 | biggestConflict Ballot 141 | ) 142 | 143 | // Broadcast the prepare request to the preparers. Observe that once 144 | // we've got confirmation from a quorum of preparers, we ignore any 145 | // subsequent messages. 146 | for i := 0; i < cap(results) && quorum > 0; i++ { 147 | result := <-results 148 | if result.err != nil { 149 | // A conflict indicates that the proposed ballot is too old and 150 | // will be rejected; the largest conflicting ballot number 151 | // should be used to fast-forward the proposer's ballot number 152 | // counter in the case of total (quorum) failure. 153 | logger.Log("addr", result.addr, "result", "conflict", "ballot", result.ballot, "err", result.err) 154 | if result.ballot.greaterThan(biggestConflict) { 155 | biggestConflict = result.ballot 156 | } 157 | } else { 158 | // A confirmation indicates the proposed ballot will succeed, 159 | // and the preparer has accepted it as a promise; the largest 160 | // confirmed ballot number is used to select which returned 161 | // value will be chosen as the current value. 162 | logger.Log("addr", result.addr, "result", "confirm", "ballot", result.ballot) 163 | if result.ballot.greaterThan(biggestConfirm) { 164 | biggestConfirm, currentState = result.ballot, result.value 165 | } 166 | quorum-- 167 | } 168 | } 169 | 170 | // If we finish collecting results and haven't achieved quorum, the 171 | // proposal fails. We should fast-forward our ballot number's counter to 172 | // the highest number we saw from the conflicted preparers, so a 173 | // subsequent proposal might succeed. We could try to re-submit the same 174 | // request with our updated ballot number, but for now let's leave that 175 | // responsibility to the caller. 176 | // 177 | // As a special case, if we have zero preparers, allow the phase to 178 | // pass. This allows us to grow the cluster from an empty state. 179 | if len(p.preparers) > 0 && quorum > 0 { 180 | logger.Log("result", "failed", "fast_forward_to", biggestConflict.Counter) 181 | p.ballot.Counter = biggestConflict.Counter // fast-forward 182 | return nil, b, ErrPrepareFailed 183 | } 184 | 185 | logger.Log("result", "success") 186 | } 187 | 188 | // We've successfully completed the prepare phase. From the paper: "The 189 | // proposer applies the change function to the current state. It will send 190 | // that new state along with the generated ballot number B (together, known 191 | // as an "accept" message) to the acceptors." 192 | newState := f(currentState) 193 | 194 | // Accept phase. 195 | { 196 | // Set up a sub-logger for this phase. 197 | logger := log.With(logger, "phase", "accept") 198 | 199 | // We collect accept results into this channel. 200 | type result struct { 201 | addr string 202 | err error 203 | } 204 | results := make(chan result, len(p.accepters)) 205 | 206 | // Broadcast accept messages to the accepters. 207 | logger.Log("broadcast_to", len(p.accepters)) 208 | for addr, target := range p.accepters { 209 | go func(addr string, target Accepter) { 210 | err := target.Accept(ctx, key, p.age, b, newState) 211 | results <- result{addr, err} 212 | }(addr, target) 213 | } 214 | 215 | // From the paper: "The proposer waits for the F+1 confirmations." 216 | // Observe that once we've got confirmation from a quorum of accepters, 217 | // we ignore any subsequent messages. 218 | quorum := qf(len(p.accepters)) 219 | for i := 0; i < cap(results) && quorum > 0; i++ { 220 | result := <-results 221 | if result.err != nil { 222 | logger.Log("addr", result.addr, "result", "conflict", "err", result.err) 223 | } else { 224 | logger.Log("addr", result.addr, "result", "confirm") 225 | quorum-- 226 | } 227 | } 228 | 229 | // If we don't get quorum, I guess we must fail the proposal. Similarly 230 | // to the first phase, make a special case when we don't have any 231 | // accepters. 232 | if len(p.accepters) > 0 && quorum > 0 { 233 | logger.Log("result", "failed", "err", "not enough confirmations") 234 | return nil, b, ErrAcceptFailed 235 | } 236 | 237 | // Log the success. 238 | logger.Log("result", "success") 239 | } 240 | 241 | // Return the new state to the caller. 242 | return newState, b, nil 243 | } 244 | 245 | // IdentityRead is an alias for propose with an identity change function. It's a 246 | // separate method to make implementing an e.g. HTTP proposer simpler; 247 | // serializing change functions is difficult. 248 | func (p *MemoryProposer) IdentityRead(ctx context.Context, key string) error { 249 | _, _, err := p.Propose(ctx, key, func(x []byte) []byte { return x }) 250 | return err 251 | } 252 | 253 | // AddAccepter adds the target acceptor to the pool of accepters used in the 254 | // second phase of proposals. It's the first step in growing the cluster, which 255 | // is a global process that needs to be orchestrated by an operator. 256 | func (p *MemoryProposer) AddAccepter(target Acceptor) error { 257 | p.mtx.Lock() 258 | defer p.mtx.Unlock() 259 | if _, ok := p.accepters[target.Address()]; ok { 260 | return ErrDuplicate 261 | } 262 | p.accepters[target.Address()] = target 263 | p.watchers[target.Address()] = target 264 | return nil 265 | } 266 | 267 | // AddPreparer adds the target acceptor to the pool of preparers used in the 268 | // first phase of proposals. It's the third step in growing the cluster, which 269 | // is a global process that needs to be orchestrated by an operator. 270 | func (p *MemoryProposer) AddPreparer(target Acceptor) error { 271 | p.mtx.Lock() 272 | defer p.mtx.Unlock() 273 | if _, ok := p.preparers[target.Address()]; ok { 274 | return ErrDuplicate 275 | } 276 | p.preparers[target.Address()] = target 277 | return nil 278 | } 279 | 280 | // RemovePreparer removes the target acceptor from the pool of preparers used in 281 | // the first phase of proposals. It's the first step in shrinking the cluster, 282 | // which is a global process that needs to be orchestrated by an operator. 283 | func (p *MemoryProposer) RemovePreparer(target Acceptor) error { 284 | p.mtx.Lock() 285 | defer p.mtx.Unlock() 286 | if _, ok := p.preparers[target.Address()]; !ok { 287 | return ErrNotFound 288 | } 289 | delete(p.preparers, target.Address()) 290 | return nil 291 | } 292 | 293 | // RemoveAccepter removes the target acceptor from the pool of accepters used in 294 | // the second phase of proposals. It's the third step in shrinking the cluster, 295 | // which is a global process that needs to be orchestrated by an operator. 296 | func (p *MemoryProposer) RemoveAccepter(target Acceptor) error { 297 | p.mtx.Lock() 298 | defer p.mtx.Unlock() 299 | if _, ok := p.accepters[target.Address()]; !ok { 300 | return ErrNotFound 301 | } 302 | delete(p.accepters, target.Address()) 303 | delete(p.watchers, target.Address()) 304 | return nil 305 | } 306 | 307 | // FullIdentityRead performs an identity read on the given key with a quorum 308 | // size of 100%. It's used as part of the garbage collection process, to delete 309 | // keys. 310 | func (p *MemoryProposer) FullIdentityRead(ctx context.Context, key string) (state []byte, b Ballot, err error) { 311 | identity := func(x []byte) []byte { return x } 312 | state, b, err = p.propose(ctx, key, identity, fullQuorum) 313 | if err == ErrPrepareFailed { 314 | // allow a single retry, to hide fast-forwards 315 | state, b, err = p.propose(ctx, key, identity, fullQuorum) 316 | } 317 | return state, b, err 318 | } 319 | 320 | // FastForwardIncrement performs part (2b) responsibilities of the GC process. 321 | func (p *MemoryProposer) FastForwardIncrement(ctx context.Context, key string, b Ballot) (Age, error) { 322 | // From the paper, this method should: "invalidate [the] cache associated 323 | // with the key ... fast-forward [the ballot number] counter to guarantee 324 | // that new ballot numbers are greater than the tombstone's ballot, and 325 | // increments the proposer's age." 326 | 327 | p.mtx.Lock() 328 | defer p.mtx.Unlock() 329 | 330 | // We have no cache associated with the key, because we don't implement the 331 | // One-round trip optimization from 2.2.1. So all we have to do is update 332 | // our counters. First, fast-forward the ballot. 333 | if p.ballot.Counter < (b.Counter + 1) { 334 | p.ballot.Counter = (b.Counter + 1) 335 | } 336 | 337 | // Then, increment our age. 338 | return p.age.inc(), nil 339 | } 340 | 341 | // ListPreparers enumerates the prepare-stage acceptors that this proposer is 342 | // configured with. 343 | func (p *MemoryProposer) ListPreparers() (addrs []string, err error) { 344 | p.mtx.Lock() 345 | defer p.mtx.Unlock() 346 | 347 | addrs = make([]string, 0, len(p.preparers)) 348 | for addr := range p.preparers { 349 | addrs = append(addrs, addr) 350 | } 351 | 352 | sort.Strings(addrs) 353 | return addrs, nil 354 | } 355 | 356 | // ListAccepters enumerates the accept-stage acceptors that this proposer is 357 | // configured with. 358 | func (p *MemoryProposer) ListAccepters() (addrs []string, err error) { 359 | p.mtx.Lock() 360 | defer p.mtx.Unlock() 361 | 362 | addrs = make([]string, 0, len(p.accepters)) 363 | for addr := range p.accepters { 364 | addrs = append(addrs, addr) 365 | } 366 | 367 | sort.Strings(addrs) 368 | return addrs, nil 369 | } 370 | 371 | // Watch the given key, emitting all states into the passed channel. 372 | func (p *MemoryProposer) Watch(ctx context.Context, key string, states chan<- []byte) error { 373 | var watchers []StateWatcher 374 | { 375 | p.mtx.Lock() 376 | for _, w := range p.watchers { 377 | watchers = append(watchers, w) 378 | } 379 | p.mtx.Unlock() 380 | } 381 | 382 | if len(watchers) <= 0 { 383 | return errors.New("no watchers available") 384 | } 385 | 386 | watcher := watchers[rand.Intn(len(watchers))] 387 | return watcher.Watch(ctx, key, states) 388 | } 389 | -------------------------------------------------------------------------------- /protocol/memory_proposer_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | var _ Proposer = (*MemoryProposer)(nil) 4 | -------------------------------------------------------------------------------- /protocol/operations.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "context" 5 | "math/rand" 6 | 7 | "github.com/go-kit/kit/log" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | // Proposer models a concrete proposer. 12 | type Proposer interface { 13 | Addresser 14 | Propose(ctx context.Context, key string, f ChangeFunc) (state []byte, b Ballot, err error) 15 | ConfigurationChanger 16 | FastForwarder 17 | AcceptorLister 18 | StateWatcher 19 | } 20 | 21 | // ConfigurationChanger models the grow/shrink cluster responsibilities of a 22 | // proposer. 23 | type ConfigurationChanger interface { 24 | IdentityRead(ctx context.Context, key string) error 25 | AddAccepter(target Acceptor) error 26 | AddPreparer(target Acceptor) error 27 | RemovePreparer(target Acceptor) error 28 | RemoveAccepter(target Acceptor) error 29 | } 30 | 31 | // FastForwarder models the garbage collection responsibilities of a proposer. 32 | type FastForwarder interface { 33 | FullIdentityRead(ctx context.Context, key string) (state []byte, b Ballot, err error) 34 | FastForwardIncrement(ctx context.Context, key string, b Ballot) (Age, error) 35 | } 36 | 37 | // AcceptorLister allows operators to introspect the state of proposers. 38 | // For debug and operational work, not necessary for the core protocol. 39 | type AcceptorLister interface { 40 | ListPreparers() ([]string, error) 41 | ListAccepters() ([]string, error) 42 | } 43 | 44 | // Acceptor models a complete, uniquely-addressable acceptor. 45 | // 46 | // Here we have a little fun with names: use Acceptor (-or) as a noun, to model 47 | // the whole composite acceptor, and Accepter (-er) as a verb, to model the 48 | // second-phase "accept" responsibilities only. 49 | type Acceptor interface { 50 | Addresser 51 | Preparer 52 | Accepter 53 | RejectRemover 54 | StateWatcher 55 | } 56 | 57 | // StateWatcher watches a key, yielding states on a user-supplied channel. 58 | type StateWatcher interface { 59 | Watch(ctx context.Context, key string, states chan<- []byte) error 60 | } 61 | 62 | // Addresser models something with a unique address. 63 | type Addresser interface { 64 | Address() string // typically "protocol://host:port" 65 | } 66 | 67 | // Preparer models the first-phase responsibilities of an acceptor. 68 | type Preparer interface { 69 | Prepare(ctx context.Context, key string, age Age, b Ballot) (value []byte, current Ballot, err error) 70 | } 71 | 72 | // Accepter models the second-phase responsibilities of an acceptor. 73 | type Accepter interface { 74 | Accept(ctx context.Context, key string, age Age, b Ballot, value []byte) error 75 | } 76 | 77 | // RejectRemover models the garbage collection responsibilities of an acceptor. 78 | type RejectRemover interface { 79 | RejectByAge(ctx context.Context, ages []Age) error 80 | RemoveIfEqual(ctx context.Context, key string, state []byte) error 81 | } 82 | 83 | // Assign special meaning to one special key, which we use to increment ballot 84 | // numbers for operations like changing cluster configuration. 85 | const zerokey = "—" 86 | 87 | // Note: When growing (or shrinking) a cluster from an odd number of acceptors 88 | // to an even number of acceptors, the implemented process is required. But when 89 | // growing (or shrinking) a cluster from an even number of acceptors to an odd 90 | // number of acceptors, an optimization is possible: we can first change the 91 | // accept and prepare lists of all proposers, and then turn the acceptor on, and 92 | // avoid the cost of a read. 93 | // 94 | // This is what's meant in this section of the paper: "The protocol for changing 95 | // the set of acceptors from A_1...A_2F+2 to A_1...A_2F+3 [from even to odd] is 96 | // more straightforward because we can treat a 2F+2 nodes cluster as a 2F+3 97 | // nodes cluster where one node had been down from the beginning: [that process 98 | // is] (1) Connect to each proposer and update its configuration to send the 99 | // prepare and accept messages to the [second] A_1...A_2F+3 set of acceptors; 100 | // (2) Turn on the A_2F+3 acceptor." 101 | // 102 | // I've chosen not to implement this for several reasons. First, cluster 103 | // membership changes are rare and operator-driven, and so don't really benefit 104 | // from the lower latency as much as reads or writes would. Second, the number 105 | // of acceptors in the cluster is not known a priori, and can in theory drift 106 | // between different proposers; calculating the correct value is difficult in 107 | // itself, probably requiring asking some other source of authority. Third, in 108 | // production environments, there's great value in having a consistent process 109 | // for any cluster change; turning a node on at different points in that process 110 | // depending on the cardinality of the node-set is fraught with peril. 111 | 112 | // GrowCluster adds the target acceptor to the cluster of proposers. 113 | func GrowCluster(ctx context.Context, target Acceptor, proposers []ConfigurationChanger) error { 114 | // If we fail, try to leave the cluster in its original state. 115 | var undo []func() 116 | defer func() { 117 | for i := len(undo) - 1; i >= 0; i-- { 118 | undo[i]() 119 | } 120 | }() 121 | 122 | // From the paper: "Connect to each proposer and update its configuration to 123 | // send the 'accept' messages to the [new] set of acceptors, and to require 124 | // F+2 confirmations during the 'accept' phase." 125 | for _, proposer := range proposers { 126 | if err := proposer.AddAccepter(target); err != nil { 127 | return errors.Wrap(err, "during grow step 1 (add accepter)") 128 | } 129 | undo = append(undo, func() { proposer.RemoveAccepter(target) }) 130 | } 131 | 132 | // From the paper: "Pick any proposer and execute the identity state 133 | // transaction x -> x." 134 | proposer := proposers[rand.Intn(len(proposers))] 135 | if err := proposer.IdentityRead(ctx, zerokey); err != nil { 136 | return errors.Wrap(err, "during grow step 2 (identity read)") 137 | } 138 | 139 | // From the paper: "Connect to each proposer and update its configuration to 140 | // send 'prepare' messages to the [new] set of acceptors, and to require F+2 141 | // confirmations [during the 'prepare' phase]." 142 | for _, proposer := range proposers { 143 | if err := proposer.AddPreparer(target); err != nil { 144 | return errors.Wrap(err, "during grow step 3 (add preparer)") 145 | } 146 | undo = append(undo, func() { proposer.RemovePreparer(target) }) 147 | } 148 | 149 | // Success! Kill the undo stack, and return. 150 | undo = []func(){} 151 | return nil 152 | } 153 | 154 | // ShrinkCluster removes the target acceptor from the cluster of proposers. 155 | func ShrinkCluster(ctx context.Context, target Acceptor, proposers []ConfigurationChanger) error { 156 | // If we fail, try to leave the cluster in its original state. 157 | var undo []func() 158 | defer func() { 159 | for i := len(undo) - 1; i >= 0; i-- { 160 | undo[i]() 161 | } 162 | }() 163 | 164 | // From the paper: "The same steps [for growing the cluster] should be 165 | // executed in the reverse order to reduce the size of the cluster." 166 | 167 | // So, remove it as a preparer. 168 | for _, proposer := range proposers { 169 | if err := proposer.RemovePreparer(target); err != nil { 170 | return errors.Wrap(err, "during shrink step 1 (remove preparer)") 171 | } 172 | undo = append(undo, func() { proposer.AddPreparer(target) }) 173 | } 174 | 175 | // Execute a no-op read. 176 | proposer := proposers[rand.Intn(len(proposers))] 177 | if err := proposer.IdentityRead(ctx, zerokey); err != nil { 178 | return errors.Wrap(err, "during shrink step 2 (identity read)") 179 | } 180 | 181 | // And then remove it as an accepter. 182 | for _, proposer := range proposers { 183 | if err := proposer.RemoveAccepter(target); err != nil { 184 | return errors.Wrap(err, "during shrink step 3 (remove accepter)") 185 | } 186 | undo = append(undo, func() { proposer.AddAccepter(target) }) 187 | } 188 | 189 | // Done. 190 | undo = []func(){} 191 | return nil 192 | } 193 | 194 | // Tombstone represents the terminal form of a key. Propose some state, likely 195 | // empty, and collect it with the resulting ballot into a Tombstone, which 196 | // becomes input to the garbage collection process. 197 | type Tombstone struct { 198 | Ballot Ballot 199 | State []byte 200 | } 201 | 202 | // GarbageCollect removes a key as described in section 3.1 "How to delete a 203 | // record" in the paper. Any error is treated as fatal to this GC attempt, and 204 | // the caller should retry if IsRetryable(err) is true. 205 | func GarbageCollect(ctx context.Context, key string, proposers []FastForwarder, acceptors []RejectRemover, logger log.Logger) error { 206 | // From the paper: "(a) Replicates an empty value to all nodes by 207 | // executing the identity transform with max quorum size (2F+1)." 208 | var killstate []byte 209 | var tombstone Ballot 210 | { 211 | var ( 212 | proposer = proposers[rand.Intn(len(proposers))] 213 | s, b, err = proposer.FullIdentityRead(ctx, key) 214 | ) 215 | if err != nil { 216 | return makeRetryable(errors.Wrap(err, "error executing identity transform")) 217 | } 218 | killstate = s 219 | tombstone = b 220 | } 221 | 222 | // From the paper: "(b) Connects to each proposer, invalidates its cache 223 | // associated with the removing key, ... fast-forwards its counter to 224 | // guarantee that new ballot numbers are greater than the tombstone's 225 | // ballot, and increments proposer's age." 226 | var ages []Age 227 | { 228 | type result struct { 229 | age Age 230 | err error 231 | } 232 | results := make(chan result, len(proposers)) 233 | for _, proposer := range proposers { 234 | go func(proposer FastForwarder) { 235 | age, err := proposer.FastForwardIncrement(ctx, key, tombstone) 236 | results <- result{age, err} 237 | }(proposer) 238 | } 239 | for i := 0; i < cap(results); i++ { 240 | result := <-results 241 | if result.err != nil { 242 | return makeRetryable(errors.Wrap(result.err, "error invalidating and incrementing proposer")) 243 | } 244 | ages = append(ages, result.age) 245 | } 246 | } 247 | 248 | // From the paper: "(c) For each acceptor, asks to reject messages from 249 | // proposers if their age is younger than the corresponding age from the 250 | // previous step." 251 | { 252 | results := make(chan error, len(acceptors)) 253 | for _, acceptor := range acceptors { 254 | go func(acceptor RejectRemover) { 255 | results <- acceptor.RejectByAge(ctx, ages) 256 | }(acceptor) 257 | } 258 | for i := 0; i < cap(results); i++ { 259 | if err := <-results; err != nil { 260 | return makeRetryable(errors.Wrap(err, "error updating ages in acceptor")) 261 | } 262 | } 263 | } 264 | 265 | // From the paper: "(d) For each acceptor, remove the register if its 266 | // value is the tombstone from the 2a step." 267 | { 268 | results := make(chan error, len(acceptors)) 269 | for _, acceptor := range acceptors { 270 | go func(acceptor RejectRemover) { 271 | results <- acceptor.RemoveIfEqual(ctx, key, killstate) 272 | }(acceptor) 273 | } 274 | for i := 0; i < cap(results); i++ { 275 | if err := <-results; err != nil { 276 | return notRetryable(errors.Wrap(err, "error removing value from acceptor")) 277 | } 278 | } 279 | } 280 | 281 | // Done! 282 | return nil 283 | } 284 | 285 | // IsRetryable indicates the error, received from the GarbageCollect function, 286 | // is non-terminal, and the client can retry the operation. 287 | func IsRetryable(err error) bool { 288 | r, ok := err.(interface{ retryable() bool }) 289 | return ok && r.retryable() 290 | } 291 | 292 | type retryableError struct{ error } 293 | 294 | func (re retryableError) retryable() bool { return true } 295 | 296 | func makeRetryable(err error) error { return retryableError{err} } 297 | 298 | // notRetryable is a no-op, it just makes the GC function nicer to read. 299 | func notRetryable(err error) error { return err } 300 | -------------------------------------------------------------------------------- /protocol/operations_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "testing" 7 | 8 | "github.com/go-kit/kit/log" 9 | ) 10 | 11 | func TestConfigurationChange(t *testing.T) { 12 | // Build the cluster. 13 | var ( 14 | logger = log.NewLogfmtLogger(testWriter{t}) 15 | a1 = NewMemoryAcceptor("1", log.With(logger, "a", 1)) 16 | a2 = NewMemoryAcceptor("2", log.With(logger, "a", 2)) 17 | a3 = NewMemoryAcceptor("3", log.With(logger, "a", 3)) 18 | p1 = NewMemoryProposer("1", log.With(logger, "p", 1), a1, a2, a3) 19 | p2 = NewMemoryProposer("2", log.With(logger, "p", 2), a1, a2, a3) 20 | p3 = NewMemoryProposer("3", log.With(logger, "p", 3), a1, a2, a3) 21 | ctx = context.Background() 22 | key = "k" 23 | val0 = "xxx" 24 | ) 25 | 26 | // Declare some verification functions. 27 | growClusterWith := func(a Acceptor) { 28 | if err := GrowCluster(ctx, a, []ConfigurationChanger{p1, p2, p3}); err != nil { 29 | t.Fatalf("grow cluster with %q: %v", a.Address(), err) 30 | } 31 | } 32 | 33 | shrinkClusterWith := func(a Acceptor) { 34 | if err := ShrinkCluster(ctx, a, []ConfigurationChanger{p1, p2, p3}); err != nil { 35 | t.Fatalf("shrink cluster with %q: %v", a.Address(), err) 36 | } 37 | } 38 | 39 | verifyReads := func() { 40 | for name, p := range map[string]Proposer{ 41 | "p1": p1, "p2": p2, "p3": p3, 42 | } { 43 | if state, _, err := p.Propose(ctx, key, changeFuncRead); err != nil { 44 | t.Errorf("read via %s after shrink: %v", name, err) 45 | } else if want, have := val0, string(state); want != have { 46 | t.Errorf("read via %s after shrink: want %q, have %q", name, want, have) 47 | } 48 | } 49 | } 50 | 51 | verifyValue := func(a *MemoryAcceptor) { 52 | if want, have := val0, string(a.dumpValue(key)); want != have { 53 | t.Errorf("acceptor %s value: want %q, have %q", a.Address(), want, have) 54 | } 55 | } 56 | 57 | // Set up an initial value. 58 | p2.Propose(ctx, key, changeFuncInitializeOnlyOnce(val0)) 59 | 60 | // Add a new acceptor. After one or more reads, 61 | // it should have the correct value. 62 | a4 := NewMemoryAcceptor("4", log.With(logger, "a", 4)) 63 | growClusterWith(a4) 64 | verifyReads() 65 | verifyValue(a4) 66 | 67 | // Add another acceptor, same deal. 68 | a5 := NewMemoryAcceptor("5", log.With(logger, "a", 5)) 69 | growClusterWith(a5) 70 | verifyReads() 71 | verifyValue(a5) 72 | 73 | // Remove one of the initial acceptors. 74 | // Reads should still work. 75 | shrinkClusterWith(a1) 76 | verifyReads() 77 | 78 | // Remove one of the new acceptors, same deal. 79 | shrinkClusterWith(a4) 80 | verifyReads() 81 | } 82 | 83 | func TestGarbageCollection(t *testing.T) { 84 | // Build the cluster. 85 | var ( 86 | logger = log.NewLogfmtLogger(testWriter{t}) 87 | a1 = NewMemoryAcceptor("1", log.With(logger, "a", 1)) 88 | a2 = NewMemoryAcceptor("2", log.With(logger, "a", 2)) 89 | a3 = NewMemoryAcceptor("3", log.With(logger, "a", 3)) 90 | p1 = NewMemoryProposer("1", log.With(logger, "p", 1), a1, a2, a3) 91 | p2 = NewMemoryProposer("2", log.With(logger, "p", 2), a1, a2, a3) 92 | p3 = NewMemoryProposer("3", log.With(logger, "p", 3), a1, a2, a3) 93 | ctx = context.Background() 94 | key = "my key" 95 | val0 = "initial value" 96 | ) 97 | 98 | // Set up an initial value. 99 | if _, _, err := p1.Propose(ctx, key, changeFuncInitializeOnlyOnce(val0)); err != nil { 100 | t.Fatalf("write initial value failed: %v", err) 101 | } 102 | 103 | // Perform a GC. 104 | var ( 105 | proposers = []FastForwarder{p1, p2, p3} 106 | acceptors = []RejectRemover{a1, a2, a3} 107 | gclogger = log.With(logger, "op", "GC") 108 | ) 109 | if err := GarbageCollect(ctx, key, proposers, acceptors, gclogger); err != nil { 110 | t.Fatalf("GC failed: %v", err) 111 | } 112 | 113 | // Read should succeed, with a nil value. 114 | state, _, err := p3.Propose(ctx, key, changeFuncRead) 115 | if want, have := (error)(nil), err; want != have { 116 | t.Fatalf("post-GC read: err: want %v, have %v", want, have) 117 | } 118 | if want, have := []byte(nil), state; !bytes.Equal(want, have) { 119 | t.Fatalf("post-GC read: ballot: want %q, have %q", want, have) 120 | } 121 | 122 | // Dump all values and check the key was actually GC'd. 123 | for _, a := range []*MemoryAcceptor{a1, a2, a3} { 124 | if want, have := []byte(nil), a.dumpValue(key); !bytes.Equal(want, have) { 125 | t.Errorf("%s: %s: want %q, have %q", a.addr, key, want, have) 126 | } 127 | } 128 | } 129 | --------------------------------------------------------------------------------