├── .gitignore ├── .gitmodules ├── LICENSE ├── README.md ├── ben.lua ├── build ├── clib └── .do_not_delete ├── dep-build ├── cjson └── libluafs ├── doc └── doc.markdown ├── example ├── common.config ├── http.config ├── mem-1 │ └── conf │ │ └── nginx.conf ├── mem-2 │ └── conf │ │ └── nginx.conf └── mem-3 │ └── conf │ └── nginx.conf ├── it ├── README.md ├── __init__.py ├── http.py ├── ngxconf.py ├── ngxctl.py ├── paxoscli.py ├── paxosclient.py └── sto.py ├── lib ├── acid │ ├── cache.lua │ ├── cluster.lua │ ├── impl │ │ ├── http.lua │ │ ├── locking_ngx.lua │ │ ├── logging_ngx.lua │ │ ├── member.lua │ │ ├── storage_ngx_fs.lua │ │ ├── storage_ngx_mc.lua │ │ ├── time_ngx.lua │ │ ├── transport_ngx_http.lua │ │ └── userdata.lua │ ├── impl_ngx.lua │ ├── logging.lua │ ├── paxos.lua │ ├── paxos │ │ ├── _sto_data_struct.lua │ │ ├── _ver.lua │ │ ├── acceptor.lua │ │ ├── base.lua │ │ ├── proposer.lua │ │ └── round.lua │ ├── paxoshelper.lua │ ├── paxosserver.lua │ ├── strutil.lua │ ├── tableutil.lua │ └── unittest.lua ├── foo.nginx.conf ├── handle_get.lua ├── nginx_cluster.lua ├── sample.lua ├── simple.lua ├── test_empty.lua ├── test_logging.lua ├── test_paxos.lua ├── test_proposer_acceptor.lua ├── test_round.lua ├── test_strutil.lua ├── test_tableutil.lua └── worker_init.lua ├── merge ├── py ├── cluster_test.py └── concurrency_test.py ├── srv ├── .gitignore └── inst └── ut /.gitignore: -------------------------------------------------------------------------------- 1 | luacov.report.out 2 | luacov.stats.out 3 | *.pyc 4 | *.so 5 | tags 6 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "dep/libluafs"] 2 | path = dep/libluafs 3 | url = https://github.com/drmingdrmer/libluafs.git 4 | branch = master 5 | [submodule "dep/cjson"] 6 | path = dep/cjson 7 | url = https://github.com/drmingdrmer/lua-cjson.git 8 | branch = master 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 张炎泼 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lua-poxos 2 | 3 | Classic Paxos implementation in lua. 4 | 5 | Nginx cluster management based on paxos. 6 | 7 | Feature: 8 | 9 | - Classic two phase paxos algorithm. 10 | 11 | - Optional phase-3 as phase 'learn' or 'commit' 12 | 13 | - Support membership changing on the fly. 14 | 15 | This is archived by making the group members a paxos instance. Running paxos 16 | on group member updates group membership. 17 | 18 | Here we borrowed the concept 'view' that stands for a single version of 19 | membership. 20 | 21 | 'view' is a no more than a normal paxos instance. 22 | 23 | 24 | ## Getting Started 25 | -------------------------------------------------------------------------------- /ben.lua: -------------------------------------------------------------------------------- 1 | 2 | function tuple() 3 | return 1, 2, 3 4 | end 5 | 6 | function tb() 7 | return {1, 2, 3} 8 | end 9 | 10 | function _ben(f, times) 11 | local t0 = os.time() 12 | for i = 1, times do 13 | f() 14 | end 15 | local t1 = os.time() 16 | 17 | local spent = t1-t0 18 | print(times, spent) 19 | return spent 20 | end 21 | 22 | function ben(f) 23 | local times = 100 24 | local spent = 0 25 | while spent < 10 do 26 | times = times * 2 27 | spent = _ben(f, times) 28 | end 29 | 30 | print( "spent:", spent, "times:", times, "rps:", math.floor(times/spent) ) 31 | end 32 | 33 | ben(tuple) 34 | ben(tb) 35 | 36 | -------------------------------------------------------------------------------- /build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | to_rm="bundle it srv merge build backup ut *.lua *.py" 4 | 5 | ./merge rc \ 6 | && cur_hash=$(git rev-parse HEAD) \ 7 | && parent=refs/heads/master \ 8 | && git rm -r $to_rm \ 9 | && echo "/srv/" >> .gitignore \ 10 | && git add .gitignore lib \ 11 | && tree_hash=$(git write-tree) \ 12 | && commit_hash=$( echo "build from ${cur_hash:0:7}" | git commit-tree $tree_hash -p $parent) \ 13 | && echo commit hash is $commit_hash \ 14 | && git update-ref $parent $commit_hash \ 15 | && git reset --hard HEAD 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /clib/.do_not_delete: -------------------------------------------------------------------------------- 1 | do not delete me 2 | -------------------------------------------------------------------------------- /dep-build/cjson: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ( 4 | p=dep/lua-cjson 5 | ( cd $p && CPATH=$CPATH:/usr/include/lua5.1 make; ) \ 6 | && cp $p/cjson.so clib/ \ 7 | && ( cd $p && make clean; ) 8 | ) 9 | -------------------------------------------------------------------------------- /dep-build/libluafs: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ( 4 | p=dep/libluafs 5 | ( cd $p && CPATH=$CPATH:/usr/include/lua5.1 make; ) \ 6 | && cp $p/libluafs.so clib/ \ 7 | && ( cd $p && make clean; ) 8 | ) 9 | -------------------------------------------------------------------------------- /doc/doc.markdown: -------------------------------------------------------------------------------- 1 | 2 | 3 | **Table of Contents** *generated with [DocToc](http://doctoc.herokuapp.com/)* 4 | 5 | - [Concept](#concept) 6 | - [Proposer](#proposer) 7 | - [Acceptor](#acceptor) 8 | - [Round](#round) 9 | - [Value](#value) 10 | - [Value Round](#value-round) 11 | - [Version](#version) 12 | - [Quorum](#quorum) 13 | - [View](#view) 14 | - [Instance](#instance) 15 | - [Internal Request and Response](#internal-request-and-response) 16 | - [Request](#request) 17 | - [Response](#response) 18 | - [Response with error](#response-with-error) 19 | - [Phase-1](#phase-1) 20 | - [Phase-1 Request](#phase-1-request) 21 | - [Phase-1 Response](#phase-1-response) 22 | - [Phase-2](#phase-2) 23 | - [Phase-2 Request](#phase-2-request) 24 | - [Phase-2 Response](#phase-2-response) 25 | - [Phase-3](#phase-3) 26 | - [Phase-3 Request](#phase-3-request) 27 | - [Phase-3 Response](#phase-3-response) 28 | - [Reference](#reference) 29 | 30 | 31 | 32 | 33 | ## Concept 34 | 35 | ### Proposer 36 | 37 | Propose a new round of paxos, to establish a value. 38 | 39 | The `value` established maybe not the `value` `proposer` specified, instead, it 40 | could be the `value` that already exists( might be accepted by `quorum` ). 41 | 42 | In this case, it is responsibility of the proposer to decide what to do: to 43 | accept this value or start another round. 44 | 45 | ### Acceptor 46 | 47 | Receive and respond requests fro Proposer. 48 | 49 | ### Round 50 | 51 | Monotonically incremental value to identify a paxos round proposed by 52 | proposer. Format: 53 | ``` 54 | (int, id_of_proposer) 55 | ``` 56 | 57 | ### Value 58 | 59 | Any data proposed, accepted or committed. 60 | 61 | ### Value Round 62 | 63 | The `Round` in which value is established. 64 | 65 | ### Version 66 | 67 | `Version` is used to identify every committed `Value`. 68 | Each time a `Value` is committed, `Version` increments by 1. 69 | 70 | The initial `Version` is 0, which means nothing has been committed. 71 | 72 | In this implementation, all of the data is stored altogether in one record: 73 | ```lua 74 | { 75 | ver = 1, 76 | val = { 77 | view = { 78 | { a=1, b=1, c=1 }, 79 | { b=1, c=1, d=1 }, 80 | }, 81 | leader = { ident="id", __expire=10000000 }, 82 | action = { 83 | { name="bla", args={} }, 84 | { name="foo", args={} }, 85 | }, 86 | } 87 | } 88 | ``` 89 | It commits the entire record into storage as a single write operation. 90 | 91 | ### Quorum 92 | 93 | By definition it is subset of member that any two quorum must have non-empty 94 | intersection. 95 | 96 | In our implementation, in simple words, it is a sub set of more than half of 97 | all members. 98 | 99 | ### View 100 | 101 | Membership of a paxos group. 102 | 103 | View is no more than a common field in table `val`. 104 | Proposer and Acceptor uses committed `view` as cluster that it belongs to. 105 | 106 | Thus partially committed value would let Proposer and Acceptor sees different 107 | cluster. But it does not break consensus. Following is how it is solved. 108 | 109 | View is capable to be updated on the fly with the same failure tolerance as 110 | normal paxos proposal. 111 | Thus there is no down time during membership changing. 112 | 113 | ```lua 114 | view = { 115 | { a=1, b=1, c=1 }, 116 | { b=1, c=1, d=1 }, 117 | } 118 | ``` 119 | View consists of 1 or 2 sub table representing membership: 120 | Being with only 1 sub table is the stable stat. 121 | 122 | To change view from A to B, an intermediary stat of A + B is introduced to 123 | connect two views that might be totally different. 124 | 125 | Quorum(A) 126 | Quorum(A)+Quorum(B) 127 | Quorum(B) 128 | 129 | At any time, committed view 130 | 131 | Version of A is 10 132 | 133 | 1. Commit dual view to quorum of cluster A. 134 | 1. Now quorum becomes quorum(A) & quorum(B). 135 | 1. Commit again to A + B. Make sure that majority of both A and B has view 136 | A+B. 137 | 1. Commit B to A + B. Make sure that majority of both A and B has the view B 138 | 1. Members in A but not in B find out that it is no more a member of the 139 | latest version of view. Then it destories itself. 140 | 141 | Without intermedia A+B, there might be a gap in time none of A or B being able 142 | to accept any value. 143 | 144 | 145 | ### Instance 146 | 147 | According to original paper [paxos][paxos_made_simple], paxos is 148 | identified by `instance_id`. 149 | In our implementation, there is only one record 150 | thus `Instance` is identified just by `Version`. 151 | 152 | On each `Acceptor`, there is only one legal `Instance` at any time: 153 | the `Instance` for next version to commit. 154 | 155 | Thus on `Acceptor`, if the version committed is 3, then only requests to 156 | version 4 will be served. Other requests would be rejected with an error. 157 | 158 | ### Internal Request and Response 159 | 160 | #### Request 161 | ```lua 162 | { 163 | cmd="phase1", 164 | cluster_id="xx", 165 | ident="receiving_acceptor", 166 | ver="next_version", 167 | 168 | -- for phase1 169 | rnd={ int, proposer_id }, 170 | 171 | -- for phase2 172 | rnd={ int, proposer_id }, 173 | val={}, 174 | 175 | -- for phase3 176 | val={}, 177 | } 178 | ``` 179 | * cmd: 180 | 181 | phase1, phase2, phase3 or something else. 182 | 183 | * cluster_id: 184 | 185 | Identifier of this cluster. 186 | 187 | * ident: 188 | 189 | Identifier of Proposer sending this request. 190 | 191 | * ver: 192 | 193 | `Version`. 194 | 195 | * rnd: 196 | 197 | `Round`. 198 | 199 | * val: 200 | 201 | The `Value` to accept or to commit. 202 | 203 | #### Response 204 | Valid Response is always a table. Table content is specific to different 205 | requests. 206 | 207 | 208 | #### Response with error 209 | ```lua 210 | { 211 | err = { 212 | Code = "string_code", 213 | Message = {}, 214 | } 215 | } 216 | ``` 217 | * Code: 218 | 219 | String error name for program. 220 | 221 | * Message: 222 | 223 | Additional error information for human or further error handling. 224 | 225 | 226 | ### Phase-1 227 | 228 | AKA prepare. 229 | 230 | #### Phase-1 Request 231 | ```lua 232 | { 233 | cmd="phase1", 234 | cluster_id="xx", 235 | ident="receiving_acceptor", 236 | ver="next_version", 237 | 238 | rnd={ int, proposer_id }, 239 | } 240 | ``` 241 | 242 | #### Phase-1 Response 243 | ```lua 244 | { 245 | rnd={ int, proposer_id }, 246 | vrnd={ int, proposer_id }, 247 | val={}, 248 | } 249 | ``` 250 | * rnd: 251 | 252 | The latest `Round` acceptor ever seen, including the one in request. 253 | 254 | * vrnd: 255 | 256 | The `Round` in which `Value` was accepted. 257 | 258 | It is `nil` if no value accepted. 259 | 260 | * val: 261 | 262 | The `Value` accepted. 263 | 264 | It is `nil` if no value accepted. 265 | 266 | ### Phase-2 267 | 268 | AKA accept. 269 | 270 | #### Phase-2 Request 271 | ```lua 272 | { 273 | cmd="phase1", 274 | cluster_id="xx", 275 | ident="receiving_acceptor", 276 | ver="next_version", 277 | 278 | rnd={ int, proposer_id }, 279 | val={} 280 | } 281 | ``` 282 | 283 | #### Phase-2 Response 284 | ```lua 285 | {} 286 | ``` 287 | If value was accepted, acceptor returns empty table. 288 | Or an `err` field describing the error. 289 | 290 | ### Phase-3 291 | 292 | AKA commit. 293 | 294 | #### Phase-3 Request 295 | ```lua 296 | { 297 | cmd="phase1", 298 | cluster_id="xx", 299 | ident="receiving_acceptor", 300 | ver="next_version", 301 | 302 | val={} 303 | } 304 | ``` 305 | 306 | #### Phase-3 Response 307 | ```lua 308 | {} 309 | ``` 310 | If value was committed, acceptor returns empty table. 311 | Or an `err` field describing the error. 312 | 313 | 314 | ## Reference 315 | 316 | [paxos made simple][paxos_made_simple] 317 | 318 | [paxos_made_simple]: http://www.google.com 319 | -------------------------------------------------------------------------------- /example/common.config: -------------------------------------------------------------------------------- 1 | 2 | # worker_processes 1; 3 | daemon off; 4 | master_process off; 5 | # error_log err.log error; 6 | error_log stderr error; 7 | pid pid.pid; 8 | 9 | events { 10 | worker_connections 256; 11 | } 12 | # vim: ft=ngx 13 | -------------------------------------------------------------------------------- /example/http.config: -------------------------------------------------------------------------------- 1 | 2 | log_format accfmt '$remote_addr [$time_local]' 3 | ' "$request" $status $bytes_sent $request_time' 4 | ; 5 | 6 | access_log acc.log accfmt; 7 | 8 | lua_package_path '$prefix/../../lib/?.lua;;'; 9 | lua_package_cpath '$prefix/../../clib/?.so;;'; 10 | 11 | # default shared dict lock storage used by paxos 12 | lua_shared_dict paxos_lock 10m; 13 | lua_socket_log_errors off; 14 | 15 | init_worker_by_lua 'require("worker_init")'; 16 | # vim: ft=ngx 17 | -------------------------------------------------------------------------------- /example/mem-1/conf/nginx.conf: -------------------------------------------------------------------------------- 1 | include ../../common.config; 2 | http { 3 | include ../../http.config; 4 | 5 | server { 6 | listen 9901; 7 | location / { content_by_lua 'require("worker_init").cc.server:handle_req()'; } 8 | } 9 | 10 | server { 11 | listen 9801; 12 | 13 | location /get/ { 14 | rewrite_by_lua 'require("handle_get").get()'; 15 | } 16 | 17 | location /www/ { 18 | content_by_lua 'ngx.say(ngx.var.uri .. " from " .. require("worker_init").ident)'; 19 | # root www/; 20 | } 21 | 22 | location /proxy/ { 23 | set_by_lua $addr 'return ngx.var.uri:sub(8)'; 24 | proxy_pass http://$addr; 25 | } 26 | } 27 | } 28 | # vim: ft=ngx 29 | -------------------------------------------------------------------------------- /example/mem-2/conf/nginx.conf: -------------------------------------------------------------------------------- 1 | # worker_processes 1; 2 | daemon off; 3 | master_process off; 4 | # error_log err.log error; 5 | error_log stderr error; 6 | pid pid.pid; 7 | 8 | events { 9 | worker_connections 256; 10 | } 11 | 12 | http { 13 | log_format accfmt '$remote_addr [$time_local]' 14 | ' "$request" $status $bytes_sent $request_time' 15 | ; 16 | 17 | access_log acc.log accfmt; 18 | 19 | lua_package_path '$prefix/../../lib/?.lua;;'; 20 | lua_package_cpath '$prefix/../../clib/?.so;;'; 21 | 22 | lua_shared_dict paxos_lock 10m; 23 | lua_socket_log_errors off; 24 | 25 | init_worker_by_lua 'require("worker_init")'; 26 | 27 | server { 28 | listen 9902; 29 | location / { content_by_lua 'require("worker_init").cc.server:handle_req()'; } 30 | } 31 | 32 | server { 33 | listen 9802; 34 | 35 | location /get/ { 36 | rewrite_by_lua 'require("handle_get").get()'; 37 | } 38 | 39 | location /www { 40 | content_by_lua 'ngx.say(ngx.var.uri .. " from " .. require("worker_init").ident)'; 41 | } 42 | 43 | location /proxy/ { 44 | set_by_lua $addr 'return ngx.var.uri:sub(8)'; 45 | proxy_pass http://$addr; 46 | } 47 | } 48 | } 49 | # vim: ft=ngx 50 | -------------------------------------------------------------------------------- /example/mem-3/conf/nginx.conf: -------------------------------------------------------------------------------- 1 | # worker_processes 1; 2 | daemon off; 3 | master_process off; 4 | # error_log err.log error; 5 | error_log stderr error; 6 | pid pid.pid; 7 | 8 | events { 9 | worker_connections 256; 10 | } 11 | 12 | http { 13 | log_format accfmt '$remote_addr [$time_local]' 14 | ' "$request" $status $bytes_sent $request_time' 15 | ; 16 | 17 | access_log acc.log accfmt; 18 | 19 | lua_package_path '$prefix/../../lib/?.lua;;'; 20 | lua_package_cpath '$prefix/../../clib/?.so;;'; 21 | 22 | lua_shared_dict paxos_lock 10m; 23 | lua_socket_log_errors off; 24 | 25 | init_worker_by_lua 'require("worker_init")'; 26 | 27 | server { 28 | listen 9903; 29 | location / { content_by_lua 'require("worker_init").cc.server:handle_req()'; } 30 | } 31 | 32 | server { 33 | listen 9803; 34 | 35 | location /get/ { 36 | rewrite_by_lua 'require("handle_get").get()'; 37 | } 38 | 39 | location /www { 40 | content_by_lua 'ngx.say(ngx.var.uri .. " from " .. require("worker_init").ident)'; 41 | } 42 | 43 | location /proxy/ { 44 | set_by_lua $addr 'return ngx.var.uri:sub(8)'; 45 | proxy_pass http://$addr; 46 | } 47 | } 48 | } 49 | # vim: ft=ngx 50 | -------------------------------------------------------------------------------- /it/README.md: -------------------------------------------------------------------------------- 1 | integration test 2 | -------------------------------------------------------------------------------- /it/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drmingdrmer/lua-paxos/85489584aaba83467bf544c6532d737205995561/it/__init__.py -------------------------------------------------------------------------------- /it/http.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import socket 3 | import select 4 | 5 | class S2HttpError(Exception): pass 6 | class BadStatus(S2HttpError): pass 7 | class LineTooLong(S2HttpError): pass 8 | class BadHeadersError(S2HttpError): pass 9 | class ChunkedSizeError(S2HttpError): pass 10 | class NotConnectedError(S2HttpError): pass 11 | class ResponseNotReady(S2HttpError): pass 12 | class HeadersError(S2HttpError): pass 13 | 14 | _MAXLINESIZE = 65536 15 | LINE_RECV_LENGTH = 1024*4 16 | BLOCK_LENGTH = 1024 * 1024 * 20 17 | SEND_BLOCK_SIZE = 8192 18 | 19 | #example, how to use 20 | # http = Http('127.0.0.1', 6003) 21 | # http.request('/file/aa') 22 | # status = http.status 23 | # headers = http.headers 24 | # buf = http.read_body(50*MB) 25 | #or 26 | # http = Http('127.0.0.1', 6003) 27 | # http.send_request('file/aa') 28 | # http.send_body(body) 29 | # http.finish_request() 30 | # status = http.status 31 | # headers = http.headers 32 | # buf = http.read_body(50*MB) 33 | 34 | class Http(object): 35 | 36 | def __init__(self, ip, port, timeout = 60): 37 | 38 | self.ip = ip 39 | self.port = port 40 | self.timeout = timeout 41 | self.sock = None 42 | 43 | self.chunked = False 44 | self.chunk_left = None 45 | self.content_len = None 46 | self.has_read = 0 47 | 48 | self.status = None 49 | self.headers = {} 50 | 51 | self.recv_iter = None 52 | 53 | def __del__( self ): 54 | 55 | if self.recv_iter is not None: 56 | try: 57 | self.recv_iter.close() 58 | except: 59 | pass 60 | self.recv_iter = None 61 | 62 | if self.sock is not None: 63 | try: 64 | self.sock.close() 65 | except: 66 | pass 67 | self.sock = None 68 | 69 | def request(self, uri, method = 'GET', headers = {}): 70 | 71 | self.send_request( uri, method = method, headers = headers ) 72 | 73 | self.finish_request() 74 | 75 | def send_request( self, uri, method = 'GET', headers = {} ): 76 | 77 | self._reset_request() 78 | 79 | sbuf = [ '{method} {uri} HTTP/1.1'.format( method=method, uri=uri ), ] 80 | sbuf += self._norm_headers(headers) 81 | 82 | self.sock = socket.socket( socket.AF_INET, socket.SOCK_STREAM ) 83 | self.sock.settimeout( self.timeout ) 84 | self.sock.connect( (self.ip, self.port) ) 85 | 86 | sbuf.extend(['', '']) 87 | msg = "\r\n".join( sbuf ) 88 | 89 | self.sock.sendall( msg ) 90 | 91 | def send_body( self, body ): 92 | if self.sock is None: 93 | raise NotConnectedError() 94 | 95 | self.sock.sendall( body ) 96 | 97 | def finish_request( self ): 98 | 99 | if self.status is not None: 100 | raise ResponseNotReady() 101 | 102 | self.recv_iter = _recv_loop( self.sock, self.timeout ) 103 | # swollow the first yield and let recv_iter wait for argument 104 | self.recv_iter.next() 105 | 106 | self._load_resp_status() 107 | self._load_resp_headers() 108 | 109 | def read_body(self, size): 110 | 111 | if size is None or size < 0: 112 | raise ValueError('size error!') 113 | 114 | if self.chunked: 115 | buf = self._read_chunked(size) 116 | self.has_read += len(buf) 117 | return buf 118 | 119 | if size > self.content_len - self.has_read: 120 | size = self.content_len - self.has_read 121 | 122 | buf = self._read(size) 123 | self.has_read += size 124 | 125 | return buf 126 | 127 | def _reset_request(self): 128 | 129 | self.chunked = False 130 | self.chunk_left = None 131 | self.content_len = None 132 | self.has_read = 0 133 | 134 | self.status = None 135 | self.headers = {} 136 | 137 | def _norm_headers( self, headers=None ): 138 | 139 | _h = { 140 | 'Host': self.ip, 141 | } 142 | _h.update( headers or {} ) 143 | 144 | hs = [] 145 | 146 | for hk, hv in _h.items(): 147 | hs.append( '%s: %s'%(hk, hv) ) 148 | 149 | return hs 150 | 151 | def _read(self, size): 152 | return self.recv_iter.send( ( 'block', size ) ) 153 | 154 | def _readline(self): 155 | return self.recv_iter.send( ( 'line', None ) ) 156 | 157 | def _read_status(self): 158 | 159 | line = self._readline() 160 | 161 | try: 162 | version, status, reason = line.strip().split(None, 2) 163 | status = int( status ) 164 | except ValueError as e: 165 | raise BadStatusLine( line ) 166 | 167 | if not version.startswith('HTTP/'): 168 | raise BadStatusLine( line ) 169 | 170 | if status < 100 or status > 999: 171 | raise BadStatusLine( line ) 172 | 173 | return status 174 | 175 | def _load_resp_status(self): 176 | 177 | while True: 178 | 179 | status = self._read_status() 180 | if status != 100: 181 | break 182 | 183 | # skip the header from the 100 response 184 | while True: 185 | skip = self._readline() 186 | if skip.strip() != '': 187 | break 188 | 189 | self.status = status 190 | 191 | def _load_resp_headers(self): 192 | 193 | while True: 194 | 195 | line = self._readline() 196 | if line == '': 197 | break 198 | 199 | kv = line.strip().split(':', 1) 200 | hname = kv[0] 201 | hval = kv[1] 202 | 203 | self.headers[ hname.lower() ] = hval.strip() 204 | 205 | code = self.headers.get('transfer-encoding', '') 206 | if code.lower() == 'chunked': 207 | self.chunked = True 208 | 209 | length = self.headers.get('content-length') 210 | if self.chunked: 211 | self.content_len = None 212 | else: 213 | if length is None: 214 | raise HeadersError('should have content-length header') 215 | elif not length.isdigit(): 216 | raise HeadersError('content-length header value error') 217 | else: 218 | self.content_len = int(length) 219 | 220 | def _get_chunk_size(self): 221 | 222 | line = self._readline() 223 | 224 | i = line.find(';') 225 | if i >= 0: 226 | # strip chunk-extensions 227 | line = line[:i] 228 | 229 | try: 230 | chunk_size = int(line, 16) 231 | except ValueError: 232 | raise ChunkedSizeError() 233 | 234 | return chunk_size 235 | 236 | def _read_chunked(self, size): 237 | 238 | buf = [] 239 | 240 | if self.chunk_left == 0: 241 | return '' 242 | 243 | while size > 0: 244 | 245 | if self.chunk_left is None: 246 | self.chunk_left = self._get_chunk_size() 247 | 248 | if self.chunk_left == 0: 249 | break 250 | 251 | toread = min(size, self.chunk_left) 252 | buf.append( self._read(toread) ) 253 | 254 | size -= toread 255 | self.chunk_left -= toread 256 | 257 | if self.chunk_left == 0: 258 | self._read( len('\r\n') ) 259 | self.chunk_left = None 260 | 261 | if self.chunk_left != 0: 262 | return ''.join(buf) 263 | 264 | # discard trailer 265 | while True: 266 | line = self._readline() 267 | if line == '': 268 | break 269 | 270 | self.chunk_left = 0 271 | 272 | return ''.join(buf) 273 | 274 | def _recv_loop( sock, timeout ): 275 | 276 | bufs = [''] 277 | mode, size = yield 278 | 279 | while True: 280 | 281 | if mode == 'line': 282 | 283 | buf = bufs[ 0 ] 284 | if '\r\n' in buf: 285 | rst, buf = buf.split( '\r\n', 1 ) 286 | bufs[ 0 ] = buf 287 | mode, size = yield rst 288 | continue 289 | else: 290 | if len( buf ) >= _MAXLINESIZE: 291 | raise LineTooLong() 292 | else: 293 | buf += _recv_raise( sock, timeout, LINE_RECV_LENGTH )[ 'buf' ] 294 | bufs[ 0 ] = buf 295 | continue 296 | else: 297 | total = len( bufs[ 0 ] ) 298 | while total < size: 299 | bufs.append( _recv_raise( sock, timeout, size-total )[ 'buf' ] ) 300 | total += len(bufs[ -1 ]) 301 | 302 | rst = ''.join( bufs ) 303 | if size < len( rst ): 304 | bufs = [ rst[ size: ] ] 305 | rst = rst[ :size ] 306 | else: 307 | bufs = [ '' ] 308 | mode, size = yield rst 309 | 310 | def _recv_raise( sock, timeout, size ): 311 | 312 | o = { 'buf':None } 313 | 314 | for ii in range( 2 ): 315 | try: 316 | o['buf'] = sock.recv( size ) 317 | break 318 | except socket.error as e: 319 | if len(e.args) > 0 and e.args[ 0 ] == errno.EAGAIN: 320 | evin, evout, everr = select.select( 321 | [ sock.fileno() ], [], [], timeout ) 322 | if len( evin ) != 0: 323 | continue 324 | else: 325 | raise socket.timeout( 'timeout %d seconds while waiting to recv' 326 | % timeout ) 327 | else: 328 | raise 329 | 330 | if o['buf'] == '': 331 | raise socket.error('want to read %d bytes, but read empty !' % size) 332 | 333 | return o 334 | 335 | 336 | -------------------------------------------------------------------------------- /it/ngxconf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import string 5 | 6 | tmpl = ''' 7 | worker_processes 1; 8 | error_log logs/err-$ident.log error; 9 | pid logs/$ident.pid; 10 | 11 | events { 12 | worker_connections 256; 13 | } 14 | 15 | http { 16 | log_format accfmt '$$remote_addr [$$time_local]' 17 | ' "$$request" $$status $$bytes_sent $$request_time' 18 | ' "$$paxos_log"' 19 | ; 20 | 21 | access_log logs/acc-$ident.log accfmt; 22 | 23 | lua_package_path '$$prefix/../../lib/?.lua;;'; 24 | lua_package_cpath '$$prefix/../../clib/?.so;;'; 25 | 26 | lua_shared_dict paxos_lock 10m; 27 | lua_socket_log_errors off; 28 | init_worker_by_lua 'local e=require("sample"); e.members_on_this_node={"$ident"}; e.init_cluster_check($enabled)'; 29 | 30 | server { 31 | listen 908$ident; 32 | 33 | location /api/ { 34 | set $$paxos_log ""; 35 | content_by_lua 'require("sample").cluster.server:handle_req()'; 36 | } 37 | 38 | location /user_api/get_leader { 39 | set $$paxos_log ""; 40 | content_by_lua ' 41 | 42 | local function output( code, ... ) 43 | ngx.status = code 44 | ngx.print( ... ) 45 | ngx.eof() 46 | ngx.exit( ngx.HTTP_OK ) 47 | end 48 | 49 | local s = require("sample").cluster.server 50 | local paxos, err, errmes = s:new_paxos({cluster_id=ngx.var.arg_cluster_id, ident=ngx.var.arg_ident}) 51 | if err then 52 | output( 500, err ) 53 | end 54 | 55 | local _l, err, errmes = paxos:local_get("leader") 56 | if err then 57 | output( 500, err ) 58 | end 59 | 60 | local _m, err, errmes = paxos:local_get_members() 61 | if err then 62 | output( 500, err ) 63 | end 64 | 65 | local ids = {} 66 | 67 | for k, _ in pairs( _m.val or {} ) do 68 | table.insert( ids, k ) 69 | end 70 | table.sort(ids) 71 | local ids = table.concat( ids, "," ) 72 | 73 | local leader, ver = _l.val, _l.ver 74 | if leader then 75 | output( 200, 76 | "ver:", _l.ver, 77 | " leader:", leader.ident, 78 | " lease:", leader.__lease, 79 | " members:", ids 80 | ) 81 | else 82 | output( 404, "- -" ) 83 | end 84 | '; 85 | } 86 | } 87 | } 88 | # vim: ft=ngx 89 | ''' 90 | 91 | pref = 'srv/nginx/conf/' 92 | 93 | def make_conf(n=3, enable_cluster_check = 'false'): 94 | for i in range( 1, n+1 ): 95 | data = { 'ident': str(i), 'enabled': enable_cluster_check } 96 | cont = string.Template( tmpl ).substitute( data ) 97 | with open( pref + str(i), 'w' ) as f: 98 | f.write( cont ) 99 | f.close() 100 | 101 | if __name__ == "__main__": 102 | make_conf(3, 'false') 103 | -------------------------------------------------------------------------------- /it/ngxctl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.6 2 | # coding: utf-8 3 | 4 | import sys 5 | import subprocess 6 | 7 | ngxpath = 'srv/nginx/sbin/nginx' 8 | 9 | def ngx_restart(idents): 10 | for ident in idents: 11 | ngx_stop(ident) 12 | ngx_start(ident) 13 | 14 | def ngx_stop(ident): 15 | subprocess.call( [ 16 | ngxpath, 17 | "-c", "conf/" + str(ident), 18 | "-s", "stop", 19 | ] ) 20 | 21 | def ngx_start(ident): 22 | subprocess.call( [ 23 | ngxpath, 24 | "-c", "conf/" + str(ident), 25 | ] ) 26 | 27 | if __name__ == "__main__": 28 | 29 | args = sys.argv[1:] 30 | cmd = args[0] 31 | ident = args[1] 32 | 33 | if cmd == 'start': 34 | ngx_start(ident) 35 | elif cmd == 'stop': 36 | ngx_stop(ident) 37 | elif cmd == 'restart': 38 | ngx_restart(ident) 39 | else: 40 | raise ValueError("Invalid cmd:", cmd) 41 | -------------------------------------------------------------------------------- /it/paxoscli.py: -------------------------------------------------------------------------------- 1 | 2 | import json 3 | import urllib 4 | 5 | import it.http 6 | import it.paxosclient 7 | 8 | PaxosError = it.paxosclient.PaxosError 9 | 10 | class PaxosClient( it.paxosclient.PaxosClient ): 11 | 12 | api_uri_prefix = '/api' 13 | 14 | def __init__( self, ident ): 15 | ip, port, cluster_id = ip_port_cid(ident) 16 | super(PaxosClient, self).__init__(ip, port, cluster_id, ident) 17 | 18 | def ip_port_cid(ident): 19 | ip = '127.0.0.1' 20 | port = 9080+int(ident) 21 | cluster_id = 'x' 22 | return ip, port, cluster_id 23 | 24 | def ids_dic(ids): 25 | return dict([ (str(x),str(x)) 26 | for x in ids ]) 27 | 28 | def init_view( ident, view_ids, ver=1 ): 29 | 30 | if type(view_ids[0]) not in (type(()), type([])): 31 | view_ids = [view_ids] 32 | 33 | view = [ ids_dic(x) for x in view_ids ] 34 | 35 | return request( 'phase3', ident, 36 | { 'ver':ver, 37 | 'val': { 38 | 'view': view, 39 | } } ) 40 | 41 | def request( cmd, ident, body=None ): 42 | if cmd == 'get_leader': 43 | cmd, body = 'get', {"key":"leader"} 44 | 45 | elif cmd == 'get_view': 46 | cmd, body = 'get', {"key":"view"} 47 | 48 | ip, port, cluster_id = ip_port_cid(ident) 49 | return it.paxosclient.request(ip, port, cluster_id, ident, cmd, body=body) 50 | 51 | def request_ex( cmd, to_ident, body=None ): 52 | ip, port, cluster_id = ip_port_cid(to_ident) 53 | return it.paxosclient.request_ex(ip, port, cluster_id, to_ident, cmd, body=body) 54 | -------------------------------------------------------------------------------- /it/paxosclient.py: -------------------------------------------------------------------------------- 1 | 2 | import sys 3 | import json 4 | import urllib 5 | 6 | import it.http 7 | 8 | _http = it.http 9 | 10 | class PaxosError( Exception ): 11 | def __init__( self, *args, **argkv ): 12 | self.Code = argkv.get( 'Code' ) 13 | self.Message = argkv.get( 'Message' ) 14 | 15 | class PaxosClient( object ): 16 | 17 | api_uri_prefix = '/api' 18 | 19 | def __init__( self, ip, port, cluster_id, ident ): 20 | self.ip = ip 21 | self.port = port 22 | self.cluster_id = cluster_id 23 | self.ident = ident 24 | 25 | def send_cmd( self, cmd, reqbody=None ): 26 | 27 | req = { 'cmd':cmd } 28 | 29 | if reqbody is not None: 30 | req.update( reqbody ) 31 | 32 | uri = self.make_uri( req ) 33 | 34 | if reqbody is not None: 35 | reqbody = json.dumps(reqbody) 36 | 37 | return self.http( uri, reqbody=reqbody ) 38 | 39 | def http( self, uri, reqbody=None ): 40 | 41 | reqbody = reqbody or '' 42 | 43 | h = _http.Http( self.ip, self.port, timeout=3 ) 44 | h.send_request( uri, headers={ 'Content-Length': len( reqbody ) } ) 45 | h.send_body( reqbody ) 46 | h.finish_request() 47 | 48 | body = h.read_body( 1024*1024 ) 49 | try: 50 | body = json.loads( body ) 51 | except: 52 | pass 53 | return { 'status': h.status, 54 | 'headers': h.headers, 55 | 'body': body, } 56 | 57 | def make_uri( self, req ): 58 | 59 | uri = self.api_uri_prefix \ 60 | + ('/{cluster_id}/{ident}/{cmd}'.format( 61 | cluster_id=self.cluster_id, 62 | ident=self.ident, 63 | cmd=req[ 'cmd' ], 64 | )) 65 | 66 | query_keys = [ 'ver' ] 67 | q = {} 68 | for k in query_keys: 69 | if k in req: 70 | q[ k ] = req[ k ] 71 | 72 | uri += '?' + urllib.urlencode( q ) 73 | return uri 74 | 75 | def init_view( ip, port, cluster_id, ident, members ): 76 | 77 | view = [ members ] 78 | 79 | return request_ex( ip, port, cluster_id, ident, 'phase3', { 80 | 'ver': 1, 81 | 'val': { 82 | 'view': [ members ], 83 | } 84 | } ) 85 | 86 | def request( ip, port, cluster_id, ident, cmd, body=None ): 87 | 88 | p = PaxosClient( ip, port, cluster_id, ident ) 89 | rst = p.send_cmd(cmd, reqbody=body) 90 | return rst 91 | 92 | def request_ex( ip, port, cluster_id, ident, cmd, body=None ): 93 | 94 | rst = request( ip, port, cluster_id, ident, cmd, body=body ) 95 | b = rst[ 'body' ] 96 | if 'err' in b: 97 | e = b[ 'err' ] 98 | raise PaxosError( **e ) 99 | 100 | return b 101 | 102 | if __name__ == "__main__": 103 | cmd = sys.argv[1] 104 | if cmd == 'init': 105 | ip, port, cluster_id, ident = sys.argv[2:6] 106 | port = int(port) 107 | 108 | members = sys.argv[6:] 109 | members = dict([(x, i+1) for i, x in enumerate(members)]) 110 | 111 | init_view(ip, port, cluster_id, ident, members) 112 | else: 113 | raise 114 | -------------------------------------------------------------------------------- /it/sto.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import os 3 | 4 | sto_base = "/tmp/paxos_test" 5 | 6 | def init_sto(): 7 | 8 | try: 9 | shutil.rmtree(sto_base) 10 | except Exception as e: 11 | print repr(e) 12 | 13 | os.mkdir(sto_base) 14 | -------------------------------------------------------------------------------- /lib/acid/cache.lua: -------------------------------------------------------------------------------- 1 | local json = require('cjson') 2 | local resty_lock = require("resty.lock") 3 | local tableutil = require("acid.tableutil") 4 | local strutil = require("acid.strutil") 5 | 6 | local to_str = strutil.to_str 7 | local ngx = ngx 8 | 9 | local _M = { _VERSION = '1.0' } 10 | 11 | -- TODO test 12 | 13 | -- use cache must be declare shared dict 'shared_dict_lock' 14 | -- in nginx configuration 15 | 16 | _M.shared_dict_lock = 'shared_dict_lock' 17 | 18 | _M.accessor = { 19 | 20 | proc = { 21 | 22 | get = function( dict, key, opts ) 23 | 24 | if opts.flush then 25 | return nil 26 | end 27 | 28 | local val = dict[key] 29 | 30 | ngx.log(ngx.DEBUG, "get [", key, "] value from proc cache: ", to_str(val)) 31 | if val ~= nil and val.expires > ngx.time() then 32 | if opts.dup ~= false then 33 | return tableutil.dup( val.data, true ) 34 | end 35 | return val.data 36 | end 37 | 38 | return nil 39 | end, 40 | 41 | set = function( dict, key, val, opts ) 42 | 43 | -- set but timeout at once 44 | if opts.exptime == 0 then 45 | dict[key] = nil 46 | return 47 | end 48 | 49 | val = { expires = os.time() + (opts.exptime or 60), 50 | data = val } 51 | 52 | dict[key] = val 53 | end, 54 | }, 55 | 56 | shdict = { 57 | 58 | get = function( dict, key, opts ) 59 | 60 | if opts.flush then 61 | return nil 62 | end 63 | 64 | local val = dict:get( key ) 65 | ngx.log(ngx.DEBUG, "get [", key, "] value from shdict cache: ", to_str(val)) 66 | if val ~= nil then 67 | 68 | val = json.decode( val ) 69 | 70 | return val 71 | end 72 | return nil 73 | end, 74 | 75 | set = function( dict, key, val, opts ) 76 | 77 | -- shareddict:set(exptime=0) means never timeout 78 | 79 | if opts.exptime == 0 then 80 | dict:delete(key) 81 | return 82 | end 83 | 84 | if val ~= nil then 85 | val = json.encode(val) 86 | end 87 | 88 | dict:set( key, val, opts.exptime or 60 ) 89 | end, 90 | }, 91 | 92 | } 93 | 94 | function _M.cacheable( dict, key, func, opts ) 95 | 96 | local val 97 | local elapsed 98 | local err_code 99 | local err_msg 100 | 101 | opts = tableutil.dup( opts or {}, true ) 102 | 103 | if opts.accessor == nil then 104 | 105 | opts.accessor = _M.accessor.proc 106 | 107 | if type(dict.flush_all) == 'function' then 108 | opts.accessor = _M.accessor.shdict 109 | end 110 | 111 | end 112 | 113 | opts.accessor = { 114 | get = opts.accessor.get or _M.accessor.proc.get, 115 | set = opts.accessor.set or _M.accessor.proc.set, 116 | } 117 | 118 | val = opts.accessor.get( dict, key, opts ) 119 | if val ~= nil then 120 | return val, nil, nil 121 | end 122 | 123 | local lock, err_msg = resty_lock:new( _M.shared_dict_lock, 124 | { exptime = 30, timeout = 30 } ) 125 | if err_msg ~= nil then 126 | return nil, 'SystemError', 127 | err_msg .. ' while new lock:' .. _M.shared_dict_lock 128 | end 129 | 130 | elapsed, err_msg = lock:lock( tostring(dict) .. key ) 131 | if err_msg ~= nil then 132 | return nil, 'LockTimeout', err_msg .. ' while lock:' .. key 133 | end 134 | 135 | val, err_code, err_msg = _M.cacheable_nolock( dict, key, func, opts ) 136 | 137 | lock:unlock() 138 | 139 | return val, err_code, err_msg 140 | end 141 | 142 | function _M.cacheable_nolock( dict, key, func, opts ) 143 | 144 | local val 145 | local err_code 146 | local err_msg 147 | 148 | val = opts.accessor.get( dict, key, opts ) 149 | if val ~= nil then 150 | return val, nil, nil 151 | end 152 | 153 | val, err_code, err_msg = func(unpack(opts.args or {})) 154 | if err_code ~= nil then 155 | return nil, err_code, err_msg 156 | end 157 | 158 | opts.accessor.set( dict, key, val, opts ) 159 | 160 | return val, nil, nil 161 | end 162 | 163 | return _M 164 | -------------------------------------------------------------------------------- /lib/acid/cluster.lua: -------------------------------------------------------------------------------- 1 | local acid_paxos = require( "acid.paxos" ) 2 | local paxoshelper = require( "acid.paxoshelper" ) 3 | local paxosserver = require( "acid.paxosserver" ) 4 | local tableutil = require( "acid.tableutil" ) 5 | 6 | local _M = { 7 | _VERSION="0.1", 8 | 9 | dead_wait = {die_away = 60 * 20, restore = 60 * 60 * 2}, 10 | dead_timeout = 86400, 11 | admin_lease = 60 * 2, 12 | longer_wait = 60 * 60 * 24, 13 | check_itv = {data_check = 60 * 2, leader_check = 10, cluster_check = 60 * 2}, 14 | restore_data_check = 60 * 10, 15 | 16 | max_dead = 4, 17 | 18 | _check = {}, 19 | _longer = {}, 20 | _dead = {}, 21 | } 22 | local _mt = { __index = _M } 23 | 24 | function _M.new(impl, opt) 25 | opt = opt or {} 26 | 27 | local cluster = { 28 | impl = impl, 29 | 30 | check_itv = tableutil.merge(_M.check_itv, opt.check_itv or {}), 31 | dead_wait = tableutil.merge(_M.dead_wait, opt.dead_wait or {}), 32 | dead_timeout = opt.dead_timeout, 33 | admin_lease = opt.admin_lease, 34 | 35 | max_dead = opt.max_dead, 36 | } 37 | setmetatable( cluster, _mt ) 38 | assert( cluster.admin_lease > 4, "lease must be long enough: > 4" ) 39 | 40 | cluster.repair_timeout = math.min(math.ceil(cluster.admin_lease / 2), 5 * 60 ) 41 | 42 | cluster.server = paxosserver.new(impl, { 43 | handlers = opt.handlers, 44 | }) 45 | 46 | return cluster 47 | end 48 | 49 | function _M:delete_cluster(paxos) 50 | 51 | local members, err, errmes = paxos:local_get_members() 52 | if err then 53 | paxos:logerr( "delete_cluster, local_get_members :", err, errmes ) 54 | return nil, err, errmes 55 | end 56 | 57 | local _, err, errmes = paxoshelper.change_view( paxos, { del = members.val } ) 58 | if err then 59 | paxos:logerr( "delete_cluster, change_view :", err, errmes ) 60 | return nil, err, errmes 61 | end 62 | 63 | return nil, nil, nil 64 | 65 | end 66 | 67 | function _M:is_member_longer( paxos, member_id ) 68 | local _mem, err, errmes = paxos:local_get_mem() 69 | if err ~= nil then 70 | if err == 'NoView' then 71 | local now = ngx.now() 72 | local rec_ts = self:record_longer( 73 | member_id.cluster_id, member_id.ident, now ) 74 | paxos:logerr(" longer member :", member_id, now - rec_ts) 75 | return now - rec_ts > self.longer_wait 76 | end 77 | 78 | self:record_longer( member_id.cluster_id, member_id.ident, nil ) 79 | return false 80 | end 81 | 82 | self:record_longer( member_id.cluster_id, member_id.ident, nil ) 83 | return _mem.val == nil 84 | end 85 | 86 | function _M:record_longer( cluster_id, ident, time ) 87 | self._longer[ident] = self._longer[ident] or {} 88 | 89 | local v = self._longer[ident] 90 | if time == nil then 91 | v[cluster_id] = nil 92 | return nil 93 | elseif time == -1 then 94 | return v[cluster_id] 95 | end 96 | 97 | v[cluster_id] = v[cluster_id] or time 98 | return v[cluster_id] 99 | end 100 | 101 | function _M:is_check_needed( check_type, member_id ) 102 | -- check_type: data_check, leader_check, cluster_check 103 | self._check[check_type] = self._check[check_type] or {} 104 | 105 | local c = self._check[check_type] 106 | local exptime = c[member_id.cluster_id] 107 | 108 | local now = self.impl:time() 109 | if exptime == nil or exptime - now <= 0 then 110 | self:reset_check_exptime( 111 | check_type, member_id, now + self.check_itv[check_type]) 112 | return true 113 | end 114 | 115 | return false 116 | end 117 | 118 | function _M:reset_check_exptime( check_type, member_id, exptime ) 119 | self._check[check_type] = self._check[check_type] or {} 120 | 121 | local c = self._check[check_type] 122 | c[member_id.cluster_id] = exptime 123 | end 124 | 125 | function _M:is_leader( leader, member_id ) 126 | return leader.ident == member_id.ident 127 | end 128 | 129 | function _M:leader_check( paxos ) 130 | 131 | local rst, err, errmes = paxoshelper.get_or_elect_leader(paxos, self.admin_lease) 132 | if err then 133 | paxos.ver = nil 134 | paxos:sync() 135 | -- Incomplete view change might cause it to be unable to form a quorum 136 | -- in either previous or next view, thus it stalls leader election. 137 | -- 138 | -- Push view to consistent state by finishing unfinished view change. 139 | local _, _err, _errmes = self:check_view( paxos ) 140 | if _err ~= nil then 141 | paxos:logerr( "check view:", _err, _errmes, paxos.member_id ) 142 | end 143 | return nil, err, 'get or elect leader, ' .. tostring(errmes) 144 | end 145 | 146 | local leader = rst.val 147 | 148 | if not self:is_leader( leader, paxos.member_id ) 149 | or not self:is_check_needed( 'leader_check', paxos.member_id ) then 150 | return leader, nil, nil 151 | end 152 | 153 | local rst, err, errmes = self:extend_lease(paxos, leader) 154 | if err then 155 | return nil, err, 'failure extend leader lease, ' .. tostring(errmes) 156 | end 157 | 158 | return leader, nil, nil 159 | end 160 | 161 | function _M:cluster_check( paxos ) 162 | 163 | if not self:is_check_needed( 'cluster_check', paxos.member_id ) then 164 | return nil, nil, nil 165 | end 166 | 167 | local _, err, errmes = self:check_view( paxos ) 168 | if err ~= nil then 169 | paxos:logerr( "check view, ", err, errmes, paxos.member_id ) 170 | end 171 | 172 | local _, err, errmes = self.impl:write_database(paxos) 173 | if err then 174 | paxos:logerr( 'write database, ', err, errmes, paxos.member_id ) 175 | 176 | if err == 'SameIDCInDB' then 177 | local _, err, errmes = self:delete_cluster(paxos) 178 | if err then 179 | paxos:logerr( "delete cluster fail, ", err, errmes, paxos.member_id ) 180 | end 181 | return nil, nil, nil 182 | end 183 | end 184 | 185 | local _, err, errmes = paxos.impl:wait_run( 186 | self.repair_timeout*0.9, self.repair_cluster, self, paxos) 187 | if err then 188 | paxos:logerr( 'repair cluster, ', paxos.member_id, err, errmes ) 189 | end 190 | 191 | return nil, nil, nil 192 | end 193 | 194 | function _M:member_check(member_id) 195 | 196 | local paxos, err, errmes = acid_paxos.new(member_id, self.impl) 197 | if err then 198 | paxos:logerr("new paxos error:", err, errmes, member_id) 199 | return nil, err, errmes 200 | end 201 | 202 | -- For leader or not: 203 | -- Local storage checking does not require version tracking for paxos. 204 | -- 205 | -- If it is not a member of any view, it is definitely correct to destory 206 | -- this member with all data removed. 207 | -- 208 | -- Race condition happens if: 209 | -- 210 | -- While new version of view has this member and someone else has been 211 | -- initiating this member. 212 | -- 213 | -- While timer triggered routine has been removing data of this 214 | -- member. 215 | -- 216 | -- But it does not matter. Data will be re-built in next checking. 217 | 218 | if self:is_member_longer( paxos, paxos.member_id ) then 219 | return self:destory_member( paxos, paxos.member_id ) 220 | elseif self:record_longer( 221 | member_id.cluster_id, member_id.ident, -1) ~= nil then 222 | return nil, nil, nil 223 | end 224 | 225 | local _, err, errmes = self:data_check(paxos) 226 | if err then 227 | paxos:logerr("data check:", err, errmes, member_id) 228 | return nil, err, errmes 229 | end 230 | 231 | local leader, err, errmes = self:leader_check(paxos) 232 | if err then 233 | paxos:logerr("leader check:", err, errmes, member_id) 234 | return nil, err, errmes 235 | end 236 | 237 | if self:is_leader( leader, member_id ) then 238 | local rst, err, errmes = self:cluster_check(paxos) 239 | if err then 240 | paxos:logerr("cluster check:", err, errmes, member_id) 241 | return nil, err, errmes 242 | end 243 | else 244 | self._dead[member_id.cluster_id] = nil 245 | end 246 | 247 | return nil, nil, nil 248 | end 249 | 250 | function _M:data_check(paxos) 251 | 252 | if not self:is_check_needed( 'data_check', paxos.member_id ) then 253 | return nil, nil, nil 254 | end 255 | 256 | local _mem, err, errmes = paxos:local_get_mem() 257 | if err then 258 | return nil, err, errmes 259 | end 260 | 261 | local _, err, errmes = self.impl:restore(paxos, _mem.val) 262 | if err then 263 | -- to data check, after restore_data_check times 264 | local now = self.impl:time() 265 | local exptime = now + self.restore_data_check 266 | self:reset_check_exptime('data_check', paxos.member_id, exptime) 267 | end 268 | end 269 | 270 | function _M:destory_member( paxos, member_id ) 271 | 272 | paxos:logerr("i am no longer a member of cluster:", member_id) 273 | 274 | local _, err, errmes = self.impl:destory(paxos) 275 | if err then 276 | paxos:logerr("destory err:", err, errmes) 277 | return nil, err, errmes 278 | else 279 | local acc, err, errmes = paxos:new_acceptor() 280 | if err then 281 | return nil, err, errmes 282 | end 283 | 284 | local _mem, err, errmes = paxos:local_get_mem() 285 | if err then 286 | return nil, err, errmes 287 | end 288 | 289 | local rst, err, errmes = acc:destory(_mem.ver) 290 | paxos:logerr("after destory acceptor:", rst, err, errmes) 291 | end 292 | 293 | self:record_longer( member_id.cluster_id, member_id.ident, nil ) 294 | end 295 | 296 | function _M:extend_lease(paxos, leader) 297 | 298 | if leader.__lease < self.repair_timeout then 299 | 300 | local rst, err, errmes = paxoshelper.elect_leader(paxos, self.admin_lease) 301 | if err then 302 | return nil, err, errmes 303 | end 304 | end 305 | return true, nil, nil 306 | end 307 | function _M:repair_cluster(paxos) 308 | 309 | local down_members, err, errmes = self:find_down(paxos) 310 | if err then 311 | return nil, err, errmes 312 | end 313 | 314 | local migrate_members = down_members.migrating or {} 315 | down_members.migrating = nil 316 | 317 | local dead_members = down_members.dead or {} 318 | 319 | if #dead_members > 0 then 320 | paxos:logerr( "dead members confirmed:", dead_members ) 321 | end 322 | 323 | local cluster_id = paxos.member_id.cluster_id 324 | 325 | local nr_down = 0 326 | for _, m in pairs(down_members) do 327 | nr_down = nr_down + #m 328 | end 329 | 330 | if nr_down > self.max_dead then 331 | paxos:logerr( cluster_id, #dead_members, nr_down, 332 | " members down, too many, can not repair" ) 333 | return 334 | end 335 | 336 | for _, _m in ipairs( dead_members ) do 337 | local ident, member = _m[1], _m[2] 338 | local _, err, errmes = self:replace_dead( paxos, ident, member ) 339 | if err then 340 | paxos:logerr( " replace_dead error, ", err, ":", errmes ) 341 | else 342 | self:record_down(cluster_id, ident, nil, nil) 343 | end 344 | 345 | -- fix only one each time 346 | return 347 | end 348 | 349 | -- only all member is alive, to migrate it 350 | -- TODO : if the first sorted member of a cluster is repairing for a long time, 351 | -- disk space of the member will be full all the time. 352 | if nr_down == 0 then 353 | for _, _m in ipairs( migrate_members ) do 354 | local ident, member = _m[1], _m[2] 355 | local _, err, errmes = self:replace_dead( paxos, ident, member ) 356 | if err then 357 | paxos:logerr( " migrate member error, ", err, ":", errmes ) 358 | end 359 | return 360 | end 361 | end 362 | end 363 | 364 | function _M:find_down(paxos) 365 | 366 | local _members, err, errmes = paxos:local_get_members() 367 | if err then 368 | return nil, err, errmes 369 | end 370 | 371 | local down_members = {} 372 | 373 | for ident, member in pairs(_members.val) do 374 | local status, ts, mes = self:confirmed_status(paxos, ident, member) 375 | if status ~= 'alive' then 376 | down_members[status] = down_members[status] or {} 377 | table.insert( down_members[status], { ident, member, ts, mes } ) 378 | end 379 | end 380 | 381 | if next(down_members) ~= nil then 382 | paxos.impl:wait_run(30, 383 | self.report_cluster, self, paxos, down_members) 384 | end 385 | 386 | return down_members, nil, nil 387 | end 388 | 389 | -- status: alive, die_away, restore, migrating, dead 390 | function _M:confirmed_status(paxos, ident, member) 391 | 392 | local cluster_id = paxos.member_id.cluster_id 393 | 394 | local rst, err, errmes = self:send_member_alive(paxos, ident) 395 | if err == nil then 396 | self:record_down(cluster_id, ident, nil, nil) 397 | return 'alive', nil, nil 398 | end 399 | 400 | local status = 'die_away' 401 | if err == 'Damaged' then 402 | status = 'restore' 403 | elseif err == 'Migrating' then 404 | paxos:logerr( "detect status : migrating ", ident, cluster_id) 405 | return 'migrating', nil, nil 406 | end 407 | 408 | local now = paxos.impl:time() 409 | local rec = self:record_down(cluster_id, ident, now, status) 410 | 411 | if rec['restore'] ~= nil then 412 | status = 'restore' 413 | end 414 | 415 | local ts = now - rec[status] 416 | paxos:logerr( "detect status :", status, 417 | ' times :', ts, ident, cluster_id, err, errmes ) 418 | 419 | if ts > ( self.dead_wait[status] or 0 ) then 420 | paxos:logerr("confirmed dead :", ident, cluster_id, status, ts) 421 | status = 'dead' 422 | end 423 | 424 | return status, ts, tostring(err) .. ':' .. tostring(errmes) 425 | end 426 | function _M:replace_dead(paxos, dead_ident, dead_mem) 427 | 428 | local _members, err, errmes = paxos:local_get_members() 429 | if err then 430 | return nil, err, errmes 431 | end 432 | 433 | local new_mem, err, errmes = self.impl:new_member(paxos, dead_ident, _members.val ) 434 | if err then 435 | return nil, err, errmes 436 | end 437 | 438 | local ec_meta, err, errmes = paxos:local_get('ec_meta') 439 | if err then 440 | return nil, err, errmes 441 | end 442 | 443 | ec_meta = ec_meta.val 444 | local set_db = ec_meta.set_db or {cur=ec_meta.ec_name, next=ec_meta.ec_name} 445 | 446 | local changes = { 447 | add = new_mem, 448 | del = { [dead_ident]=dead_mem }, 449 | merge = {ec_meta={set_db=set_db}} 450 | } 451 | 452 | local rst, err, errmes = paxoshelper.change_view( paxos, changes ) 453 | if err then 454 | return nil, err, errmes 455 | end 456 | 457 | local _, err, errmes = self.impl:write_database(paxos) 458 | if err then 459 | return nil, err, errmes 460 | end 461 | 462 | paxos:logerr( "view changed, changes: ", changes, ", view:", rst ) 463 | return rst, err, errmes 464 | end 465 | function _M:record_down(cluster_id, ident, time, status) 466 | 467 | local cd = self._dead 468 | cd[cluster_id] = cd[cluster_id] or {} 469 | 470 | local d = cd[cluster_id] 471 | 472 | if time == nil then 473 | d[ident] = nil 474 | return nil 475 | end 476 | 477 | d[ident] = tableutil.merge( {[status]=time}, d[ident] or {} ) 478 | 479 | return d[ident] 480 | end 481 | function _M:send_member_alive(paxos, ident) 482 | local rst, err, errmes = paxos:send_req(ident, { cmd = "isalive", }) 483 | if err == nil and rst.err == nil then 484 | return nil, nil, nil 485 | end 486 | 487 | local cluster_id = paxos.member_id.cluster_id 488 | 489 | if err == nil then 490 | local e = rst.err or {} 491 | err = e.Code 492 | errmes = e.Message 493 | end 494 | 495 | return nil, err, errmes 496 | end 497 | 498 | function _M:check_view(paxos) 499 | local view, err, errmes = paxos:local_get( 'view' ) 500 | if err ~= nil then 501 | return nil, err, errmes 502 | end 503 | 504 | if #view.val == 1 then 505 | return nil, nil, nil 506 | end 507 | 508 | paxos:sync() 509 | return paxoshelper.change_view( paxos, {} ) 510 | end 511 | 512 | function _M:report_cluster(paxos, down_members) 513 | local tb = {} 514 | for status, members in pairs( down_members ) do 515 | for _, member in pairs( members ) do 516 | local ident, mem, ts, mes = unpack( member ) 517 | ts = ts or 0 518 | if status ~= 'die_away' 519 | or ts > math.max(60 * 60 * 4, self.dead_wait[status]) then 520 | table.insert(tb, 521 | {status=status, index=mem.index, ident=ident, ts=ts, mes=mes}) 522 | end 523 | end 524 | end 525 | 526 | if #tb > 0 then 527 | self.impl:report_cluster( paxos, tb ) 528 | end 529 | end 530 | 531 | return _M 532 | -------------------------------------------------------------------------------- /lib/acid/impl/http.lua: -------------------------------------------------------------------------------- 1 | local strutil = require( "acid.strutil" ) 2 | local tableutil = require( "acid.tableutil" ) 3 | 4 | --example, how to use 5 | -- local h = s2http:new( ip, port, timeout ) 6 | -- h:request( uri, {method='GET', headers={}, body=''} ) 7 | -- status = h.status 8 | -- headers = h.headers 9 | -- buf = h:read_body( size ) 10 | --or 11 | -- local h = s2http:new( ip, port, timeout ) 12 | -- h:send_request( uri, {method='GET', headers={}, body=''} ) 13 | -- h:send_body( body ) 14 | -- h:finish_request() 15 | -- status = h.status 16 | -- headers = h.headers 17 | -- buf = h:read_body( size ) 18 | 19 | 20 | local DEF_PORT = 80 21 | local DEF_METHOD = 'GET' 22 | local DEF_TIMEOUT = 60000 23 | 24 | local NO_CONTENT = 204 25 | local NOT_MODIFIED = 304 26 | 27 | local _M = { _VERSION = '1.0' } 28 | local mt = { __index = _M } 29 | 30 | local function to_str(...) 31 | 32 | local argsv = {...} 33 | 34 | for i=1, select('#', ...) do 35 | argsv[i] = tableutil.str(argsv[i]) 36 | end 37 | 38 | return table.concat( argsv ) 39 | 40 | end 41 | local function _trim( s ) 42 | if type( s ) ~= 'string' then 43 | return s 44 | end 45 | return ( s:gsub( "^%s*(.-)%s*$", "%1" ) ) 46 | end 47 | 48 | local function _read_line( self ) 49 | return self.sock:receiveuntil('\r\n')() 50 | end 51 | 52 | local function _read( self, size ) 53 | if size <= 0 then 54 | return '', nil 55 | end 56 | 57 | return self.sock:receive( size ) 58 | end 59 | 60 | local function discard_lines_until( self, sequence ) 61 | local skip, err_msg 62 | sequence = sequence or '' 63 | 64 | while skip ~= sequence do 65 | skip, err_msg = _read_line( self ) 66 | if err_msg ~= nil then 67 | return 'SocketError', err_msg 68 | end 69 | end 70 | 71 | return nil, nil 72 | end 73 | 74 | local function _load_resp_status( self ) 75 | local status 76 | local line 77 | local err_code 78 | local err_msg 79 | local elems 80 | 81 | while true do 82 | line, err_msg = _read_line( self ) 83 | if err_msg ~= nil then 84 | return 'SocketError', to_str('read status line:', err_msg) 85 | end 86 | 87 | elems = strutil.split( line, ' ' ) 88 | if table.getn(elems) < 3 then 89 | return 'BadStatus', to_str('invalid status line:', line) 90 | end 91 | 92 | status = tonumber( elems[2] ) 93 | 94 | if status == nil or status < 100 or status > 999 then 95 | return 'BadStatus', to_str('invalid status value:', status) 96 | elseif 100 <= status and status < 200 then 97 | err_code, err_msg = discard_lines_until( self, '' ) 98 | if err_code ~= nil then 99 | return err_code, to_str('read header:', err_msg ) 100 | end 101 | else 102 | self.status = status 103 | break 104 | end 105 | end 106 | 107 | return nil, nil 108 | end 109 | 110 | local function _load_resp_headers( self ) 111 | local elems 112 | local err_msg 113 | local line 114 | local hname, hvalue 115 | 116 | self.ori_headers = {} 117 | self.headers = {} 118 | 119 | while true do 120 | 121 | line, err_msg = _read_line( self ) 122 | if err_msg ~= nil then 123 | return 'SocketError', to_str('read header:', err_msg) 124 | end 125 | 126 | if line == '' then 127 | break 128 | end 129 | 130 | elems = strutil.split( line, ':' ) 131 | if table.getn(elems) < 2 then 132 | return 'BadHeader', to_str('invalid header:', line) 133 | end 134 | 135 | hname = string.lower( _trim( elems[1] ) ) 136 | hvalue = _trim( line:sub(string.len(elems[1]) + 2) ) 137 | 138 | self.ori_headers[_trim(elems[1])] = hvalue 139 | self.headers[hname] = hvalue 140 | end 141 | 142 | if self.status == NO_CONTENT or self.status == NOT_MODIFIED 143 | or self.method == 'HEAD' then 144 | return nil, nil 145 | end 146 | 147 | if self.headers['transfer-encoding'] == 'chunked' then 148 | self.chunked = true 149 | return nil, nil 150 | end 151 | 152 | local cont_len = self.headers['content-length'] 153 | if cont_len ~= nil then 154 | cont_len = tonumber( cont_len ) 155 | if cont_len == nil then 156 | return 'BadHeader', to_str('invalid content-length header:', 157 | self.headers['content-length']) 158 | end 159 | self.cont_len = cont_len 160 | return nil, nil 161 | end 162 | 163 | return nil, nil 164 | end 165 | 166 | local function _norm_headers( headers ) 167 | local hs = {} 168 | 169 | for h, v in pairs( headers ) do 170 | if type( v ) ~= 'table' then 171 | v = { v } 172 | end 173 | for _, header_val in ipairs( v ) do 174 | table.insert( hs, to_str( h, ': ', header_val ) ) 175 | end 176 | end 177 | 178 | return hs 179 | end 180 | 181 | local function _read_chunk_size( self ) 182 | local line, err_msg = _read_line( self ) 183 | if err_msg ~= nil then 184 | return nil, 'SocketError', to_str('read chunk size:', err_msg) 185 | end 186 | 187 | local idx = line:find(';') 188 | if idx ~= nil then 189 | line = line:sub(1,idx-1) 190 | end 191 | 192 | local size = tonumber(line, 16) 193 | if size == nil then 194 | return nil, 'BadChunkCoding', to_str('invalid chunk size:', line) 195 | end 196 | 197 | return size, nil, nil 198 | end 199 | 200 | local function _next_chunk( self ) 201 | 202 | local size, err_code, err_msg = _read_chunk_size( self ) 203 | if err_code ~= nil then 204 | return err_code, err_msg 205 | end 206 | 207 | self.chunk_size = size 208 | self.chunk_pos = 0 209 | 210 | if size == 0 then 211 | self.body_end = true 212 | 213 | --discard trailer 214 | local err_code, err_msg = discard_lines_until( self, '' ) 215 | if err_code ~= nil then 216 | return err_code, to_str('read trailer:', err_msg ) 217 | end 218 | end 219 | 220 | return nil, nil 221 | end 222 | 223 | local function _read_chunk( self, size ) 224 | local buf 225 | local err_code 226 | local err_msg 227 | local bufs = {} 228 | 229 | while size > 0 do 230 | if self.chunk_size == nil then 231 | err_code, err_msg = _next_chunk( self ) 232 | if err_code ~= nil then 233 | return nil, err_code, err_msg 234 | end 235 | 236 | if self.body_end then 237 | break 238 | end 239 | end 240 | 241 | buf, err_msg = _read( self, math.min(size, 242 | self.chunk_size - self.chunk_pos)) 243 | if err_msg ~= nil then 244 | return nil, 'SocketError', to_str('read chunked:', err_msg) 245 | end 246 | 247 | table.insert( bufs, buf ) 248 | size = size - #buf 249 | self.chunk_pos = self.chunk_pos + #buf 250 | self.has_read = self.has_read + #buf 251 | 252 | -- chunk end, ignore '\r\n' 253 | if self.chunk_pos == self.chunk_size then 254 | buf, err_msg = _read( self, #'\r\n') 255 | if err_msg ~= nil then 256 | return nil, 'SocketError', to_str('read chunked:', err_msg) 257 | end 258 | self.chunk_size = nil 259 | self.chunk_pos = nil 260 | end 261 | end 262 | 263 | return table.concat( bufs ), nil, nil 264 | end 265 | 266 | function _M.new( _, ip, port, timeout ) 267 | 268 | timeout = timeout or DEF_TIMEOUT 269 | 270 | local sock= ngx.socket.tcp() 271 | sock:settimeout( timeout ) 272 | 273 | local h = { 274 | ip = ip, 275 | port = port or DEF_PORT, 276 | timeout = timeout, 277 | sock = sock, 278 | has_read = 0, 279 | cont_len = 0, 280 | body_end = false, 281 | chunked = false 282 | } 283 | 284 | return setmetatable( h, mt ) 285 | end 286 | 287 | function _M.request( self, uri, opts ) 288 | 289 | local err_code, err_msg = self:send_request( uri, opts ) 290 | if err_code ~= nil then 291 | return err_code, err_msg 292 | end 293 | 294 | return self:finish_request() 295 | end 296 | 297 | function _M.send_request( self, uri, opts ) 298 | 299 | opts = opts or {} 300 | 301 | self.uri = uri 302 | self.method = opts.method or DEF_METHOD 303 | 304 | local body = opts.body or '' 305 | local headers = opts.headers or {} 306 | headers.Host = headers.Host or self.ip 307 | if #body > 0 and headers['Content-Length'] == nil then 308 | headers['Content-Length'] = #body 309 | end 310 | 311 | local sbuf = {to_str(self.method, ' ', self.uri, ' HTTP/1.1'), 312 | unpack( _norm_headers( headers ) ) 313 | } 314 | table.insert( sbuf, '' ) 315 | table.insert( sbuf, body ) 316 | 317 | sbuf = table.concat( sbuf, '\r\n' ) 318 | 319 | local ret, err_msg = self.sock:connect( self.ip, self.port ) 320 | if err_msg ~= nil then 321 | return 'SocketError', to_str('connect:', err_msg) 322 | end 323 | 324 | ret, err_msg = self.sock:send( sbuf ) 325 | if err_msg ~= nil then 326 | return 'SocketError', to_str('request:', err_msg) 327 | end 328 | 329 | return nil, nil 330 | end 331 | 332 | function _M.send_body( self, body ) 333 | local bytes = 0 334 | local err_msg 335 | 336 | if body ~= nil then 337 | bytes, err_msg = self.sock:send( body ) 338 | if err_msg ~= nil then 339 | return nil, 'SocketError', 340 | to_str('send body:', err_msg) 341 | end 342 | end 343 | 344 | return bytes, nil, nil 345 | end 346 | 347 | function _M.finish_request( self ) 348 | local err_code 349 | local err_msg 350 | 351 | err_code, err_msg = _load_resp_status( self ) 352 | if err_code ~= nil then 353 | return err_code, err_msg 354 | end 355 | 356 | err_code, err_msg = _load_resp_headers( self ) 357 | if err_code ~= nil then 358 | return err_code, err_msg 359 | end 360 | 361 | return nil, nil 362 | end 363 | 364 | function _M.read_body( self, size ) 365 | 366 | if self.body_end then 367 | return '', nil, nil 368 | end 369 | 370 | if self.chunked then 371 | return _read_chunk( self, size ) 372 | end 373 | 374 | local rest_len = self.cont_len - self.has_read 375 | 376 | local buf, err_msg = _read( self, math.min(size, rest_len)) 377 | if err_msg ~= nil then 378 | return nil, 'SocketError', to_str('read body:', err_msg) 379 | end 380 | 381 | self.has_read = self.has_read + #buf 382 | 383 | if self.has_read == self.cont_len then 384 | self.body_end = true 385 | end 386 | 387 | 388 | return buf, nil, nil 389 | end 390 | 391 | function _M.set_keepalive( self, timeout, size ) 392 | local rst, err_msg = self.sock:setkeepalive( timeout, size ) 393 | if err_msg ~= nil then 394 | return 'SocketError', to_str('set keepalive:', err_msg) 395 | end 396 | 397 | return nil, nil 398 | end 399 | 400 | function _M.set_timeout( self, time ) 401 | self.sock:settimeout( time ) 402 | end 403 | 404 | function _M.close( self ) 405 | local rst, err_msg = self.sock:close() 406 | if err_msg ~= nil then 407 | return 'SocketError', to_str('close:', err_msg) 408 | end 409 | 410 | return nil, nil 411 | end 412 | 413 | return _M 414 | -------------------------------------------------------------------------------- /lib/acid/impl/locking_ngx.lua: -------------------------------------------------------------------------------- 1 | local _M = {} 2 | local _meta = { __index=_M } 3 | 4 | local resty_lock = require( "resty.lock" ) 5 | 6 | function _M.new(opt) 7 | local e = {} 8 | setmetatable( e, _meta ) 9 | return e 10 | end 11 | 12 | function _M:lock(pobj, exptime) 13 | if exptime == nil then 14 | exptime = 60 15 | end 16 | 17 | local lockname = table.concat( {'paxos', pobj.cluster_id, pobj.ident}, '/' ) 18 | 19 | local _lock = resty_lock:new( "paxos_lock", { exptime=exptime, timeout=1 } ) 20 | local elapsed, err = _lock:lock( lockname ) 21 | if err then 22 | return nil, err 23 | end 24 | 25 | return _lock, nil 26 | end 27 | function _M:unlock(_lock) 28 | if _lock ~= nil then 29 | _lock:unlock() 30 | end 31 | return nil 32 | end 33 | 34 | return _M 35 | -------------------------------------------------------------------------------- /lib/acid/impl/logging_ngx.lua: -------------------------------------------------------------------------------- 1 | local _M = {} 2 | local _meta = { __index=_M } 3 | 4 | local logging = require( "acid.logging" ) 5 | 6 | local log_enabled_phases = { 7 | set=true, 8 | rewrite=true, 9 | access=true, 10 | content=true, 11 | header_filter=true, 12 | body_filter=true, 13 | log=true, 14 | } 15 | 16 | function _M.new(opt) 17 | local e = { 18 | tracking_varname = opt.tracking_varname 19 | } 20 | setmetatable( e, _meta ) 21 | return e 22 | end 23 | 24 | function _M:_log(offset, ...) 25 | ngx.log( ngx.ERR, logging.tostr(...) ) 26 | end 27 | function _M:logerr(...) 28 | self:_log(1, ...) 29 | end 30 | 31 | function _M:track(...) 32 | 33 | local p = ngx.get_phase() 34 | 35 | if not log_enabled_phases[p] then 36 | return 37 | end 38 | 39 | local vname = self.tracking_varname 40 | if vname == nil then 41 | return 42 | end 43 | 44 | if ngx.var[vname] == nil then 45 | return 46 | end 47 | local s = logging.tostr(...) 48 | 49 | if ngx.var[vname] == "" then 50 | ngx.var[vname] = s 51 | else 52 | ngx.var[vname] = ngx.var[vname] .. ', ' .. s 53 | end 54 | end 55 | 56 | return _M 57 | -------------------------------------------------------------------------------- /lib/acid/impl/member.lua: -------------------------------------------------------------------------------- 1 | local _M = {_VERSION="0.1"} 2 | local _meta = { __index=_M } 3 | 4 | function _M.new(opt) 5 | local e = {} 6 | setmetatable( e, _meta ) 7 | return e 8 | end 9 | 10 | function _M:new_member(paxos, dead_ident, members) 11 | return nil, "NotImplemented" 12 | end 13 | 14 | return _M 15 | -------------------------------------------------------------------------------- /lib/acid/impl/storage_ngx_fs.lua: -------------------------------------------------------------------------------- 1 | local _M = {} 2 | local _meta = { __index=_M } 3 | 4 | local json = require( "cjson" ) 5 | local libluafs = require( "libluafs" ) 6 | local strutil = require( "acid.strutil" ) 7 | local cache = require( "acid.cache" ) 8 | local tableutil = require("acid.tableutil") 9 | 10 | local SP_LEN = 1 11 | 12 | local VER_START = 1 13 | local VER_LEN = 1 14 | local VER_END = VER_START + VER_LEN - 1 15 | 16 | local CHKSUM_START = VER_END + SP_LEN + 1 17 | local CHKSUM_LEN = 8 18 | local CHKSUM_END = CHKSUM_START + CHKSUM_LEN - 1 19 | 20 | local CONT_START = CHKSUM_END + SP_LEN + 1 21 | 22 | function _M.new(opt) 23 | 24 | opt = opt or {} 25 | 26 | local e = { 27 | sto_base_path = opt.sto_base_path or "/tmp" 28 | } 29 | setmetatable( e, _meta ) 30 | return e 31 | end 32 | 33 | local counter = 0 34 | local function get_incr_num() 35 | counter = counter + 1 36 | return counter 37 | end 38 | 39 | local function _chksum(cont) 40 | local c = ngx.crc32_long(cont) 41 | local chksum = string.format("%08x", c) 42 | return chksum 43 | end 44 | local function _read(path) 45 | local f, err = io.open(path, "r") 46 | if err then 47 | if string.find(err, "No such file or directory") ~= nil then 48 | return nil, nil, nil 49 | end 50 | return nil, "StorageError", 'open ' .. path .. ', ' .. tostring(err) 51 | end 52 | 53 | local cont = f:read("*a") 54 | f:close() 55 | 56 | if cont == nil then 57 | return nil, "StorageError", 'read ' .. path .. ', ' .. tostring(err) 58 | end 59 | 60 | return cont 61 | end 62 | local function _write(path, cont) 63 | 64 | local tmp_id = ngx.now() .. '_' .. ngx.worker.pid() .. '_' .. get_incr_num() 65 | 66 | local _path = path .. '_' .. tmp_id 67 | local f, err = io.open(_path, "w") 68 | if f == nil then 69 | return nil, "StorageError", 'open ' .. _path .. ', ' .. tostring(err) 70 | end 71 | 72 | local rst, err = f:write( cont ) 73 | if rst == nil then 74 | f:close() 75 | os.remove(_path) 76 | return nil, "StorageError", 'read ' .. _path .. ', ' .. tostring(err) 77 | end 78 | 79 | f:flush() 80 | f:close() 81 | 82 | local _, err = os.rename(_path, path) 83 | if err then 84 | os.remove(_path) 85 | return nil, "StorageError", 'rename '.._path..' to '..path ..', '..tostring(err) 86 | end 87 | 88 | return nil, nil, nil 89 | end 90 | 91 | local function _base(path) 92 | local elts = strutil.split( path, '/' ) 93 | elts[ #elts ] = nil 94 | local base = table.concat( elts, '/' ) 95 | return base 96 | end 97 | local function _makedir(path) 98 | 99 | if libluafs.is_dir(path) then 100 | return nil, nil, nil 101 | end 102 | 103 | local _, err, errmes = _makedir(_base(path)) 104 | if err then 105 | return nil, err, errmes 106 | end 107 | 108 | local ok = libluafs.makedir(path, 0755) 109 | if not ok then 110 | return nil, 'StorageError', 'failure makedir: ' .. tostring(path) 111 | end 112 | 113 | return nil, nil, nil 114 | end 115 | 116 | function _M:get_path(pobj) 117 | -- local path = table.concat( { pobj.cluster_id, pobj.ident }, '/' ) 118 | 119 | local elts = {self.sto_base_path, pobj.cluster_id.."_"..pobj.ident..".paxos"} 120 | local path = table.concat(elts, "/") 121 | return path 122 | end 123 | function _M:_load(pobj) 124 | 125 | local path, err, errmes = self:get_path(pobj) 126 | if err then 127 | return nil, err, errmes 128 | end 129 | 130 | local raw, err, errmes = _read(path) 131 | if err then 132 | return nil, err, errmes 133 | end 134 | 135 | if raw == nil then 136 | return nil, nil, nil 137 | end 138 | 139 | local _ver = raw:sub( VER_START, VER_END ) 140 | local ver = tonumber(_ver) 141 | if ver ~= 1 then 142 | return nil, "StorageError", "data version is invalid: " .. _ver 143 | end 144 | 145 | local chksum = raw:sub( CHKSUM_START, CHKSUM_END ) 146 | 147 | local cont = raw:sub( CONT_START ) 148 | local actual_chksum = _chksum(cont) 149 | 150 | if chksum ~= actual_chksum then 151 | return nil, "StorageError", "checksum unmatched: "..chksum .. ':' .. actual_chksum 152 | end 153 | 154 | local o = json.decode( cont ) 155 | return o, nil, nil 156 | end 157 | function _M:load(pobj, isupdate) 158 | 159 | local key = table.concat( {'paxos', pobj.cluster_id, pobj.ident}, '/' ) 160 | local opts = { exptime = 60 * 30, 161 | args = { self, pobj }, 162 | flush = isupdate} 163 | 164 | local r, err, errmes = cache.cacheable( ngx.shared.paxos_shared_dict, key, _M._load, opts ) 165 | if err then 166 | return nil, err, errmes 167 | end 168 | 169 | --temporary log error for debug cache 170 | if r ~= nil 171 | and r.committed ~= nil 172 | and r.committed.val ~= nil 173 | and r.committed.val.ec_meta ~= nil 174 | and r.committed.val.ec_meta.ec_name ~= pobj.cluster_id then 175 | local _r, _err, _errmes = self:_load( pobj ) 176 | ngx.log(ngx.ERR, 'load committed error, key:', key, ', committed:', tableutil.repr(_r)) 177 | end 178 | 179 | return r, nil, nil 180 | end 181 | function _M:store(pobj) 182 | 183 | local path, err, errmes = self:get_path(pobj) 184 | if err then 185 | return nil, err, errmes 186 | end 187 | 188 | if pobj.record == nil then 189 | ngx.log(ngx.INFO, "delete: ", path) 190 | os.remove(path) 191 | self:load( pobj, true ) 192 | return nil, nil, nil 193 | end 194 | 195 | local _, err, errmes = _makedir( _base( path ) ) 196 | if err then 197 | return nil, err, errmes 198 | end 199 | 200 | local cont = json.encode( pobj.record ) 201 | 202 | local ver = "1" 203 | local chksum = _chksum(cont) 204 | 205 | _, err, errmes = _write(path, ver .. ' ' .. chksum .. ' ' .. cont ) 206 | if err then 207 | return nil, err, errmes 208 | end 209 | 210 | self:load( pobj, true ) 211 | end 212 | 213 | return _M 214 | -------------------------------------------------------------------------------- /lib/acid/impl/storage_ngx_mc.lua: -------------------------------------------------------------------------------- 1 | local _M = {} 2 | local _meta = { __index=_M } 3 | 4 | local json = require( "cjson" ) 5 | local resty_mc = require( "resty.memcached" ) 6 | 7 | local function new_mc() 8 | local mc, err = resty_mc:new({ key_transform = { 9 | function(a) return a end, 10 | function(a) return a end, 11 | } }) 12 | if err then 13 | return nil 14 | end 15 | mc:set_timeout( 1000 ) 16 | local ok, err = mc:connect( "127.0.0.1", 8001 ) 17 | if not ok then 18 | return nil 19 | end 20 | return mc 21 | end 22 | 23 | function _M.new(opt) 24 | 25 | local e = {} 26 | setmetatable( e, _meta ) 27 | return e 28 | end 29 | 30 | function _M:load(pobj) 31 | 32 | local mc = new_mc() 33 | if mc == nil then 34 | return nil 35 | end 36 | 37 | local o, flag, err = mc:get( table.concat( { pobj.cluster_id, pobj.ident }, '/' ) ) 38 | if err then 39 | return nil 40 | end 41 | 42 | -- leave this to detect too many link 43 | -- mc:set_keepalive(10000, 100) 44 | mc:close() 45 | 46 | if o ~= nil then 47 | o = json.decode( o ) 48 | end 49 | return o 50 | end 51 | function _M:store(pobj) 52 | 53 | local ok, err 54 | 55 | local mc = new_mc() 56 | if mc == nil then 57 | return nil, nil, nil 58 | end 59 | 60 | local mckey = table.concat( { pobj.cluster_id, pobj.ident }, '/' ) 61 | 62 | -- record being nil means to delete 63 | if pobj.record == nil then 64 | ok, err = mc:delete(mckey) 65 | else 66 | local o = json.encode( pobj.record ) 67 | ok, err = mc:set( mckey, o ) 68 | end 69 | 70 | -- set_keepalive() 71 | mc:close() 72 | 73 | if not ok then 74 | return nil, (err or 'mc error'), nil 75 | end 76 | return nil, nil, nil 77 | end 78 | 79 | return _M 80 | -------------------------------------------------------------------------------- /lib/acid/impl/time_ngx.lua: -------------------------------------------------------------------------------- 1 | local _M = {} 2 | local _meta = { __index=_M } 3 | 4 | local counter = 0 5 | local function get_incr_num() 6 | counter = counter + 1 7 | return counter 8 | end 9 | 10 | function _M.new(opt) 11 | local e = {} 12 | setmetatable( e, _meta ) 13 | return e 14 | end 15 | 16 | function _M:sleep(n_sec) 17 | ngx.sleep(n_sec or 0.5) 18 | end 19 | function _M:time() 20 | return ngx.time() 21 | end 22 | function _M:new_rnd() 23 | -- round number is not required to monotonically incremental universially. 24 | -- incremental for each version is ok 25 | local ms = math.floor(ngx.now()*1000) % (86400*1000) 26 | -- distinguish different process 27 | local pid = ngx.worker.pid() % 1000 28 | -- distinguish different request 29 | local c = get_incr_num() % 1000 30 | 31 | return (ms*1000 + pid)*1000 + c 32 | end 33 | 34 | function _M:new_rnd_incr(rnd) 35 | local ms = math.ceil(rnd/(1000*1000)) + 1 36 | local pid = ngx.worker.pid() % 1000 37 | local c = get_incr_num() % 1000 38 | 39 | return (ms*1000 + pid)*1000 + c 40 | end 41 | 42 | function _M:wait_run(timeout, f, ...) 43 | 44 | local co = ngx.thread.spawn(f, ...) 45 | 46 | local expire = self:time() + timeout 47 | while self:time() < expire do 48 | if coroutine.status( co ) ~= 'running' then 49 | return nil, nil, nil 50 | end 51 | self:sleep(1) 52 | end 53 | 54 | ngx.thread.kill( co ) 55 | return nil, "Timeout", timeout 56 | end 57 | 58 | return _M 59 | -------------------------------------------------------------------------------- /lib/acid/impl/transport_ngx_http.lua: -------------------------------------------------------------------------------- 1 | local _M = {} 2 | local _meta = { __index=_M } 3 | 4 | local json = require( "cjson" ) 5 | local tableutil = require( "acid.tableutil" ) 6 | local strutil = require( "acid.strutil" ) 7 | local paxos = require( "acid.paxos" ) 8 | local http = require( "acid.impl.http" ) 9 | 10 | local errors = paxos.errors 11 | 12 | local _status = { 13 | OK = 200, 14 | BadRequest = 400, 15 | InternalError = 500, 16 | } 17 | 18 | local err_to_status = { 19 | [errors.InvalidArgument] = _status.BadRequest, 20 | [errors.InvalidMessage] = _status.BadRequest, 21 | [errors.InvalidCommand] = _status.BadRequest, 22 | [errors.InvalidCluster] = _status.BadRequest, 23 | [errors.InvalidPhase] = _status.BadRequest, 24 | [errors.VerNotExist] = _status.BadRequest, 25 | [errors.AlreadyCommitted] = _status.BadRequest, 26 | [errors.OldRound] = _status.BadRequest, 27 | [errors.NoView] = _status.InternalError, 28 | [errors.QuorumFailure] = _status.InternalError, 29 | [errors.LockTimeout] = _status.InternalError, 30 | [errors.StorageError] = _status.InternalError, 31 | [errors.NotAccepted] = _status.BadRequest, 32 | [errors.DuringChange] = _status.BadRequest, 33 | [errors.NoChange] = _status.BadRequest, 34 | [errors.Conflict] = _status.BadRequest, 35 | ["."] = _status.BadRequest, 36 | } 37 | 38 | function _M.new(opt) 39 | local e = {} 40 | setmetatable( e, _meta ) 41 | return e 42 | end 43 | 44 | function _M:send_req(pobj, id, req) 45 | 46 | req = tableutil.dup( req ) 47 | 48 | local uri = self.api_uri .. '/' .. table.concat({pobj.cluster_id, id, req.cmd}, '/') 49 | local query = ngx.encode_args({ 50 | ver = req.ver 51 | }) 52 | 53 | req.cluster_id = nil 54 | req.cmd = nil 55 | req.ver = nil 56 | 57 | local body = json.encode( req ) 58 | local members = tableutil.union( pobj.view ) 59 | local ipports = self:get_addrs({cluster_id=req.cluster_id, ident=id}, members[id]) 60 | local ipport = ipports[1] 61 | local ip, port = ipport[1], ipport[2] 62 | local timeout = 6000 -- milliseconds 63 | local uri = uri .. '?' .. query 64 | 65 | local args = { 66 | body = body, 67 | } 68 | 69 | local h = http:new( ip, port, timeout ) 70 | local err, errmes = h:request( uri, args ) 71 | if err then 72 | self:track( 73 | "send_req-err:"..tostring(err)..','..tostring(errmes) 74 | ..",to:"..tostring(args.ip)..":"..tostring(args.port)..tostring(args.url) 75 | ) 76 | return nil, err, errmes 77 | end 78 | local rstbody, err, errmes = h:read_body( 1024*1024 ) 79 | if err then 80 | return nil, err, errmes 81 | end 82 | 83 | local rst, jbody = pcall(json.decode, rstbody) 84 | if not rst then 85 | return nil, errors.InvalidMessage, "body is not valid json" 86 | end 87 | 88 | return jbody 89 | end 90 | 91 | function _M:api_recv() 92 | 93 | local uri = ngx.var.uri 94 | uri = uri:sub( #self.api_uri + 2 ) 95 | 96 | local elts = strutil.split( uri, '/' ) 97 | 98 | local cluster_id, ident, cmd = elts[1], elts[2], elts[3] 99 | local uri_args = { 100 | cluster_id = cluster_id, 101 | ident = ident, 102 | cmd = cmd, 103 | } 104 | local query_args = ngx.req.get_uri_args() 105 | query_args.ver = tonumber( query_args.ver ) 106 | 107 | ngx.req.read_body() 108 | local body = ngx.req.get_body_data() 109 | local req = {} 110 | if body ~= "" and body ~= nil then 111 | req = json.decode( body ) 112 | if req == nil then 113 | self:track( "api_recv-err:BodyIsNotJson" ) 114 | return nil, errors.InvalidMessage, "body is not valid json" 115 | end 116 | end 117 | 118 | req = tableutil.merge(req, query_args, uri_args) 119 | return req, nil, nil 120 | end 121 | function _M:api_resp(rst) 122 | 123 | self:set_resp_log(rst) 124 | 125 | local code 126 | if type(rst) == 'table' and rst.err then 127 | code = err_to_status[rst.err] or err_to_status["."] 128 | else 129 | code = _status.OK 130 | end 131 | 132 | rst = json.encode( rst ) 133 | ngx.status = code 134 | ngx.print( rst ) 135 | ngx.eof() 136 | ngx.exit( ngx.HTTP_OK ) 137 | end 138 | 139 | function _M:set_resp_log(rst) 140 | 141 | local str = tableutil.str 142 | 143 | if type(rst) == 'table' then 144 | 145 | local err = rst.err 146 | 147 | if err ~= nil then 148 | if err.Code == nil then 149 | self:logerr( "err without Code: ", err ) 150 | return 151 | end 152 | 153 | self:track('err:' .. str(err.Code) .. ',' .. str(err.Message)) 154 | else 155 | self:track('rst:' .. str(rst)) 156 | end 157 | end 158 | end 159 | 160 | function _M:get_addrs(member_id, member) 161 | return nil, "NotImplemented" 162 | end 163 | 164 | return _M 165 | -------------------------------------------------------------------------------- /lib/acid/impl/userdata.lua: -------------------------------------------------------------------------------- 1 | local _M = {} 2 | local _meta = { __index=_M } 3 | 4 | function _M.new(opt) 5 | local e = {} 6 | setmetatable( e, _meta ) 7 | return e 8 | end 9 | 10 | function _M:restore(paxos, member) end 11 | function _M:destory(paxos, _) end 12 | function _M:is_member_valid(paxos, member, opts) return true end 13 | function _M:is_needed_migrate(paxos, member) return false end 14 | function _M:report_cluster(paxos, down_members) end 15 | 16 | return _M 17 | -------------------------------------------------------------------------------- /lib/acid/impl_ngx.lua: -------------------------------------------------------------------------------- 1 | local _M = {} 2 | local _meta = { __index=_M } 3 | 4 | local tableutil = require( "acid.tableutil" ) 5 | local transport = require("acid.impl.transport_ngx_http") 6 | local storage = require("acid.impl.storage_ngx_fs") 7 | local locking = require("acid.impl.locking_ngx") 8 | local time = require("acid.impl.time_ngx") 9 | local logging = require("acid.impl.logging_ngx") 10 | local userdata = require("acid.impl.userdata") 11 | local member = require("acid.impl.member") 12 | 13 | tableutil.merge( _M, transport, storage, locking, time, logging, userdata, member ) 14 | 15 | function _M.new(opt) 16 | local e = {} 17 | tableutil.merge( e, 18 | transport.new(opt), 19 | storage.new(opt), 20 | locking.new(opt), 21 | time.new(opt), 22 | logging.new(opt), 23 | userdata.new(opt), 24 | member.new(opt) 25 | ) 26 | setmetatable( e, _meta ) 27 | return e 28 | end 29 | 30 | return _M 31 | -------------------------------------------------------------------------------- /lib/acid/logging.lua: -------------------------------------------------------------------------------- 1 | local _M = { _VERSION = "0.1" } 2 | 3 | local tableutil = require( "acid.tableutil" ) 4 | local strutil = require("acid.strutil") 5 | 6 | function _M.get_info( offset ) 7 | offset = offset or 0 8 | local thisfile = debug.getinfo(1).short_src 9 | local info 10 | for i = 2, 10 do 11 | info = debug.getinfo(i) 12 | 13 | if info.short_src ~= thisfile then 14 | if offset == 0 then 15 | break 16 | end 17 | offset = offset - 1 18 | end 19 | end 20 | return info 21 | end 22 | 23 | function _M.get_pos( offset ) 24 | local info = _M.get_info( offset ) 25 | local src = info.short_src 26 | src = strutil.split(src, "/") 27 | src = src[#src] 28 | 29 | local pos = '' 30 | pos = src .. ':' .. (info.name or '-') .. '():' .. info.currentline 31 | return pos 32 | end 33 | 34 | function _M.tostr(...) 35 | local args = {...} 36 | local n = select( "#", ... ) 37 | local t = {} 38 | for i = 1, n do 39 | table.insert( t, tableutil.str(args[i]) ) 40 | end 41 | return table.concat( t, " " ) 42 | end 43 | 44 | function _M.output(s) 45 | print(s) 46 | end 47 | 48 | function _M.dd(...) 49 | _M.output( _M.make_logline(0, ...) ) 50 | end 51 | 52 | function _M.make_logline(offset, ...) 53 | return _M.get_pos(offset) .. " " .. _M.tostr( ... ) 54 | end 55 | 56 | return _M 57 | -------------------------------------------------------------------------------- /lib/acid/paxos.lua: -------------------------------------------------------------------------------- 1 | local _M = { _VERSION = require("acid.paxos._ver") } 2 | 3 | local tableutil = require( "acid.tableutil" ) 4 | local base = require( "acid.paxos.base" ) 5 | local acceptor = require( "acid.paxos.acceptor" ) 6 | local proposer = require( "acid.paxos.proposer" ) 7 | 8 | local errors = base.errors 9 | 10 | _M.acceptor = acceptor 11 | _M.proposer = proposer 12 | _M.errors = errors 13 | 14 | local _meth = { 15 | _VERSION = _M._VERSION, 16 | acceptor = acceptor, 17 | proposer = proposer, 18 | errors = errors, 19 | field_filter = { 20 | leader = function (c) 21 | if c.val == nil or c.val.__lease < 0 then 22 | c.val = nil 23 | end 24 | return c 25 | end, 26 | }, 27 | } 28 | local _mt = { __index=_meth } 29 | 30 | function _M.new(member_id, impl, ver) 31 | 32 | local member_id, err, errmes = _M.extract_memberid( member_id ) 33 | if err then 34 | return nil, err, errmes 35 | end 36 | 37 | local p = { 38 | member_id = member_id, 39 | impl = impl, 40 | 41 | ver = ver 42 | } 43 | setmetatable(p, _mt) 44 | return p 45 | end 46 | function _M.extract_memberid(req) 47 | local member_id = tableutil.sub(req or {}, {"cluster_id", "ident"}) 48 | if member_id.cluster_id and member_id.ident then 49 | return member_id, nil 50 | else 51 | return nil, errors.InvalidArgument, "cluster_id or ident not found" 52 | end 53 | end 54 | 55 | function _meth:set(field_key, field_val) 56 | 57 | if type(field_key) ~= 'string' then 58 | return nil, errors.InvalidArgument, 'field_key must be string for set' 59 | end 60 | 61 | local c, err, errmes = self:read() 62 | if err then 63 | return nil, err, errmes 64 | end 65 | 66 | if tableutil.eq( c.val[field_key], field_val ) then 67 | 68 | local p, err, errmes = self:new_proposer() 69 | if err then 70 | return nil, err, errmes 71 | end 72 | 73 | local c, err, errmes = p:commit_specific(c) 74 | if err then 75 | return nil, err, errmes 76 | end 77 | return { ver=c.ver, key=field_key, val=field_val }, nil 78 | end 79 | 80 | c.val[field_key] = field_val 81 | 82 | local c, err, errmes = self:write(c.val) 83 | if err then 84 | return nil, err, errmes 85 | end 86 | 87 | return { ver=c.ver, key=field_key, val=c.val[field_key] } 88 | end 89 | function _meth:get(field_key) 90 | 91 | if type(field_key) ~= 'string' then 92 | return nil, errors.InvalidArgument, 'field_key must be string or nil for get' 93 | end 94 | 95 | local c, err, errmes = self:quorum_read() 96 | if err then 97 | return nil, err, errmes 98 | end 99 | 100 | return self:_make_get_rst(field_key, c) 101 | end 102 | function _meth:quorum_read() 103 | 104 | local p, err, errmes = self:new_proposer() 105 | if err then 106 | return nil, err, errmes 107 | end 108 | 109 | local c, err, errmes = p:quorum_read() 110 | if err then 111 | return nil, err, errmes 112 | end 113 | return { ver=c.ver, val=c.val }, nil 114 | end 115 | function _meth:sync() 116 | 117 | local p, err, errmes = self:new_proposer() 118 | if err then 119 | return nil, err, errmes 120 | end 121 | 122 | local c, err, errmes = p:quorum_read() 123 | if err then 124 | return nil, err, errmes 125 | end 126 | 127 | local c, err, errmes = p:commit_specific(c) 128 | if err then 129 | return nil, err, errmes 130 | end 131 | 132 | -- newer version seen 133 | if self.ver ~= nil and self.ver ~= c.ver then 134 | return nil, errors.VerNotExist, 'newer version:' .. tostring(c.ver) 135 | end 136 | 137 | return c, nil, nil 138 | end 139 | 140 | function _meth:write(val) 141 | local p, err, errmes = self:new_proposer() 142 | if err then 143 | return nil, err, errmes 144 | end 145 | 146 | local c, err, errmes = p:write(val) 147 | if err then 148 | return nil, err, errmes 149 | end 150 | 151 | if self.ver == nil or c.ver > self.ver then 152 | self.ver = c.ver 153 | end 154 | return c, nil, nil 155 | end 156 | function _meth:read() 157 | local p, err, errmes = self:new_proposer() 158 | if err then 159 | return nil, err, errmes 160 | end 161 | 162 | return p:read() 163 | end 164 | 165 | function _meth:send_req(ident, req) 166 | local p, err, errmes = self:new_proposer() 167 | if err then 168 | return nil, err, errmes 169 | end 170 | 171 | return self.impl:send_req(p, ident, req) 172 | end 173 | 174 | function _meth:local_get_mem(ident) 175 | if ident == nil then 176 | ident = self.member_id.ident 177 | end 178 | local _members, err, errmes = self:local_get_members() 179 | if err then 180 | return nil, err, errmes 181 | end 182 | return { ver=_members.ver, val=_members.val[ident] }, nil, nil 183 | end 184 | function _meth:local_get_members() 185 | local _view, err, errmes = self:local_get( 'view' ) 186 | if err then 187 | return nil, err, errmes 188 | end 189 | 190 | local members = tableutil.union( _view.val ) 191 | return {ver=_view.ver, val=members}, nil, nil 192 | end 193 | function _meth:local_get(field_key) 194 | 195 | if type(field_key) ~= 'string' then 196 | return nil, errors.InvalidArgument, 'field_key must be string or nil for get' 197 | end 198 | 199 | local c, err, errmes = self:read() 200 | if err then 201 | return nil, err, errmes 202 | end 203 | 204 | return self:_make_get_rst(field_key, c) 205 | end 206 | function _meth:_make_get_rst(key, c) 207 | -- if not found 208 | local val = c.val or {} 209 | 210 | c = { ver=c.ver, key=key, val=val[ key ] } 211 | 212 | local flt = self.field_filter[ key ] 213 | if flt ~= nil then 214 | c = flt(c) 215 | end 216 | 217 | return c, nil, nil 218 | end 219 | 220 | function _meth:logerr(...) 221 | self.impl:_log(1, ...) 222 | end 223 | 224 | function _meth:new_proposer() 225 | return proposer.new(self:_paxos_args(), self.impl) 226 | end 227 | function _meth:new_acceptor() 228 | return acceptor.new(self:_paxos_args(), self.impl) 229 | end 230 | function _meth:_paxos_args() 231 | local mid = tableutil.dup( self.member_id ) 232 | mid.ver = self.ver 233 | return mid 234 | end 235 | 236 | return _M 237 | -------------------------------------------------------------------------------- /lib/acid/paxos/_sto_data_struct.lua: -------------------------------------------------------------------------------- 1 | local member_id = { cluster_id="123.dx.GZ", ident="" } 2 | local committed = { 3 | ver = 1, 4 | val = { 5 | view = { 6 | { 7 | ['']={index=0, ip="127.0.0.1"}, 8 | ['']={index=1, ip="127.0.0.1"}, 9 | }, 10 | }, 11 | leader = { ident="id", __lease=10 }, 12 | action = { 13 | { name="bla", args={} }, 14 | { name="foo", args={} }, 15 | }, 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /lib/acid/paxos/_ver.lua: -------------------------------------------------------------------------------- 1 | return "0.1" 2 | -------------------------------------------------------------------------------- /lib/acid/paxos/acceptor.lua: -------------------------------------------------------------------------------- 1 | local _M = { _VERSION = require("acid.paxos._ver") } 2 | 3 | local tableutil = require( "acid.tableutil" ) 4 | local base = require( "acid.paxos.base" ) 5 | local round = require( "acid.paxos.round" ) 6 | 7 | local errors = base.errors 8 | 9 | local _meth = { _VERSION = _M._VERSION } 10 | local _mt = { __index = _meth } 11 | tableutil.merge( _meth, base ) 12 | 13 | local _api = { 14 | phase1 = true, 15 | phase2 = true, 16 | phase3 = true, 17 | } 18 | 19 | local function is_next_ver(self) 20 | 21 | if self.mes.ver <= self.ver then 22 | self.err = { 23 | Code = errors.AlreadyCommitted, 24 | Message = self.record.committed, 25 | } 26 | return false 27 | 28 | elseif self.mes.ver > self.ver + 1 then 29 | self.err = { 30 | Code = errors.VerNotExist, 31 | Message = self.ver, 32 | } 33 | return false 34 | end 35 | return true 36 | end 37 | local function committable(self) 38 | local c = self.record.committed 39 | 40 | if self.mes.ver < c.ver or self.mes.ver < 1 then 41 | self.err = { 42 | Code = errors.AlreadyCommitted, 43 | Message = c, 44 | } 45 | return false 46 | end 47 | return true 48 | end 49 | 50 | local function is_committed_valid( self ) 51 | 52 | -- TODO move to upper level 53 | -- local r_cluster_id = self.record.committed.val.ec_meta.ec_name 54 | -- if r_cluster_id ~= self.cluster_id then 55 | -- self.err = { 56 | -- Code = errors.InvalidCommitted, 57 | -- Message = {r_cluster_id=r_cluster_id, cluster_id=self.cluster_id} 58 | -- } 59 | -- return false 60 | -- end 61 | return true 62 | end 63 | 64 | function _M.new(args, impl) 65 | local acc = { 66 | cluster_id = args.cluster_id, 67 | ident = args.ident, 68 | ver = args.ver, 69 | 70 | mes = nil, 71 | 72 | view = nil, 73 | record = nil, 74 | err = nil, 75 | 76 | impl = impl, 77 | } 78 | setmetatable( acc, _mt ) 79 | 80 | if acc.cluster_id == nil 81 | or acc.ident == nil 82 | or acc.impl.store == nil 83 | or acc.impl.load == nil 84 | or acc.impl.lock == nil 85 | or acc.impl.unlock == nil 86 | then 87 | return nil, errors.InvalidArgument 88 | end 89 | 90 | return acc, nil 91 | end 92 | function _M.is_cmd(cmd) 93 | return _api[ cmd ] 94 | end 95 | 96 | function _meth:destory(ver) 97 | local _l, err = self.impl:lock(self) 98 | if err then 99 | return nil, errors.LockTimeout, nil 100 | end 101 | 102 | local _, err, errmes = self:load_rec() 103 | if err then 104 | return nil, err, errmes 105 | end 106 | 107 | local _, err, errmes = self:init_view() 108 | if err then 109 | return nil, err, errmes 110 | end 111 | 112 | local c = self.record.committed 113 | 114 | local rst, err, errmes 115 | if ver == c.ver then 116 | 117 | self.record = nil 118 | rst, err, errmes = self.impl:store(self) 119 | if err then 120 | rst, err, errmes = nil, errors.StorageError, nil 121 | else 122 | rst, err, errmes = nil, nil, nil 123 | end 124 | else 125 | rst, err, errmes = nil, errors.VerNotExist, c.ver 126 | end 127 | 128 | self.impl:unlock(_l) 129 | 130 | return rst, err, errmes 131 | end 132 | 133 | function _meth:process(mes) 134 | 135 | mes = mes or {} 136 | self.mes = mes 137 | 138 | if mes.cluster_id == nil then 139 | return nil, errors.InvalidMessage, "cluster_id is nil" 140 | end 141 | 142 | if not _api[ mes.cmd ] then 143 | return nil, errors.InvalidCommand, nil 144 | end 145 | 146 | if mes.cluster_id ~= self.cluster_id then 147 | return nil, errors.InvalidCluster, self.cluster_id 148 | end 149 | 150 | -- TODO pcall 151 | local _l, err = self.impl:lock(self) 152 | if err then 153 | self.impl:logerr("acceptor process lock timeout: ", self.cluster_id) 154 | return nil, errors.LockTimeout, nil 155 | end 156 | 157 | local rst, err, errmes = self[ self.mes.cmd ]( self ) 158 | 159 | self.impl:unlock(_l) 160 | 161 | return rst, err, errmes 162 | end 163 | function _meth:store_or_err() 164 | 165 | if not is_committed_valid( self ) then 166 | return nil, self.err.Code, self.err.Message 167 | end 168 | 169 | self:lease_to_expire() 170 | local _, err, errmes = self.impl:store(self) 171 | if err then 172 | return nil, errors.StorageError, nil 173 | else 174 | return nil 175 | end 176 | end 177 | function _meth:phase1() 178 | -- aka prepare 179 | 180 | local _, err, errmes = self:load_rec() 181 | if err then 182 | return nil, err, errmes 183 | end 184 | 185 | local _, err, errmes = self:init_view() 186 | if err then 187 | return nil, err, errmes 188 | end 189 | 190 | if not is_committed_valid( self ) then 191 | return nil, self.err.Code, self.err.Message 192 | end 193 | 194 | if not is_next_ver( self ) then 195 | return nil, self.err.Code, self.err.Message 196 | end 197 | 198 | local r = self.record.paxos_round 199 | 200 | if round.cmp( self.mes.rnd, r.rnd ) >= 0 then 201 | r.rnd = self.mes.rnd 202 | local _, err, errmes = self:store_or_err() 203 | if err then 204 | return nil, err, errmes 205 | end 206 | end 207 | 208 | local rst = { 209 | rnd=r.rnd, 210 | } 211 | if r.val then 212 | rst.val = r.val 213 | rst.vrnd = r.vrnd 214 | end 215 | 216 | return rst, nil, nil 217 | end 218 | function _meth:phase2() 219 | -- aka accept 220 | 221 | local _, err, errmes = self:load_rec() 222 | if err then 223 | return nil, err, errmes 224 | end 225 | 226 | local _, err, errmes = self:init_view() 227 | if err then 228 | return nil, err, errmes 229 | end 230 | 231 | if not is_committed_valid( self ) then 232 | return nil, self.err.Code, self.err.Message 233 | end 234 | 235 | if not is_next_ver( self ) then 236 | return nil, self.err.Code, self.err.Message 237 | end 238 | 239 | local r = self.record.paxos_round 240 | 241 | if round.cmp( self.mes.rnd, r.rnd ) == 0 then 242 | r.val = self.mes.val 243 | r.vrnd = self.mes.rnd 244 | 245 | return self:store_or_err() 246 | else 247 | return nil, errors.OldRound, nil 248 | end 249 | end 250 | function _meth:phase3() 251 | -- aka commit 252 | 253 | self:load_rec({ignore_err=true}) 254 | 255 | if self.record.committed.ver > 0 256 | and not is_committed_valid(self) then 257 | if self.mes.ver < 1 then 258 | return nil, self.err.Code, self.err.Message 259 | else 260 | -- initial record, make it can store the correct committed 261 | self:init_rec() 262 | end 263 | end 264 | 265 | if not committable( self ) then 266 | return nil, self.err.Code, self.err.Message 267 | end 268 | 269 | -- val with higher version is allowed to commit because some proposer has 270 | -- confirmed that it has been accepted by a quorum. 271 | 272 | local rec = self.record 273 | 274 | if self.mes.ver > rec.committed.ver then 275 | rec.paxos_round = { 276 | rnd = round.zero(), 277 | vrnd = round.zero(), 278 | } 279 | end 280 | 281 | -- for bug tracking 282 | if rec.committed.__tag ~= nil 283 | and self.mes.__tag ~= nil 284 | and rec.committed.__tag ~= self.mes.__tag 285 | and rec.committed.ver == self.mes.ver then 286 | 287 | local cval = tableutil.dup( rec.committed.val, true ) 288 | local mval = tableutil.dup( self.mes.val, true ) 289 | local cleader = cval.leader or {} 290 | local mleader = mval.leader or {} 291 | cleader.__lease = nil 292 | mleader.__lease = nil 293 | 294 | if not tableutil.eq(cval, mval) then 295 | local err = errors.Conflict 296 | local errmes = { 297 | ver=self.mes.ver, 298 | mes_tag=self.mes.__tag, 299 | committed_tag=rec.committed.__tag, 300 | mes_val=self.mes.val, 301 | committed_val=rec.committed.val, 302 | mes_val=self.mes.val, 303 | committed_val=rec.committed.val 304 | } 305 | 306 | self.impl:logerr( 'conflict: ', err, errmes ) 307 | 308 | return nil, err, errmes 309 | end 310 | end 311 | 312 | rec.committed = { 313 | ver = self.mes.ver, 314 | val = self.mes.val, 315 | __tag = self.mes.__tag, 316 | } 317 | 318 | return self:store_or_err() 319 | end 320 | return _M 321 | -------------------------------------------------------------------------------- /lib/acid/paxos/base.lua: -------------------------------------------------------------------------------- 1 | local _M = { _VERSION = require("acid.paxos._ver") } 2 | 3 | local tableutil = require( "acid.tableutil" ) 4 | local round = require( "acid.paxos.round" ) 5 | 6 | _M.errors = { 7 | InvalidArgument = 'InvalidArgument', 8 | InvalidMessage = 'InvalidMessage', 9 | InvalidCommand = 'InvalidCommand', 10 | InvalidCluster = 'InvalidCluster', 11 | InvalidPhase = 'InvalidPhase', 12 | VerNotExist = 'VerNotExist', 13 | AlreadyCommitted = 'AlreadyCommitted', 14 | OldRound = 'OldRound', 15 | NoView = 'NoView', 16 | QuorumFailure = 'QuorumFailure', 17 | LockTimeout = 'LockTimeout', 18 | StorageError = 'StorageError', 19 | NotAccepted = 'NotAccepted', 20 | DuringChange = 'DuringChange', 21 | NoChange = 'NoChange', 22 | Conflict = 'Conflict', 23 | InvalidCommitted = 'InvalidCommitted', 24 | } 25 | local errors = _M.errors 26 | 27 | function _M.init_rec( self, r ) 28 | r = r or {} 29 | r.committed = r.committed or { 30 | ver = 0, 31 | val = nil, 32 | } 33 | 34 | r.paxos_round = r.paxos_round or { 35 | rnd = round.zero(), 36 | val = nil, 37 | vrnd = round.zero(), 38 | } 39 | 40 | self.record = r 41 | 42 | self:expire_to_lease() 43 | 44 | return nil, nil, nil 45 | end 46 | 47 | function _M.load_rec( self, opt ) 48 | 49 | opt = opt or {} 50 | local ignore_err = opt.ignore_err == true 51 | 52 | local r, err, errmes = self.impl:load(self) 53 | 54 | if err and not ignore_err then 55 | return nil, err, errmes 56 | end 57 | 58 | return self:init_rec( r ) 59 | end 60 | 61 | function _M.init_view( self ) 62 | 63 | for _ = 0, 0 do 64 | 65 | local c = self.record.committed 66 | if c == nil then 67 | break 68 | end 69 | 70 | local ver, val = c.ver, c.val 71 | if ver == nil or val == nil then 72 | break 73 | end 74 | 75 | local v = val.view 76 | if v == nil or v[ 1 ] == nil then 77 | break 78 | end 79 | 80 | if self.ver ~= nil and self.ver ~= ver then 81 | return nil, errors.VerNotExist, ver 82 | end 83 | 84 | self.ver = ver 85 | self.view = v 86 | return nil 87 | end 88 | 89 | return nil, errors.NoView, nil 90 | end 91 | 92 | function _M:lease_to_expire() 93 | 94 | if self.record == nil then 95 | return nil, nil, nil 96 | end 97 | 98 | local val = self.record.committed.val 99 | if type(val) ~= 'table' then 100 | return nil, nil, nil 101 | end 102 | 103 | local now = self.impl:time() 104 | 105 | for k, v in pairs(val or {}) do 106 | if type( v ) == 'table' then 107 | if v.__lease ~= nil then 108 | v.__expire = now + v.__lease 109 | v.__lease = nil 110 | end 111 | end 112 | end 113 | return nil, nil, nil 114 | end 115 | 116 | function _M:expire_to_lease() 117 | 118 | if self.record == nil then 119 | return nil, nil, nil 120 | end 121 | 122 | local val = self.record.committed.val 123 | if type(val) ~= 'table' then 124 | return nil, nil, nil 125 | end 126 | 127 | local now = self.impl:time() 128 | 129 | for k, v in pairs(val or {}) do 130 | if type( v ) == 'table' then 131 | if v.__expire ~= nil then 132 | v.__lease = v.__expire - now 133 | v.__expire = nil 134 | end 135 | end 136 | end 137 | end 138 | 139 | return _M 140 | -------------------------------------------------------------------------------- /lib/acid/paxos/proposer.lua: -------------------------------------------------------------------------------- 1 | local _M = { _VERSION = require("acid.paxos._ver") } 2 | 3 | local tableutil = require( "acid.tableutil" ) 4 | local round = require( "acid.paxos.round" ) 5 | local base = require( "acid.paxos.base" ) 6 | 7 | local errors = base.errors 8 | 9 | local _meth = { _VERSION = _M._VERSION } 10 | local _mt = { __index = _meth } 11 | tableutil.merge( _meth, base ) 12 | 13 | function _M.new( args, impl ) 14 | 15 | local proposer = { 16 | cluster_id = args.cluster_id, 17 | ident = args.ident .. "", 18 | 19 | ver = args.ver, 20 | view = nil, 21 | acceptors = nil, 22 | 23 | -- stat becomes valid after self:decide() 24 | stat = nil, 25 | -- -- stat cummulated during paxos 26 | -- latest_committed = nil, 27 | -- latest_rnd = nil, 28 | -- stale_vers = nil, 29 | -- accepted_val = nil, 30 | 31 | rnd = round.new( { 0, args.ident } ), 32 | impl = impl, 33 | } 34 | setmetatable( proposer, _mt ) 35 | 36 | if impl.new_rnd then 37 | proposer.rnd = round.new( { impl.new_rnd( proposer ), proposer.ident } ) 38 | end 39 | 40 | -- load currently committed view for this proposer round 41 | 42 | local _, err, errmes = proposer:load_rec() 43 | if err then 44 | return nil, err, errmes 45 | end 46 | 47 | local _rnd = proposer.record.paxos_round.rnd 48 | if round.cmp( _rnd, proposer.rnd ) > 0 then 49 | proposer.rnd = round.new( { _rnd[1], proposer.ident } ) 50 | end 51 | 52 | local _, err, errmes = proposer:init_view() 53 | if err then 54 | return nil, err, errmes 55 | end 56 | 57 | -- TODO move cluster_id check to upper level 58 | -- local r_cluster_id = proposer.record.committed.val.ec_meta.ec_name 59 | -- if r_cluster_id ~= proposer.cluster_id then 60 | -- return nil, errors.InvalidCommitted, r_cluster_id 61 | -- end 62 | 63 | proposer.acceptors = tableutil.union( proposer.view, true ) 64 | 65 | return proposer 66 | end 67 | 68 | -- high level api 69 | function _meth:write( myval ) 70 | -- accepted by quorum and commit to quorum. 71 | -- This might fail in two ways: 72 | -- 1. not enough member to form a quorum. 73 | -- 2. there is currently another value accepted. 74 | 75 | local val, err, errmes = self:decide( myval ) 76 | if err then 77 | return nil, err, errmes 78 | end 79 | 80 | local c, err, errmes = self:commit() 81 | if err then 82 | return nil, err, errmes 83 | end 84 | 85 | if c.val == myval then 86 | return { ver=c.ver, val=c.val }, nil 87 | else 88 | -- other value is committed 89 | return nil, errors.VerNotExist, 'other value is committed:' .. tableutil.repr(c.val) 90 | end 91 | 92 | end 93 | function _meth:read() 94 | local c = self.record.committed 95 | return { ver=c.ver, val=c.val } 96 | end 97 | function _meth:remote_read() 98 | return self:_remote_read(false) 99 | end 100 | function _meth:quorum_read() 101 | return self:_remote_read(true) 102 | end 103 | function _meth:_remote_read(need_quorum) 104 | 105 | -- to commit with empty data, response data contains committed data stored 106 | local _resps = self:phase3({ ver=0 }) 107 | 108 | local resps = {} 109 | local err = {} 110 | for id, resp in pairs(_resps) do 111 | if resp.err.Code == errors.AlreadyCommitted then 112 | resps[ id ] = resp 113 | else 114 | err[ id ] = (resps.err or {}).Code 115 | end 116 | end 117 | 118 | if need_quorum and not self:is_quorum( resps ) then 119 | return nil, errors.QuorumFailure, 'remote read: ' .. tableutil.repr(err) 120 | end 121 | 122 | local latest = { ver=0, val=nil } 123 | for _, resp in pairs(resps) do 124 | local c = resp.err.Message 125 | if c.ver ~= nil and c.ver > latest.ver then 126 | latest = c 127 | end 128 | end 129 | 130 | if latest.ver == 0 then 131 | return nil, errors.VerNotExist, 'remote read ver is 0' 132 | end 133 | 134 | return latest 135 | end 136 | function _meth:decide( my_val ) 137 | 138 | if self.stat ~= nil then 139 | return nil, errors.InvalidPhase, 'phase 1 and 2 has already run' 140 | end 141 | 142 | if self.impl.new_rnd_incr then 143 | local _rnd = self.impl.new_rnd_incr(self, self.rnd[1]) 144 | self.rnd = round.new( {_rnd, self.ident } ) 145 | else 146 | self.rnd = round.incr( self.rnd ) 147 | end 148 | 149 | local resps = self:phase1() 150 | local accepted_resps, val, stat = self:choose_p1( resps, my_val ) 151 | 152 | self.stat = {} 153 | local st = self.stat 154 | 155 | st.phase1 = { 156 | ok = tableutil.keys( accepted_resps ), 157 | err = self:_choose_err( resps ), 158 | } 159 | 160 | tableutil.merge( st, stat ) 161 | 162 | if round.cmp( stat.latest_rnd, self.rnd ) > 0 then 163 | self.rnd = round.new({ stat.latest_rnd[1], self.rnd[2] }) 164 | return nil, errors.OldRound, 'latest round: ' .. tableutil.repr(stat.latest_rnd) 165 | end 166 | 167 | if not self:is_quorum( accepted_resps ) then 168 | return nil, errors.QuorumFailure, 'phase1: ' .. tableutil.repr(st.phase1.err) 169 | end 170 | 171 | resps = self:phase2( val ) 172 | self.p2 = resps 173 | accepted_resps = self:choose_p2( resps ) 174 | 175 | st.phase2 = { 176 | ok = tableutil.keys( accepted_resps ), 177 | err = self:_choose_err( resps ), 178 | } 179 | 180 | if not self:is_quorum( accepted_resps ) then 181 | return nil, errors.QuorumFailure, 'phase2: ' .. tableutil.repr(st.phase2.err) 182 | end 183 | 184 | self.stat.accepted_val = val 185 | 186 | return val, nil, nil 187 | end 188 | function _meth:commit() 189 | local c, err, errmes = self:make_commit_data() 190 | if err then 191 | return nil, err, errmes 192 | end 193 | return self:commit_specific(c) 194 | end 195 | function _meth:commit_specific(c) 196 | 197 | local resps = self:phase3(c) 198 | local positive = self:choose_p3( resps ) 199 | local ok = self:is_quorum( positive ) 200 | if ok then 201 | return { ver=c.ver, val=c.val }, nil 202 | else 203 | local err = self:_choose_err(resps) 204 | return nil, errors.QuorumFailure, 'commit specific: ' .. tableutil.repr(err) 205 | end 206 | end 207 | -- paxos level api 208 | function _meth:phase1() 209 | local mes = { 210 | cmd = 'phase1', 211 | cluster_id = self.cluster_id, 212 | ver = self.ver + 1, 213 | rnd = self.rnd 214 | } 215 | return self:send_mes_all( mes ) 216 | end 217 | function _meth:phase2(val) 218 | local mes = { 219 | cmd = 'phase2', 220 | cluster_id = self.cluster_id, 221 | ver = self.ver + 1, 222 | rnd = self.rnd, 223 | val = val, 224 | } 225 | return self:send_mes_all( mes ) 226 | end 227 | function _meth:phase3(c) 228 | local req = { 229 | cmd = 'phase3', 230 | cluster_id = self.cluster_id, 231 | 232 | ver = c.ver, 233 | val = c.val, 234 | 235 | __tag = c.__tag, 236 | } 237 | return self:send_mes_all( req ) 238 | end 239 | function _meth:choose_p1( resps, my_val ) 240 | 241 | local accepted = {} 242 | local latest_rnd = self.rnd 243 | local latest_v_resp = { vrnd = round.zero(), v = nil, } 244 | local latest_committed = { ver=0, val=nil } 245 | local stale_vers = {} 246 | 247 | for id, resp in pairs(resps) do 248 | 249 | if resp.err == nil then 250 | 251 | if round.cmp( resp.rnd, self.rnd ) == 0 then 252 | accepted[ id ] = resp 253 | end 254 | 255 | latest_rnd = round.max( { latest_rnd, resp.rnd } ) 256 | 257 | if round.cmp( resp.vrnd, latest_v_resp.vrnd ) > 0 then 258 | latest_v_resp = resp 259 | end 260 | 261 | elseif resp.err.Code == errors.AlreadyCommitted then 262 | 263 | local c = resp.err.Message 264 | if c.ver > latest_committed.ver then 265 | latest_committed = c 266 | end 267 | elseif resp.err.Code == errors.VerNotExist then 268 | local ver = resp.err.Message 269 | stale_vers[ id ] = ver 270 | end 271 | end 272 | 273 | if latest_committed.ver == 0 then 274 | latest_committed = nil 275 | end 276 | 277 | local stat = { 278 | latest_committed = latest_committed, 279 | latest_rnd = latest_rnd, 280 | stale_vers = stale_vers, 281 | } 282 | 283 | return accepted, ( latest_v_resp.val or my_val ), stat 284 | end 285 | function _meth:choose_p2( resps ) 286 | return self:_choose_no_err( resps ) 287 | end 288 | function _meth:choose_p3( resps ) 289 | return self:_choose_no_err( resps ) 290 | end 291 | function _meth:_choose_no_err(resps) 292 | local positive = {} 293 | for id, resp in pairs(resps) do 294 | if resp.err == nil then 295 | positive[ id ] = resp 296 | end 297 | end 298 | return positive 299 | end 300 | 301 | function _meth:_choose_err(resps) 302 | local err = {} 303 | for id, resp in pairs(resps) do 304 | if resp.err ~= nil then 305 | err[ id ] = (resp.err or {}).Code 306 | end 307 | end 308 | return err 309 | end 310 | 311 | function _meth:make_commit_data() 312 | if self.stat == nil then 313 | return nil, errors.InvalidPhase, 'phase 1 or 2 has not yet run' 314 | end 315 | if self.stat.accepted_val == nil then 316 | return nil, errors.NotAccepted 317 | end 318 | local c = { 319 | ver=self.ver+1, 320 | val=self.stat.accepted_val, 321 | __tag = table.concat({ 322 | self.cluster_id or '', 323 | self.ident or '', 324 | self.ver+1, 325 | table.concat(self.rnd, "-"), 326 | }, '/'), 327 | } 328 | return c 329 | end 330 | function _meth:is_quorum( accepted ) 331 | 332 | for _, group in ipairs(self.view) do 333 | 334 | local n = tableutil.nkeys( tableutil.intersection( { accepted, group } ) ) 335 | local total = tableutil.nkeys( group ) 336 | 337 | if total > 0 and n <= total / 2 then 338 | return false 339 | end 340 | 341 | end 342 | 343 | return true 344 | end 345 | function _meth:send_mes_all( mes ) 346 | local resps = {} 347 | for id, _ in pairs( self.acceptors ) do 348 | resps[ id ] = self.impl:send_req( self, id, mes ) 349 | end 350 | return resps 351 | end 352 | 353 | function _meth:sync_committed() 354 | -- commit if there is stale record: 355 | -- 1, version greater than current proposer found 356 | -- 2, version lower than current proposer found 357 | 358 | local c = self:get_committed_to_sync() 359 | if c == nil then 360 | return nil 361 | end 362 | 363 | assert( c.ver > 0 ) 364 | assert( c.ver >= self.ver ) 365 | assert( c.val ~= nil ) 366 | 367 | return self:commit_specific(c) 368 | end 369 | function _meth:get_committed_to_sync() 370 | local mine = self.record.committed 371 | local latest = self.stat.latest_committed or mine 372 | 373 | if latest.ver > mine.ver 374 | or tableutil.nkeys(self.stat.stale_vers) > 0 then 375 | return latest 376 | else 377 | return nil 378 | end 379 | end 380 | 381 | return _M 382 | -------------------------------------------------------------------------------- /lib/acid/paxos/round.lua: -------------------------------------------------------------------------------- 1 | local _M = { _VERSION = require("acid.paxos._ver") } 2 | 3 | local tableutil = require( "acid.tableutil" ) 4 | 5 | function _M.new( elts ) 6 | assert( elts[ 2 ] ~= nil and elts[ 3 ] == nil, 7 | "invalid nr of elts while creating new round" ) 8 | local rnd = tableutil.duplist( elts ) 9 | return rnd 10 | end 11 | 12 | function _M.zero() 13 | return _M.new({ 0, '' }) 14 | end 15 | 16 | function _M.max( rounds ) 17 | local max = _M.zero() 18 | for _, r in ipairs(rounds) do 19 | if _M.cmp( max, r ) < 0 then 20 | max = r 21 | end 22 | end 23 | return max 24 | end 25 | 26 | function _M.incr( rnd ) 27 | return _M.new({ rnd[1] + 1, rnd[2] }) 28 | end 29 | 30 | function _M.cmp( a, b ) 31 | a = a or {} 32 | b = b or {} 33 | for i = 1, 2 do 34 | local x = cmp( a[i], b[i] ) 35 | if x ~= 0 then 36 | return x 37 | end 38 | end 39 | return 0 40 | end 41 | 42 | function cmp(a, b) 43 | 44 | if a == nil then 45 | if b == nil then 46 | return 0 47 | else 48 | return -1 49 | end 50 | else 51 | if b == nil then 52 | return 1 53 | end 54 | end 55 | 56 | -- none of a or b is nil 57 | 58 | if a>b then 59 | return 1 60 | elseif a i then 221 | 222 | local sublines = _M._repr_lines(t[k], opt) 223 | sublines[ 1 ] = normkey(k, opt) ..'='.. sublines[ 1 ] 224 | extend(lst, sublines, opt) 225 | end 226 | end 227 | 228 | -- remove the last ',' 229 | lst[ #lst ] = lst[ #lst ]:sub( 1, -2 ) 230 | 231 | table.insert( lst, '}' ) 232 | return lst 233 | end 234 | 235 | function _M.iter(tbl) 236 | 237 | local ks = _M.keys(tbl) 238 | local i = 0 239 | 240 | table.sort( ks, function( a, b ) return tostring(a) 0 do 261 | 262 | local k, v = iters[#iters]() 263 | 264 | if k == nil then 265 | ks[#iters], iters[#iters] = nil, nil 266 | else 267 | ks[#iters] = k 268 | 269 | if type(v) == tabletype then 270 | table.insert(iters, _M.iter(v)) 271 | else 272 | return ks, v 273 | end 274 | end 275 | end 276 | end 277 | end 278 | 279 | function _M.has(tbl, value) 280 | 281 | if value == nil then 282 | return true 283 | end 284 | 285 | for _, v in pairs(tbl) do 286 | if v == value then 287 | return true 288 | end 289 | end 290 | 291 | return false 292 | end 293 | 294 | -- TODO test. or use has() 295 | function _M.in_table(value, tbl) 296 | for _, v in pairs(tbl) do 297 | if v == value then 298 | return true 299 | end 300 | end 301 | 302 | return false 303 | end 304 | 305 | function _M.remove(tbl, value) 306 | 307 | for k, v in pairs(tbl) do 308 | if v == value then 309 | -- int, shift 310 | if type(k) == 'number' and k % 1 == 0 then 311 | table.remove(tbl, k) 312 | else 313 | tbl[k] = nil 314 | end 315 | return v 316 | end 317 | end 318 | 319 | return nil 320 | end 321 | 322 | -- TODO test or remove this 323 | function _M.remove_value(value, tbl) 324 | local removed = false 325 | 326 | for i = #tbl, 1, -1 do 327 | if tbl[i] == value then 328 | table.remove(tbl, i) 329 | removed = true 330 | end 331 | end 332 | 333 | return removed 334 | end 335 | 336 | -- TODO test or remove this 337 | function _M.get_sub_table(tbl, keys) 338 | local sub = {} 339 | 340 | for _, k in ipairs(keys) do 341 | table.insert(sub, tbl[k]) 342 | end 343 | 344 | return sub 345 | end 346 | 347 | function _M.get_len(tbl) 348 | local len = 0 349 | for _, _ in pairs(tbl) do 350 | len = len + 1 351 | end 352 | 353 | return len 354 | end 355 | 356 | -- TODO test 357 | function _M.get_random_elements(tbl, n) 358 | local idx 359 | local rnd 360 | local tlen 361 | local elmts = {} 362 | 363 | if type(tbl) ~= 'table' then 364 | return tbl 365 | end 366 | 367 | tlen = #tbl 368 | if tlen == 0 then 369 | return {} 370 | end 371 | 372 | n = math.min(n or tlen, tlen) 373 | rnd = math.random(1, tlen) 374 | 375 | for i = 1, n, 1 do 376 | idx = (rnd+i) % tlen + 1 377 | table.insert(elmts, tbl[idx]) 378 | end 379 | 380 | return elmts 381 | end 382 | 383 | function _M.extends( tbl, tvals ) 384 | 385 | if type(tbl) ~= 'table' or tvals == nil then 386 | return tbl 387 | end 388 | 389 | -- Note: will be discarded after nil elements in tvals 390 | for i, v in ipairs( tvals ) do 391 | table.insert( tbl, v ) 392 | end 393 | 394 | return tbl 395 | end 396 | return _M 397 | -------------------------------------------------------------------------------- /lib/acid/unittest.lua: -------------------------------------------------------------------------------- 1 | local _M = { _VERSION='0.1' } 2 | 3 | local function tostr(x) 4 | return '[' .. tostring( x ) .. ']' 5 | end 6 | 7 | local function dd(...) 8 | local args = {...} 9 | local s = '' 10 | for _, mes in ipairs(args) do 11 | s = s .. tostring(mes) 12 | end 13 | _M.output( s ) 14 | end 15 | 16 | function _M.output(s) 17 | print( s ) 18 | end 19 | 20 | local function is_test_file( fn ) 21 | return fn:sub( 1, 5 ) == 'test_' and fn:sub( -4, -1 ) == '.lua' 22 | end 23 | 24 | local function scandir(directory) 25 | local t = {} 26 | for filename in io.popen('ls "'..directory..'"'):lines() do 27 | table.insert( t, filename ) 28 | end 29 | return t 30 | end 31 | 32 | local function keys(tbl) 33 | local n = 0 34 | local ks = {} 35 | for k, v in pairs( tbl ) do 36 | table.insert( ks, k ) 37 | n = n + 1 38 | end 39 | table.sort( ks, function(a, b) return tostring(a)={..}, ={..}, ... } 22 | local cluster_id = paxos.member_id.cluster_id 23 | local dead_mem = members[dead_ident] 24 | 25 | for i = 1, 4 do 26 | local ident = tostring(i) 27 | if members[ident] == nil then 28 | return { [ident]=ident } 29 | end 30 | end 31 | return nil, "NoFreeMember" 32 | end 33 | 34 | _M.cluster = acid_cluster.new(impl, { 35 | dead_wait = 3, 36 | admin_lease = 5, 37 | max_dead = 1, 38 | }) 39 | 40 | local function list_member_ids() 41 | local ms = {} 42 | for _, ident in ipairs(_M.members_on_this_node) do 43 | table.insert( ms, {cluster_id="x", ident=ident} ) 44 | end 45 | return ms 46 | end 47 | function _M.init_cluster_check(enabled) 48 | 49 | if not enabled then 50 | return 51 | end 52 | 53 | local check_interval = 1 54 | 55 | local timer_work 56 | 57 | timer_work = function (premature) 58 | 59 | if premature then 60 | -- worker is shutting down 61 | return 62 | end 63 | 64 | for _, mid in ipairs( list_member_ids() ) do 65 | local rst, err, errmes = _M.cluster:member_check(mid) 66 | if err then 67 | ngx.log( ngx.ERR, 'member_check: ', rst, ' ', err, ' ', tostring(errmes) ) 68 | end 69 | end 70 | 71 | local ok, err = ngx.timer.at( check_interval, timer_work ) 72 | end 73 | local ok, err = ngx.timer.at( check_interval, timer_work ) 74 | end 75 | 76 | return _M 77 | -------------------------------------------------------------------------------- /lib/simple.lua: -------------------------------------------------------------------------------- 1 | local nginx_cluster = require("nginx_cluster") 2 | 3 | local _M = {} 4 | 5 | local cc = nginx_cluster.new({ 6 | cluster_id = 'x', 7 | ident = '127.0.0.1:9081', 8 | path = "/tmp/paxos", 9 | 10 | standby = { 11 | '127.0.0.1:9081', 12 | '127.0.0.1:9082', 13 | '127.0.0.1:9083', 14 | '127.0.0.1:9084', 15 | '127.0.0.1:9085', 16 | '127.0.0.1:9086', 17 | }, 18 | }) 19 | 20 | return _M 21 | -------------------------------------------------------------------------------- /lib/test_empty.lua: -------------------------------------------------------------------------------- 1 | function test_foo(t) 2 | t:eq( 0, 0, '0 is 0' ) 3 | end 4 | 5 | -------------------------------------------------------------------------------- /lib/test_logging.lua: -------------------------------------------------------------------------------- 1 | local l = require( "acid.logging" ) 2 | 3 | function test_tostr(t) 4 | t:eq( "1 nil", l.tostr( 1, nil ) ) 5 | t:eq( "{1,2} {a=3,b={x=4}} nil", l.tostr( {1,2}, {a=3,b={x=4}}, nil ) ) 6 | end 7 | -------------------------------------------------------------------------------- /lib/test_paxos.lua: -------------------------------------------------------------------------------- 1 | 2 | local base = require( "acid.paxos.base" ) 3 | local paxos = require( "acid.paxos" ) 4 | local tableutil = require( "acid.tableutil" ) 5 | 6 | local errors = base.errors 7 | 8 | function test_new(t) 9 | local cases = { 10 | { 11 | mid=nil, 12 | impl=nil, 13 | rst=nil, 14 | err=errors.InvalidArgument, 15 | }, 16 | { 17 | mid={ cluster_id=nil, ident=nil }, 18 | impl=nil, 19 | rst=nil, 20 | err=errors.InvalidArgument, 21 | }, 22 | { 23 | mid={ cluster_id="x", ident=nil }, 24 | impl=nil, 25 | rst=nil, 26 | err=errors.InvalidArgument, 27 | }, 28 | { 29 | mid={ cluster_id=nil, ident="x" }, 30 | impl=nil, 31 | rst=nil, 32 | err=errors.InvalidArgument, 33 | }, 34 | { 35 | mid={ cluster_id="x", ident="x" }, 36 | impl=nil, 37 | rst=nil, 38 | err=nil, 39 | }, 40 | 41 | } 42 | for i, case in ipairs( cases ) do 43 | local mes = "" .. i .. (case.mes or "") 44 | local p, err, errmes = paxos.new(case.mid, case.impl) 45 | if case.err then 46 | t:eq( nil, p ) 47 | t:eq( case.err, err ) 48 | else 49 | t:eq( nil, err ) 50 | end 51 | end 52 | end 53 | 54 | local function make_implementation( opt ) 55 | 56 | local resps = tableutil.dup( opt.resps, true ) or {} 57 | local stores = tableutil.dup( opt.stores, true ) or {} 58 | local def_sto = tableutil.dup( opt.def_sto, true ) or {} 59 | 60 | function _sendmes( self, p, id, mes ) 61 | 62 | local r = resps[ id ] or {} 63 | 64 | if mes.cmd == 'phase1' then 65 | r = r.p1 66 | if r == nil then r = { rnd=mes.rnd } end 67 | elseif mes.cmd == 'phase2' then 68 | r = r.p2 69 | if r == nil then r = {} end 70 | elseif mes.cmd == 'phase3' then 71 | r = r.p3 72 | if r == nil then r = {} end 73 | else 74 | error( "invalid cmd" .. tableutil.repr( mes ) ) 75 | end 76 | 77 | if r == false then 78 | return nil 79 | end 80 | return r 81 | end 82 | 83 | local impl = { 84 | send_req = _sendmes, 85 | load = function( p ) 86 | local id = p.ident 87 | local c = stores[ id ] or def_sto 88 | return tableutil.dup( c, true ) 89 | end, 90 | store = function( self, p ) 91 | local id = p.ident 92 | stores[ id ] = tableutil.dup( p.record, true ) 93 | end, 94 | time= function( self ) return os.time() end, 95 | } 96 | return impl 97 | end 98 | 99 | function test_set(t) 100 | 101 | local def_sto = { 102 | committed = { 103 | ver=1, 104 | val = { 105 | view = { { a=1, b=1, c=1 } }, 106 | } 107 | } 108 | } 109 | cases = { 110 | { 111 | mes = 'set ok', 112 | key="akey", val="aval", ver=nil, 113 | rst = { ver=2, key="akey", val="aval" }, 114 | err = nil, 115 | p_ver=2, 116 | resps = { 117 | a={}, 118 | b={}, 119 | c={}, 120 | }, 121 | def_sto = def_sto, 122 | }, 123 | { 124 | mes = 'set with specific ver', 125 | key="akey", val="aval", ver=1, 126 | rst = { ver=2, key="akey", val="aval" }, 127 | err = nil, 128 | p_ver=2, 129 | resps = { 130 | a={}, 131 | b={}, 132 | c={}, 133 | }, 134 | def_sto = def_sto, 135 | }, 136 | { 137 | mes = 'set with unsatisfied ver', 138 | key="akey", val="aval", ver=3, 139 | rst = nil, 140 | err = 'VerNotExist', 141 | p_ver=3, 142 | resps = { 143 | a={}, 144 | b={}, 145 | c={}, 146 | }, 147 | def_sto = def_sto, 148 | }, 149 | { 150 | mes = 'set with quorum failure', 151 | key="akey", val="aval", ver=1, 152 | rst = nil, 153 | err = 'QuorumFailure', 154 | p_ver=1, 155 | resps = { 156 | a={p1=false}, 157 | b={p1=false}, 158 | c={}, 159 | }, 160 | def_sto = def_sto, 161 | }, 162 | } 163 | 164 | for i, case in ipairs( cases ) do 165 | 166 | mes = i .. ": " .. (case.mes or '') 167 | resps = case.resps 168 | 169 | local impl = make_implementation({ 170 | resps = case.resps, 171 | stores = case.stores, 172 | def_sto = case.def_sto 173 | }) 174 | 175 | local mid = { cluster_id="x", ident="a" } 176 | 177 | local p, err = paxos.new( mid, impl, case.ver ) 178 | t:eq( nil, err, mes ) 179 | 180 | local c, err, errmes = p:set( case.key, case.val ) 181 | t:eq( case.err, err, mes ) 182 | t:eqdict( case.rst, c, mes ) 183 | 184 | t:eq( case.p_ver, p.ver, mes ) 185 | end 186 | 187 | end 188 | function test_get(t) 189 | 190 | local def_sto = { 191 | committed = { 192 | ver=1, 193 | val = { 194 | foo = "bar", 195 | view = { { a=1, b=1, c=1 } }, 196 | } 197 | } 198 | } 199 | cases = { 200 | { 201 | mes = 'get failure 3', 202 | key="akey", ver=nil, 203 | rst = nil, 204 | err = errors.QuorumFailure, 205 | resps = { 206 | a={p3=false}, 207 | b={p3=false}, 208 | c={p3=false}, 209 | }, 210 | def_sto = def_sto, 211 | }, 212 | { 213 | mes = 'get failure 2', 214 | key="akey", ver=nil, 215 | rst = nil, 216 | err = errors.QuorumFailure, 217 | resps = { 218 | a={p3={err={Code=errors.AlreadyCommitted, Message=def_sto.committed}}}, 219 | b={p3=false}, 220 | c={p3=false}, 221 | }, 222 | def_sto = def_sto, 223 | }, 224 | { 225 | mes = 'get not found', 226 | key="akey", ver=nil, 227 | rst = {ver=1, key="akey", val=nil}, 228 | err = nil, 229 | resps = { 230 | a={p3={err={Code=errors.AlreadyCommitted, Message=def_sto.committed}}}, 231 | b={p3={err={Code=errors.AlreadyCommitted, Message=def_sto.committed}}}, 232 | c={p3=false}, 233 | }, 234 | def_sto = def_sto, 235 | }, 236 | { 237 | mes = 'get found', 238 | key="foo", ver=nil, 239 | rst = {ver=1, key="foo", val="bar"}, 240 | err = nil, 241 | resps = { 242 | a={p3={err={Code=errors.AlreadyCommitted, Message=def_sto.committed}}}, 243 | b={p3={err={Code=errors.AlreadyCommitted, Message=def_sto.committed}}}, 244 | c={p3=false}, 245 | }, 246 | def_sto = def_sto, 247 | }, 248 | { 249 | mes = 'get found latest', 250 | key="foo", ver=nil, 251 | rst = {ver=2, key="foo", val="bar2"}, 252 | err = nil, 253 | resps = { 254 | a={p3={err={Code=errors.AlreadyCommitted, Message=def_sto.committed}}}, 255 | b={p3={err={Code=errors.AlreadyCommitted, Message={ 256 | ver=2, 257 | val = { 258 | foo = "bar2", 259 | view = { { a=1, b=1, c=1 } }, 260 | } 261 | }}}}, 262 | c={p3=false}, 263 | }, 264 | def_sto = def_sto, 265 | }, 266 | { 267 | mes = 'set with unsatisfied ver', 268 | key="akey", ver=3, 269 | rst = nil, 270 | err = 'VerNotExist', 271 | resps = { 272 | a={}, 273 | b={}, 274 | c={}, 275 | }, 276 | def_sto = def_sto, 277 | }, 278 | } 279 | 280 | for i, case in ipairs( cases ) do 281 | 282 | mes = i .. ": " .. (case.mes or '') 283 | resps = case.resps 284 | 285 | local impl = make_implementation({ 286 | resps = case.resps, 287 | stores = case.stores, 288 | def_sto = case.def_sto 289 | }) 290 | 291 | local mid = { cluster_id="x", ident="a" } 292 | 293 | local p, err = paxos.new( mid, impl, case.ver ) 294 | t:eq( nil, err, mes ) 295 | 296 | local c, err, errmes = p:get( case.key ) 297 | t:eq( case.err, err, mes ) 298 | t:eqdict( case.rst, c, mes ) 299 | end 300 | 301 | end 302 | function test_sendmes(t) 303 | 304 | local def_sto = { 305 | committed = { 306 | ver=1, 307 | val = { 308 | foo = "bar", 309 | view = { { a=1, b=1, c=1 } }, 310 | } 311 | } 312 | } 313 | cases = { 314 | { 315 | mes = 'send get nothing back', 316 | key="akey", ver=nil, 317 | req = {cmd="phase3"}, 318 | to_id = 'a', 319 | rst = nil, 320 | err = nil, 321 | resps = { 322 | a={p3=false}, 323 | b={p3=false}, 324 | c={p3=false}, 325 | }, 326 | def_sto = def_sto, 327 | }, 328 | { 329 | mes = 'send get something', 330 | key="akey", ver=nil, 331 | req = {cmd="phase3"}, 332 | to_id = 'a', 333 | rst = {a=1}, 334 | err = nil, 335 | resps = { 336 | a={p3={a=1}}, 337 | b={p3={b=2}}, 338 | c={p3=false}, 339 | }, 340 | def_sto = def_sto, 341 | }, 342 | } 343 | 344 | for i, case in ipairs( cases ) do 345 | 346 | mes = i .. ": " .. (case.mes or '') 347 | resps = case.resps 348 | 349 | local impl = make_implementation({ 350 | resps = case.resps, 351 | stores = case.stores, 352 | def_sto = case.def_sto 353 | }) 354 | 355 | local mid = { cluster_id="x", ident="a" } 356 | 357 | local p, err = paxos.new( mid, impl, case.ver ) 358 | t:eq( nil, err, mes ) 359 | 360 | local c, err, errmes = p:send_req( case.to_id, case.req ) 361 | t:eq( case.err, err, mes ) 362 | t:eqdict( case.rst, c, mes ) 363 | end 364 | 365 | end 366 | -------------------------------------------------------------------------------- /lib/test_round.lua: -------------------------------------------------------------------------------- 1 | local r = require( "acid.paxos.round" ) 2 | 3 | function test_new(t) 4 | t:err( function () r.new( { 0 } ) end ) 5 | t:err( function () r.new( { 1 } ) end ) 6 | t:err( function () r.new( { a=1 } ) end ) 7 | t:err( function () r.new( { 1, 2, 3 } ) end ) 8 | 9 | local a = r.new( { 1, '' } ) 10 | t:eq( 1, a[ 1 ] ) 11 | t:eq( '', a[ 2 ] ) 12 | end 13 | 14 | function test_zero(t) 15 | local a = r.zero() 16 | t:eqlist( { 0, '' }, a, 'zero' ) 17 | end 18 | 19 | function test_max(t) 20 | 21 | t:eqlist( r.zero(), r.max( {} ), 'max of empty' ) 22 | 23 | t:eqlist( { 5, 'x' }, r.max( {{ 5, 'x' }} ), 'max of 1' ) 24 | 25 | local three = { { 0, 'x' }, { 2, 'a' }, { 1, 'b' } } 26 | local a = r.max( three ) 27 | t:eqlist( { 2, 'a' }, a, 'max of 3' ) 28 | 29 | a[ 1 ] = 10 30 | t:eq( 10, three[ 2 ][ 1 ], 'max uses reference' ) 31 | 32 | end 33 | 34 | function test_incr(t) 35 | local a = r.zero() 36 | a[ 2 ] = 'xxx' 37 | 38 | local b = r.incr( a ) 39 | 40 | t:eqlist( { 0, 'xxx' }, a ) 41 | t:eqlist( { 1, 'xxx' }, b ) 42 | end 43 | 44 | function test_cmp(t) 45 | t:eq( 0, r.cmp( nil, nil ) ) 46 | t:eq( 1, r.cmp( r.zero(), nil ) ) 47 | t:eq( -1, r.cmp( nil, r.zero() ) ) 48 | 49 | t:eq( 1, r.cmp( { 1, 'b' }, { 0 } ) ) 50 | t:eq( 1, r.cmp( { 1 }, { 0, 'a' } ) ) 51 | 52 | t:eq( 0, r.cmp( r.zero(), r.zero() ) ) 53 | t:eq( 1, r.cmp( r.incr(r.zero()), r.zero() ) ) 54 | t:eq( -1, r.cmp( { 1, 'b' }, { 2, 'a' } ) ) 55 | t:eq( 1, r.cmp( { 1, 'b' }, { 1, 'a' } ) ) 56 | t:eq( 1, r.cmp( { 2, 'a' }, { 1, 'b' } ) ) 57 | t:eq( -1, r.cmp( { 1, 'a' }, { 1, 'b' } ) ) 58 | end 59 | -------------------------------------------------------------------------------- /lib/test_strutil.lua: -------------------------------------------------------------------------------- 1 | local strutil = require("acid.strutil") 2 | 3 | 4 | function test_split(t) 5 | 6 | local str ='/v1/get/video.vic.sina.com.cn%2fmt788%2f9e%2f1a%2f81403.jpg/%7b%22xACL%22%3a%20%7b%22GRPS000000ANONYMOUSE%22%3a%20%5b%22read%22%5d, %20%22SINA00000000000SALES%22%3a%20%5b%22read%22, %20%22write%22, %20%22read_acp%22, %20%22write_acp%22%5d%7d, %20%22Info%22%3a%20null, %20%22Type%22%3a%20%22image%5c%2fjpeg%22, %20%22ver%22%3a%201042410872, %20%22Get-Location%22%3a%20%5b%7b%22CheckNumber%22%3a%201042410872, %20%22GroupID%22%3a%20341476, %20%22Partitions%22%3a%20%5b%7b%22IPs%22%3a%20%5b%2258.63.236.89%22, %20%2210.71.5.89%22%5d, %20%22PartitionID%22%3a%20%22185c3e5700014004975f90b11c13fc5e%22, %20%22IDC%22%3a%20%22.dx.GZ%22%7d, %20%7b%22IPs%22%3a%20%5b%2258.63.236.184%22, %20%2210.71.5.184%22%5d, %20%22PartitionID%22%3a%20%225a56155500014009aaa590b11c148e88%22, %20%22IDC%22%3a%20%22.dx.GZ%22%7d, %20%7b%22IPs%22%3a%20%5b%2260.28.228.36%22, %20%22172.16.228.36%22%5d, %20%22PartitionID%22%3a%20%22a41d4006000140029595d4ae52b17fe1%22, %20%22IDC%22%3a%20%22.wt.TJ%22%7d, %20%7b%22IPs%22%3a%20%5b%22111.161.78.59%22, %20%22172.16.48.59%22%5d, %20%22PartitionID%22%3a%20%22d7fee54d00014006b58090b11c145321%22, %20%22IDC%22%3a%20%22.wt.TJ%22%7d%5d%7d%5d, %20%22Info-Int%22%3a%200, %20%22ts%22%3a%201356544141, %20%22ACL%22%3a%20%7b%22SINA00000000000SALES%22%3a%20%5b%22read%22, %20%22write%22, %20%22read_acp%22, %20%22write_acp%22%5d%7d, %20%22ETag2%22%3a%20%2235b4ec0bfd826ea609054ccca4976e4fc77f3a8b%22, %20%22ETag%22%3a%20%22e14526f8858e2e0f898e72f141f108e4%22, %20%22Key%22%3a%20%22mt788%5c%2f9e%5c%2f1a%5c%2f81403.jpg%22, %20%22Owner%22%3a%20%22SINA00000000000SALES%22, %20%22Origo%22%3a%20%220000000000000000000090b11c09b4d9%22, %20%22GroupClassID%22%3a%206, %20%22File-Meta%22%3a%20%7b%22Content-Type%22%3a%20%22image%5c%2fjpeg%22%7d, %20%22Size%22%3a%2095964%7d?n=1&r=1&w=1&expire=60&ver_key=ts' 7 | 8 | t:eqdict( { '' }, strutil.split( '', '/' ), 'empty string' ) 9 | t:eqdict( { '', '' }, strutil.split( '/', '/' ), 'single pattern' ) 10 | t:eqdict( { '', '', '' }, strutil.split( '//', '/' ), 'dual pattern' ) 11 | t:eqdict( { 'a', '', '' }, strutil.split( 'a//', '/' ), '"a" and dual pattern' ) 12 | t:eqdict( { '', 'a', '' }, strutil.split( '/a/', '/' ), '/a/' ) 13 | t:eqdict( { '', '', 'a' }, strutil.split( '//a', '/' ), '//a' ) 14 | 15 | t:eqdict( { 'abcdefg', '', '' }, strutil.split( 'abcdefg//', '/' ), '"abcdefg" and dual pattern' ) 16 | t:eqdict( { '', 'abcdefg', '' }, strutil.split( '/abcdefg/', '/' ), '/abcdefg/' ) 17 | t:eqdict( { '', '', 'abcdefg' }, strutil.split( '//abcdefg', '/' ), '//abcdefg' ) 18 | 19 | t:eqdict( { 'abc', 'xyz', 'uvw' }, strutil.split( 'abc/xyz/uvw', '/' ), 'full' ) 20 | t:eqdict( { '', 'abc', '' }, strutil.split( '/abc/', '/' ), '/abc/' ) 21 | t:eqdict( { '', '', 'abc' }, strutil.split( '//abc', '/' ), '//abc' ) 22 | 23 | t:eq( str, table.concat( strutil.split( str, '/' ), '/' ) ) 24 | 25 | end 26 | 27 | function test_startswith(t) 28 | local s = strutil.startswith 29 | t:eq(true, s( '', '' ) ) 30 | t:eq(true, s( 'a', '' ) ) 31 | t:eq(false, s( 'a', 'b' ) ) 32 | t:eq(true, s( 'ab', 'a' ) ) 33 | t:eq(true, s( 'ab', 'ab' ) ) 34 | t:eq(false, s( 'ab', 'abc' ) ) 35 | end 36 | 37 | function test_endswith(t) 38 | local s = strutil.endswith 39 | t:eq(true, s( '', '' ) ) 40 | t:eq(true, s( 'a', '' ) ) 41 | t:eq(false, s( 'a', 'b' ) ) 42 | t:eq(true, s( 'ab', 'b' ) ) 43 | t:eq(true, s( 'ab', 'ab' ) ) 44 | t:eq(false, s( 'ab', 'bc' ) ) 45 | end 46 | 47 | function test_rjust(t) 48 | local f = strutil.rjust 49 | t:eq( '.......abc', f( 'abc', 10, '.' ) ) 50 | t:eq( ' abc', f( 'abc', 10 ) ) 51 | end 52 | 53 | function test_ljust(t) 54 | local f = strutil.ljust 55 | t:eq( 'abc.......', f( 'abc', 10, '.' ) ) 56 | t:eq( 'abc ', f( 'abc', 10 ) ) 57 | end 58 | 59 | function test_fnmatch(t) 60 | 61 | function t_match(s, ptn, ok) 62 | t:eq( 63 | strutil.fnmatch(s, ptn), ok, 64 | s .. ' ' .. ptn .. ' ' .. tostring(ok) 65 | ) 66 | end 67 | 68 | t_match('', '', true) 69 | t_match('a', '', false) 70 | t_match('a', 'a', true) 71 | t_match('a', 'b', false) 72 | 73 | t_match('', '?', false) 74 | t_match('?', '?', true) 75 | t_match('*', '?', true) 76 | t_match('.', '?', true) 77 | t_match('a', '?', true) 78 | 79 | t_match('', '*', true) 80 | t_match('a', '*', true) 81 | t_match('ab', '*', true) 82 | t_match('?', '*', true) 83 | t_match('??', '*', true) 84 | t_match('..', '*', true) 85 | 86 | t_match('a', '.', false) 87 | 88 | t_match('.', '*.*', true) 89 | t_match('a.', '*.*', true) 90 | t_match('.b', '*.*', true) 91 | t_match('a.b', '*.*', true) 92 | t_match('a.b.c', '*.*', true) 93 | t_match('.a.b.c', '*.*', true) 94 | t_match('.a.b.c.', '*.*', true) 95 | t_match('abc', '*.*', false) 96 | 97 | t_match('a.b', '.*', false) 98 | t_match('a.b', '*.', false) 99 | t_match('a.b', '*', true) 100 | t_match('a.b.c', '*', true) 101 | 102 | t_match('', '\\', false) 103 | t_match('\\', '\\', true) 104 | t_match('\\a', '\\', false) 105 | 106 | -- escaped 107 | t_match('*', '\\*', true) 108 | t_match('a', '\\*', false) 109 | 110 | t_match('?', '\\?', true) 111 | t_match('a', '\\?', false) 112 | t_match('ab', '\\?', false) 113 | 114 | -- non escaped 115 | t_match('a', '\\\\*', false) 116 | t_match('\\', '\\\\*', true) 117 | t_match('\\a', '\\\\*', true) 118 | t_match('\\abcd*', '\\\\*', true) 119 | 120 | t_match('?', '\\\\?', false) 121 | t_match('a', '\\\\?', false) 122 | t_match('\\?', '\\\\?', true) 123 | t_match('\\a', '\\\\?', true) 124 | end 125 | 126 | 127 | function bench_split() 128 | 129 | local str ='/v1/get/video.vic.sina.com.cn%2fmt788%2f9e%2f1a%2f81403.jpg/%7b%22xACL%22%3a%20%7b%22GRPS000000ANONYMOUSE%22%3a%20%5b%22read%22%5d, %20%22SINA00000000000SALES%22%3a%20%5b%22read%22, %20%22write%22, %20%22read_acp%22, %20%22write_acp%22%5d%7d, %20%22Info%22%3a%20null, %20%22Type%22%3a%20%22image%5c%2fjpeg%22, %20%22ver%22%3a%201042410872, %20%22Get-Location%22%3a%20%5b%7b%22CheckNumber%22%3a%201042410872, %20%22GroupID%22%3a%20341476, %20%22Partitions%22%3a%20%5b%7b%22IPs%22%3a%20%5b%2258.63.236.89%22, %20%2210.71.5.89%22%5d, %20%22PartitionID%22%3a%20%22185c3e5700014004975f90b11c13fc5e%22, %20%22IDC%22%3a%20%22.dx.GZ%22%7d, %20%7b%22IPs%22%3a%20%5b%2258.63.236.184%22, %20%2210.71.5.184%22%5d, %20%22PartitionID%22%3a%20%225a56155500014009aaa590b11c148e88%22, %20%22IDC%22%3a%20%22.dx.GZ%22%7d, %20%7b%22IPs%22%3a%20%5b%2260.28.228.36%22, %20%22172.16.228.36%22%5d, %20%22PartitionID%22%3a%20%22a41d4006000140029595d4ae52b17fe1%22, %20%22IDC%22%3a%20%22.wt.TJ%22%7d, %20%7b%22IPs%22%3a%20%5b%22111.161.78.59%22, %20%22172.16.48.59%22%5d, %20%22PartitionID%22%3a%20%22d7fee54d00014006b58090b11c145321%22, %20%22IDC%22%3a%20%22.wt.TJ%22%7d%5d%7d%5d, %20%22Info-Int%22%3a%200, %20%22ts%22%3a%201356544141, %20%22ACL%22%3a%20%7b%22SINA00000000000SALES%22%3a%20%5b%22read%22, %20%22write%22, %20%22read_acp%22, %20%22write_acp%22%5d%7d, %20%22ETag2%22%3a%20%2235b4ec0bfd826ea609054ccca4976e4fc77f3a8b%22, %20%22ETag%22%3a%20%22e14526f8858e2e0f898e72f141f108e4%22, %20%22Key%22%3a%20%22mt788%5c%2f9e%5c%2f1a%5c%2f81403.jpg%22, %20%22Owner%22%3a%20%22SINA00000000000SALES%22, %20%22Origo%22%3a%20%220000000000000000000090b11c09b4d9%22, %20%22GroupClassID%22%3a%206, %20%22File-Meta%22%3a%20%7b%22Content-Type%22%3a%20%22image%5c%2fjpeg%22%7d, %20%22Size%22%3a%2095964%7d?n=1&r=1&w=1&expire=60&ver_key=ts' 130 | 131 | local xx = strutil.split( str, '/' ) 132 | for i, v in ipairs(xx) do 133 | print( v ) 134 | end 135 | 136 | local i = 1024 * 1024 137 | while i > 0 do 138 | xx = strutil.split( str, '/') 139 | i = i - 1 140 | end 141 | 142 | end 143 | 144 | -------------------------------------------------------------------------------- /lib/test_tableutil.lua: -------------------------------------------------------------------------------- 1 | local tableutil = require("acid.tableutil") 2 | local strutil = require("acid.strutil") 3 | 4 | local tb_eq = tableutil.eq 5 | local to_str = strutil.to_str 6 | 7 | function test_nkeys(t) 8 | local cases = { 9 | {0, {}, 'nkeys of empty'}, 10 | {1, {0}, 'nkeys of 1'}, 11 | {2, {0, nil, 1}, 'nkeys of 0, nil and 1'}, 12 | {2, {0, 1}, 'nkeys of 2'}, 13 | {2, {0, 1, nil}, 'nkeys of 0, 1 and nil'}, 14 | {1, {a=0, nil}, 'nkeys of a=1'}, 15 | {2, {a=0, b=2, nil}, 'nkeys of a=1'}, 16 | } 17 | 18 | for i, case in ipairs(cases) do 19 | local n, tbl, mes = case[1], case[2], case[3] 20 | local rst = tableutil.nkeys(tbl) 21 | t:eq(n, rst, 'nkeys:' .. mes) 22 | 23 | rst = tableutil.get_len(tbl) 24 | t:eq(n, rst, 'get_len:' .. mes) 25 | end 26 | end 27 | 28 | function test_keys(t) 29 | t:eqdict( {}, tableutil.keys({}) ) 30 | t:eqdict( {1}, tableutil.keys({1}) ) 31 | t:eqdict( {1, 'a'}, tableutil.keys({1, a=1}) ) 32 | t:eqdict( {1, 2, 'a'}, tableutil.keys({1, 3, a=1}) ) 33 | end 34 | 35 | function test_duplist(t) 36 | local du = tableutil.duplist 37 | 38 | local a = { 1 } 39 | local b = tableutil.duplist( a ) 40 | a[ 2 ] = 2 41 | t:eq( nil, b[ 2 ], "dup not affected" ) 42 | 43 | t:eqdict( {1}, du( { 1, nil, 2 } ) ) 44 | t:eqdict( {1}, du( { 1, a=3 } ) ) 45 | t:eqdict( {1}, du( { 1, [3]=3 } ) ) 46 | t:eqdict( {}, du( { a=1, [3]=3 } ) ) 47 | 48 | local a = { { 1, 2, 3, a=4 } } 49 | a[2] = a[1] 50 | local b = du(a) 51 | t:eqdict({ { 1, 2, 3, a=4 }, { 1, 2, 3, a=4 } }, b) 52 | t:eq( b[1], b[2] ) 53 | end 54 | 55 | function test_sub(t) 56 | local a = { a=1, b=2, c={} } 57 | t:eqdict( {}, tableutil.sub( a, nil ) ) 58 | t:eqdict( {}, tableutil.sub( a, {} ) ) 59 | t:eqdict( {b=2}, tableutil.sub( a, {"b"} ) ) 60 | t:eqdict( {a=1, b=2}, tableutil.sub( a, {"a", "b"} ) ) 61 | 62 | local b = tableutil.sub( a, {"a", "b", "c"} ) 63 | t:neq( b, a ) 64 | t:eq( b.c, a.c, "reference" ) 65 | 66 | -- sub list 67 | 68 | local cases = { 69 | {{1, 2, 3}, {}, {}}, 70 | {{1, 2, 3}, {2, 3}, {2, 3}}, 71 | {{1, 2, 3}, {2, 3, 4}, {2, 3}}, 72 | {{1, 2, 3}, {3, 4, 2}, {3, 2}}, 73 | } 74 | 75 | for i, case in ipairs(cases) do 76 | local tbl, ks, expected = unpack(case) 77 | local rst = tableutil.sub(tbl, ks, true) 78 | t:eqdict(expected, rst, to_str(i .. 'th case: ', case)) 79 | end 80 | end 81 | 82 | function test_dup(t) 83 | local a = { a=1, 10, x={ y={z=3} } } 84 | a.self = a 85 | a.selfref = a 86 | a.x2 = a.x 87 | 88 | local b = tableutil.dup( a ) 89 | b.a = 'b' 90 | t:eq( 1, a.a, 'dup not affected' ) 91 | t:eq( 10, a[ 1 ], 'a has 10' ) 92 | t:eq( 10, b[ 1 ], 'b inherit 10' ) 93 | b[ 1 ] = 11 94 | t:eq( 10, a[ 1 ], 'a has still 10' ) 95 | 96 | a.x.y.z = 4 97 | t:eq( 4, b.x.y.z, 'no deep' ) 98 | 99 | local deep = tableutil.dup( a, true ) 100 | a.x.y.z = 5 101 | t:eq( 4, deep.x.y.z, 'deep dup' ) 102 | t:eq( deep, deep.self, 'loop reference' ) 103 | t:eq( deep, deep.selfref, 'loop reference should be dup only once' ) 104 | t:eq( deep.x, deep.x2, 'dup only once' ) 105 | t:neq( a.x, deep.x, 'dup-ed x' ) 106 | t:eq( deep.x.y, deep.x2.y ) 107 | 108 | end 109 | 110 | function test_contains(t) 111 | local c = tableutil.contains 112 | t:eq( true, c( nil, nil ) ) 113 | t:eq( true, c( 1, 1 ) ) 114 | t:eq( true, c( "", "" ) ) 115 | t:eq( true, c( "a", "a" ) ) 116 | 117 | t:eq( false, c( 1, 2 ) ) 118 | t:eq( false, c( 1, nil ) ) 119 | t:eq( false, c( nil, 1 ) ) 120 | t:eq( false, c( {}, 1 ) ) 121 | t:eq( false, c( {}, "" ) ) 122 | t:eq( false, c( "", {} ) ) 123 | t:eq( false, c( 1, {} ) ) 124 | 125 | t:eq( true, c( {}, {} ) ) 126 | t:eq( true, c( {1}, {} ) ) 127 | t:eq( true, c( {1}, {1} ) ) 128 | t:eq( true, c( {1, 2}, {1} ) ) 129 | t:eq( true, c( {1, 2}, {1, 2} ) ) 130 | t:eq( true, c( {1, 2, a=3}, {1, 2} ) ) 131 | t:eq( true, c( {1, 2, a=3}, {1, 2, a=3} ) ) 132 | 133 | t:eq( false, c( {1, 2, a=3}, {1, 2, b=3} ) ) 134 | t:eq( false, c( {1, 2 }, {1, 2, b=3} ) ) 135 | t:eq( false, c( {1}, {1, 2, b=3} ) ) 136 | t:eq( false, c( {}, {1, 2, b=3} ) ) 137 | 138 | t:eq( true, c( {1, 2, a={ x=1 }}, {1, 2} ) ) 139 | t:eq( true, c( {1, 2, a={ x=1, y=2 }}, {1, 2, a={}} ) ) 140 | t:eq( true, c( {1, 2, a={ x=1, y=2 }}, {1, 2, a={x=1}} ) ) 141 | t:eq( true, c( {1, 2, a={ x=1, y=2 }}, {1, 2, a={x=1, y=2}} ) ) 142 | 143 | t:eq( false, c( {1, 2, a={ x=1 }}, {1, 2, a={x=1, y=2}} ) ) 144 | 145 | -- self reference 146 | local a = { x=1 } 147 | local b = { x=1 } 148 | 149 | a.self = { x=1 } 150 | b.self = {} 151 | t:eq( true, c( a, b ) ) 152 | t:eq( false, c( b, a ) ) 153 | 154 | a.self = a 155 | b.self = nil 156 | t:eq( true, c( a, b ) ) 157 | t:eq( false, c( b, a ) ) 158 | 159 | a.self = a 160 | b.self = b 161 | t:eq( true, c( a, b ) ) 162 | t:eq( true, c( b, a ) ) 163 | 164 | a.self = { self=a } 165 | b.self = nil 166 | t:eq( true, c( a, b ) ) 167 | t:eq( false, c( b, a ) ) 168 | 169 | a.self = { self=a, x=1 } 170 | b.self = b 171 | t:eq( true, c( a, b ) ) 172 | 173 | a.self = { self={ self=a, x=1 }, x=1 } 174 | b.self = { self=b } 175 | t:eq( true, c( a, b ) ) 176 | 177 | -- cross reference 178 | a.self = { x=1 } 179 | b.self = { x=1 } 180 | a.self.self = b 181 | b.self.self = a 182 | t:eq( true, c( a, b ) ) 183 | t:eq( true, c( b, a ) ) 184 | 185 | end 186 | 187 | function test_eq(t) 188 | local c = tableutil.eq 189 | t:eq( true, c( nil, nil ) ) 190 | t:eq( true, c( 1, 1 ) ) 191 | t:eq( true, c( "", "" ) ) 192 | t:eq( true, c( "a", "a" ) ) 193 | 194 | t:eq( false, c( 1, 2 ) ) 195 | t:eq( false, c( 1, nil ) ) 196 | t:eq( false, c( nil, 1 ) ) 197 | t:eq( false, c( {}, 1 ) ) 198 | t:eq( false, c( {}, "" ) ) 199 | t:eq( false, c( "", {} ) ) 200 | t:eq( false, c( 1, {} ) ) 201 | 202 | t:eq( true, c( {}, {} ) ) 203 | t:eq( true, c( {1}, {1} ) ) 204 | t:eq( true, c( {1, 2}, {1, 2} ) ) 205 | t:eq( true, c( {1, 2, a=3}, {1, 2, a=3} ) ) 206 | 207 | t:eq( false, c( {1, 2}, {1} ) ) 208 | t:eq( false, c( {1, 2, a=3}, {1, 2} ) ) 209 | 210 | t:eq( false, c( {1, 2, a=3}, {1, 2, b=3} ) ) 211 | t:eq( false, c( {1, 2 }, {1, 2, b=3} ) ) 212 | t:eq( false, c( {1}, {1, 2, b=3} ) ) 213 | t:eq( false, c( {}, {1, 2, b=3} ) ) 214 | 215 | t:eq( true, c( {1, 2, a={ x=1, y=2 }}, {1, 2, a={x=1, y=2}} ) ) 216 | 217 | t:eq( false, c( {1, 2, a={ x=1 }}, {1, 2, a={x=1, y=2}} ) ) 218 | 219 | -- self reference 220 | local a = { x=1 } 221 | local b = { x=1 } 222 | 223 | a.self = { x=1 } 224 | b.self = {} 225 | t:eq( false, c( a, b ) ) 226 | 227 | a.self = { x=1 } 228 | b.self = { x=1 } 229 | t:eq( true, c( a, b ) ) 230 | 231 | a.self = a 232 | b.self = nil 233 | t:eq( false, c( b, a ) ) 234 | 235 | a.self = a 236 | b.self = b 237 | t:eq( true, c( a, b ) ) 238 | t:eq( true, c( b, a ) ) 239 | 240 | a.self = { self=a } 241 | b.self = nil 242 | t:eq( false, c( a, b ) ) 243 | 244 | a.self = { self=a, x=1 } 245 | b.self = b 246 | t:eq( true, c( a, b ) ) 247 | 248 | a.self = { self={ self=a, x=1 }, x=1 } 249 | b.self = { self=b, x=1 } 250 | t:eq( true, c( a, b ) ) 251 | 252 | -- cross reference 253 | a.self = { x=1 } 254 | b.self = { x=1 } 255 | a.self.self = b 256 | b.self.self = a 257 | t:eq( true, c( a, b ) ) 258 | t:eq( true, c( b, a ) ) 259 | 260 | end 261 | 262 | function test_intersection(t) 263 | local a = { a=1, 10 } 264 | local b = { 11, 12 } 265 | local c = tableutil.intersection( { a, b }, true ) 266 | 267 | t:eq( 1, tableutil.nkeys( c ), 'c has 1' ) 268 | t:eq( true, c[ 1 ] ) 269 | 270 | local d = tableutil.intersection( { a, { a=20 } }, true ) 271 | t:eq( 1, tableutil.nkeys( d ) ) 272 | t:eq( true, d.a, 'intersection a' ) 273 | 274 | local e = tableutil.intersection( { { a=1, b=2, c=3, d=4 }, { b=2, c=3 }, { b=2, d=5 } }, true ) 275 | t:eq( 1, tableutil.nkeys( e ) ) 276 | t:eq( true, e.b, 'intersection of 3' ) 277 | 278 | end 279 | 280 | function test_union(t) 281 | local a = tableutil.union( { { a=1, b=2, c=3 }, { a=1, d=4 } }, 0 ) 282 | t:eqdict( { a=0, b=0, c=0, d=0 }, a ) 283 | end 284 | 285 | function test_mergedict(t) 286 | t:eqdict( { a=1, b=2, c=3 }, tableutil.merge( { a=1, b=2, c=3 } ) ) 287 | t:eqdict( { a=1, b=2, c=3 }, tableutil.merge( {}, { a=1, b=2 }, { c=3 } ) ) 288 | t:eqdict( { a=1, b=2, c=3 }, tableutil.merge( { a=1 }, { b=2 }, { c=3 } ) ) 289 | t:eqdict( { a=1, b=2, c=3 }, tableutil.merge( { a=0 }, { a=1, b=2 }, { c=3 } ) ) 290 | 291 | local a = { a=1 } 292 | local b = { b=2 } 293 | local c = tableutil.merge( a, b ) 294 | t:eq( true, a==c ) 295 | a.x = 10 296 | t:eq( 10, c.x ) 297 | end 298 | 299 | function test_repr(t) 300 | local r = tableutil.repr 301 | local s1 = { sep=' ' } 302 | local s2 = { sep=' ' } 303 | 304 | t:eq( '1', r( 1 ) ) 305 | t:eq( '"1"', r( '1' ) ) 306 | t:eq( 'nil', r( nil ) ) 307 | t:eq( '{}', r( {} ) ) 308 | t:eq( '{}', r( {}, s1 ) ) 309 | t:eq( '{ 1 }', r( { 1 }, s1 ) ) 310 | t:eq( '{ 1, 2 }', r( { 1, 2 }, s1 ) ) 311 | t:eq( '{ a=1 }', r( { a=1 }, s1 ) ) 312 | t:eq( '{ 0, a=1, b=2 }', r( { 0, a=1, b=2 }, s1 ) ) 313 | t:eq( '{ 0, a=1, b=2 }', r( { 0, a=1, b=2 }, s2 ) ) 314 | 315 | local literal=[[{ 316 | 1, 317 | 2, 318 | 3, 319 | { 320 | 1, 321 | 2, 322 | 3, 323 | 4 324 | }, 325 | [100]=33333, 326 | a=1, 327 | c=100000, 328 | d=1, 329 | ["fjklY*("]={ 330 | b=3, 331 | x=1 332 | }, 333 | x={ 334 | 1, 335 | { 336 | 1, 337 | 2 338 | }, 339 | y={ 340 | a=1, 341 | b=2 342 | } 343 | } 344 | }]] 345 | local a = { 346 | 1, 2, 3, 347 | { 1, 2, 3, 4 }, 348 | a=1, 349 | c=100000, 350 | d=1, 351 | x={ 352 | 1, 353 | { 1, 2 }, 354 | y={ 355 | a=1, 356 | b=2 357 | } 358 | }, 359 | ['fjklY*(']={ 360 | x=1, 361 | b=3, 362 | }, 363 | [100]=33333 364 | } 365 | t:eq( literal, r(a, { indent=' ' }) ) 366 | 367 | 368 | end 369 | 370 | function test_str(t) 371 | local r = tableutil.str 372 | local s1 = { sep=' ' } 373 | local s2 = { sep=' ' } 374 | 375 | t:eq( '1', r( 1 ) ) 376 | t:eq( '1', r( '1' ) ) 377 | t:eq( 'nil', r( nil ) ) 378 | t:eq( '{}', r( {} ) ) 379 | t:eq( '{}', r( {}, s1 ) ) 380 | t:eq( '{ 1 }', r( { 1 }, s1 ) ) 381 | t:eq( '{ 1, 2 }', r( { 1, 2 }, s1 ) ) 382 | t:eq( '{ a=1 }', r( { a=1 }, s1 ) ) 383 | t:eq( '{ 0, a=1, b=2 }', r( { 0, a=1, b=2 }, s1 ) ) 384 | t:eq( '{ 0, a=1, b=2 }', r( { 0, a=1, b=2 }, s2 ) ) 385 | t:eq( '{0,a=1,b=2}', r( { 0, a=1, b=2 } ) ) 386 | 387 | local literal=[[{ 388 | 1, 389 | 2, 390 | 3, 391 | { 392 | 1, 393 | 2, 394 | 3, 395 | 4 396 | }, 397 | 100=33333, 398 | a=1, 399 | c=100000, 400 | d=1, 401 | fjklY*(={ 402 | b=3, 403 | x=1 404 | }, 405 | x={ 406 | 1, 407 | { 408 | 1, 409 | 2 410 | }, 411 | y={ 412 | a=1, 413 | b=2 414 | } 415 | } 416 | }]] 417 | local a = { 418 | 1, 2, 3, 419 | { 1, 2, 3, 4 }, 420 | a=1, 421 | c=100000, 422 | d=1, 423 | x={ 424 | 1, 425 | { 1, 2 }, 426 | y={ 427 | a=1, 428 | b=2 429 | } 430 | }, 431 | ['fjklY*(']={ 432 | x=1, 433 | b=3, 434 | }, 435 | [100]=33333 436 | } 437 | t:eq( literal, r(a, { indent=' ' }) ) 438 | 439 | 440 | end 441 | 442 | function test_iter(t) 443 | 444 | for ks, v in tableutil.deep_iter({}) do 445 | t:err( "should not get any keys" ) 446 | end 447 | 448 | for ks, v in tableutil.deep_iter({1}) do 449 | t:eqdict( {1}, ks ) 450 | t:eq( 1, v ) 451 | end 452 | 453 | for ks, v in tableutil.deep_iter({a="x"}) do 454 | t:eqdict( {{"a"}, "x"}, {ks, v} ) 455 | end 456 | 457 | local a = { 458 | 1, 2, 3, 459 | { 1, 2, 3, 4 }, 460 | a=1, 461 | c=100000, 462 | d=1, 463 | x={ 464 | 1, 465 | { 1, 2 }, 466 | y={ 467 | a=1, 468 | b=2 469 | } 470 | }, 471 | ['fjklY*(']={ 472 | x=1, 473 | b=3, 474 | }, 475 | [100]=33333 476 | } 477 | a.z = a.x 478 | 479 | local r = { 480 | { {1}, 1 }, 481 | { {100}, 33333 }, 482 | { {2}, 2 }, 483 | { {3}, 3 }, 484 | { {4,1}, 1 }, 485 | { {4,2}, 2 }, 486 | { {4,3}, 3 }, 487 | { {4,4}, 4 }, 488 | { {"a"}, 1 }, 489 | { {"c"}, 100000 }, 490 | { {"d"}, 1 }, 491 | { {"fjklY*(","b"}, 3 }, 492 | { {"fjklY*(","x"}, 1 }, 493 | { {"x",1}, 1 }, 494 | { {"x",2,1}, 1 }, 495 | { {"x",2,2}, 2 }, 496 | { {"x","y","a"}, 1 }, 497 | { {"x","y","b"}, 2 }, 498 | { {"z",1}, 1 }, 499 | { {"z",2,1}, 1 }, 500 | { {"z",2,2}, 2 }, 501 | { {"z","y","a"}, 1 }, 502 | { {"z","y","b"}, 2 }, 503 | } 504 | 505 | local i = 0 506 | for ks, v in tableutil.deep_iter(a) do 507 | i = i + 1 508 | t:eqdict( r[i], {ks, v} ) 509 | end 510 | 511 | end 512 | 513 | 514 | function test_has(t) 515 | local cases = { 516 | {nil, {}, true}, 517 | {1, {1}, true}, 518 | {1, {1, 2}, true}, 519 | {1, {1, 2, 'x'}, true}, 520 | {'x', {1, 2, 'x'}, true}, 521 | 522 | {1, {x=1}, true}, 523 | 524 | {'x', {x=1}, false}, 525 | {"x", {1}, false}, 526 | {"x", {1, 2}, false}, 527 | {1, {}, false}, 528 | } 529 | 530 | for i, case in ipairs(cases) do 531 | local val, tbl, expected = case[1], case[2], case[3] 532 | t:eq(expected, tableutil.has(tbl, val), i .. 'th case: ' .. to_str(val, ' ', tbl)) 533 | end 534 | end 535 | 536 | 537 | function test_remove(t) 538 | local t1 = {} 539 | local cases = { 540 | {{}, nil, {}, nil}, 541 | {{1, 2, 3}, 2, {1, 3}, 2}, 542 | {{1, 2, 3, x=4}, 2, {1, 3, x=4}, 2}, 543 | {{1, 2, 3}, 3, {1, 2}, 3}, 544 | {{1, 2, 3, x=4}, 3, {1, 2, x=4}, 3}, 545 | {{1, 2, 3, x=4}, 4, {1, 2, 3}, 4}, 546 | {{1, 2, 3, x=t1}, t1, {1, 2, 3}, t1}, 547 | 548 | {{1, 2, t1, x=t1}, t1, {1, 2, x=t1}, t1}, 549 | } 550 | 551 | for i, case in ipairs(cases) do 552 | 553 | local tbl, val, expected_tbl, expected_rst = case[1], case[2], case[3], case[4] 554 | 555 | local rst = tableutil.remove(tbl, val) 556 | 557 | t:eqdict(expected_tbl, tbl, i .. 'th tbl') 558 | t:eq(expected_rst, rst, i .. 'th rst') 559 | end 560 | end 561 | 562 | 563 | 564 | function test_extends(t) 565 | 566 | local cases = { 567 | 568 | {{1,2}, {3,4}, {1,2,3,4}}, 569 | {{1,2}, {3}, {1,2,3}}, 570 | {{1,2}, {nil}, {1,2}}, 571 | {{1,2}, {}, {1,2}}, 572 | {{}, {1,2}, {1,2}}, 573 | {nil, {1}, nil}, 574 | {{1,{2,3}}, {4,5}, {1,{2,3},4,5}}, 575 | {{1}, {{2,3},4,5}, {1,{2,3},4,5}}, 576 | {{"xx",2}, {3,"yy"}, {"xx",2,3,"yy"}}, 577 | {{1,2}, {3,nil,4}, {1,2,3}}, 578 | {{1,nil,2}, {3,4}, {1,nil,2,3,4}}, 579 | 580 | } 581 | 582 | for _, c in ipairs( cases ) do 583 | 584 | local msg = to_str(c) 585 | 586 | local exp = c[3] 587 | local rst = tableutil.extends(c[1], c[2]) 588 | 589 | t:eqdict(exp, rst, msg) 590 | end 591 | end 592 | -------------------------------------------------------------------------------- /lib/worker_init.lua: -------------------------------------------------------------------------------- 1 | 2 | local tableutil = require( "acid.tableutil" ) 3 | local libluafs = require( "libluafs" ) 4 | 5 | 6 | local _M = {} 7 | 8 | local err, errmes 9 | local mem_index = ngx.config.prefix():sub(-2, -2) 10 | 11 | local counter = 0 12 | local function get_incr_num() 13 | counter = counter + 1 14 | return counter 15 | end 16 | 17 | local function _read(path) 18 | local f, err = io.open(path, "r") 19 | if err then 20 | if string.find(err, "No such file or directory") > 0 then 21 | return nil, nil, nil 22 | end 23 | return nil, "StorageError", err 24 | end 25 | 26 | local cont = f:read("*a") 27 | f:close() 28 | 29 | if cont == nil then 30 | return nil, "StorageError", path 31 | end 32 | 33 | return cont 34 | end 35 | 36 | local function _write(path, cont) 37 | 38 | local tmp_id = ngx.now() .. '_' .. ngx.worker.pid() .. '_' .. get_incr_num() 39 | 40 | local _path = path .. '_' .. tmp_id 41 | local f, err = io.open(_path, "w") 42 | f:write(cont) 43 | f:flush() 44 | f:close() 45 | 46 | local _, err = os.rename(_path, path) 47 | if err then 48 | os.remove(_path) 49 | return nil, "StorageError", err 50 | end 51 | 52 | return nil, nil, nil 53 | end 54 | 55 | local function fn_ids(fn) 56 | local hsh = ngx.crc32_long(fn) 57 | return {hsh, hsh+1} 58 | end 59 | 60 | local function fn_mems(fn) 61 | local ids = fn_ids(fn) 62 | local mems = cc:members() 63 | local rst = {} 64 | for i, id in ipairs(ids) do 65 | rst[i] = mems[ (id % #mems) + 1 ] 66 | end 67 | return rst 68 | end 69 | 70 | local function is_data_valid() 71 | local cont = _read('is_ok') 72 | if cont == nil then 73 | return false 74 | end 75 | 76 | local time = ngx.time() 77 | 78 | if ( tonumber(cont) or 0 ) > time - 30 then 79 | return true 80 | end 81 | return false 82 | end 83 | 84 | _M.ident = "127.0.0.1:990" .. mem_index 85 | _M.cluster_id = "x" 86 | 87 | _M.cc, err, errmes = require("nginx_cluster").new({ 88 | cluster_id = _M.cluster_id, 89 | ident = _M.ident, 90 | get_standby = function() 91 | return { 92 | "127.0.0.1:9901", 93 | "127.0.0.1:9902", 94 | "127.0.0.1:9903", 95 | "127.0.0.1:9904", 96 | "127.0.0.1:9905", 97 | "127.0.0.1:9906", 98 | } 99 | end, 100 | 101 | is_data_valid = is_data_valid, 102 | 103 | restore = function() 104 | 105 | if is_data_valid() then 106 | return 107 | end 108 | 109 | local mems = _M.cc:members() 110 | local my_idx = 0 111 | for i, mm in ipairs(mems) do 112 | if mm == _M.ident then 113 | my_idx = i 114 | break 115 | end 116 | end 117 | 118 | local backups = { 119 | mems[ (my_idx-1) % #mems + 1 ], 120 | mems[ (my_idx+1) % #mems + 1 ], 121 | } 122 | 123 | -- for 124 | 125 | 126 | 127 | end, 128 | 129 | destory = function() 130 | end, 131 | 132 | }) 133 | 134 | function _M.handle_get() 135 | local cc, ident = _M.cc, _M.ident 136 | 137 | -- strip /get/ 138 | local fn = ngx.var.uri:sub(6) 139 | 140 | local mems = fn_mems(fn) 141 | 142 | local dst = mems[1] 143 | 144 | if dst == ident then 145 | ngx.req.set_uri("/www/" .. fn, true ) 146 | else 147 | dst = dst:gsub("990", "980") 148 | ngx.req.set_uri("/proxy/" .. dst .. "/get/" .. fn, true) 149 | end 150 | end 151 | 152 | function _M.handle_ls() 153 | local rst, err_msg = libluafs.readdir( 'www' ) 154 | table.sort(rst) 155 | local lst = table.concat('\n', rst) 156 | ngx.say(lst) 157 | end 158 | 159 | return _M 160 | -------------------------------------------------------------------------------- /merge: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | path=$(pwd) 4 | bs="$(find ./bundle -name "lib" -type d)" 5 | 6 | rm -rf ./lib/* 7 | mkdir -p ./lib 8 | 9 | for lib in $bs; do 10 | 11 | ( 12 | cd $lib && ( 13 | for f in $(ls); do 14 | cp -R $path/$lib/$f $path/lib/ 15 | done 16 | ) 17 | ) 18 | done 19 | 20 | if [ ".$1" = ".rc" ]; then 21 | ( 22 | cd ./lib && rm test_*.lua 23 | ) 24 | fi 25 | 26 | for modname in $(ls dep-build); do 27 | sh dep-build/$modname || { echo "Error build $modname"; exit 1; } 28 | done 29 | -------------------------------------------------------------------------------- /py/cluster_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import os 5 | import sys 6 | import datetime 7 | import time 8 | import random 9 | import threading 10 | import socket 11 | import pprint 12 | 13 | from it.paxoscli import PaxosClient, PaxosError, ids_dic, init_view, request, request_ex 14 | from it.ngxctl import ngx_start, ngx_stop, ngx_restart 15 | from it.sto import init_sto 16 | 17 | def err_rst( code ): 18 | return { "err": { "Code": code } } 19 | 20 | class ee( object ): 21 | NoChange = err_rst("NoChange") 22 | NoView = err_rst("NoView") 23 | DuringChange = err_rst("DuringChange") 24 | QuorumFailure = err_rst("QuorumFailure") 25 | VerNotExist = err_rst("VerNotExist") 26 | 27 | g1 = ids_dic("1") 28 | g2 = ids_dic("2") 29 | g3 = ids_dic("3") 30 | g12 = ids_dic("12") 31 | g23 = ids_dic("23") 32 | g13 = ids_dic("13") 33 | g123 = ids_dic("123") 34 | g234 = ids_dic("234") 35 | 36 | def out( *args ): 37 | os.write( 1, " ".join( [str(x) for x in args] ) + "\n" ) 38 | 39 | def check_test(): 40 | cases = ( 41 | # func, args, result 42 | ( "view change after dead detected", 43 | (init_view,(1,(1,2,3)), {}), 44 | (init_view,(2,(1,2,3)), {}), 45 | (init_view,(3,(1,2,3)), {}), 46 | (request,('get_view',1,),{"ver":1,"key":"view","val":[g123]}), 47 | (request,('get_view',2,),{"ver":1,"key":"view","val":[g123]}), 48 | (request,('get_view',3,),{"ver":1,"key":"view","val":[g123]}), 49 | (request,('get',1,{"key":"i"}),{"ver":1, "key":"i"}), 50 | 51 | (request,('isalive',2,),{}), 52 | (request,('isalive',4,),{"err":{"Code": "NoView"}}), 53 | 54 | (init_view,(4,(1,2,3)), {}), 55 | (request,('isalive',4,),{"err":{"Code": "NotMember"}}), 56 | 57 | # should have been destoried 58 | (time.sleep, (2,), None), 59 | (request,('isalive',4,),{"err":{"Code": "NoView"}}), 60 | 61 | # # after shut down 1, 4 become a member 62 | # (ngx_stop, ("1", ), None), 63 | # (time.sleep, (10,), None), 64 | # (request,('get_view',2,),{"ver":6,"key":"view","val":[g234]}), 65 | # (request,('get_view',3,),{"ver":6,"key":"view","val":[g234]}), 66 | # (request,('get_view',4,),{"ver":6,"key":"view","val":[g234]}), 67 | 68 | # (ngx_stop, ("2", ), None), 69 | # (time.sleep, (3,), None), 70 | # (request,('get_view',3,),{"ver":10,"key":"view","val":[g234]}), 71 | # (request,('get_view',4,),{"ver":10,"key":"view","val":[g234]}), 72 | ), 73 | ) 74 | 75 | for case in cases: 76 | ngx_restart('1234') 77 | init_sto() 78 | 79 | mes = case[0] 80 | out( "" ) 81 | out( "="*7, mes ) 82 | 83 | for actions in case[1:]: 84 | f, args, rst = actions[:3] 85 | 86 | r = f( *args ) or {} 87 | b = r.get('body') 88 | 89 | out( "" ) 90 | out( f.__name__, args ) 91 | pprint.pprint( rst ) 92 | pprint.pprint( b ) 93 | assert b == rst, "expect to be " +repr(rst) + " but: " +repr(b) 94 | 95 | out( 'OK: ', ) 96 | out( r.get('status'), r.get('body') ) 97 | 98 | # nondeterministic test 99 | 100 | ngx_stop("1") 101 | time.sleep(10) 102 | 103 | # after shutting down 1. 4 should become a member 104 | for mid in (2, 3, 4): 105 | b = request('get_view', mid)['body'] 106 | assert b['val'] == [g234] 107 | 108 | # after shutting down 2. only 3, 4 is alive. 109 | ngx_stop("2") 110 | time.sleep(10) 111 | 112 | vers = {'3':0, '4':0} 113 | while vers['3'] < 60 and vers['4'] < 60: 114 | 115 | for mid in vers: 116 | 117 | r = request('get_view', mid) 118 | b = r['body'] 119 | _ver = b['ver'] 120 | assert _ver >= vers[mid], "expect to get a greater version than: " + repr(vers[mid]) + ' but: ' + repr(b) 121 | vers[mid] = _ver 122 | 123 | assert '1' not in b['val'][0] or '2' not in b['val'][0] 124 | assert '3' in b['val'][0] 125 | assert '4' in b['val'][0] 126 | 127 | if len(b['val']) == 2: 128 | assert '1' not in b['val'][1] or '2' not in b['val'][1] 129 | assert '3' in b['val'][1] 130 | assert '4' in b['val'][1] 131 | 132 | time.sleep(1) 133 | 134 | 135 | if __name__ == "__main__": 136 | import it.ngxconf 137 | it.ngxconf.make_conf(4, 'true') 138 | 139 | check_test() 140 | -------------------------------------------------------------------------------- /py/concurrency_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import os 5 | import sys 6 | import datetime 7 | import time 8 | import random 9 | import threading 10 | import socket 11 | 12 | from it.paxoscli import PaxosClient, PaxosError, ids_dic, init_view, request, request_ex 13 | from it.ngxctl import ngx_start, ngx_stop, ngx_restart 14 | from it.sto import init_sto 15 | 16 | g1 = ids_dic("1") 17 | g2 = ids_dic("2") 18 | g3 = ids_dic("3") 19 | g12 = ids_dic("12") 20 | g23 = ids_dic("23") 21 | g13 = ids_dic("13") 22 | g123 = ids_dic("123") 23 | 24 | def err_rst( code ): 25 | return { "err": { "Code": code } } 26 | 27 | class ee( object ): 28 | NoChange = err_rst("NoChange") 29 | NoView = err_rst("NoView") 30 | DuringChange = err_rst("DuringChange") 31 | QuorumFailure = err_rst("QuorumFailure") 32 | VerNotExist = err_rst("VerNotExist") 33 | 34 | def randsleep( ratio=1 ): 35 | time.sleep( random.random()*0.4*ratio ) 36 | 37 | def dd( args, *more_args ): 38 | dt = str(datetime.datetime.now()) 39 | out( dt, *( args + list(more_args) ) ) 40 | 41 | def out( *args ): 42 | os.write( 1, " ".join( [str(x) for x in args] ) + "\n" ) 43 | 44 | def integration_test(): 45 | cases = ( 46 | # func, args, result, result_filter 47 | ( "set", 48 | (init_view,(1,(1,2,3)), {}), 49 | (init_view,(2,(1,2,3)), {}), 50 | (request,('get_view',2,),{"ver":1,"key":"view","val":[g123]}), 51 | (request,('get',1,{"key":"i"}),{"ver":1, "key":"i"}), 52 | (request,('set',1,{"key":"i", "val":100}),{"ver":2, "key":"i", "val":100}), 53 | (request,('get',2,{"key":"i"}),{"ver":2, "key":"i", "val":100}), 54 | 55 | # re-set does not change 56 | (request,('set',1,{"key":"i", "val":100}),{"ver":2, "key":"i", "val":100}), 57 | (ngx_stop,('2',), None), 58 | (ngx_stop,('3',), None), 59 | (time.sleep, (1,), None ), 60 | 61 | # set without changing require quorum too 62 | (request,('set',1,{"key":"i", "val":100}),ee.QuorumFailure), 63 | (ngx_start,('2',), None), 64 | 65 | (time.sleep, (1,), None ), 66 | (request,('set',1,{"key":"i", "val":{"foo":"bar"}}),{ "ver":3, "key":"i", "val":{ "foo":"bar" } }), 67 | 68 | # re-set table value 69 | (request,('set',1,{"key":"i", "val":{"foo":"bar"}}),{ "ver":3, "key":"i", "val":{ "foo":"bar" } }), 70 | 71 | # set with specific ver 72 | (request,('set',1,{"key":"i", "ver":2, "val":{"foo":"bar"}}),{ "err":{ "Code":"VerNotExist", "Message":3 } }), 73 | 74 | # set with different table value 75 | (request,('set',1,{"key":"i", "ver":3, "val":{"FOO":"bar"}}),{ "ver":4, "key":"i", "val":{ "FOO":"bar" } }), 76 | 77 | ), 78 | ( "get", 79 | (init_view,(1,(1,2,3)), {}), 80 | (init_view,(2,(1,2,3)), {}), 81 | (request,('get',1,{"key":"i"}),{"ver":1, "key":"i"}), 82 | (request,('set',1,{"key":"i", "val":100}),{"ver":2, "key":"i", "val":100}), 83 | (request,('get',2,{"key":"i", "ver":2}),{"ver":2, "key":"i", "val":100}), 84 | (request,('get',2,{"key":"i", "ver":0}),{ "err":{ "Code":"VerNotExist", "Message":2 } }), 85 | (request,('get',2,{"key":"i", "ver":1}),{ "err":{ "Code":"VerNotExist", "Message":2 } }), 86 | (request,('get',2,{"key":"i", "ver":3}),{ "err":{ "Code":"VerNotExist", "Message":2 } }), 87 | ), 88 | ( "unable to elect with only 1 member", 89 | (request,('get_view',1,),ee.NoView), 90 | (request,('get_view',2,),ee.NoView), 91 | (init_view,(1,(1,2,3)), {}), 92 | (request,('get_view',1,),{"ver":1,"key":"view","val":[g123]}), 93 | (request,('get_leader',1,),{"ver":1,"key":"leader"}), 94 | (request,('get_or_elect_leader',1,),ee.QuorumFailure), 95 | ), 96 | 97 | ( "able to elect with only 2 members", 98 | (init_view,(1,(1,2,3)), {}), 99 | (init_view,(2,(1,2,3)), {}), 100 | (request,('get_view',2,),{"ver":1,"key":"view","val":[g123]}), 101 | (request,('get_or_elect_leader',1,),{"ver":2, "key":"leader", "val":{"ident":"1", "__lease":1}}), 102 | (request,('get_or_elect_leader',1,),{"ver":2, "key":"leader", "val":{"ident":"1", "__lease":1}}), 103 | (request,('get_or_elect_leader',2,),{"ver":2, "key":"leader", "val":{"ident":"1", "__lease":1}}), 104 | (request,('get_leader',1,),{"ver":2,"key":"leader", "val":{"ident":"1", "__lease":1}}), 105 | (time.sleep, (1,), None ), 106 | (request,('get_leader',1,),{"ver":2,"key":"leader","val":{"ident":"1", "__lease":0}}), 107 | (time.sleep, (1,), None ), 108 | (request,('get_leader',1,),{"ver":2,"key":"leader"}), 109 | (request,('get_or_elect_leader',2,),{"ver":3,"key":"leader","val":{"ident":"2", "__lease":1}}), 110 | 111 | # get leader with version specified 112 | (request,('get',2,{"key":"leader", "ver":3}),{"ver":3,"key":"leader","val":{"ident":"2", "__lease":1}}), 113 | (request,('get',2,{"key":"leader", "ver":4}),{"err": { "Code": "VerNotExist", "Message":3 }}), 114 | ), 115 | 116 | ( "unable to elect with 2 members with different ver", 117 | (init_view,(1,(1,2,3)), {}), 118 | (request,('get_view',1,),{"ver":1,"key":"view","val":[g123]}), 119 | (init_view,(2,(1,2,3), 2), {}), 120 | (request,('get_view',2,),{"ver":2,"key":"view","val":[g123]}), 121 | # 1 will load latest version=2 from 2, and then try to elect 122 | # leader with version=2 and would found that it locally does not 123 | # have committed data with version=2 124 | (request,('get_or_elect_leader',1,),ee.QuorumFailure), 125 | (request,('get_or_elect_leader',2,),ee.QuorumFailure), 126 | (request,('get_leader',1,),{"ver":2, "key":"leader"}), 127 | ), 128 | 129 | ( "elect with dual view", 130 | (init_view,(1,((1,2,3), (1,2))), {}), 131 | (init_view,(2,((1,2,3), (1,2))), {}), 132 | (request,('get_view',2,),{"ver":1,"key":"view","val":[g123, g12]}), 133 | (request,('get_or_elect_leader',1,),{"ver":2, "key":"leader", "val":{"ident":"1", "__lease":1}}), 134 | (request,('get_or_elect_leader',1,),{"ver":2, "key":"leader", "val":{"ident":"1", "__lease":1}}), 135 | (request,('get_or_elect_leader',2,),{"ver":2, "key":"leader", "val":{"ident":"1", "__lease":1}}), 136 | (request,('get_leader',1,),{"ver":2,"key":"leader", "val":{"ident":"1", "__lease":1}}), 137 | (request,('read',1,),{"ver":2,"val":{"leader":{"ident":"1", "__lease":1}, "view":[g123, g12]}}), 138 | 139 | (time.sleep, (1,), None ), 140 | (request,('get_leader',1,),{"ver":2,"key":"leader","val":{"ident":"1", "__lease":0}}), 141 | (time.sleep, (1,), None ), 142 | (request,('get_leader',1,),{"ver":2,"key":"leader"}), 143 | (request,('get_or_elect_leader',2,),{"ver":3,"key":"leader","val":{"ident":"2", "__lease":1}}), 144 | ), 145 | 146 | ( "elect failure with dual view", 147 | (init_view,(1,((1,2,3), (1,3))), {}), 148 | (init_view,(2,((1,2,3), (1,3))), {}), 149 | (request,('get_view',2,),{"ver":1,"key":"view","val":[g123, g13]}), 150 | (request,('get_or_elect_leader',1,),ee.QuorumFailure), 151 | ), 152 | 153 | ( "change_view", 154 | (init_view,(1,(1,)), {}), 155 | (ngx_stop,(2,),None), 156 | (request,('get_view',1,),{"ver":1,"key":"view","val":[g1]}), 157 | (request,('change_view',1,{"add":g23}),{"ver":3,"key":"view","val":[g123]}), 158 | (request,('get_view',1,),{"ver":3,"key":"view","val":[g123]}), 159 | (request,('get_view',3,),{"ver":3,"key":"view","val":[g123]}), 160 | (request,('change_view',1,{"add":g23}),{"ver":3,"key":"view","val":[g123]}), 161 | ), 162 | 163 | ( "change_view without any change", 164 | (init_view,(1,(1,)), {}), 165 | (request,('get_view',1,),{"ver":1,"key":"view","val":[g1]}), 166 | (request,('change_view',1,{}), {"ver":1,"key":"view","val":[g1]}), 167 | (request,('get_view',1,),{"ver":1,"key":"view","val":[g1]}), 168 | ), 169 | ( "change_view in process, come to consistent state", 170 | (init_view,(1,((1,),(1,2)),2), {}), 171 | (request,('get_view',1,),{"ver":2,"key":"view","val":[g1, g12]}), 172 | (request,('get_view',2,),ee.NoView), 173 | (request,('change_view',1,{}), ee.DuringChange), 174 | (request,('get_view',1,),{"ver":3,"key":"view","val":[g12]}), 175 | ), 176 | ( "change_view with unmatched versions", 177 | (init_view,(1,(1,2,3),2), {}), 178 | (request,('get_view',1,),{"ver":2,"key":"view","val":[g123]}), 179 | (request,('get_view',2,),ee.NoView), 180 | (init_view,(3,(1,2,3),3), {}), 181 | (request,('get_view',3,),{"ver":3,"key":"view","val":[g123]}), 182 | 183 | # change_view fix unmatched versions 184 | (request,('change_view',1,{"del":g1}),{"ver":5,"key":"view","val":[g23]}), 185 | 186 | (request,('get_view',1,),{"ver":5,"key":"view","val":[g23]}), 187 | (request,('get_view',2,),{"ver":5,"key":"view","val":[g23]}), 188 | (request,('get_view',3,),{"ver":5,"key":"view","val":[g23]}), 189 | ), 190 | ) 191 | 192 | for case in cases: 193 | ngx_restart('123') 194 | init_sto() 195 | 196 | mes = case[0] 197 | out( "" ) 198 | out( "="*7, mes ) 199 | 200 | for actions in case[1:]: 201 | f, args, rst = actions[:3] 202 | if len(actions) == 4: 203 | rst_filter = actions[3] 204 | else: 205 | rst_filter = lambda x:x 206 | 207 | r = f( *args ) or {} 208 | b = r.get('body') 209 | b = rst_filter(b) 210 | 211 | out( "" ) 212 | out( f.__name__, args ) 213 | import pprint 214 | pprint.pprint( rst ) 215 | pprint.pprint( b ) 216 | assert b == rst, "expect to be " +repr(rst) + " but: " +repr(b) 217 | 218 | out( 'OK: ', ) 219 | out( r.get('status'), r.get('body') ) 220 | 221 | def incr_worker(incr_key, idents, n): 222 | 223 | cur_ver = 1 224 | 225 | for i in range( n ): 226 | 227 | for n_try in range( 1, 1024*1024 ): 228 | 229 | randsleep( 0.3 ) 230 | 231 | to_ident = idents[ random.randint( 0, len(idents)-1 ) ] 232 | 233 | mes = [ "key-{0}".format( incr_key ), 234 | "incr-{i} try-{n_try}".format( i=i, n_try=n_try ), 235 | "req to:", to_ident, 236 | "with ver:", cur_ver, 237 | ] 238 | 239 | try: 240 | b = request_ex( "get", to_ident, { "key":incr_key } ) 241 | 242 | remote_ver, remote_val = b[ 'ver' ], b.get('val') 243 | if remote_ver < cur_ver: 244 | # unfinished commit might be seen, 245 | continue 246 | 247 | if remote_ver >= cur_ver: 248 | # get might see uncommitted value. thus version might 249 | # not be seen in future read 250 | 251 | if remote_val == i + 1: 252 | dd( mes, "unfinished done", "get", b ) 253 | 254 | elif remote_val != i: 255 | dd( mes, "error: remote val is: {val}, i={i}, ver={ver}".format(val=remote_val, i=i, ver=remote_ver) ) 256 | sys.exit( 1 ) 257 | 258 | 259 | b = request_ex("set", to_ident, {"key":incr_key, "ver":cur_ver, "val":i+1}) 260 | dd( mes, "ok", "set", b ) 261 | 262 | cur_ver = b['ver'] 263 | 264 | 265 | b = request_ex( "read", to_ident, {"ver":b[ 'ver' ]} ) 266 | 267 | ver = b['ver'] 268 | vals = [ b['val'].get( x, 0 ) for x in idents ] 269 | total = sum(vals) 270 | 271 | dd( mes, "ver=", b['ver'], "total=", total, "vals=", *vals ) 272 | assert total == ver - 1, 'total == ver - 1: %d, %d' %( total, ver ) 273 | 274 | break 275 | 276 | except socket.error as e: 277 | pass 278 | 279 | except PaxosError as e: 280 | dd( mes, "err", e.Code, e.Message ) 281 | 282 | if e.Code == 'VerNotExist' and cur_ver < e.Message: 283 | cur_ver = e.Message 284 | dd( mes, 'refreshed ver to', cur_ver ) 285 | 286 | randsleep() 287 | 288 | except Exception as e: 289 | 290 | dd( mes, "err", repr(e) ) 291 | 292 | monkeysess = { 'enabled': True } 293 | 294 | def monkey(sess): 295 | 296 | if not monkeysess[ 'enabled' ]: 297 | return 298 | 299 | stat = dict( [ (x, True) for x in sess['idents'] ] ) 300 | 301 | while sess[ 'running' ]: 302 | 303 | ident = sess['idents'][ random.randint( 0, len(sess['idents'])-1 ) ] 304 | 305 | try: 306 | if stat[ ident ]: 307 | ngx_stop( ident ) 308 | os.write( 1, 'nginx stopped: ' + ident + '\n' ) 309 | stat[ ident ] = False 310 | else: 311 | ngx_start( ident ) 312 | os.write( 1, 'nginx started: ' + ident + '\n' ) 313 | stat[ ident ] = True 314 | 315 | randsleep() 316 | 317 | except Exception as e: 318 | os.write( 1, repr( e ) + ' while nginx operation: ' + ident + '\n' ) 319 | 320 | 321 | def concurrency_test(): 322 | ngx_restart('123') 323 | init_sto() 324 | 325 | idents = [ x for x in '123' ] 326 | nthread = 5 327 | nincr = 500 328 | nmonkey = 1 329 | 330 | for ident in idents: 331 | body = { "ver":1, 332 | "val": { 333 | "1":0, 334 | "2":0, 335 | "3":0, 336 | "view": [ { 337 | "1":"1", 338 | "2":"2", 339 | "3":"3", }, ], 340 | } 341 | } 342 | request( 'phase3', ident, body ) 343 | 344 | ths = [] 345 | for ident in idents: 346 | th = threading.Thread( target=incr_worker, args=( ident, idents, nincr ) ) 347 | th.daemon = True 348 | th.start() 349 | 350 | ths.append( th ) 351 | 352 | sess = { 'running':True, "idents":idents, 353 | 'locks': dict( [ (x, threading.RLock()) for x in idents ] ) 354 | } 355 | 356 | monkeys = [] 357 | for ii in range( nmonkey ): 358 | monkey_th = threading.Thread( target=monkey, args=( sess, ) ) 359 | monkey_th.daemon = True 360 | monkey_th.start() 361 | monkeys.append( monkey_th ) 362 | 363 | for th in ths: 364 | while th.is_alive(): 365 | th.join(0.1) 366 | 367 | sess[ 'running' ] = False 368 | for th in monkeys: 369 | th.join() 370 | 371 | if __name__ == "__main__": 372 | import it.ngxconf 373 | it.ngxconf.make_conf(3) 374 | 375 | integration_test() 376 | 377 | monkeysess[ 'enabled' ] = True 378 | concurrency_test() 379 | 380 | monkeysess[ 'enabled' ] = False 381 | concurrency_test() 382 | -------------------------------------------------------------------------------- /srv/.gitignore: -------------------------------------------------------------------------------- 1 | /bin/ 2 | /nginx/ 3 | /luajit/ 4 | /lualib/ 5 | /ngx_openresty-1.7.4.1/Makefile 6 | /ngx_openresty-1.7.4.1/build/ 7 | -------------------------------------------------------------------------------- /srv/inst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | path=$(pwd) 4 | nn=ngx_openresty-1.7.4.1 5 | 6 | ( 7 | cd $nn \ 8 | && ./configure --prefix=$path && make && make install 9 | ) \ 10 | && ( 11 | cd nginx \ 12 | && rm -rf *_temp html conf/* 13 | ) 14 | -------------------------------------------------------------------------------- /ut: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | luacov="/opt/local/share/luarocks//lib/luarocks/rocks/luacov/0.6-1/bin/luacov" 4 | 5 | case $1 in 6 | cov) 7 | cov_flag="-l luacov" 8 | ;; 9 | *) 10 | cov_flag= 11 | ;; 12 | esac 13 | 14 | export LUA_CPATH="$LUA_CPATH;$(pwd)/clib/?.so;" 15 | 16 | ( 17 | cd lib || exit 1 18 | 19 | lua $cov_flag -l acid.unittest || exit 1 20 | 21 | if [ ".$cov_flag" != "." ]; then 22 | $luacov && tail -n20 luacov.report.out 23 | fi 24 | 25 | ) && echo ok || { echo fail; exit 1; } 26 | --------------------------------------------------------------------------------