├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README ├── concoord ├── Makefile ├── __init__.py ├── asyncclientproxy.py ├── batchclientproxy.py ├── blockingclientproxy.py ├── clientproxy.py ├── codegen.py ├── connection.py ├── enums.py ├── exception.py ├── logdaemon.py ├── main.py ├── message.py ├── nameserver.py ├── node.py ├── object │ ├── __init__.py │ ├── bank.py │ ├── barrier.py │ ├── binarytree.py │ ├── boundedsemaphore.py │ ├── condition.py │ ├── counter.py │ ├── jobmanager.py │ ├── lock.py │ ├── log.py │ ├── membership.py │ ├── meshmembership.py │ ├── nameservercoord.py │ ├── pinger.py │ ├── queue.py │ ├── rlock.py │ ├── semaphore.py │ └── stack.py ├── openreplica │ ├── __init__.py │ ├── main.py │ └── nodemanager.py ├── pack.py ├── proxy │ ├── __init__.py │ ├── bank.py │ ├── barrier.py │ ├── binarytree.py │ ├── boundedsemaphore.py │ ├── condition.py │ ├── counter.py │ ├── jobmanager.py │ ├── lock.py │ ├── log.py │ ├── membership.py │ ├── meshmembership.py │ ├── nameservercoord.py │ ├── queue.py │ ├── rlock.py │ ├── semaphore.py │ ├── stack.py │ └── test.py ├── proxygenerator.py ├── pvalue.py ├── replica.py ├── responsecollector.py ├── route53.py ├── safetychecker.py ├── test │ ├── __init__.py │ ├── test_partition.py │ ├── test_replicacomeback.py │ ├── test_replicacrashfailure.py │ └── test_timeout.py ├── threadingobject │ ├── __init__.py │ ├── dboundedsemaphore.py │ ├── dcondition.py │ ├── dlock.py │ ├── drlock.py │ └── dsemaphore.py └── utils.py ├── doc ├── Makefile ├── _build │ ├── doctrees │ │ ├── environment.pickle │ │ ├── index.doctree │ │ ├── install.doctree │ │ ├── openreplica.doctree │ │ ├── overview.doctree │ │ ├── tutorial-advanced.doctree │ │ └── tutorial.doctree │ └── html │ │ ├── .buildinfo │ │ ├── _images │ │ └── concoord.jpg │ │ ├── _sources │ │ ├── index.txt │ │ ├── install.txt │ │ ├── openreplica.txt │ │ ├── overview.txt │ │ ├── tutorial-advanced.txt │ │ └── tutorial.txt │ │ ├── _static │ │ ├── ajax-loader.gif │ │ ├── basic.css │ │ ├── comment-bright.png │ │ ├── comment-close.png │ │ ├── comment.png │ │ ├── concoord.jpg │ │ ├── default.css │ │ ├── doctools.js │ │ ├── down-pressed.png │ │ ├── down.png │ │ ├── file.png │ │ ├── jquery.js │ │ ├── minus.png │ │ ├── plus.png │ │ ├── pygments.css │ │ ├── searchtools.js │ │ ├── sidebar.js │ │ ├── underscore.js │ │ ├── up-pressed.png │ │ ├── up.png │ │ └── websupport.js │ │ ├── genindex.html │ │ ├── index.html │ │ ├── install.html │ │ ├── objects.inv │ │ ├── openreplica.html │ │ ├── overview.html │ │ ├── search.html │ │ ├── searchindex.js │ │ ├── tutorial-advanced.html │ │ └── tutorial.html └── source │ ├── _static │ └── concoord.jpg │ ├── conf.py │ ├── index.rst │ ├── install.rst │ ├── openreplica.rst │ ├── overview.rst │ ├── tutorial-advanced.rst │ └── tutorial.rst ├── docs ├── .gitignore ├── 404.html ├── CNAME ├── Gemfile ├── Gemfile.lock ├── _config.yml ├── code.markdown ├── index.markdown ├── paper.markdown └── static │ ├── altinbukenOpenReplica.pdf │ ├── concoord.jpg │ └── favicon.png ├── setup.cfg ├── setup.py └── update_site.sh /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | dist/ 3 | MANIFEST 4 | secret 5 | planetlab_packages 6 | credentials.py 7 | *.tar.gz 8 | *~ 9 | *.pyc 10 | *.tar 11 | *.zip 12 | *.tar.bz2 13 | concoord_log_* 14 | .DS_Store 15 | nohup.out 16 | *.egg-info 17 | concoordlog -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, Deniz Altinbuken, Emin Gun Sirer, Cornell University 2 | 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are 7 | met: 8 | 9 | * Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following 14 | disclaimer in the documentation and/or other materials provided 15 | with the distribution. 16 | 17 | * Neither the name of the ConCoord nor the names of its 18 | contributors may be used to endorse or promote products derived 19 | from this software without specific prior written permission. 20 | 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README 2 | include LICENSE -------------------------------------------------------------------------------- /concoord/Makefile: -------------------------------------------------------------------------------- 1 | SERVERFILES := *.py 2 | CLIENTFILES := connection.py blockingclientproxy.py batchclientproxy.py asyncclientproxy.py clientproxy.py pvalue.py message.py enums.py utils.py exception.py pack.py 3 | ALL := concoordserver.tar.gz 4 | 5 | all: $(ALL) 6 | 7 | concoordserver.zip:$(SERVERFILES) 8 | zip concoordserver.zip $(SERVERFILES) 9 | 10 | concoordserver.tar.gz:$(SERVERFILES) 11 | tar czf concoordserver.tar.gz $(SERVERFILES) 12 | 13 | tar:$(SERVERFILES) 14 | tar czf concoordserver.tar $(SERVERFILES) 15 | 16 | server:$(SERVERFILES) 17 | tar czf concoordserver.tar.gz $(SERVERFILES) 18 | 19 | client:$(CLIENTFILES) 20 | tar czf concoordclient.tar.gz $(CLIENTFILES) 21 | 22 | clean: 23 | rm -f *~ 24 | 25 | clobber: clean 26 | rm -f $(ALL) 27 | rm -f concoordserver.zip 28 | rm -f concoordserver.tar 29 | rm -f concoordserver.tar.gz 30 | rm -f concoordclient.tar.gz 31 | -------------------------------------------------------------------------------- /concoord/__init__.py: -------------------------------------------------------------------------------- 1 | """concoord toolkit""" 2 | -------------------------------------------------------------------------------- /concoord/clientproxy.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: ConCoord Client Proxy 4 | @copyright: See LICENSE 5 | ''' 6 | import os, random, select, socket, sys, threading, time 7 | from threading import Lock 8 | import cPickle as pickle 9 | from concoord.pack import * 10 | from concoord.enums import * 11 | from concoord.utils import * 12 | from concoord.exception import * 13 | from concoord.connection import ConnectionPool, Connection 14 | from concoord.message import * 15 | from concoord.pvalue import PValueSet 16 | try: 17 | import dns 18 | import dns.resolver 19 | import dns.exception 20 | except: 21 | print("Install dnspython: http://www.dnspython.org/") 22 | 23 | class ClientProxy(): 24 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 25 | self.debug = debug 26 | self.timeout = timeout 27 | self.domainname = None 28 | self.token = token 29 | self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 30 | self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1) 31 | self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) 32 | self.writelock = Lock() 33 | 34 | self.bootstraplist = self.discoverbootstrap(bootstrap) 35 | if len(self.bootstraplist) == 0: 36 | raise ConnectionError("No bootstrap found") 37 | if not self.connecttobootstrap(): 38 | raise ConnectionError("Cannot connect to any bootstrap") 39 | myaddr = findOwnIP() 40 | myport = self.socket.getsockname()[1] 41 | self.me = Peer(myaddr, myport, NODE_CLIENT) 42 | self.commandnumber = random.randint(1, sys.maxint) 43 | 44 | def _getipportpairs(self, bootaddr, bootport): 45 | for node in socket.getaddrinfo(bootaddr, bootport, socket.AF_INET, socket.SOCK_STREAM): 46 | yield (node[4][0],bootport) 47 | 48 | def getbootstrapfromdomain(self, domainname): 49 | tmpbootstraplist = [] 50 | try: 51 | answers = dns.resolver.query('_concoord._tcp.'+domainname, 'SRV') 52 | for rdata in answers: 53 | for peer in self._getipportpairs(str(rdata.target), rdata.port): 54 | if peer not in tmpbootstraplist: 55 | tmpbootstraplist.append(peer) 56 | except (dns.resolver.NXDOMAIN, dns.exception.Timeout): 57 | if self.debug: print "Cannot resolve name" 58 | return tmpbootstraplist 59 | 60 | def discoverbootstrap(self, givenbootstrap): 61 | tmpbootstraplist = [] 62 | try: 63 | for bootstrap in givenbootstrap.split(","): 64 | bootstrap = bootstrap.strip() 65 | # The bootstrap list is read only during initialization 66 | if bootstrap.find(":") >= 0: 67 | bootaddr,bootport = bootstrap.split(":") 68 | for peer in self._getipportpairs(bootaddr, int(bootport)): 69 | if peer not in tmpbootstraplist: 70 | tmpbootstraplist.append(peer) 71 | else: 72 | self.domainname = bootstrap 73 | tmpbootstraplist = self.getbootstrapfromdomain(self.domainname) 74 | except ValueError: 75 | if self.debug: print "bootstrap usage: ipaddr1:port1,ipaddr2:port2 or domainname" 76 | self._graceexit() 77 | return tmpbootstraplist 78 | 79 | def connecttobootstrap(self): 80 | connected = False 81 | for boottuple in self.bootstraplist: 82 | try: 83 | self.socket.close() 84 | self.socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM) 85 | self.socket.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) 86 | self.socket.connect(boottuple) 87 | self.conn = Connection(self.socket) 88 | self.conn.settimeout(CLIENTRESENDTIMEOUT) 89 | self.bootstrap = boottuple 90 | connected = True 91 | if self.debug: print "Connected to new bootstrap: ", boottuple 92 | break 93 | except socket.error, e: 94 | if self.debug: print "Socket.Error: ", e 95 | continue 96 | return connected 97 | 98 | def trynewbootstrap(self): 99 | if self.domainname: 100 | self.bootstraplist = self.getbootstrapfromdomain(self.domainname) 101 | else: 102 | oldbootstrap = self.bootstraplist.pop(0) 103 | self.bootstraplist.append(oldbootstrap) 104 | return self.connecttobootstrap() 105 | 106 | def invoke_command(self, *args): 107 | # create a request descriptor 108 | resend = True 109 | sendcount = -1 110 | lastreplycode = -1 111 | self.commandnumber += 1 112 | clientmsg = create_message(MSG_CLIENTREQUEST, self.me, 113 | {FLD_PROPOSAL: Proposal(self.me, self.commandnumber, args), 114 | FLD_TOKEN: self.token, 115 | FLD_CLIENTBATCH: False}) 116 | while True: 117 | sendcount += 1 118 | clientmsg[FLD_SENDCOUNT] = sendcount 119 | # send the clientrequest 120 | if resend: 121 | success = self.conn.send(clientmsg) 122 | if not success: 123 | self.reconfigure() 124 | continue 125 | resend = False 126 | # Receive reply 127 | try: 128 | for reply in self.conn.received_bytes(): 129 | if reply and reply.type == MSG_CLIENTREPLY: 130 | if reply.replycode == CR_OK: 131 | return reply.reply 132 | elif reply.replycode == CR_UNBLOCK: 133 | # actionable response, wake up the thread 134 | assert lastreplycode == CR_BLOCK, "unblocked thread not previously blocked" 135 | return reply.reply 136 | elif reply.replycode == CR_EXCEPTION: 137 | raise Exception(pickle.loads(reply.reply)) 138 | elif reply.replycode == CR_INPROGRESS or reply.replycode == CR_BLOCK: 139 | # the thread is already waiting, no need to do anything 140 | lastreplycode = reply.replycode 141 | # go wait for another message 142 | continue 143 | elif reply.replycode == CR_REJECTED or reply.replycode == CR_LEADERNOTREADY: 144 | resend = True 145 | self.reconfigure() 146 | continue 147 | else: 148 | print "Unknown Client Reply Code." 149 | except ConnectionError: 150 | resend = True 151 | self.reconfigure() 152 | continue 153 | except KeyboardInterrupt: 154 | self._graceexit() 155 | 156 | def reconfigure(self): 157 | if not self.trynewbootstrap(): 158 | raise ConnectionError("Cannot connect to any bootstrap") 159 | 160 | def _graceexit(self): 161 | os._exit(0) 162 | -------------------------------------------------------------------------------- /concoord/enums.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: This class holds enums that are widely used throughout the program 4 | Because it imports itself, this module MUST NOT HAVE ANY SIDE EFFECTS!!!! 5 | @copyright: See LICENSE 6 | ''' 7 | import enums 8 | 9 | # message types 10 | MSG_CLIENTREQUEST, MSG_CLIENTREPLY, MSG_INCCLIENTREQUEST, \ 11 | MSG_PREPARE, MSG_PREPARE_ADOPTED, MSG_PREPARE_PREEMPTED, MSG_PROPOSE, \ 12 | MSG_PROPOSE_ACCEPT, MSG_PROPOSE_REJECT, \ 13 | MSG_HELO, MSG_HELOREPLY, MSG_PING, MSG_PINGREPLY, \ 14 | MSG_UPDATE, MSG_UPDATEREPLY, \ 15 | MSG_PERFORM, MSG_RESPONSE, \ 16 | MSG_GARBAGECOLLECT, MSG_STATUS, MSG_ISSUE = range(20) 17 | 18 | # message fields 19 | FLD_ID, FLD_TYPE, FLD_SRC, FLD_BALLOTNUMBER, FLD_COMMANDNUMBER, \ 20 | FLD_PROPOSAL, FLD_DECISIONS, \ 21 | FLD_REPLY, FLD_REPLYCODE, FLD_INRESPONSETO, FLD_SNAPSHOT, FLD_PVALUESET, FLD_LEADER, \ 22 | FLD_TOKEN, FLD_CLIENTBATCH, FLD_SERVERBATCH, FLD_SENDCOUNT, FLD_DECISIONBALLOTNUMBER = range(18) 23 | 24 | # node types 25 | NODE_CLIENT, NODE_REPLICA, NODE_NAMESERVER = range(3) 26 | 27 | # error_types 28 | ERR_NOERROR, ERR_NOTLEADER, ERR_INITIALIZING = range(3) 29 | 30 | # nameserver service types 31 | NS_MASTER, NS_SLAVE, NS_ROUTE53 = range(1,4) 32 | 33 | # proxy types 34 | PR_BASIC, PR_BLOCK, PR_CBATCH, PR_SBATCH = range(4) 35 | 36 | # command result 37 | META = 'META' 38 | BLOCK = 'BLOCK' 39 | UNBLOCK = 'UNBLOCK' 40 | 41 | # executed indexing 42 | EXC_RCODE, EXC_RESULT, EXC_UNBLOCKED = range(3) 43 | 44 | # client reply codes 45 | CR_OK, CR_INPROGRESS, CR_LEADERNOTREADY, CR_REJECTED, \ 46 | CR_EXCEPTION, CR_BLOCK, CR_UNBLOCK, CR_META, CR_BATCH = range(9) 47 | 48 | # timeouts 49 | ACKTIMEOUT = 1 50 | LIVENESSTIMEOUT = 5 51 | NASCENTTIMEOUT = 20 * ACKTIMEOUT 52 | CLIENTRESENDTIMEOUT = 5 53 | BACKOFFDECREASETIMEOUT = 30 54 | TESTTIMEOUT = 1 55 | BOOTSTRAPCONNECTTIMEOUT = 60 56 | 57 | # ballot 58 | BALLOTEPOCH = 0 59 | BALLOTNO = 1 60 | BALLOTNODE = 2 61 | 62 | # backoff 63 | BACKOFFINCREASE = 0.1 64 | 65 | METACOMMANDS = set(["_add_node", "_del_node", "_garbage_collect"]) 66 | WINDOW = 10 67 | GARBAGEPERIOD = 100000 68 | 69 | NOOP = "do_noop" 70 | 71 | # UDPPACKETLEN 72 | UDPMAXLEN = 1024 73 | 74 | ########################### 75 | # code to convert enum variables to strings of different kinds 76 | 77 | # convert a set of enums with a given prefix into a dictionary 78 | def get_var_mappings(prefix): 79 | """Returns a dictionary with mappings""" 80 | return dict([(getattr(enums,varname),varname.replace(prefix, "", 1)) for varname in dir(enums) if varname.startswith(prefix)]) 81 | 82 | # convert a set of enums with a given prefix into a list 83 | def get_var_list(prefix): 84 | """Returns a list of enumnames""" 85 | return [name for (value,name) in sorted(get_var_mappings(prefix).iteritems())] 86 | 87 | msg_names = get_var_list("MSG_") 88 | node_names = get_var_list("NODE_") 89 | cmd_states = get_var_list("CMD_") 90 | err_types = get_var_list("ERR_") 91 | cr_codes = get_var_list("CR_") 92 | ns_services = get_var_list("NS_") 93 | -------------------------------------------------------------------------------- /concoord/exception.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Common ConCoord exceptions. 4 | @copyright: See LICENSE 5 | ''' 6 | class ConCoordException(Exception): 7 | """Abstract base class shared by all concoord exceptions""" 8 | def __init__(self, msg=''): 9 | self.msg = msg 10 | 11 | class Timeout(ConCoordException): 12 | """The operation timed out.""" 13 | def __init__(self, value=''): 14 | self.value = value 15 | 16 | def __str__(self): 17 | return str(self.value) 18 | 19 | class ConnectionError(ConCoordException): 20 | """Connection cannot be established""" 21 | def __init__(self, value=''): 22 | self.value = value 23 | 24 | def __str__(self): 25 | return str(self.value) 26 | 27 | class BlockingReturn(ConCoordException): 28 | """Blocking Return""" 29 | def __init__(self, returnvalue=None): 30 | self.returnvalue = returnvalue 31 | 32 | def __str__(self): 33 | return str(self.returnvalue) 34 | 35 | class UnblockingReturn(ConCoordException): 36 | """Unblocking Return""" 37 | def __init__(self, returnvalue=None, unblockeddict={}): 38 | self.returnvalue = returnvalue 39 | self.unblocked = unblockeddict 40 | 41 | def __str__(self): 42 | return str(self.returnvalue) + " ".join(unblockeddict.keys()) 43 | -------------------------------------------------------------------------------- /concoord/logdaemon.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: The Logger Daemon. Receives log messages and prints them. 4 | @copyright: See LICENSE 5 | """ 6 | import socket, time, os, sys, select 7 | from concoord.utils import * 8 | 9 | def collect_input(s): 10 | msg = '' 11 | while '\n' not in msg: 12 | chunk = s.recv(1) 13 | if chunk == '': 14 | return False 15 | msg += chunk 16 | print_event(msg) 17 | return True 18 | 19 | def print_event(event): 20 | print "%s: %s" % (time.asctime(time.localtime(time.time())), event.strip()) 21 | 22 | def main(): 23 | addr = findOwnIP() 24 | port = 12000 25 | try: 26 | daemonsocket = socket.socket(socket.AF_INET,socket.SOCK_STREAM) 27 | daemonsocket.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) 28 | daemonsocket.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1) 29 | daemonsocket.bind((addr,int(port))) 30 | daemonsocket.listen(10) 31 | except socket.error: 32 | pass 33 | print_event("server ready on port %d\n" % port) 34 | 35 | socketset = [daemonsocket] 36 | while True: 37 | inputready,outputready,exceptready = select.select(socketset,[],socketset,1) 38 | for s in inputready: 39 | try: 40 | if s == daemonsocket: 41 | clientsock,clientaddr = daemonsocket.accept() 42 | print_event("accepted a connection from address %s\n" % str(clientaddr)) 43 | socketset.append(clientsock) 44 | else: 45 | if not collect_input(s): 46 | socketset.remove(s) 47 | except socket.error: 48 | socketset.remove(s) 49 | for s in exceptready: 50 | socketset.remove(s) 51 | 52 | if __name__=='__main__': 53 | main() 54 | -------------------------------------------------------------------------------- /concoord/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | @author: Deniz Altinbuken, Emin Gun Sirer 4 | @note: concoord script 5 | @date: January 20, 2012 6 | @copyright: See LICENSE 7 | ''' 8 | import argparse 9 | import signal 10 | from time import sleep,time 11 | import os, sys, time, shutil 12 | import ast, _ast 13 | import concoord 14 | from concoord.enums import * 15 | from concoord.safetychecker import * 16 | from concoord.proxygenerator import * 17 | import ConfigParser 18 | 19 | HELPSTR = "concoord, version 1.1.0-release:\n\ 20 | concoord replica [-a address -p port -o objectname -b bootstrap -l loggeraddress -w writetodisk -d debug -n domainname -r route53] - starts a replica\n\ 21 | concoord route53id [aws_access_key_id] - adds AWS_ACCESS_KEY_ID to route53 CONFIG file\n\ 22 | concoord route53key [aws_secret_access_key] - adds AWS_SECRET_ACCESS_KEY to route53 CONFIG file\n\ 23 | concoord object [objectfilepath classname] - concoordifies a python object" 24 | 25 | ROUTE53CONFIGFILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'route53.cfg') 26 | config = ConfigParser.RawConfigParser() 27 | 28 | ## ROUTE53 29 | 30 | def touch_config_file(): 31 | with open(ROUTE53CONFIGFILE, 'a'): 32 | os.utime(ROUTE53CONFIGFILE, None) 33 | 34 | def read_config_file(): 35 | config.read(ROUTE53CONFIGFILE) 36 | section = 'ENVIRONMENT' 37 | options = ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'] 38 | rewritten = True 39 | if not config.has_section(section): 40 | rewritten = True 41 | config.add_section(section) 42 | for option in options: 43 | if not config.has_option(section, option): 44 | rewritten = True 45 | config.set(section, option, '') 46 | if rewritten: 47 | # Write to CONFIG file 48 | with open(ROUTE53CONFIGFILE, 'wb') as configfile: 49 | config.write(configfile) 50 | config.read(ROUTE53CONFIGFILE) 51 | awsid = config.get('ENVIRONMENT', 'AWS_ACCESS_KEY_ID') 52 | awskey = config.get('ENVIRONMENT', 'AWS_SECRET_ACCESS_KEY') 53 | return (awsid,awskey) 54 | 55 | def print_config_file(): 56 | print "AWS_ACCESS_KEY_ID= %s\nAWS_SECRET_ACCESS_KEY= %s" % read_config_file() 57 | 58 | def add_id_to_config(newid): 59 | awsid,awskey = read_config_file() 60 | if awsid and awsid == newid: 61 | print "AWS_ACCESS_KEY_ID is already in the CONFIG file." 62 | return 63 | # Write to CONFIG file 64 | config.set('ENVIRONMENT', 'AWS_ACCESS_KEY_ID', newid) 65 | with open(ROUTE53CONFIGFILE, 'wb') as configfile: 66 | config.write(configfile) 67 | 68 | def add_key_to_config(newkey): 69 | awsid,awskey = read_config_file() 70 | if awskey and awskey == newkey: 71 | print "AWS_SECRET_ACCESS_KEY is already in the CONFIG file." 72 | return 73 | # Write to CONFIG file 74 | config.set('ENVIRONMENT', 'AWS_SECRET_ACCESS_KEY', newkey) 75 | with open(ROUTE53CONFIGFILE, 'wb') as configfile: 76 | config.write(configfile) 77 | 78 | ## REPLICA 79 | 80 | def start_replica(): 81 | node = getattr(__import__('concoord.replica', globals(), locals(), -1), 'Replica')() 82 | node.startservice() 83 | signal.signal(signal.SIGINT, node.terminate_handler) 84 | signal.signal(signal.SIGTERM, node.terminate_handler) 85 | signal.pause() 86 | 87 | def check_object(clientcode): 88 | astnode = compile(clientcode,"","exec",_ast.PyCF_ONLY_AST) 89 | v = SafetyVisitor() 90 | v.visit(astnode) 91 | return v.safe 92 | 93 | def concoordify(): 94 | parser = argparse.ArgumentParser() 95 | parser.add_argument("-o", "--objectname", action="store", dest="objectname", default='', 96 | help="client object dotted name module.Class") 97 | parser.add_argument("-t", "--token", action="store", dest="securitytoken", default=None, 98 | help="security token") 99 | parser.add_argument("-p", "--proxytype", action="store", dest="proxytype", type=int, default=0, 100 | help="0:BASIC, 1:BLOCKING, 2:CLIENT-SIDE BATCHING, 3: SERVER-SIDE BATCHING ") 101 | parser.add_argument("-s", "--safe", action="store_true", dest="safe", default=False, 102 | help="safety checking on/off") 103 | parser.add_argument("-v", "--verbose", action="store_true", dest="verbose", default=None, 104 | help="verbose mode on/off") 105 | args = parser.parse_args() 106 | 107 | if not args.objectname: 108 | print parser.print_help() 109 | return 110 | import importlib 111 | objectloc,a,classname = args.objectname.rpartition('.') 112 | object = None 113 | try: 114 | module = importlib.import_module(objectloc) 115 | if hasattr(module, classname): 116 | object = getattr(module, classname)() 117 | except (ValueError, ImportError, AttributeError): 118 | print "Can't find module %s, check your PYTHONPATH." % objectloc 119 | 120 | if module.__file__.endswith('pyc'): 121 | filename = module.__file__[:-1] 122 | else: 123 | filename = module.__file__ 124 | with open(filename, 'rU') as fd: 125 | clientcode = fd.read() 126 | if args.safe: 127 | if args.verbose: 128 | print "Checking object safety" 129 | if not check_object(clientcode): 130 | print "Object is not safe to execute." 131 | os._exit(1) 132 | elif args.verbose: 133 | print "Object is safe!" 134 | if args.verbose: 135 | print "Creating clientproxy" 136 | clientproxycode = createclientproxy(clientcode, classname, args.securitytoken, args.proxytype) 137 | clientproxycode = clientproxycode.replace('\n\n\n', '\n\n') 138 | proxyfile = open(filename[:-3]+"proxy.py", 'w') 139 | proxyfile.write(clientproxycode) 140 | proxyfile.close() 141 | print "Client proxy file created with name: ", proxyfile.name 142 | 143 | 144 | def main(): 145 | if len(sys.argv) < 2: 146 | print HELPSTR 147 | sys.exit() 148 | 149 | eventtype = sys.argv[1].upper() 150 | sys.argv.pop(1) 151 | if eventtype == 'REPLICA': 152 | start_replica() 153 | elif eventtype == 'ROUTE53ID': 154 | print "Adding AWS_ACCESS_KEY_ID to CONFIG:", sys.argv[1] 155 | add_id_to_config(sys.argv[1]) 156 | elif eventtype == 'ROUTE53KEY': 157 | print "Adding AWS_SECRET_ACCESS_KEY to CONFIG:", sys.argv[1] 158 | add_key_to_config(sys.argv[1]) 159 | elif eventtype == 'INITIALIZE': 160 | initialize() 161 | elif eventtype == 'OBJECT': 162 | concoordify() 163 | else: 164 | print HELPSTR 165 | 166 | if __name__=='__main__': 167 | main() 168 | -------------------------------------------------------------------------------- /concoord/message.py: -------------------------------------------------------------------------------- 1 | from concoord.pack import * 2 | from concoord.enums import * 3 | from concoord.pvalue import * 4 | import msgpack 5 | from threading import Lock 6 | 7 | msgidpool = 0 8 | msgidpool_lock = Lock() 9 | 10 | def assignuniqueid(): 11 | global msgidpool 12 | global msgidpool_lock 13 | with msgidpool_lock: 14 | tempid = msgidpool 15 | msgidpool += 1 16 | return tempid 17 | 18 | def create_message(msgtype, src, msgfields={}): 19 | global msgidpool 20 | global msgidpool_lock 21 | 22 | m = msgfields 23 | m[FLD_ID] = assignuniqueid() 24 | m[FLD_TYPE] = msgtype 25 | m[FLD_SRC] = src 26 | return m 27 | 28 | def parse_basic(msg): 29 | src = Peer(msg[FLD_SRC][0], msg[FLD_SRC][1], msg[FLD_SRC][2]) 30 | return Message(msg[FLD_ID], msg[FLD_TYPE], src) 31 | 32 | def parse_status(msg): 33 | return Message(msg[FLD_ID], msg[FLD_TYPE], msg[FLD_SRC]) 34 | 35 | def parse_heloreply(msg): 36 | src = Peer(msg[FLD_SRC][0], msg[FLD_SRC][1], msg[FLD_SRC][2]) 37 | return HeloReplyMessage(msg[FLD_ID], msg[FLD_TYPE], 38 | src, Peer(msg[FLD_LEADER][0], 39 | msg[FLD_LEADER][1], 40 | msg[FLD_LEADER][2])) 41 | 42 | def parse_clientrequest(msg): 43 | src = Peer(msg[FLD_SRC][0], msg[FLD_SRC][1], msg[FLD_SRC][2]) 44 | if msg[FLD_CLIENTBATCH]: 45 | proposal = ProposalClientBatch(msg[FLD_PROPOSAL][0], 46 | msg[FLD_PROPOSAL][1], 47 | msg[FLD_PROPOSAL][2]) 48 | else: 49 | proposal = Proposal(msg[FLD_PROPOSAL][0], 50 | msg[FLD_PROPOSAL][1], 51 | msg[FLD_PROPOSAL][2]) 52 | 53 | return ClientRequestMessage(msg[FLD_ID], msg[FLD_TYPE], src, 54 | proposal, msg[FLD_TOKEN], 55 | msg[FLD_SENDCOUNT], msg[FLD_CLIENTBATCH]) 56 | 57 | def parse_clientreply(msg): 58 | src = Peer(msg[FLD_SRC][0], msg[FLD_SRC][1], msg[FLD_SRC][2]) 59 | return ClientReplyMessage(msg[FLD_ID], msg[FLD_TYPE], src, 60 | msg[FLD_REPLY], msg[FLD_REPLYCODE], 61 | msg[FLD_INRESPONSETO]) 62 | 63 | def parse_prepare(msg): 64 | src = Peer(msg[FLD_SRC][0], msg[FLD_SRC][1], msg[FLD_SRC][2]) 65 | return PrepareMessage(msg[FLD_ID], msg[FLD_TYPE], src, 66 | msg[FLD_BALLOTNUMBER]) 67 | 68 | def parse_prepare_reply(msg): 69 | src = Peer(msg[FLD_SRC][0], msg[FLD_SRC][1], msg[FLD_SRC][2]) 70 | pvalueset = PValueSet() 71 | for index,pvalue in msg[FLD_PVALUESET].iteritems(): 72 | pvalueset.pvalues[Proposal(index[1][0], 73 | index[1][1], 74 | index[1][2])] = PValue(pvalue[0], pvalue[1], pvalue[2]) 75 | return PrepareReplyMessage(msg[FLD_ID], msg[FLD_TYPE], src, 76 | msg[FLD_BALLOTNUMBER], msg[FLD_INRESPONSETO], 77 | pvalueset) 78 | def parse_propose(msg): 79 | src = Peer(msg[FLD_SRC][0], msg[FLD_SRC][1], msg[FLD_SRC][2]) 80 | if msg[FLD_SERVERBATCH]: 81 | proposal = ProposalServerBatch([]) 82 | for p in msg[FLD_PROPOSAL][0]: 83 | proposal.proposals.append(Proposal(p[0], p[1], p[2])) 84 | else: 85 | proposal = Proposal(msg[FLD_PROPOSAL][0], msg[FLD_PROPOSAL][1], msg[FLD_PROPOSAL][2]) 86 | return ProposeMessage(msg[FLD_ID], msg[FLD_TYPE], src, 87 | msg[FLD_BALLOTNUMBER], msg[FLD_COMMANDNUMBER], 88 | proposal, msg[FLD_SERVERBATCH]) 89 | 90 | 91 | def parse_propose_reply(msg): 92 | src = Peer(msg[FLD_SRC][0], msg[FLD_SRC][1], msg[FLD_SRC][2]) 93 | return ProposeReplyMessage(msg[FLD_ID], msg[FLD_TYPE], src, 94 | msg[FLD_BALLOTNUMBER], msg[FLD_INRESPONSETO], 95 | msg[FLD_COMMANDNUMBER]) 96 | 97 | def parse_perform(msg): 98 | src = Peer(msg[FLD_SRC][0], msg[FLD_SRC][1], msg[FLD_SRC][2]) 99 | if msg[FLD_SERVERBATCH]: 100 | proposal = ProposalServerBatch([]) 101 | for p in msg[FLD_PROPOSAL][0]: 102 | pclient = Peer(p[0][0], p[0][1], p[0][2]) 103 | proposal.proposals.append(Proposal(pclient, p[1], p[2])) 104 | elif msg[FLD_CLIENTBATCH]: 105 | proposalclient = Peer(msg[FLD_PROPOSAL][0][0], 106 | msg[FLD_PROPOSAL][0][1], 107 | msg[FLD_PROPOSAL][0][2]) 108 | proposal = ProposalClientBatch(proposalclient, msg[FLD_PROPOSAL][1], msg[FLD_PROPOSAL][2]) 109 | else: 110 | proposalclient = Peer(msg[FLD_PROPOSAL][0][0], 111 | msg[FLD_PROPOSAL][0][1], 112 | msg[FLD_PROPOSAL][0][2]) 113 | proposal = Proposal(proposalclient, msg[FLD_PROPOSAL][1], msg[FLD_PROPOSAL][2]) 114 | return PerformMessage(msg[FLD_ID], msg[FLD_TYPE], src, 115 | msg[FLD_COMMANDNUMBER], proposal, 116 | msg[FLD_SERVERBATCH], msg[FLD_CLIENTBATCH], 117 | msg[FLD_DECISIONBALLOTNUMBER]) 118 | 119 | def parse_response(msg): 120 | src = Peer(msg[FLD_SRC][0], msg[FLD_SRC][1], msg[FLD_SRC][2]) 121 | return Message(msg[FLD_ID], msg[FLD_TYPE], src) 122 | 123 | def parse_incclientrequest(msg): 124 | src = Peer(msg[FLD_SRC][0], msg[FLD_SRC][1], msg[FLD_SRC][2]) 125 | proposalclient = Peer(msg[FLD_PROPOSAL][0][0], msg[FLD_PROPOSAL][0][1], msg[FLD_PROPOSAL][0][2]) 126 | proposal = Proposal(proposalclient, msg[FLD_PROPOSAL][1], msg[FLD_PROPOSAL][2]) 127 | return ClientRequestMessage(msg[FLD_ID], msg[FLD_TYPE], src, 128 | proposal, msg[FLD_TOKEN]) 129 | 130 | def parse_updatereply(msg): 131 | src = Peer(msg[FLD_SRC][0], msg[FLD_SRC][1], msg[FLD_SRC][2]) 132 | for commandnumber,command in msg[FLD_DECISIONS].iteritems(): 133 | try: 134 | msg[FLD_DECISIONS][commandnumber] = Proposal(msg[FLD_DECISIONS][commandnumber][0], 135 | msg[FLD_DECISIONS][commandnumber][1], 136 | msg[FLD_DECISIONS][commandnumber][2]) 137 | except IndexError as i: 138 | msg[FLD_DECISIONS][commandnumber] = Proposal(msg[FLD_DECISIONS][commandnumber][0][0][0], 139 | msg[FLD_DECISIONS][commandnumber][0][0][1], 140 | msg[FLD_DECISIONS][commandnumber][0][0][2]) 141 | return UpdateReplyMessage(msg[FLD_ID], msg[FLD_TYPE], src, 142 | msg[FLD_DECISIONS]) 143 | 144 | def parse_garbagecollect(msg): 145 | src = Peer(msg[FLD_SRC][0], msg[FLD_SRC][1], msg[FLD_SRC][2]) 146 | return GarbageCollectMessage(msg[FLD_ID], msg[FLD_TYPE], src, 147 | msg[FLD_COMMANDNUMBER], msg[FLD_SNAPSHOT]) 148 | 149 | def parse_message(msg): 150 | return parse_functions[msg[FLD_TYPE]](msg) 151 | 152 | parse_functions = [ 153 | parse_clientrequest, # MSG_CLIENTREQUEST 154 | parse_clientreply, # MSG_CLIENTREPLY 155 | parse_incclientrequest, # MSG_INCCLIENTREQUEST 156 | 157 | parse_prepare, # MSG_PREPARE 158 | parse_prepare_reply, # MSG_PREPARE_ADOPTED 159 | parse_prepare_reply, # MSG_PREPARE_PREEMPTED 160 | parse_propose, # MSG_PROPOSE 161 | parse_propose_reply, # MSG_PROPOSE_ACCEPT 162 | parse_propose_reply, # MSG_PROPOSE_REJECT 163 | 164 | parse_basic, # MSG_HELO 165 | parse_heloreply, # MSG_HELOREPLY 166 | parse_basic, # MSG_PING 167 | parse_basic, # MSG_PINGREPLY 168 | 169 | parse_basic, # MSG_UPDATE 170 | parse_updatereply, # MSG_UPDATEREPLY 171 | 172 | parse_perform, # MSG_PERFORM 173 | parse_response, # MSG_RESPONSE 174 | 175 | parse_garbagecollect, # MSG_GARBAGECOLLECT 176 | parse_status, # MSG_STATUS 177 | parse_basic # MSG_ISSUE 178 | ] 179 | 180 | -------------------------------------------------------------------------------- /concoord/object/__init__.py: -------------------------------------------------------------------------------- 1 | """concoord objects""" 2 | 3 | -------------------------------------------------------------------------------- /concoord/object/bank.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example bank object that keeps track of accounts 4 | @copyright: See LICENSE 5 | """ 6 | class Bank(): 7 | def __init__(self): 8 | self.accounts = {} 9 | 10 | def open(self, accntno): 11 | if self.accounts.has_key(accntno): 12 | return False 13 | else: 14 | self.accounts[accntno] = Account(accntno) 15 | return True 16 | 17 | def close(self, accntno): 18 | if self.accounts.has_key(accntno): 19 | del self.accounts[accntno] 20 | return True 21 | else: 22 | raise KeyError 23 | 24 | def debit(self, accntno, amount): 25 | if self.accounts.has_key(accntno): 26 | return self.accounts[accntno].debit(amount) 27 | else: 28 | raise KeyError 29 | 30 | def deposit(self, accntno, amount): 31 | if self.accounts.has_key(accntno): 32 | return self.accounts[accntno].deposit(amount) 33 | else: 34 | raise KeyError 35 | 36 | def balance(self, accntno): 37 | if self.accounts.has_key(accntno): 38 | return self.accounts[accntno].balance 39 | else: 40 | raise KeyError 41 | 42 | def __str__(self): 43 | return "\n".join(["%s" % (str(account)) for account in self.accounts.values()]) 44 | 45 | class Account(): 46 | def __init__(self, number): 47 | self.number = number 48 | self.balance = 0 49 | 50 | def __str__(self): 51 | return "Account %s: balance = $%.2f" % (self.number, self.balance) 52 | 53 | def debit(self, amount): 54 | amount = float(amount) 55 | if amount >= self.balance: 56 | self.balance = self.balance - amount 57 | return self.balance 58 | else: 59 | return False 60 | 61 | def deposit(self, amount): 62 | amount = float(amount) 63 | self.balance = self.balance + amount 64 | return self.balance 65 | -------------------------------------------------------------------------------- /concoord/object/barrier.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example barrier object 4 | @copyright: See LICENSE 5 | """ 6 | from threading import Lock 7 | from concoord.threadingobject.dcondition import DCondition 8 | 9 | class Barrier(): 10 | def __init__(self, count=1): 11 | self.count = int(count) 12 | self.current = 0 13 | self.condition = DCondition() 14 | 15 | def wait(self, _concoord_command): 16 | self.condition.acquire(_concoord_command) 17 | self.current += 1 18 | if self.current != self.count: 19 | self.condition.wait(_concoord_command) 20 | else: 21 | self.current = 0 22 | self.condition.notifyAll(_concoord_command) 23 | self.condition.release(_concoord_command) 24 | 25 | def __str__(self): 26 | return "<%s object>" % (self.__class__.__name__) 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /concoord/object/binarytree.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example binarytree 4 | @copyright: See LICENSE 5 | """ 6 | class BinaryTree: 7 | def __init__(self): 8 | self.root = None 9 | 10 | def add_node(self, data): 11 | return Node(data) 12 | 13 | def insert(self, root, data): 14 | if root == None: 15 | return self.add_node(data) 16 | else: 17 | if data <= root.data: 18 | root.left = self.insert(root.left, data) 19 | else: 20 | root.right = self.insert(root.right, data) 21 | return root 22 | 23 | def find(self, root, target): 24 | if root == None: 25 | return False 26 | else: 27 | if target == root.data: 28 | return True 29 | else: 30 | if target < root.data: 31 | return self.find(root.left, target) 32 | else: 33 | return self.find(root.right, target) 34 | 35 | def delete(self, root, target): 36 | if root == None or not self.find(root, target): 37 | return False 38 | else: 39 | if target == root.data: 40 | del root 41 | else: 42 | if target < root.data: 43 | return self.delete(root.left, target) 44 | else: 45 | return self.delete(root.right, target) 46 | 47 | def get_min(self, root): 48 | while(root.left != None): 49 | root = root.left 50 | return root.data 51 | 52 | def get_max(self, root): 53 | while(root.right != None): 54 | root = root.right 55 | return root.data 56 | 57 | def get_depth(self, root): 58 | if root == None: 59 | return 0 60 | else: 61 | ldepth = self.get_depth(root.left) 62 | rdepth = self.get_depth(root.right) 63 | return max(ldepth, rdepth) + 1 64 | 65 | def get_size(self, root): 66 | if root == None: 67 | return 0 68 | else: 69 | return self.get_size(root.left) + 1 + self.get_size(root.right) 70 | 71 | class Node: 72 | def __init__(self, data): 73 | self.left = None 74 | self.right = None 75 | self.data = data 76 | 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /concoord/object/boundedsemaphore.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example boundedsemaphore object 4 | @copyright: See LICENSE 5 | """ 6 | from concoord.threadingobject.dboundedsemaphore import DBoundedSemaphore 7 | 8 | class BoundedSemaphore(): 9 | """Semaphore object that supports following functions: 10 | - acquire: locks the object 11 | - release: unlocks the object 12 | """ 13 | def __init__(self, count=1): 14 | self.semaphore = DBoundedSemaphore(count) 15 | 16 | def __repr__(self): 17 | return repr(self.semaphore) 18 | 19 | def acquire(self, _concoord_command): 20 | try: 21 | return self.semaphore.acquire(_concoord_command) 22 | except Exception as e: 23 | raise e 24 | 25 | def release(self, _concoord_command): 26 | try: 27 | return self.semaphore.release(_concoord_command) 28 | except Exception as e: 29 | raise e 30 | 31 | def __str__(self): 32 | return str(self.semaphore) 33 | -------------------------------------------------------------------------------- /concoord/object/condition.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example condition 4 | @copyright: See LICENSE 5 | """ 6 | from concoord.threadingobject.dcondition import DCondition 7 | 8 | class Condition(): 9 | def __init__(self, lock=None): 10 | self.condition = DCondition() 11 | 12 | def __repr__(self): 13 | return repr(self.condition) 14 | 15 | def acquire(self, _concoord_command): 16 | try: 17 | self.condition.acquire(_concoord_command) 18 | except Exception as e: 19 | raise e 20 | 21 | def release(self, _concoord_command): 22 | try: 23 | self.condition.release(_concoord_command) 24 | except Exception as e: 25 | raise e 26 | 27 | def wait(self, _concoord_command): 28 | try: 29 | self.condition.wait(_concoord_command) 30 | except Exception as e: 31 | raise e 32 | 33 | def notify(self, _concoord_command): 34 | try: 35 | self.condition.notify(_concoord_command) 36 | except Exception as e: 37 | raise e 38 | 39 | def notifyAll(self, _concoord_command): 40 | try: 41 | self.condition.notifyAll(_concoord_command) 42 | except Exception as e: 43 | raise e 44 | 45 | def __str__(self): 46 | return str(self.condition) 47 | -------------------------------------------------------------------------------- /concoord/object/counter.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example counter 4 | @copyright: See LICENSE 5 | """ 6 | class Counter: 7 | def __init__(self, value=0): 8 | self.value = value 9 | 10 | def decrement(self): 11 | self.value -= 1 12 | 13 | def increment(self): 14 | self.value += 1 15 | 16 | def getvalue(self): 17 | return self.value 18 | 19 | def __str__(self): 20 | return "The counter value is %d" % self.value 21 | -------------------------------------------------------------------------------- /concoord/object/jobmanager.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example jobmanager 4 | @copyright: See LICENSE 5 | """ 6 | class JobManager(): 7 | def __init__(self): 8 | self.jobs = [] 9 | 10 | def schedule(self, job): 11 | self.jobs.append(job) 12 | 13 | def deschedule(self, job): 14 | self.jobs.remove(job) 15 | 16 | def update(self, job, key, value): 17 | self.jobe[job].setattr(value) 18 | 19 | def list_jobs(self): 20 | return self.jobs 21 | 22 | def __str__(self): 23 | return " ".join([str(j) for j in self.jobs]) 24 | 25 | class Job(): 26 | def __init__(self, jobname, jobid, jobtime): 27 | self.name = jobname 28 | self.id = jobid 29 | self.time = jobtime 30 | 31 | def __str__(self): 32 | return "Job %s: %s @ %s" % (str(job.id), str(job.name), str(job.time)) 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /concoord/object/lock.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example lock 4 | @copyright: See LICENSE 5 | """ 6 | from concoord.threadingobject.dlock import DLock 7 | 8 | class Lock(): 9 | """Lock object that supports following functions: 10 | - acquire: locks the object 11 | - release: unlocks the object 12 | """ 13 | def __init__(self): 14 | self.lock = DLock() 15 | 16 | def __repr__(self): 17 | return repr(self.lock) 18 | 19 | def acquire(self, _concoord_command): 20 | try: 21 | return self.lock.acquire(_concoord_command) 22 | except Exception as e: 23 | raise e 24 | 25 | def release(self, _concoord_command): 26 | try: 27 | self.lock.release(_concoord_command) 28 | except Exception as e: 29 | raise e 30 | 31 | def __str__(self): 32 | return str(self.lock) 33 | -------------------------------------------------------------------------------- /concoord/object/log.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example log 4 | @copyright: See LICENSE 5 | """ 6 | class Log(): 7 | def __init__(self): 8 | self.log = [] 9 | 10 | def write(self, entry): 11 | self.log = [] 12 | self.log.append(entry) 13 | 14 | def append(self, entry): 15 | self.log.append(entry) 16 | 17 | def read(self): 18 | return self.__str__() 19 | 20 | def __str__(self): 21 | return " ".join([str(e) for e in self.log]) 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /concoord/object/membership.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example membership object 4 | @copyright: See LICENSE 5 | """ 6 | class Membership(): 7 | def __init__(self): 8 | self.members = set() 9 | 10 | def add(self, member): 11 | if member not in self.members: 12 | self.members.add(member) 13 | 14 | def remove(self, member): 15 | if member in self.members: 16 | self.members.remove(member) 17 | else: 18 | raise KeyError(member) 19 | 20 | def __str__(self): 21 | return " ".join([str(m) for m in self.members]) 22 | -------------------------------------------------------------------------------- /concoord/object/meshmembership.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Membership object to coordinate a complete mesh 4 | @copyright: See LICENSE 5 | """ 6 | from threading import RLock 7 | from concoord.exception import * 8 | from concoord.threadingobject.drlock import DRLock 9 | 10 | class MeshMembership(): 11 | def __init__(self): 12 | self.groups = {} 13 | 14 | def get_group_members(self, gname): 15 | if gname in self.groups: 16 | return self.groups[gname].get_members().keys() 17 | else: 18 | raise KeyError(gname) 19 | 20 | def get_group_epoch(self, gname): 21 | if gname in self.groups: 22 | return self.groups[gname].get_epoch() 23 | else: 24 | raise KeyError(gname) 25 | 26 | def get_group_state(self, gname): 27 | if gname in self.groups: 28 | return (self.groups[gname].get_members().keys(), self.groups[gname].get_epoch()) 29 | else: 30 | raise KeyError(gname) 31 | 32 | def add_group(self, gname, minsize): 33 | if gname not in self.groups: 34 | self.groups[gname] = Group(minsize) 35 | 36 | def remove_group(self, gname): 37 | if gname in self.groups: 38 | del self.groups[gname] 39 | else: 40 | raise KeyError(gname) 41 | 42 | def approve_join(self, gname, node, epochno): 43 | if gname in self.groups: 44 | group = self.groups[gname] 45 | # Check if the epoch the node wants to be 46 | # added to is still the current epoch. 47 | success = False 48 | if group.get_epoch() == epochno: 49 | # Update the epoch and members 50 | group.inc_epoch() 51 | group.add_member(node) 52 | # Let other members know 53 | group.notifyAll() 54 | success = True 55 | return (success, group.get_epoch()) 56 | else: 57 | raise KeyError(gname) 58 | 59 | def wait(self, gname): 60 | if gname in self.groups: 61 | return self.groups[gname].wait(_concoord_command) 62 | else: 63 | raise KeyError(gname) 64 | 65 | def check_member(self, gname, node): 66 | # returns True or False and the epoch number 67 | if gname in self.groups: 68 | return (node in self.groups[gname].get_members(), self.groups[gname].get_epoch()) 69 | else: 70 | raise KeyError(gname) 71 | 72 | def notify_failure(self, gname, epoch, failednode): 73 | if gname in self.groups: 74 | # there is a failure in the group or at least 75 | # one node thinks so. take a record of it 76 | if self.groups[gname].get_epoch() != epoch: 77 | return (self.groups[gname].get_members(), self.groups[gname].get_epoch()) 78 | self.groups[gname].get_members()[failednode] += 1 79 | if self.groups[gname].get_members()[failednode] >= len(self.groups[gname].get_members())/2.0: 80 | # more than half of the nodes think that a node has failed 81 | # we'll change the view 82 | self.groups[gname].remove_member(node) 83 | self.groups[gname].inc_epoch() 84 | # notify nodes that are waiting 85 | self.groups[gname].notifyAll() 86 | return (self.groups[gname].get_members(), self.groups[gname].get_epoch()) 87 | else: 88 | raise KeyError(gname) 89 | 90 | def __str__(self): 91 | return "\n".join([str(n)+': '+str(s) for n,s in self.groups.iteritems()]) 92 | 93 | class Group(): 94 | def __init__(self, minsize): 95 | # Coordination 96 | self._epoch = 1 97 | self._minsize = int(minsize) 98 | self._members = {} # Keeps nodename:strikecount 99 | 100 | # Note: This is not a normal Condition object, it doesn't have 101 | # a lock and it doesn't provide synchronization. 102 | self.__waiters = [] # This will always include self.members.keys() wait commands 103 | self.__atomic = RLock() 104 | 105 | def wait(self, _concoord_command): 106 | # put the caller on waitinglist and take the lock away 107 | with self.__atomic: 108 | self.__waiters.append(_concoord_command) 109 | raise BlockingReturn() 110 | 111 | # This function is used only by the Coordination Object 112 | def notifyAll(self): 113 | # Notify every client on the wait list 114 | with self.__atomic: 115 | if not self.__waiters: 116 | return 117 | unblocked = {} 118 | for waitcommand in self.__waiters: 119 | # notified client should be added to the lock queue 120 | unblocked[waitcommand] = True 121 | self.__waiters = [] 122 | raise UnblockingReturn(unblockeddict=unblocked) 123 | 124 | def add_member(self, member): 125 | if member not in self._members: 126 | self._members[member] = 0 127 | return True 128 | else: 129 | return False 130 | 131 | def remove_member(self, member): 132 | if member in self._members: 133 | del self._members[member] 134 | else: 135 | raise KeyError(member) 136 | 137 | def get_size(self): 138 | return self._minsize 139 | 140 | def get_epoch(self): 141 | return self._epoch 142 | 143 | def get_members(self): 144 | return self._members 145 | 146 | def inc_epoch(self): 147 | self._epoch += 1 148 | 149 | def __str__(self): 150 | t = "Epoch: %d " % self._epoch 151 | t += "Minimum Size: %d " % self._minsize 152 | t += "Members: " 153 | t += " ".join([str(m) for m in self._members.keys()]) 154 | return t 155 | -------------------------------------------------------------------------------- /concoord/object/nameservercoord.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Nameserver coordination object that keeps subdomains and their corresponding nameservers 4 | @copyright: See LICENSE 5 | ''' 6 | from itertools import izip 7 | 8 | def pairwise(iterable): 9 | a = iter(iterable) 10 | return izip(a, a) 11 | 12 | class NameserverCoord(): 13 | def __init__(self): 14 | self._nodes = {} 15 | 16 | def addnodetosubdomain(self, subdomain, nodetype, node): 17 | nodetype = int(nodetype) 18 | if subdomain.find('openreplica') == -1: 19 | subdomain = subdomain+'.openreplica.org.' 20 | if subdomain in self._nodes: 21 | if nodetype in self._nodes[subdomain]: 22 | self._nodes[subdomain][nodetype].add(node) 23 | else: 24 | self._nodes[subdomain][nodetype] = set() 25 | self._nodes[subdomain][nodetype].add(node) 26 | else: 27 | self._nodes[subdomain] = {} 28 | self._nodes[subdomain][nodetype] = set() 29 | self._nodes[subdomain][nodetype].add(node) 30 | 31 | def delsubdomain(self, subdomain): 32 | if subdomain.find('openreplica') == -1: 33 | subdomain = subdomain+'.openreplica.org.' 34 | exists = subdomain in self._nodes 35 | if exists: 36 | del self._nodes[subdomain] 37 | return exists 38 | 39 | def delnodefromsubdomain(self, subdomain, nodetype, node): 40 | if subdomain.find('openreplica') == -1: 41 | subdomain = subdomain+'.openreplica.org.' 42 | nodetype = int(nodetype) 43 | exists = subdomain in self._nodes and nodetype in self._nodes[subdomain] and node in self._nodes[subdomain][nodetype] 44 | if exists: 45 | self._nodes[subdomain][nodetype].remove(node) 46 | return exists 47 | 48 | def updatesubdomain(self, subdomain, nodes): 49 | if subdomain.find('openreplica') == -1: 50 | subdomain = subdomain+'.openreplica.org.' 51 | if subdomain in self._nodes.keys(): 52 | self._nodes[subdomain] = nodes 53 | else: 54 | self._nodes[subdomain] = set() 55 | self._nodes[subdomain] = nodes 56 | 57 | def getnodes(self, subdomain): 58 | if subdomain.find('openreplica') == -1: 59 | subdomain = subdomain+'.openreplica.org.' 60 | return self._nodes[subdomain] 61 | 62 | def getsubdomains(self): 63 | subdomains = [] 64 | for domain in self._nodes.keys(): 65 | subdomains.append(domain.split('.')[0]) 66 | return subdomains 67 | 68 | def getdomains(self): 69 | return self._nodes.keys() 70 | 71 | def _reinstantiate(self, state): 72 | self._nodes = {} 73 | for subdomain,nodes in pairwise(state.split(';')): 74 | self._nodes[subdomain] = {} 75 | nodestypes = nodes.strip("()").split('--') 76 | for typeofnode in nodestypes: 77 | if typeofnode: 78 | typename = int(typeofnode.split('-')[0]) 79 | self._nodes[subdomain][typename] = set() 80 | nodelist = typeofnode.split('-')[1] 81 | for nodename in nodelist.split(): 82 | self._nodes[subdomain][typename].add(nodename) 83 | 84 | def __str__(self): 85 | rstr = '' 86 | for domain,nodes in self._nodes.iteritems(): 87 | if domain.find('openreplica') == -1: 88 | continue 89 | rstr += domain + ';(' 90 | for nodetype, nodes in nodes.iteritems(): 91 | if len(nodes) > 0: 92 | rstr += str(nodetype) + '-' + ' '.join(nodes) + "--" 93 | rstr += ');' 94 | return rstr 95 | -------------------------------------------------------------------------------- /concoord/object/pinger.py: -------------------------------------------------------------------------------- 1 | import socket, time 2 | from threading import RLock, Thread 3 | from concoord.exception import * 4 | from concoord.threadingobject.dcondition import DCondition 5 | 6 | MSG_PING = 8 7 | PING_DELAY = 10 8 | 9 | class Pinger(): 10 | def __init__(self): 11 | self.members = set() 12 | self.membership_condition = DCondition() 13 | 14 | self.liveness = {} 15 | 16 | self.socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM) 17 | self.socket.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) 18 | myaddr = socket.gethostbyname(socket.gethostname()) 19 | myport = self.socket.getsockname()[1] 20 | self.me = "%s:%d", (myaddr,myport) 21 | 22 | comm_thread = Thread(target=self.ping_members, name='PingThread') 23 | comm_thread.start() 24 | 25 | def add(self, member): 26 | if member not in self.members: 27 | self.members.add(member) 28 | self.liveness[member] = 0 29 | # Notify nodes of membership change 30 | self.membership_condition.notifyAll() 31 | 32 | def remove(self, member): 33 | if member in self.members: 34 | self.members.remove(member) 35 | del self.liveness[member] 36 | else: 37 | raise KeyError(member) 38 | # Notify nodes of membership change 39 | self.membership_condition.notifyAll() 40 | 41 | def get_members(self): 42 | return self.members 43 | 44 | def ping_members(self): 45 | while True: 46 | for member in self.members: 47 | print "Sending PING to %s" % str(member) 48 | pingmessage = PingMessage(self.me) 49 | success = self.send(pingmessage, peer=peer) 50 | if success < 0: 51 | print "Node not responding, marking." 52 | self.liveness[member] += 1 53 | 54 | time.sleep(PING_DELAY) 55 | 56 | def send(self, msg): 57 | messagestr = pickle.dumps(msg) 58 | message = struct.pack("I", len(messagestr)) + messagestr 59 | try: 60 | while len(message) > 0: 61 | try: 62 | bytesent = self.thesocket.send(message) 63 | message = message[bytesent:] 64 | except IOError, e: 65 | if isinstance(e.args, tuple): 66 | if e[0] == errno.EAGAIN: 67 | continue 68 | else: 69 | raise e 70 | except AttributeError, e: 71 | raise e 72 | return True 73 | except socket.error, e: 74 | if isinstance(e.args, tuple): 75 | if e[0] == errno.EPIPE: 76 | print "Remote disconnect" 77 | return False 78 | except IOError, e: 79 | print "Send Error: ", e 80 | except AttributeError, e: 81 | print "Socket deleted." 82 | return False 83 | 84 | def __str__(self): 85 | return " ".join([str(m) for m in self.members]) 86 | 87 | class PingMessage(): 88 | def __init__(self, srcname): 89 | self.type = MSG_PING 90 | self.source = srcname 91 | 92 | def __str__(self): 93 | return 'PingMessage from %s' % str(self.source) 94 | -------------------------------------------------------------------------------- /concoord/object/queue.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example queue 4 | @copyright: See LICENSE 5 | """ 6 | import Queue 7 | 8 | class Queue: 9 | def __init__(self, maxsize=0): 10 | self.queue = Queue.Queue(maxsize) 11 | 12 | def qsize(self): 13 | return self.queue.qsize() 14 | 15 | def empty(self): 16 | return self.queue.empty() 17 | 18 | def full(self): 19 | return self.queue.full() 20 | 21 | def put(self, item): 22 | return self.queue.put_nowait(item) 23 | 24 | def get(self): 25 | return self.queue.get_nowait() 26 | 27 | def __str__(self): 28 | return str(self.queue) 29 | -------------------------------------------------------------------------------- /concoord/object/rlock.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example rlock object 4 | @copyright: See LICENSE 5 | """ 6 | from concoord.threadingobject.drlock import DRLock 7 | 8 | class RLock(): 9 | def __init__(self): 10 | self.rlock = DRLock() 11 | 12 | def __repr__(self): 13 | return repr(self.rlock) 14 | 15 | def acquire(self, _concoord_command): 16 | try: 17 | return self.rlock.acquire(_concoord_command) 18 | except Exception as e: 19 | raise e 20 | 21 | def release(self, _concoord_command): 22 | try: 23 | self.rlock.release(_concoord_command) 24 | except Exception as e: 25 | raise e 26 | 27 | def __str__(self): 28 | return str(self.rlock) 29 | -------------------------------------------------------------------------------- /concoord/object/semaphore.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example semaphore object 4 | @copyright: See LICENSE 5 | """ 6 | from concoord.threadingobject.dsemaphore import DSemaphore 7 | 8 | class Semaphore(): 9 | def __init__(self, count=1): 10 | self.semaphore = DSemaphore(count) 11 | 12 | def __repr__(self): 13 | return repr(self.semaphore) 14 | 15 | def acquire(self, _concoord_command): 16 | try: 17 | return self.semaphore.acquire(_concoord_command) 18 | except Exception as e: 19 | raise e 20 | 21 | def release(self, _concoord_command): 22 | try: 23 | return self.semaphore.release(_concoord_command) 24 | except Exception as e: 25 | raise e 26 | 27 | def __str__(self): 28 | return str(self.semaphore) 29 | -------------------------------------------------------------------------------- /concoord/object/stack.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Example stack 4 | @copyright: See LICENSE 5 | """ 6 | class Stack: 7 | def __init__(self): 8 | self.stack = [] 9 | 10 | def append(self, item): 11 | self.stack.append(item) 12 | 13 | def pop(self): 14 | self.stack.pop() 15 | 16 | def get_size(self): 17 | return len(self.stack) 18 | 19 | def get_stack(self): 20 | return self.stack 21 | 22 | def __str__(self): 23 | return self.stack 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /concoord/openreplica/__init__.py: -------------------------------------------------------------------------------- 1 | """openreplica toolkit""" 2 | 3 | -------------------------------------------------------------------------------- /concoord/openreplica/nodemanager.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: NodeManager that handles remote operations. 4 | @copyright: See LICENSE 5 | ''' 6 | from threading import Thread 7 | import sys, os, socket, os.path 8 | import random, time 9 | import subprocess, signal 10 | 11 | VERSION = '1.1.0' 12 | 13 | class NodeManager(): 14 | def __init__(self, nodes, sshkey, username): 15 | self.username = username 16 | # username 17 | if not self.username: 18 | print "There is no username. Add username." 19 | return 20 | # instances 21 | if nodes: 22 | self.instances = nodes.split(',') 23 | else: 24 | self.instances = [] 25 | return 26 | # key-pair filename 27 | self.sshkey = sshkey 28 | if self.sshkey: 29 | home = os.path.expanduser("~") 30 | cmd = ['ssh-add', home+'/.ssh/' + self.sshkey] 31 | self._waitforall([self._issuecommand(cmd)]) 32 | else: 33 | print "There is no sshkey filename. Add sshkey filename." 34 | return 35 | self.alive = True 36 | 37 | def _tryconnect(self, host): 38 | cmd = ["ssh", host, "ls"] 39 | return self._system(cmd, timeout=20) == 0 40 | 41 | def _system(self, cmd, timeout=300): 42 | p = self._issuecommand(cmd) 43 | return self._waitforall([p], timeout) 44 | 45 | def _issuecommand(self, cmd): 46 | return subprocess.Popen(cmd) 47 | 48 | def _waitforall(self, pipelist, timeout=300): 49 | start = time.time() 50 | while len(pipelist) > 0: 51 | todo = [] 52 | for pipe in pipelist: 53 | rtv = pipe.poll() 54 | if rtv is None: 55 | # not done 56 | todo.append(pipe) 57 | if len(todo) > 0: 58 | time.sleep(0.1) 59 | now = time.time() 60 | if now - start > timeout: 61 | # timeout reached 62 | for p in todo: 63 | os.kill(p.pid, signal.SIGKILL) 64 | os.waitpid(p.pid, os.WNOHANG) 65 | return -len(todo) 66 | pipelist = todo 67 | return 0 68 | 69 | def executecommandall(self, command, wait=True): 70 | pipedict = {} 71 | for node in self.nodes: 72 | cmd = ["ssh", self.host, command] 73 | pipe = self._issuecommand(cmd) 74 | if wait: 75 | pipedict[pipe] = pipe.communicate() 76 | if wait: 77 | return (self._waitforall(pipedict.keys()) == 0, pipedict) 78 | return 79 | 80 | def executecommandone(self, node, command, wait=True): 81 | cmd = ["ssh", host, command] 82 | pipe = self._issuecommand(cmd) 83 | if wait: 84 | pipe.poll() 85 | stdout,stderr = pipe.communicate() 86 | return (self._waitforall([pipe]) == 0, (stdout,stderr)) 87 | return pipe 88 | 89 | def download(self, remote, local): 90 | cmd = "scp " + self.host + ":" + remote + " "+ local 91 | rtv = os.system(cmd) 92 | return rtv 93 | 94 | def upload(self, host, node, local, remote=""): 95 | """upload concoord src to amazon instance""" 96 | cmd = ["scp", local, host + ":" + remote] 97 | if os.path.isdir(local): 98 | cmd.insert(1, "-r") 99 | pipe = self._issuecommand(cmd) 100 | return self._waitforall([pipe]) == 0 101 | 102 | def __str__(self): 103 | rstr = "Username: " + self.username + '\n' 104 | rstr += "SSH key file: " + self.sshkey + '\n' 105 | rstr += "Instances:\n" 106 | rstr += '\n'.join(self.instances) 107 | return rstr 108 | -------------------------------------------------------------------------------- /concoord/pack.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken 3 | @note: Tuples used by ConCoord 4 | @copyright: See LICENSE 5 | """ 6 | from collections import namedtuple 7 | 8 | Proposal = namedtuple('Proposal', ['client', 'clientcommandnumber', 'command']) 9 | ProposalClientBatch = namedtuple('ProposalClientBatch', ['client', 'clientcommandnumber', 'command']) 10 | ProposalServerBatch = namedtuple('ProposalServerBatch', ['proposals']) 11 | 12 | class Peer(namedtuple('Peer', ['addr', 'port', 'type'])): 13 | __slots__ = () 14 | def __str__(self): 15 | return str((self.addr, self.port, self.type)) 16 | 17 | PValue = namedtuple('PValue', ['ballotnumber', 'commandnumber', 'proposal']) 18 | Message = namedtuple('Message', ['id', 'type', 'source']) 19 | IssueMessage = namedtuple('IssueMessage', ['id', 'type', 'source']) 20 | StatusMessage = namedtuple('StatusMessage', ['id', 'type', 'source']) 21 | HeloReplyMessage = namedtuple('HeloReplyMessage', ['id', 'type', 'source', 'leader']) 22 | UpdateReplyMessage = namedtuple('UpdateReplyMessage', ['id', 'type', 'source', 'decisions']) 23 | PrepareMessage = namedtuple('PrepareMessage', ['id', 'type', 'source', 'ballotnumber']) 24 | PrepareReplyMessage = namedtuple('PrepareReplyMessage', ['id', 'type','source', 25 | 'ballotnumber', 'inresponseto', 26 | 'pvalueset']) 27 | ProposeMessage = namedtuple('ProposeMessage', ['id', 'type','source', 28 | 'ballotnumber', 'commandnumber', 29 | 'proposal', 'serverbatch']) 30 | ProposeReplyMessage = namedtuple('ProposeReplyMessage', ['id', 'type','source', 31 | 'ballotnumber', 'inresponseto', 32 | 'commandnumber']) 33 | PerformMessage = namedtuple('PerformMessage', ['id', 'type','source', 34 | 'commandnumber', 'proposal', 35 | 'serverbatch', 'clientbatch', 36 | 'decisionballotnumber']) 37 | 38 | ClientRequestMessage = namedtuple('ClientRequestMessage', ['id', 'type', 'source', 39 | 'command', 'token', 40 | 'sendcount', 'clientbatch']) 41 | 42 | ClientReplyMessage = namedtuple('ClientReplyMessage', ['id', 'type', 'source', 43 | 'reply', 'replycode', 'inresponseto']) 44 | 45 | GarbageCollectMessage = namedtuple('GarbageCollectMessage', ['id', 'type', 'source', 46 | 'commandnumber', 'snapshot']) 47 | -------------------------------------------------------------------------------- /concoord/proxy/__init__.py: -------------------------------------------------------------------------------- 1 | """concoord proxies""" 2 | 3 | -------------------------------------------------------------------------------- /concoord/proxy/bank.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Bank proxy 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.clientproxy import ClientProxy 7 | class Bank: 8 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 9 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 10 | 11 | def __concoordinit__(self): 12 | return self.proxy.invoke_command('__init__') 13 | 14 | def open(self, accntno): 15 | return self.proxy.invoke_command('open', accntno) 16 | 17 | def close(self, accntno): 18 | return self.proxy.invoke_command('close', accntno) 19 | 20 | def debit(self, accntno, amount): 21 | return self.proxy.invoke_command('debit', accntno, amount) 22 | 23 | def deposit(self, accntno, amount): 24 | return self.proxy.invoke_command('deposit', accntno, amount) 25 | 26 | def balance(self, accntno): 27 | return self.proxy.invoke_command('balance', accntno) 28 | 29 | def __str__(self): 30 | return self.proxy.invoke_command('__str__') 31 | 32 | -------------------------------------------------------------------------------- /concoord/proxy/barrier.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Barrier proxy 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.blockingclientproxy import ClientProxy 7 | 8 | class Barrier: 9 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 10 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 11 | 12 | def __concoordinit__(self, count=1): 13 | return self.proxy.invoke_command('__init__', count) 14 | 15 | def wait(self): 16 | return self.proxy.invoke_command('wait') 17 | 18 | def __str__(self): 19 | return self.proxy.invoke_command('__str__') 20 | -------------------------------------------------------------------------------- /concoord/proxy/binarytree.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Binarytree proxy 4 | @copyright: See LICENSE 5 | """ 6 | from concoord.clientproxy import ClientProxy 7 | class BinaryTree: 8 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 9 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 10 | 11 | def __concoordinit__(self): 12 | return self.proxy.invoke_command('__init__') 13 | 14 | def add_node(self, data): 15 | return self.proxy.invoke_command('add_node', data) 16 | 17 | def insert(self, root, data): 18 | return self.proxy.invoke_command('insert', root, data) 19 | 20 | def find(self, root, target): 21 | return self.proxy.invoke_command('find', root, target) 22 | 23 | def delete(self, root, target): 24 | return self.proxy.invoke_command('delete', root, target) 25 | 26 | def get_min(self, root): 27 | return self.proxy.invoke_command('get_min', root) 28 | 29 | def get_max(self, root): 30 | return self.proxy.invoke_command('get_max', root) 31 | 32 | def get_depth(self, root): 33 | return self.proxy.invoke_command('get_depth', root) 34 | 35 | def get_size(self, root): 36 | return self.proxy.invoke_command('get_size', root) 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /concoord/proxy/boundedsemaphore.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Bounded semaphore proxy 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.blockingclientproxy import ClientProxy 7 | 8 | class BoundedSemaphore: 9 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 10 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 11 | 12 | def __concoordinit__(self, count=1): 13 | return self.proxy.invoke_command('__init__', count) 14 | 15 | def __repr__(self): 16 | return self.proxy.invoke_command('__repr__') 17 | 18 | def acquire(self): 19 | return self.proxy.invoke_command('acquire') 20 | 21 | def release(self): 22 | return self.proxy.invoke_command('release') 23 | 24 | def __str__(self): 25 | return self.proxy.invoke_command('__str__') 26 | -------------------------------------------------------------------------------- /concoord/proxy/condition.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Condition proxy 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.clientproxy import ClientProxy 7 | 8 | class Condition: 9 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 10 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 11 | 12 | def __concoordinit__(self, lock=None): 13 | return self.proxy.invoke_command('__init__', lock) 14 | 15 | def __repr__(self): 16 | return self.proxy.invoke_command('__repr__') 17 | 18 | def acquire(self): 19 | return self.proxy.invoke_command('acquire') 20 | 21 | def release(self): 22 | return self.proxy.invoke_command('release') 23 | 24 | def wait(self): 25 | return self.proxy.invoke_command('wait') 26 | 27 | def notify(self): 28 | return self.proxy.invoke_command('notify') 29 | 30 | def notifyAll(self): 31 | return self.proxy.invoke_command('notifyAll') 32 | 33 | def __str__(self): 34 | return self.proxy.invoke_command('__str__') 35 | -------------------------------------------------------------------------------- /concoord/proxy/counter.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Counter proxy 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.clientproxy import ClientProxy 7 | 8 | class Counter: 9 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 10 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 11 | 12 | def __concoordinit__(self, value=0): 13 | return self.proxy.invoke_command('__init__', value) 14 | 15 | def decrement(self): 16 | return self.proxy.invoke_command('decrement') 17 | 18 | def increment(self): 19 | return self.proxy.invoke_command('increment') 20 | 21 | def getvalue(self): 22 | return self.proxy.invoke_command('getvalue') 23 | 24 | def __str__(self): 25 | return self.proxy.invoke_command('__str__') 26 | -------------------------------------------------------------------------------- /concoord/proxy/jobmanager.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Jobmanager proxy 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.clientproxy import ClientProxy 7 | 8 | class JobManager: 9 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 10 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 11 | 12 | def __concoordinit__(self): 13 | return self.proxy.invoke_command('__init__') 14 | 15 | def add_job(self, job): 16 | return self.proxy.invoke_command('add_job', job) 17 | 18 | def remove_job(self, job): 19 | return self.proxy.invoke_command('remove_job', job) 20 | 21 | def list_jobs(self): 22 | return self.proxy.invoke_command('list_jobs') 23 | 24 | def __str__(self): 25 | return self.proxy.invoke_command('__str__') 26 | 27 | class Job: 28 | def __init__(self, jobname, jobid, jobtime): 29 | self.name = jobname 30 | self.id = jobid 31 | self.time = jobtime 32 | 33 | def __str__(self): 34 | return 'Job %s: %s @ %s' % (str(job.id), str(job.name), str(job.time)) 35 | -------------------------------------------------------------------------------- /concoord/proxy/lock.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Lock proxy 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.blockingclientproxy import ClientProxy 7 | 8 | class Lock: 9 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 10 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 11 | 12 | def __concoordinit__(self): 13 | return self.proxy.invoke_command('__init__') 14 | 15 | def __repr__(self): 16 | return self.proxy.invoke_command('__repr__') 17 | 18 | def acquire(self): 19 | return self.proxy.invoke_command('acquire') 20 | 21 | def release(self): 22 | return self.proxy.invoke_command('release') 23 | 24 | def __str__(self): 25 | return self.proxy.invoke_command('__str__') 26 | -------------------------------------------------------------------------------- /concoord/proxy/log.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Log proxy 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.clientproxy import ClientProxy 7 | 8 | class Log: 9 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 10 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 11 | 12 | def __concoordinit__(self): 13 | return self.proxy.invoke_command('__init__') 14 | 15 | def write(self, entry): 16 | return self.proxy.invoke_command('write', entry) 17 | 18 | def append(self, entry): 19 | return self.proxy.invoke_command('append', entry) 20 | 21 | def read(self): 22 | return self.proxy.invoke_command('read') 23 | 24 | def __str__(self): 25 | return self.proxy.invoke_command('__str__') 26 | -------------------------------------------------------------------------------- /concoord/proxy/membership.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Membership proxy 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.clientproxy import ClientProxy 7 | 8 | class Membership: 9 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 10 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 11 | 12 | def __concoordinit__(self): 13 | return self.proxy.invoke_command('__init__') 14 | 15 | def add(self, member): 16 | return self.proxy.invoke_command('add', member) 17 | 18 | def remove(self, member): 19 | return self.proxy.invoke_command('remove', member) 20 | 21 | def __str__(self): 22 | return self.proxy.invoke_command('__str__') 23 | -------------------------------------------------------------------------------- /concoord/proxy/meshmembership.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: MeshMembership proxy 4 | @copyright: See LICENSE 5 | """ 6 | from concoord.blockingclientproxy import ClientProxy 7 | 8 | class MeshMembership(): 9 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 10 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 11 | 12 | def __concoordinit__(self): 13 | return self.proxy.invoke_command('__init__') 14 | 15 | def get_group_members(self, gname): 16 | return self.proxy.invoke_command('get_group_members', gname) 17 | 18 | def get_group_epoch(self, gname): 19 | return self.proxy.invoke_command('get_group_epoch', gname) 20 | 21 | def get_group_state(self, gname): 22 | return self.proxy.invoke_command('get_group_state', gname) 23 | 24 | def add_group(self, gname, minsize): 25 | return self.proxy.invoke_command('add_group', gname, minsize) 26 | 27 | def remove_group(self, gname): 28 | return self.proxy.invoke_command('remove_group', gname) 29 | 30 | def approve_join(self, gname, node, epochno): 31 | return self.proxy.invoke_command('approve_join', gname, node, epochno) 32 | 33 | def wait(self, gname): 34 | return self.proxy.invoke_command('wait', gname) 35 | 36 | def check_member(self, gname, node): 37 | return self.proxy.invoke_command('check_member', gname, node) 38 | 39 | def notify_failure(self, gname, epoch, failednode): 40 | return self.proxy.invoke_command('notify_failure', gname, epoch, failednode) 41 | 42 | def __str__(self): 43 | return self.proxy.invoke_command('__str__') 44 | -------------------------------------------------------------------------------- /concoord/proxy/nameservercoord.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Nameserver coordination object proxy 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.clientproxy import ClientProxy 7 | 8 | class NameserverCoord: 9 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 10 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 11 | 12 | def __concoordinit__(self): 13 | return self.proxy.invoke_command('__init__') 14 | 15 | def addnodetosubdomain(self, subdomain, nodetype, node): 16 | return self.proxy.invoke_command('addnodetosubdomain', subdomain, nodetype, node) 17 | 18 | def delsubdomain(self, subdomain): 19 | return self.proxy.invoke_command('delsubdomain', subdomain) 20 | 21 | def delnodefromsubdomain(self, subdomain, nodetype, node): 22 | return self.proxy.invoke_command('delnodefromsubdomain', subdomain, nodetype, node) 23 | 24 | def updatesubdomain(self, subdomain, nodes): 25 | return self.proxy.invoke_command('updatesubdomain', subdomain, nodes) 26 | 27 | def getnodes(self, subdomain): 28 | return self.proxy.invoke_command('getnodes', subdomain) 29 | 30 | def getsubdomains(self): 31 | return self.proxy.invoke_command('getsubdomains') 32 | 33 | def _reinstantiate(self, state): 34 | return self.proxy.invoke_command('_reinstantiate', state) 35 | 36 | def __str__(self): 37 | return self.proxy.invoke_command('__str__') 38 | -------------------------------------------------------------------------------- /concoord/proxy/queue.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Queue proxy 4 | @copyright: See LICENSE 5 | """ 6 | from concoord.clientproxy import ClientProxy 7 | class Queue: 8 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 9 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 10 | 11 | def __concoordinit__(self): 12 | return self.proxy.invoke_command('__init__') 13 | 14 | def append(self, item): 15 | return self.proxy.invoke_command('append', item) 16 | 17 | def remove(self): 18 | return self.proxy.invoke_command('remove') 19 | 20 | def get_size(self): 21 | return self.proxy.invoke_command('get_size') 22 | 23 | def get_queue(self): 24 | return self.proxy.invoke_command('get_queue') 25 | 26 | def __str__(self): 27 | return self.proxy.invoke_command('__str__') 28 | -------------------------------------------------------------------------------- /concoord/proxy/rlock.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: RLock proxy 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.blockingclientproxy import ClientProxy 7 | 8 | class RLock: 9 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 10 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 11 | 12 | def __concoordinit__(self): 13 | return self.proxy.invoke_command('__init__') 14 | 15 | def __repr__(self): 16 | return self.proxy.invoke_command('__repr__') 17 | 18 | def acquire(self): 19 | return self.proxy.invoke_command('acquire') 20 | 21 | def release(self): 22 | return self.proxy.invoke_command('release') 23 | 24 | def __str__(self): 25 | return self.proxy.invoke_command('__str__') 26 | -------------------------------------------------------------------------------- /concoord/proxy/semaphore.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Semaphore proxy 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.blockingclientproxy import ClientProxy 7 | 8 | class Semaphore: 9 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 10 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 11 | 12 | def __concoordinit__(self, count=1): 13 | return self.proxy.invoke_command('__init__', count) 14 | 15 | def acquire(self): 16 | return self.proxy.invoke_command('acquire') 17 | 18 | def release(self): 19 | return self.proxy.invoke_command('release') 20 | 21 | def __str__(self): 22 | return self.proxy.invoke_command('__str__') 23 | -------------------------------------------------------------------------------- /concoord/proxy/stack.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Stack proxy 4 | @copyright: See LICENSE 5 | """ 6 | from concoord.clientproxy import ClientProxy 7 | class Stack: 8 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 9 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 10 | 11 | def __concoordinit__(self): 12 | return self.proxy.invoke_command('__init__') 13 | 14 | def append(self, item): 15 | return self.proxy.invoke_command('append', item) 16 | 17 | def pop(self): 18 | return self.proxy.invoke_command('pop') 19 | 20 | def get_size(self): 21 | return self.proxy.invoke_command('get_size') 22 | 23 | def get_stack(self): 24 | return self.proxy.invoke_command('get_stack') 25 | 26 | def __str__(self): 27 | return self.proxy.invoke_command('__str__') 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /concoord/proxy/test.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Value object proxy to test concoord implementation 4 | @copyright: See LICENSE 5 | """ 6 | from concoord.clientproxy import ClientProxy 7 | 8 | class Test(): 9 | def __init__(self, bootstrap, timeout=60, debug=False, token=None): 10 | self.proxy = ClientProxy(bootstrap, timeout, debug, token) 11 | 12 | def __concoordinit__(self): 13 | return self.proxy.invoke_command('__init__') 14 | 15 | def getvalue(self): 16 | return self.proxy.invoke_command('getvalue') 17 | 18 | def setvalue(self, newvalue): 19 | return self.proxy.invoke_command('setvalue', newvalue) 20 | 21 | def __str__(self): 22 | return self.proxy.invoke_command('__str__') 23 | 24 | -------------------------------------------------------------------------------- /concoord/proxygenerator.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Proxy Generator that creates ConCoord proxy files from regular Python objects. 4 | @copyright: See LICENSE 5 | ''' 6 | import codegen 7 | import ast, _ast 8 | import os, shutil 9 | import inspect, types, string 10 | from concoord.enums import PR_BASIC, PR_BLOCK, PR_CBATCH, PR_SBATCH 11 | 12 | class ProxyGen(ast.NodeTransformer): 13 | def __init__(self, objectname, securitytoken=None, proxytype=0): 14 | self.objectname = objectname 15 | self.classdepth = 0 16 | self.token = securitytoken 17 | self.proxytype = proxytype 18 | 19 | def generic_visit(self, node): 20 | ast.NodeTransformer.generic_visit(self, node) 21 | return node 22 | 23 | def visit_Module(self, node): 24 | if self.proxytype == PR_BASIC: 25 | importstmt = compile("from concoord.clientproxy import ClientProxy","","exec",_ast.PyCF_ONLY_AST).body[0] 26 | elif self.proxytype == PR_BLOCK: 27 | importstmt = compile("from concoord.blockingclientproxy import ClientProxy","","exec",_ast.PyCF_ONLY_AST).body[0] 28 | elif self.proxytype == PR_CBATCH: 29 | importstmt = compile("from concoord.batchclientproxy import ClientProxy","","exec",_ast.PyCF_ONLY_AST).body[0] 30 | elif self.proxytype == PR_SBATCH: 31 | importstmt = compile("from concoord.asyncclientproxy import ClientProxy","","exec",_ast.PyCF_ONLY_AST).body[0] 32 | node.body.insert(0, importstmt) 33 | return self.generic_visit(node) 34 | 35 | def visit_Import(self, node): 36 | return self.generic_visit(node) 37 | 38 | def visit_ImportFrom(self, node): 39 | return self.generic_visit(node) 40 | 41 | def visit_ClassDef(self, node): 42 | selectedclass = node.name == self.objectname 43 | if selectedclass or self.classdepth: 44 | self.classdepth += 1 45 | if self.classdepth == 1: 46 | for item in node.body: 47 | if type(item) == _ast.FunctionDef and item.name == "__init__": 48 | #node.body.remove(item) 49 | item.name = "__concoordinit__" 50 | # Add the new init method 51 | initfunc = compile("def __init__(self, bootstrap):\n" 52 | "\tself.proxy = ClientProxy(bootstrap, token=\"%s\")" % self.token, 53 | "", 54 | "exec", 55 | _ast.PyCF_ONLY_AST).body[0] 56 | node.body.insert(0, initfunc) 57 | ret = self.generic_visit(node) 58 | if selectedclass or self.classdepth: 59 | self.classdepth -= 1 60 | return ret 61 | 62 | def visit_FunctionDef(self, node): 63 | if self.classdepth == 1: 64 | for i in node.args.args: 65 | if i.id == '_concoord_command': 66 | node.args.args.remove(i) 67 | if node.name == "__init__": 68 | pass 69 | else: 70 | args = ["\'"+ node.name +"\'"]+[i.id for i in node.args.args[1:]] 71 | node.body = compile("return self.proxy.invoke_command(%s)" % ", ".join(args), 72 | "", 73 | "exec", 74 | _ast.PyCF_ONLY_AST).body 75 | return node 76 | else: 77 | return self.generic_visit(node) 78 | 79 | def createclientproxy(clientcode, objectname, securitytoken, proxytype=PR_BASIC, bootstrap=None): 80 | # Get the AST tree, transform it, convert back to string 81 | originalast = compile(clientcode, "", "exec", _ast.PyCF_ONLY_AST) 82 | newast = ProxyGen(objectname, securitytoken, proxytype).visit(originalast) 83 | return codegen.to_source(newast) 84 | 85 | -------------------------------------------------------------------------------- /concoord/pvalue.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: PValue is used to keep Paxos state in Acceptor and Leader nodes. 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.pack import * 7 | import types 8 | 9 | class PValueSet(): 10 | """PValueSet encloses a set of pvalues with the highest ballotnumber (always) 11 | and supports corresponding set functions. 12 | """ 13 | def __init__(self): 14 | self.pvalues = {} # indexed by (commandnumber,proposal): pvalue 15 | 16 | def add(self, pvalue): 17 | """Adds given PValue to the PValueSet overwriting matching 18 | (commandnumber,proposal) if it exists and has a smaller ballotnumber 19 | """ 20 | if isinstance(pvalue.proposal, ProposalServerBatch): 21 | # list of Proposals cannot be hashed, cast them to tuple 22 | index = (pvalue.commandnumber,tuple(pvalue.proposal.proposals)) 23 | else: 24 | index = (pvalue.commandnumber,pvalue.proposal) 25 | 26 | if self.pvalues.has_key(index): 27 | if self.pvalues[index].ballotnumber < pvalue.ballotnumber: 28 | self.pvalues[index] = pvalue 29 | else: 30 | self.pvalues[index] = pvalue 31 | 32 | def remove(self, pvalue): 33 | """Removes given pvalue""" 34 | if isinstance(pvalue.proposal, ProposalServerBatch): 35 | # list of Proposals cannot be hashed, cast them to tuple 36 | index = (pvalue.commandnumber,tuple(pvalue.proposal.proposals)) 37 | else: 38 | index = (pvalue.commandnumber,pvalue.proposal) 39 | del self.pvalues[index] 40 | 41 | def truncateto(self, commandnumber): 42 | """Truncates the history up to given commandnumber""" 43 | keytuples = self.pvalues.keys() 44 | allkeys = sorted(keytuples, key=lambda keytuple: keytuple[0]) 45 | # Sanity checking 46 | lastkey = allkeys[0][0] 47 | candelete = True 48 | for (cmdno,proposal) in allkeys: 49 | if cmdno == lastkey: 50 | lastkey += 1 51 | else: 52 | candelete = False 53 | break 54 | # Truncating 55 | if not candelete: 56 | return False 57 | for (cmdno,proposal) in allkeys: 58 | if cmdno < commandnumber: 59 | del self.pvalues[(cmdno,proposal)] 60 | return True 61 | 62 | def union(self, otherpvalueset): 63 | """Unionizes the pvalues of givenPValueSet with the pvalues of the 64 | PValueSet overwriting the (commandnumber,proposal) pairs with lower 65 | ballotnumber 66 | """ 67 | for candidate in otherpvalueset.pvalues.itervalues(): 68 | self.add(candidate) 69 | 70 | def pmax(self): 71 | """Returns a mapping from command numbers to proposals with the highest ballotnumbers""" 72 | pmaxresult = {} 73 | for (commandnumber,proposal) in self.pvalues.keys(): 74 | pmaxresult[commandnumber] = proposal 75 | return pmaxresult 76 | 77 | def __len__(self): 78 | """Returns the number of PValues in the PValueSet""" 79 | return len(self.pvalues) 80 | 81 | def __str__(self): 82 | """Returns PValueSet information""" 83 | return "\n".join(str(pvalue) for pvalue in self.pvalues.itervalues()) 84 | -------------------------------------------------------------------------------- /concoord/responsecollector.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Class used to collect responses to both PREPARE and PROPOSE messages 4 | @copyright: See LICENSE 5 | ''' 6 | from concoord.pvalue import PValueSet 7 | from concoord.pack import PValue 8 | 9 | class ResponseCollector(): 10 | """ResponseCollector keeps the state related to both MSG_PREPARE and 11 | MSG_PROPOSE. 12 | """ 13 | def __init__(self, replicas, ballotnumber, commandnumber, proposal): 14 | """ResponseCollector state 15 | - ballotnumber: ballotnumber for the corresponding msg 16 | - commandnumber: commandnumber for the corresponding msg 17 | - proposal: proposal for the corresponding msg 18 | - quorum: group of replica nodes for the corresponding msg 19 | - sent: msgids for the messages that have been sent 20 | - received: dictionary that keeps mappings 21 | - ntotal: # of replica nodes for the corresponding msg 22 | - nquorum: # of accepts needed for success 23 | - possiblepvalueset: Set of pvalues collected from replicas 24 | """ 25 | self.ballotnumber = ballotnumber 26 | self.commandnumber = commandnumber 27 | self.proposal = proposal 28 | self.quorum = replicas 29 | self.receivedcount = 0 30 | self.receivedfrom = set() 31 | self.ntotal = len(self.quorum) 32 | self.nquorum = self.ntotal/2+1 33 | self.possiblepvalueset = PValueSet() 34 | -------------------------------------------------------------------------------- /concoord/safetychecker.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Checks the safety of the client object 4 | @copyright: See LICENSE 5 | ''' 6 | import ast, _ast 7 | DEBUG = False 8 | 9 | blacklist = ["open","setattr","getattr","compile","exec","eval","execfile", "globals", "type"] 10 | 11 | class SafetyVisitor(ast.NodeVisitor): 12 | def __init__(self): 13 | self.safe = True 14 | self.classes = [] 15 | 16 | def generic_visit(self, node): 17 | if DEBUG: 18 | print "---", ast.dump(node) 19 | ast.NodeVisitor.generic_visit(self, node) 20 | 21 | def visit_Import(self, node): 22 | self.safe = False 23 | print "%d | No imports allowed: %s --> EXIT" % (node.lineno,node.names[0].name) 24 | 25 | def visit_ImportFrom(self, node): 26 | self.safe = False 27 | print "%d | No imports allowed: %s --> EXIT" % (node.lineno,node.module) 28 | 29 | def visit_Exec(self, node): 30 | self.safe = False 31 | print "%d | Exec not allowed --> EXIT" % node.lineno 32 | 33 | def visit_Call(self, node): 34 | if DEBUG: 35 | print 'Call : ' 36 | self.check_functioncall(node) 37 | self.generic_visit(node) 38 | 39 | def visit_ClassDef(self, node): 40 | if DEBUG: 41 | print 'ClassDef : ', node.name 42 | self.classes.append(node.name) 43 | self.generic_visit(node) 44 | 45 | def visit_Name(self, node): 46 | if DEBUG: 47 | print 'Name :', node.id 48 | 49 | def visit_Num(self, node): 50 | if DEBUG: 51 | print 'Num :', node.__dict__['n'] 52 | 53 | def visit_Str(self, node): 54 | if DEBUG: 55 | print "Str :", node.s 56 | 57 | def visit_Print(self, node): 58 | if DEBUG: 59 | print "Print :" 60 | self.generic_visit(node) 61 | 62 | def visit_Assign(self, node): 63 | if DEBUG: 64 | print "Assign :" 65 | self.check_assignment(node) 66 | self.generic_visit(node) 67 | 68 | def visit_Expr(self, node): 69 | if DEBUG: 70 | print "Expr :" 71 | self.generic_visit(node) 72 | 73 | def check_assignment(self, node): 74 | if DEBUG: 75 | print "Checking assignment.." 76 | global blacklist 77 | if type(node.value).__name__ == 'Name': 78 | if node.value.id in blacklist: 79 | self.safe = False 80 | print "%d | Function assignment: %s --> EXIT" % (node.lineno,node.value.id) 81 | 82 | def check_functioncall(self, node): 83 | if DEBUG: 84 | print "Checking function call.." 85 | global blacklist 86 | isopen = False 87 | for fname,fvalue in ast.iter_fields(node): 88 | if DEBUG: 89 | print fname, " ", fvalue 90 | if fname == 'func' and type(fvalue).__name__ == 'Name': 91 | if fvalue.id == 'open': 92 | isopen = True 93 | elif fvalue.id in blacklist: 94 | self.safe = False 95 | print "%d | Forbidden function call: %s --> EXIT" % (node.lineno,fvalue.id) 96 | if fname == 'args' and isopen: 97 | for arg in fvalue: 98 | if type(arg).__name__ == 'Str': 99 | if arg.__dict__['s'] == 'w' or arg.__dict__['s'] == 'a': 100 | self.safe = False 101 | print "%d | Write to file --> EXIT" % node.lineno 102 | elif type(arg).__name__ == 'Name': 103 | self.safe = False 104 | print "%d | File operation with variable argument: %s --> EXIT" % (node.lineno,arg.id) 105 | 106 | def main(): 107 | path = "./safetytest.py" 108 | astnode = compile(open(path, 'rU').read(),"","exec",_ast.PyCF_ONLY_AST) 109 | v = SafetyVisitor() 110 | v.visit(astnode) 111 | 112 | if __name__=='__main__': 113 | main() 114 | 115 | -------------------------------------------------------------------------------- /concoord/test/__init__.py: -------------------------------------------------------------------------------- 1 | """concoord tests""" 2 | -------------------------------------------------------------------------------- /concoord/test/test_replicacomeback.py: -------------------------------------------------------------------------------- 1 | # Create 5 replicas and kill them in turn, until 1 is left 2 | # Every time kill the leader 3 | # Then bring the leaders back in the reverse order, forcing 4 handovers. 4 | import signal, time 5 | import subprocess 6 | import time 7 | from concoord.proxy.counter import Counter 8 | 9 | class TimeoutException(Exception): 10 | pass 11 | 12 | def timeout(timeout): 13 | def timeout_function(f): 14 | def f2(*args): 15 | def timeout_handler(signum, frame): 16 | raise TimeoutException() 17 | 18 | old_handler = signal.signal(signal.SIGALRM, timeout_handler) 19 | signal.alarm(timeout) # trigger alarm in timeout seconds 20 | try: 21 | retval = f(*args) 22 | except TimeoutException: 23 | return False 24 | finally: 25 | signal.signal(signal.SIGALRM, old_handler) 26 | signal.alarm(0) 27 | return retval 28 | return f2 29 | return timeout_function 30 | 31 | @timeout(120) 32 | def test_failure(numreplicas): 33 | replicas = [] 34 | replicanames = [] 35 | 36 | print "Running replica 0" 37 | replicas.append(subprocess.Popen(['concoord', 'replica', 38 | '-o', 'concoord.object.counter.Counter', 39 | '-a', '127.0.0.1', '-p', '14000'])) 40 | replicanames.append("127.0.0.1:14000") 41 | 42 | for i in range(1, numreplicas): 43 | print "Running replica %d" %i 44 | replicas.append(subprocess.Popen(['concoord', 'replica', 45 | '-o', 'concoord.object.counter.Counter', 46 | '-a', '127.0.0.1', '-p', '1400%d'%i, 47 | '-b', '127.0.0.1:14000'])) 48 | replicanames.append("127.0.0.1:1400%d"%i) 49 | 50 | # Give the system sometime to initialize 51 | time.sleep(10) 52 | 53 | replicastring = ','.join(replicanames) 54 | # Test Clientproxy operations 55 | c = Counter(replicastring) 56 | for i in range(50): 57 | c.increment() 58 | print "Counter value after 50 increments: %d" % c.getvalue() 59 | 60 | # Start kiling replicas 61 | print "Killing replicas one by one." 62 | for i in range((numreplicas-1)/2): 63 | print "Killing replica %d" %i 64 | replicas[i].kill() 65 | 66 | # Clientproxy operations should still work 67 | c = Counter('127.0.0.1:1400%d'%(i+1)) 68 | for i in range(50): 69 | c.increment() 70 | print "Counter value after 50 more increments: %d" % c.getvalue() 71 | 72 | # Start bringing replicas back 73 | for i in reversed(xrange((numreplicas-1)/2)): 74 | print "Running replica %d" %i 75 | replicas.append(subprocess.Popen(['concoord', 'replica', 76 | '-o', 'concoord.object.counter.Counter', 77 | '-a', '127.0.0.1', '-p', '1400%d'%i, 78 | '-b', '127.0.0.1:1400%d'%(i+1)])) 79 | time.sleep(10) 80 | # Clientproxy operations should still work 81 | connected = False 82 | while(not connected): 83 | try: 84 | c = Counter('127.0.0.1:1400%d'%i) 85 | except: 86 | continue 87 | connected = True 88 | 89 | for i in range(50): 90 | c.increment() 91 | print "Counter value after 50 more increments: %d" % c.getvalue() 92 | 93 | for p in (replicas): 94 | p.kill() 95 | return True 96 | 97 | def main(): 98 | print "===== TEST 3 REPLICAS =====" 99 | s = "PASSED" if test_failure(3) else "TIMED OUT" 100 | print "===== TEST %s =====" % s 101 | print "===== TEST 5 REPLICAS =====" 102 | s = "PASSED" if test_failure(5) else "TIMED OUT" 103 | print "===== TEST %s =====" % s 104 | print "===== TEST 7 REPLICAS =====" 105 | s = "PASSED" if test_failure(7) else "TIMED OUT" 106 | print "===== TEST %s =====" % s 107 | 108 | if __name__ == '__main__': 109 | main() 110 | -------------------------------------------------------------------------------- /concoord/test/test_replicacrashfailure.py: -------------------------------------------------------------------------------- 1 | # Create 5 replicas and kill them in turn, until 1 is left 2 | # Every time kill the leader 3 | import signal, time 4 | import subprocess 5 | import time 6 | from concoord.proxy.counter import Counter 7 | 8 | class TimeoutException(Exception): 9 | pass 10 | 11 | def timeout(timeout): 12 | def timeout_function(f): 13 | def f2(*args): 14 | def timeout_handler(signum, frame): 15 | raise TimeoutException() 16 | 17 | old_handler = signal.signal(signal.SIGALRM, timeout_handler) 18 | signal.alarm(timeout) # trigger alarm in timeout seconds 19 | try: 20 | retval = f(*args) 21 | except TimeoutException: 22 | return False 23 | finally: 24 | signal.signal(signal.SIGALRM, old_handler) 25 | signal.alarm(0) 26 | return retval 27 | return f2 28 | return timeout_function 29 | 30 | @timeout(100) 31 | def test_failure(numreplicas): 32 | replicas = [] 33 | replicanames = [] 34 | 35 | print "Running replica 0" 36 | replicas.append(subprocess.Popen(['concoord', 'replica', 37 | '-o', 'concoord.object.counter.Counter', 38 | '-a', '127.0.0.1', '-p', '14000'])) 39 | replicanames.append("127.0.0.1:14000") 40 | 41 | for i in range(1,numreplicas): 42 | print "Running replica %d" %i 43 | replicas.append(subprocess.Popen(['concoord', 'replica', 44 | '-o', 'concoord.object.counter.Counter', 45 | '-a', '127.0.0.1', '-p', '1400%d'%i, 46 | '-b', '127.0.0.1:14000'])) 47 | replicanames.append("127.0.0.1:1400%d"%i) 48 | 49 | # Give the system some time to initialize 50 | time.sleep(10) 51 | 52 | replicastring = ','.join(replicanames) 53 | # Test Clientproxy operations 54 | c = Counter(replicastring) 55 | for i in range(100): 56 | c.increment() 57 | print "Counter value after 100 increments: %d" % c.getvalue() 58 | 59 | # Start kiling replicas 60 | for i in range((numreplicas-1)/2): 61 | print "Killing replica %d" %i 62 | replicas[i].kill() 63 | 64 | # Clientproxy operations should still work 65 | for i in range(100): 66 | c.increment() 67 | print "Counter value after 100 more increments: %d" % c.getvalue() 68 | 69 | for p in (replicas): 70 | p.kill() 71 | return True 72 | 73 | def main(): 74 | print "===== TEST 3 REPLICAS =====" 75 | s = "PASSED" if test_failure(3) else "TIMED OUT" 76 | print "===== TEST %s =====" % s 77 | print "===== TEST 5 REPLICAS =====" 78 | s = "PASSED" if test_failure(5) else "TIMED OUT" 79 | print "===== TEST %s =====" % s 80 | print "===== TEST 7 REPLICAS =====" 81 | s = "PASSED" if test_failure(7) else "TIMED OUT" 82 | print "===== TEST %s =====" % s 83 | 84 | if __name__ == '__main__': 85 | main() 86 | -------------------------------------------------------------------------------- /concoord/test/test_timeout.py: -------------------------------------------------------------------------------- 1 | # Cuts the connection to the leader and tests liveness 2 | import sys,os 3 | import signal, time 4 | import subprocess 5 | import time 6 | from concoord.proxy.counter import Counter 7 | from concoord.exception import ConnectionError 8 | 9 | class TimeoutException(Exception): 10 | pass 11 | 12 | def timeout(timeout): 13 | def timeout_function(f): 14 | def f2(*args): 15 | def timeout_handler(signum, frame): 16 | raise TimeoutException() 17 | 18 | old_handler = signal.signal(signal.SIGALRM, timeout_handler) 19 | signal.alarm(timeout) # trigger alarm in timeout seconds 20 | try: 21 | retval = f(*args) 22 | except TimeoutException: 23 | return False 24 | finally: 25 | signal.signal(signal.SIGALRM, old_handler) 26 | signal.alarm(0) 27 | return retval 28 | return f2 29 | return timeout_function 30 | 31 | def which(program): 32 | def is_exe(fpath): 33 | return os.path.isfile(fpath) and os.access(fpath, os.X_OK) 34 | 35 | fpath, fname = os.path.split(program) 36 | if fpath: 37 | if is_exe(program): 38 | return program 39 | else: 40 | for path in os.environ["PATH"].split(os.pathsep): 41 | path = path.strip('"') 42 | exe_file = os.path.join(path, program) 43 | if is_exe(exe_file): 44 | return exe_file 45 | return None 46 | 47 | @timeout(30) 48 | def connect_to_leader(): 49 | c_leader = Counter('127.0.0.1:14000') 50 | print "Connecting to old leader" 51 | for i in range(100): 52 | c_leader.increment() 53 | # This method will timeout before it reaches here. 54 | print "Client Made Progress: Counter value: %d" % c_minority.getvalue() 55 | return True 56 | 57 | def test_timeout(): 58 | numreplicas = 3 59 | processes = [] 60 | 61 | print "Running replica 0" 62 | processes.append(subprocess.Popen(['concoord', 'replica', 63 | '-o', 'concoord.object.counter.Counter', 64 | '-a', '127.0.0.1', '-p', '14000'])) 65 | 66 | for i in range(1, numreplicas): 67 | print "Running replica %d" %i 68 | processes.append(subprocess.Popen(['concoord', 'replica', 69 | '-o', 'concoord.object.counter.Counter', 70 | '-a', '127.0.0.1', '-p', '1400%d'%i, 71 | '-b', '127.0.0.1:14000'])) 72 | 73 | # Give the system some time to initialize 74 | time.sleep(10) 75 | 76 | # This client can only connect to the replicas in this partition 77 | c_P1 = Counter('127.0.0.1:14000', debug = True) 78 | c_P2 = Counter('127.0.0.1:14001, 127.0.0.1:14002') 79 | # The client should work 80 | print "Sending requests to the leader" 81 | for i in range(100): 82 | c_P1.increment() 83 | print "Counter value after 100 increments: %d" % c_P1.getvalue() 84 | 85 | # Save iptables settings for later recovery 86 | with open('test.iptables.rules', 'w') as output: 87 | subprocess.Popen(['sudo', 'iptables-save'], stdout=output) 88 | 89 | # Block all incoming traffic to leader 90 | iptablerule = subprocess.Popen(['sudo', 'iptables', 91 | '-I', 'INPUT', 92 | '-p', 'tcp', 93 | '--dport', '14000', 94 | '-j', 'DROP']) 95 | 96 | print "Cutting the connections to the leader. Waiting for system to stabilize." 97 | time.sleep(10) 98 | 99 | print "Connecting to old leader, which should not make progress." 100 | if connect_to_leader(): 101 | print "===== TEST FAILED =====" 102 | else: 103 | # c_P2 should make progress 104 | print "Connecting to other nodes, which should have a new leader." 105 | for i in range(100): 106 | c_P2.increment() 107 | print "Counter value after 100 increments: %d" % c_P2.getvalue() 108 | print "===== TEST PASSED =====" 109 | 110 | print "Fixing the connections and cleaning up." 111 | with open('test.iptables.rules', 'r') as input: 112 | subprocess.Popen(['sudo', 'iptables-restore'], stdin=input) 113 | subprocess.Popen(['sudo', 'rm', 'test.iptables.rules']) 114 | 115 | for p in (processes): 116 | p.kill() 117 | return True 118 | 119 | def main(): 120 | if not which('iptables'): 121 | sys.exit('Test requires iptables to run') 122 | 123 | if not os.geteuid() == 0: 124 | sys.exit('Script must be run as root') 125 | 126 | test_timeout() 127 | 128 | if __name__ == '__main__': 129 | main() 130 | -------------------------------------------------------------------------------- /concoord/threadingobject/__init__.py: -------------------------------------------------------------------------------- 1 | """distributed threading objects""" 2 | 3 | -------------------------------------------------------------------------------- /concoord/threadingobject/dboundedsemaphore.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Bounded Semaphore Coordination Object 4 | @copyright: See LICENSE 5 | """ 6 | from threading import Lock 7 | from concoord.exception import * 8 | 9 | class DBoundedSemaphore(): 10 | def __init__(self, count=1): 11 | if count < 0: 12 | raise ValueError 13 | self.__count = int(count) 14 | self.__queue = [] 15 | self.__atomic = Lock() 16 | self._initial_value = int(count) 17 | 18 | def __repr__(self): 19 | return "<%s count=%d init=%d>" % (self.__class__.__name__, self.__count, self._initial_value) 20 | 21 | def acquire(self, _concoord_command): 22 | with self.__atomic: 23 | self.__count -= 1 24 | if self.__count < 0: 25 | self.__queue.append(_concoord_command) 26 | raise BlockingReturn 27 | else: 28 | return True 29 | 30 | def release(self, _concoord_command): 31 | with self.__atomic: 32 | if self.__count == self._initial_value: 33 | return ValueError("Semaphore released too many times") 34 | else: 35 | self.__count += 1 36 | if len(self.__queue) > 0: 37 | unblockcommand = self.__queue.pop(0) 38 | # add the popped command to the exception args 39 | unblocked = {} 40 | unblocked[unblockcommand] = True 41 | raise UnblockingReturn(unblockeddict=unblocked) 42 | 43 | def __str__(self): 44 | return "<%s object>" % (self.__class__.__name__) 45 | -------------------------------------------------------------------------------- /concoord/threadingobject/dcondition.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Condition Coordination Object 4 | @copyright: See LICENSE 5 | """ 6 | from threading import RLock 7 | from concoord.exception import * 8 | from concoord.threadingobject.drlock import DRLock 9 | 10 | class DCondition(): 11 | def __init__(self, lock=None): 12 | if lock is None: 13 | lock = DRLock() 14 | self.__lock = lock 15 | # Export the lock's acquire() and release() methods 16 | self.acquire = lock.acquire 17 | self.release = lock.release 18 | self.__waiters = [] 19 | self.__atomic = RLock() 20 | 21 | def __repr__(self): 22 | return "" % (self.__lock, len(self.__waiters)) 23 | 24 | def wait(self, _concoord_command): 25 | # put the caller on waitinglist and take the lock away 26 | with self.__atomic: 27 | if not self.__lock._is_owned(_concoord_command.client): 28 | raise RuntimeError("cannot wait on un-acquired lock") 29 | self.__waiters.append(_concoord_command) 30 | self.__lock.release(_concoord_command) 31 | raise BlockingReturn() 32 | 33 | def notify(self, _concoord_command): 34 | # Notify the next client on the wait list 35 | with self.__atomic: 36 | if not self.__lock._is_owned(_concoord_command.client): 37 | raise RuntimeError("cannot notify on un-acquired lock") 38 | if not self.__waiters: 39 | return 40 | waitcommand = self.__waiters.pop(0) 41 | # notified client should be added to the lock queue 42 | self.__lock._add_to_queue(waitcommand) 43 | 44 | def notifyAll(self, _concoord_command): 45 | # Notify every client on the wait list 46 | with self.__atomic: 47 | if not self.__lock._is_owned(_concoord_command.client): 48 | raise RuntimeError("cannot notify on un-acquired lock") 49 | if not self.__waiters: 50 | return 51 | for waitcommand in self.__waiters: 52 | # notified client should be added to the lock queue 53 | self.__lock._add_to_queue(waitcommand) 54 | self.__waiters = [] 55 | 56 | def __str__(self): 57 | return "<%s object>" % (self.__class__.__name__) 58 | 59 | -------------------------------------------------------------------------------- /concoord/threadingobject/dlock.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Lock Coordination Object 4 | @copyright: See LICENSE 5 | """ 6 | from threading import Lock 7 | from concoord.enums import * 8 | from concoord.exception import * 9 | 10 | class DLock(): 11 | def __init__(self): 12 | self.__locked = False 13 | self.__owner = None 14 | self.__queue = [] 15 | self.__atomic = Lock() 16 | 17 | def __repr__(self): 18 | return "<%s owner=%s>" % (self.__class__.__name__, str(self.__owner)) 19 | 20 | def acquire(self, _concoord_command): 21 | with self.__atomic: 22 | if self.__locked: 23 | self.__queue.append(_concoord_command) 24 | raise BlockingReturn() 25 | else: 26 | self.__locked = True 27 | self.__owner = _concoord_command.client 28 | 29 | def release(self, _concoord_command): 30 | with self.__atomic: 31 | if self.__owner != _concoord_command.client: 32 | raise RuntimeError("cannot release un-acquired lock") 33 | if len(self.__queue) > 0: 34 | unblockcommand = self.__queue.pop(0) 35 | self.__owner = unblockcommand.client 36 | # add the popped command to the exception args 37 | unblocked = {} 38 | unblocked[unblockcommand] = True 39 | raise UnblockingReturn(unblockeddict=unblocked) 40 | elif len(self.__queue) == 0: 41 | self.__owner = None 42 | self.__locked = False 43 | 44 | def __str__(self): 45 | return "<%s object>" % (self.__class__.__name__) 46 | 47 | -------------------------------------------------------------------------------- /concoord/threadingobject/drlock.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: RLock Coordination Object 4 | @copyright: See LICENSE 5 | """ 6 | from threading import Lock 7 | from concoord.enums import * 8 | from concoord.exception import * 9 | 10 | class DRLock(): 11 | def __init__(self): 12 | self.__count = 0 13 | self.__owner = None 14 | self.__queue = [] 15 | self.__atomic = Lock() 16 | 17 | def __repr__(self): 18 | return "<%s owner=%r count=%d>" % (self.__class__.__name__, self.__owner, self.__count) 19 | 20 | def acquire(self, _concoord_command): 21 | with self.__atomic: 22 | if self.__count > 0 and self.__owner != _concoord_command.client: 23 | self.__queue.append(_concoord_command) 24 | raise BlockingReturn() 25 | elif self.__count > 0 and self.__owner == _concoord_command.client: 26 | self.__count += 1 27 | return 1 28 | else: 29 | self.__count = 1 30 | self.__owner = _concoord_command.client 31 | return True 32 | 33 | def release(self, _concoord_command): 34 | with self.__atomic: 35 | if self.__owner != _concoord_command.client: 36 | raise RuntimeError("cannot release un-acquired lock") 37 | self.__count -= 1 38 | 39 | if self.__count == 0 and len(self.__queue) > 0: 40 | self.__count += 1 41 | unblockcommand = self.__queue.pop(0) 42 | self.__owner = unblockcommand.client 43 | # add the popped command to the exception args 44 | unblocked = {} 45 | unblocked[unblockcommand] = True 46 | raise UnblockingReturn(unblockeddict=unblocked) 47 | elif self.__count == 0 and len(self.__queue) == 0: 48 | self.__owner = None 49 | 50 | # Internal methods used by condition variables 51 | def _is_owned(self, client): 52 | return self.__owner == client 53 | 54 | def _add_to_queue(self, clientcommand): 55 | self.__queue.append(clientcommand) 56 | 57 | def __str__(self): 58 | return "<%s object>" % (self.__class__.__name__) 59 | 60 | -------------------------------------------------------------------------------- /concoord/threadingobject/dsemaphore.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Semaphore Coordination Object 4 | @copyright: See LICENSE 5 | """ 6 | from threading import Lock 7 | from concoord.exception import * 8 | from concoord.enums import * 9 | 10 | class DSemaphore(): 11 | def __init__(self, count=1): 12 | if count < 0: 13 | raise ValueError 14 | self.__count = int(count) 15 | self.__queue = [] 16 | self.__atomic = Lock() 17 | 18 | def __repr__(self): 19 | return "<%s count=%d>" % (self.__class__.__name__, self.__count) 20 | 21 | def acquire(self, _concoord_command): 22 | with self.__atomic: 23 | self.__count -= 1 24 | if self.__count < 0: 25 | self.__queue.append(_concoord_command) 26 | raise BlockingReturn() 27 | else: 28 | return True 29 | 30 | def release(self, _concoord_command): 31 | with self.__atomic: 32 | self.__count += 1 33 | if len(self.__queue) > 0: 34 | unblockcommand = self.__queue.pop(0) 35 | unblocked = {} 36 | unblocked[unblockcommand] = True 37 | raise UnblockingReturn(unblockeddict=unblocked) 38 | 39 | def __str__(self): 40 | return "<%s object>" % (self.__class__.__name__) 41 | -------------------------------------------------------------------------------- /concoord/utils.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Deniz Altinbuken, Emin Gun Sirer 3 | @note: Utility functions for the runtime. Includes a timer module for collecting measurements. 4 | @copyright: See LICENSE 5 | ''' 6 | import socket 7 | import os, sys 8 | import time 9 | import string 10 | import threading 11 | from concoord.enums import * 12 | 13 | def findOwnIP(): 14 | """Retrieves the hostname of the caller""" 15 | return socket.gethostbyname(socket.gethostname()) 16 | 17 | def get_addressportpairs(group): 18 | for peer in group.iterkeys(): 19 | yield (peer.addr,peer.port) 20 | 21 | def get_addresses(group): 22 | for peer in group.iterkeys(): 23 | yield peer.addr 24 | 25 | # A logger will always print to the screen. It can also log to a file or to a network log daemon. 26 | class NoneLogger(): 27 | def write(self, cls, string): 28 | return 29 | 30 | def close(self): 31 | return 32 | 33 | class Logger(): 34 | def __init__(self, name, filename=None, lognode=None): 35 | self.prefix = name 36 | self.log = None 37 | if filename is not None: 38 | self.log = open("concoord_log_"+name, 'w') 39 | if lognode is not None: 40 | logaddr,logport = lognode.split(':') 41 | try: 42 | self.log = socket.socket(socket.AF_INET,socket.SOCK_STREAM) 43 | self.log.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) 44 | self.log.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1) 45 | self.log.connect((logaddr,int(logport))) 46 | except IOError: 47 | self.log = None 48 | return 49 | 50 | def write(self, cls, string): 51 | print "[%s] %s %s: %s" % (self.prefix + '_' + threading.current_thread().name, 52 | time.time(), # time.asctime(time.localtime(time.time())), 53 | cls, string) 54 | if self.log is not None: 55 | self.log.write("[%s] %s %s: %s" % (self.prefix + '_' + threading.current_thread().name, 56 | time.time(), # time.asctime(time.localtime(time.time())), 57 | cls, string)) 58 | self.log.flush() 59 | 60 | def close(self): 61 | if self.log is not None: 62 | self.log.close() 63 | 64 | # PERFORMANCE MEASUREMENT UTILS 65 | timers = {} 66 | def starttimer(timerkey, timerno): 67 | global timers 68 | index = "%s-%s" % (str(timerkey),str(timerno)) 69 | if not timers.has_key(index): 70 | timers[index] = [time.time(), 0] 71 | 72 | def endtimer(timerkey, timerno): 73 | global timers 74 | index = "%s-%s" % (str(timerkey),str(timerno)) 75 | try: 76 | if timers[index][1] == 0: 77 | timers[index][1] = time.time() 78 | except: 79 | print "Can't stop timer %s %s." % (str(timerkey),str(timerno)) 80 | 81 | def dumptimers(numreplicas, ownertype, outputdict): 82 | global timers 83 | filename = "output/replica/%s" % str(numreplicas) 84 | try: 85 | outputfile = open(outputdict+filename, "w") 86 | except: 87 | outputfile = open("./"+filename, "w") 88 | for index,numbers in timers.iteritems(): 89 | timerkey, timerno = index.rsplit("-") 90 | if not numbers[1]-numbers[0] < 0: 91 | outputfile.write("%s:\t%s\t%s\n" % (str(timerno), 92 | str(numreplicas), 93 | str(numbers[1]-numbers[0]))) 94 | outputfile.close() 95 | 96 | def starttiming(fn): 97 | """ 98 | Decorator used to start timing. 99 | Keeps track of the count for the first and second calls. 100 | """ 101 | def new(*args, **kw): 102 | obj = args[0] 103 | if obj.firststarttime == 0: 104 | obj.firststarttime = time.time() 105 | elif obj.secondstarttime == 0: 106 | obj.secondstarttime = time.time() 107 | profile_on() 108 | return fn(*args, **kw) 109 | return new 110 | 111 | def endtiming(fn): 112 | """ 113 | Decorator used to end timing. 114 | Keeps track of the count for the first and second calls. 115 | """ 116 | NITER = 10000 117 | def new(*args, **kw): 118 | ret = fn(*args, **kw) 119 | obj = args[0] 120 | if obj.firststoptime == 0: 121 | obj.firststoptime = time.time() 122 | elif obj.secondstoptime == 0: 123 | obj.secondstoptime = time.time() 124 | elif obj.count == NITER: 125 | now = time.time() 126 | total = now - obj.secondstarttime 127 | perrequest = total/NITER 128 | filename = "output/%s" % (str(len(obj.groups[NODE_REPLICA])+1)) 129 | outputfile = open("./"+filename, "a") 130 | # numreplicas #perrequest #total 131 | outputfile.write("%s\t%s\t%s\n" % (str(len(obj.groups[NODE_REPLICA])+1), 132 | str(perrequest), str(total))) 133 | outputfile.close() 134 | obj.count += 1 135 | sys.stdout.flush() 136 | profile_off() 137 | profilerdict = get_profile_stats() 138 | for key, value in sorted(profilerdict.iteritems(), 139 | key=lambda (k,v): (v[2],k)): 140 | print "%s: %s" % (key, value) 141 | time.sleep(10) 142 | sys.stdout.flush() 143 | os._exit(0) 144 | else: 145 | obj.count += 1 146 | return ret 147 | return new 148 | 149 | def throughput_test(fn): 150 | """Decorator used to measure throughput.""" 151 | def new(*args, **kw): 152 | ret = fn(*args, **kw) 153 | obj = args[0] 154 | obj.throughput_runs += 1 155 | if obj.throughput_runs == 100: 156 | obj.throughput_start = time.time() 157 | elif obj.throughput_runs == 1100: 158 | obj.throughput_stop = time.time() 159 | totaltime = obj.throughput_stop - obj.throughput_start 160 | print "********************************************" 161 | print "TOTAL: ", totaltime 162 | print "TPUT: ", 1000/totaltime, "req/s" 163 | print "********************************************" 164 | obj._graceexit(1) 165 | return ret 166 | return new 167 | 168 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/ConCoord.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ConCoord.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/ConCoord" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ConCoord" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /doc/_build/doctrees/environment.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/doctrees/environment.pickle -------------------------------------------------------------------------------- /doc/_build/doctrees/index.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/doctrees/index.doctree -------------------------------------------------------------------------------- /doc/_build/doctrees/install.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/doctrees/install.doctree -------------------------------------------------------------------------------- /doc/_build/doctrees/openreplica.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/doctrees/openreplica.doctree -------------------------------------------------------------------------------- /doc/_build/doctrees/overview.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/doctrees/overview.doctree -------------------------------------------------------------------------------- /doc/_build/doctrees/tutorial-advanced.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/doctrees/tutorial-advanced.doctree -------------------------------------------------------------------------------- /doc/_build/doctrees/tutorial.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/doctrees/tutorial.doctree -------------------------------------------------------------------------------- /doc/_build/html/.buildinfo: -------------------------------------------------------------------------------- 1 | # Sphinx build info version 1 2 | # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. 3 | config: 1df50fb5a5078830334630fe1457eb89 4 | tags: a205e9ed8462ae86fdd2f73488852ba9 5 | -------------------------------------------------------------------------------- /doc/_build/html/_images/concoord.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/html/_images/concoord.jpg -------------------------------------------------------------------------------- /doc/_build/html/_sources/index.txt: -------------------------------------------------------------------------------- 1 | .. ConCoord documentation master file, created by 2 | sphinx-quickstart on Tue Mar 27 16:31:41 2012. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to ConCoord's documentation 7 | =================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 3 13 | 14 | overview 15 | install 16 | tutorial 17 | openreplica 18 | tutorial-advanced 19 | 20 | Indices and tables 21 | ================== 22 | 23 | * :ref:`genindex` 24 | * :ref:`modindex` 25 | * :ref:`search` 26 | -------------------------------------------------------------------------------- /doc/_build/html/_sources/install.txt: -------------------------------------------------------------------------------- 1 | Installing ConCoord 2 | =================== 3 | 4 | Requirements 5 | ------------------------ 6 | The minimum requirements for ConCoord are:: 7 | 8 | - python 2.7.2 or later 9 | - dnspython-1.9.4 10 | - msgpack-python 11 | 12 | Installation 13 | ------------------------ 14 | ConCoord can be installed from source with: 15 | 16 | .. sourcecode:: console 17 | 18 | $ python setup.py install 19 | 20 | ConCoord is also available for install through PyPI: 21 | 22 | .. sourcecode:: console 23 | 24 | $ pip install concoord 25 | 26 | -------------------------------------------------------------------------------- /doc/_build/html/_sources/overview.txt: -------------------------------------------------------------------------------- 1 | Overview 2 | ======== 3 | 4 | ConCoord is a novel coordination service that provides replication and 5 | synchronization support for large-scale distributed systems. ConCoord 6 | employs an object-oriented approach, in which the system creates 7 | and maintains live replicas for Python objects written by the user. 8 | ConCoord converts these Python objects into Paxos Replicated State 9 | Machines (RSM) and enables clients to do method invocations on them 10 | transparently as if they are local objects. ConCoord uses these 11 | replicated objects to implement coordination and synchronization 12 | constructs in large-scale distributed systems, in effect establishing 13 | a transparent way of providing a coordination service. 14 | 15 | .. image:: _static/concoord.jpg 16 | :align: center 17 | 18 | 19 | -------------------------------------------------------------------------------- /doc/_build/html/_sources/tutorial-advanced.txt: -------------------------------------------------------------------------------- 1 | ConCoordifying Python Objects 2 | ----------------------------- 3 | For cases when the objects included in the ConCoord distribution do 4 | not meet your coordination needs, ConCoord lets you convert your 5 | local Python objects into distributable objects very easily. 6 | 7 | To walk through the ConCoord approach, you will use a different 8 | Counter object that increments and decrements by ten, namely 9 | tencounter.py. Once you install ConCoord, you can create coordination 10 | objects and save them anywhere in your filesystem. After you create 11 | ``tencounter.py``, you can save it under ``/foo/tencounter.py``: 12 | 13 | .. sourcecode:: python 14 | 15 | class TenCounter: 16 | def __init__(self, value=0): 17 | self.value = value 18 | 19 | def decrement(self): 20 | self.value -= 10 21 | 22 | def increment(self): 23 | self.value += 10 24 | 25 | def getvalue(self): 26 | return self.value 27 | 28 | def __str__(self): 29 | return "The counter value is %d" % self.value 30 | 31 | Once you have created the object, update your ``PYTHONPATH`` accordingly, 32 | so that the objects can be found and imported: 33 | 34 | .. sourcecode:: console 35 | 36 | $ export PYTHONPATH=$PYTHONPATH:/foo/ 37 | 38 | Clients will use a proxy object to do method calls on the object. 39 | To create the proxy object automatically, you can use the following 40 | command: 41 | 42 | .. sourcecode:: console 43 | 44 | $ concoord object -o tencounter.TenCounter 45 | 46 | ``Usage: concoord object [-h] [-o OBJECTNAME] [-t SECURITYTOKEN] [-p PROXYTYPE] [-s] [-v]`` 47 | 48 | where, 49 | ``-h, --help`` show this help message and exit 50 | 51 | ``-o OBJECTNAME, --objectname OBJECTNAME`` client object dotted name module.Class 52 | 53 | ``-t SECURITYTOKEN, --token SECURITYTOKEN`` security token 54 | 55 | ``-p PROXYTYPE, --proxytype PROXYTYPE`` 0: basic, 1: blocking, 2: client-side batching, 3: server-side batching 56 | 57 | ``-s, --safe`` safety checking on/off 58 | 59 | ``-v, --verbose`` verbose mode on/off 60 | 61 | This script will create a proxy file under the directory that the 62 | object resides (i.e. ``/foo/``): 63 | 64 | ``/foo/tencounterproxy.py`` := the proxy that can be used by the client 65 | 66 | IMPORTANT NOTE: ConCoord objects treat ``__init__`` functions specially in 67 | two ways: 68 | 69 | 1) When Replicas go live, the object is instantiated calling the 70 | ``__init__`` without any arguments. Therefore, while implementing 71 | coordination objects, the ``__init__`` method should be implemented to 72 | be able to run without explicit arguments. You can use defaults to 73 | implement an ``__init__`` method that accepts arguments. 74 | 75 | 2) In the proxy created, the ``__init__`` function is used to initialize 76 | the Client-Replica connection. This way, multiple clients can 77 | connect to a ConCoord instance without reinitializing the 78 | object. During proxy generation, the original ``__init__`` function is 79 | renamed as ``__concoordinit__``, to reinitialize the object the user can 80 | call ``__concoordinit__`` at any point. 81 | 82 | After this point on, you can use TenCounter just like we use Counter before. 83 | 84 | Creating Source Bundles 85 | ----------------------- 86 | 87 | You can create bundles to use at the server and client sides using the 88 | ``Makefile`` provided under ``concoord/`` 89 | 90 | Remember to add the objects you have created in these bundles. 91 | 92 | Creating a Server Bundle 93 | ~~~~~~~~~~~~~~~~~~~~~~~~ 94 | 95 | To create a bundle that can run replicas: 96 | 97 | .. sourcecode:: console 98 | 99 | $ make server 100 | 101 | Creating a Client Bundle 102 | ~~~~~~~~~~~~~~~~~~~~~~~~ 103 | 104 | To create a bundle that can run a client and connect to an existing 105 | ConCoord instance: 106 | 107 | .. sourcecode:: console 108 | 109 | $ make client 110 | 111 | Logging 112 | ------- 113 | 114 | We have two kinds of loggers for ConCoord: 115 | 116 | * Console Logger 117 | * Network Logger 118 | 119 | Both of these loggers are included under ``utils.py``. To start the 120 | ``NetworkLogger``, use the ``logdaemon.py`` on the host you want to keep the 121 | Logger. 122 | 123 | Synchronization & Threading 124 | --------------------------- 125 | 126 | ConCoord provides a distributed and fault-tolerant threading 127 | library. The library includes: 128 | 129 | * Lock 130 | * RLock 131 | * Semaphore 132 | * BoundedSemaphore 133 | * Barrier 134 | * Condition 135 | 136 | The implementations of distributed synchronization objects follow the 137 | implementations in the Python threading library. We will walk through 138 | an example below using the ``Semaphore`` object under 139 | ``concoord/object/semaphore.py`` 140 | 141 | In the blocking object implementation, the method invocations that use 142 | an object from the threading library requires an extra argument 143 | ``_concoord_command``. This argument is used by the calling Replica node 144 | to relate any blocking/unblocking method invocation to a specific 145 | client. This way, even if the client disconnects and reconnects, the 146 | ConCoord instance will remain in a safe state. 147 | 148 | .. sourcecode:: python 149 | 150 | from concoord.threadingobject.dsemaphore import DSemaphore 151 | 152 | class Semaphore: 153 | def __init__(self, count=1): 154 | self.semaphore = DSemaphore(count) 155 | 156 | def __repr__(self): 157 | return repr(self.semaphore) 158 | 159 | def acquire(self, _concoord_command): 160 | try: 161 | return self.semaphore.acquire(_concoord_command) 162 | except Exception as e: 163 | raise e 164 | 165 | def release(self, _concoord_command): 166 | try: 167 | return self.semaphore.release(_concoord_command) 168 | except Exception as e: 169 | raise e 170 | 171 | def __str__(self): 172 | return str(self.semaphore) 173 | 174 | To create the proxy for this blocking object we will use the following command: 175 | 176 | .. sourcecode:: console 177 | 178 | $ concoord object -o concoord.object.semaphore.Semaphore -p 1 179 | 180 | This command creates the proxy that supports blocking operations. Now 181 | you can use blocking objects just like basic ConCoord objects. First, 182 | we start the replica nodes the same way we did before as follows: 183 | 184 | .. sourcecode:: console 185 | 186 | $ concoord replica -o concoord.object.semaphore.Semaphore -a 127.0.0.1 -p 14000 187 | 188 | To test the functionality, you can use multiple clients or print out the ``Semaphore`` object as follows: 189 | 190 | .. sourcecode:: pycon 191 | 192 | >>> from semaphoreproxy import Semaphore 193 | >>> s = Semaphore("127.0.0.1:14000") 194 | >>> s.acquire() 195 | True 196 | >>> i = 10 197 | >>> i += 5 198 | >>> s 199 | 200 | >>> s.release() 201 | >>> s 202 | 203 | >>> 204 | -------------------------------------------------------------------------------- /doc/_build/html/_static/ajax-loader.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/html/_static/ajax-loader.gif -------------------------------------------------------------------------------- /doc/_build/html/_static/comment-bright.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/html/_static/comment-bright.png -------------------------------------------------------------------------------- /doc/_build/html/_static/comment-close.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/html/_static/comment-close.png -------------------------------------------------------------------------------- /doc/_build/html/_static/comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/html/_static/comment.png -------------------------------------------------------------------------------- /doc/_build/html/_static/concoord.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/html/_static/concoord.jpg -------------------------------------------------------------------------------- /doc/_build/html/_static/default.css: -------------------------------------------------------------------------------- 1 | /* 2 | * default.css_t 3 | * ~~~~~~~~~~~~~ 4 | * 5 | * Sphinx stylesheet -- default theme. 6 | * 7 | * :copyright: Copyright 2007-2013 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | @import url("basic.css"); 13 | 14 | /* -- page layout ----------------------------------------------------------- */ 15 | 16 | body { 17 | font-family: sans-serif; 18 | font-size: 100%; 19 | background-color: #11303d; 20 | color: #000; 21 | margin: 0; 22 | padding: 0; 23 | } 24 | 25 | div.document { 26 | background-color: #1c4e63; 27 | } 28 | 29 | div.documentwrapper { 30 | float: left; 31 | width: 100%; 32 | } 33 | 34 | div.bodywrapper { 35 | margin: 0 0 0 230px; 36 | } 37 | 38 | div.body { 39 | background-color: #ffffff; 40 | color: #000000; 41 | padding: 0 20px 30px 20px; 42 | } 43 | 44 | div.footer { 45 | color: #ffffff; 46 | width: 100%; 47 | padding: 9px 0 9px 0; 48 | text-align: center; 49 | font-size: 75%; 50 | } 51 | 52 | div.footer a { 53 | color: #ffffff; 54 | text-decoration: underline; 55 | } 56 | 57 | div.related { 58 | background-color: #133f52; 59 | line-height: 30px; 60 | color: #ffffff; 61 | } 62 | 63 | div.related a { 64 | color: #ffffff; 65 | } 66 | 67 | div.sphinxsidebar { 68 | } 69 | 70 | div.sphinxsidebar h3 { 71 | font-family: 'Trebuchet MS', sans-serif; 72 | color: #ffffff; 73 | font-size: 1.4em; 74 | font-weight: normal; 75 | margin: 0; 76 | padding: 0; 77 | } 78 | 79 | div.sphinxsidebar h3 a { 80 | color: #ffffff; 81 | } 82 | 83 | div.sphinxsidebar h4 { 84 | font-family: 'Trebuchet MS', sans-serif; 85 | color: #ffffff; 86 | font-size: 1.3em; 87 | font-weight: normal; 88 | margin: 5px 0 0 0; 89 | padding: 0; 90 | } 91 | 92 | div.sphinxsidebar p { 93 | color: #ffffff; 94 | } 95 | 96 | div.sphinxsidebar p.topless { 97 | margin: 5px 10px 10px 10px; 98 | } 99 | 100 | div.sphinxsidebar ul { 101 | margin: 10px; 102 | padding: 0; 103 | color: #ffffff; 104 | } 105 | 106 | div.sphinxsidebar a { 107 | color: #98dbcc; 108 | } 109 | 110 | div.sphinxsidebar input { 111 | border: 1px solid #98dbcc; 112 | font-family: sans-serif; 113 | font-size: 1em; 114 | } 115 | 116 | 117 | 118 | /* -- hyperlink styles ------------------------------------------------------ */ 119 | 120 | a { 121 | color: #355f7c; 122 | text-decoration: none; 123 | } 124 | 125 | a:visited { 126 | color: #355f7c; 127 | text-decoration: none; 128 | } 129 | 130 | a:hover { 131 | text-decoration: underline; 132 | } 133 | 134 | 135 | 136 | /* -- body styles ----------------------------------------------------------- */ 137 | 138 | div.body h1, 139 | div.body h2, 140 | div.body h3, 141 | div.body h4, 142 | div.body h5, 143 | div.body h6 { 144 | font-family: 'Trebuchet MS', sans-serif; 145 | background-color: #f2f2f2; 146 | font-weight: normal; 147 | color: #20435c; 148 | border-bottom: 1px solid #ccc; 149 | margin: 20px -20px 10px -20px; 150 | padding: 3px 0 3px 10px; 151 | } 152 | 153 | div.body h1 { margin-top: 0; font-size: 200%; } 154 | div.body h2 { font-size: 160%; } 155 | div.body h3 { font-size: 140%; } 156 | div.body h4 { font-size: 120%; } 157 | div.body h5 { font-size: 110%; } 158 | div.body h6 { font-size: 100%; } 159 | 160 | a.headerlink { 161 | color: #c60f0f; 162 | font-size: 0.8em; 163 | padding: 0 4px 0 4px; 164 | text-decoration: none; 165 | } 166 | 167 | a.headerlink:hover { 168 | background-color: #c60f0f; 169 | color: white; 170 | } 171 | 172 | div.body p, div.body dd, div.body li { 173 | text-align: justify; 174 | line-height: 130%; 175 | } 176 | 177 | div.admonition p.admonition-title + p { 178 | display: inline; 179 | } 180 | 181 | div.admonition p { 182 | margin-bottom: 5px; 183 | } 184 | 185 | div.admonition pre { 186 | margin-bottom: 5px; 187 | } 188 | 189 | div.admonition ul, div.admonition ol { 190 | margin-bottom: 5px; 191 | } 192 | 193 | div.note { 194 | background-color: #eee; 195 | border: 1px solid #ccc; 196 | } 197 | 198 | div.seealso { 199 | background-color: #ffc; 200 | border: 1px solid #ff6; 201 | } 202 | 203 | div.topic { 204 | background-color: #eee; 205 | } 206 | 207 | div.warning { 208 | background-color: #ffe4e4; 209 | border: 1px solid #f66; 210 | } 211 | 212 | p.admonition-title { 213 | display: inline; 214 | } 215 | 216 | p.admonition-title:after { 217 | content: ":"; 218 | } 219 | 220 | pre { 221 | padding: 5px; 222 | background-color: #eeffcc; 223 | color: #333333; 224 | line-height: 120%; 225 | border: 1px solid #ac9; 226 | border-left: none; 227 | border-right: none; 228 | } 229 | 230 | tt { 231 | background-color: #ecf0f3; 232 | padding: 0 1px 0 1px; 233 | font-size: 0.95em; 234 | } 235 | 236 | th { 237 | background-color: #ede; 238 | } 239 | 240 | .warning tt { 241 | background: #efc2c2; 242 | } 243 | 244 | .note tt { 245 | background: #d6d6d6; 246 | } 247 | 248 | .viewcode-back { 249 | font-family: sans-serif; 250 | } 251 | 252 | div.viewcode-block:target { 253 | background-color: #f4debf; 254 | border-top: 1px solid #ac9; 255 | border-bottom: 1px solid #ac9; 256 | } -------------------------------------------------------------------------------- /doc/_build/html/_static/doctools.js: -------------------------------------------------------------------------------- 1 | /* 2 | * doctools.js 3 | * ~~~~~~~~~~~ 4 | * 5 | * Sphinx JavaScript utilities for all documentation. 6 | * 7 | * :copyright: Copyright 2007-2013 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | /** 13 | * select a different prefix for underscore 14 | */ 15 | $u = _.noConflict(); 16 | 17 | /** 18 | * make the code below compatible with browsers without 19 | * an installed firebug like debugger 20 | if (!window.console || !console.firebug) { 21 | var names = ["log", "debug", "info", "warn", "error", "assert", "dir", 22 | "dirxml", "group", "groupEnd", "time", "timeEnd", "count", "trace", 23 | "profile", "profileEnd"]; 24 | window.console = {}; 25 | for (var i = 0; i < names.length; ++i) 26 | window.console[names[i]] = function() {}; 27 | } 28 | */ 29 | 30 | /** 31 | * small helper function to urldecode strings 32 | */ 33 | jQuery.urldecode = function(x) { 34 | return decodeURIComponent(x).replace(/\+/g, ' '); 35 | }; 36 | 37 | /** 38 | * small helper function to urlencode strings 39 | */ 40 | jQuery.urlencode = encodeURIComponent; 41 | 42 | /** 43 | * This function returns the parsed url parameters of the 44 | * current request. Multiple values per key are supported, 45 | * it will always return arrays of strings for the value parts. 46 | */ 47 | jQuery.getQueryParameters = function(s) { 48 | if (typeof s == 'undefined') 49 | s = document.location.search; 50 | var parts = s.substr(s.indexOf('?') + 1).split('&'); 51 | var result = {}; 52 | for (var i = 0; i < parts.length; i++) { 53 | var tmp = parts[i].split('=', 2); 54 | var key = jQuery.urldecode(tmp[0]); 55 | var value = jQuery.urldecode(tmp[1]); 56 | if (key in result) 57 | result[key].push(value); 58 | else 59 | result[key] = [value]; 60 | } 61 | return result; 62 | }; 63 | 64 | /** 65 | * highlight a given string on a jquery object by wrapping it in 66 | * span elements with the given class name. 67 | */ 68 | jQuery.fn.highlightText = function(text, className) { 69 | function highlight(node) { 70 | if (node.nodeType == 3) { 71 | var val = node.nodeValue; 72 | var pos = val.toLowerCase().indexOf(text); 73 | if (pos >= 0 && !jQuery(node.parentNode).hasClass(className)) { 74 | var span = document.createElement("span"); 75 | span.className = className; 76 | span.appendChild(document.createTextNode(val.substr(pos, text.length))); 77 | node.parentNode.insertBefore(span, node.parentNode.insertBefore( 78 | document.createTextNode(val.substr(pos + text.length)), 79 | node.nextSibling)); 80 | node.nodeValue = val.substr(0, pos); 81 | } 82 | } 83 | else if (!jQuery(node).is("button, select, textarea")) { 84 | jQuery.each(node.childNodes, function() { 85 | highlight(this); 86 | }); 87 | } 88 | } 89 | return this.each(function() { 90 | highlight(this); 91 | }); 92 | }; 93 | 94 | /** 95 | * Small JavaScript module for the documentation. 96 | */ 97 | var Documentation = { 98 | 99 | init : function() { 100 | this.fixFirefoxAnchorBug(); 101 | this.highlightSearchWords(); 102 | this.initIndexTable(); 103 | }, 104 | 105 | /** 106 | * i18n support 107 | */ 108 | TRANSLATIONS : {}, 109 | PLURAL_EXPR : function(n) { return n == 1 ? 0 : 1; }, 110 | LOCALE : 'unknown', 111 | 112 | // gettext and ngettext don't access this so that the functions 113 | // can safely bound to a different name (_ = Documentation.gettext) 114 | gettext : function(string) { 115 | var translated = Documentation.TRANSLATIONS[string]; 116 | if (typeof translated == 'undefined') 117 | return string; 118 | return (typeof translated == 'string') ? translated : translated[0]; 119 | }, 120 | 121 | ngettext : function(singular, plural, n) { 122 | var translated = Documentation.TRANSLATIONS[singular]; 123 | if (typeof translated == 'undefined') 124 | return (n == 1) ? singular : plural; 125 | return translated[Documentation.PLURALEXPR(n)]; 126 | }, 127 | 128 | addTranslations : function(catalog) { 129 | for (var key in catalog.messages) 130 | this.TRANSLATIONS[key] = catalog.messages[key]; 131 | this.PLURAL_EXPR = new Function('n', 'return +(' + catalog.plural_expr + ')'); 132 | this.LOCALE = catalog.locale; 133 | }, 134 | 135 | /** 136 | * add context elements like header anchor links 137 | */ 138 | addContextElements : function() { 139 | $('div[id] > :header:first').each(function() { 140 | $('\u00B6'). 141 | attr('href', '#' + this.id). 142 | attr('title', _('Permalink to this headline')). 143 | appendTo(this); 144 | }); 145 | $('dt[id]').each(function() { 146 | $('\u00B6'). 147 | attr('href', '#' + this.id). 148 | attr('title', _('Permalink to this definition')). 149 | appendTo(this); 150 | }); 151 | }, 152 | 153 | /** 154 | * workaround a firefox stupidity 155 | */ 156 | fixFirefoxAnchorBug : function() { 157 | if (document.location.hash && $.browser.mozilla) 158 | window.setTimeout(function() { 159 | document.location.href += ''; 160 | }, 10); 161 | }, 162 | 163 | /** 164 | * highlight the search words provided in the url in the text 165 | */ 166 | highlightSearchWords : function() { 167 | var params = $.getQueryParameters(); 168 | var terms = (params.highlight) ? params.highlight[0].split(/\s+/) : []; 169 | if (terms.length) { 170 | var body = $('div.body'); 171 | window.setTimeout(function() { 172 | $.each(terms, function() { 173 | body.highlightText(this.toLowerCase(), 'highlighted'); 174 | }); 175 | }, 10); 176 | $('

' + _('Hide Search Matches') + '

') 178 | .appendTo($('#searchbox')); 179 | } 180 | }, 181 | 182 | /** 183 | * init the domain index toggle buttons 184 | */ 185 | initIndexTable : function() { 186 | var togglers = $('img.toggler').click(function() { 187 | var src = $(this).attr('src'); 188 | var idnum = $(this).attr('id').substr(7); 189 | $('tr.cg-' + idnum).toggle(); 190 | if (src.substr(-9) == 'minus.png') 191 | $(this).attr('src', src.substr(0, src.length-9) + 'plus.png'); 192 | else 193 | $(this).attr('src', src.substr(0, src.length-8) + 'minus.png'); 194 | }).css('display', ''); 195 | if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) { 196 | togglers.click(); 197 | } 198 | }, 199 | 200 | /** 201 | * helper function to hide the search marks again 202 | */ 203 | hideSearchWords : function() { 204 | $('#searchbox .highlight-link').fadeOut(300); 205 | $('span.highlighted').removeClass('highlighted'); 206 | }, 207 | 208 | /** 209 | * make the url absolute 210 | */ 211 | makeURL : function(relativeURL) { 212 | return DOCUMENTATION_OPTIONS.URL_ROOT + '/' + relativeURL; 213 | }, 214 | 215 | /** 216 | * get the current relative url 217 | */ 218 | getCurrentURL : function() { 219 | var path = document.location.pathname; 220 | var parts = path.split(/\//); 221 | $.each(DOCUMENTATION_OPTIONS.URL_ROOT.split(/\//), function() { 222 | if (this == '..') 223 | parts.pop(); 224 | }); 225 | var url = parts.join('/'); 226 | return path.substring(url.lastIndexOf('/') + 1, path.length - 1); 227 | } 228 | }; 229 | 230 | // quick alias for translations 231 | _ = Documentation.gettext; 232 | 233 | $(document).ready(function() { 234 | Documentation.init(); 235 | }); 236 | -------------------------------------------------------------------------------- /doc/_build/html/_static/down-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/html/_static/down-pressed.png -------------------------------------------------------------------------------- /doc/_build/html/_static/down.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/html/_static/down.png -------------------------------------------------------------------------------- /doc/_build/html/_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/html/_static/file.png -------------------------------------------------------------------------------- /doc/_build/html/_static/minus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/html/_static/minus.png -------------------------------------------------------------------------------- /doc/_build/html/_static/plus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/html/_static/plus.png -------------------------------------------------------------------------------- /doc/_build/html/_static/pygments.css: -------------------------------------------------------------------------------- 1 | .highlight .hll { background-color: #ffffcc } 2 | .highlight { background: #eeffcc; } 3 | .highlight .c { color: #408090; font-style: italic } /* Comment */ 4 | .highlight .err { border: 1px solid #FF0000 } /* Error */ 5 | .highlight .k { color: #007020; font-weight: bold } /* Keyword */ 6 | .highlight .o { color: #666666 } /* Operator */ 7 | .highlight .cm { color: #408090; font-style: italic } /* Comment.Multiline */ 8 | .highlight .cp { color: #007020 } /* Comment.Preproc */ 9 | .highlight .c1 { color: #408090; font-style: italic } /* Comment.Single */ 10 | .highlight .cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */ 11 | .highlight .gd { color: #A00000 } /* Generic.Deleted */ 12 | .highlight .ge { font-style: italic } /* Generic.Emph */ 13 | .highlight .gr { color: #FF0000 } /* Generic.Error */ 14 | .highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ 15 | .highlight .gi { color: #00A000 } /* Generic.Inserted */ 16 | .highlight .go { color: #333333 } /* Generic.Output */ 17 | .highlight .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ 18 | .highlight .gs { font-weight: bold } /* Generic.Strong */ 19 | .highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ 20 | .highlight .gt { color: #0044DD } /* Generic.Traceback */ 21 | .highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ 22 | .highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ 23 | .highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ 24 | .highlight .kp { color: #007020 } /* Keyword.Pseudo */ 25 | .highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ 26 | .highlight .kt { color: #902000 } /* Keyword.Type */ 27 | .highlight .m { color: #208050 } /* Literal.Number */ 28 | .highlight .s { color: #4070a0 } /* Literal.String */ 29 | .highlight .na { color: #4070a0 } /* Name.Attribute */ 30 | .highlight .nb { color: #007020 } /* Name.Builtin */ 31 | .highlight .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ 32 | .highlight .no { color: #60add5 } /* Name.Constant */ 33 | .highlight .nd { color: #555555; font-weight: bold } /* Name.Decorator */ 34 | .highlight .ni { color: #d55537; font-weight: bold } /* Name.Entity */ 35 | .highlight .ne { color: #007020 } /* Name.Exception */ 36 | .highlight .nf { color: #06287e } /* Name.Function */ 37 | .highlight .nl { color: #002070; font-weight: bold } /* Name.Label */ 38 | .highlight .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ 39 | .highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */ 40 | .highlight .nv { color: #bb60d5 } /* Name.Variable */ 41 | .highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */ 42 | .highlight .w { color: #bbbbbb } /* Text.Whitespace */ 43 | .highlight .mf { color: #208050 } /* Literal.Number.Float */ 44 | .highlight .mh { color: #208050 } /* Literal.Number.Hex */ 45 | .highlight .mi { color: #208050 } /* Literal.Number.Integer */ 46 | .highlight .mo { color: #208050 } /* Literal.Number.Oct */ 47 | .highlight .sb { color: #4070a0 } /* Literal.String.Backtick */ 48 | .highlight .sc { color: #4070a0 } /* Literal.String.Char */ 49 | .highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ 50 | .highlight .s2 { color: #4070a0 } /* Literal.String.Double */ 51 | .highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ 52 | .highlight .sh { color: #4070a0 } /* Literal.String.Heredoc */ 53 | .highlight .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ 54 | .highlight .sx { color: #c65d09 } /* Literal.String.Other */ 55 | .highlight .sr { color: #235388 } /* Literal.String.Regex */ 56 | .highlight .s1 { color: #4070a0 } /* Literal.String.Single */ 57 | .highlight .ss { color: #517918 } /* Literal.String.Symbol */ 58 | .highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */ 59 | .highlight .vc { color: #bb60d5 } /* Name.Variable.Class */ 60 | .highlight .vg { color: #bb60d5 } /* Name.Variable.Global */ 61 | .highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */ 62 | .highlight .il { color: #208050 } /* Literal.Number.Integer.Long */ -------------------------------------------------------------------------------- /doc/_build/html/_static/sidebar.js: -------------------------------------------------------------------------------- 1 | /* 2 | * sidebar.js 3 | * ~~~~~~~~~~ 4 | * 5 | * This script makes the Sphinx sidebar collapsible. 6 | * 7 | * .sphinxsidebar contains .sphinxsidebarwrapper. This script adds 8 | * in .sphixsidebar, after .sphinxsidebarwrapper, the #sidebarbutton 9 | * used to collapse and expand the sidebar. 10 | * 11 | * When the sidebar is collapsed the .sphinxsidebarwrapper is hidden 12 | * and the width of the sidebar and the margin-left of the document 13 | * are decreased. When the sidebar is expanded the opposite happens. 14 | * This script saves a per-browser/per-session cookie used to 15 | * remember the position of the sidebar among the pages. 16 | * Once the browser is closed the cookie is deleted and the position 17 | * reset to the default (expanded). 18 | * 19 | * :copyright: Copyright 2007-2013 by the Sphinx team, see AUTHORS. 20 | * :license: BSD, see LICENSE for details. 21 | * 22 | */ 23 | 24 | $(function() { 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | // global elements used by the functions. 34 | // the 'sidebarbutton' element is defined as global after its 35 | // creation, in the add_sidebar_button function 36 | var bodywrapper = $('.bodywrapper'); 37 | var sidebar = $('.sphinxsidebar'); 38 | var sidebarwrapper = $('.sphinxsidebarwrapper'); 39 | 40 | // for some reason, the document has no sidebar; do not run into errors 41 | if (!sidebar.length) return; 42 | 43 | // original margin-left of the bodywrapper and width of the sidebar 44 | // with the sidebar expanded 45 | var bw_margin_expanded = bodywrapper.css('margin-left'); 46 | var ssb_width_expanded = sidebar.width(); 47 | 48 | // margin-left of the bodywrapper and width of the sidebar 49 | // with the sidebar collapsed 50 | var bw_margin_collapsed = '.8em'; 51 | var ssb_width_collapsed = '.8em'; 52 | 53 | // colors used by the current theme 54 | var dark_color = $('.related').css('background-color'); 55 | var light_color = $('.document').css('background-color'); 56 | 57 | function sidebar_is_collapsed() { 58 | return sidebarwrapper.is(':not(:visible)'); 59 | } 60 | 61 | function toggle_sidebar() { 62 | if (sidebar_is_collapsed()) 63 | expand_sidebar(); 64 | else 65 | collapse_sidebar(); 66 | } 67 | 68 | function collapse_sidebar() { 69 | sidebarwrapper.hide(); 70 | sidebar.css('width', ssb_width_collapsed); 71 | bodywrapper.css('margin-left', bw_margin_collapsed); 72 | sidebarbutton.css({ 73 | 'margin-left': '0', 74 | 'height': bodywrapper.height() 75 | }); 76 | sidebarbutton.find('span').text('»'); 77 | sidebarbutton.attr('title', _('Expand sidebar')); 78 | document.cookie = 'sidebar=collapsed'; 79 | } 80 | 81 | function expand_sidebar() { 82 | bodywrapper.css('margin-left', bw_margin_expanded); 83 | sidebar.css('width', ssb_width_expanded); 84 | sidebarwrapper.show(); 85 | sidebarbutton.css({ 86 | 'margin-left': ssb_width_expanded-12, 87 | 'height': bodywrapper.height() 88 | }); 89 | sidebarbutton.find('span').text('«'); 90 | sidebarbutton.attr('title', _('Collapse sidebar')); 91 | document.cookie = 'sidebar=expanded'; 92 | } 93 | 94 | function add_sidebar_button() { 95 | sidebarwrapper.css({ 96 | 'float': 'left', 97 | 'margin-right': '0', 98 | 'width': ssb_width_expanded - 28 99 | }); 100 | // create the button 101 | sidebar.append( 102 | '
«
' 103 | ); 104 | var sidebarbutton = $('#sidebarbutton'); 105 | light_color = sidebarbutton.css('background-color'); 106 | // find the height of the viewport to center the '<<' in the page 107 | var viewport_height; 108 | if (window.innerHeight) 109 | viewport_height = window.innerHeight; 110 | else 111 | viewport_height = $(window).height(); 112 | sidebarbutton.find('span').css({ 113 | 'display': 'block', 114 | 'margin-top': (viewport_height - sidebar.position().top - 20) / 2 115 | }); 116 | 117 | sidebarbutton.click(toggle_sidebar); 118 | sidebarbutton.attr('title', _('Collapse sidebar')); 119 | sidebarbutton.css({ 120 | 'color': '#FFFFFF', 121 | 'border-left': '1px solid ' + dark_color, 122 | 'font-size': '1.2em', 123 | 'cursor': 'pointer', 124 | 'height': bodywrapper.height(), 125 | 'padding-top': '1px', 126 | 'margin-left': ssb_width_expanded - 12 127 | }); 128 | 129 | sidebarbutton.hover( 130 | function () { 131 | $(this).css('background-color', dark_color); 132 | }, 133 | function () { 134 | $(this).css('background-color', light_color); 135 | } 136 | ); 137 | } 138 | 139 | function set_position_from_cookie() { 140 | if (!document.cookie) 141 | return; 142 | var items = document.cookie.split(';'); 143 | for(var k=0; k 4 | 5 | 6 | 7 | 8 | 9 | 10 | Index — ConCoord 1.1.0 documentation 11 | 12 | 13 | 14 | 15 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |
31 |

Navigation

32 | 38 |
39 | 40 |
41 |
42 |
43 |
44 | 45 | 46 |

Index

47 | 48 |
49 | 50 |
51 | 52 | 53 |
54 |
55 |
56 |
57 |
58 | 59 | 60 | 61 | 73 | 74 |
75 |
76 |
77 |
78 |
79 |

Navigation

80 | 86 |
87 |
88 | © Copyright 2012, Deniz Altinbuken, Emin Gun Sirer. 89 | Created using Sphinx 1.2b1. 90 |
91 | 92 | -------------------------------------------------------------------------------- /doc/_build/html/install.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Installing ConCoord — ConCoord 1.1.0 documentation 10 | 11 | 12 | 13 | 14 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
32 |

Navigation

33 | 45 |
46 | 47 |
48 |
49 |
50 |
51 | 52 |
53 |

Installing ConCoord

54 |
55 |

Requirements

56 |

The minimum requirements for ConCoord are:

57 | 
- python 2.7.2 or later
 58 | - dnspython-1.9.4
 59 | - msgpack-python
60 |
61 |
62 |
63 |

Installation

64 |

ConCoord can be installed from source with:

65 | 
$ python setup.py install
 66 |
67 |
68 |

ConCoord is also available for install through PyPI:

69 | 
$ pip install concoord
 70 |
71 |
72 |
73 |
74 | 75 | 76 |
77 |
78 |
79 |
80 |
81 |

Table Of Contents

82 | 89 | 90 |

Previous topic

91 |

Overview

93 |

Next topic

94 |

Getting Started

96 |

This Page

97 | 101 | 113 | 114 |
115 |
116 |
117 |
118 |
119 |

Navigation

120 | 132 |
133 |
134 | © Copyright 2012, Deniz Altinbuken, Emin Gun Sirer. 135 | Created using Sphinx 1.2b1. 136 |
137 | 138 | -------------------------------------------------------------------------------- /doc/_build/html/objects.inv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/_build/html/objects.inv -------------------------------------------------------------------------------- /doc/_build/html/overview.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Overview — ConCoord 1.1.0 documentation 10 | 11 | 12 | 13 | 14 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
32 |

Navigation

33 | 45 |
46 | 47 |
48 |
49 |
50 |
51 | 52 |
53 |

Overview

54 |

ConCoord is a novel coordination service that provides replication and 55 | synchronization support for large-scale distributed systems. ConCoord 56 | employs an object-oriented approach, in which the system creates 57 | and maintains live replicas for Python objects written by the user. 58 | ConCoord converts these Python objects into Paxos Replicated State 59 | Machines (RSM) and enables clients to do method invocations on them 60 | transparently as if they are local objects. ConCoord uses these 61 | replicated objects to implement coordination and synchronization 62 | constructs in large-scale distributed systems, in effect establishing 63 | a transparent way of providing a coordination service.

64 | _images/concoord.jpg 65 |
66 | 67 | 68 |
69 |
70 |
71 |
72 |
73 |

Previous topic

74 |

Welcome to ConCoord’s documentation

76 |

Next topic

77 |

Installing ConCoord

79 |

This Page

80 | 84 | 96 | 97 |
98 |
99 |
100 |
101 |
102 |

Navigation

103 | 115 |
116 |
117 | © Copyright 2012, Deniz Altinbuken, Emin Gun Sirer. 118 | Created using Sphinx 1.2b1. 119 |
120 | 121 | -------------------------------------------------------------------------------- /doc/_build/html/search.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Search — ConCoord 1.1.0 documentation 10 | 11 | 12 | 13 | 14 | 23 | 24 | 25 | 26 | 27 | 28 | 31 | 32 | 33 | 34 | 35 | 36 | 37 |
38 |

Navigation

39 | 45 |
46 | 47 |
48 |
49 |
50 |
51 | 52 |

Search

53 |
54 | 55 |

56 | Please activate JavaScript to enable the search 57 | functionality. 58 |

59 |
60 |

61 | From here you can search these documents. Enter your search 62 | words into the box below and click "search". Note that the search 63 | function will automatically search for all of the words. Pages 64 | containing fewer words won't appear in the result list. 65 |

66 |
67 | 68 | 69 | 70 |
71 | 72 |
73 | 74 |
75 | 76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |

Navigation

87 | 93 |
94 |
95 | © Copyright 2012, Deniz Altinbuken, Emin Gun Sirer. 96 | Created using Sphinx 1.2b1. 97 |
98 | 99 | -------------------------------------------------------------------------------- /doc/_build/html/searchindex.js: -------------------------------------------------------------------------------- 1 | Search.setIndex({envversion:42,terms:{all:[5,4],code:5,queri:[5,4],secondli:4,unblock:0,follow:[0,5,4],disk:[5,4],privat:4,depend:5,zone:[5,4],send:[5,4],under:[0,5,4],esc:4,emploi:2,sourc:[],everi:[5,4],util:0,veri:0,retriev:[5,4],tri:4,did:[0,4],tencount:0,dig:[5,4],"try":0,round:[],configfilepath:[],ten:0,pass:4,download:4,port:[5,4],counterproxi:[],index:1,deleg:[5,4],addr:[5,4],section:4,abl:[0,5,4],current:[5,4],directori:[0,4],"new":[5,4],method:[0,2,5],gener:0,rlock:0,let:[0,5,4],address:[5,4],sinc:5,valu:0,acceptor:[],search:1,"__repr__":0,chang:[5,4],extra:0,reiniti:[0,5],modul:[0,1],"__concoordinit__":[0,5],filenam:4,instal:[],decrement:0,txt:[5,4],establish:2,from:[0,5,3,4],would:5,regist:4,two:[0,5,4],live:[0,2],call:[0,5],type:[],exit:[0,5,4],relat:0,warn:4,sourcecod:[],behalf:[5,4],account:[5,4],acquir:0,join:[5,4],aws_secret_access_kei:[5,4],setup:[5,3,4],work:5,remain:0,can:[0,5,3,4],meet:0,def:0,lock:0,sudo:[5,4],accept:[0,5,4],minimum:3,want:[0,5,4],multipl:0,secur:[0,5,4],anoth:[5,4],paxo:2,write:[5,4],how:[],answer:[5,4],config:4,updat:[0,5,4],after:[0,5,4],befor:[0,5,4],demonstr:[],ani:[0,5,4],bind:[5,4],counter:[0,5,4],explicit:0,inform:[5,4],maintain:2,concoordifi:[],counterdomain:[5,4],oper:0,route53kei:5,elig:4,help:[0,5,4],move:4,dsemaphor:0,through:[0,5,3,4],reconnect:0,dynam:[5,4],paramet:4,disconnect:0,consensu:[],therefor:0,them:[0,2],messag:[0,5,4],"return":[0,5,4],thei:[2,4],python:[],safe:[0,4],initi:0,now:[0,4],name:[],edit:[],writetodisk:[5,4],getvalu:[0,5],token:0,mode:[0,5,4],debug:[5,4],found:0,side:0,domain:[5,4],keypair:4,usernam:[5,4],connect:[],our:[],orient:2,special:0,out:0,shown:[5,4],msgpack:3,safeti:0,network:0,addsshkei:4,content:1,print:[0,4],boundedsemaphor:0,proxi:[0,5],barrier:0,advanc:[],given:[5,4],releas:0,bootrstrap:[],thread:[],launch:4,synchron:[],keep:[0,5,4],registrar:[5,4],support:[0,2,4],first:[0,5,4],origin:[0,5],semaphor:0,onc:[0,5,4],public_dn:4,alreadi:[5,4],done:[5,4],least:5,open:4,iam:[5,4],differ:0,script:[0,4],interact:[],system:[2,5,4],construct:2,networklogg:0,master:[],similarli:4,includ:[0,5,4],rsm:2,shell:[],consol:[0,5,4],option:5,especi:4,copi:[],specifi:5,kind:0,"_concoord_command":0,provid:[0,2,5,4],"wr\u0131tten":[],str:0,bootstrap:[5,4],comput:4,credenti:[5,4],argument:[0,4],have:[0,5,4],need:[0,5,4],built:4,self:0,ipaddr:5,note:[0,5],also:[5,3,4],client:[],build:[],which:2,even:0,sure:[5,4],distribut:[0,2,5],track:[5,4],object:[],most:[5,4],deploi:5,pair:[5,4],"class":0,renam:0,later:3,request:[5,4],dot:[0,5,4],route53id:5,show:[0,5,4],text:4,"__str__":0,verbos:0,find:4,absolut:5,onli:5,explicitli:4,locat:[5,4],just:0,configur:[],activ:[5,4],written:2,should:[0,5,4],local:[0,2,4],get:[],familiar:4,pypi:3,repr:0,cannot:4,ssh:4,namecheap:[5,4],requir:[],enabl:[2,4],"public":4,remot:4,common:5,privileg:[5,4],where:[0,5,4],view:[5,4],respond:[5,4],set:[],commandlin:4,see:[5,4],result:5,fail:4,best:4,state:[0,2],between:[],"import":[0,5],approach:[0,2],kei:4,toler:0,both:[0,5],addusernam:4,boto:[5,4],fault:0,visudo:4,etc:[5,4],instanc:[],dnspython:3,com:[5,4],point:[0,5,4],instanti:0,overview:[],openreplica:[],walk:0,linux:4,respect:5,pycon:[],west:4,three:[],secret:[],treat:0,basic:0,convert:[0,2],anywher:0,coordin:[0,2,5],nameserv:[],"case":0,replic:2,novel:2,look:[5,4],servic:2,batch:0,"while":[0,5,4],exist:0,invoc:[0,2,5],increment:[0,5],readi:[5,4],itself:[5,4],"__init__":0,develop:4,receiv:[5,4],make:[0,5,4],same:0,tutori:[],tencounterproxi:0,http:[5,4],openreplicanameserv:[],effect:2,rais:0,user:[0,2,4],login:4,off:[0,5,4],firstli:4,well:4,without:[0,5,4],domainnam:[5,4],exampl:[0,5,4],command:[0,5,4],thi:[0,5,4],filesystem:0,"_concoord":[5,4],protocol:[],execut:4,boot:[5,4],route53:[],aws_access_key_id:[5,4],web:4,easi:5,makefil:0,except:0,add:[0,4],logger:[0,5,4],save:[0,4],transpar:2,threadingobject:0,know:4,press:4,requiretti:4,bit:4,insert:4,resid:0,like:[0,4],specif:[0,5],amazonaw:4,manual:[],server:[],page:1,"export":0,proper:5,librari:0,scale:2,sshkeyfilenam:4,easili:[0,5,4],pem:4,larg:2,condit:0,foo:0,machin:[2,5,4],run:[0,5,4],usag:[0,5,4],host:[0,5,4],peer:[5,4],manag:4,srv:[5,4],act:[5,4],"_tcp":[5,4],block:0,own:[5,4],pythonpath:0,objectnam:[0,5,4],lastli:4,bound:5,automat:[0,4],your:[0,5,4],durabl:[],accordingli:0,log:[],wai:[0,2,5,4],addnod:4,"long":5,avail:3,start:[],interfac:4,editor:4,sshkei:4,"function":[0,5],godaddi:[5,4],offer:5,tupl:[5,4],bundl:[],semaphoreproxi:0,nsipaddr:[],line:4,"true":0,count:0,consist:[],"default":[0,4],access:[5,4],record:[5,4],below:[0,5,4],logdaemon:0,securitytoken:0,creat:[],cover:[5,4],dure:0,doesn:[5,4],replica:[],implement:[0,2],aliv:5,file:[0,5,4],publicdn:4,pip:[5,3,4],check:[0,4],when:[0,5,4],masterdomain:[],rememb:0,test:0,you:[0,5,4],node:[],semaphoredomain:[],nameservercoord:[],proxytyp:0,public_dns_nam:4,time:[5,4]},objtypes:{},objnames:{},filenames:["tutorial-advanced","index","overview","install","openreplica","tutorial"],titles:["ConCoordifying Python Objects","Welcome to ConCoord’s documentation","Overview","Installing ConCoord","OpenReplica","Getting Started"],objects:{},titleterms:{set:4,creat:0,overview:2,indic:1,replica:[5,4],connect:5,tabl:1,instal:3,concoord:[1,5,3,4],route53:[5,4],welcom:1,start:[5,4],rout:[],configur:4,acceptor:[],how:4,instanc:4,master:[5,4],document:1,node:[5,4],sourc:0,slave:[],get:5,python:0,bundl:0,object:[0,5],standalon:[],amazon:[5,4],openreplica:4,nameserv:[],requir:3,log:0,name:[5,4],thread:0,tutori:[],synchron:0,advanc:[],server:[0,5,4],client:0,concoordifi:0,ec2:4}}) -------------------------------------------------------------------------------- /doc/source/_static/concoord.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/doc/source/_static/concoord.jpg -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | .. ConCoord documentation master file, created by 2 | sphinx-quickstart on Tue Mar 27 16:31:41 2012. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to ConCoord's documentation 7 | =================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 3 13 | 14 | overview 15 | install 16 | tutorial 17 | openreplica 18 | tutorial-advanced 19 | 20 | Indices and tables 21 | ================== 22 | 23 | * :ref:`genindex` 24 | * :ref:`modindex` 25 | * :ref:`search` 26 | -------------------------------------------------------------------------------- /doc/source/install.rst: -------------------------------------------------------------------------------- 1 | Installing ConCoord 2 | =================== 3 | 4 | Requirements 5 | ------------------------ 6 | The minimum requirements for ConCoord are:: 7 | 8 | - python 2.7.2 or later 9 | - dnspython-1.9.4 10 | - msgpack-python 11 | 12 | Installation 13 | ------------------------ 14 | ConCoord can be installed from source with: 15 | 16 | .. sourcecode:: console 17 | 18 | $ python setup.py install 19 | 20 | ConCoord is also available for install through PyPI: 21 | 22 | .. sourcecode:: console 23 | 24 | $ pip install concoord 25 | 26 | -------------------------------------------------------------------------------- /doc/source/overview.rst: -------------------------------------------------------------------------------- 1 | Overview 2 | ======== 3 | 4 | ConCoord is a novel coordination service that provides replication and 5 | synchronization support for large-scale distributed systems. ConCoord 6 | employs an object-oriented approach, in which the system creates 7 | and maintains live replicas for Python objects written by the user. 8 | ConCoord converts these Python objects into Paxos Replicated State 9 | Machines (RSM) and enables clients to do method invocations on them 10 | transparently as if they are local objects. ConCoord uses these 11 | replicated objects to implement coordination and synchronization 12 | constructs in large-scale distributed systems, in effect establishing 13 | a transparent way of providing a coordination service. 14 | 15 | .. image:: _static/concoord.jpg 16 | :align: center 17 | 18 | 19 | -------------------------------------------------------------------------------- /doc/source/tutorial-advanced.rst: -------------------------------------------------------------------------------- 1 | ConCoordifying Python Objects 2 | ----------------------------- 3 | For cases when the objects included in the ConCoord distribution do 4 | not meet your coordination needs, ConCoord lets you convert your 5 | local Python objects into distributable objects very easily. 6 | 7 | To walk through the ConCoord approach, you will use a different 8 | Counter object that increments and decrements by ten, namely 9 | tencounter.py. Once you install ConCoord, you can create coordination 10 | objects and save them anywhere in your filesystem. After you create 11 | ``tencounter.py``, you can save it under ``/foo/tencounter.py``: 12 | 13 | .. sourcecode:: python 14 | 15 | class TenCounter: 16 | def __init__(self, value=0): 17 | self.value = value 18 | 19 | def decrement(self): 20 | self.value -= 10 21 | 22 | def increment(self): 23 | self.value += 10 24 | 25 | def getvalue(self): 26 | return self.value 27 | 28 | def __str__(self): 29 | return "The counter value is %d" % self.value 30 | 31 | Once you have created the object, update your ``PYTHONPATH`` accordingly, 32 | so that the objects can be found and imported: 33 | 34 | .. sourcecode:: console 35 | 36 | $ export PYTHONPATH=$PYTHONPATH:/foo/ 37 | 38 | Clients will use a proxy object to do method calls on the object. 39 | To create the proxy object automatically, you can use the following 40 | command: 41 | 42 | .. sourcecode:: console 43 | 44 | $ concoord object -o tencounter.TenCounter 45 | 46 | ``Usage: concoord object [-h] [-o OBJECTNAME] [-t SECURITYTOKEN] [-p PROXYTYPE] [-s] [-v]`` 47 | 48 | where, 49 | ``-h, --help`` show this help message and exit 50 | 51 | ``-o OBJECTNAME, --objectname OBJECTNAME`` client object dotted name module.Class 52 | 53 | ``-t SECURITYTOKEN, --token SECURITYTOKEN`` security token 54 | 55 | ``-p PROXYTYPE, --proxytype PROXYTYPE`` 0: basic, 1: blocking, 2: client-side batching, 3: server-side batching 56 | 57 | ``-s, --safe`` safety checking on/off 58 | 59 | ``-v, --verbose`` verbose mode on/off 60 | 61 | This script will create a proxy file under the directory that the 62 | object resides (i.e. ``/foo/``): 63 | 64 | ``/foo/tencounterproxy.py`` := the proxy that can be used by the client 65 | 66 | IMPORTANT NOTE: ConCoord objects treat ``__init__`` functions specially in 67 | two ways: 68 | 69 | 1) When Replicas go live, the object is instantiated calling the 70 | ``__init__`` without any arguments. Therefore, while implementing 71 | coordination objects, the ``__init__`` method should be implemented to 72 | be able to run without explicit arguments. You can use defaults to 73 | implement an ``__init__`` method that accepts arguments. 74 | 75 | 2) In the proxy created, the ``__init__`` function is used to initialize 76 | the Client-Replica connection. This way, multiple clients can 77 | connect to a ConCoord instance without reinitializing the 78 | object. During proxy generation, the original ``__init__`` function is 79 | renamed as ``__concoordinit__``, to reinitialize the object the user can 80 | call ``__concoordinit__`` at any point. 81 | 82 | After this point on, you can use TenCounter just like we use Counter before. 83 | 84 | Creating Source Bundles 85 | ----------------------- 86 | 87 | You can create bundles to use at the server and client sides using the 88 | ``Makefile`` provided under ``concoord/`` 89 | 90 | Remember to add the objects you have created in these bundles. 91 | 92 | Creating a Server Bundle 93 | ~~~~~~~~~~~~~~~~~~~~~~~~ 94 | 95 | To create a bundle that can run replicas: 96 | 97 | .. sourcecode:: console 98 | 99 | $ make server 100 | 101 | Creating a Client Bundle 102 | ~~~~~~~~~~~~~~~~~~~~~~~~ 103 | 104 | To create a bundle that can run a client and connect to an existing 105 | ConCoord instance: 106 | 107 | .. sourcecode:: console 108 | 109 | $ make client 110 | 111 | Logging 112 | ------- 113 | 114 | We have two kinds of loggers for ConCoord: 115 | 116 | * Console Logger 117 | * Network Logger 118 | 119 | Both of these loggers are included under ``utils.py``. To start the 120 | ``NetworkLogger``, use the ``logdaemon.py`` on the host you want to keep the 121 | Logger. 122 | 123 | Synchronization & Threading 124 | --------------------------- 125 | 126 | ConCoord provides a distributed and fault-tolerant threading 127 | library. The library includes: 128 | 129 | * Lock 130 | * RLock 131 | * Semaphore 132 | * BoundedSemaphore 133 | * Barrier 134 | * Condition 135 | 136 | The implementations of distributed synchronization objects follow the 137 | implementations in the Python threading library. We will walk through 138 | an example below using the ``Semaphore`` object under 139 | ``concoord/object/semaphore.py`` 140 | 141 | In the blocking object implementation, the method invocations that use 142 | an object from the threading library requires an extra argument 143 | ``_concoord_command``. This argument is used by the calling Replica node 144 | to relate any blocking/unblocking method invocation to a specific 145 | client. This way, even if the client disconnects and reconnects, the 146 | ConCoord instance will remain in a safe state. 147 | 148 | .. sourcecode:: python 149 | 150 | from concoord.threadingobject.dsemaphore import DSemaphore 151 | 152 | class Semaphore: 153 | def __init__(self, count=1): 154 | self.semaphore = DSemaphore(count) 155 | 156 | def __repr__(self): 157 | return repr(self.semaphore) 158 | 159 | def acquire(self, _concoord_command): 160 | try: 161 | return self.semaphore.acquire(_concoord_command) 162 | except Exception as e: 163 | raise e 164 | 165 | def release(self, _concoord_command): 166 | try: 167 | return self.semaphore.release(_concoord_command) 168 | except Exception as e: 169 | raise e 170 | 171 | def __str__(self): 172 | return str(self.semaphore) 173 | 174 | To create the proxy for this blocking object we will use the following command: 175 | 176 | .. sourcecode:: console 177 | 178 | $ concoord object -o concoord.object.semaphore.Semaphore -p 1 179 | 180 | This command creates the proxy that supports blocking operations. Now 181 | you can use blocking objects just like basic ConCoord objects. First, 182 | we start the replica nodes the same way we did before as follows: 183 | 184 | .. sourcecode:: console 185 | 186 | $ concoord replica -o concoord.object.semaphore.Semaphore -a 127.0.0.1 -p 14000 187 | 188 | To test the functionality, you can use multiple clients or print out the ``Semaphore`` object as follows: 189 | 190 | .. sourcecode:: pycon 191 | 192 | >>> from semaphoreproxy import Semaphore 193 | >>> s = Semaphore("127.0.0.1:14000") 194 | >>> s.acquire() 195 | True 196 | >>> i = 10 197 | >>> i += 5 198 | >>> s 199 | 200 | >>> s.release() 201 | >>> s 202 | 203 | >>> 204 | -------------------------------------------------------------------------------- /doc/source/tutorial.rst: -------------------------------------------------------------------------------- 1 | Getting Started 2 | --------------- 3 | 4 | To use ConCoord you need a Python object that can be used for the 5 | coordination of your distributed system. In the ConCoord distribution, 6 | we offer ready-to-use objects that cover the most common coordination 7 | needs. So first, let's start a ConCoord instance with an object in 8 | the distribution, namely Counter under ``concoord/object/counter.py``. 9 | 10 | Starting Nodes 11 | -------------- 12 | 13 | To distribute your object you should start at least one replica. 14 | 15 | Starting Replica Nodes 16 | ~~~~~~~~~~~~~~~~~~~~~~ 17 | 18 | To start a bootstrap replica node that doesn't need to be connected to 19 | another replica: 20 | 21 | .. sourcecode:: console 22 | 23 | $ concoord replica -o concoord.object.counter.Counter -a 127.0.0.1 -p 14000 24 | 25 | To start replica nodes to join an active ConCoord instance, use the 26 | following command to connect to another replica: 27 | 28 | .. sourcecode:: console 29 | 30 | $ concoord replica -o concoord.object.counter.Counter -b 127.0.0.1:14000 -a 127.0.0.1 -p 14001 31 | 32 | Note that you can specify the port and the address of any node with 33 | options ``-p`` and ``-a`` respectively. The nodes can also be run in 34 | the debug mode or with a logger with the commands shown below: 35 | 36 | ``Usage: concoord replica [-h] [-a ADDR] [-p PORT] [-b BOOTSTRAP] [-o OBJECTNAME] [-l LOGGER] [-n DOMAIN] [-r] [-w] [-d]`` 37 | 38 | where, 39 | ``-h, --help`` show this help message and exit 40 | 41 | ``-a ADDR, --addr ADDR`` address for the node 42 | 43 | ``-p PORT, --port PORT`` port for the node 44 | 45 | ``-b BOOTSTRAP, --boot BOOTSTRAP`` address:port tuple for the bootstrap peer 46 | 47 | ``-o OBJECTNAME, --objectname OBJECTNAME`` client object dotted name 48 | 49 | ``-l LOGGER, --logger LOGGER`` logger address 50 | 51 | ``-n DOMAIN, --domainname DOMAIN`` domain name that the name server will accept queries for 52 | 53 | ``-r, --route53`` use Route53 54 | 55 | ``-w, --writetodisk`` writing to disk on/off 56 | 57 | ``-d, --debug`` debug on/off 58 | 59 | Starting Replicas as Name Servers 60 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 61 | 62 | You can dynamically locate nodes in a given ConCoord instance using 63 | DNS queries if the instance includes replicas that can act as name 64 | servers. There are two ways you can run a ConCoord Replica as a name 65 | server. 66 | 67 | * **Master Name Server:** Keeps track of the view and responds to DNS 68 | queries itself. Requires su privileges to bind to port 53. 69 | 70 | * **Route53 Name Server:** Keeps track of the view and updates an Amazon 71 | Route53 account. Amazon Route53 answers to DNS queries on behalf of 72 | the slave name server. Requires a ready-to-use Amazon Route53 73 | account. 74 | 75 | Master Name Server 76 | +++++++++++++++++++++ 77 | 78 | To use a replica node as a master name server first you have to setup 79 | the name server delegations (you can do this by updating the domain 80 | name server information of any domain name you own from the domain 81 | registrar you use (godaddy, namecheap etc.)). Once all the delegations 82 | are setup for the ip address the replica uses, you can start a replica 83 | node as a name server for ``counterdomain.com`` as follows: 84 | 85 | .. sourcecode:: console 86 | 87 | $ sudo concoord replica -o concoord.object.counter.Counter -a 127.0.0.1 -n counterdomain.com 88 | 89 | And to start the replica to join an already running ConCoord instance, 90 | provide the bootstrap: 91 | 92 | .. sourcecode:: console 93 | 94 | $ sudo concoord replica -o concoord.object.counter.Counter -a 127.0.0.1 -b 127.0.0.1:14000 -n counterdomain.com 95 | 96 | When the replica starts running, you can send queries for 97 | ``counterdomain.com`` and see the most current set of nodes as 98 | follows: 99 | 100 | .. sourcecode:: console 101 | 102 | $ dig -t a counterdomain.com # returns set of Replicas 103 | 104 | $ dig -t srv _concoord._tcp.counterdomain.com # returns set of Replicas with ports 105 | 106 | $ dig -t txt counterdomain.com # returns set of all nodes 107 | 108 | $ dig -t ns counterdomain.com # returns set of name servers 109 | 110 | 111 | If you want to run the name server without proper delegation setup, you 112 | can query the name server bound to ``127.0.0.1`` specifically as follows: 113 | 114 | .. sourcecode:: console 115 | 116 | $ dig -t txt counterdomain.com @127.0.0.1 # returns set of all nodes 117 | 118 | Note that this would only work for a, srv and txt queries, since ns 119 | queries require absolute dns names or origins, not an ip address. 120 | 121 | Amazon Route53 Name Server 122 | ++++++++++++++++++++++++++ 123 | 124 | First make sure that boto is installed on the machine you want to run 125 | the Route53 name server. You can easily install boto as follows:: 126 | 127 | $ pip install boto 128 | 129 | Before starting a name server connected to Amazon Route53, you should 130 | have a Route53 account set up and ready to receive requests. This is 131 | done through the AWS Console (http://console.aws.amazon.com/route53), by 132 | creating a new Hosted Zone to host your domain name. 133 | 134 | After your Route53 account is set up, the name server can update 135 | Route53 records every time the view of the system changes. 136 | 137 | To use the Name Server to update Amazon Route53, you should provide 138 | your ``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY``. You can retrieve 139 | these from the AWS Console (http://console.aws.amazon.com/iam/), by 140 | looking under the security credentials of the username that you used 141 | while creating the Hosted Zone for your domain name. Once you have the 142 | information, you can set up Route53 configuration easily as follows: 143 | 144 | .. sourcecode:: console 145 | 146 | $ concoord route53id [AWS_ACCESS_KEY_ID] 147 | $ concoord route53key [AWS_SECRET_ACCESS_KEY] 148 | 149 | Once you make sure that your Route53 account is set up and the 150 | configuration file includes your AWS credentials, you can start the 151 | replica with a name server as follows: 152 | 153 | .. sourcecode:: console 154 | 155 | $ concoord replica -o concoord.object.counter.Counter -n counterdomain.com -r 156 | 157 | When the replica starts running, you can send queries for 158 | ``counterdomain.com`` and see the most current set of nodes as follows: 159 | 160 | .. sourcecode:: console 161 | 162 | $ dig -t a counterdomain.com # returns set of Replicas 163 | 164 | $ dig -t srv _concoord._tcp.counterdomain.com # returns set of Replicas with ports 165 | 166 | $ dig -t txt counterdomain.com # returns set of all nodes 167 | 168 | $ dig -t ns counterdomain.com # returns set of name servers 169 | 170 | Connecting to ConCoord Objects 171 | ------------------------------ 172 | 173 | Once you have a ConCoord instance running with your object, it is easy 174 | to access your object. 175 | 176 | The proxy for the Counter object is also included in the distribution. 177 | You can import and use this proxy object in your code. Depending on 178 | how you set your name server up, you can access your object with the 179 | ``ipaddr:port`` pair or the domainname. In the example below, the 180 | ``ipaddr:port`` of both replica nodes are used. As a result, the 181 | client will be able to do method invocations on the object as long as 182 | one of the replicas is alive: 183 | 184 | .. sourcecode:: pycon 185 | 186 | >>> from concoord.proxy.counter import Counter 187 | >>> c = Counter("127.0.0.1:14000, 127.0.0.1:14001") 188 | >>> c.increment() 189 | >>> c.increment() 190 | >>> c.getvalue() 191 | 2 192 | 193 | At any point to reinitialize an object after it is deployed on 194 | replicas, you should call ``__concoordinit__`` function: 195 | 196 | .. sourcecode:: pycon 197 | 198 | >>> from concoord.proxy.counter import Counter 199 | >>> c = Counter("127.0.0.1:14000, 127.0.0.1:14001") 200 | >>> c.increment() 201 | >>> c.__concoordinit__() 202 | >>> c.increment() 203 | >>> c.getvalue() 204 | 1 -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | _site 2 | .sass-cache 3 | .jekyll-cache 4 | .jekyll-metadata 5 | vendor 6 | .DS_Store 7 | -------------------------------------------------------------------------------- /docs/404.html: -------------------------------------------------------------------------------- 1 | --- 2 | permalink: /404.html 3 | layout: default 4 | --- 5 | 6 | 19 | 20 |
21 |

404

22 | 23 |

Page not found :(

24 |

The requested page could not be found.

25 |
26 | -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | openreplica.org 2 | -------------------------------------------------------------------------------- /docs/Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | # Hello! This is where you manage which Jekyll version is used to run. 3 | # When you want to use a different version, change it below, save the 4 | # file and run `bundle install`. Run Jekyll with `bundle exec`, like so: 5 | # 6 | # bundle exec jekyll serve 7 | # 8 | # This will help ensure the proper Jekyll version is running. 9 | # Happy Jekylling! 10 | gem "jekyll", "~> 4.1.1" 11 | # This is the default theme for new Jekyll sites. You may change this to anything you like. 12 | gem "minima", "~> 2.5" 13 | # If you want to use GitHub Pages, remove the "gem "jekyll"" above and 14 | # uncomment the line below. To upgrade, run `bundle update github-pages`. 15 | # gem "github-pages", group: :jekyll_plugins 16 | # If you have any plugins, put them here! 17 | group :jekyll_plugins do 18 | gem "jekyll-feed", "~> 0.12" 19 | end 20 | 21 | # Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem 22 | # and associated library. 23 | platforms :mingw, :x64_mingw, :mswin, :jruby do 24 | gem "tzinfo", "~> 1.2" 25 | gem "tzinfo-data" 26 | end 27 | 28 | # Performance-booster for watching directories on Windows 29 | gem "wdm", "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin] 30 | 31 | -------------------------------------------------------------------------------- /docs/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | addressable (2.8.0) 5 | public_suffix (>= 2.0.2, < 5.0) 6 | colorator (1.1.0) 7 | concurrent-ruby (1.1.7) 8 | em-websocket (0.5.1) 9 | eventmachine (>= 0.12.9) 10 | http_parser.rb (~> 0.6.0) 11 | eventmachine (1.2.7) 12 | ffi (1.13.1) 13 | forwardable-extended (2.6.0) 14 | http_parser.rb (0.6.0) 15 | i18n (1.8.5) 16 | concurrent-ruby (~> 1.0) 17 | jekyll (4.1.1) 18 | addressable (~> 2.4) 19 | colorator (~> 1.0) 20 | em-websocket (~> 0.5) 21 | i18n (~> 1.0) 22 | jekyll-sass-converter (~> 2.0) 23 | jekyll-watch (~> 2.0) 24 | kramdown (~> 2.1) 25 | kramdown-parser-gfm (~> 1.0) 26 | liquid (~> 4.0) 27 | mercenary (~> 0.4.0) 28 | pathutil (~> 0.9) 29 | rouge (~> 3.0) 30 | safe_yaml (~> 1.0) 31 | terminal-table (~> 1.8) 32 | jekyll-feed (0.15.0) 33 | jekyll (>= 3.7, < 5.0) 34 | jekyll-sass-converter (2.1.0) 35 | sassc (> 2.0.1, < 3.0) 36 | jekyll-seo-tag (2.6.1) 37 | jekyll (>= 3.3, < 5.0) 38 | jekyll-watch (2.2.1) 39 | listen (~> 3.0) 40 | kramdown (2.3.1) 41 | rexml 42 | kramdown-parser-gfm (1.1.0) 43 | kramdown (~> 2.0) 44 | liquid (4.0.3) 45 | listen (3.2.1) 46 | rb-fsevent (~> 0.10, >= 0.10.3) 47 | rb-inotify (~> 0.9, >= 0.9.10) 48 | mercenary (0.4.0) 49 | minima (2.5.1) 50 | jekyll (>= 3.5, < 5.0) 51 | jekyll-feed (~> 0.9) 52 | jekyll-seo-tag (~> 2.1) 53 | pathutil (0.16.2) 54 | forwardable-extended (~> 2.6) 55 | public_suffix (4.0.6) 56 | rb-fsevent (0.10.4) 57 | rb-inotify (0.10.1) 58 | ffi (~> 1.0) 59 | rexml (3.2.5) 60 | rouge (3.23.0) 61 | safe_yaml (1.0.5) 62 | sassc (2.4.0) 63 | ffi (~> 1.9) 64 | terminal-table (1.8.0) 65 | unicode-display_width (~> 1.1, >= 1.1.1) 66 | unicode-display_width (1.7.0) 67 | 68 | PLATFORMS 69 | ruby 70 | 71 | DEPENDENCIES 72 | jekyll (~> 4.1.1) 73 | jekyll-feed (~> 0.12) 74 | minima (~> 2.5) 75 | tzinfo (~> 1.2) 76 | tzinfo-data 77 | wdm (~> 0.1.1) 78 | 79 | BUNDLED WITH 80 | 2.1.4 81 | -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | # If you need help with YAML syntax, here are some quick references for you: 2 | # https://learn-the-web.algonquindesign.ca/topics/markdown-yaml-cheat-sheet/#yaml 3 | # https://learnxinyminutes.com/docs/yaml/ 4 | # 5 | # Site settings 6 | # If you look in the HTML files, 7 | # you will see the settings below accessed via {{ site.title }}, {{ site.email }}, and so on. 8 | # You can create any custom variable you would like, and they will be accessible 9 | # in the templates via {{ site.myvariable }}. 10 | 11 | title: OpenReplica 12 | google_analytics: UA-9339334-4 13 | email: 14 | description: >- # this means to ignore newlines until "baseurl:" 15 | Website for the educational coordination service OpenReplica for Distributed Systems. 16 | baseurl: "" 17 | url: "https://openreplica.org" # the base hostname & protocol for your site, e.g. http://example.com 18 | address: openreplica.org 19 | paper_path: ../static/altinbukenOpenReplica.pdf 20 | github_username: denizalti 21 | 22 | # Build settings 23 | theme: minima 24 | plugins: 25 | - jekyll-feed 26 | 27 | header_pages: 28 | - documentation.markdown 29 | - code.markdown 30 | - paper.markdown 31 | 32 | # Exclude from processing. 33 | # The following items will not be processed, by default. 34 | # Any item listed under the `exclude:` key here will be automatically added to 35 | # the internal "default list". 36 | # 37 | # Excluded items can be processed by explicitly listing the directories or 38 | # their entries' file path in the `include:` list. 39 | # 40 | # exclude: 41 | # - .sass-cache/ 42 | # - .jekyll-cache/ 43 | # - gemfiles/ 44 | # - Gemfile 45 | # - Gemfile.lock 46 | # - node_modules/ 47 | # - vendor/bundle/ 48 | # - vendor/cache/ 49 | # - vendor/gems/ 50 | # - vendor/ruby/ 51 | -------------------------------------------------------------------------------- /docs/code.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Code 4 | permalink: /code/ 5 | --- 6 | 7 | The Python source code for OpenReplica is available in the following 8 | [git repository](https://github.com/denizalti/openreplica): 9 | 10 | 
https://github.com/denizalti/openreplica
11 | 12 | You can clone the repo as follows: 13 | 
$ git clone https://github.com/denizalti/openreplica.git
14 |
15 | 16 | The repository also includes a detailed documentation of OpenReplica. 17 | 18 | Note that, OpenReplica is not actively maintained at this time. 19 | -------------------------------------------------------------------------------- /docs/index.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | --- 4 | 5 | OpenReplica was built to provide an object-oriented coordination service for Distributed Systems. 6 | Unlike the widely-used counterparts such as ZooKeeper and Chubby, OpenReplica does not 7 | use a file-system API, instead it enables servers to coordinate over objects 8 | that can run code. This way required coordination behavior can be implemented 9 | as an object! 10 | 11 | OpenReplica provides availability, reliability and fault-tolerance in distributed systems. It is designed to maintain long-lived, critical state (such as configuration information) and to synchronize distributed components. It works as follows: you define a Python object that encapsulates the state you want replicated, along with methods that can update it, and can synchronize threads that access it. You give it to OpenReplica, your object gets geographically distributed automatically, and you receive a proxy through which multiple clients can access the replicated object transparently. To the rest of your application, your replicated object appears as a regular Python object when you use the provided proxy. 12 | 13 | OpenReplica ensures that new object replicas are dynamically created to compensate for any node or network failures involving your nodes. Our current implementation executes replicas on PlanetLab hosts distributed at academic sites around the globe, on failure independent hosts. You could also use the code behind OpenReplica to deploy on other hosts, and integrate with DNS and Amazon Route 53. 14 | 15 | OpenReplica is similar to services such as Google's Chubby and Yahoo's ZooKeeper, except for a critical difference: OpenReplica provides an object-oriented interface to applications. Overall, OpenReplica differs from existing systems in the following ways: 16 | 17 | * **High Performance:** Coupled with the concurrent Paxos protocol our implementation uses, the object-based API obviates the need for costly serialization and achieves much higher performance than other systems. 18 | * **Dynamic Implementation:** OpenReplica enables any server to be replaced at run-time. There are no statically designated servers, or configuration files -- any and all servers can be changed on the fly. 19 | * **DNS Integration:** Clients locate the up-to-date replicas through DNS. OpenReplica enables you to run your own authoritative DNS server, or to use a subdomain under openreplica.org, or to use Amazon's Route 53. 20 | * **Easy to use:** Smoothly integrates into your existing Python program, no external interfaces needed. Even complex coordination functionality, such as implementing synchronization objects like semaphores, is straight-forward. 21 | 22 | **How does OpenReplica work?** 23 | 24 | OpenReplica is powered by ConCoord, a novel coordination service that provides replication and synchronization support for large-scale distributed systems. ConCoord employs an object-oriented approach, in which the system actively creates and maintains live replicas for user-provided objects. Through ConCoord, the clients are able to access these replicated objects transparently as if they are local objects. The ConCoord approach proposes using these replicated objects to implement coordination constructs in large-scale distributed systems, in effect establishing a transparent way of providing a coordination service. 25 | 26 | 27 | 28 | **ConCoord Design** 29 | 30 | To support complex distributed synchronization constructs, ConCoord presents a novel mechanism that enables the replicated objects to control the execution flow of their clients, in essence providing blocking and non-blocking method invocations on a replicated object. ConCoord employs Paxos as the underlying consensus protocol to tolerate crash failures of hosts and the underlying network. To facilitate deployments in dynamic cloud environments, the implementation supports view changes, which permit any number of servers to be replaced at runtime. Integration with DNS enables clients to easily locate the most current set of replicas. 31 | 32 |
33 |
34 | This website is built and maintained by [Deniz Altınbüken, Ph.D.](https://denizaltinbuken.com) OpenReplica is not actively maintained but it is available as an open-source project. If you have questions, you can contact me at [hello@{{ site.address }}](mailto:hello@{{ site.address }}) 35 | -------------------------------------------------------------------------------- /docs/paper.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Paper 4 | permalink: /paper/ 5 | --- 6 | 7 | Deniz Altınbüken and Emin Gün Sirer. [*Commodifying Replicated State Machines with OpenReplica*](https://ecommons.cornell.edu/handle/1813/29009). Technical Report. 2012. 8 | 9 | 10 | -------------------------------------------------------------------------------- /docs/static/altinbukenOpenReplica.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/docs/static/altinbukenOpenReplica.pdf -------------------------------------------------------------------------------- /docs/static/concoord.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/docs/static/concoord.jpg -------------------------------------------------------------------------------- /docs/static/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/denizalti/openreplica/5a51ba2b475f44da221304ea7f0b9118f2e8511a/docs/static/favicon.png -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # concoord configuration file 2 | 3 | [metadata] 4 | name = concoord 5 | version = 1.1.0 6 | author = Deniz Altinbuken, Emin Gun Sirer 7 | author-email = deniz@systems.cs.cornell.edu, egs@systems.cs.cornell.edu 8 | maintainer = Deniz Altinbuken 9 | maintainer-email = concoord@systems.cs.cornell.edu 10 | summary = Coordination service for distributed systems. 11 | description-file = README 12 | keywords = concoord, openreplica, coordination, replication, synchronization, paxos, rsm 13 | 14 | requires-python = >=2.7 15 | requires-dist = pkginfo 16 | requires-external = python-dnspython, msgpack-python 17 | provides-dist = concoord (1.1.0) 18 | 19 | project-url = concoord, http://openreplica.org/ 20 | home-page = http://openreplica.org/ 21 | download-url = http://openreplica.org/src/ 22 | 23 | [global] 24 | # Auto-detect modules, packages, scripts, files 25 | autodetect = 1 26 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | 5 | from setuptools import setup 6 | from setuptools import find_packages 7 | 8 | classifiers = [ 9 | 'Development Status :: 3 - Alpha', 10 | 'Intended Audience :: Developers', 11 | 'Intended Audience :: System Administrators', 12 | 'License :: OSI Approved :: BSD License', 13 | 'Operating System :: MacOS :: MacOS X', 14 | 'Operating System :: POSIX :: Linux', 15 | 'Operating System :: Unix', 16 | 'Programming Language :: Python', 17 | 'Programming Language :: Python :: 2.7', 18 | ] 19 | 20 | setup( 21 | name='concoord', 22 | version='1.1.0', 23 | author='Deniz Altinbuken, Emin Gun Sirer', 24 | author_email='deniz@systems.cs.cornell.edu, egs@systems.cs.cornell.edu', 25 | packages=find_packages(), 26 | license='3-Clause BSD', 27 | url='http://openreplica.org/', 28 | description='Coordination service for distributed systems.', 29 | long_description=open('README').read(), 30 | classifiers=classifiers, 31 | entry_points={ 32 | 'console_scripts': ['concoord = concoord.main:main', 33 | 'openreplica = concoord.openreplica.main:main'] 34 | }, 35 | install_requires=[ 36 | 'python>=2.7', 37 | 'msgpack-python', 38 | 'dnspython', 39 | ], 40 | ) 41 | -------------------------------------------------------------------------------- /update_site.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # This script pushes the contents of the website folder to 3 | # branch gh-pages. 4 | git subtree push --prefix website/ origin gh-pages --------------------------------------------------------------------------------