├── .gitattributes ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── _config.yml ├── benchmarks ├── .gitignore ├── 2pcommit │ ├── orig.da │ └── spec.da ├── clpaxos │ └── spec.da ├── controller.da ├── crleader │ └── orig.da ├── dscrash │ ├── orig.da │ └── spec.da ├── hsleader │ ├── orig.da │ └── spec.da ├── lamutex │ ├── C │ │ ├── Makefile │ │ ├── lamport.c │ │ ├── pqueue.c │ │ └── pqueue.h │ ├── Erlang │ │ ├── Makefile │ │ └── lamutex.erl │ ├── Python │ │ └── lamutex.py │ ├── orig.da │ ├── spec.da │ └── spec_lam.da ├── lapaxos │ └── orig.da ├── pingpong │ └── ping.da ├── plot.py ├── ramutex │ ├── orig.da │ └── spec.da ├── ratoken │ └── spec.da ├── run.py ├── sktoken │ └── orig.da └── vrpaxos │ ├── orig.da │ └── spec.da ├── bin ├── dac ├── dac.bat ├── dar └── dar.bat ├── da ├── __init__.py ├── __main__.py ├── api.py ├── common.py ├── compiler │ ├── __init__.py │ ├── __main__.py │ ├── daast_dict.py │ ├── daast_nest.py │ ├── dast.py │ ├── incgen.py │ ├── parser.py │ ├── pseudo.py │ ├── pygen.py │ ├── symtab.py │ ├── ui.py │ └── utils.py ├── examples │ ├── 2pcommit │ │ ├── __init__.py │ │ ├── orig.da │ │ └── spec.da │ ├── __init__.py │ ├── clpaxos │ │ ├── __init__.py │ │ └── spec.da │ ├── crleader │ │ ├── __init__.py │ │ └── orig.da │ ├── dscrash │ │ ├── __init__.py │ │ ├── orig.da │ │ └── spec.da │ ├── hsleader │ │ ├── __init__.py │ │ ├── orig.da │ │ └── spec.da │ ├── lamutex │ │ ├── __init__.py │ │ ├── orig.da │ │ ├── spec.da │ │ └── spec_lam.da │ ├── lapaxos │ │ ├── __init__.py │ │ └── orig.da │ ├── pingpong │ │ ├── __init__.py │ │ └── ping.da │ ├── raft │ │ ├── __init__.py │ │ └── orig.da │ ├── ramutex │ │ ├── __init__.py │ │ ├── orig.da │ │ └── spec.da │ ├── ratoken │ │ ├── __init__.py │ │ └── spec.da │ ├── sktoken │ │ ├── __init__.py │ │ └── orig.da │ └── vrpaxos │ │ ├── __init__.py │ │ └── orig.da ├── freeze.py ├── importer │ ├── __init__.py │ ├── py36.py │ └── py37.py ├── lib │ ├── __init__.py │ ├── base.da │ └── laclock.da ├── pattern.py ├── sim.py ├── tools │ ├── __init__.py │ └── unparse.py └── transport │ ├── __init__.py │ ├── base.py │ ├── manager.py │ ├── mesgloop.py │ └── sock.py ├── doc ├── language.tex └── screenshot.png ├── setup.py └── tests ├── __init__.py ├── test_freezer.py ├── test_manager.py ├── test_mesgloop.py └── test_sock.py /.gitattributes: -------------------------------------------------------------------------------- 1 | *.py diff=python 2 | *.da linguist-language=python 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.dap 3 | __pycache__/ 4 | *.log 5 | *.bak 6 | da/lib/**/*.py 7 | da/examples/**/*.py 8 | benchmarks/**/*.py 9 | dist/ 10 | build/ 11 | pyDistAlgo.egg-info/ 12 | doc/auto/ 13 | doc/*.aux 14 | doc/*.pdf 15 | MANIFEST 16 | .dir-locals.el 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Most of DistAlgo is covered by the terms of the following (MIT) license: 2 | 3 | ``` 4 | Permission is hereby granted, free of charge, to any person obtaining a 5 | copy of this software and associated documentation files (the "Software"), 6 | to deal in the Software without restriction, including without limitation 7 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | and/or sell copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | DEALINGS IN THE SOFTWARE. 21 | ``` 22 | 23 | Please see each individual source file for a list of copyright holders. 24 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | prune doc/ 2 | include doc/language.pdf 3 | include doc/screenshot.png 4 | include README.md 5 | include setup.py 6 | include LICENSE 7 | 8 | recursive-include da/examples __init__.py *.da 9 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-midnight -------------------------------------------------------------------------------- /benchmarks/.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.beam 3 | graphs 4 | results 5 | *.dump 6 | -------------------------------------------------------------------------------- /benchmarks/2pcommit/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | from random import randint 3 | controller = import_da('controller') 4 | 5 | class Coordinator(process, controller.Controllee): 6 | def setup(ctl, cohorts:set): 7 | super().setup(ctl) 8 | 9 | def to_commit(): 10 | send(('prepare'), to= cohorts) 11 | await(each(c in cohorts, has= some(received(('vote', _), from_= _c)))) 12 | if some(c in cohorts, has= received(('vote', 'aborting'), from_= c)): 13 | s = setof(c, c in cohorts, received(('vote','ready'), from_= c)) 14 | send(('abort'), to= s) 15 | abort() 16 | if each(c in cohorts, has= received(('vote', 'ready'), from_= c)): 17 | send(('commit'), to= cohorts) 18 | await(each(c in cohorts, has= received(('done'), from_= c))) 19 | commit() 20 | 21 | def abort(): output('abort') 22 | def commit(): output('commit') 23 | 24 | @controller.run 25 | def run(): 26 | to_commit() 27 | 28 | class Cohort(process, controller.Controllee): 29 | def setup(ctl, failure_rate): 30 | super().setup(ctl) 31 | self.terminate = False 32 | 33 | def receive(msg= ('prepare'), from_= coord): 34 | if prepared(): 35 | send(('vote', 'ready'), to= coord) 36 | ready() 37 | else: 38 | send(('vote', 'aborting'), to= coord) 39 | abort() 40 | 41 | def receive(msg= ('abort')): 42 | abort() 43 | 44 | def receive(msg= ('commit'), from_= coord): 45 | send(('done'), to= coord) 46 | commit() 47 | 48 | def prepared(): return randint(0, 100) > failure_rate 49 | def ready(): output('ready') 50 | def abort(): output('abort'); terminate = True 51 | def commit(): output('commit'); terminate = True 52 | 53 | @controller.run 54 | def run(): 55 | await(terminate) 56 | 57 | def main(): 58 | ncohorts = int(sys.argv[1]) if len(sys.argv) > 1 else 10 59 | fail_rate = int(sys.argv[2]) if len(sys.argv) > 2 else 10 60 | 61 | ctl = new(controller.Controller, num= 1) 62 | setup(ctl, (ncohorts + 1,)) 63 | start(ctl) 64 | 65 | cohorts = new(Cohort, (ctl, fail_rate,), num= ncohorts) 66 | coordinators = new(Coordinator, (ctl, cohorts,), num= 1) 67 | start(cohorts | coordinators) 68 | 69 | # This is an executable specification of the algorithm described in 70 | # Kifer, M., et al. (2006). Database Systems: An Application Oriented Approach 71 | # (pages 1010-1012). Addison-Wesley. 72 | 73 | # This code includes setup for trying to commit a transaction with a given 74 | # number of cohorts. 75 | -------------------------------------------------------------------------------- /benchmarks/2pcommit/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | from random import randint 3 | controller = import_da('controller') 4 | 5 | @controller.rugroup('bo_measured') 6 | class Coordinator(process, controller.Controllee): 7 | def setup(ctl, tid:str, cohorts:set): 8 | super().setup(ctl) 9 | 10 | def to_commit(): 11 | send(('prepare', tid), to= cohorts) 12 | await(each(c in cohorts, has= some(received(('vote', _, _tid), from_= _c)))) 13 | if some(c in cohorts, has= received(('vote', 'abort', tid), from_= c)): 14 | s = setof(c, c in cohorts, received(('vote','ready', _tid), from_=c)) 15 | send(('abort', tid), to= s) 16 | abort(tid) 17 | if each(c in cohorts, has= received(('vote', 'ready', tid), from_= c)): 18 | send(('commit', tid), to= cohorts) 19 | await(each(c in cohorts, has= received(('done', tid), from_= c))) 20 | commit(tid) 21 | 22 | def abort(tid): output('abort:' + str(tid)) 23 | def commit(tid): output('commit:' + str(tid)) 24 | 25 | @controller.run 26 | def run(): 27 | to_commit() 28 | 29 | send(('done',), to= cohorts) 30 | 31 | class Cohort(process, controller.Controllee): 32 | def setup(ctl, failure_rate:float): 33 | super().setup(ctl) 34 | 35 | def receive(msg=('prepare', tid), from_= coord): 36 | if prepared(tid): 37 | send(('vote', 'ready', tid), to= coord) 38 | ready(tid) 39 | else: 40 | send(('vote', 'abort', tid), to= coord) 41 | abort(tid) 42 | 43 | def receive(msg=('abort', tid)): 44 | abort(tid) 45 | 46 | def receive(msg=('commit', tid), from_= coord): 47 | send(('done', tid), to= coord) 48 | commit(tid) 49 | 50 | def prepared(tid): return randint(0, 100) > failure_rate 51 | def ready(tid): output('ready:' + str(tid)) 52 | def abort(tid): output('abort:' + str(tid)) 53 | def commit(tid): output('commit:' + str(tid)) 54 | 55 | @controller.run 56 | def run(): 57 | await(received(('done',))) 58 | 59 | def main(): 60 | ncohorts = int(sys.argv[1]) if len(sys.argv) > 1 else 10 61 | fail_rate = int(sys.argv[2]) if len(sys.argv) > 2 else 10 62 | 63 | ctl = new(controller.Controller, num= 1) 64 | setup(ctl, (ncohorts + 1,)) 65 | start(ctl) 66 | 67 | cohorts = new(Cohort, (ctl, fail_rate,), num= ncohorts) 68 | coordinators = new(Coordinator, (ctl, '001', cohorts), num= 1) 69 | start(cohorts | coordinators) 70 | 71 | # this code includes setup and termination for trying to commit a 72 | # transaction with a given number of cohorts. 73 | -------------------------------------------------------------------------------- /benchmarks/clpaxos/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | controller = import_da('controller') 3 | 4 | class Proposer(process, controller.Controllee): 5 | def setup(ctl, acceptors:set, quorumsize:int, 6 | f:float, nrounds:int, timeout:float): 7 | super().setup(ctl) 8 | self.propNum = (0, self.id) # Current proposal(ballot) number 9 | self.propVal = self.id # Own value to propose 10 | 11 | @controller.run 12 | def run(): 13 | count = 0 14 | while count < nrounds: 15 | work() 16 | --prepare 17 | send(('Prepare', propNum, self), to=acceptors) 18 | if await(len(setof(a, received(('Promise', _propNum, _, _, a)))) > quorumsize): 19 | --propose 20 | _, voted = max(setof((n, v), 21 | received(('Promise', _propNum, n, v, _)), 22 | len(setof(a, 23 | received(('Promise', 24 | _propNum, _n, _v, a)))) > f) | 25 | {((-1, self), propVal)}) 26 | 27 | send(('OneC', propNum, voted, self), to=acceptors) 28 | if await(len(setof(a, received(('TwoAv', _propNum, _voted, a)))) > quorumsize): 29 | --end 30 | output("Succeeded proposing %s" % (voted,)) 31 | count += 1 32 | continue 33 | elif timeout(timeout): 34 | output("Failed to Propose in time, retrying.") 35 | elif timeout(timeout): 36 | output("Failed to Prepare in time, retrying.") 37 | propNum = (propNum[0] + 1, self) 38 | send(('Done',), to=acceptors) 39 | 40 | class Acceptor(process, controller.Controllee): 41 | def setup(ctl, acceptors:set, proposers:set, quorumsize:int, f:float): 42 | super().setup(ctl) 43 | self.peers = acceptors | proposers 44 | 45 | @controller.run 46 | def run(): 47 | while True: 48 | if await(some( 49 | received(('TwoAv', n, v, _)), 50 | has=(len(setof(a, received(('TwoAv', _n, _v, a)))) > 51 | quorumsize and 52 | not sent(('TwoB', n, v))))): 53 | send(('TwoB', n, v), to=peers) 54 | elif each(p in proposers, has=received(('Done',), from_=p)): 55 | break 56 | 57 | def receive(msg=('Prepare', n, p)): 58 | if n > maxpromised(): 59 | vn, vv = max(setof((vpn, vv), sent(('TwoAv', vpn, vv, self))) | 60 | {((-1, self), None)}) 61 | send(('Promise', n, vn, vv, self), to=peers) 62 | 63 | def receive(msg=('OneC', n, v, p)): 64 | if (n >= maxpromised() and islegal(n, v) and 65 | (not some(sent(('TwoAv', _n, _, self))))): 66 | send(('TwoAv', n, v, self), to=peers) 67 | 68 | def maxpromised(): 69 | return max(setof(n, sent(('Promise', n, _, _, _))) | {(-2, self)}) 70 | 71 | def islegal(n, v): 72 | voted = setof((vn, vv), received(('Promise', _n, vn, vv, _)), 73 | len(setof(a, received(('Promise', _n, _vn, _vv, a)))) > f) 74 | if voted and (max(voted)[1] is not None): 75 | return v == max(voted)[1] 76 | else: 77 | return True 78 | 79 | def main(): 80 | nproposers = int(sys.argv[1]) if len(sys.argv) > 1 else 5 81 | nacceptors = int(sys.argv[2]) if len(sys.argv) > 2 else 10 82 | nrounds = int(sys.argv[3]) if len(sys.argv) > 3 else 1 83 | timeout = int(sys.argv[4]) if len(sys.argv) > 4 else 5 84 | f = int((nacceptors-1)/3) 85 | quorum = int(nacceptors/2 + f) 86 | 87 | ctl = new(controller.Controller, num= 1) 88 | setup(ctl, (nacceptors + nproposers,)) 89 | start(ctl) 90 | 91 | acceptors = new(Acceptor, num=nacceptors) 92 | proposers = new(Proposer, num=nproposers) 93 | setup(acceptors, (ctl, acceptors, proposers, quorum, f)) 94 | setup(proposers, (ctl, acceptors, quorum, f, nrounds, timeout)) 95 | start(acceptors) 96 | start(proposers) 97 | -------------------------------------------------------------------------------- /benchmarks/crleader/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | controller = import_da('controller') 3 | 4 | class P(process, controller.Controllee): 5 | def setup(ctl, left:P): 6 | super().setup(ctl) 7 | self.leaderid = None 8 | 9 | def initiate(): 10 | send(('Election', self.id), to=left) 11 | 12 | def receive(msg=('Election', p)): 13 | if p > self.id: 14 | send(('Election', p), to=left) 15 | if p < self.id: 16 | if not sent(('Election', self.id)): 17 | send(('Election', self.id), to=left) 18 | if p == self.id: 19 | send(('Leader', self.id), to=left) 20 | 21 | def receive(msg=('Leader', leader)): 22 | leaderid = leader 23 | if leader != self.id: 24 | send(('Leader', leader), to=left) 25 | 26 | @controller.run 27 | def run(): 28 | initiate() 29 | await (some(received(('Leader', _)))) 30 | output("Leader is %r."%(leaderid)) 31 | 32 | def main(): 33 | n = int(sys.argv[1]) if len(sys.argv) > 1 else 5 34 | config(channel="fifo") 35 | 36 | ctl = new(controller.Controller, num= 1) 37 | setup(ctl, (n,)) 38 | start(ctl) 39 | 40 | ps = list(new(P, num= n)) 41 | for i, p in enumerate(ps): 42 | setup({p}, (ctl, ps[(i+1 if i < (len(ps)-1) else 0)],)) 43 | start(ps) 44 | -------------------------------------------------------------------------------- /benchmarks/dscrash/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | config(Channel is Reliable, Handling is one) 3 | controller = import_da('controller') 4 | 5 | class P(process, controller.Controllee): 6 | def setup(ctl, ps:set, v:P, maxfail:int): 7 | super().setup(ctl) 8 | self.x = -1 9 | self.V = {v: False} 10 | self.receiveflag = False 11 | 12 | @controller.run 13 | def run(): 14 | --start 15 | for i in range(maxfail): 16 | --one_round 17 | for k in V: 18 | if not V[k]: 19 | send(('Value', k), to=ps) 20 | V[k] = True 21 | await(receiveflag) 22 | receiveflag = False 23 | --end 24 | x = max(listof(v, v in V, V[v])) 25 | output("x = %r" % (x)) 26 | 27 | def receive(msg=('Value', v)): 28 | receiveflag = True 29 | if v not in V: 30 | V[v] = False 31 | 32 | def main(): 33 | n = int(sys.argv[1]) if len(sys.argv) > 1 else 10 34 | f = int(sys.argv[2]) if len(sys.argv) > 2 else 50 35 | 36 | ctl = new(controller.Controller, num= 1) 37 | setup(ctl, (n,)) 38 | start(ctl) 39 | 40 | ps = new(P, num= n) 41 | for i, p in enumerate(list(ps)): 42 | setup({p}, (ctl, ps, i, f)) 43 | start(ps) 44 | -------------------------------------------------------------------------------- /benchmarks/dscrash/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | controller = import_da('controller') 3 | 4 | class P(process, controller.Controllee): 5 | def setup(ctl, ps:set, v:int, maxfail:int): 6 | super().setup(ctl) 7 | self.V = {v} # V is set of values to agree on, initially own value 8 | 9 | @controller.run 10 | def run(): 11 | for rnd in range(1, maxfail): 12 | send(('Value', 13 | setof(v, v in V, 14 | not some(sent(('Value', V2, _)), has=(v in V2))), 15 | self.id), to=ps) 16 | reset(received) 17 | await(some(received(('Value', _, _)))) 18 | V.update(setof(v, received(('Value', V2, _)), v in V2)) 19 | x = min(V) 20 | output(x) 21 | 22 | def main(): 23 | n = int(sys.argv[1]) if len(sys.argv) > 1 else 10 24 | f = int(sys.argv[2]) if len(sys.argv) > 2 else 10 25 | 26 | ctl = new(controller.Controller, num= 1) 27 | setup(ctl, (n,)) 28 | start(ctl) 29 | 30 | ps = new(P, num= n) 31 | for i, p in enumerate(list(ps)): 32 | setup({p}, (ctl, ps, i, f)) 33 | start(ps) 34 | -------------------------------------------------------------------------------- /benchmarks/hsleader/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | import random 3 | controller = import_da('controller') 4 | 5 | class P(process, controller.Controllee): 6 | def setup(ctl, left:P, right:P): 7 | super().setup(ctl) 8 | self.status = "Unknown" # Current status, can be {"Unknown", "Leader"} 9 | self.phase_left, self.phase_right = False, False 10 | self.phase = 0 11 | 12 | @controller.run 13 | def run(): 14 | while True: 15 | send(('Token', self.id, 'out', 1 << phase), to={left, right}) 16 | 17 | if await(status == "Leader"): 18 | output("I am leader at phase %d!"%phase) 19 | send(('Leader', self.id), to={left, right}) 20 | break 21 | elif (phase_left and phase_right): 22 | phase += 1 23 | phase_left, phase_right = False, False 24 | elif some(received(('Leader', leader))): 25 | output("Leader is " + str(leader)) 26 | break 27 | 28 | def receive(msg=('Token', v, direction, h), from_=source): 29 | if source == left and direction == 'out': 30 | if v > self.id and h > 1: 31 | send(('Token', v, 'out', h-1), to=right) 32 | elif v > self.id and h == 1: 33 | send(('Token', v, 'in', 1), to=left) 34 | elif v == self.id: 35 | status = "Leader" 36 | 37 | elif source == right and direction == 'out': 38 | if v > self.id and h > 1: 39 | send(('Token', v, 'out', h-1), to=left) 40 | elif v > self.id and h == 1: 41 | send(('Token', v, 'in', 1), to=right) 42 | elif v == self.id: 43 | status = "Leader" 44 | 45 | elif source == left and direction == 'in': 46 | if v > self.id: 47 | send(('Token', v, 'in', 1), to=right) 48 | elif v == self.id: 49 | phase_left = True 50 | 51 | elif source == right and direction == 'in': 52 | if v > self.id: 53 | send(('Token', v, 'in', 1), to=left) 54 | elif v == self.id: 55 | phase_right = True 56 | 57 | def receive(msg=('Leader', leader), from_=source): 58 | if source == left: 59 | send(('Leader', leader), to=right) 60 | else: 61 | send(('Leader', leader), to=left) 62 | 63 | def main(): 64 | n = int(sys.argv[1]) if len(sys.argv) > 1 else 10 65 | config(channel="fifo") 66 | 67 | ctl = new(controller.Controller, num= 1) 68 | setup(ctl, (n,)) 69 | start(ctl) 70 | 71 | topology = list(new(P, num= n)) 72 | random.shuffle(topology) 73 | for i, p in enumerate(topology): 74 | if i == len(topology)-1: 75 | setup({p}, (ctl, topology[i-1], topology[0])) 76 | else: 77 | setup({p}, (ctl, topology[i-1], topology[i+1])) 78 | start(topology) 79 | 80 | -------------------------------------------------------------------------------- /benchmarks/hsleader/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | import random 3 | controller = import_da('controller') 4 | 5 | class P(process, controller.Controllee): 6 | def setup(ctl, left:P, right:P): 7 | super().setup(ctl) 8 | 9 | @controller.run 10 | def run(): 11 | distance = 1 12 | while True: 13 | send(('Out', self.id, distance), to={left, right}) 14 | if await(some(received(('Out', self.id, d)))): 15 | output("I am leader at distance %d!"%d) 16 | send(('Leader', self.id), to={left, right}) 17 | break 18 | elif some(received(('Leader', leader))): 19 | output("Leader is " + str(leader)) 20 | break 21 | elif (received(('In', self.id), from_=left) and 22 | received(('In', self.id), from_=right)): 23 | distance *= 2 24 | reset(received) 25 | 26 | def receive(msg=('Out', v, d), from_=source): 27 | if v > self.id: 28 | if d > 1: 29 | send(('Out', v, d-1), to=(right if source == left else left)) 30 | elif d == 1: 31 | send(('In', v), to=source) 32 | 33 | def receive(msg=('In', v), from_=source): 34 | if v > self.id: 35 | send(('In', v), to=(right if source == left else left)) 36 | 37 | def receive(msg=("Leader", leader), from_=source): 38 | send(('Leader', leader), to=(right if source == left else left)) 39 | 40 | def main(): 41 | n = int(sys.argv[1]) if len(sys.argv) > 1 else 10 42 | config(channel="fifo") 43 | 44 | ctl = new(controller.Controller, num= 1) 45 | setup(ctl, (n,)) 46 | start(ctl) 47 | 48 | topology = list(new(P, num= n)) 49 | random.shuffle(topology) 50 | for i, p in enumerate(topology): 51 | if i == len(topology)-1: 52 | setup({p}, (ctl, topology[i-1], topology[0])) 53 | else: 54 | setup({p}, (ctl, topology[i-1], topology[i+1])) 55 | start(topology) 56 | -------------------------------------------------------------------------------- /benchmarks/lamutex/C/Makefile: -------------------------------------------------------------------------------- 1 | 2 | CC=gcc 3 | CFLAGS=-O3 -std=c99 -fms-extensions 4 | LDFLAGS=-lrt 5 | 6 | all : lamport 7 | strip lamport 8 | 9 | lamport : lamport.o pqueue.o 10 | 11 | clean : 12 | rm -f *.o lamport 13 | 14 | .PHONY : all clean 15 | -------------------------------------------------------------------------------- /benchmarks/lamutex/C/pqueue.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2010 Volkan Yazıcı 3 | * Copyright 2006-2010 The Apache Software Foundation 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 6 | * use this file except in compliance with the License. You may obtain a copy of 7 | * the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "pqueue.h" 24 | 25 | 26 | #define left(i) ((i) << 1) 27 | #define right(i) (((i) << 1) + 1) 28 | #define parent(i) ((i) >> 1) 29 | 30 | 31 | pqueue_t * 32 | pqueue_init(size_t n, 33 | pqueue_cmp_pri_f cmppri, 34 | pqueue_get_pri_f getpri, 35 | pqueue_set_pri_f setpri, 36 | pqueue_get_pos_f getpos, 37 | pqueue_set_pos_f setpos) 38 | { 39 | pqueue_t *q; 40 | 41 | if (!(q = malloc(sizeof(pqueue_t)))) 42 | return NULL; 43 | 44 | /* Need to allocate n+1 elements since element 0 isn't used. */ 45 | if (!(q->d = malloc((n + 1) * sizeof(void *)))) { 46 | free(q); 47 | return NULL; 48 | } 49 | 50 | q->size = 1; 51 | q->avail = q->step = (n+1); /* see comment above about n+1 */ 52 | q->cmppri = cmppri; 53 | q->setpri = setpri; 54 | q->getpri = getpri; 55 | q->getpos = getpos; 56 | q->setpos = setpos; 57 | 58 | return q; 59 | } 60 | 61 | 62 | void 63 | pqueue_free(pqueue_t *q) 64 | { 65 | free(q->d); 66 | free(q); 67 | } 68 | 69 | 70 | size_t 71 | pqueue_size(pqueue_t *q) 72 | { 73 | /* queue element 0 exists but doesn't count since it isn't used. */ 74 | return (q->size - 1); 75 | } 76 | 77 | 78 | static void 79 | bubble_up(pqueue_t *q, size_t i) 80 | { 81 | size_t parent_node; 82 | void *moving_node = q->d[i]; 83 | pqueue_pri_t moving_pri = q->getpri(moving_node); 84 | 85 | for (parent_node = parent(i); 86 | ((i > 1) && q->cmppri(q->getpri(q->d[parent_node]), moving_pri)); 87 | i = parent_node, parent_node = parent(i)) 88 | { 89 | q->d[i] = q->d[parent_node]; 90 | q->setpos(q->d[i], i); 91 | } 92 | 93 | q->d[i] = moving_node; 94 | q->setpos(moving_node, i); 95 | } 96 | 97 | 98 | static size_t 99 | maxchild(pqueue_t *q, size_t i) 100 | { 101 | size_t child_node = left(i); 102 | 103 | if (child_node >= q->size) 104 | return 0; 105 | 106 | if ((child_node+1) < q->size && 107 | q->cmppri(q->getpri(q->d[child_node]), q->getpri(q->d[child_node+1]))) 108 | child_node++; /* use right child instead of left */ 109 | 110 | return child_node; 111 | } 112 | 113 | 114 | static void 115 | percolate_down(pqueue_t *q, size_t i) 116 | { 117 | size_t child_node; 118 | void *moving_node = q->d[i]; 119 | pqueue_pri_t moving_pri = q->getpri(moving_node); 120 | 121 | while ((child_node = maxchild(q, i)) && 122 | q->cmppri(moving_pri, q->getpri(q->d[child_node]))) 123 | { 124 | q->d[i] = q->d[child_node]; 125 | q->setpos(q->d[i], i); 126 | i = child_node; 127 | } 128 | 129 | q->d[i] = moving_node; 130 | q->setpos(moving_node, i); 131 | } 132 | 133 | 134 | int 135 | pqueue_insert(pqueue_t *q, void *d) 136 | { 137 | void *tmp; 138 | size_t i; 139 | size_t newsize; 140 | 141 | if (!q) return 1; 142 | 143 | /* allocate more memory if necessary */ 144 | if (q->size >= q->avail) { 145 | newsize = q->size + q->step; 146 | if (!(tmp = realloc(q->d, sizeof(void *) * newsize))) 147 | return 1; 148 | q->d = tmp; 149 | q->avail = newsize; 150 | } 151 | 152 | /* insert item */ 153 | i = q->size++; 154 | q->d[i] = d; 155 | bubble_up(q, i); 156 | 157 | return 0; 158 | } 159 | 160 | 161 | void 162 | pqueue_change_priority(pqueue_t *q, 163 | pqueue_pri_t new_pri, 164 | void *d) 165 | { 166 | size_t posn; 167 | pqueue_pri_t old_pri = q->getpri(d); 168 | 169 | q->setpri(d, new_pri); 170 | posn = q->getpos(d); 171 | if (q->cmppri(old_pri, new_pri)) 172 | bubble_up(q, posn); 173 | else 174 | percolate_down(q, posn); 175 | } 176 | 177 | 178 | int 179 | pqueue_remove(pqueue_t *q, void *d) 180 | { 181 | size_t posn = q->getpos(d); 182 | q->d[posn] = q->d[--q->size]; 183 | if (q->cmppri(q->getpri(d), q->getpri(q->d[posn]))) 184 | bubble_up(q, posn); 185 | else 186 | percolate_down(q, posn); 187 | 188 | return 0; 189 | } 190 | 191 | 192 | void * 193 | pqueue_pop(pqueue_t *q) 194 | { 195 | void *head; 196 | 197 | if (!q || q->size == 1) 198 | return NULL; 199 | 200 | head = q->d[1]; 201 | q->d[1] = q->d[--q->size]; 202 | percolate_down(q, 1); 203 | 204 | return head; 205 | } 206 | 207 | 208 | void * 209 | pqueue_peek(pqueue_t *q) 210 | { 211 | void *d; 212 | if (!q || q->size == 1) 213 | return NULL; 214 | d = q->d[1]; 215 | return d; 216 | } 217 | 218 | 219 | void 220 | pqueue_dump(pqueue_t *q, 221 | FILE *out, 222 | pqueue_print_entry_f print) 223 | { 224 | int i; 225 | 226 | fprintf(stdout,"posn\tleft\tright\tparent\tmaxchild\t...\n"); 227 | for (i = 1; i < q->size ;i++) { 228 | fprintf(stdout, 229 | "%d\t%d\t%d\t%d\t%ul\t", 230 | i, 231 | left(i), right(i), parent(i), 232 | (unsigned int) maxchild(q, i)); 233 | print(out, q->d[i]); 234 | } 235 | } 236 | 237 | 238 | static void 239 | set_pos(void *d, size_t val) 240 | { 241 | /* do nothing */ 242 | } 243 | 244 | 245 | static void 246 | set_pri(void *d, pqueue_pri_t pri) 247 | { 248 | /* do nothing */ 249 | } 250 | 251 | 252 | void 253 | pqueue_print(pqueue_t *q, 254 | FILE *out, 255 | pqueue_print_entry_f print) 256 | { 257 | pqueue_t *dup; 258 | void *e; 259 | 260 | dup = pqueue_init(q->size, 261 | q->cmppri, q->getpri, set_pri, 262 | q->getpos, set_pos); 263 | dup->size = q->size; 264 | dup->avail = q->avail; 265 | dup->step = q->step; 266 | 267 | memcpy(dup->d, q->d, (q->size * sizeof(void *))); 268 | 269 | while ((e = pqueue_pop(dup))) 270 | print(out, e); 271 | 272 | pqueue_free(dup); 273 | } 274 | 275 | 276 | static int 277 | subtree_is_valid(pqueue_t *q, int pos) 278 | { 279 | if (left(pos) < q->size) { 280 | /* has a left child */ 281 | if (q->cmppri(q->getpri(q->d[pos]), q->getpri(q->d[left(pos)]))) 282 | return 0; 283 | if (!subtree_is_valid(q, left(pos))) 284 | return 0; 285 | } 286 | if (right(pos) < q->size) { 287 | /* has a right child */ 288 | if (q->cmppri(q->getpri(q->d[pos]), q->getpri(q->d[right(pos)]))) 289 | return 0; 290 | if (!subtree_is_valid(q, right(pos))) 291 | return 0; 292 | } 293 | return 1; 294 | } 295 | 296 | 297 | int 298 | pqueue_is_valid(pqueue_t *q) 299 | { 300 | return subtree_is_valid(q, 1); 301 | } 302 | -------------------------------------------------------------------------------- /benchmarks/lamutex/C/pqueue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2010 Volkan Yazıcı 3 | * Copyright 2006-2010 The Apache Software Foundation 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 6 | * use this file except in compliance with the License. You may obtain a copy of 7 | * the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | 18 | 19 | /** 20 | * @file pqueue.h 21 | * @brief Priority Queue function declarations 22 | * 23 | * @{ 24 | */ 25 | 26 | 27 | #ifndef PQUEUE_H 28 | #define PQUEUE_H 29 | 30 | /** priority data type */ 31 | typedef void* pqueue_pri_t; 32 | 33 | /** callback functions to get/set/compare the priority of an element */ 34 | typedef pqueue_pri_t (*pqueue_get_pri_f)(void *a); 35 | typedef void (*pqueue_set_pri_f)(void *a, pqueue_pri_t pri); 36 | typedef int (*pqueue_cmp_pri_f)(pqueue_pri_t next, pqueue_pri_t curr); 37 | 38 | 39 | /** callback functions to get/set the position of an element */ 40 | typedef size_t (*pqueue_get_pos_f)(void *a); 41 | typedef void (*pqueue_set_pos_f)(void *a, size_t pos); 42 | 43 | 44 | /** debug callback function to print a entry */ 45 | typedef void (*pqueue_print_entry_f)(FILE *out, void *a); 46 | 47 | 48 | /** the priority queue handle */ 49 | typedef struct pqueue_t 50 | { 51 | size_t size; 52 | size_t avail; 53 | size_t step; 54 | pqueue_cmp_pri_f cmppri; 55 | pqueue_get_pri_f getpri; 56 | pqueue_set_pri_f setpri; 57 | pqueue_get_pos_f getpos; 58 | pqueue_set_pos_f setpos; 59 | void **d; 60 | } pqueue_t; 61 | 62 | 63 | /** 64 | * initialize the queue 65 | * 66 | * @param n the initial estimate of the number of queue items for which memory 67 | * should be preallocated 68 | * @param pri the callback function to run to assign a score to a element 69 | * @param get the callback function to get the current element's position 70 | * @param set the callback function to set the current element's position 71 | * 72 | * @Return the handle or NULL for insufficent memory 73 | */ 74 | pqueue_t * 75 | pqueue_init(size_t n, 76 | pqueue_cmp_pri_f cmppri, 77 | pqueue_get_pri_f getpri, 78 | pqueue_set_pri_f setpri, 79 | pqueue_get_pos_f getpos, 80 | pqueue_set_pos_f setpos); 81 | 82 | 83 | /** 84 | * free all memory used by the queue 85 | * @param q the queue 86 | */ 87 | void pqueue_free(pqueue_t *q); 88 | 89 | 90 | /** 91 | * return the size of the queue. 92 | * @param q the queue 93 | */ 94 | size_t pqueue_size(pqueue_t *q); 95 | 96 | 97 | /** 98 | * insert an item into the queue. 99 | * @param q the queue 100 | * @param d the item 101 | * @return 0 on success 102 | */ 103 | int pqueue_insert(pqueue_t *q, void *d); 104 | 105 | 106 | /** 107 | * move an existing entry to a different priority 108 | * @param q the queue 109 | * @param old the old priority 110 | * @param d the entry 111 | */ 112 | void 113 | pqueue_change_priority(pqueue_t *q, 114 | pqueue_pri_t new_pri, 115 | void *d); 116 | 117 | 118 | /** 119 | * pop the highest-ranking item from the queue. 120 | * @param p the queue 121 | * @param d where to copy the entry to 122 | * @return NULL on error, otherwise the entry 123 | */ 124 | void *pqueue_pop(pqueue_t *q); 125 | 126 | 127 | /** 128 | * remove an item from the queue. 129 | * @param p the queue 130 | * @param d the entry 131 | * @return 0 on success 132 | */ 133 | int pqueue_remove(pqueue_t *q, void *d); 134 | 135 | 136 | /** 137 | * access highest-ranking item without removing it. 138 | * @param q the queue 139 | * @param d the entry 140 | * @return NULL on error, otherwise the entry 141 | */ 142 | void *pqueue_peek(pqueue_t *q); 143 | 144 | 145 | /** 146 | * print the queue 147 | * @internal 148 | * DEBUG function only 149 | * @param q the queue 150 | * @param out the output handle 151 | * @param the callback function to print the entry 152 | */ 153 | void 154 | pqueue_print(pqueue_t *q, 155 | FILE *out, 156 | pqueue_print_entry_f print); 157 | 158 | 159 | /** 160 | * dump the queue and it's internal structure 161 | * @internal 162 | * debug function only 163 | * @param q the queue 164 | * @param out the output handle 165 | * @param the callback function to print the entry 166 | */ 167 | void 168 | pqueue_dump(pqueue_t *q, 169 | FILE *out, 170 | pqueue_print_entry_f print); 171 | 172 | 173 | /** 174 | * checks that the pq is in the right order, etc 175 | * @internal 176 | * debug function only 177 | * @param q the queue 178 | */ 179 | int pqueue_is_valid(pqueue_t *q); 180 | 181 | 182 | #endif /* PQUEUE_H */ 183 | /** @} */ 184 | -------------------------------------------------------------------------------- /benchmarks/lamutex/Erlang/Makefile: -------------------------------------------------------------------------------- 1 | 2 | all: lamutex.beam 3 | 4 | lamutex.beam: lamutex.erl 5 | erlc lamutex.erl 6 | 7 | run: lamutex.beam 8 | erl -noshell -run lamutex start 10 5 9 | 10 | clean: 11 | rm *.beam 12 | 13 | .PHONY: all run run2 clean 14 | -------------------------------------------------------------------------------- /benchmarks/lamutex/Erlang/lamutex.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2010-2015 Bo Lin 2 | %% Copyright (c) 2010-2015 Yanhong Annie Liu 3 | %% Copyright (c) 2010-2015 Stony Brook University 4 | %% Copyright (c) 2010-2015 The Research Foundation of SUNY 5 | 6 | %% Permission is hereby granted, free of charge, to any person 7 | %% obtaining a copy of this software and associated documentation files 8 | %% (the "Software"), to deal in the Software without restriction, 9 | %% including without limitation the rights to use, copy, modify, merge, 10 | %% publish, distribute, sublicense, and/or sell copies of the Software, 11 | %% and to permit persons to whom the Software is furnished to do so, 12 | %% subject to the following conditions: 13 | 14 | %% The above copyright notice and this permission notice shall be 15 | %% included in all copies or substantial portions of the Software. 16 | 17 | %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | %% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | %% MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | %% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | %% LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | %% OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | %% WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | -module(lamutex). 26 | -export([run/2, start/1, start/0, run_processes/2]). 27 | -export([site1/2, site/2, comm/5, broadcast_tracker/2]). 28 | 29 | %% ------------------------------ 30 | %% Command line entry point 31 | %% ------------------------------ 32 | start() -> 33 | run(10, 10). 34 | 35 | start([NPeers]) -> 36 | run(list_to_integer(NPeers), 5); 37 | start([NPeers, RTime]) -> 38 | run(list_to_integer(NPeers), list_to_integer(RTime)). 39 | 40 | %% ------------------------------ 41 | %% Program entry point 42 | %% ------------------------------ 43 | run(NumPeers, RunTime) 44 | when is_integer(NumPeers), is_integer(RunTime) -> 45 | io:format("Starting Comm processes...\n"), 46 | Comms = create_comm_processes(NumPeers), 47 | %% Register the broadcast process so Comms would know where to send 48 | %% broadcast requests: 49 | register(tracker, spawn(lamutex, broadcast_tracker, [Comms, self()])), 50 | io:format("Starting Site processes... ~w~n \n", [length(Comms)]), 51 | Sites = create_site_processes(Comms, RunTime), 52 | statistics(runtime), 53 | {WallclockTime, _Result} = timer:tc(lamutex, run_processes, [Sites, NumPeers]), 54 | {_, CpuTime} = statistics(runtime), 55 | dump_perf_data(WallclockTime, NumPeers, CpuTime), 56 | halt(0). 57 | 58 | run_processes(Sites, NumPeers) -> 59 | do_broadcast(Sites, start), 60 | wait_all_done(NumPeers). 61 | 62 | wait_all_done(0) -> 63 | ok; 64 | wait_all_done(NumLeft) 65 | when NumLeft > 0 -> 66 | receive 67 | {finished, _Id} -> 68 | wait_all_done(NumLeft-1) 69 | end. 70 | 71 | get_memory_stats() -> 72 | {ok, StatsFile} = file:open(filename:join(['/proc', os:getpid(), 'status']), 73 | [read]), 74 | {ok, Matcher} = re:compile("^VmHWM:[^0-9]*\([0-9]+\) kB"), 75 | case first_matching_line(Matcher, StatsFile) of 76 | {vmhwm, Value} -> Value; 77 | notfound -> "0" 78 | end. 79 | 80 | first_matching_line(Matcher, Device) -> 81 | case io:get_line(Device, "") of 82 | eof -> 83 | file:close(Device), 84 | notfound; 85 | Line -> 86 | case re:run(Line, Matcher, [{capture, all_but_first, list}]) of 87 | {match, [Cap]} -> 88 | file:close(Device), 89 | {vmhwm, Cap}; 90 | nomatch -> 91 | first_matching_line(Matcher, Device) 92 | end 93 | end. 94 | 95 | dump_perf_data(WallclockTime, NumPeers, CpuTime) -> 96 | io:format("###OUTPUT: {\"Total_memory\": ~s, " 97 | "\"Wallclock_time\": ~f, \"Total_processes\": ~w, " 98 | "\"Total_process_time\": ~f, \"Total_user_time\": ~f}~n", 99 | [get_memory_stats(), WallclockTime/1000000, 100 | NumPeers, CpuTime/1000, CpuTime/1000]). 101 | 102 | %% -------------------- 103 | %% This helper process handles all broadcast requests. It's a small hackery to 104 | %% get around the circular dependency of letting every peer process know about 105 | %% all other peers before they are actually created. Maybe there's a better 106 | %% way to do this... 107 | %% -------------------- 108 | broadcast_tracker(Targets, ParentId) -> 109 | receive 110 | {broadcast, Message} -> 111 | do_broadcast(Targets, Message), 112 | broadcast_tracker(Targets, ParentId); 113 | {finished, Time} -> 114 | ParentId ! {finished, Time}, 115 | broadcast_tracker(Targets, ParentId) 116 | end. 117 | do_broadcast([], _) -> 118 | ok; 119 | do_broadcast([To|Rest], Message) -> 120 | To ! Message, 121 | do_broadcast(Rest, Message). 122 | 123 | 124 | %% -------------------- 125 | %% Create all the "Comm" processes. 126 | %% -------------------- 127 | create_comm_processes(N) -> 128 | [spawn(lamutex, comm, [Id, N, [], 0, ignore]) || Id <- lists:seq(1, N)]. 129 | 130 | %% -------------------- 131 | %% Create all the "Site" processes. 132 | %% -------------------- 133 | create_site_processes(CommList, Rounds) -> 134 | [spawn(lamutex, site1, [Id, Rounds]) || Id <- CommList]. 135 | 136 | %% -------------------- 137 | %% Site entry point. 138 | %% -------------------- 139 | site1(CommId, Rounds) -> 140 | io:format("Site ~w starting...\n", [CommId]), 141 | receive 142 | start -> ok 143 | end, 144 | site(CommId, Rounds), 145 | io:format("Process ~w done. \n", [CommId]), 146 | tracker ! {finished, CommId}. 147 | 148 | %% -------------------- 149 | %% The "Site" process is now extremely simple and straight forward: 1. do 150 | %% non-critical stuff; 2. try enter critical section by notifying our "Comm" 151 | %% and waiting for the "goahead" signal; 3. do critical work; 4. leave CS. It 152 | %% maintains no data apart from the corresponding "Comm" PID that handles CS 153 | %% requests on its behalf. 154 | %% -------------------- 155 | site(_CommId, 0) -> 156 | ok; 157 | site(CommId, Rounds) -> 158 | %% Non CS: 159 | enter_critical_section(CommId), 160 | %% In CS: 161 | io:format("In critical section: ~w \n", [CommId]), 162 | %procrastinate(), 163 | %% Leave CS: 164 | io:format("Leaving critical section: ~w \n", [CommId]), 165 | leave_critical_section(CommId), 166 | site(CommId, Rounds-1). 167 | 168 | %% -------------------- 169 | %% All data structures pertaining to critical section management is now moved 170 | %% into the "Comm" process. 171 | %% -------------------- 172 | comm(SiteId, NPeers, ReqQueue, Clock, AckSet) -> 173 | %% First check whether our Site is ready to enter CS: 174 | Result = check_can_enter_cs(self(), ReqQueue, AckSet, NPeers), 175 | if 176 | Result -> 177 | SiteId ! goahead, 178 | comm(SiteId, NPeers, ReqQueue, Clock, ignore); 179 | true -> 180 | ok 181 | end, 182 | %% Then handle the next message: 183 | receive 184 | {request, From, Clk} -> 185 | From ! {ack, self(), Clock}, % Send ack reply 186 | comm(SiteId, NPeers, 187 | lists:keystore(From, 2, ReqQueue, {Clk, From}), 188 | max(Clock, Clk) + 1, 189 | AckSet); 190 | 191 | {ack, From, _Clk} -> 192 | %% 'ignore' is used as a minor optimiazation: If our Site is not 193 | %% trying to enter CS, then we can safely ignore all ack messages: 194 | if 195 | AckSet == ignore -> 196 | comm(SiteId, NPeers, ReqQueue, Clock, AckSet); 197 | true -> 198 | comm(SiteId, NPeers, ReqQueue, Clock, 199 | sets:add_element(From, AckSet)) 200 | end; 201 | 202 | {release, From, _Clk} -> 203 | comm(SiteId, NPeers, 204 | lists:keydelete(From, 2, ReqQueue), Clock, AckSet); 205 | 206 | %% These messages are used to communicate with our Site: 207 | {enter, Id} -> % Our own site wishes to enter 208 | broadcast({request, self(), Clock}), 209 | comm(Id, NPeers, ReqQueue, Clock, sets:new()); 210 | {release, Id} when Id == SiteId-> % Our own site is done 211 | broadcast({release, self(), Clock}), 212 | comm(Id, NPeers, ReqQueue, Clock, ignore) 213 | end. 214 | 215 | %% ================================================== 216 | %% Helpers 217 | %% ================================================== 218 | 219 | %% Just send the message to the broadcast tracker process: 220 | broadcast(Message) -> 221 | tracker ! {broadcast, Message}. 222 | 223 | %% CS condition check. If successful send goahead back to site: 224 | check_can_enter_cs(CommId, ReqQueue, AckSet, NumPeers) -> 225 | AckSet /= ignore andalso 226 | sets:size(AckSet) == NumPeers andalso 227 | length(ReqQueue) > 0 andalso 228 | element(2, lists:min(ReqQueue)) == CommId. 229 | 230 | %% Time waster: 231 | procrastinate() -> 232 | receive 233 | after random:uniform(3) * 1000 -> 234 | ok 235 | end. 236 | 237 | enter_critical_section(CommId) -> 238 | CommId ! {enter, self()}, 239 | receive 240 | goahead -> 241 | ok 242 | end. 243 | 244 | leave_critical_section(CommId) -> 245 | CommId ! {release, self()}. 246 | -------------------------------------------------------------------------------- /benchmarks/lamutex/Python/lamutex.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) 2010-2015 Bo Lin 3 | # Copyright (c) 2010-2015 Yanhong Annie Liu 4 | # Copyright (c) 2010-2015 Stony Brook University 5 | # Copyright (c) 2010-2015 The Research Foundation of SUNY 6 | # 7 | # Permission is hereby granted, free of charge, to any person 8 | # obtaining a copy of this software and associated documentation files 9 | # (the "Software"), to deal in the Software without restriction, 10 | # including without limitation the rights to use, copy, modify, merge, 11 | # publish, distribute, sublicense, and/or sell copies of the Software, 12 | # and to permit persons to whom the Software is furnished to do so, 13 | # subject to the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be 16 | # included in all copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 21 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 22 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 24 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 | 26 | import sys 27 | import time 28 | import json 29 | import queue 30 | import random 31 | import resource 32 | import threading 33 | import multiprocessing 34 | from heapq import heappush, heappop, heapify 35 | 36 | BROADCAST = -1 37 | LOG = -2 38 | 39 | # ============================== 40 | # The 'Site' class represents a peer in the network. Each 'Site' is run in a 41 | # separate process 42 | # ============================== 43 | class Site(multiprocessing.Process): 44 | # ==================== 45 | # Each 'Site' will have one 'Comm' thread, its sole purpose is to reply all 46 | # incoming requests: 47 | # ==================== 48 | class Comm(threading.Thread): 49 | def __init__(self, pid, net, queue): 50 | threading.Thread.__init__(self) 51 | self.peerid = pid # self's unique process id 52 | self.queue = queue # Queue used to communicate with our main Site 53 | # process 54 | self.net = net # Interface to the "network" (which is a 55 | # dictionary containing an 'in' queue and an 56 | # 'out' queue.) 57 | self.daemon = True 58 | 59 | # Simple algorithm for Comm: Take a message from net 'in' and put it 60 | # into the queue for Site. If the message is a request we reply it: 61 | def run(self): 62 | while True: 63 | mtype, source, clock = receive(self.net) 64 | self.queue.put((mtype, source, clock)) 65 | if (mtype == "request"): 66 | send(self.net, source, ("ack", self.peerid, clock)) 67 | 68 | def __init__(self, pid, net, iterations, peers): 69 | multiprocessing.Process.__init__(self) 70 | self.peerid = pid # Our unique peer id 71 | self.net = net # Interface to the "network" (which is a dict 72 | # containing an in queue and an out queue.) 73 | self.peers = peers # List of all processes, including self 74 | self.iterations = iterations # Number of iterations to run 75 | self.thread_queue = queue.Queue() # Queue used to communicate with Comm 76 | self.acks = set() # The set of replies we've received 77 | self.reqs = [] # Request priority queue, in clock order 78 | self.clock = 0 # Our logical clock 79 | self.comm = Site.Comm(pid, net, self.thread_queue) 80 | self.running = False 81 | 82 | def run(self): 83 | self.comm.start() # Start the "Comm" thread 84 | while not self.running: 85 | self.handle_message(True) 86 | 87 | rudata_start = resource.getrusage(resource.RUSAGE_SELF) 88 | count = 0 89 | while(self.running and count < self.iterations): 90 | # Non CS: 91 | self.label() # labels mark the points where we can "break" 92 | self.work() 93 | self.label() 94 | self.enter_critical_section() 95 | print("Process %d(clock:%d) has entered critical section."% 96 | (self.peerid,self.clock)) 97 | self.label() 98 | self.work() 99 | self.label() 100 | print("Process %d is leaving critical section."%self.peerid) 101 | self.leave_critical_section() 102 | count += 1 103 | 104 | rudata_end = resource.getrusage(resource.RUSAGE_SELF) 105 | utime = rudata_end.ru_utime - rudata_start.ru_utime 106 | stime = rudata_end.ru_stime - rudata_start.ru_stime 107 | send(self.net, LOG, (utime, stime, rudata_end.ru_maxrss)) 108 | while self.running: 109 | self.handle_message(True) 110 | 111 | # This simulates the controlled "label" mechanism. Currently we simply 112 | # handle one message on one label call: 113 | def label(self): 114 | self.handle_message(False) 115 | 116 | # Handles one message. 'block' indicates whether to blocking waiting for 117 | # next message to come in if the queue is currently empty: 118 | def handle_message(self, block): 119 | try: 120 | (mtype, source, clock) = self.thread_queue.get(block) 121 | if mtype == "request": 122 | # Put request on queue and update clock 123 | heappush(self.reqs, (clock, source)) 124 | self.clock = max(self.clock, clock) + 1 125 | elif mtype == "ack": 126 | self.acks.add(source) 127 | elif mtype == "release": 128 | # Remove the sender from our request queue 129 | for (clk, peer) in self.reqs: 130 | if peer == source: 131 | self.reqs.remove((clk, peer)) 132 | # Note that we have to re-heapify after the remove: 133 | heapify(self.reqs) 134 | elif mtype == "terminate": 135 | self.running = False 136 | elif mtype == "start": 137 | self.running = True 138 | else: 139 | raise RuntimeError("Unknown message type " + mtype) 140 | except queue.Empty: 141 | pass 142 | 143 | # We keep handling messages until the wait condition becomes true: 144 | def enter_critical_section(self): 145 | self.acks = set() 146 | send(self.net, BROADCAST, ("request", self.peerid, self.clock)) 147 | while (len(self.reqs) == 0 or 148 | self.reqs[0][1] != self.peerid or 149 | len(self.acks) != len(self.peers)): 150 | self.handle_message(True) 151 | 152 | def leave_critical_section(self): 153 | send(self.net, BROADCAST, ("release", self.peerid, self.clock)) 154 | 155 | # Simulate work, waste some random amount of time: 156 | def work(self): 157 | # time.sleep(random.randint(1, 5)) 158 | return 159 | 160 | 161 | # ============================== 162 | # The "Simulator" class takes care of message routing between the peers: 163 | # ============================== 164 | class Simulator(threading.Thread): 165 | def __init__(self, sites, netq): 166 | threading.Thread.__init__(self) 167 | self.sites = sites 168 | self.netq = netq # This is the shared 'out' queue of all the 169 | # processes 170 | 171 | self.perf_usrtime = 0 172 | self.perf_systime = 0 173 | self.perf_memory = 0 174 | 175 | self.completed = 0 176 | 177 | def run(self): 178 | t1 = time.perf_counter() 179 | for s in self.sites: 180 | s.net['in'].put(('start', -1, -1)) 181 | 182 | while self.completed < len(self.sites): 183 | message = self.netq.get(True) 184 | self.dispatch_message(message) 185 | 186 | t2 = time.perf_counter() 187 | for s in self.sites: 188 | s.net['in'].put(('terminate', -1, -1)) 189 | 190 | jsondata = {'Wallclock_time': t2 - t1, 191 | 'Total_processes': self.completed, 192 | 'Total_process_time': self.perf_usrtime + self.perf_systime, 193 | 'Total_user_time': self.perf_usrtime, 194 | 'Total_memory': self.perf_memory} 195 | jsonoutput = json.dumps(jsondata) 196 | print("###OUTPUT: " + jsonoutput) 197 | 198 | for site in self.sites: 199 | site.join() 200 | 201 | def dispatch_message(self, message): 202 | (to, m) = message 203 | if to == BROADCAST: 204 | for site in self.sites: 205 | site.net['in'].put(m) 206 | elif to == LOG: 207 | self.completed += 1 208 | 209 | utime, stime, maxrss = m 210 | self.perf_usrtime += utime 211 | self.perf_systime += stime 212 | self.perf_memory += maxrss 213 | 214 | else: 215 | self.sites[to].net['in'].put(m) 216 | 217 | # Create nproc "Sites": 218 | def create_sites(nproc, q, iterations): 219 | return [Site(id, {'out':q, 'in':multiprocessing.Queue()}, 220 | iterations, range(0, nproc)) 221 | for id in range(0, nproc)] 222 | 223 | # ============================== 224 | # Main program entry point: 225 | # ============================== 226 | def main(nproc, iterations): 227 | queue = multiprocessing.Queue() 228 | print("Creating sites...") 229 | sites = create_sites(nproc, queue, iterations) 230 | print("Creating simulator...") 231 | sim = Simulator(sites, queue) 232 | print("Starting %d sites...\n"%nproc) 233 | for site in sites: 234 | site.start() 235 | print("Starting simulator...") 236 | sim.start() 237 | sim.join() 238 | 239 | # Wrapper functions for message passing: 240 | def send(net, to, message): 241 | net['out'].put((to, message)) 242 | 243 | def receive(net): 244 | return net['in'].get() 245 | 246 | if __name__ == "__main__": 247 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 248 | iterations = int(sys.argv[2]) if len(sys.argv) > 2 else 5 249 | main(nprocs, iterations) 250 | -------------------------------------------------------------------------------- /benchmarks/lamutex/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | controller = import_da('controller') 3 | 4 | class P(process, controller.Controllee): 5 | def setup(ctl, s:set, nrequests:int): # s is set of all other processes 6 | super().setup(ctl) 7 | self.q = set() 8 | 9 | def mutex(task): 10 | -- request 11 | c = logical_clock() 12 | send(('request', c, self), to= s) 13 | q.add(('request', c, self)) 14 | await(each(('request', c2, p) in q, 15 | has= (c2, p)==(c, self) or (c, self) < (c2, p)) and 16 | each(p in s, has= some(received(('ack', c2, _p)), has= c2 > c))) 17 | -- critical_section 18 | task() 19 | -- release 20 | q.remove(('request', c, self)) 21 | send(('release', logical_clock(), self), to= s) 22 | 23 | def receive(msg= ('request', c2, p)): 24 | q.add(('request', c2, p)) 25 | send(('ack', logical_clock(), self), to= p) 26 | 27 | def receive(msg= ('release', _, p)): 28 | # q.remove(('request', _, p)) # pattern matching needed for _ 29 | # q.remove(anyof(setof(('request', c, p), ('request', c, _p) in q))) 30 | for x in setof(('request', c, p), ('request', c, _p) in q): 31 | q.remove(x); break 32 | # for ('request', c, _p) in q: q.remove('request', c, p); break 33 | # for (tag, c, p2) in q: 34 | # if tag == 'request' and p2 == p: 35 | # q.remove((tag, c, p2)); break 36 | 37 | @controller.run 38 | def run(): # change to run 39 | def task(): output('in cs') 40 | for i in range(nrequests): 41 | mutex(task) 42 | 43 | def main(): 44 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 45 | nrequests = int(sys.argv[2]) if len(sys.argv) > 2 else 1 46 | 47 | config(channel= 'fifo', clock= 'Lamport') 48 | 49 | ctl = new(controller.Controller, num= 1) 50 | setup(ctl, (nprocs,)) 51 | start(ctl) 52 | 53 | ps = new(P, num= nprocs) 54 | for p in ps: setup(p, (ctl, ps-{p}, nrequests)) 55 | start(ps) 56 | 57 | # This is an executable specification of the algorithm described in 58 | # Lamport, L. (1978). "Time, clocks, and the ordering of events in a 59 | # distributed system". Communications of the ACM, 21(7):558-565. 60 | 61 | # This code includes setup and termination for serving a given number of 62 | # requests per process. 63 | 64 | # All labels are not needed, 65 | # leaving 14 or 15 lines total for the algorithm body and message handlers. 66 | -------------------------------------------------------------------------------- /benchmarks/lamutex/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | controller = import_da('controller') 3 | 4 | class P(process, controller.Controllee): 5 | def setup(ctl, s:set, nrequests:int): # s is set of all other processes 6 | super().setup(ctl) 7 | 8 | def mutex(task): 9 | -- request 10 | c = logical_clock() 11 | send(('request', c, self.id), to= s) 12 | await(each(received(('request', c2, p)), 13 | has= received(('release', c2, p)) or (c, self.id) < (c2, p)) 14 | and each(p in s, has= received(('ack', c, p)))) 15 | -- critical_section 16 | task() 17 | -- release 18 | send(('release', c, self.id), to= s) 19 | 20 | def receive(msg= ('request', c, p)): 21 | send(('ack', c, self.id), to= p) 22 | 23 | @controller.run 24 | def run(): 25 | def task(): output('in cs') 26 | for i in range(nrequests): 27 | mutex(task) 28 | 29 | def main(): 30 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 31 | nrequests = int(sys.argv[2]) if len(sys.argv) > 2 else 1 32 | 33 | config(channel= 'fifo', clock= 'Lamport') 34 | 35 | ctl = new(controller.Controller, num= 1) 36 | setup(ctl, (nprocs,)) 37 | start(ctl) 38 | 39 | ps = new(P, num= nprocs) 40 | for p in ps: setup(p, (ctl, ps-{p}, nrequests)) 41 | start(ps) 42 | 43 | # This is an executable specification that simplifies the algorithm in 44 | # Lamport, L. (1978). "Time, clocks, and the ordering of events in a 45 | # distributed system". Communications of the ACM, 21(7):558-565. 46 | 47 | # The simplification is to remove an unnecessary q and message handler; 48 | # release and ack messages include the request time, 49 | # not timestamps unused or used for unnecessary expensive comparison. 50 | 51 | # This code includes setup and termination for serving a given number of 52 | # requests per process. 53 | 54 | # All labels are not needed, 55 | # leaving a total of 9 lines for the algorithm body and message handler. 56 | -------------------------------------------------------------------------------- /benchmarks/lamutex/spec_lam.da: -------------------------------------------------------------------------------- 1 | import sys 2 | controller = import_da('controller') 3 | 4 | class P(process, controller.Controllee): 5 | def setup(ctl, s:set, nrequests:int): # s is set of all other processes 6 | super().setup(ctl) 7 | 8 | def mutex(task): 9 | -- request 10 | c = logical_clock() 11 | send(('request', c, self.id), to= s) 12 | await(each(received(('request', c2, p)), 13 | has= received(('release', c2, p)) or (c, self.id) < (c2, p)) 14 | and each(p in s, has= some(received(('ack', c2, _p)), has= c2 > c))) 15 | -- critical_section 16 | task() 17 | -- release 18 | send(('release', c, self.id), to= s) 19 | 20 | def receive(msg= ('request', _, p)): 21 | send(('ack', logical_clock(), self.id), to= p) 22 | 23 | @controller.run 24 | def run(): 25 | def task(): output('in cs') 26 | for i in range(nrequests): 27 | mutex(task) 28 | 29 | def main(): 30 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 31 | nrequests = int(sys.argv[2]) if len(sys.argv) > 2 else 1 32 | 33 | config(channel= 'fifo', clock= 'Lamport') 34 | 35 | ctl = new(controller.Controller, num= 1) 36 | setup(ctl, (nprocs,)) 37 | start(ctl) 38 | 39 | ps = new(P, num= nprocs) 40 | for p in ps: setup(p, (ctl, ps-{p}, nrequests)) 41 | start(ps) 42 | 43 | # This is an executable specification that simplifies the algorithm in 44 | # Lamport, L. (1978). "Time, clocks, and the ordering of events in a 45 | # distributed system". Communications of the ACM, 21(7):558-565. 46 | 47 | # The simplification is to remove an unnecessary queue and message handler; 48 | # release msgs use the request time, not unused timestamps, to replace queue. 49 | 50 | # This code includes setup and termination of a given number of processes 51 | # each serving a given number of requests. 52 | 53 | # All labels are not needed, 54 | # leaving 9 lines total for the algorithm body and message handler. 55 | -------------------------------------------------------------------------------- /benchmarks/lapaxos/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | from random import randint 3 | controller = import_da('controller') 4 | 5 | class Proposer(process, controller.Controllee): 6 | def setup(ctl, acceptors:set, timeout): 7 | super().setup(ctl) 8 | self.n = None # proposal number 9 | self.majority = acceptors # majority of acceptors; all in other papers 10 | 11 | def to_consent(): 12 | n = (0, self.id) if n == None else (n[0]+1, self.id) # pick a prop num 13 | send(('prepare', n), to= majority) 14 | 15 | if await(len(setof(a, received(('respond', _n, _), from_ =a))) 16 | > len(acceptors)/2): 17 | v = anyof(setof(v, received(('respond', _n, (n2, v))), 18 | n2==max(setof(n2, received(('respond', _n, (n2, _)))))) 19 | or {randint(1,100)}) # any value, pick in 1..100 20 | responded = setof(a, received(('respond', _n, _), from_ =a)) 21 | send(('accept', n, v), to= responded) 22 | #output('### chose', n, v) 23 | 24 | elif timeout(timeout): 25 | #output('failed proposal number', n) 26 | pass 27 | 28 | @controller.run 29 | def run(): 30 | while not received(('done',)): 31 | to_consent() 32 | 33 | def anyof(s): 34 | return next(iter(s)) if s else None 35 | 36 | @controller.rugroup('bo_measured') 37 | class Acceptor(process, controller.Controllee): 38 | def setup(ctl, learners:set): 39 | super().setup(ctl) 40 | 41 | def receive(msg= ('prepare', n), from_= p): 42 | if each(sent(('respond', n2, _)), has= n > n2): 43 | maxprop = anyof(setof((n, v), sent(('accepted', n, v)), 44 | n==max(setof(n, sent(('accepted', n, _)))))) 45 | send(('respond', n, maxprop), to =p) 46 | 47 | def receive(msg= ('accept', n, v)): 48 | if not some(sent(('respond', n2, _)), has= n2 > n): 49 | send(('accepted', n, v), to= learners) 50 | 51 | @controller.run 52 | def run(): 53 | return 0 54 | 55 | def anyof(s): 56 | """return any element of set s if s is not empty or 'None' otherwise""" 57 | return next(iter(s)) if s else None 58 | 59 | class Learner(process, controller.Controllee): 60 | def setup(ctl, acceptors:set, proposer:Proposer, timeout): 61 | super().setup(ctl) 62 | 63 | def learn(): 64 | if await(some(received(('accepted', n, v)), 65 | has= len(setof(a, received(('accepted', _n, _v), from_=a))) 66 | > len(acceptors)/2)): 67 | output('learned', n, v) 68 | 69 | elif timeout(timeout): 70 | output('failed learning anything') 71 | 72 | @controller.run 73 | def run(): 74 | learn() 75 | send(('done',), to= proposer) 76 | 77 | def main(): 78 | nacceptors = int(sys.argv[1]) if len(sys.argv) > 1 else 3 79 | nproposers = int(sys.argv[2]) if len(sys.argv) > 2 else 3 80 | timeout = int(sys.argv[3]) if len(sys.argv) > 3 else 1 81 | 82 | ctl = new(controller.Controller, num= 1) 83 | setup(ctl, (nacceptors + nproposers * 2, )) 84 | start(ctl) 85 | 86 | acceptors = new(Acceptor, num= nacceptors) 87 | proposers = new(Proposer, (ctl, acceptors, timeout), num= nproposers) 88 | learners = new(Learner, num= nproposers) 89 | for a in acceptors: setup(a, (ctl, learners,)) 90 | for p, l in zip(proposers, learners): setup(l, (ctl, acceptors, p, timeout)) 91 | start(acceptors | proposers | learners) 92 | 93 | # This is an executable specification of the algorithm described in 94 | # Lamport, L. (2001). Paxos Made Simple. ACM SIGACT News 95 | # (Distributed Computing Column), 32(4):51-58, December. 96 | 97 | # This code includes setup and termination for running repeated rounds until 98 | # the learners all terminate after learning the consent value or timeout. 99 | -------------------------------------------------------------------------------- /benchmarks/pingpong/ping.da: -------------------------------------------------------------------------------- 1 | import sys 2 | controller = import_da('controller') 3 | 4 | class Pong(process, controller.Controllee): 5 | def setup(ctl, total_pings): 6 | super().setup(ctl) 7 | 8 | @controller.run 9 | def run(): 10 | await(len(listof(p, received(('Ping',), from_=p))) == total_pings) 11 | 12 | def receive(msg=('Ping',), from_=p): 13 | output("Pinged") 14 | send(('Pong',), to=p) 15 | 16 | class Ping(process, controller.Controllee): 17 | def setup(ctl, p, nrounds): 18 | super().setup(ctl) 19 | 20 | @controller.run 21 | def run(): 22 | for i in range(nrounds): 23 | clk = logical_clock() 24 | send(('Ping',), to=p) 25 | await(some(received(('Pong',), clk=rclk), has=(rclk > clk))) 26 | 27 | def receive(msg=('Pong',)): 28 | output("Ponged.") 29 | 30 | def main(): 31 | nrounds = int(sys.argv[1]) if len(sys.argv) > 1 else 3 32 | npings = int(sys.argv[2]) if len(sys.argv) > 2 else 3 33 | config(clock='Lamport') 34 | 35 | ctl = new(controller.Controller, num= 1) 36 | setup(ctl, (npings + 1,)) 37 | start(ctl) 38 | 39 | pong = new(Pong, [ctl, nrounds * npings], num= 1) 40 | ping = new(Ping, num= npings) 41 | setup(ping, (ctl, pong, nrounds)) 42 | start(pong) 43 | start(ping) 44 | -------------------------------------------------------------------------------- /benchmarks/ramutex/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | controller = import_da('controller') 3 | 4 | class P(process, controller.Controllee): 5 | def setup(ctl, ps:set, nrounds:int): 6 | super().setup(ctl) 7 | self.reqc = None 8 | self.waiting = set() 9 | self.replied = set() 10 | 11 | def cs(task): 12 | # to enter cs, enque and send request to all, then await replies from all 13 | --start 14 | reqc = logical_clock() 15 | send(('Request', reqc), to=ps) 16 | 17 | await(len(replied) == len(ps)) 18 | 19 | # critical section 20 | task() 21 | 22 | # to exit cs, deque and send releases to all 23 | --release 24 | reqc = None 25 | send(('Reply', logical_clock()), to=waiting) 26 | --end 27 | waiting = set() 28 | replied = set() 29 | 30 | @controller.run 31 | def run(): 32 | def anounce(): 33 | output("In cs!") 34 | for i in range(nrounds) : 35 | cs(anounce) 36 | 37 | # when receiving requests from others, enque and reply 38 | def receive(msg=('Request', timestamp), from_=source): 39 | if (reqc == None or (timestamp, source) < (reqc, self.id)): 40 | send(('Reply', logical_clock()), to=source) 41 | else: 42 | waiting.add(source) 43 | 44 | def receive(msg=('Reply', c1), from_=source): 45 | if reqc is not None and c1 > reqc: 46 | replied.add(source) 47 | 48 | def main(): 49 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 50 | nrounds = int(sys.argv[2]) if len(sys.argv) > 2 else 1 51 | config(clock='Lamport') 52 | 53 | ctl = new(controller.Controller, num= 1) 54 | setup(ctl, (nprocs,)) 55 | start(ctl) 56 | 57 | ps = new(P, num= nprocs) 58 | # setup the processes 59 | for p in ps: setup({p}, (ctl, ps-{p}, nrounds)) 60 | # start the processes 61 | start(ps) 62 | -------------------------------------------------------------------------------- /benchmarks/ramutex/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | controller = import_da('controller') 3 | 4 | class P(process, controller.Controllee): 5 | def setup(ctl, s:set, n:int): # pass in set of all processes 6 | super().setup(ctl) 7 | 8 | def cs(task): 9 | --request 10 | ownc = logical_clock() 11 | send(('request', ownc, self.id), to=s) # send request to all processes 12 | --cs 13 | await(each(p in s, 14 | has=some(received(('ack', c, _p)), has=(c > ownc)))) 15 | task() # critical section 16 | --release 17 | send(('ack', logical_clock(), self.id), 18 | to=setof(p, p in s, 19 | some(received(('request', c, _p)), 20 | has=((c, p) >= (ownc, self.id))))) 21 | 22 | def receive(msg=('request', c, p)): 23 | ownc = max(setof(c, sent(('request', c, self.id)), 24 | not each(p in s, has=some(received(('ack', c2, _p)), 25 | has= c2 > c))) | 26 | {-1}) 27 | if ownc == -1 or (c, p) < (ownc, self.id): 28 | send(('ack', logical_clock(), self.id), to=p) 29 | 30 | @controller.run 31 | def run(): 32 | def anounce(): 33 | output("In cs!") 34 | for i in range(n): 35 | cs(anounce) 36 | 37 | def main(): 38 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 39 | nrounds = int(sys.argv[2]) if len(sys.argv) > 2 else 1 40 | config(clock='Lamport') 41 | 42 | ctl = new(controller.Controller, num= 1) 43 | setup(ctl, (nprocs,)) 44 | start(ctl) 45 | 46 | ps = new(P, num= nprocs) 47 | # setup the processes 48 | for p in ps: setup({p}, (ctl, ps-{p}, nrounds)) 49 | # start the processes 50 | start(ps) 51 | -------------------------------------------------------------------------------- /benchmarks/ratoken/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | controller = import_da('controller') 3 | 4 | class P (process, controller.Controllee): 5 | def setup(ctl, ps:set, nrounds:int, orig_token:bool): 6 | super().setup(ctl) 7 | self.clock = 0 8 | self.token = dict((p, 0) for p in ps) 9 | 10 | def cs(task): 11 | --request 12 | if not token_present(): 13 | clock += 1 14 | send(('request', clock, self.id), to=ps) 15 | await(token_present()) 16 | token[self.id] = clock 17 | 18 | task() # critical section 19 | 20 | --release 21 | for p in ps: 22 | if request_pending(p) and token_present(): 23 | #output("sending %r-> %r" % (token, p)) 24 | send(('access', token), to=p) 25 | break 26 | 27 | def receive(msg=('access', newtok)): 28 | token = newtok 29 | 30 | def receive(msg=('request', c, p)): 31 | if request_pending(p) and token_present(): 32 | send(('access', token), to=p) 33 | 34 | def request_pending(p): 35 | # p has a request after it last had the token 36 | return some(received(('request', c, _p)), has=(c > token[p])) 37 | 38 | def token_present(): 39 | return (orig_token and not some(sent(('access', _))) or 40 | some(received(('access', token1)), 41 | has= (not some(sent(('access', token2)), 42 | has= (token2[self.id] > token1[self.id]))))) 43 | 44 | @controller.run 45 | def run(): 46 | def anounce(): 47 | output("In cs!") 48 | if token_present(): 49 | output("I'm lucky!") 50 | for i in range(nrounds): 51 | cs(anounce) 52 | 53 | def main(): 54 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 55 | nrounds = int(sys.argv[2]) if len(sys.argv) > 2 else 1 56 | 57 | ctl = new(controller.Controller, num= 1) 58 | setup(ctl, (nprocs,)) 59 | start(ctl) 60 | 61 | # create n process 62 | ps = new(P, num= nprocs) 63 | 64 | p = ps.pop() 65 | setup(ps, (ctl, ps|{p}, nrounds, False)) 66 | setup([p], (ctl, ps|{p}, nrounds, True)) 67 | start(ps|{p}) 68 | -------------------------------------------------------------------------------- /benchmarks/sktoken/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | controller = import_da('controller') 3 | 4 | class P(process, controller.Controllee): 5 | def setup(ctl, ps:set, orig_token:bool, 6 | nrounds:int): # other procs, and whether self.id holds token 7 | super().setup(ctl) 8 | self.RN = dict((p, 0) for p in ps) # last request number received 9 | # Token structure: 10 | self.Q = [] # queue of pending requests 11 | self.LN = dict((p, 0) for p in ps) # last request number for which token was granted 12 | 13 | def cs(task): 14 | -- request 15 | if not haveToken(): 16 | RN[self.id] += 1 17 | send(('request', self.id, RN[self.id]), to=ps) 18 | #await(some(rcvd(('token', _, _LN1)), LN1[self.id] < rn)) 19 | await(haveToken()) 20 | 21 | task() #critical section 22 | 23 | LN[self.id] = RN[self.id] 24 | Q.extend(listof(p, p in ps, p not in Q, RN[p] == LN[p] + 1)) 25 | listlen = len 26 | if listlen(Q) > 0: 27 | p = Q.pop() 28 | send(('token', Q, LN), to=p) 29 | 30 | def receive(msg=('token', Q1, LN1)): 31 | Q = Q1 32 | LN = LN1 33 | 34 | def receive(msg=('request', p, n)): 35 | RN[p] = max((RN[p], n)) 36 | if (haveToken() and RN[p] == LN[p] + 1): 37 | send(('token', Q, LN), to=p) 38 | 39 | def haveToken(): 40 | return (orig_token and not some(sent(('token', _, _)))) or\ 41 | some(received(('token', _, LN1)), 42 | has=(not some(sent(('token', _, LN2)), 43 | has=(LN2[self.id] > LN1[self.id])))) 44 | 45 | @controller.run 46 | def run(): 47 | def anounce(): 48 | output("In cs!") 49 | if haveToken(): 50 | output("I'm lucky!") 51 | for i in range(nrounds): 52 | cs(anounce) 53 | 54 | def main(): 55 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 56 | nrounds = int(sys.argv[2]) if len(sys.argv) > 2 else 1 57 | 58 | ctl = new(controller.Controller, num= 1) 59 | setup(ctl, (nprocs,)) 60 | start(ctl) 61 | 62 | # create n process 63 | ps = new(P, num= nprocs) 64 | 65 | # setup the processes 66 | lucky = ps.pop() 67 | 68 | setup(ps, (ctl, ps|{lucky}, False, nrounds)) 69 | setup({lucky}, (ctl, ps|{lucky}, True, nrounds)) 70 | start(ps|{lucky}) 71 | -------------------------------------------------------------------------------- /benchmarks/vrpaxos/orig.da: -------------------------------------------------------------------------------- 1 | # Note: Acceptor, Replica, Leader, and Client are all managed and timed 2 | # by the controller. Commander and Scout are created by processes 3 | # rather than by main(), so they are unknown to the controller. 4 | # Consequently their time is not included in the reported total 5 | # process time. 6 | 7 | import sys 8 | import time 9 | import random 10 | controller = import_da('controller') 11 | 12 | NOPS = 10 # number of different operations that the state machine can do 13 | # operation i maps a state to a pair of new state and result of i on state 14 | def operation(i): return lambda state: (state+[i], ['result',i,'on',state]) 15 | operations = {i: operation(i) for i in range(NOPS)} 16 | 17 | # wrong: all closures get i = NOPS - 1 18 | # ops = {i: (lambda state: (state+[i], ['res',i]+state)) for i in range(NOPS)} 19 | 20 | class Replica(process, controller.Controllee): 21 | def setup(ctl, leaders:set, initial_state): 22 | super().setup(ctl) 23 | self.state = initial_state; self.slot_num = 1 24 | self.proposals = set(); self.decisions = set() 25 | 26 | def propose(p): 27 | if not some((_, _p) in decisions): 28 | # find the maximum used slot number, or 0 29 | maxs = max(setof(s, (s, _) in proposals | decisions) or {0}) 30 | # within the maximum + 1, find smallest not yet used 31 | s1 = min(setof(s, s in range(1, maxs + 1 + 1), 32 | not some((_s, _) in proposals | decisions))) 33 | proposals.add((s1, p)) 34 | send(('propose', s1, p), to= leaders) 35 | 36 | def perform(p): 37 | output('### perform', p) 38 | client, cid, op = p 39 | if some((s, _p) in decisions, has= s < slot_num): 40 | slot_num += 1 41 | else: 42 | output('===', state, op) 43 | next, result = operations[op](state) 44 | output('===', next, result) 45 | state = next; slot_num += 1 46 | send(('response', cid, result), to= client) 47 | 48 | @controller.run 49 | def run(): 50 | await(received(('done',))) 51 | 52 | def receive(msg= ('request', p)): 53 | output('### request', p) 54 | propose(p) 55 | 56 | def receive(msg= ('decision', s, p)): 57 | output('### decision', s, p) 58 | decisions.add((s, p)) 59 | while some((_slot_num, p1) in decisions): 60 | if some((_slot_num, p2) in proposals, has= p2 != p1): 61 | propose(p2) 62 | perform(p1) 63 | 64 | class Acceptor(process, controller.Controllee): 65 | def setup(ctl): 66 | super().setup(ctl) 67 | self.ballot_num = None; self.accepted = set() 68 | 69 | @controller.run 70 | def run(): 71 | await(received(('done',))) 72 | 73 | def receive(msg= m): 74 | BOTTOM = (-1, -1); 75 | ballot_num = max((setof(b, received(('p1a', _, b))) | 76 | setof(b, received(('p2a', _, b, _, _)))) or {BOTTOM}) 77 | 78 | def receive(msg= ('p1a', leader_scout, b)): 79 | output('### p1a', leader_scout, b) 80 | send(('p1b', self.id, ballot_num, accepted), to= leader_scout) 81 | 82 | def receive(msg= ('p2a', leader_commander, b, s, p)): 83 | output('### p2a', leader_commander, b, s, p) 84 | if b == ballot_num : accepted.add((b, s, p)) 85 | send(('p2b', self.id, ballot_num), to= leader_commander) 86 | 87 | class Commander(process): 88 | def setup(leader, acceptors:set, replicas:set, b, s, p): pass 89 | 90 | def run(): 91 | output('### start') 92 | send(('p2a', self.id, b, s, p), to= acceptors) 93 | if await(len(setof(a, received(('p2b', a, _b)))) > len(acceptors)/2): 94 | send(('decision', s, p), to= replicas) 95 | elif some(received(('p2b', _, b1)), has= b1 != b): 96 | send(('preempted', b1), to= leader) 97 | 98 | class Scout(process): 99 | def setup(leader, acceptors:set, b): 100 | self.pvalues = set() 101 | 102 | def run(): 103 | output('### start') 104 | # add sleep 105 | time.sleep(random.random()) # next random float in [0.0, 1.0) 106 | send(('p1a', self.id, b), to= acceptors) 107 | if await(len(setof(a, received(('p1b', a, _b, _)))) > len(acceptors)/2): 108 | pvalues = setof(v, received(('p1b', _, _b, r)), v in r) #accepted 109 | send(('adopted', b, pvalues), to= leader) 110 | elif some(received(('p1b', a, b1, _)), has= b1 != b): 111 | send(('preempted', b1), to= leader) 112 | 113 | class Leader(process, controller.Controllee): 114 | def setup(ctl, acceptors:set, replicas:set): 115 | super().setup(ctl) 116 | self.ballot_num = (0, self.id); 117 | self.active = False; self.proposals = set() 118 | self.started = set() 119 | 120 | @controller.run 121 | def run(): 122 | # start(new(Scout, (self.id, acceptors, ballot_num))) 123 | sub = new(Scout, (self.id, acceptors, ballot_num)) 124 | start(sub); started.add(sub) 125 | await(received(('done',))) 126 | for tmp in started: tmp.join() 127 | 128 | def receive(msg= ('propose', s, p)): 129 | output('### propose', s, p) 130 | if not some((_s, _) in proposals): 131 | proposals.add((s,p)) 132 | if active: 133 | # start(new(Commander, 134 | # (self.id, acceptors, replicas, ballot_num, s, p))) 135 | sub = new(Commander, 136 | (self.id, acceptors, replicas, ballot_num, s, p)) 137 | start(sub); started.add(sub) 138 | 139 | def receive(msg= ('adopted', ballot_num, pvals)): 140 | output('### adopted', ballot_num, pvals) 141 | proposals = circle_plus(proposals, pmax(pvals)) 142 | for (s, p) in proposals: 143 | # start(new(Commander, 144 | # (self.id, acceptors, replicas, ballot_num, s, p))) 145 | sub = new(Commander, 146 | (self.id, acceptors, replicas, ballot_num, s, p)) 147 | start(sub); started.add(sub) 148 | active = True 149 | 150 | def receive(msg= ('preempted', (r1, leader1))): 151 | if (r1, leader1) > ballot_num: 152 | active = False 153 | ballot_num = (r1 + 1, self.id) 154 | # start(new(Scout, (self.id, acceptors, ballot_num))) 155 | sub = new(Scout, (self.id, acceptors, ballot_num)) 156 | start(sub); started.add(sub) 157 | 158 | def circle_plus(x, y): 159 | return y | setof((s, p), (s, p) in x, not some((_s, _) in y)) 160 | 161 | def pmax(pvals): 162 | return setof((s, p), (b, s, p) in pvals, 163 | each((b1, _s, _) in pvals, has= b1 <= b)) 164 | 165 | class Client(process, controller.Controllee): 166 | def setup(ctl, replicas:set, nops): 167 | super().setup(ctl) 168 | self.cid = 0 # command id 169 | self.results = dict() # map of command id to result of command 170 | self.count = dict() # map of command id to number of responses 171 | 172 | @controller.run 173 | def run(): 174 | for i in range(nops): 175 | send(('request', (self.id, cid, random.randint(0, NOPS-1))), 176 | to= replicas) 177 | await(cid in results) 178 | output('received result', cid, results[cid]) 179 | cid += 1 180 | await(each(cid in range(nops), has= count[cid] == len(replicas))) 181 | 182 | def receive(msg= ('response', cid, result)): 183 | output('### response', cid, result) 184 | if cid not in results: 185 | results[cid] = result 186 | elif results[cid] != result: 187 | output('different result', cid, result, 'than', results[cid]) 188 | count[cid] = 1 if cid not in count else count[cid] + 1 189 | 190 | def main(): 191 | nacceptors = int(sys.argv[1]) if len(sys.argv) > 1 else 3 # 8 192 | nreplicas = int(sys.argv[2]) if len(sys.argv) > 2 else 3 # 4 193 | nleaders = int(sys.argv[3]) if len(sys.argv) > 3 else 1 # 2 194 | nclients = int(sys.argv[4]) if len(sys.argv) > 4 else 1 # 4 195 | nops = int(sys.argv[5]) if len(sys.argv) > 5 else 2 # 2 196 | 197 | ctl = new(controller.Controller, num= 1) 198 | setup(ctl, (nacceptors + nreplicas + nleaders + nclients,)) 199 | start(ctl) 200 | 201 | acceptors = new(Acceptor, (ctl,), num= nacceptors) 202 | replicas = new(Replica, num= nreplicas) 203 | leaders = new(Leader, (ctl, acceptors, replicas), num= nleaders) 204 | initial_state = []; setup(replicas, (ctl, leaders, initial_state)) 205 | clients = new(Client, (ctl, replicas, nops), num= nclients) 206 | 207 | start(acceptors) 208 | start(replicas | leaders) 209 | start(clients) 210 | 211 | for c in clients: c.join() 212 | print('done') 213 | send(('done',), to= (acceptors|replicas|leaders)) 214 | 215 | # This code includes setup and termination for each client to request and 216 | # complete a number of operations. 217 | 218 | # Not properly terminating when there are live Commanders or Scounts. 219 | # This happens usually when there are multiple leaders or clients; 220 | # adding count in client doesn't help. 221 | -------------------------------------------------------------------------------- /benchmarks/vrpaxos/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | import random 4 | controller = import_da('controller') 5 | 6 | BACKOFF_LIMIT = 8 7 | NOPS = 10 # number of different operations that the state machine can do 8 | # operation i maps a state to a pair of new state and result of i on state 9 | def operation(i): return lambda state: (state+[i], ['result',i,'on',state]) 10 | operations = {i: operation(i) for i in range(NOPS)} 11 | 12 | # wrong: all closures get i = NOPS - 1 13 | # ops = {i: (lambda state: (state+[i], ['res',i]+state)) for i in range(NOPS)} 14 | 15 | class Replica(process, controller.Controllee): 16 | def setup(ctl, leaders:set, initial_state): 17 | super().setup(ctl) 18 | self.state = initial_state; self.slot_num = 1 19 | 20 | def serve(): 21 | backoff = 0 22 | while True: 23 | output('### replica') 24 | if await(some(received(('request', p)), has= 25 | each(sent(('propose', s, _p)), has= 26 | some(received(('decision', _s, p2)), has= p2!=p)))): 27 | output('### replica 1', 'p', p) 28 | # find the maximum used slot 29 | used_s = max((setof(s, sent(('propose', s, _))) | 30 | setof(s, received(('decision', s, _)))) | {0}) 31 | output('used_s', used_s) 32 | # within used+1, find the smallest that is not yet used 33 | s = min(setof(s, s in range(1, used_s + 1 + 1), 34 | not(some(sent(('propose', _s ,_))) or 35 | some(received(('decision', _s, _)))))) 36 | output('s', s, 'p', p) 37 | send(('propose', s, p), to= leaders) 38 | elif some(received(('decision', _slot_num, p))): 39 | output('### replica 2', 'p', p) 40 | if not some(received(('decision', s, _p)), has= s < slot_num): 41 | client, cmd_id, op = p 42 | state, result = operations[op](state) 43 | output('### replica 2.1', 'state', state, 'result', result) 44 | send(('response', cmd_id, result), to= client) 45 | backoff = 0 46 | else: 47 | await(False, timeout=random.randrange((1< len(acceptors)/2): 69 | output('### leader 1') 70 | 71 | for (s,p) in pmax(setof(t, received(('1b', _ballot, accepted)), 72 | t in accepted)): 73 | send(('2a', ballot, s, p), to= acceptors) 74 | 75 | while True: 76 | if await(some(received(('propose', s, p)), has= 77 | not some(sent(('2a', _, _s, _))))): 78 | output('### leader 1.1 b={}, s={}, p={}'.format(ballot, s, p)) 79 | send(('2a', ballot, s, p), to= acceptors) 80 | elif some(received(('2b', _ballot, s, p)), has= 81 | len(setof(a, received(('2b', _ballot, _s, _p), from_= a))) 82 | > len(acceptors)/2 and 83 | not sent(('decision', s, p))): # not in DistAlgo vesion 84 | output('### leader 1.2 s={}, p={}'.format(s, p)) 85 | send(('decision', s, p,), to= replicas) 86 | backoff = 0 87 | elif (some(received(('1b',b,_)), has= b > ballot) or \ 88 | some(received(('2b',b,_,_)), has= b > ballot)): 89 | output('### leader 1.3') 90 | await(False, timeout=random.randrange((1< ballot) or \ 96 | some(received(('2b',b,_,_)), has= b > ballot): 97 | output('### leader 2') 98 | await(False, timeout=random.randrange((1< 1 else 8 175 | nreplicas = int(sys.argv[2]) if len(sys.argv) > 2 else 8 176 | nleaders = int(sys.argv[3]) if len(sys.argv) > 3 else 2 177 | nclients = int(sys.argv[4]) if len(sys.argv) > 4 else 5 178 | nops = int(sys.argv[5]) if len(sys.argv) > 5 else 3 179 | 180 | ctl = new(controller.Controller, num= 1) 181 | nprocs = nacceptors + nreplicas + nleaders + nclients 182 | setup(ctl, (nprocs, nclients)) 183 | start(ctl) 184 | 185 | acceptors = new(Acceptor, (ctl,), num= nacceptors) 186 | replicas = new(Replica, num= nreplicas) 187 | leaders = new(Leader, (ctl, acceptors, replicas), num= nleaders) 188 | initial_state = []; setup(replicas, (ctl, leaders, initial_state)) 189 | clients = new(Client, (ctl, replicas, nops), num= nclients) 190 | 191 | start(acceptors) 192 | start(replicas | leaders) 193 | start(clients) 194 | -------------------------------------------------------------------------------- /bin/dac: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import runpy 5 | import os.path as path 6 | 7 | binpath = sys.path[0] 8 | rootpath = path.dirname(path.abspath(binpath)) 9 | sys.path.insert(0, rootpath) 10 | 11 | del binpath 12 | del rootpath 13 | del path 14 | 15 | sys._real_argv = sys.argv[0] 16 | runpy.run_module("da.compiler", 17 | run_name="__main__", alter_sys=True) 18 | -------------------------------------------------------------------------------- /bin/dac.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | python %~p0\dac %* 3 | -------------------------------------------------------------------------------- /bin/dar: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import runpy 5 | import os.path as path 6 | 7 | binpath = sys.path[0] 8 | rootpath = path.dirname(path.abspath(binpath)) 9 | sys.path.insert(0, rootpath) 10 | 11 | del binpath 12 | del rootpath 13 | del path 14 | 15 | sys._real_argv = sys.argv[0] 16 | runpy.run_module("da", 17 | run_name="__main__", alter_sys=True) 18 | -------------------------------------------------------------------------------- /bin/dar.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | python %~p0\dar %* 3 | -------------------------------------------------------------------------------- /da/__init__.py: -------------------------------------------------------------------------------- 1 | # runtime package 2 | 3 | from . import common 4 | from . import importer 5 | from . import pattern as pat 6 | from .common import global_init 7 | from .sim import DistProcess, NodeProcess 8 | 9 | __version__ = VERSION = common.__version__ 10 | modules = common.modules 11 | 12 | __all__ = ["global_init", "DistProcess", "NodeProcess"] 13 | 14 | for name in common.api_registry.keys(): 15 | globals()[name] = common.api_registry[name] 16 | __all__.append(name) 17 | 18 | # Hook into multiprocessing.spawn: 19 | common._install() 20 | -------------------------------------------------------------------------------- /da/compiler/__init__.py: -------------------------------------------------------------------------------- 1 | # Compiler package for Distalgo 2 | 3 | from .pygen import PythonGenerator 4 | from .parser import Parser 5 | from .parser import daast_from_file 6 | from .parser import daast_from_str 7 | from .ui import dafile_to_pyast 8 | from .ui import dafile_to_pyfile 9 | from .ui import dafile_to_pycode 10 | from .ui import dastr_to_pycode 11 | from .ui import dafile_to_pycfile 12 | from .ui import main 13 | 14 | __all__ = ['PythonGenerator', 'Parser', 15 | 'daast_from_file', 'daast_from_str', 16 | 'dafile_to_pyast', 'dafile_to_pyfile', 17 | 'dafile_to_pycode', 'dastr_to_pycode', 18 | 'dafile_to_pycfile', 19 | 'main'] 20 | -------------------------------------------------------------------------------- /da/compiler/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2017 Bo Lin 2 | # Copyright (c) 2010-2017 Yanhong Annie Liu 3 | # Copyright (c) 2010-2017 Stony Brook University 4 | # Copyright (c) 2010-2017 The Research Foundation of SUNY 5 | # 6 | # Permission is hereby granted, free of charge, to any person 7 | # obtaining a copy of this software and associated documentation files 8 | # (the "Software"), to deal in the Software without restriction, 9 | # including without limitation the rights to use, copy, modify, merge, 10 | # publish, distribute, sublicense, and/or sell copies of the Software, 11 | # and to permit persons to whom the Software is furnished to do so, 12 | # subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be 15 | # included in all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | # Main module entry point 26 | import sys 27 | from da.compiler import ui 28 | 29 | if hasattr(sys, '_real_argv'): 30 | sys.argv[0] = sys._real_argv 31 | 32 | if __name__ == '__main__': 33 | ui.main() 34 | -------------------------------------------------------------------------------- /da/compiler/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2017 Bo Lin 2 | # Copyright (c) 2010-2017 Yanhong Annie Liu 3 | # Copyright (c) 2010-2017 Stony Brook University 4 | # Copyright (c) 2010-2017 The Research Foundation of SUNY 5 | # 6 | # Permission is hereby granted, free of charge, to any person 7 | # obtaining a copy of this software and associated documentation files 8 | # (the "Software"), to deal in the Software without restriction, 9 | # including without limitation the rights to use, copy, modify, merge, 10 | # publish, distribute, sublicense, and/or sell copies of the Software, 11 | # and to permit persons to whom the Software is furnished to do so, 12 | # subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be 15 | # included in all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | import io 26 | import sys 27 | 28 | import da 29 | from da.tools.unparse import Unparser 30 | 31 | DB_ERROR = 0 32 | DB_WARN = 1 33 | DB_INFO = 2 34 | DB_DEBUG =3 35 | Debug = DB_INFO 36 | 37 | VERSION_HEADER = "# -*- generated by {} -*-" 38 | 39 | ########## 40 | # Exceptions: 41 | class CompilerException(Exception): 42 | def __init__(self, reason=None, node=None, name=None, msg=None): 43 | super().__init__(reason, node, name, msg) 44 | self.reason = reason 45 | self.node = node 46 | self.name = name 47 | self.msg = msg 48 | 49 | class MalformedStatementError(CompilerException): pass 50 | class ResolverException(CompilerException): pass 51 | 52 | 53 | def to_source(tree): 54 | textbuf = io.StringIO(newline='') 55 | textbuf.write(VERSION_HEADER.format(da.__version__)) 56 | Unparser(tree, textbuf) 57 | return textbuf.getvalue() 58 | 59 | def to_file(tree, fd): 60 | fd.write(VERSION_HEADER.format(da.__version__)) 61 | return Unparser(tree, fd).counter 62 | 63 | def set_debug_level(level): 64 | global Debug 65 | if is_valid_debug_level(level): 66 | Debug = level 67 | 68 | def is_valid_debug_level(level): 69 | return type(level) is int and DB_ERROR <= level and level <= DB_DEBUG 70 | 71 | # Common utility functions 72 | 73 | def printe(mesg, lineno=0, col_offset=0, filename="", outfd=sys.stderr): 74 | if Debug >= DB_ERROR: 75 | fs = "%s:%d:%d: error: %s" 76 | print(fs % (filename, lineno, col_offset, mesg), file=outfd) 77 | 78 | def printw(mesg, lineno=0, col_offset=0, filename="", outfd=sys.stderr): 79 | if Debug >= DB_WARN: 80 | fs = "%s:%d:%d: warning: %s" 81 | print(fs % (filename, lineno, col_offset, mesg), file=outfd) 82 | 83 | def printd(mesg, lineno=0, col_offset=0, filename="", outfd=sys.stderr): 84 | if Debug >= DB_DEBUG: 85 | fs = "%s:%d:%d: DEBUG: %s" 86 | print(fs % (filename, lineno, col_offset, mesg), file=outfd) 87 | 88 | def printi(mesg, lineno=0, col_offset=0, filename="", outfd=sys.stdout): 89 | if Debug >= DB_INFO: 90 | fs = "%s:%d:%d: %s" 91 | print(fs % (filename, lineno, col_offset, mesg), file=outfd) 92 | 93 | class Namespace: 94 | """A simple container for storing arbitrary attributes.""" 95 | pass 96 | 97 | class OptionsManager: 98 | def __init__(self, cmdline_args, module_args, default=False): 99 | self.cmdline_args = cmdline_args 100 | self.module_args = module_args 101 | self.default = default 102 | 103 | def __getattribute__(self, option): 104 | if option in {'cmdline_args', 'module_args', 'default'}: 105 | return super().__getattribute__(option) 106 | 107 | if hasattr(self.cmdline_args, option): 108 | return getattr(self.cmdline_args, option) 109 | elif hasattr(self.module_args, option): 110 | return getattr(self.module_args, option) 111 | else: 112 | return self.default 113 | 114 | class CompilerMessagePrinter: 115 | 116 | def __init__(self, filename, _parent=None): 117 | self.filename = filename 118 | self._parent = _parent 119 | if not _parent: 120 | self._errcnt = 0 121 | self._warncnt = 0 122 | else: 123 | assert isinstance(_parent, CompilerMessagePrinter) 124 | 125 | def incerr(self): 126 | if self._parent: 127 | self._parent.incerr() 128 | else: 129 | self._errcnt += 1 130 | 131 | def incwarn(self): 132 | if self._parent: 133 | self._parent.incwarn() 134 | else: 135 | self._warncnt += 1 136 | 137 | @property 138 | def errcnt(self): 139 | if self._parent: 140 | return self._parent.errcnt 141 | else: 142 | return self._errcnt 143 | 144 | @property 145 | def warncnt(self): 146 | if self._parent: 147 | return self._parent.warncnt 148 | else: 149 | return self._warncnt 150 | 151 | def accumulate_counters(self, printer): 152 | """Add the counter values from `printer` into ours.""" 153 | assert isinstance(printer, CompilerMessagePrinter) 154 | if self._parent: 155 | self._parent.accumulate_counters(printer) 156 | else: 157 | self._errcnt += printer.errcnt 158 | self._warncnt += printer.warncnt 159 | 160 | def error(self, mesg, node): 161 | self.incerr() 162 | if node is not None: 163 | printe(mesg, node.lineno, node.col_offset, self.filename) 164 | else: 165 | printe(mesg, 0, 0, self.filename) 166 | 167 | def warn(self, mesg, node): 168 | self.incwarn() 169 | if node is not None: 170 | printw(mesg, node.lineno, node.col_offset, self.filename) 171 | else: 172 | printw(mesg, 0, 0, self.filename) 173 | 174 | def debug(self, mesg, node=None): 175 | if node is not None: 176 | printd(mesg, node.lineno, node.col_offset, self.filename) 177 | else: 178 | printd(mesg, 0, 0, self.filename) 179 | -------------------------------------------------------------------------------- /da/examples/2pcommit/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/2pcommit/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | from random import randint 3 | 4 | class Coordinator(process): 5 | def setup(cohorts:set): pass 6 | 7 | def to_commit(): 8 | send(('prepare'), to= cohorts) 9 | await(each(c in cohorts, has= some(received(('vote', _), from_= _c)))) 10 | if some(c in cohorts, has= received(('vote', 'aborting'), from_= c)): 11 | s = setof(c, c in cohorts, received(('vote','ready'), from_= c)) 12 | send(('abort'), to= s) 13 | abort() 14 | if each(c in cohorts, has= received(('vote', 'ready'), from_= c)): 15 | send(('commit'), to= cohorts) 16 | await(each(c in cohorts, has= received(('done'), from_= c))) 17 | commit() 18 | 19 | def abort(): output('abort') 20 | def commit(): output('commit') 21 | 22 | def run(): 23 | to_commit() 24 | 25 | output('terminating') 26 | 27 | class Cohort(process): 28 | def setup(failure_rate): 29 | self.terminate = False 30 | 31 | def receive(msg= ('prepare'), from_= coord): 32 | if prepared(): 33 | send(('vote', 'ready'), to= coord) 34 | ready() 35 | else: 36 | send(('vote', 'aborting'), to= coord) 37 | abort() 38 | 39 | def receive(msg= ('abort')): 40 | abort() 41 | 42 | def receive(msg= ('commit'), from_= coord): 43 | send(('done'), to= coord) 44 | commit() 45 | 46 | def prepared(): return randint(0, 100) > failure_rate 47 | def ready(): output('ready') 48 | def abort(): output('abort'); terminate = True 49 | def commit(): output('commit'); terminate = True 50 | 51 | def run(): 52 | await(terminate) 53 | 54 | def main(): 55 | ncohorts = int(sys.argv[1]) if len(sys.argv) > 1 else 10 56 | fail_rate = int(sys.argv[2]) if len(sys.argv) > 2 else 10 57 | 58 | cohorts = new(Cohort, (fail_rate,), num= ncohorts) 59 | coordinators = new(Coordinator, (cohorts,), num= 1) 60 | start(cohorts | coordinators) 61 | 62 | # This is an executable specification of the algorithm described in 63 | # Kifer, M., et al. (2006). Database Systems: An Application Oriented Approach 64 | # (pages 1010-1012). Addison-Wesley. 65 | 66 | # This code includes setup for trying to commit a transaction with a given 67 | # number of cohorts. 68 | -------------------------------------------------------------------------------- /da/examples/2pcommit/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | from random import randint 3 | 4 | class Coordinator(process): 5 | def setup(tid:str, cohorts:set): pass 6 | 7 | def to_commit(): 8 | send(('prepare', tid), to= cohorts) 9 | await(each(c in cohorts, has= some(received(('vote', _, _tid), from_= _c)))) 10 | if some(c in cohorts, has= received(('vote', 'abort', tid), from_= c)): 11 | s = setof(c, c in cohorts, received(('vote','ready', _tid), from_=c)) 12 | send(('abort', tid), to= s) 13 | abort(tid) 14 | if each(c in cohorts, has= received(('vote', 'ready', tid), from_= c)): 15 | send(('commit', tid), to= cohorts) 16 | await(each(c in cohorts, has= received(('done', tid), from_= c))) 17 | commit(tid) 18 | 19 | def abort(tid): output('abort:' + str(tid)) 20 | def commit(tid): output('commit:' + str(tid)) 21 | 22 | def run(): 23 | to_commit() 24 | 25 | send(('done',), to= cohorts) 26 | output('terminating') 27 | 28 | class Cohort(process): 29 | def setup(failure_rate:float): pass 30 | 31 | def receive(msg=('prepare', tid), from_= coord): 32 | if prepared(tid): 33 | send(('vote', 'ready', tid), to= coord) 34 | ready(tid) 35 | else: 36 | send(('vote', 'abort', tid), to= coord) 37 | abort(tid) 38 | 39 | def receive(msg=('abort', tid)): 40 | abort(tid) 41 | 42 | def receive(msg=('commit', tid), from_= coord): 43 | send(('done', tid), to= coord) 44 | commit(tid) 45 | 46 | def prepared(tid): return randint(0, 100) > failure_rate 47 | def ready(tid): output('ready:' + str(tid)) 48 | def abort(tid): output('abort:' + str(tid)) 49 | def commit(tid): output('commit:' + str(tid)) 50 | 51 | def run(): 52 | await(received(('done',))) 53 | 54 | def main(): 55 | ncohorts = int(sys.argv[1]) if len(sys.argv) > 1 else 10 56 | fail_rate = int(sys.argv[2]) if len(sys.argv) > 2 else 10 57 | 58 | cohorts = new(Cohort, (fail_rate,), num= ncohorts) 59 | coordinators = new(Coordinator, ('001', cohorts), num= 1) 60 | start(cohorts | coordinators) 61 | 62 | # this code includes setup and termination for trying to commit a 63 | # transaction with a given number of cohorts. 64 | -------------------------------------------------------------------------------- /da/examples/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/clpaxos/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/clpaxos/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | class Proposer(process): 4 | def setup(acceptors:set, quorumsize:int, 5 | f:float, nrounds:int, timeout:float): 6 | self.propNum = (0, self) # Current proposal(ballot) number 7 | self.propVal = self # Own value to propose 8 | 9 | def run(): 10 | count = 0 11 | while count < nrounds: 12 | work() 13 | --prepare 14 | send(('Prepare', propNum, self), to=acceptors) 15 | if await(len(setof(a, received(('Promise', _propNum, _, _, a)))) > quorumsize): 16 | --propose 17 | _, voted = max(setof((n, v), 18 | received(('Promise', _propNum, n, v, _)), 19 | len(setof(a, 20 | received(('Promise', 21 | _propNum, _n, _v, a)))) > f) | 22 | {((-1, self), propVal)}) 23 | 24 | send(('OneC', propNum, voted, self), to=acceptors) 25 | if await(len(setof(a, received(('TwoAv', _propNum, _voted, a)))) > quorumsize): 26 | --end 27 | output("Succeeded proposing %s" % (voted,)) 28 | count += 1 29 | continue 30 | elif timeout(timeout): 31 | output("Failed to Propose in time, retrying.") 32 | elif timeout(timeout): 33 | output("Failed to Prepare in time, retrying.") 34 | propNum = (propNum[0] + 1, self) 35 | send(('Done',), to=acceptors) 36 | 37 | class Acceptor(process): 38 | def setup(acceptors:set, proposers:set, quorumsize:int, f:float): 39 | self.peers = acceptors | proposers 40 | 41 | def run(): 42 | while True: 43 | if await(some( 44 | received(('TwoAv', n, v, _)), 45 | has=(len(setof(a, received(('TwoAv', _n, _v, a)))) > 46 | quorumsize and 47 | not sent(('TwoB', n, v))))): 48 | send(('TwoB', n, v), to=peers) 49 | elif each(p in proposers, has=received(('Done',), from_=p)): 50 | break 51 | 52 | def receive(msg=('Prepare', n, p)): 53 | if n > maxpromised(): 54 | vn, vv = max(setof((vpn, vv), sent(('TwoAv', vpn, vv, self))) | 55 | {((-1, self), None)}) 56 | send(('Promise', n, vn, vv, self), to=peers) 57 | 58 | def receive(msg=('OneC', n, v, p)): 59 | if (n >= maxpromised() and islegal(n, v) and 60 | (not some(sent(('TwoAv', _n, _, self))))): 61 | send(('TwoAv', n, v, self), to=peers) 62 | 63 | def maxpromised(): 64 | return max(setof(n, sent(('Promise', n, _, _, _))) | {(-2, self)}) 65 | 66 | def islegal(n, v): 67 | voted = setof((vn, vv), received(('Promise', _n, vn, vv, _)), 68 | len(setof(a, received(('Promise', _n, _vn, _vv, a)))) > f) 69 | if voted and (max(voted)[1] is not None): 70 | return v == max(voted)[1] 71 | else: 72 | return True 73 | 74 | def main(): 75 | nproposers = int(sys.argv[1]) if len(sys.argv) > 1 else 5 76 | nacceptors = int(sys.argv[2]) if len(sys.argv) > 2 else 10 77 | nrounds = int(sys.argv[3]) if len(sys.argv) > 3 else 1 78 | timeout = int(sys.argv[4]) if len(sys.argv) > 4 else 1 79 | f = int((nacceptors-1)/3) 80 | quorum = int(nacceptors/2 + f) 81 | 82 | acceptors = new(Acceptor, num=nacceptors) 83 | proposers = new(Proposer, num=nproposers) 84 | setup(acceptors, (acceptors, proposers, quorum, f)) 85 | setup(proposers, (acceptors, quorum, f, nrounds, timeout)) 86 | start(acceptors) 87 | start(proposers) 88 | -------------------------------------------------------------------------------- /da/examples/crleader/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/crleader/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | class P(process): 4 | def setup(left:P): 5 | self.leaderid = None 6 | 7 | def initiate(): 8 | send(('Election', self), to=left) 9 | 10 | def receive(msg=('Election', p)): 11 | if p > self: 12 | send(('Election', p), to=left) 13 | if p < self: 14 | if not sent(('Election', self)): 15 | send(('Election', self), to=left) 16 | if p == self: 17 | send(('Leader', self), to=left) 18 | 19 | def receive(msg=('Leader', leader)): 20 | leaderid = leader 21 | if leader != self: 22 | send(('Leader', leader), to=left) 23 | 24 | def run(): 25 | initiate() 26 | await (some(received(('Leader', _)))) 27 | output("Leader is", leaderid) 28 | 29 | def main(): 30 | n = int(sys.argv[1]) if len(sys.argv) > 1 else 10 31 | config(channel="fifo") 32 | ps = list(new(P, num= n)) 33 | for i, p in enumerate(ps): 34 | setup({p}, (ps[(i+1 if i < (len(ps)-1) else 0)],)) 35 | start(ps) 36 | -------------------------------------------------------------------------------- /da/examples/dscrash/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/dscrash/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | config(Channel is Reliable, Handling is one) 3 | 4 | class P(process): 5 | def setup(ps:set, v:P, maxfail:int): 6 | self.x = -1 7 | self.V = {v: False} 8 | self.receiveflag = False 9 | 10 | def run(): 11 | --start 12 | for i in range(maxfail): 13 | --one_round 14 | for k in V: 15 | if not V[k]: 16 | send(('Value', k), to=ps) 17 | V[k] = True 18 | await(receiveflag) 19 | receiveflag = False 20 | --end 21 | send(('done', ), to=ps) 22 | x = max(listof(v, v in V, V[v])) 23 | output("x = %r" % (x)) 24 | await(each(p in ps, has=received(('done',), from_=p))) 25 | 26 | def receive(msg=('Value', v)): 27 | receiveflag = True 28 | if v not in V: 29 | V[v] = False 30 | 31 | def main(): 32 | n = int(sys.argv[1]) if len(sys.argv) > 1 else 10 33 | f = int(sys.argv[2]) if len(sys.argv) > 2 else 50 34 | ps = new(P, num= n) 35 | for i, p in enumerate(list(ps)): 36 | setup({p}, (ps, i, f)) 37 | start(ps) 38 | -------------------------------------------------------------------------------- /da/examples/dscrash/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | class P(process): 4 | def setup(ps:set, v:int, maxfail:int): 5 | self.V = {v} # set of values to agree on, intially own value v 6 | 7 | def run(): 8 | for rnd in range(1, maxfail): 9 | send(('Value', 10 | setof(v, v in V, 11 | not some(sent(('Value', V2, _)), has=(v in V2))), 12 | self), to=ps) 13 | reset(received) 14 | await(some(received(('Value', _, _)))) 15 | V |= setof(v, received(('Value', V2, _)), v in V2) 16 | x = min(V) 17 | output(x) 18 | 19 | def main(): 20 | n = int(sys.argv[1]) if len(sys.argv) > 1 else 10 21 | f = int(sys.argv[2]) if len(sys.argv) > 2 else 10 22 | ps = new(P, num= n) 23 | for i, p in enumerate(list(ps)): 24 | setup({p}, (ps, i, f)) 25 | start(ps) 26 | -------------------------------------------------------------------------------- /da/examples/hsleader/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/hsleader/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | import random 3 | 4 | class P(process): 5 | def setup(left:P, right:P): 6 | self.status = "Unknown" # Current status, can be {"Unknown", "Leader"} 7 | self.phase_left, self.phase_right = False, False 8 | self.phase = 0 9 | 10 | def run(): 11 | while True: 12 | send(('Token', self, 'out', 1 << phase), to={left, right}) 13 | 14 | if await(status == "Leader"): 15 | output("I am leader at phase %d!"%phase) 16 | send(('Leader', self), to={left, right}) 17 | break 18 | elif (phase_left and phase_right): 19 | phase += 1 20 | phase_left, phase_right = False, False 21 | elif some(received(('Leader', leader))): 22 | output("Leader is " + str(leader)) 23 | break 24 | 25 | def receive(msg=('Token', v, direction, h), from_=source): 26 | if source == left and direction == 'out': 27 | if v > self and h > 1: 28 | send(('Token', v, 'out', h-1), to=right) 29 | elif v > self and h == 1: 30 | send(('Token', v, 'in', 1), to=left) 31 | elif v == self: 32 | status = "Leader" 33 | 34 | elif source == right and direction == 'out': 35 | if v > self and h > 1: 36 | send(('Token', v, 'out', h-1), to=left) 37 | elif v > self and h == 1: 38 | send(('Token', v, 'in', 1), to=right) 39 | elif v == self: 40 | status = "Leader" 41 | 42 | elif source == left and direction == 'in': 43 | if v > self: 44 | send(('Token', v, 'in', 1), to=right) 45 | elif v == self: 46 | phase_left = True 47 | 48 | elif source == right and direction == 'in': 49 | if v > self: 50 | send(('Token', v, 'in', 1), to=left) 51 | elif v == self: 52 | phase_right = True 53 | 54 | def receive(msg=('Leader', leader), from_=source): 55 | if source == left: 56 | send(('Leader', leader), to=right) 57 | else: 58 | send(('Leader', leader), to=left) 59 | 60 | def main(): 61 | n = int(sys.argv[1]) if len(sys.argv) > 1 else 10 62 | config(channel="fifo") 63 | topology = list(new(P, num= n)) 64 | random.shuffle(topology) 65 | for i, p in enumerate(topology): 66 | if i == len(topology)-1: 67 | setup({p}, (topology[i-1], topology[0])) 68 | else: 69 | setup({p}, (topology[i-1], topology[i+1])) 70 | start(topology) 71 | 72 | -------------------------------------------------------------------------------- /da/examples/hsleader/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | import random 3 | 4 | class P(process): 5 | def setup(left:P, right:P): pass 6 | 7 | def run(): 8 | distance = 1 9 | while True: 10 | send(('Out', self, distance), to={left, right}) 11 | if await(some(received(('Out', self, d)))): 12 | output("I am leader at distance %d!"%d) 13 | send(('Leader', self), to={left, right}) 14 | break 15 | elif some(received(('Leader', leader))): 16 | output("Leader is", leader) 17 | break 18 | elif (received(('In', self), from_=left) and 19 | received(('In', self), from_=right)): 20 | distance *= 2 21 | reset(received) 22 | 23 | def receive(msg=('Out', v, d), from_=source): 24 | if v > self: 25 | if d > 1: 26 | send(('Out', v, d-1), to=(right if source == left else left)) 27 | elif d == 1: 28 | send(('In', v), to=source) 29 | 30 | def receive(msg=('In', v), from_=source): 31 | if v > self: 32 | send(('In', v), to=(right if source == left else left)) 33 | 34 | def receive(msg=("Leader", leader), from_=source): 35 | send(('Leader', leader), to=(right if source == left else left)) 36 | 37 | def main(): 38 | n = int(sys.argv[1]) if len(sys.argv) > 1 else 10 39 | config(channel="fifo") 40 | topology = list(new(P, num= n)) 41 | random.shuffle(topology) 42 | for i, p in enumerate(topology): 43 | if i == len(topology)-1: 44 | setup({p}, (topology[i-1], topology[0])) 45 | else: 46 | setup({p}, (topology[i-1], topology[i+1])) 47 | start(topology) 48 | -------------------------------------------------------------------------------- /da/examples/lamutex/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/lamutex/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | config(channel is fifo, clock is lamport) 3 | 4 | class P(process): 5 | def setup(s:set, nrequests:int): # s is set of all other processes 6 | self.q = set() 7 | 8 | def mutex(task): 9 | -- request 10 | c = logical_clock() 11 | send(('request', c, self), to= s) 12 | q.add(('request', c, self)) 13 | await(each(('request', c2, p) in q, 14 | has= (c2, p)==(c, self) or (c, self) < (c2, p)) and 15 | each(p in s, has= some(received(('ack', c2, _p)), has= c2 > c))) 16 | -- critical_section 17 | task() 18 | -- release 19 | q.remove(('request', c, self)) 20 | send(('release', logical_clock(), self), to= s) 21 | 22 | def receive(msg= ('request', c2, p)): 23 | q.add(('request', c2, p)) 24 | send(('ack', logical_clock(), self), to= p) 25 | 26 | def receive(msg= ('release', _, p)): 27 | # q.remove(('request', _, p)) # pattern matching needed for _ 28 | # q.remove(anyof(setof(('request', c, p), ('request', c, _p) in q))) 29 | for x in setof(('request', c, p), ('request', c, _p) in q): 30 | q.remove(x) 31 | break 32 | # for ('request', c, _p) in q: q.remove('request', c, p); break 33 | # for (tag, c, p2) in q: 34 | # if tag == 'request' and p2 == p: 35 | # q.remove((tag, c, p2)); break 36 | 37 | def run(): 38 | def task(): 39 | output('in cs') 40 | for i in range(nrequests): 41 | mutex(task) 42 | 43 | send(('done', self), to= parent()) 44 | await(received(('done',), from_=parent())) 45 | output('terminating') 46 | 47 | def main(): 48 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 49 | nrequests = int(sys.argv[2]) if len(sys.argv) > 2 else 1 50 | 51 | ps = new(P, num=nprocs) 52 | for p in ps: setup(p, (ps-{p}, nrequests)) 53 | start(ps) 54 | await(each(p in ps, has=received(('done', p)))) 55 | send(('done',), to=ps) 56 | 57 | # This is an executable specification of the algorithm described in 58 | # Lamport, L. (1978). "Time, clocks, and the ordering of events in a 59 | # distributed system". Communications of the ACM, 21(7):558-565. 60 | 61 | # This code includes setup and termination for serving a given number of 62 | # requests per process. 63 | 64 | # All labels are not needed, 65 | # leaving 14 or 15 lines total for the algorithm body and message handlers. 66 | -------------------------------------------------------------------------------- /da/examples/lamutex/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | config(channel= 'fifo', clock= 'Lamport') 3 | 4 | class P(process): 5 | def setup(s:set, nrequests:int): pass # s is set of all other processes 6 | 7 | def mutex(task): 8 | -- request 9 | c = logical_clock() 10 | send(('request', c, self), to= s) 11 | await(each(received(('request', c2, p)), 12 | has= received(('release', c2, p)) or (c, self) < (c2, p)) 13 | and each(p in s, has= received(('ack', c, p)))) 14 | -- critical_section 15 | task() 16 | -- release 17 | send(('release', c, self), to= s) 18 | 19 | def receive(msg= ('request', c, p)): 20 | send(('ack', c, self), to= p) 21 | 22 | def run(): 23 | def task(): 24 | output('in cs') 25 | output('releasing cs') 26 | for i in range(nrequests): 27 | mutex(task) 28 | 29 | send(('done', self), to= s) 30 | await(each(p in s, has= received(('done', p)))) 31 | output('terminating') 32 | 33 | def main(): 34 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 35 | nrequests = int(sys.argv[2]) if len(sys.argv) > 2 else 1 36 | 37 | ps = new(P, num= nprocs) 38 | for p in ps: setup(p, (ps-{p}, nrequests)) 39 | start(ps) 40 | 41 | # This is an executable specification that simplifies the algorithm in 42 | # Lamport, L. (1978). "Time, clocks, and the ordering of events in a 43 | # distributed system". Communications of the ACM, 21(7):558-565. 44 | 45 | # The simplification is to remove an unnecessary q and message handler; 46 | # release and ack messages include the request time, 47 | # not timestamps unused or used for unnecessary expensive comparison. 48 | 49 | # This code includes setup and termination for serving a given number of 50 | # requests per process. 51 | 52 | # All labels are not needed, 53 | # leaving a total of 9 lines for the algorithm body and message handler. 54 | -------------------------------------------------------------------------------- /da/examples/lamutex/spec_lam.da: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | class P(process): 4 | def setup(s:set, nrequests:int): pass # s is set of all other processes 5 | 6 | def mutex(task): 7 | -- request 8 | c = logical_clock() 9 | send(('request', c, self), to= s) 10 | await(each(received(('request', c2, p)), 11 | has= received(('release', c2, p)) or (c, self) < (c2, p)) 12 | and each(p in s, has= some(received(('ack', c2, _p)), has= c2 > c))) 13 | -- critical_section 14 | task() 15 | -- release 16 | send(('release', c, self), to= s) 17 | 18 | def receive(msg= ('request', _, p)): 19 | send(('ack', logical_clock(), self), to= p) 20 | 21 | def run(): 22 | def task(): output('in critical section') 23 | for i in range(nrequests): mutex(task) 24 | 25 | send(('done', self), to= s) 26 | await(each(p in s, has= received(('done', p)))) 27 | output('terminating') 28 | 29 | def main(): 30 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 31 | nrequests = int(sys.argv[2]) if len(sys.argv) > 2 else 1 32 | 33 | config(channel= Fifo, clock= Lamport) 34 | 35 | ps = new(P, num= nprocs) 36 | for p in ps: setup(p, (ps-{p}, nrequests)) 37 | start(ps) 38 | 39 | # This is an executable specification that simplifies the algorithm in 40 | # Lamport, L. (1978). "Time, clocks, and the ordering of events in a 41 | # distributed system". Communications of the ACM, 21(7):558-565. 42 | 43 | # The simplification is to remove an unnecessary queue and message handler; 44 | # release msgs use the request time, not unused timestamps, to replace queue. 45 | 46 | # This code includes setup and termination of a given number of processes 47 | # each serving a given number of requests. 48 | 49 | # All labels are not needed, 50 | # leaving 9 lines total for the algorithm body and message handler. 51 | -------------------------------------------------------------------------------- /da/examples/lapaxos/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/lapaxos/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | from random import randint 3 | TIMEOUT = 1 4 | 5 | class Proposer(process): 6 | def setup(acceptors:set): 7 | self.n = None # proposal number 8 | self.majority = acceptors # majority of acceptors; all in other papers 9 | 10 | def to_consent(): 11 | n = (0, self) if n == None else (n[0]+1, self) # pick a prop num 12 | send(('prepare', n), to= majority) 13 | 14 | if await(len(setof(a, received(('respond', _n, _), from_ =a))) 15 | > len(acceptors)/2): 16 | v = anyof(setof(v, received(('respond', _n, (n2, v))), 17 | n2==max(setof(n2, received(('respond', _n, (n2, _)))))) 18 | or {randint(1,100)}) # any value, pick in 1..100 19 | responded = setof(a, received(('respond', _n, _), from_ =a)) 20 | send(('accept', n, v), to= responded) 21 | debug('### chose', n, v) 22 | 23 | elif timeout(TIMEOUT): 24 | output('failed proposal number', n) 25 | 26 | def run(): 27 | while not received(('done',)): 28 | to_consent() 29 | output('terminating') 30 | 31 | def anyof(s): 32 | return next(iter(s)) if s else None 33 | 34 | class Acceptor(process): 35 | def setup(learners:set): pass 36 | 37 | def receive(msg= ('prepare', n), from_= p): 38 | if each(sent(('respond', n2, _)), has= n > n2): 39 | maxprop = anyof(setof((n, v), sent(('accepted', n, v)), 40 | n==max(setof(n, sent(('accepted', n, _)))))) 41 | send(('respond', n, maxprop), to =p) 42 | 43 | def receive(msg= ('accept', n, v)): 44 | if not some(sent(('respond', n2, _)), has= n2 > n): 45 | send(('accepted', n, v), to= learners) 46 | 47 | def run(): 48 | await(received(('done',))) 49 | output('terminating') 50 | 51 | def anyof(s): 52 | """return any element of set s if s is not empty or 'None' otherwise""" 53 | return next(iter(s)) if s else None 54 | 55 | class Learner(process): 56 | def setup(acceptors:set): pass 57 | 58 | def learn(): 59 | if await(some(received(('accepted', n, v)), 60 | has= len(setof(a, received(('accepted', _n, _v), from_=a))) 61 | > len(acceptors)/2)): 62 | output('learned', n, v) 63 | 64 | elif timeout(TIMEOUT * 10): 65 | output('failed learning anything') 66 | 67 | def run(): 68 | learn() 69 | output('terminating') 70 | send(('learned', ), to=nodeof(self)) 71 | 72 | def main(): 73 | nacceptors = int(sys.argv[1]) if len(sys.argv) > 1 else 3 74 | nproposers = int(sys.argv[2]) if len(sys.argv) > 2 else 5 75 | nlearners = int(sys.argv[3]) if len(sys.argv) > 3 else 3 76 | 77 | acceptors = new(Acceptor, num= nacceptors) 78 | proposers = new(Proposer, (acceptors,), num= nproposers) 79 | learners = new(Learner, (acceptors,), num= nlearners) 80 | for p in acceptors: setup(p, (learners,)) 81 | start(acceptors | proposers | learners) 82 | 83 | await(each(l in learners, has=received(('learned',), from_=l))) 84 | output('done') 85 | send(('done',), to= (acceptors|proposers)) 86 | 87 | # This is an executable specification of the algorithm described in 88 | # Lamport, L. (2001). Paxos Made Simple. ACM SIGACT News 89 | # (Distributed Computing Column), 32(4):51-58, December. 90 | 91 | # This code includes setup and termination for running repeated rounds until 92 | # the learners all terminate after learning the consent value or timeout. 93 | -------------------------------------------------------------------------------- /da/examples/pingpong/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/pingpong/ping.da: -------------------------------------------------------------------------------- 1 | import sys 2 | class Pong(process): 3 | def setup(total_pings:int): pass 4 | 5 | def run(): 6 | await(total_pings == 0) 7 | 8 | def receive(msg=('Ping',), from_=p): 9 | output("Pinged") 10 | send(('Pong',), to=p) 11 | total_pings -= 1 12 | 13 | class Ping(process): 14 | def setup(p:Pong, nrounds:int): pass 15 | 16 | def run(): 17 | for i in range(nrounds): 18 | clk = logical_clock() 19 | send(('Ping',), to=p) 20 | await(some(received(('Pong',), clk=rclk), has=(rclk > clk))) 21 | 22 | def receive(msg=('Pong',)): 23 | output("Ponged.") 24 | 25 | def main(): 26 | nrounds = int(sys.argv[1]) if len(sys.argv) > 1 else 3 27 | npings = int(sys.argv[2]) if len(sys.argv) > 2 else 3 28 | config(clock='Lamport') 29 | pong = new(Pong, [nrounds * npings], num= 1) 30 | ping = new(Ping, num= npings) 31 | setup(ping, (pong, nrounds)) 32 | start(pong) 33 | start(ping) 34 | -------------------------------------------------------------------------------- /da/examples/raft/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/ramutex/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/ramutex/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | class P(process): 4 | def setup(ps:set, nrounds:int): 5 | self.reqc = None 6 | self.waiting = set() 7 | self.replied = set() 8 | 9 | def cs(task): 10 | # to enter cs, enque and send request to all, then await replies from all 11 | --start 12 | reqc = logical_clock() 13 | send(('Request', reqc), to=ps) 14 | 15 | await(len(replied) == len(ps)) 16 | 17 | # critical section 18 | task() 19 | 20 | # to exit cs, deque and send releases to all 21 | --release 22 | reqc = None 23 | send(('Reply', logical_clock()), to=waiting) 24 | --end 25 | waiting = set() 26 | replied = set() 27 | 28 | def run(): 29 | def anounce(): 30 | output("In cs!") 31 | for i in range(nrounds) : 32 | cs(anounce) 33 | send(('Done', self), to=ps) 34 | await(each(p in ps, has= received(('Done', p)))) 35 | output("Terminating..") 36 | 37 | # when receiving requests from others, enque and reply 38 | def receive(msg=('Request', timestamp), from_=source): 39 | if (reqc == None or (timestamp, source) < (reqc, self)): 40 | send(('Reply', logical_clock()), to=source) 41 | else: 42 | waiting.add(source) 43 | 44 | def receive(msg=('Reply', c1), from_=source): 45 | if reqc is not None and c1 > reqc: 46 | replied.add(source) 47 | 48 | def main(): 49 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 50 | nrounds = int(sys.argv[2]) if len(sys.argv) > 2 else 1 51 | config(clock='Lamport') 52 | ps = new(P, num= nprocs) 53 | # setup the processes 54 | for p in ps: setup({p}, (ps-{p}, nrounds)) 55 | # start the processes 56 | start(ps) 57 | -------------------------------------------------------------------------------- /da/examples/ramutex/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | class P(process): 4 | def setup(s:set, n:int): # pass in set of all processes 5 | pass 6 | 7 | def cs(task): 8 | --request 9 | ownc = logical_clock() 10 | send(('request', ownc, self), to=s) # send request to all processes 11 | --cs 12 | await(each(p in s, 13 | has=some(received(('ack', c, _p)), has=(c > ownc)))) 14 | task() # critical section 15 | --release 16 | output("Release!") 17 | send(('ack', logical_clock(), self), 18 | to=setof(p, p in s, 19 | some(received(('request', c, _p)), 20 | has=((c, p) >= (ownc, self))))) 21 | 22 | def receive(msg=('request', c, p)): 23 | ownc = max(setof(c, sent(('request', c, self)), 24 | not each(p in s, has=some(received(('ack', c2, _p)), 25 | has= c2 > c))) | 26 | {-1}) 27 | if ownc == -1 or (c, p) < (ownc, self): 28 | send(('ack', logical_clock(), self), to=p) 29 | 30 | def run(): 31 | def anounce(): 32 | output("In cs!") 33 | for i in range(n): 34 | cs(anounce) 35 | send(('Done',), to=parent()) 36 | await(received(('Done',), from_=parent())) 37 | 38 | def main(): 39 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 40 | nrounds = int(sys.argv[2]) if len(sys.argv) > 2 else 1 41 | config(clock='Lamport') 42 | ps = new(P, num= nprocs) 43 | # setup the processes 44 | for p in ps: setup({p}, (ps-{p}, nrounds)) 45 | # start the processes 46 | start(ps) 47 | await(each(p in ps, has=received(('Done',), from_=p))) 48 | output("All processes done.") 49 | send(('Done',), to=ps) 50 | -------------------------------------------------------------------------------- /da/examples/ratoken/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/ratoken/spec.da: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | class P (process): 4 | def setup(ps:set, nrounds:int, orig_token:bool): 5 | self.clock = 0 6 | self.token = dict((p, 0) for p in ps) 7 | 8 | def cs(task): 9 | --request 10 | if not token_present(): 11 | clock += 1 12 | send(('request', clock, self), to=ps) 13 | await(token_present()) 14 | token[self] = clock 15 | 16 | task() # critical section 17 | 18 | --release 19 | for p in ps: 20 | if request_pending(p) and token_present(): 21 | #output("sending %r-> %r" % (token, p)) 22 | send(('access', token), to=p) 23 | break 24 | 25 | def receive(msg=('access', newtok)): 26 | token = newtok 27 | 28 | def receive(msg=('request', c, p)): 29 | if request_pending(p) and token_present(): 30 | send(('access', token), to=p) 31 | 32 | def request_pending(p): 33 | # p has a request after it last had the token 34 | return some(received(('request', c, _p)), has=(c > token[p])) 35 | 36 | def token_present(): 37 | return (orig_token and not some(sent(('access', _))) or 38 | some(received(('access', token1)), 39 | has= (not some(sent(('access', token2)), 40 | has= (token2[self] > token1[self]))))) 41 | 42 | def run(): 43 | def anounce(): 44 | output("In cs!") 45 | if token_present(): 46 | output("I'm lucky!") 47 | for i in range(nrounds): 48 | cs(anounce) 49 | send(('Done',), to=ps) 50 | await(each(p in ps, has= received(('Done',), from_=p))) 51 | output("Done!") 52 | 53 | def main(): 54 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 55 | nrounds = int(sys.argv[2]) if len(sys.argv) > 2 else 1 56 | # create n process 57 | ps = new(P, num= nprocs) 58 | 59 | p = ps.pop() 60 | setup(ps, (ps|{p}, nrounds, False)) 61 | setup([p], (ps|{p}, nrounds, True)) 62 | start(ps|{p}) 63 | -------------------------------------------------------------------------------- /da/examples/sktoken/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/sktoken/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | class P(process): 4 | def setup(ps:set, orig_token:bool, 5 | nrounds:int): # other procs, and whether self holds token 6 | self.RN = dict((p, 0) for p in ps) # last request number received 7 | # Token structure: 8 | self.Q = [] # queue of pending requests 9 | self.LN = dict((p, 0) for p in ps) # last request number for which token was granted 10 | 11 | def cs(task): 12 | -- request 13 | if not haveToken(): 14 | RN[self] += 1 15 | send(('request', self, RN[self]), to=ps) 16 | await(haveToken()) 17 | 18 | task() #critical section 19 | 20 | LN[self] = RN[self] 21 | Q.extend(listof(p, p in ps, p not in Q, RN[p] == LN[p] + 1)) 22 | if len(Q) > 0: 23 | p = Q.pop() 24 | send(('token', Q, LN), to=p) 25 | 26 | def receive(msg=('token', Q1, LN1)): 27 | Q = Q1 28 | LN = LN1 29 | 30 | def receive(msg=('request', p, n)): 31 | RN[p] = max((RN[p], n)) 32 | if (haveToken() and RN[p] == LN[p] + 1): 33 | send(('token', Q, LN), to=p) 34 | 35 | def haveToken(): 36 | return (orig_token and not some(sent(('token', _, _)))) or\ 37 | some(received(('token', _, LN1)), 38 | has=(not some(sent(('token', _, LN2)), 39 | has=(LN2[self] > LN1[self])))) 40 | 41 | def run(): 42 | def anounce(): 43 | output("In cs!") 44 | if haveToken(): 45 | output("I'm lucky!") 46 | for i in range(nrounds): 47 | cs(anounce) 48 | send(('Done',), to=ps) 49 | await(each(p in ps, has= received(('Done',), from_=p))) 50 | output("Done!") 51 | 52 | def main(): 53 | nprocs = int(sys.argv[1]) if len(sys.argv) > 1 else 10 54 | nrounds = int(sys.argv[2]) if len(sys.argv) > 2 else 1 55 | # create n process 56 | ps = new(P, num= nprocs) 57 | 58 | # setup the processes 59 | lucky = ps.pop() 60 | 61 | setup(ps, (ps|{lucky}, False, nrounds)) 62 | setup({lucky}, (ps|{lucky}, True, nrounds)) 63 | start(ps|{lucky}) 64 | -------------------------------------------------------------------------------- /da/examples/vrpaxos/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/examples/vrpaxos/orig.da: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | import random 4 | 5 | NOPS = 10 # number of different operations that the state machine can do 6 | # operation i maps a state to a pair of new state and result of i on state 7 | def operation(i): return lambda state: (state+[i], ['result',i,'on',state]) 8 | operations = {i: operation(i) for i in range(NOPS)} 9 | 10 | # wrong: all closures get i = NOPS - 1 11 | # ops = {i: (lambda state: (state+[i], ['res',i]+state)) for i in range(NOPS)} 12 | 13 | class Replica(process): 14 | def setup(leaders:set, initial_state:list): 15 | self.state = initial_state; self.slot_num = 1 16 | self.proposals = set(); self.decisions = set() 17 | 18 | def propose(p): 19 | if not some((_, _p) in decisions): 20 | # find the maximum used slot number, or 0 21 | maxs = max(setof(s, (s, _) in proposals | decisions) or {0}) 22 | # within the maximum + 1, find smallest not yet used 23 | s1 = min(setof(s, s in range(1, maxs + 1 + 1), 24 | not some((_s, _) in proposals | decisions))) 25 | proposals.add((s1, p)) 26 | send(('propose', s1, p), to= leaders) 27 | 28 | def perform(p): 29 | debug('### perform', p) 30 | client, cid, op = p 31 | if some((s, _p) in decisions, has= s < slot_num): 32 | slot_num += 1 33 | else: 34 | debug('===', state, op) 35 | next, result = operations[op](state) 36 | debug('===', next, result) 37 | state = next; slot_num += 1 38 | send(('response', cid, result), to= client) 39 | 40 | def run(): 41 | debug('### start') 42 | await(received(('done',))) 43 | output('terminating') 44 | 45 | def receive(msg= ('request', p)): 46 | debug('### request', p) 47 | propose(p) 48 | 49 | def receive(msg= ('decision', s, p)): 50 | debug('### decision', s, p) 51 | decisions.add((s, p)) 52 | while some((_slot_num, p1) in decisions): 53 | if some((_slot_num, p2) in proposals, has= p2 != p1): 54 | propose(p2) 55 | perform(p1) 56 | 57 | class Acceptor(process): 58 | def setup(): 59 | self.ballot_num = None; self.accepted = set() 60 | 61 | def run(): 62 | debug('### start') 63 | await(received(('done',))) 64 | output('terminating') 65 | 66 | def receive(msg= m): 67 | BOTTOM = (-1, -1); 68 | ballot_num = max((setof(b, received(('p1a', _, b))) | 69 | setof(b, received(('p2a', _, b, _, _)))) or {BOTTOM}) 70 | 71 | def receive(msg= ('p1a', leader_scout, b)): 72 | debug('### p1a', leader_scout, b) 73 | send(('p1b', self, ballot_num, accepted), to= leader_scout) 74 | 75 | def receive(msg= ('p2a', leader_commander, b, s, p)): 76 | debug('### p2a', leader_commander, b, s, p) 77 | if b == ballot_num : accepted.add((b, s, p)) 78 | send(('p2b', self, ballot_num), to= leader_commander) 79 | 80 | class Commander(process): 81 | def setup(leader:Leader, acceptors:set, replicas:set, 82 | b, s, p): pass 83 | 84 | def run(): 85 | debug('### start') 86 | send(('p2a', self, b, s, p), to= acceptors) 87 | if await(len(setof(a, received(('p2b', a, _b)))) > len(acceptors)/2): 88 | send(('decision', s, p), to= replicas) 89 | elif some(received(('p2b', _, b1)), has= b1 != b): 90 | send(('preempted', b1), to= leader) 91 | 92 | class Scout(process): 93 | def setup(leader, acceptors:set, b): 94 | self.pvalues = set() 95 | 96 | def run(): 97 | debug('### start') 98 | # add sleep 99 | time.sleep(random.random()) # next random float in [0.0, 1.0) 100 | send(('p1a', self, b), to= acceptors) 101 | if await(len(setof(a, received(('p1b', a, _b, _)))) > len(acceptors)/2): 102 | pvalues = setof(v, received(('p1b', _, _b, r)), v in r) #accepted 103 | send(('adopted', b, pvalues), to= leader) 104 | elif some(received(('p1b', a, b1, _)), has= b1 != b): 105 | send(('preempted', b1), to= leader) 106 | 107 | class Leader(process): 108 | def setup(acceptors:set, replicas:set): 109 | self.ballot_num = (0, self); 110 | self.active = False; self.proposals = set() 111 | 112 | def run(): 113 | debug('### start') 114 | sub = new(Scout, (self, acceptors, ballot_num), 115 | method='thread', daemon=True) 116 | start(sub) 117 | await(received(('done',))) 118 | output('terminating') 119 | 120 | def receive(msg= ('propose', s, p)): 121 | debug('### propose', s, p) 122 | if not some((_s, _) in proposals): 123 | proposals.add((s,p)) 124 | if active: 125 | sub = new(Commander, 126 | (self, acceptors, replicas, ballot_num, s, p), 127 | method='thread', daemon=True) 128 | start(sub) 129 | 130 | def receive(msg= ('adopted', ballot_num, pvals)): 131 | debug('### adopted', ballot_num, pvals) 132 | proposals = circle_plus(proposals, pmax(pvals)) 133 | for (s, p) in proposals: 134 | sub = new(Commander, 135 | (self, acceptors, replicas, ballot_num, s, p), 136 | method='thread', daemon=True) 137 | start(sub) 138 | active = True 139 | 140 | def receive(msg= ('preempted', (r1, leader1))): 141 | if (r1, leader1) > ballot_num: 142 | active = False 143 | ballot_num = (r1 + 1, self) 144 | sub = new(Scout, (self, acceptors, ballot_num), 145 | method='thread', daemon=True) 146 | start(sub) 147 | 148 | def circle_plus(x, y): 149 | return y | setof((s, p), (s, p) in x, not some((_s, _) in y)) 150 | 151 | def pmax(pvals): 152 | return setof((s, p), (b, s, p) in pvals, 153 | each((b1, _s, _) in pvals, has= b1 <= b)) 154 | 155 | class Client(process): 156 | def setup(replicas:set, nops:int): 157 | self.cid = 0 # command id 158 | self.results = dict() # map of command id to result of command 159 | self.count = dict() # map of command id to number of responses 160 | 161 | def run(): 162 | for i in range(nops): 163 | send(('request', (self, cid, random.randint(0, NOPS-1))), 164 | to= replicas) 165 | await(cid in results) 166 | output('received result', cid, results[cid]) 167 | cid += 1 168 | await(each(cid in range(nops), has= count[cid] == len(replicas))) 169 | output('terminating') 170 | send(('done',), to=parent()) 171 | 172 | def receive(msg= ('response', cid, result)): 173 | debug('### response', cid, result) 174 | if cid not in results: 175 | results[cid] = result 176 | elif results[cid] != result: 177 | error('different result', cid, result, 'than', results[cid]) 178 | count[cid] = 1 if cid not in count else count[cid] + 1 179 | 180 | def main(): 181 | nacceptors = int(sys.argv[1]) if len(sys.argv) > 1 else 5 182 | nreplicas = int(sys.argv[2]) if len(sys.argv) > 2 else 4 183 | nleaders = int(sys.argv[3]) if len(sys.argv) > 3 else 2 184 | nclients = int(sys.argv[4]) if len(sys.argv) > 4 else 3 185 | nops = int(sys.argv[5]) if len(sys.argv) > 5 else 3 186 | 187 | acceptors = new(Acceptor, (), num= nacceptors) 188 | replicas = new(Replica, num= nreplicas) 189 | leaders = new(Leader, (acceptors, replicas), num= nleaders) 190 | initial_state = []; setup(replicas, (leaders, initial_state)) 191 | clients = new(Client, (replicas, nops), num= nclients) 192 | 193 | start(acceptors) 194 | start(replicas | leaders) 195 | start(clients) 196 | 197 | await(each(c in clients, has=received(('done',), from_=c))) 198 | output('All clients done.') 199 | send(('done',), to= (acceptors|replicas|leaders)) 200 | 201 | # This code includes setup and termination for each client to request and 202 | # complete a number of operations. 203 | 204 | # Not properly terminating when there are live Commanders or Scounts. 205 | # This happens usually when there are multiple leaders or clients; 206 | # adding count in client doesn't help. 207 | -------------------------------------------------------------------------------- /da/freeze.py: -------------------------------------------------------------------------------- 1 | """Utility for making (best-effort) read-only copies of objects. 2 | 3 | This is adapted from the `copy.deepcopy` standard library API. To freeze builtin 4 | types, we create `frozenset`, `frozendict`, and `frozenlist` in place of `set`, 5 | `dict`, and `list`, respectively. For user-defined types, if it has a 6 | `__deepfreeze__` special method, then that method is called to create a 7 | read-only deep-copy. Otherwise, we simply fall-back to making a deep copy, which 8 | is better than nothing. Refer to the documentation for module `copy` on how deep 9 | copy works. 10 | 11 | """ 12 | 13 | import types 14 | import weakref 15 | from copyreg import dispatch_table 16 | 17 | __all__ = ['frozendict', 'frozenlist', 'deepfreeze'] 18 | 19 | 20 | class frozendict(dict): 21 | """Hashable immutable dict implementation 22 | 23 | Copied from http://code.activestate.com/recipes/414283/ 24 | 25 | """ 26 | 27 | def _blocked_attribute(obj): 28 | raise AttributeError("A frozendict cannot be modified.") 29 | _blocked_attribute = property(_blocked_attribute) 30 | 31 | __delitem__ = __setitem__ = clear = _blocked_attribute 32 | pop = popitem = setdefault = update = _blocked_attribute 33 | 34 | def __new__(cls, *args, **kws): 35 | new = dict.__new__(cls) 36 | dict.__init__(new, *args, **kws) 37 | return new 38 | 39 | def __init__(self, *args, **kws): 40 | pass 41 | 42 | def __hash__(self): 43 | try: 44 | return self._cached_hash 45 | except AttributeError: 46 | h = self._cached_hash = hash(tuple(sorted(self.items()))) 47 | return h 48 | 49 | def __repr__(self): 50 | return "frozendict(%s)" % dict.__repr__(self) 51 | 52 | def _build_set_keyvalue_(self, key, val): 53 | """Backdoor updater for recursively building the frozendict.""" 54 | if not hasattr(self, '_cached_hash'): 55 | return super().__setitem__(key, val) 56 | else: 57 | raise AttributeError("Attempting to update frozendict after " 58 | "hash value has been read.") 59 | 60 | 61 | class frozenlist(list): 62 | """Hashable immutable list implementation 63 | 64 | Copied from http://code.activestate.com/recipes/414283/ 65 | 66 | """ 67 | 68 | def _blocked_attribute(obj): 69 | raise AttributeError("A frozenlist cannot be modified.") 70 | _blocked_attribute = property(_blocked_attribute) 71 | 72 | append = extend = insert = remove = sort = clear = pop = reverse = \ 73 | __iadd__ = __imul__ = __delitem__ = __setitem__ = _blocked_attribute 74 | 75 | def __new__(cls, *args, **kws): 76 | new = list.__new__(cls) 77 | list.__init__(new, *args, **kws) 78 | return new 79 | 80 | def __init__(self, *args, **kws): 81 | pass 82 | 83 | def __hash__(self): 84 | try: 85 | return self._cached_hash 86 | except AttributeError: 87 | h = self._cached_hash = hash(tuple(sorted(self))) 88 | return h 89 | 90 | def __repr__(self): 91 | return "frozenlist(%s)" % list.__repr__(self) 92 | 93 | def _build_add_elem_(self, elem): 94 | """Backdoor updater for recursively building the frozenlist.""" 95 | if not hasattr(self, '_cached_hash'): 96 | return super().append(elem) 97 | else: 98 | raise AttributeError("Attempting to modify frozenlist after " 99 | "hash value has been read.") 100 | 101 | 102 | def deepfreeze(x, memo=None, _nil=[]): 103 | """Deep freeze operation on arbitrary Python objects. 104 | 105 | See the module's __doc__ string for more info. 106 | 107 | """ 108 | 109 | if memo is None: 110 | memo = {} 111 | 112 | d = id(x) 113 | y = memo.get(d, _nil) 114 | if y is not _nil: 115 | return y 116 | 117 | cls = type(x) 118 | 119 | copier = _deepfreeze_dispatch.get(cls) 120 | if copier: 121 | y = copier(x, memo) 122 | else: 123 | try: 124 | issc = issubclass(cls, type) 125 | except TypeError: # cls is not a class (old Boost; see SF #502085) 126 | issc = 0 127 | if issc: 128 | y = _deepfreeze_atomic(x, memo) 129 | else: 130 | copier = getattr(x, "__deepfreeze__", None) 131 | if copier: 132 | y = copier(memo) 133 | else: 134 | reductor = dispatch_table.get(cls) 135 | if reductor: 136 | rv = reductor(x) 137 | else: 138 | reductor = getattr(x, "__reduce_ex__", None) 139 | if reductor: 140 | rv = reductor(4) 141 | else: 142 | reductor = getattr(x, "__reduce__", None) 143 | if reductor: 144 | rv = reductor() 145 | else: 146 | raise Error( 147 | "un(deep)copyable object of type %s" % cls) 148 | if isinstance(rv, str): 149 | y = x 150 | else: 151 | y = _reconstruct(x, memo, *rv) 152 | 153 | # If is its own copy, don't memoize. 154 | if y is not x: 155 | memo[d] = y 156 | _keep_alive(x, memo) # Make sure x lives at least as long as d 157 | return y 158 | 159 | _deepfreeze_dispatch = d = {} 160 | 161 | def _deepfreeze_atomic(x, memo): 162 | return x 163 | d[type(None)] = _deepfreeze_atomic 164 | d[type(Ellipsis)] = _deepfreeze_atomic 165 | d[type(NotImplemented)] = _deepfreeze_atomic 166 | d[int] = _deepfreeze_atomic 167 | d[float] = _deepfreeze_atomic 168 | d[bool] = _deepfreeze_atomic 169 | d[complex] = _deepfreeze_atomic 170 | d[bytes] = _deepfreeze_atomic 171 | d[str] = _deepfreeze_atomic 172 | try: 173 | d[types.CodeType] = _deepfreeze_atomic 174 | except AttributeError: 175 | pass 176 | d[type] = _deepfreeze_atomic 177 | d[types.BuiltinFunctionType] = _deepfreeze_atomic 178 | d[types.FunctionType] = _deepfreeze_atomic 179 | d[weakref.ref] = _deepfreeze_atomic 180 | 181 | def _deepfreeze_set(x, memo, deepfreeze=deepfreeze): 182 | # A set can not contain itself, so we can freeze its elements before putting 183 | # the set in the memo: 184 | y = frozenset(deepfreeze(a, memo) for a in x) 185 | # We still have to put the set on the memo because other mutable structures 186 | # might contain a reference to it: 187 | memo[id(x)] = y 188 | return y 189 | d[set] = _deepfreeze_set 190 | 191 | def _deepfreeze_bytearray(x, memo, deepfreeze=deepfreeze): 192 | y = bytes(x) 193 | memo[id(x)] = y 194 | return y 195 | d[bytearray] = _deepfreeze_bytearray 196 | 197 | def _deepfreeze_list(x, memo, deepfreeze=deepfreeze): 198 | y = frozenlist() 199 | memo[id(x)] = y 200 | append = y._build_add_elem_ 201 | for a in x: 202 | append(deepfreeze(a, memo)) 203 | return y 204 | d[list] = _deepfreeze_list 205 | 206 | def _deepfreeze_tuple(x, memo, deepfreeze=deepfreeze): 207 | y = [deepfreeze(a, memo) for a in x] 208 | # We're not going to put the tuple in the memo, but it's still important we 209 | # check for it, in case the tuple contains recursive mutable structures. 210 | try: 211 | return memo[id(x)] 212 | except KeyError: 213 | pass 214 | for k, j in zip(x, y): 215 | if k is not j: 216 | y = tuple(y) 217 | break 218 | else: 219 | y = x 220 | return y 221 | d[tuple] = _deepfreeze_tuple 222 | 223 | def _deepfreeze_dict(x, memo, deepfreeze=deepfreeze): 224 | y = frozendict() 225 | memo[id(x)] = y 226 | update = y._build_set_keyvalue_ 227 | for key, value in x.items(): 228 | update(deepfreeze(key, memo), deepfreeze(value, memo)) 229 | return y 230 | d[dict] = _deepfreeze_dict 231 | 232 | def _deepfreeze_method(x, memo): # Copy instance methods 233 | return type(x)(x.__func__, deepfreeze(x.__self__, memo)) 234 | d[types.MethodType] = _deepfreeze_method 235 | 236 | del d 237 | 238 | def _keep_alive(x, memo): 239 | """Keeps a reference to the object x in the memo. 240 | 241 | Because we remember objects by their id, we have 242 | to assure that possibly temporary objects are kept 243 | alive by referencing them. 244 | We store a reference at the id of the memo, which should 245 | normally not be used unless someone tries to deepfreeze 246 | the memo itself... 247 | """ 248 | try: 249 | memo[id(memo)].append(x) 250 | except KeyError: 251 | # aha, this is the first one :-) 252 | memo[id(memo)]=[x] 253 | 254 | def _reconstruct(x, memo, func, args, 255 | state=None, listiter=None, dictiter=None, 256 | deepfreeze=deepfreeze): 257 | deep = memo is not None 258 | if deep and args: 259 | args = (deepfreeze(arg, memo) for arg in args) 260 | y = func(*args) 261 | if deep: 262 | memo[id(x)] = y 263 | 264 | if state is not None: 265 | if deep: 266 | state = deepfreeze(state, memo) 267 | if hasattr(y, '__setstate__'): 268 | y.__setstate__(state) 269 | else: 270 | if isinstance(state, tuple) and len(state) == 2: 271 | state, slotstate = state 272 | else: 273 | slotstate = None 274 | if state is not None: 275 | y.__dict__.update(state) 276 | if slotstate is not None: 277 | for key, value in slotstate.items(): 278 | setattr(y, key, value) 279 | 280 | if listiter is not None: 281 | if deep: 282 | for item in listiter: 283 | item = deepfreeze(item, memo) 284 | y.append(item) 285 | else: 286 | for item in listiter: 287 | y.append(item) 288 | if dictiter is not None: 289 | if deep: 290 | for key, value in dictiter: 291 | key = deepfreeze(key, memo) 292 | value = deepfreeze(value, memo) 293 | y[key] = value 294 | else: 295 | for key, value in dictiter: 296 | y[key] = value 297 | return y 298 | 299 | del types, weakref 300 | -------------------------------------------------------------------------------- /da/importer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2017 Bo Lin 2 | # Copyright (c) 2010-2017 Yanhong Annie Liu 3 | # Copyright (c) 2010-2017 Stony Brook University 4 | # Copyright (c) 2010-2017 The Research Foundation of SUNY 5 | # 6 | # Permission is hereby granted, free of charge, to any person 7 | # obtaining a copy of this software and associated documentation files 8 | # (the "Software"), to deal in the Software without restriction, 9 | # including without limitation the rights to use, copy, modify, merge, 10 | # publish, distribute, sublicense, and/or sell copies of the Software, 11 | # and to permit persons to whom the Software is furnished to do so, 12 | # subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be 15 | # included in all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | import sys 26 | 27 | PythonVersion = sys.version_info 28 | if PythonVersion < (3, 7): 29 | from . import py36 as real_lib 30 | elif PythonVersion.major == 3 and PythonVersion.minor in [7, 8, 9]: 31 | from . import py37 as real_lib 32 | else: 33 | raise RuntimeError("Python version {} is not supported." 34 | .format(PythonVersion)) 35 | 36 | da_cache_from_source = real_lib.da_cache_from_source 37 | 38 | __all__ = ["da_cache_from_source"] 39 | 40 | # Inject the DistAlgo module loader: 41 | real_lib._install() 42 | -------------------------------------------------------------------------------- /da/importer/py36.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2017 Bo Lin 2 | # Copyright (c) 2010-2017 Yanhong Annie Liu 3 | # Copyright (c) 2010-2017 Stony Brook University 4 | # Copyright (c) 2010-2017 The Research Foundation of SUNY 5 | # 6 | # Permission is hereby granted, free of charge, to any person 7 | # obtaining a copy of this software and associated documentation files 8 | # (the "Software"), to deal in the Software without restriction, 9 | # including without limitation the rights to use, copy, modify, merge, 10 | # publish, distribute, sublicense, and/or sell copies of the Software, 11 | # and to permit persons to whom the Software is furnished to do so, 12 | # subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be 15 | # included in all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | import sys 26 | import os.path 27 | 28 | from importlib import machinery 29 | from importlib import util 30 | # XXX: these are Python internal stuff that may very likely break in future 31 | # releases, but is current as of 3.5 and 3.6: 32 | from importlib._bootstrap import _call_with_frames_removed 33 | from importlib._bootstrap import _verbose_message 34 | from importlib._bootstrap_external import _get_supported_file_loaders 35 | from importlib._bootstrap_external import _validate_bytecode_header 36 | from importlib._bootstrap_external import _compile_bytecode 37 | from importlib._bootstrap_external import _code_to_bytecode 38 | 39 | # XXX: any use of `common.get_runtime_option` in this module must always provide 40 | # a 'default' argument since GlobalOptions may not have been initialized yet: 41 | from .. import common 42 | 43 | DISTALGO_SUFFIXES = ['.da'] 44 | 45 | def da_cache_from_source(source_path, optimization=None): 46 | """Given the path to a .da file, return the path to its .pyc file. 47 | 48 | This function modifies the filename generated by importlib's 49 | `cache_from_source` to include the DistAlgo version number. 50 | 51 | """ 52 | bytecode_path = util.cache_from_source(source_path, optimization=optimization) 53 | bytecode_dir, bytecode_file = os.path.split(bytecode_path) 54 | components = bytecode_file.split('.') 55 | components.insert(-1, 'da-{}'.format(common.__version__)) 56 | bytecode_file = '.'.join(components) 57 | return os.path.join(bytecode_dir, bytecode_file) 58 | 59 | 60 | class DASourceFileLoader(machinery.SourceFileLoader): 61 | """A source loader that loads '.da' files. 62 | 63 | We rely on the parent class to do most of the heavy lifting, the only places 64 | where we need to hook into is `source_to_code`, which is called from 65 | `get_code` to load the bytecode for the given source file, and 66 | `exec_module`, which allows us to maintain a list of all loaded DistAlgo 67 | modules. 68 | 69 | """ 70 | 71 | def __init__(self, fullname, path): 72 | """This is called from the finder. 73 | 74 | """ 75 | super().__init__(fullname, path) 76 | 77 | def exec_module(self, module): 78 | """Execute the module.""" 79 | super().exec_module(module) 80 | # Once we get here, we can be sure that the module has been successfully 81 | # loaded into the system, so we register it in our system: 82 | common.add_da_module(module) 83 | 84 | def source_to_code(self, data, path, *, _optimize=-1): 85 | """Return the Python code object compiled from DistAlgo source. 86 | 87 | """ 88 | from .. import compiler 89 | 90 | codeobj = _call_with_frames_removed(compiler.dastr_to_pycode, 91 | data, path, _optimize=_optimize, 92 | args=common.get_runtime_option( 93 | 'compiler_args', 94 | default=[])) 95 | if codeobj is None: 96 | raise ImportError("Unable to compile {}.".format(path)) 97 | return codeobj 98 | 99 | # XXX: copied wholesale from `machinery.SourceLoader`! All we really need is 100 | # simply alter the `bytecode_path` returned by `cache_from_source` to 101 | # include the DistAlgo version number: 102 | def get_code(self, fullname): 103 | """Concrete implementation of InspectLoader.get_code. 104 | 105 | Reading of bytecode requires path_stats to be implemented. To write 106 | bytecode, set_data must also be implemented. 107 | 108 | """ 109 | source_path = self.get_filename(fullname) 110 | source_mtime = None 111 | try: 112 | # XXX: begin our change 113 | bytecode_path = da_cache_from_source(source_path) 114 | # XXX: end our change 115 | except NotImplementedError: 116 | bytecode_path = None 117 | else: 118 | # XXX: begin our change 119 | if not common.get_runtime_option('recompile', default=False): 120 | # XXX: end our change 121 | try: 122 | st = self.path_stats(source_path) 123 | except IOError: 124 | pass 125 | else: 126 | source_mtime = int(st['mtime']) 127 | try: 128 | data = self.get_data(bytecode_path) 129 | except OSError: 130 | pass 131 | else: 132 | try: 133 | bytes_data = _validate_bytecode_header(data, 134 | source_stats=st, name=fullname, 135 | path=bytecode_path) 136 | except (ImportError, EOFError): 137 | pass 138 | else: 139 | _verbose_message('{} matches {}', bytecode_path, 140 | source_path) 141 | return _compile_bytecode(bytes_data, name=fullname, 142 | bytecode_path=bytecode_path, 143 | source_path=source_path) 144 | source_bytes = self.get_data(source_path) 145 | code_object = self.source_to_code(source_bytes, source_path) 146 | _verbose_message('code object from {}', source_path) 147 | if (not sys.dont_write_bytecode and bytecode_path is not None and 148 | source_mtime is not None): 149 | data = _code_to_bytecode(code_object, source_mtime, 150 | len(source_bytes)) 151 | try: 152 | self._cache_bytecode(source_path, bytecode_path, data) 153 | _verbose_message('wrote {!r}', bytecode_path) 154 | except NotImplementedError: 155 | pass 156 | return code_object 157 | 158 | def _install(): 159 | """Hooks our loader into the import machinery. 160 | 161 | """ 162 | da_loader = [(DASourceFileLoader, DISTALGO_SUFFIXES)] 163 | python_loaders = _get_supported_file_loaders() 164 | # Append Python's own loaders after ours, so that '.da' files are preferred 165 | # over same-named '.py' files: 166 | loaders = da_loader + python_loaders 167 | # make sure we are in front of the default FileFinder: 168 | sys.path_hooks.insert(1, machinery.FileFinder.path_hook(*loaders)) 169 | -------------------------------------------------------------------------------- /da/importer/py37.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2017 Bo Lin 2 | # Copyright (c) 2010-2017 Yanhong Annie Liu 3 | # Copyright (c) 2010-2017 Stony Brook University 4 | # Copyright (c) 2010-2017 The Research Foundation of SUNY 5 | # 6 | # Permission is hereby granted, free of charge, to any person 7 | # obtaining a copy of this software and associated documentation files 8 | # (the "Software"), to deal in the Software without restriction, 9 | # including without limitation the rights to use, copy, modify, merge, 10 | # publish, distribute, sublicense, and/or sell copies of the Software, 11 | # and to permit persons to whom the Software is furnished to do so, 12 | # subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be 15 | # included in all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | import sys 26 | import os.path 27 | 28 | import _imp 29 | from importlib import machinery 30 | from importlib import util 31 | # XXX: these are Python internal stuff that may very likely break in future 32 | # releases, but is current as of 3.7: 33 | from importlib._bootstrap import _call_with_frames_removed 34 | from importlib._bootstrap import _verbose_message 35 | from importlib._bootstrap_external import _get_supported_file_loaders 36 | from importlib._bootstrap_external import _classify_pyc 37 | from importlib._bootstrap_external import _compile_bytecode 38 | from importlib._bootstrap_external import _validate_hash_pyc 39 | from importlib._bootstrap_external import _validate_timestamp_pyc 40 | from importlib._bootstrap_external import _code_to_hash_pyc 41 | from importlib._bootstrap_external import _code_to_timestamp_pyc 42 | 43 | # XXX: any use of `get_runtime_option` in this module must always provide a 44 | # 'default' argument since GlobalOptions may not have been initialized yet: 45 | from .. import common 46 | 47 | DISTALGO_SUFFIXES = ['.da'] 48 | 49 | def da_cache_from_source(source_path, optimization=None): 50 | """Given the path to a .da file, return the path to its .pyc file. 51 | 52 | This function modifies the filename generated by importlib's 53 | `cache_from_source` to include the DistAlgo version number. 54 | 55 | """ 56 | bytecode_path = util.cache_from_source(source_path, optimization=optimization) 57 | bytecode_dir, bytecode_file = os.path.split(bytecode_path) 58 | components = bytecode_file.split('.') 59 | components.insert(-1, 'da-{}'.format(common.__version__)) 60 | bytecode_file = '.'.join(components) 61 | return os.path.join(bytecode_dir, bytecode_file) 62 | 63 | 64 | class DASourceFileLoader(machinery.SourceFileLoader): 65 | """A source loader that loads '.da' files. 66 | 67 | We rely on the parent class to do most of the heavy lifting, the only places 68 | where we need to hook into is `source_to_code`, which is called from 69 | `get_code` to load the bytecode for the given source file, and 70 | `exec_module`, which allows us to maintain a list of all loaded DistAlgo 71 | modules. 72 | 73 | """ 74 | 75 | def __init__(self, fullname, path): 76 | """This is called from the finder. 77 | 78 | """ 79 | super().__init__(fullname, path) 80 | 81 | def exec_module(self, module): 82 | """Execute the module.""" 83 | super().exec_module(module) 84 | # Once we get here, we can be sure that the module has been successfully 85 | # loaded into the system, so we register it in our system: 86 | common.add_da_module(module) 87 | 88 | def source_to_code(self, data, path, *, _optimize=-1): 89 | """Return the Python code object compiled from DistAlgo source. 90 | 91 | """ 92 | from .. import compiler 93 | 94 | codeobj = _call_with_frames_removed(compiler.dastr_to_pycode, 95 | data, path, _optimize=_optimize, 96 | args=common.get_runtime_option( 97 | 'compiler_args', 98 | default=[])) 99 | if codeobj is None: 100 | raise ImportError("Unable to compile {}.".format(path)) 101 | return codeobj 102 | 103 | # XXX: copied wholesale from `machinery.SourceLoader`! All we really need is 104 | # simply alter the `bytecode_path` returned by `cache_from_source` to 105 | # include the DistAlgo version number: 106 | def get_code(self, fullname): 107 | """Concrete implementation of InspectLoader.get_code. 108 | 109 | Reading of bytecode requires path_stats to be implemented. To write 110 | bytecode, set_data must also be implemented. 111 | 112 | """ 113 | source_path = self.get_filename(fullname) 114 | source_mtime = None 115 | source_bytes = None 116 | source_hash = None 117 | hash_based = False 118 | check_source = True 119 | try: 120 | # XXX: begin our change 121 | bytecode_path = da_cache_from_source(source_path) 122 | # XXX: end our change 123 | except NotImplementedError: 124 | bytecode_path = None 125 | else: 126 | # XXX: begin our change 127 | if not common.get_runtime_option('recompile', default=False): 128 | # XXX: end our change 129 | try: 130 | st = self.path_stats(source_path) 131 | except OSError: 132 | pass 133 | else: 134 | source_mtime = int(st['mtime']) 135 | try: 136 | data = self.get_data(bytecode_path) 137 | except OSError: 138 | pass 139 | else: 140 | exc_details = { 141 | 'name': fullname, 142 | 'path': bytecode_path, 143 | } 144 | try: 145 | flags = _classify_pyc(data, fullname, exc_details) 146 | bytes_data = memoryview(data)[16:] 147 | hash_based = flags & 0b1 != 0 148 | if hash_based: 149 | check_source = flags & 0b10 != 0 150 | if (_imp.check_hash_based_pycs != 'never' and 151 | (check_source or 152 | _imp.check_hash_based_pycs == 'always')): 153 | source_bytes = self.get_data(source_path) 154 | source_hash = _imp.source_hash( 155 | _RAW_MAGIC_NUMBER, 156 | source_bytes, 157 | ) 158 | _validate_hash_pyc(data, source_hash, fullname, 159 | exc_details) 160 | else: 161 | _validate_timestamp_pyc( 162 | data, 163 | source_mtime, 164 | st['size'], 165 | fullname, 166 | exc_details, 167 | ) 168 | except (ImportError, EOFError): 169 | pass 170 | else: 171 | _verbose_message('{} matches {}', bytecode_path, 172 | source_path) 173 | return _compile_bytecode(bytes_data, name=fullname, 174 | bytecode_path=bytecode_path, 175 | source_path=source_path) 176 | if source_bytes is None: 177 | source_bytes = self.get_data(source_path) 178 | code_object = self.source_to_code(source_bytes, source_path) 179 | _verbose_message('code object from {}', source_path) 180 | if (not sys.dont_write_bytecode and bytecode_path is not None and 181 | source_mtime is not None): 182 | if hash_based: 183 | if source_hash is None: 184 | source_hash = _imp.source_hash(source_bytes) 185 | data = _code_to_hash_pyc(code_object, source_hash, check_source) 186 | else: 187 | data = _code_to_timestamp_pyc(code_object, source_mtime, 188 | len(source_bytes)) 189 | try: 190 | self._cache_bytecode(source_path, bytecode_path, data) 191 | _verbose_message('wrote {!r}', bytecode_path) 192 | except NotImplementedError: 193 | pass 194 | return code_object 195 | 196 | def _install(): 197 | """Hooks our loader into the import machinery. 198 | 199 | """ 200 | da_loader = [(DASourceFileLoader, DISTALGO_SUFFIXES)] 201 | python_loaders = _get_supported_file_loaders() 202 | # Append Python's own loaders after ours, so that '.da' files are preferred 203 | # over same-named '.py' files: 204 | loaders = da_loader + python_loaders 205 | # make sure we are in front of the default FileFinder: 206 | sys.path_hooks.insert(1, machinery.FileFinder.path_hook(*loaders)) 207 | -------------------------------------------------------------------------------- /da/lib/__init__.py: -------------------------------------------------------------------------------- 1 | # Just so Python thinks this is a package 2 | -------------------------------------------------------------------------------- /da/lib/base.da: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2017 Bo Lin 2 | # Copyright (c) 2010-2017 Yanhong Annie Liu 3 | # Copyright (c) 2010-2017 Stony Brook University 4 | # Copyright (c) 2010-2017 The Research Foundation of SUNY 5 | # 6 | # Permission is hereby granted, free of charge, to any person 7 | # obtaining a copy of this software and associated documentation files 8 | # (the "Software"), to deal in the Software without restriction, 9 | # including without limitation the rights to use, copy, modify, merge, 10 | # publish, distribute, sublicense, and/or sell copies of the Software, 11 | # and to permit persons to whom the Software is furnished to do so, 12 | # subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be 15 | # included in all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | # This is the base DistAlgo module 26 | 27 | def main(): 28 | return 0 29 | -------------------------------------------------------------------------------- /da/lib/laclock.da: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2017 Bo Lin 2 | # Copyright (c) 2010-2017 Yanhong Annie Liu 3 | # Copyright (c) 2010-2017 Stony Brook University 4 | # Copyright (c) 2010-2017 The Research Foundation of SUNY 5 | # 6 | # Permission is hereby granted, free of charge, to any person 7 | # obtaining a copy of this software and associated documentation files 8 | # (the "Software"), to deal in the Software without restriction, 9 | # including without limitation the rights to use, copy, modify, merge, 10 | # publish, distribute, sublicense, and/or sell copies of the Software, 11 | # and to permit persons to whom the Software is furnished to do so, 12 | # subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be 15 | # included in all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | class ClockTag: pass 26 | 27 | class LamportTimestamp(process): 28 | """An implementation of Lamport's logical timestamp algorithm. 29 | 30 | See "Leslie Lamport. 1978. Time, clocks, and the ordering of events in a 31 | distributed system. Commun. ACM 21, 7 (July 1978), 558-565. 32 | DOI=http://dx.doi.org/10.1145/359545.359563" 33 | 34 | """ 35 | def setup(): 36 | self._logical_clock = 0 37 | 38 | def send(message, to, channel=None, noclock=False, **rest): 39 | if noclock: 40 | # allows sending messages to processes that does not use a logical 41 | # clock: 42 | return super().send(message, to, channel, **rest) 43 | else: 44 | return super().send((ClockTag, self._logical_clock, message), 45 | to, channel, **rest) 46 | 47 | def receive(msg=(_ClockTag, rclock, message), from_=src): 48 | self._logical_clock = max(self._logical_clock, rclock) + 1 49 | super().send(message, to=self, impersonate=src) 50 | 51 | def logical_time(): 52 | """Returns the current value of the logical clock.""" 53 | return self._logical_clock 54 | 55 | def incr_logical_time(): 56 | """Increments the logical clock.""" 57 | self._logical_clock += 1 58 | 59 | # Need this to avoid the compiler warning: 60 | def run(): pass 61 | -------------------------------------------------------------------------------- /da/tools/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /da/transport/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2017 Bo Lin 2 | # Copyright (c) 2010-2017 Yanhong Annie Liu 3 | # Copyright (c) 2010-2017 Stony Brook University 4 | # Copyright (c) 2010-2017 The Research Foundation of SUNY 5 | # 6 | # Permission is hereby granted, free of charge, to any person 7 | # obtaining a copy of this software and associated documentation files 8 | # (the "Software"), to deal in the Software without restriction, 9 | # including without limitation the rights to use, copy, modify, merge, 10 | # publish, distribute, sublicense, and/or sell copies of the Software, 11 | # and to permit persons to whom the Software is furnished to do so, 12 | # subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be 15 | # included in all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | from .manager import * 26 | from .base import * 27 | from .mesgloop import * 28 | from .sock import * 29 | 30 | __all__ = (manager.__all__ + 31 | base.__all__ + 32 | mesgloop.__all__ + 33 | sock.__all__) 34 | -------------------------------------------------------------------------------- /da/transport/base.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2017 Bo Lin 2 | # Copyright (c) 2010-2017 Yanhong Annie Liu 3 | # Copyright (c) 2010-2017 Stony Brook University 4 | # Copyright (c) 2010-2017 The Research Foundation of SUNY 5 | # 6 | # Permission is hereby granted, free of charge, to any person 7 | # obtaining a copy of this software and associated documentation files 8 | # (the "Software"), to deal in the Software without restriction, 9 | # including without limitation the rights to use, copy, modify, merge, 10 | # publish, distribute, sublicense, and/or sell copies of the Software, 11 | # and to permit persons to whom the Software is furnished to do so, 12 | # subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be 15 | # included in all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | import logging 26 | 27 | from ..common import get_runtime_option 28 | 29 | __all__ = ["ChannelCaps", "TransportException", 30 | "VersionMismatchException", "AuthenticationException", 31 | "BindingException", "NoAvailablePortsException", 32 | "NoTargetTransportException", "InvalidTransportStateException", 33 | "PacketSizeExceededException", 34 | "Transport"] 35 | 36 | logger = logging.getLogger(__name__) 37 | 38 | class ChannelCaps: 39 | """An enum of channel capabilities.""" 40 | FIFO = 1 41 | RELIABLE = 2 42 | INTERHOST = 4 43 | BROADCAST = 8 44 | RELIABLEFIFO = FIFO | RELIABLE 45 | 46 | class TransportException(Exception): pass 47 | class VersionMismatchException(TransportException): pass 48 | class AuthenticationException(TransportException): pass 49 | class BindingException(TransportException): pass 50 | class NoAvailablePortsException(BindingException): pass 51 | class NoTargetTransportException(TransportException): pass 52 | class InvalidTransportStateException(TransportException): pass 53 | class PacketSizeExceededException(TransportException): pass 54 | 55 | class Transport: 56 | """Implements an inter-process communication channel for sending of data. 57 | 58 | Transports are responsible for sending and receiving messages across OS 59 | process boundaries, and optionally, across processes running on remote 60 | hosts. This is the abstract base class for all types of transports in 61 | DistAlgo. 62 | 63 | """ 64 | slot_index = 0 65 | capabilities = ~0 66 | def __init__(self, authkey): 67 | super().__init__() 68 | self.queue = None 69 | self.hostname = None 70 | self.authkey = authkey 71 | self.mesgloop = None 72 | self._name = None 73 | 74 | def initialize(self, hostname=None, **_): 75 | if hostname is None: 76 | hostname = get_runtime_option('hostname', default='localhost') 77 | self.hostname = hostname 78 | 79 | def start(self, queue, mesgloop=None): 80 | """Starts the transport. 81 | 82 | """ 83 | raise NotImplementedError() 84 | 85 | def close(self): 86 | """Stops the transport and clean up resources. 87 | 88 | """ 89 | pass 90 | 91 | def send(self, data, dest, **params): 92 | """Send `data` to `dest`. 93 | 94 | `data` should be a `bytes` or `bytearray` object. `dest` should be a 95 | DistAlgo process id. 96 | 97 | """ 98 | raise NotImplementedError() 99 | 100 | def setname(self, name): 101 | self._name = name 102 | 103 | @classmethod 104 | def address_from_id(cls, target): 105 | """Returns the transport address of `target`. 106 | 107 | Given process id `target`, return the address of it's corresponding 108 | transport, or None if `target` does not have a corresponding transport. 109 | 110 | """ 111 | address = target.transports[cls.slot_index] 112 | if address is None: 113 | return None 114 | else: 115 | return (target.hostname, address) 116 | 117 | @property 118 | def address(self): 119 | return None 120 | 121 | def __str__(self): 122 | return self.__class__.__name__ 123 | 124 | -------------------------------------------------------------------------------- /da/transport/manager.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2017 Bo Lin 2 | # Copyright (c) 2010-2017 Yanhong Annie Liu 3 | # Copyright (c) 2010-2017 Stony Brook University 4 | # Copyright (c) 2010-2017 The Research Foundation of SUNY 5 | # 6 | # Permission is hereby granted, free of charge, to any person 7 | # obtaining a copy of this software and associated documentation files 8 | # (the "Software"), to deal in the Software without restriction, 9 | # including without limitation the rights to use, copy, modify, merge, 10 | # publish, distribute, sublicense, and/or sell copies of the Software, 11 | # and to permit persons to whom the Software is furnished to do so, 12 | # subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be 15 | # included in all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | import logging 26 | 27 | from .base import ChannelCaps, TransportException 28 | from .mesgloop import SelectorLoop 29 | from ..common import WaitableQueue 30 | 31 | __all__ = ["TransportTypes", "TransportManager", "transport"] 32 | 33 | logger = logging.getLogger(__name__) 34 | 35 | TransportTypes = [] 36 | 37 | class TransportManager: 38 | """Manages all DistAlgo transports within a process. 39 | 40 | """ 41 | log = None 42 | 43 | def __init__(self, cookie=None): 44 | self.queue = None 45 | self.mesgloop = None 46 | self.transports = [] 47 | self.initialized = False 48 | self.started = False 49 | self.authkey = cookie 50 | if self.__class__.log is None: 51 | self.__class__.log = logger.getChild(self.__class__.__name__) 52 | 53 | def __getstate__(self): 54 | return (self.queue, 55 | self.mesgloop, 56 | self.transports, 57 | self.initialized, 58 | self.started, 59 | self.authkey) 60 | 61 | def __setstate__(self, state): 62 | self.queue, self.mesgloop, self.transports, self.initialized, self.started, self.authkey = state 63 | if self.__class__.log is None: 64 | self.__class__.log = logger.getChild(self.__class__.__name__) 65 | 66 | @property 67 | def transport_addresses(self): 68 | return tuple(t.address if t is not None else None 69 | for t in self.transports) 70 | 71 | @property 72 | def transport_addresses_str(self): 73 | return ", ".join(["{}={}".format(typ.__name__, 74 | tr.address 75 | if tr is not None else "") 76 | for typ, tr in zip(TransportTypes, self.transports)]) 77 | 78 | def initialize(self, pipe=None, **params): 79 | """Initialize all transports. 80 | 81 | """ 82 | self.log.debug("Initializing with key %r...", self.authkey) 83 | total = len(TransportTypes) 84 | cnt = 0 85 | res = [] 86 | for cls in TransportTypes: 87 | try: 88 | if pipe is not None: 89 | assert pipe.recv() is cls 90 | trsp = cls(self.authkey) 91 | trsp.initialize(pipe=pipe, **params) 92 | cnt += 1 93 | res.append(trsp) 94 | except Exception as err: 95 | self.log.debug("Failed to initialize transport %s: %r", 96 | transport, err, exc_info=1) 97 | res.append(None) 98 | self.transports = tuple(res) 99 | if pipe: 100 | pipe.send('done') 101 | if cnt != total: 102 | self.log.warning( 103 | "Initialization failed for {}/{} transports.".format( 104 | (total - cnt), total)) 105 | self.initialized = True 106 | 107 | def start(self): 108 | """Start all transports. 109 | 110 | """ 111 | self.log.debug("Starting...") 112 | self.queue = WaitableQueue() 113 | self.mesgloop = SelectorLoop() 114 | started, total = 0, 0 115 | res = [] 116 | for trsp in self.transports: 117 | if trsp is not None: 118 | total += 1 119 | try: 120 | trsp.start(self.queue, self.mesgloop) 121 | started += 1 122 | res.append(trsp) 123 | except Exception as err: 124 | self.log.error("Failed to start transport %s: %r", 125 | transport, err) 126 | res.append(None) 127 | else: 128 | res.append(None) 129 | if started != total: 130 | self.log.warning( 131 | "Start failed for {}/{} transports.".format( 132 | (total - started), total)) 133 | self.started = True 134 | self.transports = tuple(res) 135 | 136 | def close(self): 137 | """Shut down all transports. 138 | 139 | """ 140 | self.log.debug("Stopping...") 141 | total = len(TransportTypes) 142 | cnt = 0 143 | for trsp in self.transports: 144 | try: 145 | if trsp is not None: 146 | trsp.close() 147 | cnt += 1 148 | except Exception as err: 149 | self.log.warning("Exception when stopping transport %s: %r", 150 | transport, err) 151 | if self.mesgloop: 152 | self.mesgloop.stop() 153 | self.started = False 154 | self.initialized = False 155 | self.log.debug("%d/%d transports stopped.", cnt, total) 156 | 157 | def serialize(self, pipe, pid): 158 | """Sends all transports to child process. 159 | """ 160 | for trsp in self.transports: 161 | pipe.send(trsp.__class__) 162 | trsp.serialize(pipe, pid) 163 | 164 | def get_transport(self, flags): 165 | """Returns the first transport instance satisfying `flags`, or None if 166 | no transport satisfies `flags`. 167 | """ 168 | flags &= ~(ChannelCaps.BROADCAST) 169 | for tr in self.transports: 170 | if tr is not None and (flags & tr.capabilities) == 0: 171 | return tr 172 | return None 173 | 174 | def transport(cls): 175 | """Decorator to register `cls` as a transport. 176 | 177 | """ 178 | cls.slot_index = len(TransportTypes) 179 | TransportTypes.append(cls) 180 | return cls 181 | -------------------------------------------------------------------------------- /da/transport/mesgloop.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2017 Bo Lin 2 | # Copyright (c) 2010-2017 Yanhong Annie Liu 3 | # Copyright (c) 2010-2017 Stony Brook University 4 | # Copyright (c) 2010-2017 The Research Foundation of SUNY 5 | # 6 | # Permission is hereby granted, free of charge, to any person 7 | # obtaining a copy of this software and associated documentation files 8 | # (the "Software"), to deal in the Software without restriction, 9 | # including without limitation the rights to use, copy, modify, merge, 10 | # publish, distribute, sublicense, and/or sell copies of the Software, 11 | # and to permit persons to whom the Software is furnished to do so, 12 | # subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be 15 | # included in all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | 25 | import logging 26 | import socket 27 | import selectors 28 | import threading 29 | 30 | __all__ = ["SelectorLoop"] 31 | 32 | logger = logging.getLogger(__name__) 33 | 34 | RECV_BUF_SIZE = 32 35 | 36 | class TerminateLoop(Exception): pass 37 | class SelectorLoop(object): 38 | """Wrapper around a Selector object providing a background message loop. 39 | 40 | """ 41 | def __init__(self, selectorcls=selectors.DefaultSelector): 42 | super().__init__() 43 | # Multiplexer for all sockets: 44 | self.selector = selectorcls() 45 | # Class logger instance: 46 | self._log = logger.getChild(self.__class__.__name__) 47 | # A dummy socket pair for waking up the message-loop: 48 | self.notifier, self.event = None, None 49 | # Background thread: 50 | self.worker = None 51 | 52 | def _handle_event(self, sock, _): 53 | # Just drain the event socket buffer: 54 | data = sock.recv(RECV_BUF_SIZE) 55 | if not data: 56 | raise TerminateLoop() 57 | 58 | def __len__(self): 59 | """Returns the number of registered callbacks.""" 60 | reg = self.selector.get_map() 61 | if self.event and self.event in reg: 62 | return len(reg) - 1 63 | else: 64 | return len(reg) 65 | 66 | def register(self, conn, callback, data=None): 67 | """Registers a new connection object. 68 | 69 | """ 70 | try: 71 | self.selector.register(conn, selectors.EVENT_READ, (callback, data)) 72 | self.notify() 73 | except ValueError as e: 74 | # The conn object was already closed, so call the callback to 75 | # trigger any cleanup routines from the caller 76 | self._log.debug("Registering invalid connection %s: %r", 77 | conn, e, exc_info=1) 78 | callback(conn, data) 79 | 80 | def deregister(self, conn): 81 | try: 82 | self.selector.unregister(conn) 83 | # No need to wake the main loop here 84 | except (KeyError, ValueError): 85 | pass 86 | 87 | def notify(self): 88 | """Wake the main message loop.""" 89 | if self.notifier: 90 | try: 91 | self.notifier.send(b'x') 92 | except (AttributeError, OSError): 93 | # socket already closed, just ignore 94 | pass 95 | 96 | def is_alive(self): 97 | return self.worker is not None and self.worker.is_alive() 98 | 99 | def start(self): 100 | """Starts the message loop thread.""" 101 | if self.worker is None: 102 | self.worker = threading.Thread(target=self.run, daemon=True) 103 | self.worker.start() 104 | 105 | def stop(self): 106 | """Stops the message loop thread.""" 107 | if self.notifier: 108 | try: 109 | self.notifier.close() 110 | except (AttributeError, OSError): 111 | pass 112 | 113 | def run(self): 114 | try: 115 | self.notifier, self.event = socket.socketpair() 116 | self.selector.register(self.event, selectors.EVENT_READ, 117 | (self._handle_event, None)) 118 | while True: 119 | events = self.selector.select() 120 | for key, _ in events: 121 | callback, aux = key.data 122 | try: 123 | callback(key.fileobj, aux) 124 | except socket.error as e: 125 | if key.fileobj is self.event: 126 | self._log.error("Notifier socket failed: %r", e) 127 | break 128 | else: 129 | self._log.debug( 130 | "socket.error when receiving from %s: %r", 131 | key, e, exc_info=1) 132 | self.deregister(key.fileobj) 133 | except TerminateLoop: 134 | pass 135 | except Exception as e: 136 | self._log.error("Message loop terminated abnormally: %r", e) 137 | self._log.debug("Uncaught exception %r", e, exc_info=1) 138 | finally: 139 | if self.notifier: 140 | try: 141 | self.notifier.close() 142 | except OSError: 143 | pass 144 | if self.event: 145 | self.deregister(self.event) 146 | try: 147 | self.event.close() 148 | except OSError: 149 | pass 150 | self.notifier, self.event = None, None 151 | self.worker = None 152 | -------------------------------------------------------------------------------- /doc/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DistAlgo/distalgo/24f710e5e9ddfb49e119fc0bde7f34d9cdfc5ab4/doc/screenshot.png -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | import sys 4 | import subprocess 5 | import setuptools 6 | import distutils.command.build 7 | import setuptools.command.build_py 8 | import setuptools.command.install_lib 9 | import setuptools.command.sdist 10 | from setuptools import setup 11 | from distutils import log 12 | from distutils.cmd import Command 13 | from distutils.dep_util import newer 14 | 15 | import da 16 | 17 | class CompileDocCommand(Command): 18 | """A custom command to run `pdflatex` on 'doc/language.tex'.""" 19 | 20 | description = "generate 'doc/language.pdf' from 'doc/language.tex'." 21 | user_options = [] 22 | 23 | def initialize_options(self): 24 | pass 25 | 26 | def finalize_options(self): 27 | pass 28 | 29 | def run(self): 30 | rootdir = os.getcwd() 31 | os.chdir('./doc') 32 | command = ['pdflatex', 'language.tex'] 33 | self.announce('Running command: {}'.format(command)) 34 | subprocess.check_call(command) 35 | os.chdir(rootdir) 36 | 37 | class DABuildCommand(distutils.command.build.build): 38 | """build everything needed to install.""" 39 | 40 | sub_commands = [('build_doc', None)] + \ 41 | distutils.command.build.build.sub_commands 42 | 43 | # auxiliary function adapted from `distutils.util.byte_compile`: 44 | def _byte_compile(files, optimize=-1, force=False, prefix=None, 45 | base_dir=None, dry_run=False): 46 | from da.compiler import dafile_to_pycfile 47 | from da.importer import da_cache_from_source 48 | 49 | # XXX: do we need "indirect" mode?? 50 | for file in files: 51 | if file[-3:] != ".da": 52 | continue 53 | if optimize >= 0: 54 | opt = '' if optimize == 0 else optimize 55 | cfile = da_cache_from_source(file, optimization=opt) 56 | else: 57 | cfile = da_cache_from_source(file) 58 | dfile = file 59 | if prefix: 60 | if file[:len(prefix)] != prefix: 61 | raise ValueError("invalid prefix: filename {} doesn't start with {}".format(file, prefix)) 62 | dfile = dfile[len(prefix):] 63 | if base_dir: 64 | dfile = os.path.join(base_dir, dfile) 65 | if force or newer(file, cfile): 66 | log.info("byte-compiling {} to {}".format(file, cfile)) 67 | if not dry_run: 68 | dafile_to_pycfile(file, outname=cfile, optimize=optimize, 69 | dfile=dfile) 70 | else: 71 | log.debug("skipping byte-compilation of {} to {}." 72 | .format(file, cfile)) 73 | 74 | class DABuildPyCommand(setuptools.command.build_py.build_py): 75 | """Auto build all examples before packaging.""" 76 | 77 | def byte_compile(self, files): 78 | super().byte_compile(files) 79 | if sys.dont_write_bytecode: 80 | self.warn('byte-compiling is disabled, skipping.') 81 | return 82 | 83 | prefix = self.build_lib 84 | if prefix[-1] != os.sep: 85 | prefix = prefix + os.sep 86 | if self.compile: 87 | _byte_compile(files, optimize=0, force=self.force, 88 | prefix=prefix, dry_run=self.dry_run) 89 | if self.optimize: 90 | _byte_compile(files, optimize=self.optimize, force=self.force, 91 | prefix=prefix, dry_run=self.dry_run) 92 | 93 | class DAInstallCommand(setuptools.command.install_lib.install_lib): 94 | """Install all modules.""" 95 | 96 | def byte_compile(self, files): 97 | super().byte_compile(files) 98 | if sys.dont_write_bytecode: 99 | self.warn('byte-compiling is disabled, skipping.') 100 | return 101 | 102 | install_root = self.get_finalized_command('install').root 103 | if self.compile: 104 | _byte_compile(files, optimize=0, 105 | force=self.force, prefix=install_root, 106 | dry_run=self.dry_run) 107 | if self.optimize > 0: 108 | _byte_compile(files, optimize=self.optimize, 109 | force=self.force, prefix=install_root, 110 | verbose=self.verbose, dry_run=self.dry_run) 111 | 112 | class DASdistCommand(setuptools.command.sdist.sdist): 113 | """Generate doc/language.pdf before packaging.""" 114 | 115 | sub_commands = [('build_doc', None)] + \ 116 | setuptools.command.sdist.sdist.sub_commands 117 | 118 | setup(name = "pyDistAlgo", 119 | version = da.__version__, 120 | url = "https://github.com/DistAlgo/distalgo", 121 | description = "A high-level language for distributed algorithms.", 122 | author = "bolin", 123 | author_email = "bolin@cs.stonybrook.edu", 124 | classifiers = [ 125 | 'Development Status :: 4 - Beta', 126 | 'Intended Audience :: Developers', 127 | 'Programming Language :: Python :: 3', 128 | 'Programming Language :: Python :: 3.5', 129 | 'Programming Language :: Python :: 3.6', 130 | 'Programming Language :: Python :: 3.7', 131 | 'Topic :: Software Development :: Compilers', 132 | ], 133 | 134 | packages = [ 135 | 'da', 136 | 'da.compiler', 137 | 'da.examples', 138 | 'da.importer', 139 | 'da.lib', 140 | 'da.tools', 141 | 'da.transport', 142 | ], 143 | include_package_data = True, 144 | package_data = { 145 | 'da.examples' : ['**/*.da'] 146 | }, 147 | 148 | cmdclass = { 149 | 'build_doc' : CompileDocCommand, 150 | 'build_py' : DABuildPyCommand, 151 | 'install_lib' : DAInstallCommand, 152 | 'sdist' : DASdistCommand, 153 | } 154 | ) 155 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DistAlgo/distalgo/24f710e5e9ddfb49e119fc0bde7f34d9cdfc5ab4/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_freezer.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from da.freeze import frozendict, frozenlist, deepfreeze 4 | 5 | 6 | class TestFrozenDictList(unittest.TestCase): 7 | def setUp(self): 8 | self.d = {1: "a"} 9 | self.l = [1, 2, 3] 10 | 11 | def test_frozendict(self): 12 | fd = frozendict(self.d) 13 | self.assertTrue(1 in fd) 14 | self.assertEqual(fd[1], "a") 15 | with self.assertRaises(AttributeError): 16 | fd[1] = "b" 17 | with self.assertRaises(AttributeError): 18 | fd[2] = "a" 19 | with self.assertRaises(AttributeError): 20 | fd.pop(1) 21 | with self.assertRaises(AttributeError): 22 | fd.popitem() 23 | with self.assertRaises(AttributeError): 24 | fd.setdefault(2) 25 | with self.assertRaises(AttributeError): 26 | fd.update({1 : "b", 2 : "a"}) 27 | with self.assertRaises(AttributeError): 28 | del fd[1] 29 | self.assertEqual(fd, {1 : "a"}) 30 | self.assertEqual(hash(fd), hash(frozendict(self.d))) 31 | 32 | def test_frozenlist(self): 33 | fl = frozenlist(self.l) 34 | self.assertEqual(len(fl), len(self.l)) 35 | self.assertEqual(fl, [1, 2, 3]) 36 | with self.assertRaises(AttributeError): 37 | fl.append(4) 38 | with self.assertRaises(AttributeError): 39 | fl.extend([4, 5]) 40 | with self.assertRaises(AttributeError): 41 | fl += (1, 2) 42 | with self.assertRaises(AttributeError): 43 | fl *= 4 44 | self.assertEqual(fl + [1, 2], [1, 2, 3, 1, 2]) 45 | self.assertEqual(fl * 2, [1, 2, 3, 1, 2, 3]) 46 | self.assertEqual(len({fl}), 1) 47 | self.assertTrue(fl in {fl}) 48 | 49 | 50 | class UserObj: 51 | def __init__(self): 52 | self.f0 = [1] 53 | self.f1 = {10} 54 | self.f2 = {1 : 2, 2 : self.f1} 55 | 56 | def __eq__(self, other): 57 | return isinstance(other, type(self)) and \ 58 | self.f0 == other.f0 and \ 59 | self.f1 == other.f1 and \ 60 | self.f2 == other.f2 61 | 62 | class TestDeepFreeze(unittest.TestCase): 63 | def setUp(self): 64 | self.s = {1, 2, 0} 65 | self.l = [1, 3, 2] 66 | self.rl = [10, 9, self.s, self.s] 67 | self.rl.append(self.rl) 68 | self.d = {"a" : 1, ("b", 0) : self.l} 69 | self.rd = {1 : self.rl} 70 | self.rd["self"] = self.rd 71 | 72 | def test_freeze_set(self): 73 | fs = deepfreeze(self.s) 74 | self.assertTrue(isinstance(fs, frozenset)) 75 | self.assertEqual(fs, self.s) 76 | 77 | def test_freeze_list(self): 78 | fl = deepfreeze(self.l) 79 | self.assertTrue(isinstance(fl, frozenlist)) 80 | self.assertEqual(self.l, fl) 81 | with self.assertRaises(AttributeError): 82 | fl.append(4) 83 | fl = deepfreeze(self.rl) 84 | self.assertIs(fl[-1], fl) 85 | self.assertIs(fl[3], fl[2]) 86 | with self.assertRaises(AttributeError): 87 | fl.update(self.l) 88 | 89 | def test_freeze_dict(self): 90 | fd = deepfreeze(self.d) 91 | self.assertTrue(isinstance(fd, frozendict)) 92 | fd = deepfreeze(self.rd) 93 | self.assertTrue(isinstance(fd, frozendict)) 94 | 95 | def test_freeze_object(self): 96 | obj = UserObj() 97 | fobj = deepfreeze(obj) 98 | self.assertTrue(isinstance(fobj, UserObj)) 99 | self.assertFalse(obj is fobj) 100 | self.assertEqual(obj, fobj) 101 | with self.assertRaises(AttributeError): 102 | fobj.f0.insert(1, 2) 103 | with self.assertRaises(AttributeError): 104 | fobj.f2[1] = 3 105 | 106 | 107 | if __name__ == '__main__': 108 | unittest.main() 109 | -------------------------------------------------------------------------------- /tests/test_manager.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import unittest 3 | 4 | from da.common import WaitableQueue 5 | from da.transport import TransportManager, TransportTypes,\ 6 | UdpTransport, TcpTransport 7 | 8 | KEY = b'1abc' 9 | DATA = b'1234' 10 | LOCALHOST = socket.gethostbyname('localhost') 11 | DEFAULT_PORT = 16000 12 | class TestTransportManager(unittest.TestCase): 13 | def setUp(self): 14 | self.manager = TransportManager(cookie=KEY) 15 | 16 | def test_init_close(self): 17 | self.manager.initialize() 18 | self.assertTrue(self.manager.initialized) 19 | self.manager.close() 20 | self.assertFalse(self.manager.initialized) 21 | 22 | def test_start(self): 23 | self.manager.initialize() 24 | self.assertTrue(self.manager.initialized) 25 | self.manager.start() 26 | self.assertTrue(self.manager.started) 27 | self.assertEqual( 28 | len([t for t in self.manager.transports if t is not None]), 29 | len(TransportTypes)) 30 | self.manager.close() 31 | self.assertFalse(self.manager.initialized) 32 | 33 | def test_sendrecv(self): 34 | self.manager.initialize(port=DEFAULT_PORT, strict=True) 35 | self.manager.start() 36 | 37 | queue = WaitableQueue() 38 | udpsender = UdpTransport(KEY) 39 | udpsender.initialize() 40 | udpsender.start(queue) 41 | udpsender.send(DATA, (LOCALHOST, DEFAULT_PORT)) 42 | udpsender.close() 43 | tcpsender = TcpTransport(KEY) 44 | tcpsender.initialize() 45 | tcpsender.start(queue) 46 | tcpsender.send(DATA, (LOCALHOST, DEFAULT_PORT)) 47 | tcpsender.close() 48 | 49 | queue = self.manager.queue 50 | transport, packet, _ = queue.pop(block=True) 51 | self.assertEqual(packet[transport.data_offset:], DATA) 52 | transport, packet, _ = queue.pop(block=True) 53 | self.assertEqual(packet[transport.data_offset:], DATA) 54 | 55 | self.manager.close() 56 | 57 | if __name__ == '__main__': 58 | unittest.main() 59 | -------------------------------------------------------------------------------- /tests/test_mesgloop.py: -------------------------------------------------------------------------------- 1 | import time 2 | import socket 3 | import unittest 4 | 5 | from da.transport.mesgloop import * 6 | 7 | DATA = b'1234' 8 | DATA2 = b'9876' 9 | 10 | class SelectorLoopTest(unittest.TestCase): 11 | def setUp(self): 12 | self.sender, self.recver = socket.socketpair() 13 | self.loop = SelectorLoop() 14 | 15 | def test_size(self): 16 | self.assertEqual(len(self.loop), 0) 17 | self.loop.start() 18 | self.loop.register(self.recver, (lambda a, b: None)) 19 | time.sleep(0.01) 20 | self.assertEqual(len(self.loop), 1) 21 | self.loop.deregister(self.recver) 22 | self.assertEqual(len(self.loop), 0) 23 | self.loop.stop() 24 | time.sleep(0.01) 25 | self.assertFalse(self.loop.is_alive()) 26 | 27 | def test_sendrecv1(self): 28 | loop = self.loop 29 | self.assertFalse(loop.is_alive()) 30 | loop.start() 31 | data = None 32 | def callback(conn, aux): 33 | nonlocal data 34 | self.assertEqual(aux, self) 35 | data = conn.recv(100) 36 | time.sleep(0.01) 37 | self.assertTrue(loop.is_alive()) 38 | loop.register(self.recver, callback, data=self) 39 | self.sender.send(DATA) 40 | time.sleep(0.01) 41 | loop.stop() 42 | time.sleep(0.01) 43 | self.assertEqual(data, DATA) 44 | loop.deregister(self.recver) 45 | self.assertFalse(loop.is_alive()) 46 | 47 | def test_sendrecv2(self): 48 | loop = self.loop 49 | sender2, recver2 = socket.socketpair() 50 | data1, data2 = None, None 51 | def callback1(conn, aux): 52 | nonlocal data1 53 | self.assertEqual(aux, self) 54 | data1 = conn.recv(100) 55 | def callback2(conn, aux): 56 | nonlocal data2 57 | self.assertEqual(aux, self) 58 | data2 = conn.recv(100) 59 | loop.register(self.recver, callback1, data=self) 60 | loop.start() 61 | time.sleep(0.01) 62 | loop.register(recver2, callback2, data=self) 63 | self.assertTrue(loop.is_alive()) 64 | self.sender.send(DATA) 65 | sender2.send(DATA2) 66 | time.sleep(0.01) 67 | loop.stop() 68 | time.sleep(0.01) 69 | self.assertEqual(data1, DATA) 70 | self.assertEqual(data2, DATA2) 71 | loop.deregister(self.recver) 72 | self.assertFalse(loop.is_alive()) 73 | sender2.close() 74 | recver2.close() 75 | 76 | def test_recver_close(self): 77 | loop = self.loop 78 | data = None 79 | def callback(conn, _): 80 | nonlocal data 81 | data = 1 82 | conn.close() 83 | data = conn.recv(100) 84 | loop.register(self.recver, callback) 85 | loop.start() 86 | time.sleep(0.01) 87 | with self.assertLogs('da.transport.mesgloop', level='DEBUG') as cm: 88 | self.sender.send(DATA) 89 | time.sleep(0.01) 90 | self.assertEqual(data, 1) 91 | self.assertEqual(len(cm.output), 1) 92 | self.assertRegex(cm.output[0], "socket.error when receiving") 93 | loop.stop() 94 | 95 | def test_sender_close(self): 96 | loop = self.loop 97 | loop.start() 98 | data = None 99 | def callback(conn, _): 100 | nonlocal data 101 | data = conn.recv(100) 102 | loop.register(self.recver, callback) 103 | time.sleep(0.01) 104 | self.sender.close() 105 | time.sleep(0.01) 106 | self.assertEqual(data, b'') 107 | 108 | def test_register_closed_connection(self): 109 | self.recver.close() 110 | cleaned = False 111 | def cleanup(k, _): 112 | nonlocal cleaned 113 | self.assertIs(k, self.recver) 114 | cleaned = True 115 | self.loop.register(self.recver, cleanup) 116 | self.assertEqual(cleaned, True) 117 | 118 | def tearDown(self): 119 | self.sender.close() 120 | self.recver.close() 121 | self.loop.stop() 122 | 123 | if __name__ == '__main__': 124 | unittest.main() 125 | -------------------------------------------------------------------------------- /tests/test_sock.py: -------------------------------------------------------------------------------- 1 | import time 2 | import socket 3 | import unittest 4 | 5 | from da.common import WaitableQueue, QueueEmpty 6 | from da.transport import UdpTransport, TcpTransport, SelectorLoop, \ 7 | AuthenticationException 8 | 9 | LOCALHOST = socket.gethostbyname('localhost') 10 | 11 | KEY = b'1' 12 | DATA = b'1234' 13 | DEFAULT_PORT = 16000 14 | class TestUdpTransport(unittest.TestCase): 15 | def setUp(self): 16 | self.transport = UdpTransport(KEY) 17 | self.queue = WaitableQueue() 18 | 19 | def test_init(self): 20 | self.transport.initialize() 21 | self.assertIsNotNone(self.transport.conn) 22 | self.transport.close() 23 | self.assertIsNone(self.transport.conn) 24 | 25 | def test_init_strict(self): 26 | self.transport.initialize(port=DEFAULT_PORT, strict=True) 27 | self.assertEqual(self.transport.conn.getsockname()[1], DEFAULT_PORT) 28 | self.transport.close() 29 | 30 | def test_sendrecv(self): 31 | self.transport.initialize(port=DEFAULT_PORT, strict=True) 32 | self.transport.start(self.queue) 33 | self.assertTrue(self.transport.started) 34 | sender = UdpTransport(KEY) 35 | sender.initialize() 36 | sender.send(DATA, (LOCALHOST, DEFAULT_PORT)) 37 | transport, packet, _ = self.queue.pop(block=True) 38 | self.assertIs(transport, self.transport) 39 | self.assertEqual(packet[transport.data_offset:], DATA) 40 | self.transport.close() 41 | sender.close() 42 | 43 | def test_auth(self): 44 | self.transport.initialize(port=DEFAULT_PORT, strict=True) 45 | self.transport.start(self.queue) 46 | sender = UdpTransport(authkey=b'2') 47 | sender.initialize() 48 | with self.assertLogs('da.transport.sock.UdpTransport', level='WARN'): 49 | sender.send(DATA, (LOCALHOST, DEFAULT_PORT)) 50 | with self.assertRaises(QueueEmpty): 51 | self.queue.pop(timeout=0.01) 52 | sender.close() 53 | self.transport.close() 54 | 55 | class TestTcpTransport(unittest.TestCase): 56 | def setUp(self): 57 | self.transport = TcpTransport(KEY) 58 | self.queue = WaitableQueue() 59 | self.mesgloop = SelectorLoop() 60 | 61 | def test_init(self): 62 | self.transport.initialize() 63 | self.assertIsNotNone(self.transport.conn) 64 | self.transport.close() 65 | self.assertIsNone(self.transport.conn) 66 | 67 | def test_init_strict(self): 68 | self.transport.initialize(port=DEFAULT_PORT, strict=True) 69 | self.assertEqual(self.transport.conn.getsockname()[1], DEFAULT_PORT) 70 | self.transport.close() 71 | 72 | def test_start(self): 73 | self.transport.initialize() 74 | self.transport.start(self.queue, self.mesgloop) 75 | self.assertIs(self.transport.mesgloop, self.mesgloop) 76 | self.assertEqual(len(self.mesgloop), 1) 77 | self.transport.close() 78 | self.assertEqual(len(self.mesgloop), 0) 79 | 80 | def test_sendrecv(self): 81 | self.transport.initialize(port=DEFAULT_PORT, strict=True) 82 | self.transport.start(self.queue, self.mesgloop) 83 | self.assertTrue(self.transport.started) 84 | self.assertEqual(len(self.mesgloop), 1) 85 | sender = TcpTransport(KEY) 86 | sender.initialize() 87 | sender.start(self.queue, self.mesgloop) 88 | self.assertEqual(len(self.mesgloop), 2) 89 | sender.send(DATA, (LOCALHOST, DEFAULT_PORT)) 90 | transport, packet, _ = self.queue.pop(block=True) 91 | self.assertIs(transport, self.transport) 92 | self.assertEqual(packet[transport.data_offset:], DATA) 93 | self.assertEqual(len(self.transport.cache), 1) 94 | 95 | # The other way: 96 | self.transport.send(DATA, sender.conn.getsockname()) 97 | transport, packet, _ = self.queue.pop(block=True) 98 | self.assertIs(transport, sender) 99 | self.assertEqual(packet[transport.data_offset:], DATA) 100 | self.assertEqual(len(self.transport.cache), 1) 101 | self.transport.close() 102 | sender.close() 103 | 104 | def test_auth(self): 105 | self.transport.initialize(port=DEFAULT_PORT, strict=True) 106 | self.transport.start(self.queue, self.mesgloop) 107 | sender = TcpTransport(authkey=b'wrong key') 108 | sender.initialize() 109 | sender.start(self.queue, self.mesgloop) 110 | with self.assertLogs('da.transport.sock.TcpTransport', level='WARN'): 111 | with self.assertRaises(AuthenticationException): 112 | sender.send(DATA, (LOCALHOST, DEFAULT_PORT)) 113 | with self.assertRaises(QueueEmpty): 114 | self.queue.pop(timeout=0.01) 115 | sender.close() 116 | self.transport.close() 117 | 118 | if __name__ == '__main__': 119 | unittest.main() 120 | --------------------------------------------------------------------------------