├── .gitignore ├── tompkins ├── examples │ ├── __init__.py │ ├── simple_split_problem.py │ └── simple_scheduling_problem.py ├── __init__.py ├── tests │ ├── test_util.py │ ├── test_ilp.py │ └── test_dag.py ├── util.py ├── dag.py └── ilp.py └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /tompkins/examples/__init__.py: -------------------------------------------------------------------------------- 1 | import simple_scheduling_problem 2 | import simple_split_problem 3 | -------------------------------------------------------------------------------- /tompkins/__init__.py: -------------------------------------------------------------------------------- 1 | """ Static Scheduling for Heterogeneous Computing 2 | 3 | See also: 4 | schedule(dag, agents, compcost, commcost, R, B, M) 5 | 6 | """ 7 | 8 | import ilp 9 | import dag 10 | import examples 11 | from dag import schedule, orderings 12 | -------------------------------------------------------------------------------- /tompkins/tests/test_util.py: -------------------------------------------------------------------------------- 1 | from tompkins.util import * 2 | 3 | def test_iterable(): 4 | assert iterable((1,2)) 5 | assert iterable([3]) 6 | assert not iterable(3) 7 | 8 | 9 | def test_dictify(): 10 | f = lambda a,b: a+b 11 | g = dictify(f) 12 | assert g[1,2] == 3 13 | 14 | def test_reverse_dict(): 15 | d = {'a': (1, 2), 'b': (2, 3), 'c':()} 16 | assert reverse_dict(d) == {1: ('a',), 2: ('a', 'b'), 3: ('b',)} 17 | 18 | def test_merge(): 19 | d = {1:2, 3:4} 20 | e = {4:5} 21 | assert merge(d, e) == {1:2, 3:4, 4:5} 22 | 23 | def test_merge_many(): 24 | d = {1:2, 3:4} 25 | e = {4:5} 26 | f = {6:7} 27 | assert merge(d, e, f) == {1:2, 3:4, 4:5, 6:7} 28 | 29 | def test_intersection(): 30 | assert set(intersection((1,2,3), (2,3))) == set((2,3)) 31 | assert set(intersection((1,2,3), {2:'a' ,3:'b'})) == set((2,3)) 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Static Scheduling with Integer Programming 2 | ========================================== 3 | 4 | This module describes a highly flexible scheduling problem by encoding it as a 5 | Mixed Integer Linear Program (MILP) using the 6 | [pulp framework](http://code.google.com/p/pulp-or/) 7 | This framework can then call any of several external codes[2] to solve 8 | 9 | This formulation of the MILP was specified in chapters 4.1, 4.2 in the 10 | following masters thesis. Section and equation numbers are cited. 11 | 12 | ["Optimization Techniques for Task Allocation and Scheduling in Distributed Multi-Agent Operations"](http://dspace.mit.edu/bitstream/handle/1721.1/16974/53816027.pdf?sequence=1) 13 | 14 | by 15 | 16 | Mark F. Tompkins, June 2003, Masters thesis for MIT Dept EECS 17 | 18 | [1] sudo easy_install pulp 19 | 20 | [2] On ubuntu we used "sudo apt-get install glpk" 21 | -------------------------------------------------------------------------------- /tompkins/tests/test_ilp.py: -------------------------------------------------------------------------------- 1 | from tompkins.ilp import schedule, PtoQ, jobs_when_where 2 | from collections import defaultdict 3 | from tompkins.examples.simple_scheduling_problem import Jobs, Agents, D, C, R, B, P, M 4 | 5 | def test_schedule(): 6 | prob, X, S, Cmax = schedule(Jobs, Agents, D, C, R, B, P, M) 7 | prob.solve() 8 | assert Cmax.value() == 14 9 | assert S['start'].value() == 0 10 | assert S['end'].value() == Cmax.value() 11 | 12 | def test_PtoQ(): 13 | P = defaultdict(lambda : 0) 14 | P['a', 'b'] = 1 15 | P['b', 'c'] = 1 16 | Q = PtoQ(P) 17 | assert Q['a','b'] and Q['b', 'c'] and Q['a', 'c'] 18 | 19 | def test_jobs_when_where(): 20 | prob, X, S, Cmax = schedule(Jobs, Agents, D, C, R, B, P, M) 21 | result = jobs_when_where(prob, X, S, Cmax) 22 | assert isinstance(result, list) 23 | assert len(result) == len(S) 24 | job, time, machine = result[0] 25 | assert job == 'start' 26 | assert time == 0 27 | assert machine in Agents 28 | 29 | -------------------------------------------------------------------------------- /tompkins/examples/simple_split_problem.py: -------------------------------------------------------------------------------- 1 | # a -> 1 -> b -> 2 -> d 2 | # -> c -> 3 -> e 3 | 4 | from tompkins.dag import send, recv 5 | 6 | a,b,c,d,e = 'abcde' 7 | A,B,C = 'ABC' 8 | usedby = {a: (1,), b: (2,), c: (3,), d:(), e:()} 9 | outputsof = {1: (b, c), 2: (d,), 3: (e, )} 10 | 11 | # unidag = bidag_to_unidag(usedby, outputsof) 12 | unidag = {1: (2, 3), 2: (), 3: ()} 13 | agents = (A, B, C) 14 | 15 | def computation_cost(job, agent): 16 | if (job, agent) in [(1,'A'), (2, 'B'), (3, 'C')]: 17 | return 0 18 | else: 19 | return 3 20 | 21 | def communication_cost(job, agent1, agent2): 22 | if agent1 == agent2: 23 | return 0 24 | return 1 25 | def bicommunication_cost(var, agent1, agent2): 26 | if agent1 == agent2: 27 | return 0 28 | return 1 29 | 30 | def R(job): 31 | return 0 32 | 33 | def B(job, agent): 34 | return 1 35 | 36 | M = 10 37 | 38 | solution = { 39 | 'dags': 40 | {'A': {1: (send('A', 'B', 1, 2), send('A', 'C', 1, 3)), 41 | send('A', 'B', 1, 2): (), 42 | send('A', 'C', 1, 3): ()}, 43 | 'B': {2: (), ('recv', 'A', 'B', 1, 2): (2,)}, 44 | 'C': {3: (), ('recv', 'A', 'C', 1, 3): (3,)}}, 45 | 'makespan': 1, 46 | 'sched': [(1, 0.0, 'A'), (2, 1.0, 'B'), (3, 1.0, 'C')], 47 | 'orders': {'A': [1], 'B': [2], 'C': [3]}, 48 | # 'bidags': 49 | # {'A': ({a: (1,), b: (send('A', 'B'), ), c: (send('A', 'C'),)}, 50 | # {1: (b, c), send('A', 'B'): (), send('A', 'C'): ()}), 51 | # 'B': ({b: (2,), d: ()}, 52 | # {recv('A', 'B'): (b,), 2: (d,)}), 53 | # 'C': ({c: (3,), e: ()}, 54 | # {recv('A', 'C'): (c,), 3: (e,)})} 55 | } 56 | -------------------------------------------------------------------------------- /tompkins/util.py: -------------------------------------------------------------------------------- 1 | 2 | def iterable(x): 3 | try: 4 | iter(x) 5 | return True 6 | except TypeError: 7 | return False 8 | 9 | class fakedict(object): 10 | def __init__(self, fn): 11 | self.fn = fn 12 | try: 13 | self.multi_input = fn.func_code.co_argcount > 1 14 | except AttributeError: 15 | self.multi_input = True 16 | def __getitem__(self, item): 17 | if self.multi_input: 18 | return self.fn(*item) 19 | else: 20 | return self.fn(item) 21 | 22 | def dictify(f): 23 | """ 24 | Turns a function into something that looks like a dict 25 | 26 | >>> f = lambda a,b : a+b 27 | >>> g = dictify(f) 28 | >>> g[1,2] 29 | 3 30 | """ 31 | return fakedict(f) 32 | 33 | def reverse_dict(d): 34 | """ Reverses direction of dependence dict 35 | 36 | >>> d = {'a': (1, 2), 'b': (2, 3), 'c':()} 37 | >>> reverse_dict(d) 38 | {1: ('a',), 2: ('a', 'b'), 3: ('b',)} 39 | """ 40 | result = {} 41 | for key in d: 42 | for val in d[key]: 43 | result[val] = result.get(val, tuple()) + (key, ) 44 | return result 45 | 46 | def merge(*args): 47 | return dict(sum([arg.items() for arg in args], [])) 48 | 49 | def intersection(t1, t2): 50 | return tuple(set(t1).intersection(set(t2))) 51 | 52 | def unique(coll): 53 | return len(coll) == len(set(coll)) 54 | 55 | def groupby(f, coll): 56 | """ Group a collection by a key function 57 | 58 | >>> from tompkins.util import groupby 59 | >>> names = ['Alice', 'Bob', 'Charlie', 'Dan', 'Edith', 'Frank'] 60 | >>> groupby(len, names) 61 | {3: ['Bob', 'Dan'], 5: ['Alice', 'Edith', 'Frank'], 7: ['Charlie']} 62 | """ 63 | d = dict() 64 | for item in coll: 65 | key = f(item) 66 | if key not in d: 67 | d[key] = [] 68 | d[key].append(item) 69 | return d 70 | -------------------------------------------------------------------------------- /tompkins/examples/simple_scheduling_problem.py: -------------------------------------------------------------------------------- 1 | from tompkins.ilp import schedule, jobs_when_where 2 | from collections import defaultdict 3 | from pulp import value 4 | """ 5 | A sample problem 6 | >>> x = matrix('x') 7 | >>> y = x*x 8 | >>> z = y.sum() 9 | >>> f = function([x], z) 10 | 11 | Architecture 12 | CPU --- GPU 13 | 14 | Data starts and must end on the CPU 15 | """ 16 | 17 | # 4.2.1 problem Variables 18 | # Input Parameters - Sets 19 | Jobs = ['start', 'gemm', 'sum', 'end'] 20 | Agents = ['cpu', 'gpu'] # workers 21 | n = len(Jobs) 22 | m = len(Agents) 23 | 24 | # Indicator Variables 25 | # Immediate job precedence graph - specifies DAG 26 | # P[job1, job2] == 1 if job1 directly precedes job2 27 | P = defaultdict(lambda:0) 28 | P['start','gemm'] = 1 29 | P['gemm','sum'] = 1 30 | P['sum','end'] = 1 31 | 32 | # Agent ability matrix 33 | # B[job, agent] == 1 if job can be performed by agent 34 | B = defaultdict(lambda:1) 35 | for agent in Agents: 36 | if agent != 'cpu': 37 | B['start', agent] = 0 # must start on cpu 38 | B['end', agent] = 0 # must start on cpu 39 | 40 | # DATA Values 41 | # Execution Delay Matrix - Time cost of performing Jobs[i] on Agent[j] 42 | D = defaultdict(lambda:0) 43 | D['gemm','cpu'] = 10 # gemm on cpu 44 | D['gemm','gpu'] = 3 # gemm on gpu 45 | D['sum','cpu'] = 5 # sum on cpu 46 | D['sum','gpu'] = 9 # sum on gpu 47 | 48 | # Communication Delay matrix - Cost of sending results of job from 49 | # agent to agent 50 | C = defaultdict(lambda:0) 51 | for a,b in [('cpu', 'gpu'), ('gpu', 'cpu')]: 52 | # cost to communicate matrices is 3 (symmetric) 53 | C['start', a,b] = 3 54 | C['gemm', a,b] = 3 55 | # cost to communicate scalar is .01 (symmetric) 56 | C['sum', a,b] = .01 57 | 58 | # Job Release Times - Additional constraints on availablility of Jobs 59 | # R = np.zeros(n) 60 | R = defaultdict(lambda:0) 61 | 62 | # Maximum makespan 63 | M = 100 64 | 65 | if __name__ == '__main__': 66 | # Set up the Mixed Integer Linear Program 67 | prob, X, S, Cmax = schedule(Jobs, Agents, D, C, R, B, P, M) 68 | 69 | prob.solve() 70 | 71 | print "Makespan: ", value(Cmax) 72 | sched = jobs_when_where(prob, X, S, Cmax) 73 | 74 | print "Schedule: ", sched 75 | -------------------------------------------------------------------------------- /tompkins/tests/test_dag.py: -------------------------------------------------------------------------------- 1 | from tompkins.dag import (manydags, unidag_to_P, send, recv, schedule, issend, 2 | isrecv, replace_send_recv, orderings) 3 | from tompkins.examples.simple_split_problem import (unidag, agents, 4 | computation_cost, communication_cost, R, B, M, solution) 5 | 6 | def test_jobs_where(): 7 | pass 8 | 9 | def test_unidag_to_P(): 10 | d = unidag_to_P({1: (2, 3)}) 11 | assert d[1,2] == 1 and d[1,3] == 1 and d[2,3] == 0 12 | assert unidag_to_P({1: (2, 3), 3: (4,)}) == {(1,2): 1, (1,3): 1, (3,4): 1} 13 | 14 | def test_manydags_simple(): 15 | # 1 -> 2 -> 3 16 | dag = {1: (2,), 2: (3, ), 3:()} 17 | jobson = {'A': (1, 2), 'B': (3, )} 18 | 19 | dags = manydags(dag, jobson) 20 | assert dags['A'] == {1: (2, ), 21 | 2: (send('A', 'B', 2, 3), ), 22 | send('A', 'B', 2, 3): ()} 23 | assert dags['B'] == {recv('A', 'B', 2, 3): (3, ), 3:()} 24 | 25 | def test_manydags_send_recv(): 26 | # 1 -> 2 -> 3 27 | dag = {1: (2,), 2: (3, ), 3:()} 28 | jobson = {'A': (1, 2), 'B': (3, )} 29 | 30 | class Send(object): 31 | def __init__(self, *args): 32 | self.args = args 33 | def __eq__(self, other): 34 | return type(self) == type(other) and self.args == other.args 35 | def __hash__(self): 36 | return hash((type(self), self.args)) 37 | class Recv(Send): pass 38 | 39 | dags = manydags(dag, jobson, send=Send, recv=Recv) 40 | assert dags['A'] == {1: (2, ), 41 | 2: (Send('A', 'B', 2, 3), ), 42 | Send('A', 'B', 2, 3): ()} 43 | assert dags['B'] == {Recv('A', 'B', 2, 3): (3, ), 3:()} 44 | 45 | def test_manydags_less_simple(): 46 | # 1 -> 2 47 | # -> 3 48 | 49 | dag = {1: (2, 3), 2: (), 3:()} 50 | jobson = {'A': (1,), 'B': (2, ), 'C':(3,)} 51 | 52 | dags = manydags(dag, jobson) 53 | assert dags['A'] == {1: (send('A', 'B', 1, 2), send('A', 'C', 1, 3)), 54 | send('A', 'B', 1, 2): (), 55 | send('A', 'C', 1, 3): ()} 56 | assert dags['B'] == {recv('A', 'B', 1, 2): (2, ), 2: ()} 57 | assert dags['C'] == {recv('A', 'C', 1, 3): (3, ), 3: ()} 58 | 59 | def test_manydags_multivar(): 60 | # 1 -> 2 -> 6 61 | # -> 3 -> 62 | # -> 4 -> 63 | # -> 5 -> 64 | dag = {1: (2, 3, 4, 5), 2: (6,), 3: (6,), 4: (6,), 5: (6,), 6: ()} 65 | jobson = {'A': (1, 2, 6), 'B': (3, ), 'C':(4,), 'D': (5,)} 66 | dags = manydags(dag, jobson) 67 | assert dags == {'A': {1: (2, ('send', 'A', 'B', 1, 3), 68 | ('send', 'A', 'C', 1, 4), 69 | ('send', 'A', 'D', 1, 5)), 70 | 2: (6,), 71 | 6: (), 72 | ('recv', 'B', 'A', 3, 6): (6,), 73 | ('recv', 'C', 'A', 4, 6): (6,), 74 | ('recv', 'D', 'A', 5, 6): (6,), 75 | ('send', 'A', 'B', 1, 3): (), 76 | ('send', 'A', 'C', 1, 4): (), 77 | ('send', 'A', 'D', 1, 5): ()}, 78 | 'B': {3: (('send', 'B', 'A', 3, 6),), 79 | ('recv', 'A', 'B', 1, 3): (3,), 80 | ('send', 'B', 'A', 3, 6): ()}, 81 | 'C': {4: (('send', 'C', 'A', 4, 6),), 82 | ('recv', 'A', 'C', 1, 4): (4,), 83 | ('send', 'C', 'A', 4, 6): ()}, 84 | 'D': {5: (('send', 'D', 'A', 5, 6),), 85 | ('recv', 'A', 'D', 1, 5): (5,), 86 | ('send', 'D', 'A', 5, 6): ()}} 87 | 88 | def test_simple_split_problem_integrative(): 89 | dags, sched, makespan = schedule(unidag, agents, computation_cost, 90 | communication_cost, R, B, M) 91 | orders = orderings(sched) 92 | 93 | assert dags == solution['dags'] 94 | 95 | assert sched == solution['sched'] 96 | 97 | assert orders == solution['orders'] 98 | 99 | assert makespan == solution['makespan'] 100 | 101 | def test_issend(): 102 | assert issend(send(1,2,3,4)) 103 | assert not issend(recv(1,2,3,4)) 104 | def test_isrecv(): 105 | assert isrecv(recv(1,2,3,4)) 106 | assert not isrecv(send(1,2,3,4)) 107 | 108 | def test_replace_send_recv(): 109 | dag = {1: (2, send(1,2,3,4)), 2: (), send(1,2,3,4): (), recv(1,2,3,4): (1,)} 110 | o = replace_send_recv(dag, lambda a,b,c,d: a+b+c+d, lambda a,b,c,d: a*b*c*d) 111 | 112 | assert o == {1: (2, 1+2+3+4), 2: (), 1+2+3+4: (), 1*2*3*4: (1,)} 113 | -------------------------------------------------------------------------------- /tompkins/dag.py: -------------------------------------------------------------------------------- 1 | from tompkins.ilp import schedule as schedule_tompkins 2 | from tompkins.ilp import jobs_when_where 3 | from tompkins.util import (reverse_dict, dictify, intersection, merge, unique, 4 | groupby) 5 | from collections import defaultdict 6 | 7 | 8 | def precedes_to_dag(jobs, precedes): 9 | return {a: [b for b in jobs if precedes(a, b)] for a in jobs} 10 | 11 | def transform_args(dag, agents, compcost, commcost, R, B, M): 12 | """ Transform arguments given to dag.schedule to those expected by tompkins 13 | 14 | inputs: 15 | dag - unipartite dag of the form {1: (2, 3)} if job 1 precedes 2 and 3 16 | agents - a list of machines on which we can run each job 17 | compcost - a function (job, agent) -> runtime 18 | commcost - a function (job, agent, agent) -> communication time 19 | R - a function (job) -> start time (usually lambda j: 0) 20 | B - a function (job, agent) -> 1/0 - 1 if job can be run on agent 21 | M - a maximum makespan 22 | outputs: 23 | see tompkins.schedule's inputs 24 | """ 25 | P = unidag_to_P(dag) 26 | D = dictify(compcost) 27 | C = dictify(commcost) 28 | R = dictify(R) 29 | B = dictify(B) 30 | jobs = dag.keys() 31 | return jobs, agents, D, C, R, B, P, M 32 | 33 | def schedule(dag, agents, compcost, commcost, 34 | R=lambda job: 0, # Data available immediately 35 | B=lambda job, agent: 1, # All agents can run all jobs 36 | M=100000.0, # Large cutoff 37 | **kwargs): 38 | """ Statically Schedule a DAG of jobs on a set of machines 39 | 40 | This function wraps tompkins.ilp.schedule 41 | 42 | inputs: 43 | dag - unipartite dag of the form {1: (2, 3)} if job 1 precedes 2 and 3 44 | agents - a list of machines on which we can run each job 45 | compcost - a function (job, agent) -> runtime 46 | commcost - a function (job, agent, agent) -> communication time 47 | R - a function (job) -> start time (usually lambda job: 0) 48 | B - a function (job, agent) -> 1/0 - 1 if job can be run on agent 49 | M - a maximum makespan 50 | 51 | outputs: 52 | dags - a dict mapping machine to local dag 53 | sched - a list of (job, start_time, machine) 54 | makespan - the total runtime of the computation 55 | """ 56 | args = schedule_tompkins( 57 | *transform_args(dag, agents, compcost, commcost, R, B, M)) 58 | prob, X, S, Cmax = args 59 | sched = jobs_when_where(*args) 60 | jobson = compute_jobson(sched) 61 | dags = manydags(dag, jobson, **kwargs) 62 | return dags, sched, Cmax.value() 63 | 64 | def unidag_to_P(dag): 65 | """ Converts a dag dict into one suitable for the tompkins algorithm 66 | 67 | input like {1: (2, 3)} 68 | output like {(1,2): 1, (1, 3): 1} 69 | """ 70 | d = defaultdict(lambda : 0) 71 | for k, v in {(a,b): 1 for a in dag for b in dag[a]}.items(): 72 | d[k] = v 73 | return d 74 | 75 | def compute_jobson(sched): 76 | """ 77 | Given the output of jobs_when_where produce a dict mapping machine to jobs 78 | {machine: [jobs]} 79 | """ 80 | result = {} 81 | for job, _, agent in sched: 82 | result[agent] = result.get(agent, ()) + (job,) 83 | return result 84 | 85 | def send(from_machine, to_machine, from_job, to_job): 86 | return ("send", from_machine, to_machine, from_job, to_job) 87 | def recv(from_machine, to_machine, from_job, to_job): 88 | return ("recv", from_machine, to_machine, from_job, to_job) 89 | 90 | def issend(x): 91 | return isinstance(x, tuple) and x[0] == "send" 92 | def isrecv(x): 93 | return isinstance(x, tuple) and x[0] == "recv" 94 | 95 | def replace_send_recv(dag, fn_send, fn_recv): 96 | """ Replaces all instances of tompkins send with a user defined send 97 | 98 | inputs functions like: 99 | fn_send - (from_machine, to_machine, from_job, to_job) -> a send 100 | fn_recv - (from_machine, to_machine, from_job, to_job) -> a recv 101 | """ 102 | def convert(x): 103 | if issend(x): return fn_send(*x[1:]) 104 | if isrecv(x): return fn_recv(*x[1:]) 105 | return x 106 | return {convert(key): tuple(map(convert, values)) 107 | for key, values in dag.items()} 108 | 109 | def manydags(dag, jobson, send=send, recv=recv): 110 | """ Given a dag and a schedule return many dags with sends/receives 111 | 112 | inputs: 113 | dag - Dictionary containing precedence constraints - specifies DAG: 114 | dag[job1] == (job2, job3) 1 if job1 immediately precedes job2 and job3 115 | jobson - Dictionary mapping machine to list of jobs 116 | 117 | returns: 118 | dags - a dict of dags mapping machine to dag {machine: dag} 119 | Each dag is represented like dag (see above) 120 | New send and receive jobs have been added 121 | """ 122 | 123 | onmachine = {value:key for key, values in jobson.items() 124 | for value in values} 125 | revdag = reverse_dict(dag) 126 | assert unique(sum(jobson.values(), ())) 127 | 128 | return {machine: 129 | merge( 130 | # Standard local dag 131 | {fromjob: tuple( 132 | tojob if tojob in jobson[machine] 133 | else send(machine, onmachine[tojob], fromjob, tojob) 134 | for tojob in dag[fromjob]) 135 | for fromjob in jobson[machine]} 136 | , 137 | # Add in all of the receives 138 | {recv(onmachine[fromjob], machine, fromjob, tojob): 139 | intersection(dag[fromjob], jobson[machine]) 140 | for tojob in jobson[machine] 141 | for fromjob in revdag.get(tojob, ()) # might not have parent 142 | if fromjob not in jobson[machine]} 143 | , 144 | # Add in all of the sends 145 | {send(machine, onmachine[tojob], fromjob, tojob): () 146 | for fromjob in jobson[machine] 147 | for tojob in dag.get(fromjob, ()) 148 | if onmachine[tojob] != machine 149 | } 150 | ) 151 | for machine in jobson.keys()} 152 | 153 | 154 | def orderings(sched): 155 | nth = lambda n: (lambda x: x[n]) 156 | byagent = groupby(nth(2), sched) 157 | return {k: map(nth(0), sorted(v, key=nth(1))) for k,v in byagent.items()} 158 | -------------------------------------------------------------------------------- /tompkins/ilp.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module describes a highly flexible scheduling problem by encoding it as a 3 | Mixed Integer Linear Program (MILP) using the pulp framework[1]. 4 | This framework can then call any of several external codes[2] to solve 5 | 6 | This formulation of the MILP was specified in chapters 4.1, 4.2 in the 7 | following masters thesis. Section and equation numbers are cited. 8 | 9 | "Optimization Techniques for Task Allocation and Scheduling in Distributed Multi-Agent Operations" 10 | by 11 | Mark F. Tompkins, June 2003, Masters thesis for MIT Dept EECS[3] 12 | 13 | [1] http://code.google.com/p/pulp-or/ -- "sudo easy_install pulp" 14 | [2] On ubuntu we used "sudo apt-get install glpk" 15 | [3] http://dspace.mit.edu/bitstream/handle/1721.1/16974/53816027.pdf?sequence=1 16 | """ 17 | 18 | from pulp import (LpVariable, LpProblem, LpMinimize, LpInteger, LpContinuous, 19 | lpSum, LpStatus) 20 | from collections import defaultdict 21 | from tompkins.util import reverse_dict, dictify, merge, intersection 22 | 23 | def schedule(Jobs, Agents, D, C, R, B, P, M): 24 | """ 25 | Finds the optimal scheduling of Jobs to Agents (workers) given 26 | Jobs - a set of Jobs (an iterable of hashable objects) 27 | Agents - a set of Agents/workers (an iterable of hashable objects) 28 | D - Dictionary detailing execution cost of running jobs on agents : 29 | D[job, agent] = time to run job on agent 30 | C - Dictionary detailing Communication cost sending results of jobs 31 | between agents : 32 | C[job, a1, a2] = time to comm results of job on from a1 to a2 33 | R - Additional constraint on start time of jobs, usually defaultdict(0) 34 | B - Dict saying which jobs can run on which agents: 35 | B[job, agent] == 1 if job can run on agent 36 | P - Dictionary containing precedence constraints - specifies DAG: 37 | P[job1, job2] == 1 if job1 immediately precedes job2 38 | M - Upper bound on makespan 39 | 40 | Returns 41 | prob - the pulp problem instance 42 | X - Dictionary detailing which jobs were run on which agents: 43 | X[job][agent] == 1 iff job was run on agent 44 | S - Starting time of each job 45 | Cmax - Optimal makespan 46 | 47 | """ 48 | 49 | # Set up global precedence matrix 50 | Q = PtoQ(P) 51 | 52 | 53 | # PULP SETUP 54 | # The prob variable is created to contain the problem data 55 | prob = LpProblem("Scheduling Problem - Tompkins Formulation", LpMinimize) 56 | 57 | # The problem variables are created 58 | # X - 1 if job is scheduled to be completed by agent 59 | X = LpVariable.dicts("X", (Jobs, Agents), 0, 1, LpInteger) 60 | # S - Scheduled start time of job 61 | S = LpVariable.dicts("S", Jobs, 0, M, LpContinuous) 62 | 63 | # Theta - Whether two jobs overlap 64 | Theta = LpVariable.dicts("Theta", (Jobs, Jobs), 0, 1, LpInteger) 65 | # Makespan 66 | Cmax = LpVariable("C_max", 0, M, LpContinuous) 67 | 68 | ##################### 69 | # 4.2.2 CONSTRAINTS # 70 | ##################### 71 | 72 | # Objective function 73 | prob += Cmax 74 | 75 | # Subject to: 76 | 77 | # 4-1 Cmax is greater than the ending schedule time of all jobs 78 | for job in Jobs: 79 | prob += Cmax >= S[job] + lpSum([D[job, agent] * X[job][agent] 80 | for agent in Agents if B[job, agent]>0]) 81 | 82 | # 4-2 an agent cannot be assigned a job unless it provides the services 83 | # necessary to complete that job 84 | for job in Jobs: 85 | for agent in Agents: 86 | if B[job, agent] == 0: 87 | prob += X[job][agent] == 0 88 | 89 | # 4-3 specifies that each job must be assigned once to exactly one agent 90 | for job in Jobs: 91 | prob += lpSum([X[job][agent] for agent in Agents]) == 1 92 | 93 | # 4-4 a job cannot start until its predecessors are completed and data has 94 | # been communicated to it if the preceding jobs were executed on a 95 | # different agent 96 | for (j,k), prec in P.items(): 97 | if prec>0: # if j precedes k in the DAG 98 | prob += S[k]>=S[j] 99 | for a in Agents: 100 | for b in Agents: 101 | if B[j,a] and B[k,b]: # a is capable of j and b capable of k 102 | prob += S[k] >= (S[j] + 103 | (D[j,a] + C[j,a,b]) * (X[j][a] + X[k][b] -1)) 104 | 105 | # 4-5 a job cannot start until after its release time 106 | for job in Jobs: 107 | if R[job]>0: 108 | prob += S[job] >= R[job] 109 | 110 | # Collectively, (4-6) and (4-7) specify that an agent may process at most 111 | # one job at a time 112 | 113 | # 4-6 114 | for j in Jobs: 115 | for k in Jobs: 116 | if j==k or Q[j,k]!=0: 117 | continue 118 | prob += S[k] - lpSum([D[j,a]*X[j][a] for a in Agents]) - S[j] >= ( 119 | -M*Theta[j][k]) 120 | # The following line had a < in the paper. We've switched to <= 121 | # Uncertain if this is a good idea 122 | prob += S[k] - lpSum([D[j,a]*X[j][a] for a in Agents]) - S[j] <= ( 123 | M*(1-Theta[j][k])) 124 | # 4-7 if two jobs j and k are assigned to the same agent, their execution 125 | # times may not overlap 126 | for j in Jobs: 127 | for k in Jobs: 128 | for a in Agents: 129 | prob += X[j][a] + X[k][a] + Theta[j][k] + Theta[k][j] <= 3 130 | 131 | return prob, X, S, Cmax 132 | 133 | def PtoQ(P): 134 | """ 135 | Construct full job precedence graph from immediate precedence graph 136 | 137 | Inputs: 138 | P - Dictionary encoding the immediate precedence graph: 139 | P[job1, job2] == 1 iff job1 immediately precedes job2 140 | 141 | Outputs: 142 | Q - Dictionary encoding full precedence graph: 143 | Q[job1, job2] == 1 if job1 comes anytime before job2 144 | """ 145 | Q = defaultdict(lambda:0) 146 | # Add one-time-step knowledge 147 | for (i,j), prec in P.items(): 148 | Q[i,j] = prec 149 | 150 | changed = True 151 | while(changed): 152 | changed = False 153 | for (i,j), prec in P.items(): # i comes immediately before j 154 | if not prec: 155 | continue 156 | for (jj, k), prec in Q.items(): # j comes sometime before k 157 | if jj != j or not prec: 158 | continue 159 | if not Q[i,k]: # Didn't know i comes sometime before k? 160 | changed = True # We changed Q 161 | Q[i,k] = 1 # Now we do. 162 | return Q 163 | 164 | def jobs_when_where(prob, X, S, Cmax): 165 | """ 166 | Take the outputs of schedule and produce a list of the form 167 | [(job, start_time, agent), 168 | (job, start_time, agent), 169 | ... ] 170 | 171 | sorted by start_time. 172 | 173 | >>> sched = jobs_when_where(*make_ilp(env, ... )) 174 | """ 175 | status = LpStatus[prob.solve()] 176 | if status != 'Optimal': 177 | print "ILP solver status: ", status 178 | 179 | def runs_on(job, X): 180 | return [k for k,v in X[job].items() if v.value()][0] 181 | 182 | sched = [(job, time.value(), runs_on(job,X)) for job, time in S.items()] 183 | return list(sorted(sched, key=lambda x:x[1:])) 184 | --------------------------------------------------------------------------------