├── README
└── coreit.py


/README:
--------------------------------------------------------------------------------
 1 | coreit
 2 | -------
 3 | 
 4 | Helper function to throw code on cores that can be run in parallel.
 5 | 
 6 | If you have code like
 7 | 
 8 | >>> for i in xrange(start, stop):
 9 | ...     slow_function(i)
10 | 
11 | You can speed it up on a 4-core machine by running it like so:
12 | 
13 | >>> functions = [slow_function for i in xrange(start, stop)]
14 | >>> args = [(i,) for i in xrange(start, stop)]
15 | >>> coreit(functions, args, 4)
16 | 
17 | Note that 
18 | 
19 | >>> p = multiprocessing.Pool(4)
20 | >>> p.map(slow_function, xrange(start, stop))
21 | 
22 | is built in and does the equivalent. 
23 | 
24 | As such, this code is somewhat obsolete, but has a distinction: if you
25 | have *different* functions that you want to run concurrently, coreit
26 | will do it.  Pool.map won't.
27 | 
28 | Requirements
29 | -------------
30 | 
31 | * multiprocessing  (python 2.6 or http://code.google.com/p/python-multiprocessing/)
32 | 
33 | Todo
34 | ----
35 | 
36 | * ?
37 | 
38 | Licensed under PSF license
39 | Copyright (c) 2010 Matt Harrison (matthewharrison@gmail.com)
40 | 


--------------------------------------------------------------------------------
/coreit.py:
--------------------------------------------------------------------------------
 1 | """
 2 | coreit
 3 | -------
 4 | 
 5 | Helper function to throw code on cores that can be run in parallel.
 6 | 
 7 | If you have code like
 8 | 
 9 | >>> for i in xrange(start, stop):
10 | ...     slow_function(i)
11 | 
12 | You can speed it up on a 4-core machine by running it like so:
13 | 
14 | >>> functions = [slow_function for i in xrange(start, stop)]
15 | >>> args = [(i,) for i in xrange(start, stop)]
16 | >>> coreit(functions, args, 4)
17 | 
18 | Licensed under PSF license
19 | Copyright (c) 2010 Matt Harrison
20 | """
21 | 
22 | __license__ = 'psf'
23 | __author__ = 'matthewharrison@gmail.com'
24 | __version__ = '0.0.1'
25 | 
26 | from multiprocessing import Process, Queue
27 | from Queue import Empty
28 | 
29 | 
30 | def coreit(functions, args, num_procs):
31 |     """
32 |     Given a list of functions (and a corresponding list of thier
33 |     arguments (in tuples or empty tuples)), run them concurrently
34 |     using num_procs.
35 |     """
36 |     def runit(func_args_queue):
37 |         """
38 |         Target for multiprocessing.Process.  It takes a Queue instance
39 |         that has tuples of (function, args) in it.
40 |         Just pops off the functions and runs them.
41 |         """
42 |         while True:
43 |             try:
44 |                 f, args = func_args_queue.get(block=False)
45 |                 f(*args)
46 |             except Empty:
47 |                 break
48 |     work_q = Queue()
49 |     for i, f in enumerate(functions):
50 |         work_q.put((f, args[i]))
51 |     # only create as many processes as we have num_procs
52 |     # they will run until work_q is empty
53 |     processes = [Process(target=runit, args=(work_q,)) for i in range(num_procs)]
54 |     for p in processes:
55 |         p.start()
56 |     for p in processes:
57 |         p.join()
58 | 
59 | def slow_function():
60 |     import math
61 |     total = 0
62 |     for i in xrange(10000000):
63 |         total = total + math.sqrt(i)
64 |     print total
65 | 
66 | def test():    
67 |     import timeit
68 |     num_runs = 1
69 |     print '2 core'
70 |     t = timeit.Timer("coreit([slow_function, slow_function], [tuple(), tuple()], 2)", 'from coreit import *')
71 |     print "%s seconds" % t.timeit(num_runs)
72 |     print '1 core'
73 |     t = timeit.Timer("slow_function();slow_function()", 'from coreit import *')
74 |     print "%s seconds" % t.timeit(num_runs)
75 | 
76 | if __name__ == '__main__':
77 |     test()
78 | 


--------------------------------------------------------------------------------