├── .gitignore
├── README.md
├── worker.py
├── think.cpp
├── slave.py
├── LICENSE
├── ventilatorsink.py
└── master.py


/.gitignore:
--------------------------------------------------------------------------------
1 | think
2 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ### Easy cluster parallelization in ZeroMQ
 2 | 
 3 | This is meant as a support code for [this blog
 4 | post](http://mdup.fr/blog/easy-cluster-parallelization-with-zeromq).
 5 | 
 6 | TL;DR: two skeletons to dispatch jobs in a cluster using zeromq. The first, bad
 7 | skeleton is "ventilatorsink" + "worker" (PUSH/PULL). It's not good because it
 8 | can block if a worker is too long. The good pattern is "master" + "slave"
 9 | (REQ/REP).
10 | 
11 | Marc Dupont [mdup.fr](http://mdup.fr)
12 | 


--------------------------------------------------------------------------------
/worker.py:
--------------------------------------------------------------------------------
 1 | import zmq
 2 | import subprocess
 3 | 
 4 | def worker():
 5 |     # Setup ZMQ sockets.
 6 |     context = zmq.Context()
 7 |     sock_in = context.socket(zmq.PULL)
 8 |     sock_in.connect("tcp://192.168.196.1:5557") # IP of master
 9 |     sock_out = context.socket(zmq.PUSH)
10 |     sock_out.connect("tcp://192.168.196.1:5558") # IP of master
11 | 
12 |     while True:
13 |         work = sock_in.recv_json()
14 |         p = work['p']
15 |         q = work['q']
16 |         print "running computation p=%d, q=%d" % (p, q)
17 |         result = run_computation(p, q)
18 |         sock_out.send(result)
19 | 
20 | # Delegate the hard work to the C++ binary, and retrieve its results.
21 | def run_computation(p, q):
22 |     return subprocess.check_output(["./think", str(p), str(q)])
23 | 


--------------------------------------------------------------------------------
/think.cpp:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <chrono>
 3 | #include <thread>
 4 | #include <cmath>
 5 | #include <cassert>
 6 | 
 7 | void hardcore_computation(double p, double q) {
 8 |     // Your big number crunching work should go here.
 9 |     // Instead we calculate some fake result.
10 |     std::this_thread::sleep_for(std::chrono::seconds(2));
11 |     const double result = std::sin(p + 2*q);
12 | 
13 |     // Output the result.
14 |     std::cout <<
15 |         "{ \"params\":"                    "\n"
16 |         "  { \"p\": " << p << ","          "\n"
17 |         "    \"q\": " << q << " },"        "\n"
18 |         "  \"result\": " << result << " }" "\n";
19 | }
20 | 
21 | int main(int argc, const char **argv) {
22 |     // Retrieve parameters for this run.
23 |     assert(argc > 2);
24 |     const double p = std::stod(argv[1]);
25 |     const double q = std::stod(argv[2]);
26 | 
27 |     // Start the computation.
28 |     hardcore_computation(p, q);
29 | 
30 |     return 0;
31 | }
32 | 


--------------------------------------------------------------------------------
/slave.py:
--------------------------------------------------------------------------------
 1 | import zmq
 2 | import subprocess
 3 | 
 4 | def slave():
 5 |     # Setup ZMQ.
 6 |     context = zmq.Context()
 7 |     sock = context.socket(zmq.REQ)
 8 |     sock.connect("tcp://192.168.196.1:5557") # IP of master
 9 | 
10 |     while True:
11 |         # Say we're available.
12 |         sock.send_json({ "msg": "available" })
13 | 
14 |         # Retrieve work and run the computation.
15 |         work = sock.recv_json()
16 |         if work == {}:
17 |             continue
18 |         p = work['p']
19 |         q = work['q']
20 |         print "running computation p=%d, q=%d" % (p, q)
21 |         result = run_computation(p, q)
22 | 
23 |         # We have a result, let's inform the master about that, and receive the
24 |         # "thanks".
25 |         sock.send_json({ "msg": "result", "result": result})
26 |         sock.recv()
27 | 
28 | # Delegate the hard work to the C++ binary and retrieve its results from the
29 | # standard output.
30 | def run_computation(p, q):
31 |     return subprocess.check_output(["./think", str(p), str(q)])
32 | 
33 | if __name__ == "__main__":
34 |     slave()
35 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2015 Marc Dupont
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/ventilatorsink.py:
--------------------------------------------------------------------------------
 1 | import zmq
 2 | import sys
 3 | 
 4 | def ventilator(p_range, q_range):
 5 |     # Setup ZMQ socket
 6 |     context = zmq.Context()
 7 |     sock = context.socket(zmq.PUSH)
 8 |     sock.bind("tcp://0.0.0.0:5557")
 9 | 
10 |     # Iterate over the grid, send each piece of computation to a worker.
11 |     for p in p_range:
12 |         for q in [0.01, 0.1, 1, 10]:
13 |             work = { 'p' : p, 'q': q };
14 |             print "sending work p=%d, q=%d..." % (p, q)
15 |             sock.send_json(work)
16 | 
17 | def sink(p_range, q_range):
18 |     # Total number of computations.
19 |     n_total = len(p_range) * len(q_range)
20 | 
21 |     # Setup ZMQ socket.
22 |     context = zmq.Context()
23 |     sock = context.socket(zmq.PULL)
24 |     sock.bind("tcp://0.0.0.0:5558")
25 | 
26 |     # Accumulate the results until we know all computations are done.
27 |     results = []
28 |     n_processed = 0
29 |     while n_processed < n_total:
30 |         r = sock.recv()
31 |         results.append(r)
32 |         n_processed += 1
33 |     for r in results:
34 |         print r
35 | 
36 | if __name__ == "__main__":
37 |     p_range = [pow(10, n) for n in xrange(-6, 6)] # All values for the first parameter
38 |     q_range = [pow(10, n) for n in xrange(-6, 6)] # All values for the second parameter
39 | 
40 |     if len(sys.argv) < 2:
41 |         print "usage: %s (ventilator | sink)" % sys.argv[0]
42 |     elif sys.argv[1] == 'ventilator':
43 |         ventilator(p_range, q_range)
44 |     elif sys.argv[1] == 'sink':
45 |         sink(p_range, q_range)
46 |     else:
47 |         print "usage: %s (ventilator | sink)" % sys.argv[0]
48 | 


--------------------------------------------------------------------------------
/master.py:
--------------------------------------------------------------------------------
 1 | import zmq
 2 | 
 3 | def master(p_range, q_range):
 4 |     # Setup ZMQ.
 5 |     context = zmq.Context()
 6 |     sock = context.socket(zmq.REP)
 7 |     sock.bind("tcp://0.0.0.0:5557")
 8 | 
 9 |     # Generate the json messages for all computations.
10 |     works = generate_works(p_range, q_range)
11 | 
12 |     # How many calculations are expected?
13 |     n_total = len(p_range) * len(q_range)
14 | 
15 |     # Loop until all results arrived.
16 |     results = []
17 |     while len(results) < n_total:
18 |         # Receive;
19 |         j = sock.recv_json()
20 | 
21 |         # First case: worker says "I'm available". Send him some work.
22 |         if j['msg'] == "available":
23 |             send_next_work(sock, works)
24 | 
25 |         # Second case: worker says "Here's your result". Store it, say thanks.
26 |         elif j['msg'] == "result":
27 |             r = j['result']
28 |             results.append(r)
29 |             send_thanks(sock)
30 | 
31 |     # Results are all in.
32 |     print "=== Results ==="
33 |     for r in results:
34 |         print r
35 | 
36 | def generate_works(p_range, q_range):
37 |     # We want to span all (p, q) combinations.
38 |     for p in p_range:
39 |         for q in q_range:
40 |             work = { 'p' : p, 'q': q };
41 |             print "sending work p=%f, q=%f..." % (p, q)
42 |             yield work
43 | 
44 | def send_next_work(sock, works):
45 |     try:
46 |         work = works.next()
47 |         sock.send_json(work)
48 |     except StopIteration:
49 |         # If no more work is available, we still have to reply something.
50 |         sock.send_json({})
51 | 
52 | def send_thanks(sock):
53 |     sock.send("") # Nothing more to say actually
54 | 
55 | if __name__ == "__main__":
56 |     p_range = [pow(10, n) for n in xrange(-6, 6)] # All values for the first parameter
57 |     q_range = [pow(10, n) for n in xrange(-6, 6)] # All values for the second parameter
58 | 
59 |     master(p_range, q_range)
60 | 


--------------------------------------------------------------------------------