├── .gitignore
├── LICENSE
├── README.md
├── checkipnb.py
└── launch_ipynb.sh


/.gitignore:
--------------------------------------------------------------------------------
 1 | *.py[cod]
 2 | 
 3 | # C extensions
 4 | *.so
 5 | 
 6 | # Packages
 7 | *.egg
 8 | *.egg-info
 9 | dist
10 | build
11 | eggs
12 | parts
13 | bin
14 | var
15 | sdist
16 | develop-eggs
17 | .installed.cfg
18 | lib
19 | lib64
20 | __pycache__
21 | 
22 | # Installer logs
23 | pip-log.txt
24 | 
25 | # Unit test / coverage reports
26 | .coverage
27 | .tox
28 | nosetests.xml
29 | 
30 | # Translations
31 | *.mo
32 | 
33 | # Mr Developer
34 | .mr.developer.cfg
35 | .project
36 | .pydevproject
37 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2014 Thomas Wiecki
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of
 6 | this software and associated documentation files (the "Software"), to deal in
 7 | the Software without restriction, including without limitation the rights to
 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 9 | the Software, and to permit persons to whom the Software is furnished to do so,
10 | subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | ipython notebook cluster
2 | ========================
3 | 
4 | Scripts to launch IPython parallel on a cluster and run an IPython Notebook to run the analysis.
5 | 
6 | For a full description, see [this blog post](http://twiecki.github.com/blog/2014/02/24/ipython-nb-cluster/).
7 | 


--------------------------------------------------------------------------------
/checkipnb.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | """
 3 | simple example script for running notebooks and reporting exceptions.
 4 | 
 5 | Usage: `checkipnb.py foo.ipynb [bar.ipynb [...]]`
 6 | 
 7 | Each cell is submitted to the kernel, and checked for errors.
 8 | """
 9 | 
10 | import os,sys,time
11 | 
12 | from Queue import Empty
13 | 
14 | try:
15 |     from IPython.kernel import KernelManager
16 | except ImportError:
17 |     from IPython.zmq.blockingkernelmanager import BlockingKernelManager as KernelManager
18 | 
19 | from IPython.nbformat.current import reads, NotebookNode
20 | 
21 | def run_notebook(nb):
22 |     km = KernelManager()
23 |     km.start_kernel(stderr=open(os.devnull, 'w'))
24 |     try:
25 |         kc = km.client()
26 |     except AttributeError:
27 |         # 0.13
28 |         kc = km
29 |     kc.start_channels()
30 |     shell = kc.shell_channel
31 |     # simple ping:
32 |     shell.execute("pass")
33 |     shell.get_msg()
34 |     
35 |     cells = 0
36 |     failures = 0
37 |     for ws in nb.worksheets:
38 |         for cell in ws.cells:
39 |             if cell.cell_type != 'code':
40 |                 continue
41 |             shell.execute(cell.input)
42 |             # wait for finish, maximum 20s
43 |             reply = shell.get_msg(timeout=20)['content']
44 |             if reply['status'] == 'error':
45 |                 failures += 1
46 |                 print "\nFAILURE:"
47 |                 print cell.input
48 |                 print '-----'
49 |                 print "raised:"
50 |                 print '\n'.join(reply['traceback'])
51 |             cells += 1
52 |             sys.stdout.write('.')
53 | 
54 |     print
55 |     print "ran notebook %s" % nb.metadata.name
56 |     print "    ran %3i cells" % cells
57 |     if failures:
58 |         print "    %3i cells raised exceptions" % failures
59 |     kc.stop_channels()
60 |     km.shutdown_kernel()
61 |     del km
62 | 
63 | if __name__ == '__main__':
64 |     for ipynb in sys.argv[1:]:
65 |         print "running %s" % ipynb
66 |         with open(ipynb) as f:
67 |             nb = reads(f.read(), 'json')
68 |         run_notebook(nb)
69 | 


--------------------------------------------------------------------------------
/launch_ipynb.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | #SBATCH -J ipython
 3 | #SBATCH -n 64
 4 | #SBATCH --time=48:00:00
 5 | #SBATCH --ntasks-per-node=8
 6 | #SBATCH --mem-per-cpu=1024
 7 | 
 8 | echo "Launching controller"
 9 | $HOME/anaconda/bin/ipcontroller --ip='*' &
10 | sleep 10
11 | 
12 | echo "Launching engines"
13 | srun $HOME/anaconda/bin/ipengine &
14 | sleep 25
15 | 
16 | echo "Launching job"
17 | $HOME/anaconda/bin/python checkipnb.py $1
18 | 
19 | echo "Done!"
20 | 


--------------------------------------------------------------------------------