├── run_async ├── __init__.pyc ├── __init__.py ├── handlers.py ├── utils.py ├── settings.py ├── async_run_magic.py └── run_server.py ├── requirements.txt ├── async_run_ipython_magic.py ├── LICENSE ├── examples ├── script2run.py ├── async_run magic notebook examples.ipynb └── iPython (default) Run Magic Test.ipynb └── README.md /run_async/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leriomaggio/async-ipython-magic/HEAD/run_async/__init__.pyc -------------------------------------------------------------------------------- /run_async/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ 3 | # Author: Valerio Maggio 4 | # Copyright (c) 2015 Valerio Maggio 5 | # License: BSD 3 clause -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ipython==5.1.0 2 | jupyter==1.0.0 3 | jupyter-client==4.2.2 4 | jupyter-console==4.1.1 5 | jupyter-core==4.1.0 6 | nbconvert==4.2.0 7 | nbformat==4.0.1 8 | notebook==6.1.5 9 | psutil==5.6.6 10 | tornado==4.3 11 | traitlets==4.2.1 -------------------------------------------------------------------------------- /async_run_ipython_magic.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ 3 | # Author: Valerio Maggio 4 | # Copyright (c) 2015 Valerio Maggio 5 | # License: BSD 3 clause 6 | 7 | from run_async.async_run_magic import AsyncRunMagic 8 | 9 | # from IPython.core.magic import register_cell_magic 10 | # async_run = register_cell_magic(async_run) 11 | 12 | # ----------------------------- 13 | # Register the new Class Magic 14 | # ----------------------------- 15 | 16 | # noinspection PyUnresolvedReferences 17 | ip = get_ipython() 18 | ip.register_magics(AsyncRunMagic) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Valerio Maggio 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /examples/script2run.py: -------------------------------------------------------------------------------- 1 | """This is a sample script file, run and invoked directly 2 | by an iPython notebook (i.e. "iPython Run Test.ipynb"). 3 | 4 | This is just to see how the `%run` iPython magic works 5 | with external executions. 6 | """ 7 | 8 | import os 9 | import argparse 10 | import sys 11 | 12 | class DummyClass(): 13 | """Dummy class just defined to see how namespace 14 | integration works""" 15 | 16 | def __init__(self, class_name): 17 | self._class_name = class_name 18 | 19 | @property 20 | def name(self): 21 | return self._class_name 22 | 23 | 24 | dummy_global = DummyClass('Global Scope') 25 | print('Created Instance: dummy_global') 26 | 27 | 28 | def main(): 29 | current_path = os.path.abspath(os.path.curdir) 30 | print('Current Path: ', current_path) 31 | 32 | print('Sys Argv: ', sys.argv) 33 | 34 | dummy_local = DummyClass('Local Scope') 35 | print('Instance: dummy_local') 36 | print('Name: ', dummy_local.name) 37 | 38 | 39 | def main_heavy(): 40 | N = 10**7 41 | s = sum([i**3 for i in range(N)]) 42 | return s 43 | 44 | 45 | if __name__ == '__main__': 46 | 47 | parser = argparse.ArgumentParser(description='Simple or Heavy Execution.') 48 | parser.add_argument('--mode', dest='mode', type=str, choices=['simple', 'heavy'], 49 | default='simple', help='the execution mode (i.e. Simple or Heavy)') 50 | 51 | args = parser.parse_args() 52 | 53 | if args.mode == 'simple': 54 | # Simple, Lightweight, execution 55 | main() 56 | 57 | dummy_main = DummyClass('Main Scope') 58 | print('Instance: dummy_main') 59 | print('Name (in Main): ', dummy_main.name) 60 | else: # heavy 61 | s = main_heavy() 62 | print('sum: ', s) -------------------------------------------------------------------------------- /run_async/handlers.py: -------------------------------------------------------------------------------- 1 | """Collection of handler classes (dictionary-like objects) 2 | """ 3 | # Author: Valerio Maggio 4 | # Copyright (c) 2015 Valerio Maggio 5 | # License: BSD 3 clause 6 | 7 | from collections import defaultdict 8 | 9 | # from queue import Queue 10 | from multiprocessing import SimpleQueue 11 | try: 12 | from tornado.websocket import WebSocketHandler 13 | except ImportError: 14 | pass 15 | 16 | from .settings import JS_ROLE 17 | from .utils import format_ws_connection_id 18 | 19 | 20 | class Handler(): 21 | """Container object for data management. The handler contains 22 | a dictionary whose default type is determined according to the 23 | class provided in the constructor""" 24 | 25 | def __init__(self, factory=None): 26 | if factory is None: 27 | factory = str # Default type of data 28 | self._data = defaultdict(factory) 29 | 30 | def add(self, key, value): 31 | self._data[key] = value 32 | 33 | def remove(self, key): 34 | _ = self._data.pop(key, None) 35 | 36 | def get(self, key): 37 | return self._data.get(key, None) 38 | 39 | def __contains__(self, key): 40 | return key in self._data 41 | 42 | @property 43 | def entries(self): 44 | return list(self._data.keys()) 45 | 46 | 47 | class WebSocketConnectionHandler(Handler): 48 | """Handler for `tornado.websocket.WebSocketHandler` connections. 49 | 50 | Entries' keys are the _connection_id[s], namely 51 | --- 52 | """ 53 | 54 | def __init__(self): 55 | super(WebSocketConnectionHandler, self).__init__( 56 | factory=WebSocketHandler) 57 | 58 | 59 | class ResultCache(Handler): 60 | """Handler for caching execution results, 61 | namely JSON (string) output. 62 | 63 | Entries' keys are the (clients) _connection_id[s], namely 64 | --- 65 | """ 66 | 67 | def __init__(self): 68 | super(ResultCache, self).__init__(factory=str) 69 | 70 | def add(self, session_id, value): 71 | cache_id = format_ws_connection_id(JS_ROLE, session_id) 72 | super(ResultCache, self).add(cache_id, value) 73 | 74 | 75 | class ExecutionHandler(Handler): 76 | """Handler to store the execution queues in order to 77 | make clients to wait on correct thread queues. 78 | Entries' keys are the session_id[s], namely "one queue per session_id". 79 | """ 80 | 81 | def __init__(self): 82 | super(ExecutionHandler, self).__init__(factory=SimpleQueue) 83 | 84 | -------------------------------------------------------------------------------- /run_async/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ 3 | # Author: Valerio Maggio 4 | # Copyright (c) 2015 Valerio Maggio 5 | # License: BSD 3 clause 6 | 7 | from IPython.utils.coloransi import TermColors, color_templates 8 | from .settings import SERVER_ADDR, SERVER_PORT, CONNECTION_ID_SEP 9 | 10 | COLORS = [color[1] for color in color_templates] 11 | 12 | def strip_ansi_color(text): 13 | """ 14 | Removes ANSI colors from the text 15 | 16 | Parameters 17 | ---------- 18 | text : str 19 | The input text string to process 20 | 21 | Returns 22 | ------- 23 | str : the plain text with all ANSI colors stripped. 24 | 25 | """ 26 | text = text.replace(TermColors.Normal, TermColors.NoColor) 27 | for color in COLORS: 28 | text = text.replace(TermColors._base % (color), TermColors.NoColor) 29 | return text 30 | 31 | 32 | def connection_string(web_socket=True, extra=''): 33 | if web_socket: 34 | protocol = 'ws' 35 | else: 36 | protocol = 'http' 37 | return '{proto}://{server}:{port}/{extra}'.format(proto=protocol, server=SERVER_ADDR, 38 | port= SERVER_PORT, extra=extra) 39 | 40 | def format_ws_connection_id(role_name, session_id): 41 | """ 42 | Format and return a (likely) unique string 43 | to be fed to the Websocket server in the 44 | url. This string will be used to uniquely 45 | identify the open connection. 46 | See `run_server.WebSocketConnectionHandler` for further 47 | details. 48 | 49 | Parameters 50 | ---------- 51 | role_name : str 52 | The name of the role of the client trying to connect 53 | to the WebSocket 54 | session_id : str 55 | The uniquely defined `uuid` generated for the 56 | connecting client. 57 | 58 | Returns 59 | ------- 60 | ws_conn_id : str 61 | String representing the connection ID to be 62 | fed to the WebSocket server (url) 63 | """ 64 | 65 | return "{0}{1}{2}".format(role_name, CONNECTION_ID_SEP, session_id) 66 | 67 | 68 | def parse_ws_connection_id(connection_id): 69 | """ 70 | Get and return the role name and the 71 | session id associated to the input connection id. 72 | 73 | Parameters 74 | ---------- 75 | connection_id : str 76 | The connection ID formatted according to the 77 | `format_ws_connection_id` function. 78 | 79 | Returns 80 | ------- 81 | _connection_id : str 82 | The name of the role of the connected client associated to the connection ID 83 | (namely, "JS" or "PY") 84 | session_id : str 85 | The session id associated to the client at connection time. 86 | 87 | Note 88 | ---- 89 | This function is the counterpart of the `format_ws_connection_id` function. 90 | This function decompose a connection id, while the former composes it. 91 | """ 92 | 93 | role_name, session_id = connection_id.split(CONNECTION_ID_SEP) 94 | return role_name, session_id -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `%%async_run`: an IPython notebook magic for asynchronous cell execution 2 | 3 | 4 | ## Description (Talk Abstract @EuroScipy 2016) ## 5 | 6 | 7 | 8 | The IPython `notebook` project is one the preferred tools of data scientists, 9 | and it is nowadays the bastion for *Reproducible Research*. 10 | 11 | In fact, notebooks are now used as in-browser IDE (*Integrated Development Environment*) to implement the whole data analysis process, along with the corresponding documentation. 12 | 13 | However, since this kind of processes usually include heavy-weight computations, 14 | it may likely happen that execution results get lost if something wrong happens, e.g. the connection to the 15 | notebook server hangs or an accidental page refresh is issued. 16 | 17 | To this end, `[%]%async_run` notebook line/cell magic to the rescue. 18 | 19 | In this talk, I would like to talk about some of the technologies I played with since I decided to develop 20 | this extension. 21 | These technologies include **asynchronous I/O** libraries (e.g. `asyncio`, `tornado.websocket`), 22 | **`multiprocessing`**, along with IPython `kernels` and `notebooks`. 23 | 24 | During the talk, I would like to discuss pitfalls, failures, and adopted solutions (e.g. *namespace management 25 | among processes*) , aiming at getting as many feedbacks as possible from the community. 26 | 27 | A general introduction to the actual state-of-the-art of the **Jupyter** projects (an libraries) will be 28 | presented as well, in order to help those who are willing to know some more details about the internals of 29 | IPython. 30 | 31 | ### Slides ### 32 | 33 | Slides of the talk are available on my 34 | [SpeakerDeck](https://speakerdeck.com/valeriomaggio/percent-percent-async-run-an-ipython-notebook-extension-for-asynchronous-cell-execution) 35 | profile. 36 | 37 | ## Enabling the Magic(s) 38 | 39 | Enabling the magic is simple as *copying files into a directory*. Open the terminal and: 40 | 41 | ``` 42 | cp -R async_run_ipython_magic.py run_async/ ~/.ipython/profile_default/startup/ 43 | ``` 44 | 45 | After that, the magic will be enabled by default at the startup of each Jupyter/IPython sessions. 46 | 47 | ## Requirements ## 48 | 49 | The **only** two main requirements for this Magic are `notebook` and `tornado` (which will be 50 | indeed installed by the *jupyter notebook* itself). Moreover, the `psutil` module is needed. 51 | 52 | To ease the installation of all the requirements, a `requirements.txt` file is provided in the repo 53 | for pip installing: 54 | 55 | ```pip install -f requirements.txt``` 56 | 57 | ### Python 2 Users 58 | 59 | So far, this magic works **only** with Python 3. 60 | For example, it relies on the `concurrent.futures` module to allow for the multiprocessing execution. 61 | 62 | This module is only available in **Python 3** standard library. For Python 2, you have to `pip install` 63 | the [futures](https://pypi.python.org/pypi/futures) 64 | 65 | A couple of fix could be included in the code to easily support Python 2 as well, leveraging on the `six` 66 | module. These fixes will be soon included in the code base. 67 | 68 | ## Usage ## 69 | 70 | Three `[%]%async_*` magics are provided within this package: 71 | 72 | * `%async_run_server` : Spawns the `AsyncRunServer` process, which is in charge of handling the async cell execution inside a Tornado `WebApplication` and `IOLoop`. 73 | 74 | * `%async_stop_server` : Stops the `AsyncRunServer` running process, if any. 75 | 76 | * `[%]%async_run` : Line/Cell Magic to asynchronously execute the content of the line/cell, respectively. 77 | 78 | ### Examples ### 79 | 80 | Please, check out the `examples` folder for examples and hints for usage (so far, very few examples available. More to come!) 81 | 82 | 83 | ### Note: ### 84 | 85 | If you want to run the server in a terminal and get the log output, move to the `startup` folder and execute: 86 | 87 | - `python -m run_async.run_server` 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /run_async/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Main Configuration Settings 3 | """ 4 | # Author: Valerio Maggio 5 | # Copyright (c) 2015 Valerio Maggio 6 | # License: BSD 3 clause 7 | 8 | JS_ROLE = 'JS' 9 | PY_ROLE = 'PYTHON' 10 | 11 | SERVER_ADDR = '127.0.0.1' 12 | SERVER_PORT = 5678 13 | 14 | # Separator String for WebSocket connections 15 | CONNECTION_ID_SEP = '---' 16 | 17 | LIGHT_HTML_OUTPUT_CELL = ''' 18 | 

 19 | 
 20 | '''
 21 | 
 22 | # Not Yet Used
 23 | CSS_CODE = '''
 24 | 
 25 | 
 89 | '''
 90 | JS_WEBSOCKET_CODE = '''
 91 | 
 92 | 
170 | 
171 | '''
172 | 
173 | EXEC_OUTPUT = 'exec_output'
174 | 
175 | # List of names to be excluded from pickling during the async process
176 | DEFAULT_BLACKLIST = ['__builtin__', '__builtins__', '__doc__',
177 |                      '__loader__', '__name__', '__package__',
178 |                      '__spec__', '_sh', 'exit', 'quit', 'MyMagics',
179 |                      'AsyncRunMagic', 'Magics', 'cmagic', 'magics_class']


--------------------------------------------------------------------------------
/examples/async_run magic notebook examples.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": null,
  6 |    "metadata": {
  7 |     "collapsed": false
  8 |    },
  9 |    "outputs": [],
 10 |    "source": [
 11 |     "%async_start_server"
 12 |    ]
 13 |   },
 14 |   {
 15 |    "cell_type": "markdown",
 16 |    "metadata": {},
 17 |    "source": [
 18 |     "To verify that the server is **actually** running, go to [http://127.0.0.1:5678/ping](http://127.0.0.1:5678/ping).\n",
 19 |     "\n",
 20 |     "This is just a ping test"
 21 |    ]
 22 |   },
 23 |   {
 24 |    "cell_type": "markdown",
 25 |    "metadata": {},
 26 |    "source": [
 27 |     "## Very first run"
 28 |    ]
 29 |   },
 30 |   {
 31 |    "cell_type": "code",
 32 |    "execution_count": null,
 33 |    "metadata": {
 34 |     "collapsed": false
 35 |    },
 36 |    "outputs": [],
 37 |    "source": [
 38 |     "N = 10\n",
 39 |     "N"
 40 |    ]
 41 |   },
 42 |   {
 43 |    "cell_type": "code",
 44 |    "execution_count": null,
 45 |    "metadata": {
 46 |     "collapsed": false
 47 |    },
 48 |    "outputs": [],
 49 |    "source": [
 50 |     "s = 5"
 51 |    ]
 52 |   },
 53 |   {
 54 |    "cell_type": "code",
 55 |    "execution_count": null,
 56 |    "metadata": {
 57 |     "collapsed": false
 58 |    },
 59 |    "outputs": [],
 60 |    "source": [
 61 |     "s"
 62 |    ]
 63 |   },
 64 |   {
 65 |    "cell_type": "code",
 66 |    "execution_count": null,
 67 |    "metadata": {
 68 |     "collapsed": true
 69 |    },
 70 |    "outputs": [],
 71 |    "source": [
 72 |     "l = 0"
 73 |    ]
 74 |   },
 75 |   {
 76 |    "cell_type": "markdown",
 77 |    "metadata": {},
 78 |    "source": [
 79 |     "#### Update the value of a variable, asynchronously"
 80 |    ]
 81 |   },
 82 |   {
 83 |    "cell_type": "code",
 84 |    "execution_count": null,
 85 |    "metadata": {
 86 |     "collapsed": false
 87 |    },
 88 |    "outputs": [],
 89 |    "source": [
 90 |     "%%async_run \n",
 91 |     "N = 10**s\n",
 92 |     "print(N)"
 93 |    ]
 94 |   },
 95 |   {
 96 |    "cell_type": "markdown",
 97 |    "metadata": {},
 98 |    "source": [
 99 |     "### The following cell has no output, just computation"
100 |    ]
101 |   },
102 |   {
103 |    "cell_type": "code",
104 |    "execution_count": null,
105 |    "metadata": {
106 |     "collapsed": false
107 |    },
108 |    "outputs": [],
109 |    "source": [
110 |     "%%async_run\n",
111 |     "s += 1\n",
112 |     "l += 1"
113 |    ]
114 |   },
115 |   {
116 |    "cell_type": "code",
117 |    "execution_count": null,
118 |    "metadata": {
119 |     "collapsed": false
120 |    },
121 |    "outputs": [],
122 |    "source": [
123 |     "s, l"
124 |    ]
125 |   },
126 |   {
127 |    "cell_type": "code",
128 |    "execution_count": null,
129 |    "metadata": {
130 |     "collapsed": false
131 |    },
132 |    "outputs": [],
133 |    "source": [
134 |     "l += 1"
135 |    ]
136 |   },
137 |   {
138 |    "cell_type": "code",
139 |    "execution_count": null,
140 |    "metadata": {
141 |     "collapsed": false
142 |    },
143 |    "outputs": [],
144 |    "source": [
145 |     "l"
146 |    ]
147 |   },
148 |   {
149 |    "cell_type": "markdown",
150 |    "metadata": {},
151 |    "source": [
152 |     "## 1. Output History is correctly updated"
153 |    ]
154 |   },
155 |   {
156 |    "cell_type": "code",
157 |    "execution_count": null,
158 |    "metadata": {
159 |     "collapsed": false
160 |    },
161 |    "outputs": [],
162 |    "source": [
163 |     "_oh"
164 |    ]
165 |   },
166 |   {
167 |    "cell_type": "markdown",
168 |    "metadata": {},
169 |    "source": [
170 |     "## 2. Exception Output Handling\n",
171 |     "\n",
172 |     "In this particular example, the `L` variable has been never defined, thus leading to a `NameError` exception"
173 |    ]
174 |   },
175 |   {
176 |    "cell_type": "code",
177 |    "execution_count": null,
178 |    "metadata": {
179 |     "collapsed": false
180 |    },
181 |    "outputs": [],
182 |    "source": [
183 |     "%async_run L += 1"
184 |    ]
185 |   },
186 |   {
187 |    "cell_type": "markdown",
188 |    "metadata": {},
189 |    "source": [
190 |     "## 3. Let's do an heavy computation example"
191 |    ]
192 |   },
193 |   {
194 |    "cell_type": "code",
195 |    "execution_count": null,
196 |    "metadata": {
197 |     "collapsed": false,
198 |     "scrolled": false
199 |    },
200 |    "outputs": [],
201 |    "source": [
202 |     "%%async_run \n",
203 |     "N = 10**7\n",
204 |     "s = sum([i**2 for i in range(N)])\n",
205 |     "print(s)"
206 |    ]
207 |   },
208 |   {
209 |    "cell_type": "code",
210 |    "execution_count": null,
211 |    "metadata": {
212 |     "collapsed": false
213 |    },
214 |    "outputs": [],
215 |    "source": [
216 |     "s"
217 |    ]
218 |   },
219 |   {
220 |    "cell_type": "markdown",
221 |    "metadata": {},
222 |    "source": [
223 |     "## 4. Module Import in `async_run` cells"
224 |    ]
225 |   },
226 |   {
227 |    "cell_type": "code",
228 |    "execution_count": null,
229 |    "metadata": {
230 |     "collapsed": true
231 |    },
232 |    "outputs": [],
233 |    "source": [
234 |     "from sklearn.datasets import load_iris\n",
235 |     "\n",
236 |     "iris = load_iris()\n",
237 |     "X = iris.data"
238 |    ]
239 |   },
240 |   {
241 |    "cell_type": "code",
242 |    "execution_count": null,
243 |    "metadata": {
244 |     "collapsed": false
245 |    },
246 |    "outputs": [],
247 |    "source": [
248 |     "%%async_run\n",
249 |     "import numpy as np\n",
250 |     "s = np.sum(X)\n",
251 |     "print(s)"
252 |    ]
253 |   },
254 |   {
255 |    "cell_type": "markdown",
256 |    "metadata": {},
257 |    "source": [
258 |     "#### Now the `numpy` module is available in the notebook namespace to be used"
259 |    ]
260 |   },
261 |   {
262 |    "cell_type": "code",
263 |    "execution_count": null,
264 |    "metadata": {
265 |     "collapsed": false
266 |    },
267 |    "outputs": [],
268 |    "source": [
269 |     "a = np.array(range(10))"
270 |    ]
271 |   },
272 |   {
273 |    "cell_type": "code",
274 |    "execution_count": null,
275 |    "metadata": {
276 |     "collapsed": false
277 |    },
278 |    "outputs": [],
279 |    "source": [
280 |     "a"
281 |    ]
282 |   },
283 |   {
284 |    "cell_type": "code",
285 |    "execution_count": null,
286 |    "metadata": {
287 |     "collapsed": false
288 |    },
289 |    "outputs": [],
290 |    "source": [
291 |     "%%async_run\n",
292 |     "s = np.sum(a)"
293 |    ]
294 |   },
295 |   {
296 |    "cell_type": "code",
297 |    "execution_count": null,
298 |    "metadata": {
299 |     "collapsed": false,
300 |     "scrolled": false
301 |    },
302 |    "outputs": [],
303 |    "source": [
304 |     "s"
305 |    ]
306 |   },
307 |   {
308 |    "cell_type": "code",
309 |    "execution_count": null,
310 |    "metadata": {
311 |     "collapsed": false
312 |    },
313 |    "outputs": [],
314 |    "source": [
315 |     "np.sum(range(10))"
316 |    ]
317 |   },
318 |   {
319 |    "cell_type": "code",
320 |    "execution_count": null,
321 |    "metadata": {
322 |     "collapsed": false
323 |    },
324 |    "outputs": [],
325 |    "source": [
326 |     "%async_stop_server"
327 |    ]
328 |   },
329 |   {
330 |    "cell_type": "code",
331 |    "execution_count": null,
332 |    "metadata": {
333 |     "collapsed": true
334 |    },
335 |    "outputs": [],
336 |    "source": []
337 |   }
338 |  ],
339 |  "metadata": {
340 |   "kernelspec": {
341 |    "display_name": "Python 3",
342 |    "language": "python",
343 |    "name": "python3"
344 |   },
345 |   "language_info": {
346 |    "codemirror_mode": {
347 |     "name": "ipython",
348 |     "version": 3
349 |    },
350 |    "file_extension": ".py",
351 |    "mimetype": "text/x-python",
352 |    "name": "python",
353 |    "nbconvert_exporter": "python",
354 |    "pygments_lexer": "ipython3",
355 |    "version": "3.5.2"
356 |   }
357 |  },
358 |  "nbformat": 4,
359 |  "nbformat_minor": 0
360 | }
361 | 


--------------------------------------------------------------------------------
/run_async/async_run_magic.py:
--------------------------------------------------------------------------------
  1 | """IPython Magic to run cells asynchronously
  2 | """
  3 | # Author: Valerio Maggio 
  4 | # Copyright (c) 2015 Valerio Maggio 
  5 | # License: BSD 3 clause
  6 | 
  7 | from __future__ import print_function  # Python 2 compatibility
  8 | 
  9 | import json
 10 | from pickle import dumps as pickle_dumps
 11 | from pickle import loads as pickle_loads
 12 | from pickle import PicklingError
 13 | ## -- Python2
 14 | # from six.moves.urllib.request import URLError, urlopen
 15 | from urllib.request import URLError, urlopen
 16 | from uuid import uuid4
 17 | 
 18 | try:
 19 |     from tornado.websocket import websocket_connect
 20 | except ImportError:
 21 |     pass
 22 | 
 23 | from importlib import import_module
 24 | from inspect import ismodule as inspect_ismodule
 25 | 
 26 | from .settings import JS_ROLE, PY_ROLE, EXEC_OUTPUT
 27 | from .settings import DEFAULT_BLACKLIST
 28 | from .settings import JS_WEBSOCKET_CODE, LIGHT_HTML_OUTPUT_CELL
 29 | from .utils import (strip_ansi_color,
 30 |                     connection_string, format_ws_connection_id)
 31 | 
 32 | from IPython.display import HTML
 33 | from IPython.core.magic import (Magics, magics_class, line_magic,
 34 |                                 line_cell_magic)
 35 | 
 36 | from .run_server import AsyncRunServer
 37 | from threading import Thread
 38 | # used to reliably kill possibly running
 39 | # server process
 40 | import psutil
 41 | 
 42 | 
 43 | # -------------------------
 44 | # IPython (Line/Cell) Magic
 45 | # -------------------------
 46 | 
 47 | class WSConnector:
 48 | 
 49 |     def __init__(self, connection_id, code_to_run, shell):
 50 |         """
 51 |         Parameters
 52 |         ----------
 53 |         connection_id: str
 54 |             The unique ID of the connection to establish on the websocket.
 55 |         code_to_run: str
 56 |             The content of the async cell to run_async_cell_execution
 57 |         shell: `IPython.core.interactiveshell.InteractiveShell`
 58 |             Instance of the current IPython shell running in the
 59 |             notebook.
 60 |         """
 61 |         self.ws_conn = None
 62 |         self.connection_id = connection_id
 63 |         self.cell_source = code_to_run
 64 |         self.shell = shell
 65 |         self.exec_count = shell.execution_count
 66 | 
 67 |     def connect(self):
 68 |         """
 69 |         Creates the connection to the Tornado Web Socket
 70 |         """
 71 |         # 'ws://localhost:5678/ws/'
 72 |         conn_string = connection_string(web_socket=True,
 73 |                                         extra='ws/{}'.format(self.connection_id))
 74 |         websocket_connect(conn_string, callback=self.on_connected,
 75 |                           on_message_callback=self.on_message)
 76 | 
 77 |     def on_connected(self, f):
 78 |         """Callback fired /on_connection/ established.
 79 | 
 80 |         Once the connection to the websocket has been established,
 81 |         all the currenct namespace is pickled and written to the
 82 |         corresponding web_socket connection.
 83 |         """
 84 |         try:
 85 |             ws_conn = f.result()
 86 |             self.ws_conn = ws_conn
 87 |             data = {'connection_id': self.connection_id,
 88 |                     'nb_code_to_run_async': self.cell_source,}
 89 |             msg = json.dumps(data)
 90 |             ws_conn.write_message(message=msg)
 91 |             white_ns = self._pack_namespace()
 92 |             ws_conn.write_message(message=pickle_dumps(white_ns), binary=True)
 93 |         except PicklingError as e:
 94 |             print(str(e))
 95 | 
 96 |     def _pack_namespace(self):
 97 |         """Collect all the /pickable/ objects from the namespace
 98 |         so to pass them to the async execution environment."""
 99 |         white_ns = dict()
100 |         white_ns.setdefault('import_modules', list())
101 |         for k, v in self.shell.user_ns.items():
102 |             if not k in DEFAULT_BLACKLIST:
103 |                 try:
104 |                     if inspect_ismodule(v):
105 |                         white_ns['import_modules'].append((k, v.__name__))
106 |                     else:
107 |                         _ = pickle_dumps({k: v})
108 |                         white_ns[k] = v
109 |                 except PicklingError:
110 |                     continue
111 |                 except Exception:
112 |                     continue
113 |         white_ns['connection_id'] = self.connection_id
114 |         return white_ns
115 | 
116 |     def on_message(self, message):
117 |         """Callback fired /on_message/.
118 | 
119 |         This hand of the web socket (Python side) will be
120 |         fired whenever the asynch execution is
121 |         completed.
122 |         """
123 |         if message is not None:
124 |             msg = dict(pickle_loads(message))
125 |             exec_output = None
126 |             if EXEC_OUTPUT in msg:
127 |                 exec_output = msg.pop(EXEC_OUTPUT)
128 | 
129 |             # Look for modules to Import
130 |             self._check_modules_import(msg)
131 |             # Update Output History
132 |             self._update_output_history(exec_output)
133 |             self.ws_conn.close()
134 | 
135 |     def _check_modules_import(self, msg):
136 |         """
137 |         Check if any module has been imported in the
138 |         async cell. If that is the case, import
139 |         modules again in the current shell to make
140 |         them available in the current namespace.
141 | 
142 |         Parameters
143 |         ----------
144 |         msg : dict
145 |             Message dictionary returned by Async execution
146 |         """
147 |         module_names = msg.pop('import_modules')
148 |         modules = dict()
149 |         if module_names:
150 |             for alias, mname in module_names:
151 |                 module = import_module(mname)
152 |                 modules[alias] = module
153 |         self.shell.user_ns.update(msg)
154 |         if modules:
155 |             self.shell.user_ns.update(modules)
156 | 
157 |     def _update_output_history(self, exec_output):
158 |         """Update the Output history in the current
159 |         IPython Shell.
160 | 
161 |         Parameters
162 |         ----------
163 |         exec_output : str
164 |             Output of the Async execution.
165 |         """
166 |         out_cell_key_in_namespace = '_{}'.format(str(self.exec_count))
167 |         if exec_output:  # Update Output history
168 |             exec_output = strip_ansi_color(exec_output)
169 |             self.shell.user_ns['_oh'][self.exec_count] = exec_output
170 |             self.shell.user_ns[out_cell_key_in_namespace] = exec_output
171 |         else:
172 |             # This is necessary to avoid that `_oh` contains the
173 |             # the `LIGHT_HTML_CODE` as output
174 |             # (which is not even rendered in the notebook)
175 |             _ = self.shell.user_ns['_oh'].pop(self.exec_count, None)
176 |             _ = self.shell.user_ns.pop(out_cell_key_in_namespace, None)
177 | 
178 | 
179 | @magics_class
180 | class AsyncRunMagic(Magics):
181 | 
182 |     def __init__(self, shell, **kwargs):
183 |         super(AsyncRunMagic, self).__init__(shell, **kwargs)
184 |         self._server_process = None
185 |         
186 |     @line_cell_magic
187 |     def async_run(self, line, cell=None):
188 |         """Run code into cell asynchronously
189 |             Usage:\\
190 |               %async_run  (cell content)
191 |         """
192 |         if cell is None:
193 |             code_to_run = line
194 |         else:
195 |             code_to_run = cell
196 | 
197 |         session_id = str(uuid4())
198 |         connection_id = format_ws_connection_id(PY_ROLE, session_id)
199 |         try:
200 |             _ = urlopen(connection_string(web_socket=False, extra='ping'))
201 |         except URLError:
202 |             print("Connection to server refused!", end='  ')
203 |             print("Use %async_run_server first!")
204 |         else:
205 |             connector = WSConnector(connection_id, code_to_run, self.shell)
206 |             connector.connect()
207 | 
208 |             html_output = LIGHT_HTML_OUTPUT_CELL.format(session_id=session_id)
209 |             js_code = JS_WEBSOCKET_CODE.replace('__sessionid__', session_id)
210 |             js_code = js_code.replace('__connection_id__', format_ws_connection_id(JS_ROLE,
211 |                                                                                    session_id))
212 |             html_output += js_code
213 |             return HTML(html_output)
214 | 
215 |     def _spawn_server_process(self):
216 |         self._server_process.start()
217 |         print('Process Started with PID ', self._server_process.pid)
218 |         self._server_process.join()
219 | 
220 |     @line_magic
221 |     def async_start_server(self, line):
222 |         if (not self._server_process is None) and (self._server_process.is_alive()):
223 |             print("Cannot Start process twice")
224 |         else:
225 |             self._server_process = AsyncRunServer()
226 |             th_runner = Thread(target=self._spawn_server_process,
227 |                                daemon=True)
228 |             th_runner.start()
229 | 
230 |     @line_magic
231 |     def async_stop_server(self, line):
232 |         if self._server_process is None or not self._server_process.is_alive():
233 |             print('No Server is Running')
234 |         else:
235 |             print("Killing SIGINT to PID ", self._server_process.pid)
236 |             while self._server_process.is_alive():
237 |                 try:
238 |                     p = psutil.Process(self._server_process.pid)
239 |                     p.terminate()  # or p.kill()
240 |                 except ProcessLookupError:
241 |                     pass
242 |             else:
243 |                 self._server_process = None
244 | 
245 | 


--------------------------------------------------------------------------------
/run_async/run_server.py:
--------------------------------------------------------------------------------
  1 | """
  2 | """
  3 | # Author: Valerio Maggio 
  4 | # Copyright (c) 2015 Valerio Maggio 
  5 | # License: BSD 3 clause
  6 | 
  7 | # Tornado Import
  8 | try:
  9 |     from tornado.httpserver import HTTPServer
 10 |     from tornado.ioloop import IOLoop
 11 |     from tornado.web import Application, RequestHandler
 12 |     from tornado.websocket import WebSocketHandler
 13 | except ImportError:
 14 |     WebSocketHandler = RequestHandler = Application = object
 15 | 
 16 | # Execution
 17 | from multiprocessing import Process as mp_Process
 18 | # -- Python2 Compatibility WARNING
 19 | # Python 2 requires **futures**
 20 | # https://github.com/agronholm/pythonfutures
 21 | from concurrent.futures import ProcessPoolExecutor
 22 | from threading import Thread
 23 | 
 24 | # Shell Namespace restoring
 25 | from inspect import ismodule as inspect_ismodule
 26 | from importlib import import_module
 27 | 
 28 | # Messaging
 29 | import json
 30 | import pickle
 31 | 
 32 | # IPython
 33 | from IPython.utils.io import capture_output
 34 | from IPython.core.interactiveshell import InteractiveShell
 35 | 
 36 | # Handlers and Utils
 37 | from .handlers import (WebSocketConnectionHandler, ResultCache,
 38 |                        ExecutionHandler)
 39 | from .settings import JS_ROLE, PY_ROLE, SERVER_PORT, SERVER_ADDR
 40 | from .utils import parse_ws_connection_id
 41 | 
 42 | 
 43 | def execute_cell(raw_cell, current_ns):
 44 |     """
 45 |     Perform the execution of the async cell
 46 |     """
 47 |     # Create a new InteractiveShell
 48 |     shell = InteractiveShell()
 49 |     # Disable Debugger
 50 |     shell.call_pdb = False
 51 |     shell.pdb = False
 52 | 
 53 |     # Process and Inject in the Namespace imported modules
 54 |     module_names = current_ns.pop('import_modules')
 55 |     modules = {}
 56 |     if module_names:
 57 |         for alias, mname in module_names:
 58 |             module = import_module(mname)
 59 |             modules[alias] = module
 60 |     shell.user_ns.update(current_ns)
 61 |     if modules:
 62 |         shell.user_ns.update(modules)
 63 | 
 64 |     output = ''
 65 |     with capture_output() as io:
 66 |         _ = shell.run_cell(raw_cell,silent=True,
 67 |                            shell_futures=False)
 68 | 
 69 |     # Update Namespace
 70 |     updated_namespace = dict()
 71 |     updated_namespace.setdefault('import_modules', list())
 72 |     for k, v in shell.user_ns.items():
 73 |         try:
 74 |             if inspect_ismodule(v):
 75 |                 updated_namespace['import_modules'].append((k, v.__name__))
 76 |             else:
 77 |                 _ = pickle.dumps({k:v})
 78 |                 updated_namespace[k] = v
 79 |         except TypeError:
 80 |             continue
 81 |         except pickle.PicklingError:
 82 |             continue
 83 |         except AttributeError:
 84 |             continue
 85 | 
 86 |     # if not output:
 87 |     output += io.stdout
 88 |     return output, updated_namespace
 89 | 
 90 | class AsyncRunHandler(WebSocketHandler):
 91 |     """
 92 |     Tornado WebSocket Handlers.
 93 |     This class is responsible to handle the
 94 |     actual communication occuring on the
 95 |     web socket between the (JS) client and
 96 |     (PY) server.
 97 |     """
 98 | 
 99 |     def __init__(self, application, request, **kwargs):
100 |         super(AsyncRunHandler, self).__init__(application,
101 |                                               request, **kwargs)
102 |         self._session_id = ''
103 |         self._code_to_run = None
104 |         self._user_ns = None
105 | 
106 |     # noinspection PyMethodOverriding
107 |     def initialize(self, connection_handler, result_cache, io_loop):
108 |         """Initialize the WebsocketHandler injecting proper handlers
109 |         instances.
110 |         These handlers will be used to store reference to client connections,
111 |         to cache execution results, and to manage
112 |         a system of output queues, respectively.
113 |         """
114 |         self._connection_handler = connection_handler
115 |         self._execution_cache = result_cache
116 |         self._ioloop = io_loop
117 | 
118 |     def check_origin(self, origin):
119 |         return True
120 | 
121 |     def open(self, connection_id):
122 |         """
123 |         """
124 |         print('Connection Opened for: ', connection_id)
125 |         self._connection_id = connection_id
126 |         _, session_id = parse_ws_connection_id(connection_id)
127 |         self._session_id = session_id
128 | 
129 |         # ADD Websocket Connection
130 |         self._connection_handler.add(connection_id, self)
131 | 
132 |     def process_work_completed(self, future):
133 |         """
134 |         Callback injected in Tornado IOLoop to be called
135 |         whenever the future (concurrent.ProcessPoolExecutor) is completed.
136 |         """
137 |         # This output will go to the server stdout
138 |         # to be removed
139 |         print('Future Completed')
140 | 
141 |         # Get Execution results
142 |         output, namespace = future.result()  # potentially blocking call
143 | 
144 |         # Post-execution processing
145 |         data = {'session_id': self._session_id,
146 |                 'output': output}
147 | 
148 |         # ADD Cache Result
149 |         # print('Caching results for ', self.cache_id)
150 |         # FIXME: This does not work if the output includes Images
151 |         # TODO: Try using pickle here, instead of json
152 |         jsonified = json.dumps(data)
153 |         self._execution_cache.add(self._session_id, jsonified)
154 | 
155 |         # Get WebSocket Connection of the client to receive updates in
156 |         # the namespace of the cell
157 |         ws_conn = self._connection_handler.get(self._connection_id)
158 |         if ws_conn:
159 |             # Send to the client the updated namespace
160 |             # Add Execution output to allow for *Output History UPDATE*
161 |             message = {'exec_output': output}
162 |             message.update(namespace)
163 |             bin_message = pickle.dumps(message)
164 |             # Write again on the web socket so to fire JS Client side.
165 |             ws_conn.write_message(bin_message, binary=True)
166 |         else:
167 |             print("No Connection found for ", self._connection_id)
168 | 
169 |     def run_async_cell_execution(self):
170 |         with ProcessPoolExecutor() as executor:
171 |             future = executor.submit(execute_cell, self._code_to_run, self._user_ns)
172 |             self._ioloop.add_future(future, self.process_work_completed)
173 |             # self.process_work_completed(future)
174 | 
175 |     def on_message(self, message):
176 |         """
177 |         Handler method activated every time a new
178 |         message is received on the web socket.
179 |         """
180 |         try:
181 |             data = json.loads(message)
182 |         except TypeError:
183 |             # Binary Message
184 |             data = pickle.loads(message)
185 | 
186 |         connection_id = data.get('connection_id', '')
187 |         role_name, _ = parse_ws_connection_id(connection_id)
188 | 
189 |         if role_name == JS_ROLE:
190 |             # GET Cache Result
191 |             json_data = self._execution_cache.get(connection_id)
192 |             # GET WebSocketConnection
193 |             ws_conn = self._connection_handler.get(connection_id)
194 |             if ws_conn and json_data:
195 |                 ws_conn.write_message(json_data)  # JS Client
196 |             else:
197 |                 print('No connection nor data stored for ', role_name)
198 | 
199 |         elif role_name == PY_ROLE:  # parse the code to run_async_cell_execution
200 |             if 'nb_code_to_run_async' in data:
201 |                 self._code_to_run = data['nb_code_to_run_async']
202 |             else:  # namespace
203 |                 self._user_ns = data
204 | 
205 |             if self._code_to_run and self._user_ns:
206 |                 # Start the execution of the cell
207 |                 print("Starting Execution")
208 |                 # t = Thread(target=self.run_async_cell_execution)
209 |                 # t.start()
210 |                 self.run_async_cell_execution()
211 |         else:
212 |             print('No Action found for Role: ', role_name)
213 | 
214 |     def on_close(self):
215 |         # REMOVE WebSocketConnection
216 |         print('Closing Connection for ', self._connection_id)
217 |         self._connection_handler.remove(self._connection_id)
218 | 
219 | 
220 | class PingRequestHandler(RequestHandler):
221 |     """Dummy Request Handler used to test
222 |     connectivity to webserver"""
223 |     def get(self):
224 |         self.write("Server is Up'n'Running!")
225 | 
226 | 
227 | class AsyncRunServer(mp_Process):
228 |     """The main `multiprocessing.Process` class
229 |     controlling the execution of the
230 |     Asynch Server running.
231 | 
232 |     This class is in charge to handle
233 |     references to the IO Loop (Tornado Loop
234 |     so far) and the Http Server.
235 |     """
236 | 
237 |     def __init__(self):
238 |         super(AsyncRunServer, self).__init__()
239 |         self.io_loop = None
240 |         self.http_server = None
241 | 
242 |     def run(self):
243 |         #logging.basicConfig(filename='runserver.log',level=logging.DEBUG)
244 | 
245 |         IOLoop.clear_current()
246 |         IOLoop.clear_instance()
247 |         self.io_loop = IOLoop.instance()
248 | 
249 |         ws_connection_handler = WebSocketConnectionHandler()
250 |         results_cache = ResultCache()
251 |         tornado_app = Application(handlers=[
252 |             (r"/ws/(.*)", AsyncRunHandler, {'connection_handler': ws_connection_handler,
253 |                                             'result_cache': results_cache,
254 |                                             'io_loop': self.io_loop,
255 |                                             }),
256 |             (r"/ping", PingRequestHandler)])
257 |         self.http_server = HTTPServer(tornado_app)
258 |         try:
259 |             self.http_server.listen(port=SERVER_PORT,
260 |                                     address=SERVER_ADDR)
261 |             if not self.io_loop._running:
262 |                 print('Running Server Loop')
263 |                 self.io_loop.start()
264 |             else:
265 |                 print("IOLoop already running")
266 |         except OSError:
267 |             print("Server is already running!")
268 |         except KeyboardInterrupt:  # SIGINT, SIGTERM
269 |             print('Closing Server Loop')
270 |             self.http_server.close_all_connections()
271 |             self.io_loop.stop()
272 | 
273 | 
274 | if __name__ == '__main__':
275 | 
276 |     server = AsyncRunServer()
277 |     try:
278 |         server.start()
279 |         server.join()
280 |     except KeyboardInterrupt:
281 |         pass
282 | 
283 | 


--------------------------------------------------------------------------------
/examples/iPython (default) Run Magic Test.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "## Sample Notebook to see how the `%run` iPython magic works\n",
  8 |     "\n",
  9 |     "**Note**: If this will work as expected, this could be a good way to proceed with \n",
 10 |     "heavy executions to run in notebooks (on likely \"remote\" machines)."
 11 |    ]
 12 |   },
 13 |   {
 14 |    "cell_type": "markdown",
 15 |    "metadata": {
 16 |     "collapsed": true
 17 |    },
 18 |    "source": [
 19 |     "```python\n",
 20 |     "    TARGET_SCRIPT = 'script2run'  # script2run.py file will be executed\n",
 21 |     "```"
 22 |    ]
 23 |   },
 24 |   {
 25 |    "cell_type": "code",
 26 |    "execution_count": null,
 27 |    "metadata": {
 28 |     "collapsed": false
 29 |    },
 30 |    "outputs": [],
 31 |    "source": [
 32 |     "%run script2run"
 33 |    ]
 34 |   },
 35 |   {
 36 |    "cell_type": "markdown",
 37 |    "metadata": {},
 38 |    "source": [
 39 |     "### Checking Namespace\n",
 40 |     "\n",
 41 |     "See whether the namespace of the iPython notebook has been populated with variables declared in the script"
 42 |    ]
 43 |   },
 44 |   {
 45 |    "cell_type": "code",
 46 |    "execution_count": null,
 47 |    "metadata": {
 48 |     "collapsed": false
 49 |    },
 50 |    "outputs": [],
 51 |    "source": [
 52 |     "print(dummy_global.name)"
 53 |    ]
 54 |   },
 55 |   {
 56 |    "cell_type": "code",
 57 |    "execution_count": null,
 58 |    "metadata": {
 59 |     "collapsed": false
 60 |    },
 61 |    "outputs": [],
 62 |    "source": [
 63 |     "print(dummy_main.name)"
 64 |    ]
 65 |   },
 66 |   {
 67 |    "cell_type": "markdown",
 68 |    "metadata": {},
 69 |    "source": [
 70 |     "## Now test Heavyweight computations\n",
 71 |     "\n",
 72 |     "Now try to execute a script whose execution takes some time, in order to see if the control comes back to the notebook after the `%run` magic"
 73 |    ]
 74 |   },
 75 |   {
 76 |    "cell_type": "code",
 77 |    "execution_count": null,
 78 |    "metadata": {
 79 |     "collapsed": false
 80 |    },
 81 |    "outputs": [],
 82 |    "source": [
 83 |     "%run -t script2run --mode=heavy"
 84 |    ]
 85 |   },
 86 |   {
 87 |    "cell_type": "code",
 88 |    "execution_count": null,
 89 |    "metadata": {
 90 |     "collapsed": false
 91 |    },
 92 |    "outputs": [],
 93 |    "source": [
 94 |     "print(dummy_main.name)"
 95 |    ]
 96 |   },
 97 |   {
 98 |    "cell_type": "markdown",
 99 |    "metadata": {},
100 |    "source": [
101 |     "### Testing the `iPythonPexpect` extension"
102 |    ]
103 |   },
104 |   {
105 |    "cell_type": "code",
106 |    "execution_count": null,
107 |    "metadata": {
108 |     "collapsed": true
109 |    },
110 |    "outputs": [],
111 |    "source": [
112 |     "import os"
113 |    ]
114 |   },
115 |   {
116 |    "cell_type": "code",
117 |    "execution_count": null,
118 |    "metadata": {
119 |     "collapsed": false
120 |    },
121 |    "outputs": [],
122 |    "source": [
123 |     "install_ext = False\n",
124 |     "if not os.path.exists('/Users/valerio/.ipython/extensions/ipythonPexpect.py'):\n",
125 |     "    print('ipythonPexpect must be installed')\n",
126 |     "else:\n",
127 |     "    print('Extension already installed')"
128 |    ]
129 |   },
130 |   {
131 |    "cell_type": "code",
132 |    "execution_count": null,
133 |    "metadata": {
134 |     "collapsed": true
135 |    },
136 |    "outputs": [],
137 |    "source": [
138 |     "%install_ext https://cdcvs.fnal.gov/redmine/projects/ipython_ext/repository/revisions/master/raw/ipythonPexpect.py"
139 |    ]
140 |   },
141 |   {
142 |    "cell_type": "code",
143 |    "execution_count": null,
144 |    "metadata": {
145 |     "collapsed": false
146 |    },
147 |    "outputs": [],
148 |    "source": [
149 |     "%load_ext ipythonPexpect"
150 |    ]
151 |   },
152 |   {
153 |    "cell_type": "code",
154 |    "execution_count": null,
155 |    "metadata": {
156 |     "collapsed": false
157 |    },
158 |    "outputs": [],
159 |    "source": [
160 |     "%pexpect_spawn_python"
161 |    ]
162 |   },
163 |   {
164 |    "cell_type": "code",
165 |    "execution_count": null,
166 |    "metadata": {
167 |     "collapsed": true
168 |    },
169 |    "outputs": [],
170 |    "source": [
171 |     "N = 10**7"
172 |    ]
173 |   },
174 |   {
175 |    "cell_type": "code",
176 |    "execution_count": null,
177 |    "metadata": {
178 |     "collapsed": true
179 |    },
180 |    "outputs": [],
181 |    "source": [
182 |     "s = sum([i**3 for i in range(N)])"
183 |    ]
184 |   },
185 |   {
186 |    "cell_type": "code",
187 |    "execution_count": null,
188 |    "metadata": {
189 |     "collapsed": false
190 |    },
191 |    "outputs": [],
192 |    "source": [
193 |     "%pexpect_close"
194 |    ]
195 |   },
196 |   {
197 |    "cell_type": "markdown",
198 |    "metadata": {
199 |     "collapsed": true
200 |    },
201 |    "source": [
202 |     "## Python `multiprocessing`: `Pool.apply_async`"
203 |    ]
204 |   },
205 |   {
206 |    "cell_type": "code",
207 |    "execution_count": null,
208 |    "metadata": {
209 |     "collapsed": true
210 |    },
211 |    "outputs": [],
212 |    "source": [
213 |     "import multiprocessing as mp"
214 |    ]
215 |   },
216 |   {
217 |    "cell_type": "code",
218 |    "execution_count": null,
219 |    "metadata": {
220 |     "collapsed": false
221 |    },
222 |    "outputs": [],
223 |    "source": [
224 |     "print(mp.cpu_count())"
225 |    ]
226 |   },
227 |   {
228 |    "cell_type": "code",
229 |    "execution_count": null,
230 |    "metadata": {
231 |     "collapsed": false
232 |    },
233 |    "outputs": [],
234 |    "source": [
235 |     "from script2run import main_heavy"
236 |    ]
237 |   },
238 |   {
239 |    "cell_type": "code",
240 |    "execution_count": null,
241 |    "metadata": {
242 |     "collapsed": true
243 |    },
244 |    "outputs": [],
245 |    "source": [
246 |     "pool = mp.Pool(processes=mp.cpu_count())"
247 |    ]
248 |   },
249 |   {
250 |    "cell_type": "code",
251 |    "execution_count": null,
252 |    "metadata": {
253 |     "collapsed": true
254 |    },
255 |    "outputs": [],
256 |    "source": [
257 |     "r = pool.apply_async(main_heavy)"
258 |    ]
259 |   },
260 |   {
261 |    "cell_type": "code",
262 |    "execution_count": null,
263 |    "metadata": {
264 |     "collapsed": false
265 |    },
266 |    "outputs": [],
267 |    "source": [
268 |     "output = r.get()\n",
269 |     "print(output)"
270 |    ]
271 |   },
272 |   {
273 |    "cell_type": "markdown",
274 |    "metadata": {},
275 |    "source": [
276 |     "#### Trying get the name of current notebook"
277 |    ]
278 |   },
279 |   {
280 |    "cell_type": "code",
281 |    "execution_count": null,
282 |    "metadata": {
283 |     "collapsed": false
284 |    },
285 |    "outputs": [],
286 |    "source": [
287 |     "from ipykernel import get_connection_info\n",
288 |     "import json\n",
289 |     "\n",
290 |     "info = json.loads(get_connection_info())\n",
291 |     "kernel_id = info['key']\n",
292 |     "\n",
293 |     "info"
294 |    ]
295 |   },
296 |   {
297 |    "cell_type": "code",
298 |    "execution_count": null,
299 |    "metadata": {
300 |     "collapsed": false
301 |    },
302 |    "outputs": [],
303 |    "source": [
304 |     "kernel_id"
305 |    ]
306 |   },
307 |   {
308 |    "cell_type": "code",
309 |    "execution_count": null,
310 |    "metadata": {
311 |     "collapsed": false
312 |    },
313 |    "outputs": [],
314 |    "source": [
315 |     "import os\n",
316 |     "os.getcwd()"
317 |    ]
318 |   },
319 |   {
320 |    "cell_type": "code",
321 |    "execution_count": null,
322 |    "metadata": {
323 |     "collapsed": false
324 |    },
325 |    "outputs": [],
326 |    "source": [
327 |     "from notebook.notebookapp import list_running_servers\n",
328 |     "\n",
329 |     "for server in list_running_servers():\n",
330 |     "    print(server)"
331 |    ]
332 |   },
333 |   {
334 |    "cell_type": "code",
335 |    "execution_count": null,
336 |    "metadata": {
337 |     "collapsed": false
338 |    },
339 |    "outputs": [],
340 |    "source": [
341 |     "import urllib\n",
342 |     "import json\n",
343 |     "response = urllib.request.urlopen('http://127.0.0.1:8188/api/sessions')\n",
344 |     "encoding = response.read().decode('utf-8')\n",
345 |     "sessions = json.loads(encoding)\n",
346 |     "for sess in sessions:\n",
347 |     "    print(sess)"
348 |    ]
349 |   },
350 |   {
351 |    "cell_type": "code",
352 |    "execution_count": null,
353 |    "metadata": {
354 |     "collapsed": true
355 |    },
356 |    "outputs": [],
357 |    "source": [
358 |     "from ipykernel.kernelapp import IPKernelApp"
359 |    ]
360 |   },
361 |   {
362 |    "cell_type": "code",
363 |    "execution_count": null,
364 |    "metadata": {
365 |     "collapsed": false
366 |    },
367 |    "outputs": [],
368 |    "source": [
369 |     "kernel_app = IPKernelApp.instance()"
370 |    ]
371 |   },
372 |   {
373 |    "cell_type": "code",
374 |    "execution_count": null,
375 |    "metadata": {
376 |     "collapsed": false
377 |    },
378 |    "outputs": [],
379 |    "source": [
380 |     "kernel_app.connection_file"
381 |    ]
382 |   },
383 |   {
384 |    "cell_type": "code",
385 |    "execution_count": null,
386 |    "metadata": {
387 |     "collapsed": false
388 |    },
389 |    "outputs": [],
390 |    "source": [
391 |     "kernel_app.connection_file"
392 |    ]
393 |   },
394 |   {
395 |    "cell_type": "code",
396 |    "execution_count": null,
397 |    "metadata": {
398 |     "collapsed": false
399 |    },
400 |    "outputs": [],
401 |    "source": [
402 |     "ipy_kernel = kernel_app.kernel\n",
403 |     "ipy_kernel.execution_count"
404 |    ]
405 |   },
406 |   {
407 |    "cell_type": "code",
408 |    "execution_count": null,
409 |    "metadata": {
410 |     "collapsed": false
411 |    },
412 |    "outputs": [],
413 |    "source": [
414 |     "ipy_kernel.shell.reset()"
415 |    ]
416 |   },
417 |   {
418 |    "cell_type": "markdown",
419 |    "metadata": {},
420 |    "source": [
421 |     "### Testing Notebook APIs"
422 |    ]
423 |   },
424 |   {
425 |    "cell_type": "code",
426 |    "execution_count": null,
427 |    "metadata": {
428 |     "collapsed": false
429 |    },
430 |    "outputs": [],
431 |    "source": [
432 |     "from nbformat import current_nbformat\n",
433 |     "\n",
434 |     "current_nbformat"
435 |    ]
436 |   },
437 |   {
438 |    "cell_type": "code",
439 |    "execution_count": null,
440 |    "metadata": {
441 |     "collapsed": false
442 |    },
443 |    "outputs": [],
444 |    "source": [
445 |     "from nbformat import v4 as nbf\n",
446 |     "from nbformat import read as nbf_read"
447 |    ]
448 |   },
449 |   {
450 |    "cell_type": "code",
451 |    "execution_count": null,
452 |    "metadata": {
453 |     "collapsed": true
454 |    },
455 |    "outputs": [],
456 |    "source": [
457 |     "from nbformat import NO_CONVERT"
458 |    ]
459 |   },
460 |   {
461 |    "cell_type": "code",
462 |    "execution_count": null,
463 |    "metadata": {
464 |     "collapsed": true
465 |    },
466 |    "outputs": [],
467 |    "source": [
468 |     "from nbformat import current_nbformat"
469 |    ]
470 |   },
471 |   {
472 |    "cell_type": "code",
473 |    "execution_count": null,
474 |    "metadata": {
475 |     "collapsed": false
476 |    },
477 |    "outputs": [],
478 |    "source": [
479 |     "current_nbformat"
480 |    ]
481 |   },
482 |   {
483 |    "cell_type": "code",
484 |    "execution_count": null,
485 |    "metadata": {
486 |     "collapsed": false
487 |    },
488 |    "outputs": [],
489 |    "source": [
490 |     "from nbformat import versions"
491 |    ]
492 |   },
493 |   {
494 |    "cell_type": "code",
495 |    "execution_count": null,
496 |    "metadata": {
497 |     "collapsed": false
498 |    },
499 |    "outputs": [],
500 |    "source": [
501 |     "versions"
502 |    ]
503 |   },
504 |   {
505 |    "cell_type": "code",
506 |    "execution_count": null,
507 |    "metadata": {
508 |     "collapsed": false
509 |    },
510 |    "outputs": [],
511 |    "source": []
512 |   },
513 |   {
514 |    "cell_type": "code",
515 |    "execution_count": null,
516 |    "metadata": {
517 |     "collapsed": false
518 |    },
519 |    "outputs": [],
520 |    "source": [
521 |     "nb = nbf_read('iPython Run Magic Test.ipynb', as_version=4)"
522 |    ]
523 |   },
524 |   {
525 |    "cell_type": "code",
526 |    "execution_count": null,
527 |    "metadata": {
528 |     "collapsed": false
529 |    },
530 |    "outputs": [],
531 |    "source": [
532 |     "code_cells = list(filter(lambda c: c['cell_type'] == 'code', nb.cells))\n",
533 |     "code_cells"
534 |    ]
535 |   },
536 |   {
537 |    "cell_type": "code",
538 |    "execution_count": null,
539 |    "metadata": {
540 |     "collapsed": false
541 |    },
542 |    "outputs": [],
543 |    "source": [
544 |     "new_output_cell = nbf.new_output('stream', text='Output Text!')\n",
545 |     "new_code_cell = nbf.new_code_cell(source=\"print('Output Text!)\")"
546 |    ]
547 |   },
548 |   {
549 |    "cell_type": "code",
550 |    "execution_count": null,
551 |    "metadata": {
552 |     "collapsed": true
553 |    },
554 |    "outputs": [],
555 |    "source": [
556 |     "last_cell = code_cells[-1]"
557 |    ]
558 |   },
559 |   {
560 |    "cell_type": "code",
561 |    "execution_count": null,
562 |    "metadata": {
563 |     "collapsed": false
564 |    },
565 |    "outputs": [],
566 |    "source": [
567 |     "last_cell"
568 |    ]
569 |   },
570 |   {
571 |    "cell_type": "code",
572 |    "execution_count": null,
573 |    "metadata": {
574 |     "collapsed": true
575 |    },
576 |    "outputs": [],
577 |    "source": [
578 |     "last_cell.source = \"print('Output Text!)\"\n",
579 |     "last_cell.outputs.append(new_output_cell)\n",
580 |     "last_cell.metadata['collapsed'] = False"
581 |    ]
582 |   },
583 |   {
584 |    "cell_type": "code",
585 |    "execution_count": null,
586 |    "metadata": {
587 |     "collapsed": false
588 |    },
589 |    "outputs": [],
590 |    "source": [
591 |     "from notebook.notebookapp import NotebookApp"
592 |    ]
593 |   },
594 |   {
595 |    "cell_type": "code",
596 |    "execution_count": null,
597 |    "metadata": {
598 |     "collapsed": false
599 |    },
600 |    "outputs": [],
601 |    "source": [
602 |     "import notebook"
603 |    ]
604 |   },
605 |   {
606 |    "cell_type": "code",
607 |    "execution_count": null,
608 |    "metadata": {
609 |     "collapsed": false
610 |    },
611 |    "outputs": [],
612 |    "source": [
613 |     "notebook.services.kernels.kernelmanager.getcwd()"
614 |    ]
615 |   },
616 |   {
617 |    "cell_type": "code",
618 |    "execution_count": null,
619 |    "metadata": {
620 |     "collapsed": true
621 |    },
622 |    "outputs": [],
623 |    "source": []
624 |   }
625 |  ],
626 |  "metadata": {
627 |   "kernelspec": {
628 |    "display_name": "Python 3",
629 |    "language": "python",
630 |    "name": "python3"
631 |   },
632 |   "language_info": {
633 |    "codemirror_mode": {
634 |     "name": "ipython",
635 |     "version": 3
636 |    },
637 |    "file_extension": ".py",
638 |    "mimetype": "text/x-python",
639 |    "name": "python",
640 |    "nbconvert_exporter": "python",
641 |    "pygments_lexer": "ipython3",
642 |    "version": "3.5.2"
643 |   }
644 |  },
645 |  "nbformat": 4,
646 |  "nbformat_minor": 0
647 | }
648 | 


--------------------------------------------------------------------------------