├── screenshot.jpg ├── client.py ├── README.org └── ob-ipython.el /screenshot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregsexton/ob-ipython/HEAD/screenshot.jpg -------------------------------------------------------------------------------- /client.py: -------------------------------------------------------------------------------- 1 | import jupyter_client as client 2 | import threading 3 | import json 4 | import sys 5 | import argparse 6 | 7 | find_connection_file = client.find_connection_file 8 | 9 | semaphore = threading.Semaphore(value=0) 10 | interested_lock = threading.Lock() 11 | interested = [] 12 | 13 | def msg_router(io, shell): 14 | while True: 15 | msg = io() 16 | msg['channel'] = 'io' 17 | msgid = msg['parent_header'].get('msg_id', None) 18 | with interested_lock: 19 | if msgid not in interested: 20 | continue 21 | print(json.dumps(msg, default=str)) 22 | if (msg.get('msg_type', '') == 'status' and 23 | msg['content']['execution_state'] == 'idle'): 24 | break 25 | 26 | while True: 27 | msg = shell() 28 | msg['channel'] = 'shell' 29 | msgid = msg['parent_header'].get('msg_id', None) 30 | with interested_lock: 31 | if msgid not in interested: 32 | continue 33 | print(json.dumps(msg, default=str)) 34 | if msg.get('msg_type', '') in ['execute_reply', 35 | 'inspect_reply', 36 | 'complete_reply']: 37 | semaphore.release() 38 | 39 | def create_client(name): 40 | if name.endswith('.json'): 41 | cf = find_connection_file(name) 42 | else: 43 | cf = find_connection_file('emacs-' + name) 44 | c = client.BlockingKernelClient(connection_file=cf) 45 | c.load_connection_file() 46 | c.start_channels() 47 | io, shell = c.get_iopub_msg, c.get_shell_msg 48 | t = threading.Thread(target=msg_router, args=(io, shell)) 49 | t.setDaemon(True) 50 | t.start() 51 | return c 52 | 53 | parser = argparse.ArgumentParser() 54 | parser.add_argument('--conn-file') 55 | parser.add_argument('--execute', action='store_true') 56 | parser.add_argument('--inspect', action='store_true') 57 | parser.add_argument('--complete', action='store_true') 58 | args = parser.parse_args() 59 | 60 | c = create_client(args.conn_file) 61 | 62 | with interested_lock: 63 | if args.execute: 64 | msgid = c.execute(sys.stdin.read(), allow_stdin=False) 65 | interested.append(msgid) 66 | 67 | elif args.inspect: 68 | req = json.loads(sys.stdin.read()) 69 | code = req['code'] 70 | msgid = c.inspect(code, 71 | cursor_pos=req.get('pos', len(code)), 72 | detail_level=req.get('detail', 0)) 73 | interested.append(msgid) 74 | 75 | elif args.complete: 76 | req = json.loads(sys.stdin.read()) 77 | code = req['code'] 78 | pos = req.get('pos', len(code)) 79 | # causes things to hang as kernel doesn't come back with a 80 | # complete_reply 81 | if code[pos-1] in ['\n', '\r']: 82 | sys.exit(0) 83 | msgid = c.complete(code, 84 | cursor_pos=pos) 85 | interested.append(msgid) 86 | 87 | semaphore.acquire() 88 | -------------------------------------------------------------------------------- /README.org: -------------------------------------------------------------------------------- 1 | #+TITLE: Readme 2 | 3 | ** What is this? 4 | 5 | An Emacs library that allows Org mode to evaluate code blocks using 6 | a Jupyter kernel (Python by default). 7 | 8 | ** Screenshot 9 | 10 | [[./screenshot.jpg]] 11 | 12 | ** How do I install this? 13 | 14 | *** First, you need IPython 15 | 16 | Before installing, you'll need Jupyter (>= 1.0) and IPython (>= 17 | 5.0) installed and working. You will also need the [[http://jupyter.readthedocs.org/en/latest/install.html][Jupyter]] console 18 | and client (~jupyter_console~, ~jupyter_client~) libraries. All of 19 | this should be trivially installable using pip. 20 | 21 | *** Install the Emacs plugin 22 | 23 | This package is in MELPA. I recommend installing from there. 24 | 25 | Otherwise, for manual installation, you'll need the following 26 | elisp dependencies first: 27 | 28 | * https://github.com/magnars/dash.el 29 | * Including dash-functional 30 | * https://github.com/magnars/s.el 31 | * https://github.com/rejeep/f.el 32 | 33 | Then just drop this somewhere in your load path and ~(require 34 | 'ob-ipython)~. 35 | 36 | Lastly, activate ~ipython~ in Org-Babel: 37 | 38 | #+BEGIN_SRC emacs-lisp 39 | (org-babel-do-load-languages 40 | 'org-babel-load-languages 41 | '((ipython . t) 42 | ;; other languages.. 43 | )) 44 | #+END_SRC 45 | 46 | ** How do I use it? 47 | 48 | Open an org file, add a SRC block and evaluate as you would any Org 49 | SRC block (usually =C-c C-c=). Here I will run through some example 50 | blocks. 51 | 52 | This is the most basic ipython block. You *must* provide a session 53 | argument. You can name the session if you wish to separate state. 54 | You can also pass a connection json of an existing ipython session 55 | as a session name in order to connect to it. 56 | 57 | The result returned by ob-ipython should be renderable by org so 58 | it's recommended to always use ~:results raw drawer~. 59 | 60 | #+BEGIN_SRC org 61 | ,#+BEGIN_SRC ipython :session :results raw drawer 62 | %matplotlib inline 63 | import matplotlib.pyplot as plt 64 | import numpy as np 65 | ,#+END_SRC 66 | #+END_SRC 67 | 68 | Here we evaluate some code with a function definition using a named 69 | session. 70 | 71 | #+BEGIN_SRC org 72 | ,#+BEGIN_SRC ipython :session mysession :exports both :results raw drawer 73 | def foo(x): 74 | return x + 9 75 | 76 | [foo(x) + 7 for x in range(7)] 77 | ,#+END_SRC 78 | 79 | ,#+RESULTS: 80 | : [16, 17, 18, 19, 20, 21, 22] 81 | #+END_SRC 82 | 83 | To get a graphic out, you must ensure that you have evaluated 84 | ~%matplotlib inline~ first. A file will be generated for you (see 85 | the ~ob-ipython-resources-dir~ custom var if you want to change the 86 | path). 87 | 88 | #+BEGIN_SRC org 89 | ,#+BEGIN_SRC ipython :session :exports both :results raw drawer 90 | plt.hist(np.random.randn(20000), bins=200) 91 | ,#+END_SRC 92 | #+END_SRC 93 | 94 | If you provide an ipyfile argument, this filename will be used 95 | instead of generating one. 96 | 97 | #+BEGIN_SRC org 98 | ,#+BEGIN_SRC ipython :session :ipyfile /tmp/image.png :exports both :results raw drawer 99 | plt.hist(np.random.randn(20000), bins=200) 100 | ,#+END_SRC 101 | #+END_SRC 102 | 103 | In order to make an svg graphic rather than a png, you may specify 104 | the output format globally to IPython. 105 | 106 | #+BEGIN_EXAMPLE 107 | %config InlineBackend.figure_format = 'svg' 108 | #+END_EXAMPLE 109 | 110 | If you wish to use a specific Jupyter kernel, you may pass the 111 | kernel option. This enables you to use ob-ipython with languages 112 | other than Python. You need to have the Jupyter kernel installed 113 | and working before you can use this. 114 | 115 | When mixing code from different languages you will need to make use 116 | of the session argument. 117 | 118 | #+BEGIN_SRC org 119 | ,#+BEGIN_SRC ipython :session :kernel clojure 120 | (+ 1 2) 121 | ,#+END_SRC 122 | 123 | ,#+RESULTS: 124 | : 3 125 | #+END_SRC 126 | 127 | ob-ipython supports providing variables and even tables to code. 128 | 129 | #+BEGIN_SRC org 130 | ,#+TBLNAME: data_table 131 | | a | 1 | 2 | 132 | | b | 2 | 3 | 133 | | c | 3 | 4 | 134 | 135 | ,#+BEGIN_SRC ipython :session :exports both :var x=2 :var data=data_table 136 | (x, data) 137 | ,#+END_SRC 138 | 139 | ,#+RESULTS: 140 | : (2, [['a', 1, 2], ['b', 2, 3], ['c', 3, 4]]) 141 | #+END_SRC 142 | 143 | Asynchronous execution is supported. Use the ~:async t~ option. 144 | 145 | #+BEGIN_SRC org 146 | ,#+BEGIN_SRC ipython :session :ipyfile /tmp/image.png :exports both :async t :results raw drawer 147 | import time 148 | time.sleep(3) 149 | plt.hist(np.random.randn(20000), bins=200) 150 | ,#+END_SRC 151 | #+END_SRC 152 | 153 | ** Experimental: Jupyter kernel support 154 | 155 | This package is starting to transition from the original 156 | ipython-only support to full jupyter support. 157 | 158 | If you have other kernels installed, you should be able to evaluate 159 | blocks by providing jupyter-X as the language, where X is the 160 | language name recognised by jupyter. For example, you can do 161 | something like this: 162 | 163 | #+BEGIN_SRC org 164 | ,#+BEGIN_SRC jupyter-R :results raw drawer 165 | x <- 3 166 | x 167 | ,#+END_SRC 168 | #+END_SRC 169 | 170 | Notice, when providing languages like this, you do not need to 171 | (although you may) provide a session argument. A default session is 172 | created per language. This should also try to provide support for 173 | per-language modes when editing. 174 | 175 | ** Working with a remote session 176 | 177 | First, follow the instructions [[https://github.com/ipython/ipython/wiki/Cookbook:-Connecting-to-a-remote-kernel-via-ssh][here]] to get access to a remote kernel. 178 | You can then pass the name of the local json file as a session arg to 179 | use this tunnel. 180 | 181 | Essentially the instructions boil down to 182 | 183 | - scp the remote json file over (will be in =/run/user/*/jupyter/=) 184 | - use =jupyter console --existing ./kernel-remote.json --ssh 185 | remote-host= to connect locally. This will give you a prompt that 186 | should allow you to evaluate expressions locally but have them run 187 | on the remote machine. 188 | - use the newly created local config file printed by the previous step 189 | as the argument to =:session=. Be sure to include the =.json= suffix 190 | so that ob-ipython knows to use this as the config rather than 191 | spawning a new session. 192 | 193 | ** What features are there outside of Org SRC block evaluation? 194 | 195 | * You can ask the running IPython kernel for documentation. Open a 196 | SRC block, place the point on the thing you're interested in and 197 | run =M-x ob-ipython-inspect=. I recommend you bind this to a key. 198 | 199 | * ~ob-ipython-completions~ queries the kernel for completions at a 200 | position. You may use this to hook up any completion mechanism. 201 | We already provide a company backend. With company installed, add 202 | ~(add-to-list 'company-backends 'company-ob-ipython)~ somewhere 203 | in your config. This should then work while editing a src block. 204 | 205 | * It's often easier to play with code using a REPL. With the point 206 | in an ipython SRC block, you can open a REPL connected to the 207 | current kernel by running =C-c C-v C-z=. 208 | 209 | * If evaluated code produces an error, this will be displayed 210 | nicely in a buffer using IPython's traceback support. 211 | 212 | * Stdout/err from code evaluation is displayed in a popup buffer. 213 | This is great for debugging or getting verbose output that is 214 | best left out of documents (e.g progress updates). If you wish to 215 | capture output in your document use the =:results output= SRC 216 | block header. 217 | 218 | * You can interrupt or kill a running kernel. This is helpful if 219 | things get stuck or really broken. See =M-x 220 | ob-ipython-interrupt-kernel= and =M-x ob-ipython-kill-kernel=, 221 | respectively. 222 | 223 | ** Tips and tricks 224 | 225 | Here are a few things I have setup to make life better. These 226 | aren't provided with ob-ipython, but are recommended. 227 | 228 | * Be sure to use ~%matplotlib inline~, otherwise graphics won't work. 229 | 230 | * I use yasnippet to create src blocks. Here is the snippet I use. 231 | It takes care of generating unique file names (when I want one) 232 | so I don't have to think about this. 233 | 234 | #+BEGIN_SRC snippet 235 | # -*- mode: snippet -*- 236 | # name: ipython block 237 | # key: py 238 | # -- 239 | ,#+BEGIN_SRC ipython :session ${1::ipyfile ${2:$$(let ((temporary-file-directory "./")) (make-temp-file "py" nil ".png"))} }:exports ${3:both} 240 | $0 241 | ,#+END_SRC 242 | #+END_SRC 243 | 244 | These days I use this more often: 245 | 246 | #+BEGIN_SRC snippet 247 | ,#+BEGIN_SRC ipython :session :exports both :results raw drawer 248 | $0 249 | ,#+END_SRC 250 | #+END_SRC 251 | 252 | * I use the following Org settings: 253 | 254 | #+BEGIN_SRC emacs-lisp 255 | (setq org-confirm-babel-evaluate nil) ;don't prompt me to confirm everytime I want to evaluate a block 256 | 257 | ;;; display/update images in the buffer after I evaluate 258 | (add-hook 'org-babel-after-execute-hook 'org-display-inline-images 'append) 259 | #+END_SRC 260 | 261 | * Export with the =LaTeX= backend using the =minted= package for 262 | source block highlighting fails for =ipython= blocks by default 263 | with the error 264 | : Error: no lexer for alias 'ipython' found 265 | 266 | To use the =python= lexer for =ipython= blocks, add this setting: 267 | #+BEGIN_SRC emacs-lisp 268 | (add-to-list 'org-latex-minted-langs '(ipython "python")) 269 | #+END_SRC 270 | 271 | # * Install pandoc and anything ipython renders as html will be 272 | # converted to org. This is mostly useful for getting nice tables 273 | # automatically. 274 | 275 | * ob-ipython will display anything back from ipython with the mime 276 | type 'text/org' verbatim. This allows you and others to create 277 | [[https://www.safaribooksonline.com/blog/2014/02/11/altering-display-existing-classes-ipython/][formatters]] that output raw org. For example, drop this in your 278 | ipython startup file to have arrays and dataframes rendered as 279 | org tables: 280 | 281 | #+BEGIN_SRC python 282 | import IPython 283 | from tabulate import tabulate 284 | 285 | class OrgFormatter(IPython.core.formatters.BaseFormatter): 286 | def __call__(self, obj): 287 | try: 288 | return tabulate(obj, headers='keys', 289 | tablefmt='orgtbl', showindex='always') 290 | except: 291 | return None 292 | 293 | ip = get_ipython() 294 | ip.display_formatter.formatters['text/org'] = OrgFormatter() 295 | #+END_SRC 296 | 297 | ** Help, it doesn't work 298 | 299 | First thing to do is check that you have all of the required 300 | dependencies. Several common problems have been resolved in the 301 | project's issues, so take a look there to see if your problem has a 302 | quick fix. Otherwise feel free to cut an issue - I'll do my best to 303 | help. 304 | 305 | ** Alternatives 306 | *** Why not use IPython notebook? 307 | 308 | I tried using the IPython notebook but quickly became frustrated 309 | with trying to write code in a web browser. This provides another 310 | option for creating documents containing executable Python code, 311 | but in Emacs - with everything that entails. 312 | 313 | *** Why not use [[https://millejoh.github.io/emacs-ipython-notebook/][EIN]]? 314 | 315 | EIN is really great. It kept me happy for quite a while but I 316 | started to feel constrained by the cell format of IPython 317 | notebooks. What I really wanted was to embed code in Org 318 | documents. It's hard to compete with Org mode! A few key points in 319 | favour of Org: 320 | 321 | * In my opinion, Org's markup is better than Markdown. 322 | * Org's organisational, editing and navigation facilities are much 323 | better than EIN. 324 | * Org's tables... 325 | * Org can export to multiple formats. 326 | * I like how Org opens a new buffer when editing code so that you 327 | can use a Python major mode rather than trying to handle 328 | multiple major modes in one. 329 | 330 | I also found myself hitting bugs in EIN where evaluation and doc 331 | lookup would just stop working. I regularly had to kill and reopen 332 | buffers or restart the IPython kernel and this was getting 333 | frustrating. 334 | 335 | *** How does this compare to regular Org Python integration (ob-python)? 336 | 337 | I think this is more robust. The executed code is sent to a 338 | running IPython kernel which has an architecture designed for this 339 | purpose. The way ob-python works feels like a bit of a hack. I ran 340 | in to race conditions using ob-python where the Org buffer would 341 | update its results before the Python REPL had finished evaluating 342 | the code block. This is what eventually drove me to write this. 343 | 344 | It's easier to get plots and images out of this. I also provide 345 | several features I missed when using plain ob-python, such as 346 | looking up documentation and getting IPython-style tracebacks when 347 | things go wrong. 348 | 349 | You can also use IPython-specific features such as ~%timeit~. 350 | -------------------------------------------------------------------------------- /ob-ipython.el: -------------------------------------------------------------------------------- 1 | ;;; ob-ipython.el --- org-babel functions for IPython evaluation 2 | 3 | ;; Author: Greg Sexton 4 | ;; Keywords: literate programming, reproducible research 5 | ;; Homepage: http://www.gregsexton.org 6 | ;; Package-Requires: ((s "1.9.0") (dash "2.10.0") (dash-functional "1.2.0") (f "0.17.2") (emacs "24")) 7 | 8 | ;; The MIT License (MIT) 9 | 10 | ;; Copyright (c) 2015 Greg Sexton 11 | 12 | ;; Permission is hereby granted, free of charge, to any person obtaining a copy 13 | ;; of this software and associated documentation files (the "Software"), to deal 14 | ;; in the Software without restriction, including without limitation the rights 15 | ;; to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 | ;; copies of the Software, and to permit persons to whom the Software is 17 | ;; furnished to do so, subject to the following conditions: 18 | 19 | ;; The above copyright notice and this permission notice shall be included in 20 | ;; all copies or substantial portions of the Software. 21 | 22 | ;; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 | ;; IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 | ;; FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 | ;; AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 | ;; LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 | ;; OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 28 | ;; THE SOFTWARE. 29 | 30 | ;;; Commentary: 31 | 32 | ;; Org-Babel support for evaluating Python source code using IPython. 33 | 34 | ;;; Code: 35 | 36 | (require 'ob) 37 | (require 'ob-python) 38 | (require 'dash) 39 | (require 'dash-functional) 40 | (require 's) 41 | (require 'f) 42 | (require 'json) 43 | (require 'python) 44 | (require 'cl) 45 | 46 | ;; variables 47 | 48 | (defcustom ob-ipython-kernel-extra-args '() 49 | "List of extra args to pass when creating a kernel." 50 | :group 'ob-ipython) 51 | 52 | (defcustom ob-ipython-client-path 53 | (f-expand "./client.py" 54 | (or (-when-let (f load-file-name) (f-dirname f)) default-directory)) 55 | "Path to the client script." 56 | :group 'ob-ipython) 57 | 58 | (defcustom ob-ipython-command 59 | "jupyter" 60 | "Command to launch ipython. Usually ipython or jupyter." 61 | :group 'ob-ipython) 62 | 63 | (defcustom ob-ipython-resources-dir "./obipy-resources/" 64 | "Directory where resources (e.g images) are stored so that they 65 | can be displayed.") 66 | 67 | ;; utils 68 | 69 | (defun ob-ipython--write-string-to-file (file string) 70 | (if string 71 | (with-temp-buffer 72 | (let ((require-final-newline nil)) 73 | (insert string) 74 | (write-file file))) 75 | (error "No output was produced to write to a file."))) 76 | 77 | (defun ob-ipython--write-base64-string (file b64-string) 78 | (if b64-string 79 | (with-temp-buffer 80 | (let ((buffer-file-coding-system 'binary) 81 | (require-final-newline nil)) 82 | (insert b64-string) 83 | (base64-decode-region (point-min) (point-max)) 84 | (write-file file))) 85 | (error "No output was produced to write to a file."))) 86 | 87 | (defun ob-ipython--create-traceback-buffer (traceback) 88 | (let ((buf (get-buffer-create "*ob-ipython-traceback*"))) 89 | (with-current-buffer buf 90 | (special-mode) 91 | (let ((inhibit-read-only t)) 92 | (erase-buffer) 93 | (-each traceback 94 | (lambda (line) (insert (format "%s\n" line)))) 95 | (ansi-color-apply-on-region (point-min) (point-max)))) 96 | (pop-to-buffer buf))) 97 | 98 | (defun ob-ipython--create-inspect-buffer (doc) 99 | (let ((buf (get-buffer-create "*ob-ipython-inspect*"))) 100 | (with-current-buffer buf 101 | (special-mode) 102 | (let ((inhibit-read-only t)) 103 | (erase-buffer) 104 | (insert doc) 105 | (ansi-color-apply-on-region (point-min) (point-max)) 106 | (whitespace-cleanup) 107 | (goto-char (point-min)))) 108 | (pop-to-buffer buf))) 109 | 110 | (defun ob-ipython--clear-output-buffer () 111 | (let ((buf (get-buffer-create "*ob-ipython-out*"))) 112 | (with-current-buffer buf 113 | (let ((inhibit-read-only t)) 114 | (erase-buffer))))) 115 | 116 | (defun ob-ipython--output (output append-p) 117 | (when (not (s-blank? output)) 118 | (let ((buf (get-buffer-create "*ob-ipython-out*"))) 119 | (with-current-buffer buf 120 | (special-mode) 121 | (let ((inhibit-read-only t)) 122 | (unless append-p (erase-buffer)) 123 | (when (s-blank? (buffer-string)) (pop-to-buffer buf)) 124 | (let ((p (point))) 125 | (if (= p (point-max)) ;allow tailing 126 | (progn (insert output) 127 | (-when-let (w (get-buffer-window buf 'visible)) 128 | (set-window-point w (point-max)))) 129 | (save-excursion 130 | (goto-char (point-max)) 131 | (insert output))) 132 | (ansi-color-apply-on-region p (point-max)) 133 | ;; this adds some support for control chars 134 | (comint-carriage-motion p (point-max))) 135 | (unless append-p (goto-char (point-min)))))))) 136 | 137 | (defun ob-ipython--dump-error (err-msg) 138 | (with-current-buffer (get-buffer-create "*ob-ipython-debug*") 139 | (special-mode) 140 | (let ((inhibit-read-only t)) 141 | (erase-buffer) 142 | (insert err-msg) 143 | (goto-char (point-min)))) 144 | (error "There was a fatal error trying to process the request. See *ob-ipython-debug*")) 145 | 146 | (defun ob-ipython--generate-file-name (suffix) 147 | (s-concat (make-temp-name ob-ipython-resources-dir) suffix)) 148 | 149 | ;; process management 150 | 151 | (defun ob-ipython--kernel-file (name) 152 | (if (s-ends-with-p ".json" name) 153 | name 154 | (format "emacs-%s.json" name))) 155 | 156 | (defun ob-ipython--kernel-repl-cmd (name) 157 | (list ob-ipython-command "console" "--simple-prompt" "--existing" 158 | (ob-ipython--kernel-file name))) 159 | 160 | ;;; TODO: could setup a default sentinel that outputs error on process 161 | ;;; early termination 162 | (defun ob-ipython--create-process (name cmd) 163 | (let ((buf (get-buffer-create (format "*ob-ipython-%s*" name)))) 164 | (with-current-buffer buf (erase-buffer)) 165 | (apply 'start-process name buf (car cmd) (cdr cmd)))) 166 | 167 | (defun ob-ipython--get-python () 168 | (locate-file (if (eq system-type 'windows-nt) 169 | "python.exe" 170 | (or python-shell-interpreter "python")) 171 | exec-path)) 172 | 173 | (defun ob-ipython--create-kernel (name &optional kernel) 174 | (when (and (not (ignore-errors (process-live-p (get-process (format "kernel-%s" name))))) 175 | (not (s-ends-with-p ".json" name))) 176 | (ob-ipython--create-process 177 | (format "kernel-%s" name) 178 | (append 179 | (list ob-ipython-command "console" "--simple-prompt") 180 | (list "-f" (ob-ipython--kernel-file name)) 181 | (if kernel (list "--kernel" kernel) '()) 182 | ;;should be last in the list of args 183 | ob-ipython-kernel-extra-args)) 184 | (sleep-for 1))) 185 | 186 | (defun ob-ipython--get-kernel-processes () 187 | (let ((procs (-filter (lambda (p) 188 | (s-starts-with? "kernel-" (process-name p))) 189 | (process-list)))) 190 | (-zip (-map (-compose (-partial 's-replace "kernel-" "") 191 | 'process-name) 192 | procs) 193 | procs))) 194 | 195 | (defun ob-ipython--create-repl (name) 196 | (let ((python-shell-completion-native-enable nil) 197 | (cmd (s-join " " (ob-ipython--kernel-repl-cmd name)))) 198 | (if (string= "default" name) 199 | (progn 200 | (run-python cmd nil nil) 201 | (format "*%s*" python-shell-buffer-name)) 202 | (let ((process-name (format "Python:%s" name))) 203 | (get-buffer-process 204 | (python-shell-make-comint cmd process-name nil)) 205 | (format "*%s*" process-name))))) 206 | 207 | ;; kernel management 208 | 209 | (defun ob-ipython--choose-kernel () 210 | (let ((procs (ob-ipython--get-kernel-processes))) 211 | (-> (ido-completing-read "kernel? " (-map 'car procs) nil t) 212 | (assoc procs) 213 | cdr 214 | list))) 215 | 216 | ;;; TODO: make this work on windows 217 | ;;; NOTE: interrupting remote kernel not currently possible, cf https://github.com/jupyter/jupyter_console/issues/150 218 | (defun ob-ipython-interrupt-kernel (proc) 219 | "Interrupt a running kernel. Useful for terminating infinite 220 | loops etc. If things get really desparate try `ob-ipython-kill-kernel'." 221 | (interactive (ob-ipython--choose-kernel)) 222 | (when proc 223 | ;; send SIGINT to "python -m ipykernel_launcher", a child of proc 224 | (let ((proc-name (process-name proc))) 225 | (accept-process-output 226 | ;; get the child pid with pgrep -P 227 | ;; NOTE assumes proc has only 1 child (seems to be true always) 228 | (make-process 229 | :name (concat proc-name "-child") 230 | :command (list "pgrep" "-P" (number-to-string 231 | (process-id proc))) 232 | ;; send SIGINT to child-proc 233 | :filter 234 | (lambda (proc child-proc-id) 235 | (make-process 236 | :name (concat "interrupt-" proc-name) 237 | :command (list "kill" "-2" 238 | (string-trim child-proc-id))))))))) 239 | 240 | (defun ob-ipython-kill-kernel (proc) 241 | "Kill a kernel process. If you then re-evaluate a source block 242 | a new kernel will be started." 243 | (interactive (ob-ipython--choose-kernel)) 244 | (when proc 245 | (delete-process proc) 246 | (message (format "Killed %s" (process-name proc))))) 247 | 248 | ;; evaluation 249 | 250 | (defvar ob-ipython--async-queue nil) 251 | 252 | (defun ob-ipython--enqueue (q x) 253 | (set q (append (symbol-value q) (list x)))) 254 | 255 | (defun ob-ipython--dequeue (q) 256 | (let ((ret (car (symbol-value q)))) 257 | (set q (cdr (symbol-value q))) 258 | ret)) 259 | 260 | (defun ob-ipython--collect-json () 261 | ;; this function assumes that we're in a buffer with the json lines 262 | (let ((json-array-type 'list)) 263 | (let (acc) 264 | (while (not (= (point) (point-max))) 265 | (setq acc (cons (json-read) acc)) 266 | (forward-line)) 267 | (nreverse acc)))) 268 | 269 | (defun ob-ipython--running-p () 270 | (get-process "execute")) 271 | 272 | (defun ob-ipython--run-async (code name callback args) 273 | (let ((proc (ob-ipython--create-process 274 | "execute" 275 | (list (ob-ipython--get-python) 276 | "--" ob-ipython-client-path "--conn-file" name "--execute")))) 277 | ;; TODO: maybe add a way of disabling streaming output? 278 | ;; TODO: cleanup and break out - we parse twice, can we parse once? 279 | (set-process-filter 280 | proc 281 | (lexical-let ((parse-pos 0)) 282 | (lambda (proc output) 283 | ;; not guaranteed to be given lines - we need to handle buffering 284 | (with-current-buffer (process-buffer proc) 285 | (goto-char (point-max)) 286 | (insert output) 287 | (let ((json-array-type 'list)) 288 | (goto-char parse-pos) 289 | (while (not (= (point) (point-max))) 290 | (condition-case nil 291 | (progn (-> (json-read) 292 | list 293 | ob-ipython--extract-output 294 | (ob-ipython--output t)) 295 | (forward-line) 296 | (setq parse-pos (point))) 297 | (error (goto-char (point-max)))))))))) 298 | (set-process-sentinel 299 | proc 300 | (lexical-let ((callback callback) 301 | (args args)) 302 | (lambda (proc state) 303 | (when (not (process-live-p proc)) 304 | (with-current-buffer (process-buffer proc) 305 | (goto-char (point-min)) 306 | (apply callback (-> (ob-ipython--collect-json) 307 | ob-ipython--eval 308 | (cons args)))) 309 | (ob-ipython--maybe-run-async))))) 310 | (process-send-string proc code) 311 | (process-send-string proc "\n") 312 | (process-send-eof proc))) 313 | 314 | (defun ob-ipython--maybe-run-async () 315 | (when (not (ob-ipython--running-p)) 316 | (when-let (val (ob-ipython--dequeue 'ob-ipython--async-queue)) 317 | (cl-destructuring-bind (code name callback args) val 318 | (ob-ipython--run-async code name callback args))))) 319 | 320 | (defun ob-ipython--execute-request-async (code name callback args) 321 | (ob-ipython--enqueue 'ob-ipython--async-queue (list code name callback args)) 322 | (ob-ipython--maybe-run-async)) 323 | 324 | (defun ob-ipython--execute-request (code name) 325 | (with-temp-buffer 326 | (let ((ret (apply 'call-process-region code nil 327 | (ob-ipython--get-python) nil t nil 328 | (list "--" ob-ipython-client-path "--conn-file" name "--execute")))) 329 | (if (> ret 0) 330 | (ob-ipython--dump-error (buffer-string)) 331 | (goto-char (point-min)) 332 | (ob-ipython--collect-json))))) 333 | 334 | (defun ob-ipython--extract-output (msgs) 335 | (->> msgs 336 | (-filter (lambda (msg) (string= "stream" (cdr (assoc 'msg_type msg))))) 337 | (-filter (lambda (msg) (-contains? '("stdout" "stderr") 338 | (->> msg (assoc 'content) 339 | (assoc 'name) 340 | cdr)))) 341 | (-map (lambda (msg) (->> msg (assoc 'content) (assoc 'text) cdr))) 342 | (-reduce 's-concat))) 343 | 344 | (defun ob-ipython--extract-result (msgs) 345 | `((:value . ,(->> msgs 346 | (-filter (lambda (msg) 347 | (s-equals? "execute_result" 348 | (cdr (assoc 'msg_type msg))))) 349 | (-mapcat (lambda (msg) 350 | (->> msg (assoc 'content) (assoc 'data) cdr))))) 351 | (:display . ,(->> msgs 352 | (-filter (lambda (msg) 353 | (s-equals? "display_data" 354 | (cdr (assoc 'msg_type msg))))) 355 | (-mapcat (lambda (msg) 356 | (->> msg (assoc 'content) (assoc 'data) cdr))))))) 357 | 358 | (defun ob-ipython--extract-error (msgs) 359 | (let ((error-content 360 | (->> msgs 361 | (-filter (lambda (msg) (-contains? '("execute_reply" "inspect_reply") 362 | (cdr (assoc 'msg_type msg))))) 363 | car 364 | (assoc 'content) 365 | cdr))) 366 | ;; TODO: this doesn't belong in this abstraction 367 | (ob-ipython--create-traceback-buffer (cdr (assoc 'traceback error-content))) 368 | (format "%s: %s" (cdr (assoc 'ename error-content)) (cdr (assoc 'evalue error-content))))) 369 | 370 | (defun ob-ipython--extract-status (msgs) 371 | (->> msgs 372 | (-filter (lambda (msg) (-contains? '("execute_reply" "inspect_reply" "complete_reply") 373 | (cdr (assoc 'msg_type msg))))) 374 | car 375 | (assoc 'content) 376 | (assoc 'status) 377 | cdr)) 378 | 379 | (defun ob-ipython--extract-execution-count (msgs) 380 | (->> msgs 381 | (-filter (lambda (msg) (-contains? '("execute_reply") 382 | (cdr (assoc 'msg_type msg))))) 383 | car 384 | (assoc 'content) 385 | (assoc 'execution_count) 386 | cdr)) 387 | 388 | (defun ob-ipython--eval (service-response) 389 | (let ((status (ob-ipython--extract-status service-response))) 390 | (cond ((string= "ok" status) `((:result . ,(ob-ipython--extract-result service-response)) 391 | (:output . ,(ob-ipython--extract-output service-response)) 392 | (:exec-count . ,(ob-ipython--extract-execution-count service-response)))) 393 | ((string= "abort" status) (error "Kernel execution aborted.")) 394 | ((string= "error" status) (error (ob-ipython--extract-error service-response)))))) 395 | 396 | ;; inspection 397 | 398 | (defun ob-ipython--inspect-request (code &optional pos detail) 399 | (let ((input (json-encode `((code . ,code) 400 | (pos . ,(or pos (length code))) 401 | (detail . ,(or detail 0))))) 402 | (args (list "--" ob-ipython-client-path 403 | "--conn-file" 404 | (ob-ipython--get-session-from-edit-buffer (current-buffer)) 405 | "--inspect"))) 406 | (with-temp-buffer 407 | (let ((ret (apply 'call-process-region input nil 408 | (ob-ipython--get-python) nil t nil 409 | args))) 410 | (if (> ret 0) 411 | (ob-ipython--dump-error (buffer-string)) 412 | (goto-char (point-min)) 413 | (ob-ipython--collect-json)))))) 414 | 415 | (defun ob-ipython--inspect (code pos) 416 | "Given a piece of code and a point position, return inspection results." 417 | (let* ((resp (ob-ipython--inspect-request code pos 0)) 418 | (status (ob-ipython--extract-status resp))) 419 | (if (string= "ok" status) 420 | (->> resp 421 | (-filter (lambda (msg) 422 | (-contains? '("execute_result" "display_data" "inspect_reply") 423 | (cdr (assoc 'msg_type msg))))) 424 | (-mapcat (lambda (msg) 425 | (->> msg 426 | (assoc 'content) 427 | (assoc 'data) 428 | cdr)))) 429 | (error (ob-ipython--extract-error resp))))) 430 | 431 | (defun ob-ipython-inspect (buffer pos) 432 | "Ask a kernel for documentation on the thing at POS in BUFFER." 433 | (interactive (list (current-buffer) (point))) 434 | (let ((code (with-current-buffer buffer 435 | (buffer-substring-no-properties (point-min) (point-max))))) 436 | (-if-let (result (->> (ob-ipython--inspect code pos) 437 | (assoc 'text/plain) 438 | cdr)) 439 | (ob-ipython--create-inspect-buffer result) 440 | (message "No documentation was found.")))) 441 | 442 | ;; completion 443 | 444 | (defun ob-ipython--complete-request (code &optional pos) 445 | (let ((input (json-encode `((code . ,code) 446 | (pos . ,(or pos (length code)))))) 447 | (args (list "--" ob-ipython-client-path "--conn-file" 448 | (ob-ipython--get-session-from-edit-buffer (current-buffer)) 449 | "--complete"))) 450 | (with-temp-buffer 451 | (let ((ret (apply 'call-process-region input nil 452 | (ob-ipython--get-python) nil t nil 453 | args))) 454 | (if (> ret 0) 455 | (ob-ipython--dump-error (buffer-string)) 456 | (goto-char (point-min)) 457 | (ob-ipython--collect-json)))))) 458 | 459 | (defun ob-ipython-completions (buffer pos) 460 | "Ask a kernel for completions on the thing at POS in BUFFER." 461 | (let* ((code (with-current-buffer buffer 462 | (buffer-substring-no-properties (point-min) (point-max)))) 463 | (resp (ob-ipython--complete-request code pos)) 464 | (status (ob-ipython--extract-status resp))) 465 | (if (not (string= "ok" status)) 466 | '() 467 | (->> resp 468 | (-filter (lambda (msg) 469 | (-contains? '("complete_reply") 470 | (cdr (assoc 'msg_type msg))))) 471 | (-mapcat (lambda (msg) 472 | (->> msg 473 | (assoc 'content) 474 | cdr))))))) 475 | 476 | (defun ob-ipython--company-doc-buffer (doc) 477 | "Make company-suggested doc-buffer with ansi-color support." 478 | (let ((buf (company-doc-buffer doc))) 479 | (with-current-buffer buf 480 | (ansi-color-apply-on-region (point-min) (point-max))) 481 | buf)) 482 | 483 | (defun company-ob-ipython (command &optional arg &rest ignored) 484 | (interactive (list 'interactive)) 485 | (cl-case command 486 | (interactive (company-begin-backend 'company-ob-ipython)) 487 | (prefix (and ob-ipython-mode 488 | (let ((res (ob-ipython-completions (current-buffer) (1- (point))))) 489 | (substring-no-properties (buffer-string) 490 | (cdr (assoc 'cursor_start res)) 491 | (cdr (assoc 'cursor_end res)))))) 492 | (candidates (cons :async (lambda (cb) 493 | (let ((res (ob-ipython-completions 494 | (current-buffer) (1- (point))))) 495 | (funcall cb (cdr (assoc 'matches res))))))) 496 | (sorted t) 497 | (doc-buffer (ob-ipython--company-doc-buffer 498 | (cdr (assoc 'text/plain (ob-ipython--inspect arg (length arg)))))))) 499 | 500 | ;; mode 501 | 502 | (define-minor-mode ob-ipython-mode 503 | "" 504 | nil 505 | " ipy" 506 | '()) 507 | 508 | ;; babel framework 509 | 510 | (add-to-list 'org-src-lang-modes '("ipython" . python)) 511 | (add-hook 'org-mode-hook 'ob-ipython-auto-configure-kernels) 512 | 513 | (defvar ob-ipython-configured-kernels nil) 514 | 515 | (defun ob-ipython--get-kernels () 516 | "Return a list of available jupyter kernels and their corresponding languages. 517 | The elements of the list have the form (\"kernel\" \"language\")." 518 | (and ob-ipython-command 519 | (let ((kernelspecs (cdar (json-read-from-string 520 | (shell-command-to-string 521 | (s-concat ob-ipython-command " kernelspec list --json")))))) 522 | (-map (lambda (spec) 523 | (cons (symbol-name (car spec)) 524 | (->> (cdr spec) 525 | (assoc 'spec) 526 | cdr 527 | (assoc 'language) 528 | cdr))) 529 | kernelspecs)))) 530 | 531 | (defun ob-ipython--configure-kernel (kernel-lang) 532 | "Configure org mode to use specified kernel." 533 | (let* ((kernel (car kernel-lang)) 534 | (language (cdr kernel-lang)) 535 | (jupyter-lang (concat "jupyter-" language)) 536 | (mode (intern (or (cdr (assoc language org-src-lang-modes)) 537 | (replace-regexp-in-string "[0-9]*" "" language)))) 538 | (header-args (intern (concat "org-babel-default-header-args:" jupyter-lang)))) 539 | (add-to-list 'org-src-lang-modes `(,jupyter-lang . ,mode)) 540 | ;; Only set defaults if the corresponding variable is nil or does not 541 | ;; exist yet. 542 | (unless (and (boundp header-args) (symbol-value header-args)) 543 | (set (intern (concat "org-babel-default-header-args:" jupyter-lang)) 544 | `((:session . ,language) 545 | (:kernel . ,kernel)))) 546 | (defalias (intern (concat "org-babel-execute:" jupyter-lang)) 547 | 'org-babel-execute:ipython) 548 | (defalias (intern (concat "org-babel-" jupyter-lang "-initiate-session")) 549 | 'org-babel-ipython-initiate-session) 550 | kernel-lang)) 551 | 552 | (defun ob-ipython-auto-configure-kernels (&optional replace) 553 | "Auto-configure kernels for use with org-babel based on the 554 | available kernelspecs of the current jupyter installation. If 555 | REPLACE is non-nil, force configuring the kernels even if they 556 | have previously been configured." 557 | (interactive (list t)) 558 | (when (or replace (not ob-ipython-configured-kernels)) 559 | (setq ob-ipython-configured-kernels 560 | (-map 'ob-ipython--configure-kernel (ob-ipython--get-kernels))))) 561 | 562 | (defvar org-babel-default-header-args:ipython '()) 563 | 564 | (defun org-babel-edit-prep:ipython (info) 565 | ;; TODO: based on kernel, should change the major mode 566 | (ob-ipython--create-kernel (->> info (nth 2) (assoc :session) cdr 567 | ob-ipython--normalize-session) 568 | (->> info (nth 2) (assoc :kernel) cdr)) 569 | (ob-ipython-mode +1)) 570 | 571 | (defun ob-ipython--normalize-session (session) 572 | (if (string= "default" session) 573 | (error "default is reserved for when no name is provided. Please use a different session name.") 574 | (or session "default"))) 575 | 576 | (defun ob-ipython--get-session-from-edit-buffer (buffer) 577 | (with-current-buffer buffer 578 | (->> org-src--babel-info 579 | (nth 2) 580 | (assoc :session) 581 | cdr 582 | ob-ipython--normalize-session))) 583 | 584 | (defun org-babel-execute:ipython (body params) 585 | "Execute a block of IPython code with Babel. 586 | This function is called by `org-babel-execute-src-block'." 587 | (ob-ipython--clear-output-buffer) 588 | (if (cdr (assoc :async params)) 589 | (ob-ipython--execute-async body params) 590 | (ob-ipython--execute-sync body params))) 591 | 592 | (defun ob-ipython--execute-async (body params) 593 | (let* ((file (cdr (assoc :ipyfile params))) 594 | (session (cdr (assoc :session params))) 595 | (result-type (cdr (assoc :result-type params))) 596 | (sentinel (ipython--async-gen-sentinel))) 597 | (ob-ipython--create-kernel (ob-ipython--normalize-session session) 598 | (cdr (assoc :kernel params))) 599 | (ob-ipython--execute-request-async 600 | (org-babel-expand-body:generic (encode-coding-string body 'utf-8) 601 | params (org-babel-variable-assignments:python params)) 602 | (ob-ipython--normalize-session session) 603 | (lambda (ret sentinel buffer file result-type) 604 | (let ((replacement (ob-ipython--process-response ret file result-type))) 605 | (ipython--async-replace-sentinel sentinel buffer replacement))) 606 | (list sentinel (current-buffer) file result-type)) 607 | (format "%s - %s" (length ob-ipython--async-queue) sentinel))) 608 | 609 | (defun ob-ipython--execute-sync (body params) 610 | (let* ((file (cdr (assoc :ipyfile params))) 611 | (session (cdr (assoc :session params))) 612 | (result-type (cdr (assoc :result-type params)))) 613 | (ob-ipython--create-kernel (ob-ipython--normalize-session session) 614 | (cdr (assoc :kernel params))) 615 | (-when-let (ret (ob-ipython--eval 616 | (ob-ipython--execute-request 617 | (org-babel-expand-body:generic (encode-coding-string body 'utf-8) 618 | params (org-babel-variable-assignments:python params)) 619 | (ob-ipython--normalize-session session)))) 620 | (ob-ipython--process-response ret file result-type)))) 621 | 622 | (defun ob-ipython--process-response (ret file result-type) 623 | (let ((result (cdr (assoc :result ret))) 624 | (output (cdr (assoc :output ret)))) 625 | (if (eq result-type 'output) 626 | output 627 | (ob-ipython--output output nil) 628 | (s-concat 629 | (format "# Out[%d]:\n" (cdr (assoc :exec-count ret))) 630 | (s-join "\n" (->> (-map (-partial 'ob-ipython--render file) 631 | (list (cdr (assoc :value result)) 632 | (cdr (assoc :display result)))) 633 | (remove-if-not nil))))))) 634 | 635 | (defun ob-ipython--render (file-or-nil values) 636 | (let ((org (lambda (value) value)) 637 | (png (lambda (value) 638 | (let ((file (or file-or-nil (ob-ipython--generate-file-name ".png")))) 639 | (ob-ipython--write-base64-string file value) 640 | (format "[[file:%s]]" file)))) 641 | (svg (lambda (value) 642 | (let ((file (or file-or-nil (ob-ipython--generate-file-name ".svg")))) 643 | (ob-ipython--write-string-to-file file value) 644 | (format "[[file:%s]]" file)))) 645 | (html (lambda (value) 646 | ;; ((eq (car value) 'text/html) 647 | ;; (let ((pandoc (executable-find "pandoc"))) 648 | ;; (and pandoc (with-temp-buffer 649 | ;; (insert value) 650 | ;; (shell-command-on-region 651 | ;; (point-min) (point-max) 652 | ;; (format "%s -f html -t org" pandoc) t t) 653 | ;; (s-trim (buffer-string)))))) 654 | )) 655 | (txt (lambda (value) 656 | (let ((lines (s-lines value))) 657 | (if (cdr lines) 658 | (->> lines 659 | (-map 's-trim) 660 | (s-join "\n ") 661 | (s-concat " ") 662 | (format "#+BEGIN_EXAMPLE\n%s\n#+END_EXAMPLE")) 663 | (s-concat ": " (car lines))))))) 664 | (or (-when-let (val (cdr (assoc 'text/org values))) (funcall org val)) 665 | (-when-let (val (cdr (assoc 'image/png values))) (funcall png val)) 666 | (-when-let (val (cdr (assoc 'image/svg+xml values))) (funcall svg val)) 667 | (-when-let (val (cdr (assoc 'text/plain values))) (funcall txt val))))) 668 | 669 | (defun org-babel-prep-session:ipython (session params) 670 | "Prepare SESSION according to the header arguments in PARAMS. 671 | VARS contains resolved variable references" 672 | ;; c-u c-c c-v c-z 673 | (error "Currently unsupported.")) 674 | 675 | (defun org-babel-load-session:ipython (session body params) 676 | "Load BODY into SESSION." 677 | ;; c-c c-v c-l 678 | (error "Currently unsupported.")) 679 | 680 | (defun org-babel-ipython-initiate-session (&optional session params) 681 | "Create a session named SESSION according to PARAMS." 682 | (if (string= session "none") 683 | (error "ob-ipython currently only supports evaluation using a session. 684 | Make sure your src block has a :session param.") 685 | (when (not (s-ends-with-p ".json" session)) 686 | (ob-ipython--create-kernel (ob-ipython--normalize-session session) 687 | (cdr (assoc :kernel params)))) 688 | (ob-ipython--create-repl (ob-ipython--normalize-session session)))) 689 | 690 | ;; async 691 | 692 | (defun ipython--async-gen-sentinel () 693 | ;; lifted directly from org-id. thanks. 694 | (let ((rnd (md5 (format "%s%s%s%s%s%s%s" 695 | (random) 696 | (current-time) 697 | (user-uid) 698 | (emacs-pid) 699 | (user-full-name) 700 | user-mail-address 701 | (recent-keys))))) 702 | (format "%s-%s-4%s-%s%s-%s" 703 | (substring rnd 0 8) 704 | (substring rnd 8 12) 705 | (substring rnd 13 16) 706 | (format "%x" 707 | (logior 708 | #b10000000 709 | (logand 710 | #b10111111 711 | (string-to-number 712 | (substring rnd 16 18) 16)))) 713 | (substring rnd 18 20) 714 | (substring rnd 20 32)))) 715 | 716 | (defun ipython--async-replace-sentinel (sentinel buffer replacement) 717 | (save-window-excursion 718 | (save-excursion 719 | (save-restriction 720 | (with-current-buffer buffer 721 | (goto-char (point-min)) 722 | (re-search-forward sentinel) 723 | (re-search-backward "\\(call\\|src\\)_\\|^[ \t]*#\\+\\(BEGIN_SRC\\|CALL:\\)") 724 | (org-babel-remove-result) 725 | (org-babel-insert-result 726 | replacement 727 | (cdr (assoc :result-params (nth 2 (org-babel-get-src-block-info))))) 728 | (org-redisplay-inline-images)))))) 729 | 730 | ;; lib 731 | 732 | (provide 'ob-ipython) 733 | 734 | ;;; ob-ipython.el ends here 735 | --------------------------------------------------------------------------------