├── .travis.yml ├── LICENSE ├── README.org ├── docs ├── internals.org ├── matplotlib.org ├── matplotlib_1.png ├── matplotlib_2.png ├── matplotlib_3.png ├── matplotlib_3d.png ├── matplotlib_contour.png ├── matplotlib_scatter.png ├── numpy.org └── scipy.org ├── install.sh ├── py4cl.asd ├── py4cl.py ├── src ├── callpython.lisp ├── config.lisp ├── do-after-load.lisp ├── import-export.lisp ├── lisp-classes.lisp ├── package.lisp ├── python-process.lisp ├── reader.lisp └── writer.lisp └── tests └── tests.lisp /.travis.yml: -------------------------------------------------------------------------------- 1 | language: lisp 2 | dist: xenial 3 | sudo: required 4 | 5 | git: 6 | depth: 3 7 | 8 | env: 9 | matrix: 10 | #- LISP=abcl 11 | #- LISP=allegro 12 | - LISP=sbcl 13 | - LISP=ccl 14 | #- LISP=clisp 15 | #- LISP=ecl 16 | #- LISP=cmucl 17 | 18 | addons: 19 | apt: 20 | packages: &standard_packages 21 | - python-numpy 22 | - python3-numpy 23 | 24 | matrix: 25 | allow_failures: 26 | - env: LISP=clisp 27 | #- env: LISP=ccl 28 | 29 | install: 30 | - ./install.sh 31 | #- curl https://raw.githubusercontent.com/luismbo/cl-travis/master/install.sh | bash 32 | 33 | script: 34 | - cl -e "(in-package :cl-user) 35 | (ql:quickload :py4cl) 36 | (ql:quickload :py4cl/tests) 37 | 38 | (if (or 39 | (let ((report (py4cl/tests:run))) 40 | (when (or (plusp (slot-value report 'clunit::failed)) 41 | (plusp (slot-value report 'clunit::errors))) 42 | (princ report))) 43 | (let ((py4cl:*python-command* \"python3\")) 44 | (let ((report (py4cl/tests:run))) 45 | (when (or (plusp (slot-value report 'clunit::failed)) 46 | (plusp (slot-value report 'clunit::errors))) 47 | (princ report))))) 48 | (uiop:quit 1))" 49 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Ben Dudson 4 | 5 | Parts based on cl4py 6 | Copyright (c) 2018 Marco Heisig 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy 9 | of this software and associated documentation files (the "Software"), to deal 10 | in the Software without restriction, including without limitation the rights 11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | copies of the Software, and to permit persons to whom the Software is 13 | furnished to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice shall be included in 16 | all copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 24 | THE SOFTWARE. 25 | -------------------------------------------------------------------------------- /README.org: -------------------------------------------------------------------------------- 1 | [[https://travis-ci.org/bendudson/py4cl][https://travis-ci.org/bendudson/py4cl.svg?branch=master]] 2 | 3 | * Introduction 4 | 5 | Py4CL is a bridge between Common Lisp and Python, which enables Common 6 | Lisp to interact with Python code. It uses streams to communicate with 7 | a separate python process, the approach taken by [[https://github.com/marcoheisig/cl4py][cl4py]]. This is 8 | different to the CFFI approach used by [[https://github.com/pinterface/burgled-batteries][burgled-batteries]], but has the 9 | same goal. 10 | 11 | ** Installing 12 | 13 | Depends on: 14 | 15 | - Currently tested with SBCL, CCL and ECL (after 2016-09-06). CLISP 16 | doesn't (yet) have =uiop:launch-program=. 17 | - ASDF3 version 3.2.0 (Jan 2017) or later, as =uiop:launch-program= 18 | is used to run and communicate with python asyncronously. 19 | - [[https://common-lisp.net/project/trivial-garbage/][Trivial-garbage]], available through Quicklisp. 20 | - Python 2 or 3 21 | - (optional) The [[http://www.numpy.org/][NumPy]] python library for multidimensional arrays 22 | 23 | Clone this repository into =~/quicklisp/local-projects/= or other 24 | location where it can be found by ASDF: 25 | #+BEGIN_SRC bash 26 | $ git clone https://github.com/bendudson/py4cl.git 27 | #+END_SRC 28 | 29 | then load into Lisp with 30 | #+BEGIN_SRC lisp 31 | (ql:quickload :py4cl) 32 | #+END_SRC 33 | 34 | ** Tests 35 | 36 | Tests use [[https://github.com/tgutu/clunit][clunit]], and run on [[https://travis-ci.org/][Travis]] using [[https://github.com/luismbo/cl-travis][cl-travis]]. Most development 37 | is done under Arch linux with SBCL and Python3. To run the tests 38 | yourself: 39 | #+BEGIN_SRC lisp 40 | (asdf:test-system :py4cl) 41 | #+END_SRC 42 | or 43 | #+BEGIN_SRC lisp 44 | (ql:quickload :py4cl/tests) 45 | (py4cl/tests:run) 46 | #+END_SRC 47 | 48 | * Examples 49 | 50 | Py4CL allows python modules to be imported as Lisp packages, python 51 | functions to be called from lisp, and lisp functions called from 52 | python. In the example below, [[https://www.scipy.org/][SciPy]]'s [[https://docs.scipy.org/doc/scipy/reference/generated/scipy.integrate.odeint.html][odeint]] function is used to 53 | integrate ODEs defined by a Lisp function. The result is a Lisp array, 54 | which is then plotted using the [[https://matplotlib.org/][matplotlib]] plotting library. 55 | 56 | #+BEGIN_SRC lisp 57 | (ql:quickload :py4cl) 58 | 59 | (py4cl:import-module "numpy" :as "np") 60 | (py4cl:import-module "scipy.integrate" :as "integrate") 61 | 62 | ;; Integrate some ODEs 63 | (defparameter *data* 64 | (integrate:odeint 65 | (lambda (y time) 66 | (vector (aref y 1) ; dy[0]/dt = y[1] 67 | (- (aref y 0)))) ; dy[1]/dt = -y[0] 68 | #(1.0 0.0) ; Initial state 69 | (np:linspace 0.0 (* 2 pi) 20))) ; Vector of times 70 | 71 | ; (array-dimensions *data*) => (20 2) 72 | 73 | ;; Make a plot, save and show it in a window 74 | (py4cl:import-module "matplotlib.pyplot" :as "plt") 75 | 76 | (plt:plot *data*) 77 | (plt:xlabel "Time") 78 | (plt:savefig "result.pdf") 79 | (plt:show) 80 | #+END_SRC 81 | 82 | More detailed examples of using python packages using =py4cl=: 83 | - [[./docs/numpy.org][Numpy arrays]] 84 | - [[./docs/matplotlib.org][Matplotlib plotting]] 85 | - [[./docs/scipy.org][Scipy scientific library]] 86 | 87 | * Reference 88 | ** Direct evaluation of python code: =python-eval=, =python-exec= 89 | 90 | For direct access to the python subprocess, =python-eval= 91 | evaluates an expression, converting the result to a suitable lisp 92 | type. Note that there are nicer, more lispy wrappers around this function, 93 | described below, but they are mostly built on top of =python-eval=. 94 | 95 | #+BEGIN_SRC lisp 96 | (asdf:load-system "py4cl") 97 | 98 | (py4cl:python-eval "[i**2 for i in range(5)]") ; => #(0 1 4 9 16) 99 | #+END_SRC 100 | 101 | #+RESULTS: 102 | | 0 | 1 | 4 | 9 | 16 | 103 | 104 | #+BEGIN_SRC lisp 105 | (py4cl:python-eval "{'hello':'world', 'answer':42}") ; => # 106 | #+END_SRC 107 | 108 | #+RESULTS: 109 | : # 110 | 111 | Data is passed between python and lisp as text. The python function 112 | =lispify= converts values to a form which can be read by the lisp 113 | reader; the lisp function =pythonize= outputs strings which can be 114 | =eval='d in python. The following type conversions are done: 115 | 116 | | Lisp type | Python type | 117 | |-----------+--------------------| 118 | | NIL | None | 119 | | integer | int | 120 | | ratio | fractions.Fraction | 121 | | real | float | 122 | | complex | complex float | 123 | | string | str | 124 | | hash map | dict | 125 | | list | tuple | 126 | | vector | list | 127 | | array | NumPy array | 128 | | symbol | Symbol class | 129 | | function | function | 130 | 131 | Note that python does not have all the numerical types which lisp has, 132 | for example complex integers. 133 | 134 | Because =python-eval= and =python-exec= evaluate strings as python 135 | expressions, strings passed to them are not escaped or converted as 136 | other types are. To pass a string to python as an argument, call =py4cl::pythonize= 137 | 138 | #+BEGIN_SRC lisp 139 | (let ((my-str "testing")) 140 | (py4cl:python-eval "len(" (py4cl::pythonize my-str) ")" )) 141 | #+END_SRC 142 | 143 | #+RESULTS: 144 | : 7 145 | 146 | Note that this escaping is done automatically by higher-level interfaces like 147 | =python-call= and =chain=: 148 | #+BEGIN_SRC lisp 149 | (let ((my-str "testing")) 150 | (py4cl:python-call "len" my-str)) 151 | #+END_SRC 152 | 153 | #+RESULTS: 154 | : 7 155 | 156 | #+BEGIN_SRC lisp 157 | (let ((my-str "testing")) 158 | (py4cl:chain (len my-str))) 159 | #+END_SRC 160 | 161 | #+RESULTS: 162 | : 7 163 | 164 | If python objects cannot be converted into a lisp value, then they are 165 | stored and a handle is returned to lisp. This handle can be used to 166 | manipulate the object, and when it is garbage collected the python 167 | object is also deleted (using the [[https://common-lisp.net/project/trivial-garbage/][trivial-garbage]] package). 168 | 169 | #+BEGIN_SRC lisp 170 | (destructuring-bind (fig ax) (plt:subplots) 171 | ;; fig is #S(PY4CL::PYTHON-OBJECT :TYPE "" :HANDLE 6) 172 | (py4cl:python-eval ax ".plot(" #(0 1 0 1) ")") 173 | (plt:show)) 174 | #+END_SRC 175 | 176 | The interface to python objects is nicer using =chain= (see below): 177 | #+BEGIN_SRC lisp 178 | (destructuring-bind (fig ax) (plt:subplots) 179 | (py4cl:chain ax (plot #(0 1 0 1))) 180 | (plt:show)) 181 | #+END_SRC 182 | 183 | The python process can be explicitly started and stopped using 184 | =python-start= and =python-stop=, but =py4cl= functions start python 185 | automatically if needed by calling =python-start-if-not-alive=. 186 | 187 | ** Calling python functions: =python-call= 188 | 189 | =python-call= can be used to pass arguments to any python callable, 190 | such as a function in a module: 191 | 192 | #+BEGIN_SRC lisp 193 | (py4cl:python-exec "import math") 194 | (py4cl:python-call "math.sqrt" 42) 195 | #+END_SRC 196 | 197 | #+RESULTS: 198 | : 6.4807405 199 | 200 | or a lambda function: 201 | #+BEGIN_SRC lisp 202 | (py4cl:python-call "lambda x: 2*x" 21) 203 | #+END_SRC 204 | 205 | #+RESULTS: 206 | : 42 207 | 208 | Keywords are translated, with the symbol made lowercase: 209 | #+BEGIN_SRC lisp 210 | (py4cl:python-call "lambda a=0, b=1: a-b" :b 2 :a 1) 211 | #+END_SRC 212 | 213 | #+RESULTS: 214 | : -1 215 | 216 | ** Calling python methods: =python-method= 217 | 218 | Python methods on objects can be called by using the =python-method= function. The first argument 219 | is the object (including strings, arrays, tuples); the second argument is either a string or a symbol 220 | specifying the method, followed by any arguments: 221 | #+BEGIN_SRC lisp 222 | (py4cl:python-method "hello {0}" 'format "world") ; => "hello world" 223 | #+END_SRC 224 | 225 | #+RESULTS: 226 | : hello world 227 | 228 | #+BEGIN_SRC lisp 229 | (py4cl:python-method '(1 2 3) '__len__) ; => 3 230 | #+END_SRC 231 | 232 | #+RESULTS: 233 | : 3 234 | 235 | ** Getting python attributes: =python-getattr= 236 | 237 | The attributes of a python object can be accessed using the generic 238 | functon =python-getattr=, with the python object as first argument and 239 | a string as the name of the attribute: 240 | #+BEGIN_SRC lisp 241 | (py4cl:python-getattr '(1 2 3) "__doc__") 242 | #+END_SRC 243 | 244 | 245 | Note: Methods for this function can also be defined for lisp classes, 246 | enabling python code to access attributes of lisp objects. See below 247 | for details. 248 | 249 | ** Chaining python methods: =chain= 250 | 251 | In python it is quite common to apply a chain of method calls, data 252 | member access, and indexing operations to an object. To make this work 253 | smoothly in Lisp, there is the =chain= macro (Thanks to @kat-co and 254 | [[https://common-lisp.net/project/parenscript/reference.html][parenscript]] for the inspiration). This consists of a target object, 255 | followed by a chain of operations to apply. For example 256 | #+BEGIN_SRC lisp 257 | (py4cl:chain "hello {0}" (format "world") (capitalize)) ; => "Hello world" 258 | #+END_SRC 259 | 260 | #+RESULTS: 261 | : Hello world 262 | 263 | which is converted to python 264 | #+BEGIN_SRC python 265 | return "hello {0}".format("world").capitalize() 266 | #+END_SRC 267 | 268 | #+RESULTS: 269 | : Hello world 270 | 271 | The only things which are treated specially by this macro are lists 272 | and symbols at the top level. The first element of lists are treated as 273 | python method names, top-level symbols are treated as data 274 | members. Everything else is evaluated as lisp before being converted 275 | to a python value. 276 | 277 | If the first argument is a list, then it is assumed to be a python 278 | function to be called; otherwise it is evaluated before converting to 279 | a python value. For example 280 | #+BEGIN_SRC lisp 281 | (py4cl:chain (slice 3) stop) 282 | #+END_SRC 283 | 284 | #+RESULTS: 285 | : 3 286 | 287 | is converted to the python: 288 | #+BEGIN_SRC python 289 | return slice(3).stop 290 | #+END_SRC 291 | 292 | #+RESULTS: 293 | : 3 294 | 295 | Symbols as first argument, or arguments to python methods, are 296 | evaluated, so the following works: 297 | #+BEGIN_SRC lisp 298 | (let ((format-str "hello {0}") 299 | (argument "world")) 300 | (py4cl:chain format-str (format argument))) ; => "hello world" 301 | #+END_SRC 302 | 303 | #+RESULTS: 304 | : hello world 305 | 306 | Arguments to methods are lisp, since only the top level forms in =chain= are treated specially: 307 | #+BEGIN_SRC lisp 308 | (py4cl:chain "result: {0}" (format (+ 1 2))) ; => "result: 3" 309 | #+END_SRC 310 | 311 | #+RESULTS: 312 | : result: 3 313 | 314 | Indexing with =[]= brackets is commonly used in python, which calls the =__getitem__= method. 315 | This method can be called like any other method 316 | #+BEGIN_SRC lisp 317 | (py4cl:chain "hello" (__getitem__ 4)) ; => "o" 318 | #+END_SRC 319 | 320 | #+RESULTS: 321 | : o 322 | 323 | but since this is a common method an alias =[]= is supported: 324 | #+BEGIN_SRC lisp 325 | (py4cl:chain "hello" ([] 4)) ; => "o" 326 | #+END_SRC 327 | 328 | #+RESULTS: 329 | : o 330 | 331 | which is converted to the python 332 | #+BEGIN_SRC python 333 | return "hello"[4] 334 | #+END_SRC 335 | 336 | #+RESULTS: 337 | : o 338 | 339 | For simple cases where the index is a value like a number or string 340 | (not a symbol or a list), the brackets can be omitted: 341 | #+BEGIN_SRC lisp 342 | (py4cl:chain "hello" 4) ; => "o" 343 | #+END_SRC 344 | 345 | #+RESULTS: 346 | : o 347 | 348 | Slicing can be done by calling the python =slice= function: 349 | #+BEGIN_SRC lisp 350 | (py4cl:chain "hello" ([] (py4cl:python-call "slice" 2 4))) ; => "ll" 351 | #+END_SRC 352 | 353 | #+RESULTS: 354 | : ll 355 | 356 | which could be imported as a lisp function (see below): 357 | #+BEGIN_SRC lisp 358 | (py4cl:import-function "slice") 359 | (py4cl:chain "hello" ([] (slice 2 4))) ; => "ll" 360 | #+END_SRC 361 | 362 | #+RESULTS: 363 | : ll 364 | 365 | This of course also works with multidimensional arrays: 366 | #+BEGIN_SRC lisp 367 | (py4cl:chain #2A((1 2 3) (4 5 6)) ([] 1 (slice 0 2))) ;=> #(4 5) 368 | #+END_SRC 369 | 370 | #+RESULTS: 371 | | 4 | 5 | 372 | 373 | Sometimes the python functions or methods may contain upper case 374 | characters; class names often start with a capital letter. All symbols 375 | are converted to lower case, but the case can be controlled by passing 376 | a string rather than a symbol as the first element: 377 | #+BEGIN_SRC lisp 378 | ;; Define a class 379 | (py4cl:python-exec 380 | "class TestClass: 381 | def doThing(self, value = 42): 382 | return value") 383 | 384 | ;; Create an object and call the method 385 | (py4cl:chain ("TestClass") ("doThing" :value 31)) ; => 31 386 | #+END_SRC 387 | Note that the keyword is converted, converting to lower case. 388 | 389 | ** Printing from python 390 | 391 | Since standard output is used for communication between lisp and python, this is 392 | redirected (to a =StringIO= buffer) while user python code is running. The 393 | output from python functions is then sent to lisp, to be printed to 394 | =*standard-output*=. This means that anything printed by the python process may 395 | only appear in chunks, as it is sent to lisp. The following does however work as 396 | expected: 397 | #+BEGIN_SRC lisp :results output 398 | (py4cl:chain (print "hello world")) 399 | ; => prints "hello world", returns NIL 400 | #+END_SRC 401 | 402 | #+RESULTS: 403 | : hello world 404 | 405 | In python =print_function= is imported from =__future__=, so should be available 406 | as a function in python 2.6+, as well as in version 3+. 407 | 408 | ** Asynchronous python functions: =python-call-async= 409 | 410 | One of the advantages of using streams to communicate with a separate 411 | python process, is that the python and lisp processes can run at the 412 | same time. =python-call-async= calls python but returns a closure 413 | immediately. The python process continues running, and the result can 414 | be retrieved by calling the returned closure. 415 | 416 | #+BEGIN_SRC lisp 417 | (defparameter thunk (py4cl:python-call-async "lambda x: 2*x" 21)) 418 | 419 | (funcall thunk) ; => 42 420 | #+END_SRC 421 | 422 | #+RESULTS: 423 | : 42 424 | 425 | If the function call requires callbacks to lisp, then these will only 426 | be serviced when a =py4cl= function is called. In that case the python 427 | function may not be able to finish until the thunk is called. This 428 | should not result in deadlocks, because all =py4cl= functions can 429 | service callbacks while waiting for a result. 430 | 431 | ** Importing functions: =import-function= 432 | 433 | Python functions can be made available in Lisp by using =import-function=. By 434 | default this makes a function which can take any number of arguments, and then 435 | translates these into a call to the python function. 436 | #+BEGIN_SRC lisp 437 | (asdf:load-system "py4cl") 438 | 439 | (py4cl:python-exec "import math") 440 | (py4cl:import-function "math.sqrt") 441 | (math.sqrt 42) ; => 6.4807405 442 | #+END_SRC 443 | 444 | #+RESULTS: 445 | : 6.4807405 446 | 447 | If a different symbol is needed in Lisp then the =:as= keyword can be 448 | used with either a string or symbol: 449 | #+BEGIN_SRC lisp 450 | (py4cl:import-function "sum" :as "pysum") 451 | (pysum '(1 2 3)) ; => 6 452 | #+END_SRC 453 | 454 | #+RESULTS: 455 | : 6 456 | 457 | This is implemented as a macro which defines a function which in turn calls =python-call=. 458 | 459 | ** Importing modules: =import-module= 460 | 461 | Python modules can be imported as lisp packages using =import-module=. 462 | For example, to import the [[https://matplotlib.org/][matplotlib]] plotting library, and make its functions 463 | available in the package =PLT= from within Lisp: 464 | #+BEGIN_SRC lisp :session import-example 465 | (asdf:load-system "py4cl") 466 | (py4cl:import-module "matplotlib.pyplot" :as "plt") ; Creates PLT package 467 | #+END_SRC 468 | 469 | #+RESULTS: 470 | : T 471 | 472 | This will also import it into the python process as the module =plt=, so that 473 | =python-call= or =python-eval= can also make use of the =plt= module. 474 | 475 | Like =python-exec=, =python-call= and other similar functions, 476 | =import-module= starts python if it is not already running, so that 477 | the available functions can be discovered. 478 | 479 | The python docstrings are made available as Lisp function docstrings, so we can see them 480 | using =describe=: 481 | #+BEGIN_SRC lisp :session import-example 482 | (describe 'plt:plot) 483 | #+END_SRC 484 | 485 | Functions in the =PLT= package can be used to make simple plots: 486 | #+BEGIN_SRC lisp :session import-example 487 | (plt:plot #(1 2 3 2 1) :color "r") 488 | (plt:show) 489 | #+END_SRC 490 | 491 | #+RESULTS: 492 | : NIL 493 | 494 | Notes: 495 | - =import-module= should be used as a top-level form, to ensure that 496 | the package is defined before it is used. 497 | 498 | - If using =import-module= within [[https://orgmode.org/worg/org-contrib/babel/][org-mode babel]] then the import 499 | should be done in a separate code block to the first use of the 500 | imported package, or a condition will be raised like "Package NP 501 | does not exist." 502 | 503 | ** Exporting a function to python: =export-function= 504 | 505 | Lisp functions can be passed as arguments to =python-call= 506 | or imported functions: 507 | #+BEGIN_SRC lisp 508 | (py4cl:python-exec "from scipy.integrate import romberg") 509 | 510 | (py4cl:python-call "romberg" 511 | (lambda (x) (/ (exp (- (* x x))) 512 | (sqrt pi))) 513 | 0.0 1.0) ; Range of integration 514 | #+END_SRC 515 | 516 | #+RESULTS: 517 | : 0.4213504 518 | 519 | Lisp functions can be made available to python code using =export-function=: 520 | #+BEGIN_SRC lisp 521 | (py4cl:python-exec "from scipy.integrate import romberg") 522 | 523 | (py4cl:export-function (lambda (x) (/ (exp (- (* x x))) 524 | (sqrt pi))) "gaussian") 525 | 526 | (py4cl:python-eval "romberg(gaussian, 0.0, 1.0)") ; => 0.4213504 527 | #+END_SRC 528 | 529 | #+RESULTS: 530 | : 0.4213504 531 | 532 | ** Manipulating objects remotely: =remote-objects= 533 | 534 | If a sequence of python functions and methods are being used to manipulate data, 535 | then data may be passed between python and lisp. This is fine for small amounts 536 | of data, but inefficient for large datasets. 537 | 538 | The =remote-objects= and =remote-objects*= macros provide =unwind-protect= environments 539 | in which all python functions return handles rather than values to lisp. This enables 540 | python functions to be combined without transferring much data. 541 | 542 | The difference between these macros is =remote-objects= returns a handle, but 543 | =remote-objects*= evaluates the result, and so will return a value if possible. 544 | 545 | #+BEGIN_SRC lisp 546 | (py4cl:remote-objects (py4cl:python-eval "1+2")) ; => #S(PY4CL::PYTHON-OBJECT :TYPE "" :HANDLE 0) 547 | #+END_SRC 548 | 549 | #+RESULTS: 550 | : #S(PY4CL::PYTHON-OBJECT :TYPE "" :HANDLE 4) 551 | 552 | #+BEGIN_SRC lisp 553 | (py4cl:remote-objects* (py4cl:python-eval "1+2")) ; => 3 554 | #+END_SRC 555 | 556 | #+RESULTS: 557 | : 3 558 | 559 | The advantage comes when dealing with large arrays or other datasets: 560 | #+BEGIN_SRC lisp 561 | (time (np:sum (np:arange 1000000))) 562 | ; => 3.672 seconds of real time 563 | ; 390,958,896 bytes consed 564 | #+END_SRC 565 | 566 | #+BEGIN_SRC lisp 567 | (time (py4cl:remote-objects* (np:sum (np:arange 1000000)))) 568 | ; => 0.025 seconds of real time 569 | ; 32,544 bytes consed 570 | #+END_SRC 571 | ** =setf=-able places 572 | 573 | The =python-eval= function is =setf=-able, so that python objects can 574 | be assigned to by using =setf=. Since =chain= uses =python-eval=, it is also 575 | =setf=-able. This can be used to set elements in an array, entries in a dict/hash-table, 576 | or object data members, for example: 577 | #+BEGIN_SRC lisp 578 | (py4cl:import-module "numpy" :as "np") 579 | #+END_SRC 580 | 581 | #+RESULTS: 582 | : T 583 | 584 | #+BEGIN_SRC lisp 585 | (py4cl:remote-objects* 586 | (let ((array (np:zeros '(2 2)))) 587 | (setf (py4cl:chain array ([] 0 1)) 1.0 588 | (py4cl:chain array ([] 1 0)) -1.0) 589 | array)) 590 | ; => #2A((0.0 1.0) 591 | ; (-1.0 0.0)) 592 | #+END_SRC 593 | 594 | #+RESULTS: 595 | : #2A((0.0 1.0) (-1.0 0.0)) 596 | 597 | Note that this modifies the value in python, so the above example only 598 | works because =array= is a handle to a python object, rather than an 599 | array which is stored in lisp. The following therefore does not work: 600 | #+BEGIN_SRC lisp 601 | (let ((array (np:zeros '(2 2)))) 602 | (setf (py4cl:chain array ([] 0 1)) 1.0 603 | (py4cl:chain array ([] 1 0)) -1.0) 604 | array) 605 | ; => #2A((0.0 0.0) 606 | ; (0.0 0.0)) 607 | #+END_SRC 608 | 609 | #+RESULTS: 610 | : #2A((0.0 0.0) (0.0 0.0)) 611 | 612 | The =np:zeros= function returned an array to lisp; the array was then 613 | sent to python and modified in python. The modified array is not 614 | returned, since this would mean transferring the whole array. If the 615 | value is in lisp then just use the lisp functions: 616 | #+BEGIN_SRC lisp 617 | (let ((array (np:zeros '(2 2)))) 618 | (setf (aref array 0 1) 1.0 619 | (aref array 1 0) -1.0) 620 | array) 621 | ; => #2A((0.0 1.0) 622 | ; (-1.0 0.0)) 623 | #+END_SRC 624 | 625 | #+RESULTS: 626 | : #2A((0.0 1.0) (-1.0 0.0)) 627 | 628 | ** Passing lisp objects to python: =python-getattr= 629 | 630 | Lisp structs and class objects can be passed to python, put into data structures and 631 | returned: 632 | #+BEGIN_SRC lisp 633 | (py4cl:import-function "dict") ; Makes python dictionaries 634 | 635 | (defstruct test-struct 636 | x y) 637 | 638 | (let ((map (dict :key (make-test-struct :x 1 :y 2)))) ; Make a dictionary, return as hash-map 639 | ;; Get the struct from the hash-map, and get the Y slot 640 | (test-struct-y 641 | (py4cl:chain map "key"))) ; => 2 642 | #+END_SRC 643 | 644 | #+RESULTS: 645 | : 2 646 | 647 | In python this is handled using an object of class =UnknownLispObject=, which 648 | contains a handle. The lisp object is stored in a hash map 649 | =*lisp-objects*=. When the python object is deleted, a message is sent to remove 650 | the object from the hash map. 651 | 652 | To enable python to access slots, or call methods on a struct or class, a 653 | handler function needs to be registered. This is done by providing a method 654 | for generic function =python-getattr=. This function will be called when a 655 | python function attempts to access attributes of an object (=__getattr__= 656 | method). 657 | 658 | #+BEGIN_SRC lisp 659 | ;; Define a class with some slots 660 | (defclass test-class () 661 | ((value :initarg :value))) 662 | 663 | ;; Define a method to handle calls from python 664 | (defmethod py4cl:python-getattr ((object test-class) slot-name) 665 | (cond 666 | ((string= slot-name "value") ; data member 667 | (slot-value object 'value)) 668 | ((string= slot-name "func") ; method, return a function 669 | (lambda (arg) (* 2 arg))) 670 | (t (call-next-method)))) ; Otherwise go to next method 671 | 672 | (let ((instance (make-instance 'test-class :value 21))) 673 | ;; Get the value from the slot, call the method 674 | ;; python: instance.func(instance.value) 675 | (py4cl:chain instance (func (py4cl:chain instance value)))) ; => 42 676 | #+END_SRC 677 | 678 | #+RESULTS: 679 | : 42 680 | 681 | Inheritance then works as usual with CLOS methods: 682 | #+BEGIN_SRC lisp 683 | ;; Class inheriting from test-class 684 | (defclass child-class (test-class) 685 | ((other :initarg :other))) 686 | 687 | ;; Define method which passes to the next method if slot not recognised 688 | (defmethod py4cl:python-getattr ((object child-class) slot-name) 689 | (cond 690 | ((string= slot-name "other") 691 | (slot-value object 'other)) 692 | (t (call-next-method)))) 693 | 694 | (let ((object (make-instance 'child-class :value 42 :other 3))) 695 | (list 696 | (py4cl:chain object value) ; Call TEST-CLASS getattr method via CALL-NEXT-METHOD 697 | (py4cl:chain object other))) ;=> (42 3) 698 | #+END_SRC 699 | 700 | #+RESULTS: 701 | | 42 | 3 | 702 | -------------------------------------------------------------------------------- /docs/internals.org: -------------------------------------------------------------------------------- 1 | * Internal details 2 | 3 | ** Messages formats 4 | 5 | Messages consist of a single character, followed by a number (N) in string 6 | format e.g. "12", a newline, then N characters. 7 | 8 | *** Lisp to Python 9 | 10 | The first character of the message describes the type of message: 11 | - `x` means run `exec`, and is used for statements where no return 12 | value is expected e.g. `import` statements 13 | - `e` means run `eval`, to evaluate an expression and return the 14 | result 15 | 16 | *** Python to Lisp 17 | 18 | The first character can be: 19 | - `r` indicates a return value 20 | - `e` indicates an error, followed by a message 21 | - `c` indicates a callback, calling a lisp function and expecting a 22 | value to be sent back to python 23 | 24 | Strings sent to Lisp are processed with `read-from-string` to obtain a 25 | value. 26 | 27 | ** Garbage collection 28 | 29 | When lisp objects are garbage collected, which may be triggered by calling: 30 | #+BEGIN_SRC lisp 31 | (tg:gc :full t) 32 | #+END_SRC 33 | 34 | #+RESULTS: 35 | : NIL 36 | 37 | the finalisation function puts the corresponding python handle into a list 38 | #+BEGIN_SRC lisp 39 | py4cl::*freed-python-objects* 40 | #+END_SRC 41 | 42 | #+RESULTS: 43 | : NIL 44 | 45 | When =python-eval= or =python-exec= are called, these call =delete-freed-python-objects= which 46 | calls python to delete the corresponding python objects from the object store. 47 | To see how many objects are in the store: 48 | #+BEGIN_SRC lisp 49 | (py4cl:python-eval "len(_py4cl_objects)") 50 | #+END_SRC 51 | 52 | #+RESULTS: 53 | : 2 54 | 55 | -------------------------------------------------------------------------------- /docs/matplotlib.org: -------------------------------------------------------------------------------- 1 | * Matplotlib: plotting 2 | 3 | This section was adapted from the [[https://scipy-lectures.org/intro/matplotlib/index.html][Scipy lecture notes]], which was 4 | written by Nicolas Rougier, Mike Müller, and Gaël Varoquaux. 5 | 6 | License: [[http://creativecommons.org/licenses/by/4.0/][Creative Commons Attribution 4.0 International License (CC-by)]] 7 | 8 | Adapted for [[https://github.com/bendudson/py4cl][py4cl]] by B.Dudson (2019). 9 | 10 | The following examples assume that you have already loaded =py4cl= 11 | #+BEGIN_SRC lisp 12 | (ql:quickload :py4cl) 13 | #+END_SRC 14 | 15 | #+RESULTS: 16 | | :PY4CL | 17 | 18 | and then imported python modules as lisp packages using: 19 | #+BEGIN_SRC lisp 20 | (py4cl:import-module "numpy" :as "np") 21 | (py4cl:import-module "matplotlib.pyplot" :as "plt") 22 | #+END_SRC 23 | 24 | #+RESULTS: 25 | : T 26 | 27 | ** Simple plot 28 | 29 | #+CAPTION: Plot of sin and cos in range -pi to pi 30 | #+NAME: fig:simple 31 | [[./matplotlib_1.png]] 32 | 33 | #+BEGIN_SRC lisp 34 | (let* ((x (np:linspace (- pi) pi 256 :endpoint t)) 35 | (c (np:cos x)) 36 | (s (np:sin x))) 37 | 38 | (plt:plot x c) 39 | (plt:plot x s)) 40 | 41 | (plt:show) 42 | #+END_SRC 43 | 44 | #+RESULTS: 45 | : NIL 46 | 47 | ** Instantiating defaults 48 | 49 | #+BEGIN_SRC lisp 50 | ;; Create a figure of size 8x6 inches, 80 dots per inch 51 | (plt:figure :figsize '(8 6) :dpi 80) 52 | 53 | ;; Create a new subplot from a grid of 1x1 54 | (plt:subplot 1 1 1) 55 | 56 | (let* ((x (np:linspace (- pi) pi 256 :endpoint t)) 57 | (c (np:cos x)) 58 | (s (np:sin x))) 59 | 60 | ;; Plot cosine with a blue continuous line of width 1 (pixels) 61 | (plt:plot x c :color "blue" :linewidth 1.0 :linestyle "-") 62 | 63 | ;; Plot sine with a green continuous line of width 1 (pixels) 64 | (plt:plot x s :color "green" :linewidth 1.0 :linestyle "-") 65 | 66 | ;; Set x limits 67 | (plt:xlim -4.0 4.0) 68 | 69 | ;; Set x ticks 70 | (plt:xticks (np:linspace -4 4 9 :endpoint t)) 71 | 72 | ;; Set y limits 73 | (plt:ylim -1.0 1.0) 74 | 75 | ;; Set y ticks 76 | (plt:yticks (np:linspace -1 1 5 :endpoint t)) 77 | 78 | ;; Save figure using 72 dots per inch 79 | ;; (plt:savefig "exercise_2.png" :dpi 72) 80 | 81 | ;; Show result on screen 82 | (plt:show)) 83 | #+END_SRC 84 | 85 | #+RESULTS: 86 | : NIL 87 | 88 | ** Moving spines 89 | 90 | Spines are the lines connecting the axis tick marks and noting the 91 | boundaries of the data area. They can be placed at arbitrary positions 92 | and until now, they were on the border of the axis. We’ll change that 93 | since we want to have them in the middle. Since there are four of them 94 | (top/bottom/left/right), we’ll discard the top and right by setting 95 | their color to none and we’ll move the bottom and left ones to 96 | coordinate 0 in data space coordinates. 97 | 98 | This illustrates the use of =py4cl:chain= to access data members and methods 99 | of python objects. 100 | 101 | #+CAPTION: Plot with spines moved to origin 102 | #+NAME: fig:moving_spines 103 | [[./matplotlib_2.png]] 104 | 105 | #+BEGIN_SRC lisp 106 | (let* ((x (np:linspace (- pi) pi 256 :endpoint t)) 107 | (c (np:cos x)) 108 | (s (np:sin x))) 109 | 110 | (plt:plot x c) 111 | (plt:plot x s)) 112 | 113 | (let ((ax (plt:gca))) ; gca stands for 'get current axis' 114 | (py4cl:chain ax spines "right" (set_color "none")) 115 | (py4cl:chain ax spines "top" (set_color "none")) 116 | 117 | (py4cl:chain ax xaxis (set_ticks_position "bottom")) 118 | (py4cl:chain ax spines "bottom" (set_position '("data" 0))) 119 | 120 | (py4cl:chain ax yaxis (set_ticks_position "left")) 121 | (py4cl:chain ax spines "left" (set_position '("data" 0)))) 122 | 123 | (plt:show) 124 | #+END_SRC 125 | 126 | #+RESULTS: 127 | : NIL 128 | 129 | ** Annotating some points 130 | 131 | Let’s annotate some interesting points using the annotate command. We 132 | chose the 2π/3 value and we want to annotate both the sine and the 133 | cosine. We’ll first draw a marker on the curve as well as a straight 134 | dotted line. Then, we’ll use the annotate command to display some text 135 | with an arrow. 136 | 137 | #+CAPTION: Annotating a plot with labels and arrows 138 | #+NAME: fig:annotation 139 | [[./matplotlib_3.png]] 140 | 141 | #+BEGIN_SRC lisp 142 | (py4cl:import-function "dict") 143 | 144 | (let* ((x (np:linspace (- pi) pi 256 :endpoint t)) 145 | (c (np:cos x)) 146 | (s (np:sin x))) 147 | 148 | (plt:plot x c) 149 | (plt:plot x s)) 150 | 151 | (let ((ax (plt:gca))) ; gca stands for 'get current axis' 152 | (py4cl:chain ax spines "right" (set_color "none")) 153 | (py4cl:chain ax spines "top" (set_color "none")) 154 | 155 | (py4cl:chain ax xaxis (set_ticks_position "bottom")) 156 | (py4cl:chain ax spines "bottom" (set_position '("data" 0))) 157 | 158 | (py4cl:chain ax yaxis (set_ticks_position "left")) 159 | (py4cl:chain ax spines "left" (set_position '("data" 0)))) 160 | 161 | ;; Annotate 162 | 163 | (let ((y (* 2 pi (/ 3)))) 164 | (plt:plot (vector y y) (vector 0 (cos y)) :color "blue" :linewidth 2.5 :linestyle "--") 165 | (plt:scatter (list y) (list (cos y)) 50 :color "blue") 166 | 167 | (plt:annotate "$cos(\\frac{2\\pi}{3})=-\\frac{1}{2}$" :xy (list y (cos y)) :xycoords "data" 168 | :xytext '(-90 -50) :textcoords "offset points" :fontsize 16 169 | :arrowprops (dict :arrowstyle "->" :connectionstyle "arc3,rad=.2")) 170 | 171 | (plt:plot (list y y) (list 0 (sin y)) :color "red" :linewidth 2.5 :linestyle "--") 172 | (plt:scatter (list y) (list (sin y)) 50 :color "red") 173 | 174 | (plt:annotate "$sin(\\frac{2\\pi}{3})=\\frac{\\sqrt{3}}{2}$" 175 | :xy (list y (sin y)) :xycoords "data" 176 | :xytext '(10 30) :textcoords "offset points" :fontsize 16 177 | :arrowprops (dict :arrowstyle "->" :connectionstyle "arc3,rad=.2"))) 178 | 179 | (plt:show) 180 | #+END_SRC 181 | 182 | #+RESULTS: 183 | : NIL 184 | 185 | ** Plotting a scatter of points 186 | 187 | #+CAPTION: A simple example showing how to plot a scatter of points with matplotlib. 188 | #+NAME: fig:scatter 189 | [[./matplotlib_scatter.png]] 190 | 191 | We'll load Numpy's random routines to generate random numbers, 192 | in addition to the =np= and =plt= modules already loaded: 193 | #+BEGIN_SRC lisp 194 | (py4cl:import-module "numpy.random" :as "nprand") 195 | #+END_SRC 196 | 197 | #+RESULTS: 198 | : T 199 | 200 | #+BEGIN_SRC lisp 201 | (let* ((n 1024) 202 | (x (nprand:normal 0 1 n)) 203 | (y (nprand:normal 0 1 n)) 204 | (theta (np:arctan2 y x))) 205 | 206 | (plt:axes #(0.025 0.025 0.95 0.95)) 207 | (plt:scatter x y :s 75 :c theta :alpha 0.5) 208 | 209 | (plt:xlim -1.5 1.5) 210 | (plt:xticks #()) 211 | (plt:ylim -1.5 1.5) 212 | (plt:yticks #())) 213 | 214 | (plt:show) 215 | #+END_SRC 216 | 217 | #+RESULTS: 218 | : NIL 219 | 220 | ** Displaying the contours of a function 221 | 222 | #+CAPTION: An example showing how to display the contours of a function with matplotlib. 223 | #+NAME: fig:contour 224 | [[./matplotlib_contour.png]] 225 | 226 | #+BEGIN_SRC lisp 227 | (defun f (x y) 228 | (* 229 | (+ 1 (* -0.5 x) (expt x 5) (expt y 3)) 230 | (exp (- 0 (* x x) (* y y))))) 231 | 232 | (let* ((n 256) 233 | (xy (np:meshgrid (np:linspace -3 3 n) 234 | (np:linspace -3 3 n))) 235 | (x (aref xy 0)) 236 | (y (aref xy 1))) 237 | 238 | (plt:axes #(0.025 0.025 0.95 0.95)) 239 | 240 | (plt:contourf x y (py4cl:python-call (np:vectorize #'f) x y) 8 :alpha 0.75) 241 | 242 | (let ((c (plt:contour x y (py4cl:python-call (np:vectorize #'f) x y) 8 :colors "black" :linewidth 0.5))) 243 | (plt:clabel c :inline 1 :fontsize 10))) 244 | 245 | (plt:xticks #()) 246 | (plt:yticks #()) 247 | (plt:show) 248 | #+END_SRC 249 | 250 | #+RESULTS: 251 | : NIL 252 | 253 | or a slightly more efficient method, using =remote-objects=: 254 | #+BEGIN_SRC lisp 255 | (defun f (x y) 256 | (* 257 | (+ 1 (* -0.5 x) (expt x 5) (expt y 3)) 258 | (exp (- 0 (* x x) (* y y))))) 259 | 260 | (py4cl:remote-objects 261 | (let* ((n 256) 262 | (xy (np:meshgrid (np:linspace -3 3 n) 263 | (np:linspace -3 3 n))) 264 | (x (py4cl:chain xy 0)) 265 | (y (py4cl:chain xy 1))) 266 | 267 | (plt:axes #(0.025 0.025 0.95 0.95)) 268 | 269 | (plt:contourf x y (py4cl:python-call (np:vectorize #'f) x y) 8 :alpha 0.75) 270 | 271 | (let ((c (plt:contour x y (py4cl:python-call (np:vectorize #'f) x y) 8 :colors "black" :linewidth 0.5))) 272 | (plt:clabel c :inline 1 :fontsize 10)))) 273 | 274 | (plt:xticks #()) 275 | (plt:yticks #()) 276 | (plt:show) 277 | #+END_SRC 278 | 279 | #+RESULTS: 280 | : NIL 281 | 282 | The majority of the time is spent in calling lisp for every point in (x,y). We could 283 | just do the calculation in lisp using =array-operations=: 284 | #+BEGIN_SRC lisp 285 | (ql:quickload :array-operations) 286 | #+END_SRC 287 | 288 | #+RESULTS: 289 | | :ARRAY-OPERATIONS | 290 | 291 | #+BEGIN_SRC lisp 292 | (defun f (x y) 293 | (* 294 | (+ 1 (* -0.5 x) (expt x 5) (expt y 3)) 295 | (exp (- 0 (* x x) (* y y))))) 296 | 297 | (let* ((n 256) 298 | (xy (np:meshgrid (np:linspace -3 3 n) 299 | (np:linspace -3 3 n))) 300 | (x (aref xy 0)) 301 | (y (aref xy 1)) 302 | (z (aops:vectorize (x y) (f x y)))) 303 | 304 | (plt:axes #(0.025 0.025 0.95 0.95)) 305 | 306 | (plt:contourf x y z 8 :alpha 0.75) 307 | 308 | (let ((c (plt:contour x y z 8 :colors "black" :linewidth 0.5))) 309 | (plt:clabel c :inline 1 :fontsize 10))) 310 | 311 | (plt:xticks #()) 312 | (plt:yticks #()) 313 | (plt:show) 314 | #+END_SRC 315 | 316 | #+RESULTS: 317 | : NIL 318 | 319 | ** 3D plots 320 | 321 | #+CAPTION: A simple example of 3D plotting. 322 | #+NAME: fig:3d 323 | [[./matplotlib_3d.png]] 324 | 325 | #+BEGIN_SRC lisp 326 | (py4cl:import-module "numpy" :as "np") 327 | (py4cl:import-module "matplotlib.pyplot" :as "plt") 328 | (py4cl:import-function "Axes3D" :from "mpl_toolkits.mplot3d") 329 | 330 | ;; Load array-operations (aops) for VECTORIZE macro 331 | (ql:quickload :array-operations) 332 | #+END_SRC 333 | 334 | #+RESULTS: 335 | | :ARRAY-OPERATIONS | 336 | 337 | #+BEGIN_SRC lisp 338 | (let* ((fig (plt:figure)) 339 | (ax (Axes3D fig)) 340 | (xy (np:meshgrid (np:arange -4 4 0.25) 341 | (np:arange -4 4 0.25))) 342 | (x (aref xy 0)) 343 | (y (aref xy 1)) 344 | 345 | (r (aops:vectorize (x y) (sqrt (+ (* x x) (* y y))))) 346 | (z (aops:vectorize (r) (sin r)))) 347 | 348 | (py4cl:chain ax (plot_surface x y z :rstride 1 :cstride 1)) 349 | (py4cl:chain ax (contourf x y z :zdir "z" :offset -2)) 350 | (py4cl:chain ax (set_zlim -2 2))) 351 | (plt:show) 352 | #+END_SRC 353 | 354 | #+RESULTS: 355 | : NIL 356 | 357 | -------------------------------------------------------------------------------- /docs/matplotlib_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bendudson/py4cl/e74192eee065b96df7c99e133f6b957dffad24b7/docs/matplotlib_1.png -------------------------------------------------------------------------------- /docs/matplotlib_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bendudson/py4cl/e74192eee065b96df7c99e133f6b957dffad24b7/docs/matplotlib_2.png -------------------------------------------------------------------------------- /docs/matplotlib_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bendudson/py4cl/e74192eee065b96df7c99e133f6b957dffad24b7/docs/matplotlib_3.png -------------------------------------------------------------------------------- /docs/matplotlib_3d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bendudson/py4cl/e74192eee065b96df7c99e133f6b957dffad24b7/docs/matplotlib_3d.png -------------------------------------------------------------------------------- /docs/matplotlib_contour.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bendudson/py4cl/e74192eee065b96df7c99e133f6b957dffad24b7/docs/matplotlib_contour.png -------------------------------------------------------------------------------- /docs/matplotlib_scatter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bendudson/py4cl/e74192eee065b96df7c99e133f6b957dffad24b7/docs/matplotlib_scatter.png -------------------------------------------------------------------------------- /docs/numpy.org: -------------------------------------------------------------------------------- 1 | * NumPy: creating and manipulating numerical data 2 | 3 | This section was adapted from the [[http://scipy-lectures.org/intro/numpy/][Scipy lecture notes]], which was written by 4 | Emmanuelle Gouillart, Didrik Pinte, Gaël Varoquaux, and Pauli Virtanen. 5 | 6 | License: [[http://creativecommons.org/licenses/by/4.0/][Creative Commons Attribution 4.0 International License (CC-by)]] 7 | 8 | Adapted for [[https://github.com/bendudson/py4cl][py4cl]] by B.Dudson (2019). 9 | 10 | The following examples assume that you have already loaded =py4cl= 11 | #+BEGIN_SRC lisp 12 | (ql:quickload :py4cl) 13 | #+END_SRC 14 | and then imported python modules as lisp packages using: 15 | #+BEGIN_SRC lisp 16 | (py4cl:import-module "numpy" :as "np") 17 | #+END_SRC 18 | 19 | ** Functions for creating arrays 20 | 21 | #+BEGIN_SRC lisp 22 | (np:ones '(3 3)) ; => #2A((1.0 1.0 1.0) 23 | ; (1.0 1.0 1.0) 24 | ; (1.0 1.0 1.0)) 25 | #+END_SRC 26 | 27 | #+RESULTS: 28 | : #2A((1.0 1.0 1.0) (1.0 1.0 1.0) (1.0 1.0 1.0)) 29 | 30 | #+BEGIN_SRC lisp 31 | (np:zeros '(2 2)) ; => #2A((0.0 0.0) 32 | ; => (0.0 0.0)) 33 | #+END_SRC 34 | 35 | #+RESULTS: 36 | : #2A((0.0 0.0) (0.0 0.0)) 37 | 38 | #+BEGIN_SRC lisp 39 | (np:arange 10) ; => #(0 1 2 3 4 5 6 7 8 9) 40 | #+END_SRC 41 | 42 | #+RESULTS: 43 | | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 44 | 45 | #+BEGIN_SRC lisp 46 | (np:eye 3) ; => #2A((1.0 0.0 0.0) 47 | ; (0.0 1.0 0.0) 48 | ; (0.0 0.0 1.0)) 49 | #+END_SRC 50 | 51 | #+RESULTS: 52 | : #2A((1.0 0.0 0.0) (0.0 1.0 0.0) (0.0 0.0 1.0)) 53 | 54 | #+BEGIN_SRC lisp 55 | (np:diag #(1 2 3 4)) ; => #2A((1 0 0 0) 56 | ; (0 2 0 0) 57 | ; (0 0 3 0) 58 | ; (0 0 0 4)) 59 | #+END_SRC 60 | 61 | #+RESULTS: 62 | : #2A((1 0 0 0) (0 2 0 0) (0 0 3 0) (0 0 0 4)) 63 | 64 | ** Indexing and slicing 65 | 66 | For creating slices of arrays the python =slice= function can be imported: 67 | #+BEGIN_SRC lisp 68 | (py4cl:import-function "slice") 69 | #+END_SRC 70 | 71 | Using this, arrays can be sliced: 72 | #+BEGIN_SRC lisp 73 | (py4cl:chain (np.arange 10) ([] (slice 2 9 3))) ; (slice start end step) 74 | ; => #(2 5 8) 75 | #+END_SRC 76 | 77 | #+RESULTS: 78 | | 2 | 5 | 8 | 79 | 80 | Note that the last index is not included: 81 | #+BEGIN_SRC lisp 82 | (py4cl:chain (np.arange 10) ([] (slice nil 4))) ; => #(0 1 2 3) 83 | #+END_SRC 84 | 85 | #+RESULTS: 86 | | 0 | 1 | 2 | 3 | 87 | 88 | All three slice components are not required: by default, start is 0, end is the last and step is 1 89 | 90 | Reverse an array 91 | #+BEGIN_SRC lisp 92 | 93 | (py4cl:chain #(1 2 3 4) ([] (slice nil nil -1))) 94 | ; => #(4 3 2 1) 95 | #+END_SRC 96 | 97 | #+RESULTS: 98 | | 4 | 3 | 2 | 1 | 99 | 100 | ** Elementwise operations 101 | 102 | Important: Array multiplication is not matrix multiplication: 103 | #+BEGIN_SRC lisp 104 | (let ((c (np:ones '(3 3)))) 105 | (py4cl:python-eval c "*" c)) 106 | 107 | ; => #2A((1.0 1.0 1.0) 108 | ; (1.0 1.0 1.0) 109 | ; (1.0 1.0 1.0)) 110 | #+END_SRC 111 | 112 | #+RESULTS: 113 | : #2A((1.0 1.0 1.0) (1.0 1.0 1.0) (1.0 1.0 1.0)) 114 | 115 | Matrix multiplication: 116 | #+BEGIN_SRC lisp 117 | (let ((c (np:ones '(3 3)))) 118 | (py4cl:chain c (dot c))) 119 | ; => #2A((3.0 3.0 3.0) 120 | ; (3.0 3.0 3.0) 121 | ; (3.0 3.0 3.0)) 122 | #+END_SRC 123 | 124 | #+RESULTS: 125 | : #2A((3.0 3.0 3.0) (3.0 3.0 3.0) (3.0 3.0 3.0)) 126 | 127 | ** Basic reductions 128 | 129 | Computing sums 130 | #+BEGIN_SRC lisp 131 | (np:sum #(1 2 3 4)) ;=> 10 132 | #+END_SRC 133 | 134 | #+RESULTS: 135 | : 10 136 | 137 | #+BEGIN_SRC lisp 138 | (py4cl:chain (np.array #(1 2 3 4)) (sum)) ; => 10 139 | #+END_SRC 140 | 141 | #+RESULTS: 142 | : 10 143 | 144 | Sum by rows and by columns: 145 | #+BEGIN_SRC lisp 146 | (let ((x #2A((1 1) (2 2)))) 147 | (np:sum x :axis 0)) ; => #(3 3) 148 | #+END_SRC 149 | 150 | #+RESULTS: 151 | | 3 | 3 | 152 | 153 | #+BEGIN_SRC lisp 154 | (let ((x #2A((1 1) (2 2)))) 155 | (py4cl:chain x (sum :axis 1))) ; => #(2 4) 156 | #+END_SRC 157 | 158 | #+RESULTS: 159 | | 2 | 4 | 160 | 161 | Slicing and then summing: 162 | #+BEGIN_SRC lisp 163 | (let ((x #2A((1 1) (2 2)))) 164 | (py4cl:chain x ([] 1 (slice 0 nil)) (sum))) ; => 4 165 | #+END_SRC 166 | 167 | #+RESULTS: 168 | : 4 169 | 170 | 171 | -------------------------------------------------------------------------------- /docs/scipy.org: -------------------------------------------------------------------------------- 1 | * Scipy : high-level scientific computing 2 | 3 | This section was adapted from the [[https://scipy-lectures.org/intro/scipy.html][Scipy lecture notes]], which was written by Gaël 4 | Varoquaux, Adrien Chauve, Andre Espaze, Emmanuelle Gouillart, and Ralf Gommers. 5 | 6 | License: [[http://creativecommons.org/licenses/by/4.0/][Creative Commons Attribution 4.0 International License (CC-by)]] 7 | 8 | Adapted for [[https://github.com/bendudson/py4cl][py4cl]] by B.Dudson (2019). 9 | 10 | The following examples assume that you have already loaded =py4cl= 11 | #+BEGIN_SRC lisp 12 | (ql:quickload :py4cl) 13 | #+END_SRC 14 | 15 | #+RESULTS: 16 | | :PY4CL | 17 | 18 | ** File input/output: =scipy.io= 19 | 20 | *** Matlab files: Loading and saving 21 | 22 | Import the =io= module: 23 | #+BEGIN_SRC lisp 24 | (py4cl:import-module "numpy" :as "np") 25 | (py4cl:import-module "scipy.io" :as "spio") 26 | #+END_SRC 27 | 28 | #+RESULTS: 29 | : T 30 | 31 | #+BEGIN_SRC lisp 32 | (py4cl:import-function "dict") ; Use to make dictionaries (hash tables) 33 | 34 | ;; Create a 3x3 matrix and save to file.mat as "a" 35 | (spio:savemat "file.mat" (dict :a (np:ones '(3 3)))) 36 | 37 | ;; Load file.mat returning a hash table 38 | (let ((data (spio:loadmat "file.mat"))) 39 | (gethash "a" data)) 40 | 41 | ; => #2A((1.0 1.0 1.0) (1.0 1.0 1.0) (1.0 1.0 1.0)) 42 | #+END_SRC 43 | 44 | #+RESULTS: 45 | : #2A((1.0 1.0 1.0) (1.0 1.0 1.0) (1.0 1.0 1.0)) 46 | : T 47 | 48 | ** Linear algebra operations: =scipy.linalg= 49 | 50 | The scipy.linalg module provides standard linear algebra operations, 51 | relying on an underlying efficient implementation (BLAS, LAPACK). 52 | 53 | #+BEGIN_SRC lisp 54 | (py4cl:import-module "scipy.linalg" :as "linalg") 55 | #+END_SRC 56 | 57 | #+RESULTS: 58 | : T 59 | 60 | The =scipy.linalg.det= function computes the determinant of a square matrix: 61 | #+BEGIN_SRC lisp 62 | (linalg:det #2A((1 2) (3 4))) ; => -2.0 63 | #+END_SRC 64 | 65 | #+RESULTS: 66 | : -2.0 67 | 68 | #+BEGIN_SRC lisp 69 | (linalg:det (np:ones '(3 4))) ; => PY4CL:PYTHON-ERROR "expected square matrix" 70 | #+END_SRC 71 | 72 | The =scipy.linalg.inv= function computes the inverse of a square matrix: 73 | #+BEGIN_SRC lisp 74 | (linalg:inv #2A((1 2) 75 | (3 4))) 76 | ; => #2A((-2.0 1.0) 77 | ; (1.5 -0.5)) 78 | #+END_SRC 79 | 80 | #+RESULTS: 81 | : #2A((-2.0 1.0) (1.5 -0.5)) 82 | 83 | #+BEGIN_SRC lisp 84 | (let* ((arr #2A((1 2) 85 | (3 4))) 86 | (iarr (linalg:inv arr))) 87 | (np:allclose (np:dot arr iarr) (np:eye 2))) ; => T 88 | #+END_SRC 89 | 90 | #+RESULTS: 91 | : T 92 | 93 | Finally computing the inverse of a singular matrix (its determinant is 94 | zero) will raise a condition: 95 | #+BEGIN_SRC lisp 96 | (linalg:inv #2A((3 2) (6 4))) ; => PY4CL:PYTHON-ERROR "singular matrix" 97 | #+END_SRC 98 | 99 | More advanced operations are available, for example singular-value 100 | decomposition (SVD): 101 | #+BEGIN_SRC lisp 102 | (let ((arr (py4cl:python-eval 103 | (np:reshape (np:arange 9) '(3 3)) 104 | "+" 105 | (np:diag '(1 0 1))))) 106 | (destructuring-bind (uarr spec vharr) (linalg:svd arr) 107 | spec)) ; => #(14.889826 0.45294237 0.29654968) 108 | #+END_SRC 109 | 110 | #+RESULTS: 111 | | 14.889826 | 0.45294237 | 0.29654968 | 112 | 113 | The original matrix can be re-composed by matrix multiplication of the 114 | outputs of svd with np.dot: 115 | #+BEGIN_SRC lisp 116 | (let ((arr (py4cl:python-eval 117 | (np:reshape (np:arange 9) '(3 3)) 118 | "+" 119 | (np:diag '(1 0 1))))) 120 | (destructuring-bind (uarr spec vharr) (linalg:svd arr) 121 | (let* ((sarr (np:diag spec)) 122 | (svd_mat (py4cl:chain uarr (dot sarr) (dot vharr)))) 123 | (np:allclose svd_mat arr)))) ; => T 124 | #+END_SRC 125 | 126 | #+RESULTS: 127 | : T 128 | 129 | SVD is commonly used in statistics and signal processing. Many other 130 | standard decompositions (QR, LU, Cholesky, Schur), as well as solvers 131 | for linear systems, are available in [[https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg][scipy.linalg]]. 132 | 133 | ** Interpolation: =scipy.interpolate= 134 | 135 | scipy.interpolate is useful for fitting a function from experimental 136 | data and thus evaluating points where no measure exists. The module is 137 | based on the FITPACK Fortran subroutines. 138 | 139 | By imagining experimental data close to a sine function, 140 | scipy.interpolate.interp1d can build a linear interpolation function: 141 | 142 | #+BEGIN_SRC lisp 143 | (py4cl:import-function "interp1d" :from "scipy.interpolate") 144 | 145 | (py4cl:import-module "numpy" :as "np") 146 | (py4cl:import-module "numpy.random" :as "nprandom") 147 | (py4cl:import-module "operator" :as "op") ; For arithmetic 148 | #+END_SRC 149 | 150 | #+RESULTS: 151 | : T 152 | 153 | #+BEGIN_SRC lisp 154 | (py4cl:remote-objects* ; Keep data in python except final result 155 | (let* ((measured_time (np:linspace 0 1 10)) 156 | (noise (op:mul 157 | (op:sub 158 | (op:mul (nprandom:random 10) 2) 159 | 1.0) 160 | 1e-1)) 161 | (measures (op:add 162 | (np:sin (op:mul (* 2 pi) measured_time)) 163 | noise))) 164 | 165 | ;; scipy.interpolate.interp1d can build a linear interpolation function: 166 | (let ((linear_interp (interp1d measured_time measures)) 167 | (interpolation_time (np:linspace 0 1 50))) 168 | ;;Then the result can be evaluated at the time of interest: 169 | (py4cl:python-call linear_interp interpolation_time)))) 170 | #+END_SRC 171 | 172 | #+RESULTS: 173 | | 0.023268929 | 0.15154132 | 0.2798137 | 0.40808612 | 0.53635854 | 0.6646309 | 0.7510545 | 0.80399907 | 0.8569436 | 0.90988815 | 0.96283275 | 1.005534 | 0.96628934 | 0.9270447 | 0.88780004 | 0.8485553 | 0.8093107 | 0.7399575 | 0.65555006 | 0.5711427 | 0.48673522 | 0.40232778 | 0.31176445 | 0.19965541 | 0.08754636 | -0.024562676 | -0.13667172 | -0.24878076 | -0.37039456 | -0.49472398 | -0.6190534 | -0.7433829 | -0.86771226 | -0.9565787 | -0.97451895 | -0.99245924 | -1.0103995 | -1.0283397 | -1.04628 | -0.9682796 | -0.8782866 | -0.78829354 | -0.69830054 | -0.60830754 | -0.5144631 | -0.41580448 | -0.31714582 | -0.2184872 | -0.11982856 | -0.021169921 | 174 | 175 | Note that arithmetic with NumPy arrays is currently best done in a 176 | =remote-objects= environment. A 1D array returned to lisp is 177 | converted to a list when sent to python. This list then doesn't 178 | support arithmetic, even though the original 1D array did support 179 | arithmetic. 180 | 181 | 182 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # cl-travis install script. Don't remove this line. 3 | set -e 4 | 5 | # get 6 | get() { 7 | destination=$1; shift 8 | for url in "$@"; do 9 | echo "Downloading ${url}..." 10 | if curl --no-progress-bar --retry 10 -o "$destination" -L "$url"; then 11 | return 0; 12 | else 13 | echo "Failed to download ${url}." 14 | fi 15 | done 16 | 17 | return 1; 18 | } 19 | 20 | # unpack 21 | unpack() { 22 | opt=$1 23 | file=$2; 24 | destination=$3; 25 | 26 | echo "Unpacking tarball $1 into $3..." 27 | mkdir -p "$destination" 28 | tar -C "$destination" --strip-components=1 "$opt" -xf "$file" 29 | } 30 | 31 | install_i386_arch() { 32 | # hack for issue #17 33 | sudo sed -i -e 's/deb http/deb [arch=amd64] http/' "/etc/apt/sources.list.d/google-chrome.list" 34 | sudo dpkg --add-architecture i386 35 | sudo apt-get update -qq 36 | sudo apt-get install -y libc6:i386 37 | } 38 | 39 | # add_to_lisp_rc 40 | add_to_lisp_rc() { 41 | string=$1 42 | 43 | case "$LISP" in 44 | abcl) rc=".abclrc" ;; 45 | allegro*) rc=".clinit.cl" ;; 46 | sbcl*) rc=".sbclrc" ;; 47 | ccl*) rc=".ccl-init.lisp" ;; 48 | cmucl) rc=".cmucl-init.lisp" ;; 49 | clisp*) rc=".clisprc.lisp" ;; 50 | ecl) rc=".eclrc" ;; 51 | *) 52 | echo "Unable to determine RC file for '$LISP'." 53 | exit 1 54 | ;; 55 | esac 56 | 57 | echo "$string" >> "$HOME/.cim/init.lisp" 58 | echo "$string" >> "$HOME/$rc" 59 | } 60 | 61 | ASDF_URL="https://common-lisp.net/project/asdf/archives/asdf.lisp" 62 | ASDF_LOCATION="$HOME/asdf" 63 | 64 | install_asdf() { 65 | get asdf.lisp "$ASDF_URL" 66 | add_to_lisp_rc "(load \"$ASDF_LOCATION\")" 67 | } 68 | 69 | compile_asdf() { 70 | echo "Compiling ASDF..." 71 | cl -c "$ASDF_LOCATION.lisp" -Q 72 | } 73 | 74 | ASDF_SR_CONF_DIR="$HOME/.config/common-lisp/source-registry.conf.d" 75 | ASDF_SR_CONF_FILE="$ASDF_SR_CONF_DIR/cl-travis.conf" 76 | LOCAL_LISP_TREE="$HOME/lisp" 77 | 78 | setup_asdf_source_registry() { 79 | mkdir -p "$LOCAL_LISP_TREE" 80 | mkdir -p "$ASDF_SR_CONF_DIR" 81 | 82 | echo "(:tree \"$TRAVIS_BUILD_DIR/\")" > "$ASDF_SR_CONF_FILE" 83 | echo "(:tree \"$LOCAL_LISP_TREE/\")" >> "$ASDF_SR_CONF_FILE" 84 | 85 | echo "Created $ASDF_SR_CONF_FILE" 86 | cat -n "$ASDF_SR_CONF_FILE" 87 | } 88 | 89 | # install_script 90 | install_script() { 91 | path=$1; shift 92 | tmp=$(mktemp) 93 | 94 | echo "#!/bin/sh" > "$tmp" 95 | for line; do 96 | echo "$line" >> "$tmp" 97 | done 98 | chmod 755 "$tmp" 99 | 100 | sudo mv "$tmp" "$path" 101 | } 102 | 103 | ABCL_TARBALL_URL1="https://common-lisp.net/project/armedbear/releases/1.3.2/abcl-bin-1.3.2.tar.gz" 104 | ABCL_TARBALL_URL2="http://cddr.org/ci/abcl-bin-1.3.2.tar.gz" 105 | ABCL_TARBALL="abcl.tar.gz" 106 | ABCL_DIR="$HOME/abcl" 107 | ABCL_SCRIPT="/usr/local/bin/abcl" 108 | 109 | install_abcl() { 110 | sudo apt-get install -y default-jre 111 | get "$ABCL_TARBALL" "$ABCL_TARBALL_URL1" "$ABCL_TARBALL_URL2" 112 | unpack -z "$ABCL_TARBALL" "$ABCL_DIR" 113 | 114 | install_script "$ABCL_SCRIPT" \ 115 | "java -cp \"$ABCL_DIR/abcl-contrib.jar\" \ 116 | -jar \"$ABCL_DIR/abcl.jar\" \"\$@\"" 117 | 118 | cim use abcl-system --default 119 | } 120 | 121 | SBCL_TARBALL_URL1="http://prdownloads.sourceforge.net/sbcl/sbcl-1.2.13-x86-64-linux-binary.tar.bz2" 122 | SBCL_TARBALL_URL2="http://cddr.org/ci/sbcl-1.2.13-x86-64-linux-binary.tar.bz2" 123 | SBCL_TARBALL="sbcl.tar.bz2" 124 | SBCL_DIR="$HOME/sbcl" 125 | 126 | install_sbcl() { 127 | echo "Installing SBCL..." 128 | get "$SBCL_TARBALL" "$SBCL_TARBALL_URL1" "$SBCL_TARBALL_URL2" 129 | unpack -j "$SBCL_TARBALL" "$SBCL_DIR" 130 | ( cd "$SBCL_DIR" && sudo bash install.sh ) 131 | 132 | cim use sbcl-system --default 133 | } 134 | 135 | SBCL32_TARBALL_URL1="http://prdownloads.sourceforge.net/sbcl/sbcl-1.2.7-x86-linux-binary.tar.bz2" 136 | SBCL32_TARBALL_URL2="http://cddr.org/ci/sbcl-1.2.7-x86-linux-binary.tar.bz2" 137 | SBCL32_TARBALL="sbcl32.tar.bz2" 138 | SBCL32_DIR="$HOME/sbcl32" 139 | 140 | install_sbcl32() { 141 | echo "Installing 32-bit SBCL..." 142 | install_i386_arch 143 | 144 | get "$SBCL32_TARBALL" "$SBCL32_TARBALL_URL1" "$SBCL32_TARBALL_URL2" 145 | unpack -j "$SBCL32_TARBALL" "$SBCL32_DIR" 146 | ( cd "$SBCL32_DIR" && sudo bash install.sh ) 147 | sudo ln -s /usr/local/bin/sbcl /usr/local/bin/sbcl32 148 | 149 | cim use sbcl-system --default 150 | } 151 | 152 | CCL_TARBALL_URL1="https://github.com/Clozure/ccl/releases/download/v1.11.5/ccl-1.11.5-linuxx86.tar.gz" 153 | CCL_TARBALL_URL2="http://kerno.org/~luis/ci/ccl-1.11-linuxx86.tar.gz" 154 | CCL_TARBALL_URL3="http://common-lisp.net/~loliveira/tarballs/ci/ccl-1.11-linuxx86.tar.gz" 155 | CCL_TARBALL="ccl.tar.gz" 156 | CCL_DIR="$HOME/ccl" 157 | CCL_SCRIPT_PREFIX="/usr/local/bin" 158 | 159 | install_ccl() { 160 | if [ "$LISP" = "ccl32" ]; then 161 | echo "Installing 32-bit CCL..." 162 | install_i386_arch 163 | bin="lx86cl" 164 | script="ccl32" 165 | else 166 | echo "Installing CCL..." 167 | bin="lx86cl64" 168 | script="ccl" 169 | fi 170 | get "$CCL_TARBALL" "$CCL_TARBALL_URL1" "$CCL_TARBALL_URL2" "$CCL_TARBALL_URL3" 171 | unpack -z "$CCL_TARBALL" "$CCL_DIR" 172 | 173 | install_script "$CCL_SCRIPT_PREFIX/$script" "\"$CCL_DIR/$bin\" \"\$@\"" 174 | if [ "$LISP" = "ccl32" ]; then 175 | # also install the 'ccl' script so that CIM can pick it up. 176 | install_script "$CCL_SCRIPT_PREFIX/ccl" "\"$CCL_DIR/$bin\" \"\$@\"" 177 | fi 178 | 179 | cim use ccl-system --default 180 | } 181 | 182 | CMUCL_TARBALL_URL1="https://common-lisp.net/project/cmucl/downloads/snapshots/2015/07/cmucl-2015-07-x86-linux.tar.bz2" 183 | CMUCL_EXTRA_TARBALL_URL1="https://common-lisp.net/project/cmucl/downloads/snapshots/2015/07/cmucl-2015-07-x86-linux.extra.tar.bz2" 184 | CMUCL_TARBALL_URL2="http://cddr.org/ci/cmucl-2015-07-x86-linux.tar.bz2" 185 | CMUCL_EXTRA_TARBALL_URL2="http://cddr.org/ci/cmucl-2015-07-x86-linux.extra.tar.bz2" 186 | CMUCL_TARBALL="cmucl.tar.bz2" 187 | CMUCL_EXTRA_TARBALL="cmucl-extra.tar.bz2" 188 | CMUCL_DIR="$HOME/cmucl" 189 | CMUCL_SCRIPT="/usr/local/bin/cmucl" 190 | 191 | install_cmucl() { 192 | echo "Installing CMUCL..." 193 | install_i386_arch 194 | get "$CMUCL_TARBALL" "$CMUCL_TARBALL_URL1" "$CMUCL_TARBALL_URL2" 195 | get "$CMUCL_EXTRA_TARBALL" "$CMUCL_EXTRA_TARBALL_URL" "$CMUCL_EXTRA_TARBALL_URL2" 196 | mkdir -p "$CMUCL_DIR" 197 | tar -C "$CMUCL_DIR" -xjf "$CMUCL_TARBALL" 198 | tar -C "$CMUCL_DIR" -xjf "$CMUCL_EXTRA_TARBALL" 199 | 200 | install_script "$CMUCL_SCRIPT" \ 201 | "CMUCLLIB=\"$CMUCL_DIR/lib/cmucl/lib\" \"$CMUCL_DIR/bin/lisp\" \"\$@\"" 202 | 203 | # XXX: no CIM support for CMUCL 204 | } 205 | 206 | ECL_TARBALL_URL1="http://common-lisp.net/~loliveira/tarballs/ecl-13.5.1-linux-amd64.tar.gz" 207 | ECL_TARBALL_URL2="http://kerno.org/~luis/ci/ecl-13.5.1-linux-amd64.tar.gz" 208 | ECL_TARBALL="ecl.tar.gz" 209 | 210 | install_ecl() { 211 | echo "Installing ECL..." 212 | #get "$ECL_TARBALL" "$ECL_TARBALL_URL1" "$ECL_TARBALL_URL2" 213 | #sudo tar -C / -xzf "$ECL_TARBALL" 214 | sudo apt-get install -y ecl 215 | 216 | cim use ecl-system --default 217 | } 218 | 219 | install_clisp() { 220 | if [ "$LISP" = "clisp32" ]; then 221 | echo "Installing 32-bit CLISP..." 222 | install_i386_arch 223 | sudo apt-get remove -y libsigsegv2 224 | sudo apt-get install -y libsigsegv2:i386 225 | sudo apt-get install -y clisp:i386 226 | sudo ln -s /usr/bin/clisp /usr/local/bin/clisp32 227 | else 228 | echo "Installing CLISP..." 229 | sudo apt-get install -y clisp 230 | fi 231 | cim use clisp-system --default 232 | } 233 | 234 | ACL_TARBALL_URL="https://franz.com/ftp/pub/acl10.1express/linux86/acl10.1express-linux-x86.tbz2" 235 | ACL_TARBALL="acl.tbz2" 236 | ACL_DIR="$HOME/acl" 237 | install_acl() { 238 | echo "Installing Allegro CL..." 239 | install_i386_arch 240 | 241 | case "$LISP" in 242 | allegro) acl=alisp ;; 243 | allegromodern) acl=mlisp ;; 244 | *) 245 | echo "Unrecognised lisp: '$LISP'" 246 | exit 1 247 | ;; 248 | esac 249 | 250 | get "$ACL_TARBALL" "$ACL_TARBALL_URL" 251 | unpack -j "$ACL_TARBALL" "$ACL_DIR" 252 | 253 | sudo ln -vs "$ACL_DIR"/alisp "/usr/local/bin/$acl" 254 | 255 | cim use alisp-system --default 256 | } 257 | 258 | QUICKLISP_URL="http://beta.quicklisp.org/quicklisp.lisp" 259 | 260 | install_quicklisp() { 261 | get quicklisp.lisp "$QUICKLISP_URL" 262 | echo 'Installing Quicklisp...' 263 | cl -f quicklisp.lisp -e '(quicklisp-quickstart:install)' 264 | add_to_lisp_rc '(let ((quicklisp-init (merge-pathnames "quicklisp/setup.lisp" 265 | (user-homedir-pathname)))) 266 | (when (probe-file quicklisp-init) 267 | (load quicklisp-init)))' 268 | } 269 | 270 | # this variable is used to grab a specific version of the 271 | # cim_installer which itself looks at this variable to figure out 272 | # which version of CIM it should install. 273 | CIM_INSTALL_BRANCH=c9f4ea960ce4504d5ddd229b9f0f83ddc6dce773 274 | CL_SCRIPT="/usr/local/bin/cl" 275 | CIM_SCRIPT="/usr/local/bin/cim" 276 | QL_SCRIPT="/usr/local/bin/ql" 277 | 278 | install_cim() { 279 | curl -L "https://raw.github.com/KeenS/CIM/$CIM_INSTALL_BRANCH/scripts/cim_installer" | /bin/sh 280 | 281 | install_script "$CL_SCRIPT" ". \"$HOME\"/.cim/init.sh; exec cl \"\$@\"" 282 | install_script "$CIM_SCRIPT" ". \"$HOME\"/.cim/init.sh; exec cim \"\$@\"" 283 | install_script "$QL_SCRIPT" ". \"$HOME\"/.cim/init.sh; exec ql \"\$@\"" 284 | } 285 | 286 | ( 287 | cd "$HOME" 288 | 289 | sudo apt-get update 290 | install_cim 291 | install_asdf 292 | 293 | case "$LISP" in 294 | abcl) install_abcl ;; 295 | allegro|allegromodern) install_acl ;; 296 | sbcl) install_sbcl ;; 297 | sbcl32) install_sbcl32 ;; 298 | ccl|ccl32) install_ccl ;; 299 | cmucl) install_cmucl; exit 0 ;; # no CIM support 300 | clisp|clisp32) install_clisp ;; 301 | ecl) install_ecl ;; 302 | *) 303 | echo "Unrecognised lisp: '$LISP'" 304 | exit 1 305 | ;; 306 | esac 307 | 308 | compile_asdf 309 | 310 | cl -e '(format t "~%~a ~a up and running! (ASDF ~a)~%~%" 311 | (lisp-implementation-type) 312 | (lisp-implementation-version) 313 | (asdf:asdf-version))' 314 | 315 | install_quicklisp 316 | setup_asdf_source_registry 317 | ) 318 | -------------------------------------------------------------------------------- /py4cl.asd: -------------------------------------------------------------------------------- 1 | ;;;; py4cl.asd 2 | 3 | (asdf:defsystem "py4cl" 4 | :serial t 5 | :description "Call Python libraries from Common Lisp" 6 | :author "Ben Dudson " 7 | :license "MIT" 8 | :depends-on ("trivial-garbage" "uiop" "cl-json" "numpy-file-format") 9 | :pathname #P"src/" 10 | :serial t 11 | :components ((:file "package") 12 | (:file "config") 13 | (:file "reader") 14 | (:file "writer") 15 | (:file "python-process") 16 | (:file "lisp-classes") 17 | (:file "callpython") 18 | (:file "import-export") 19 | (:file "do-after-load")) 20 | :in-order-to ((test-op (test-op "py4cl/tests")))) 21 | 22 | ;; This is to store the path to the source code 23 | ;; suggested here https://xach.livejournal.com/294639.html 24 | (defpackage #:py4cl/config (:export #:*base-directory*)) 25 | (defparameter py4cl/config:*base-directory* 26 | (asdf:system-source-directory "py4cl")) 27 | 28 | (asdf:defsystem "py4cl/tests" 29 | :serial t 30 | :description "Unit tests for the py4cl library." 31 | :author "Ben Dudson " 32 | :license "MIT" 33 | :depends-on ("py4cl" 34 | "clunit" 35 | "trivial-garbage" 36 | "bordeaux-threads") 37 | :pathname #P"tests/" 38 | :components ((:file "tests")) 39 | :perform (test-op (o c) (symbol-call :py4cl/tests :run))) 40 | -------------------------------------------------------------------------------- /py4cl.py: -------------------------------------------------------------------------------- 1 | # Python interface for py4cl 2 | # 3 | # This code handles messages from lisp, marshals and unmarshals data, 4 | # and defines classes which forward all interactions to lisp. 5 | # 6 | # Should work with python 2.7 or python 3 7 | 8 | from __future__ import print_function 9 | 10 | import sys 11 | import numbers 12 | import itertools 13 | import os 14 | import json 15 | 16 | try: 17 | from io import StringIO # Python 3 18 | except: 19 | from io import BytesIO as StringIO 20 | 21 | is_py2 = sys.version_info[0] < 3 22 | 23 | # Direct stdout to a StringIO buffer, 24 | # to prevent commands from printing to the output stream 25 | 26 | write_stream = sys.stdout 27 | redirect_stream = StringIO() 28 | 29 | sys.stdout = redirect_stream 30 | 31 | config = {} 32 | def load_config(): 33 | if (os.path.exists(os.path.dirname(__file__) + "/.config")): 34 | with open(os.path.dirname(__file__) + "/.config") as conf: 35 | global config 36 | config = json.load(conf) 37 | try: 38 | eval_globals['_py4cl_config'] = config 39 | except: 40 | pass 41 | load_config() 42 | 43 | class Symbol(object): 44 | """ 45 | A wrapper around a string, representing a Lisp symbol. 46 | """ 47 | def __init__(self, name): 48 | self._name = name 49 | def __str__(self): 50 | return self._name 51 | def __repr__(self): 52 | return "Symbol("+self._name+")" 53 | 54 | class LispCallbackObject (object): 55 | """ 56 | Represents a lisp function which can be called. 57 | 58 | An object is used rather than a lambda, so that the lifetime 59 | can be monitoried, and the function removed from a hash map 60 | """ 61 | def __init__(self, handle): 62 | """ 63 | handle A number, used to refer to the object in Lisp 64 | """ 65 | self.handle = handle 66 | 67 | def __del__(self): 68 | """ 69 | Delete this object, sending a message to Lisp 70 | """ 71 | try: 72 | sys.stdout = write_stream 73 | write_stream.write("d") 74 | send_value(self.handle) 75 | finally: 76 | sys.stdout = redirect_stream 77 | 78 | def __call__(self, *args, **kwargs): 79 | """ 80 | Call back to Lisp 81 | 82 | args Arguments to be passed to the function 83 | """ 84 | global return_values 85 | 86 | # Convert kwargs into a sequence of ":keyword value" pairs 87 | # appended to the positional arguments 88 | allargs = args 89 | for key, value in kwargs.items(): 90 | allargs += (Symbol(":"+str(key)), value) 91 | 92 | old_return_values = return_values # Save to restore after 93 | try: 94 | return_values = 0 # Need to send the values 95 | sys.stdout = write_stream 96 | write_stream.write("c") 97 | send_value((self.handle, allargs)) 98 | finally: 99 | return_values = old_return_values 100 | sys.stdout = redirect_stream 101 | 102 | # Wait for a value to be returned. 103 | # Note that the lisp function may call python before returning 104 | return message_dispatch_loop() 105 | 106 | 107 | class UnknownLispObject (object): 108 | """ 109 | Represents an object in Lisp, which could not be converted to Python 110 | """ 111 | 112 | __during_init = True # Don't send changes during __init__ 113 | 114 | def __init__(self, lisptype, handle): 115 | """ 116 | lisptype A string describing the type. Mainly for debugging 117 | handle A number, used to refer to the object in Lisp 118 | """ 119 | self.lisptype = lisptype 120 | self.handle = handle 121 | self.__during_init = False # Further changes are sent to Lisp 122 | 123 | def __del__(self): 124 | """ 125 | Delete this object, sending a message to Lisp 126 | """ 127 | try: 128 | sys.stdout = write_stream 129 | write_stream.write("d") 130 | send_value(self.handle) 131 | finally: 132 | sys.stdout = redirect_stream 133 | 134 | def __str__(self): 135 | return "UnknownLispObject(\""+self.lisptype+"\", "+str(self.handle)+")" 136 | 137 | def __getattr__(self, attr): 138 | # Check if there is a slot with this name 139 | try: 140 | sys.stdout = write_stream 141 | write_stream.write("s") # Slot read 142 | send_value((self.handle, attr)) 143 | finally: 144 | sys.stdout = redirect_stream 145 | 146 | # Wait for the result 147 | return message_dispatch_loop() 148 | 149 | def __setattr__(self, attr, value): 150 | if self.__during_init: 151 | return object.__setattr__(self, attr, value) 152 | try: 153 | sys.stdout = write_stream 154 | write_stream.write("S") # Slot write 155 | send_value((self.handle, attr, value)) 156 | finally: 157 | sys.stdout = redirect_stream 158 | # Wait until finished, to syncronise 159 | return message_dispatch_loop() 160 | 161 | # These store the environment used when eval'ing strings from Lisp 162 | eval_globals = {} 163 | eval_locals = {} 164 | 165 | # Settings 166 | 167 | return_values = 0 # Try to return values to lisp. If > 0, always return a handle 168 | # A counter is used, rather than Boolean, to allow nested environments. 169 | 170 | ################################################################## 171 | # This code adapted from cl4py 172 | # 173 | # https://github.com/marcoheisig/cl4py 174 | # 175 | # Copyright (c) 2018 Marco Heisig 176 | # 2019 Ben Dudson 177 | 178 | lispifiers = { 179 | bool : lambda x: "T" if x else "NIL", 180 | type(None) : lambda x: "NIL", 181 | int : str, 182 | float : str, 183 | complex : lambda x: "#C(" + lispify(x.real) + " " + lispify(x.imag) + ")", 184 | list : lambda x: "#(" + " ".join(lispify(elt) for elt in x) + ")", 185 | tuple : lambda x: "(" + " ".join(lispify(elt) for elt in x) + ")", 186 | # Note: With dict -> hash table, use :test 'equal so that string keys work as expected 187 | dict : lambda x: "#.(let ((table (make-hash-table :test 'equal))) " + " ".join("(setf (gethash (cl:quote {}) table) (cl:quote {}))".format(lispify(key), lispify(value)) for key, value in x.items()) + " table)", 188 | str : lambda x: "\"" + x.replace("\\", "\\\\").replace('"', '\\"') + "\"", 189 | type(u'unicode') : lambda x: "\"" + x.replace("\\", "\\\\").replace('"', '\\"') + "\"", # Unicode in python 2 190 | Symbol : str, 191 | UnknownLispObject : lambda x: "#.(py4cl::lisp-object {})".format(x.handle), 192 | } 193 | 194 | # This is used to test if a value is a numeric type 195 | numeric_base_classes = (numbers.Number,) 196 | 197 | eval_globals["_py4cl_numpy_is_loaded"] = False 198 | try: 199 | # Use NumPy for multi-dimensional arrays 200 | import numpy 201 | eval_globals["_py4cl_numpy_is_loaded"] = True 202 | NUMPY_PICKLE_INDEX = 0 # optional increment in lispify_ndarray and reset to 0 203 | def load_pickled_ndarray(filename): 204 | arr = numpy.load(filename, allow_pickle = True) 205 | return arr 206 | 207 | def delete_numpy_pickle_arrays(): 208 | global NUMPY_PICKLE_INDEX 209 | while NUMPY_PICKLE_INDEX > 0: 210 | NUMPY_PICKLE_INDEX -= 1 211 | numpy_pickle_location = config["numpyPickleLocation"] \ 212 | + ".from." + str(NUMPY_PICKLE_INDEX) 213 | if os.path.exists(numpy_pickle_location): 214 | os.remove(numpy_pickle_location) 215 | 216 | def lispify_ndarray(obj): 217 | """Convert a NumPy array to a string which can be read by lisp 218 | Example: 219 | array([[1, 2], => '#2A((1 2) (3 4))' 220 | [3, 4]]) 221 | """ 222 | global NUMPY_PICKLE_INDEX 223 | if "numpyPickleLowerBound" in config and \ 224 | "numpyPickleLocation" in config and \ 225 | obj.size >= config["numpyPickleLowerBound"]: 226 | numpy_pickle_location = config["numpyPickleLocation"] \ 227 | + ".from." + str(NUMPY_PICKLE_INDEX) 228 | NUMPY_PICKLE_INDEX += 1 229 | with open(numpy_pickle_location, "wb") as f: 230 | numpy.save(f, obj, allow_pickle = True) 231 | return ('#.(numpy-file-format:load-array "' 232 | + numpy_pickle_location + '")') 233 | if obj.ndim == 0: 234 | # Convert to scalar then lispify 235 | return lispify(numpy.asscalar(obj)) 236 | 237 | def nested(obj): 238 | """Turns an array into nested ((1 2) (3 4))""" 239 | if obj.ndim == 1: 240 | return "("+" ".join([lispify(i) for i in obj])+")" 241 | return "(" + " ".join([nested(obj[i,...]) for i in range(obj.shape[0])]) + ")" 242 | 243 | return "#{:d}A".format(obj.ndim) + nested(obj) 244 | 245 | # Register the handler to convert Python -> Lisp strings 246 | lispifiers[numpy.ndarray] = lispify_ndarray 247 | 248 | # Register numeric base class 249 | numeric_base_classes += (numpy.number,) 250 | except: 251 | pass 252 | 253 | def lispify_handle(obj): 254 | """ 255 | Store an object in a dictionary, and return a handle 256 | """ 257 | handle = next(python_handle) 258 | python_objects[handle] = obj 259 | return "#.(py4cl::make-python-object-finalize :type \""+str(type(obj))+"\" :handle "+str(handle)+")" 260 | 261 | def lispify(obj): 262 | """ 263 | Turn a python object into a string which can be parsed by Lisp's reader. 264 | 265 | If return_values is false then always creates a handle 266 | """ 267 | if return_values > 0: 268 | return lispify_handle(obj) 269 | 270 | try: 271 | return lispifiers[type(obj)](obj) 272 | except KeyError: 273 | # Special handling for numbers. This should catch NumPy types 274 | # as well as built-in numeric types 275 | if isinstance(obj, numeric_base_classes): 276 | return str(obj) 277 | 278 | # Another unknown type. Return a handle to a python object 279 | return lispify_handle(obj) 280 | 281 | def generator(function, stop_value): 282 | temp = None 283 | while True: 284 | temp = function() 285 | if temp == stop_value: break 286 | yield temp 287 | 288 | ################################################################## 289 | 290 | def recv_string(): 291 | """ 292 | Get a string from the input stream 293 | """ 294 | # First a line containing the length as a string 295 | length = int(sys.stdin.readline()) 296 | # Then the specified number of bytes 297 | return sys.stdin.read(length) 298 | 299 | def recv_value(): 300 | """ 301 | Get a value from the input stream 302 | Return could be any type 303 | """ 304 | if is_py2: 305 | return eval(recv_string(), eval_globals, eval_locals) 306 | return eval(recv_string(), eval_globals) 307 | 308 | def send_value(value): 309 | """ 310 | Send a value to stdout as a string, with length of string first 311 | """ 312 | try: 313 | value_str = lispify(value) 314 | except Exception as e: 315 | # At this point the message type has been sent, 316 | # so we can't change to throw an exception/signal condition 317 | value_str = "Lispify error: " + str(e) 318 | print(len(value_str)) 319 | write_stream.write(value_str) 320 | write_stream.flush() 321 | 322 | def return_stdout(): 323 | """ 324 | Return the contents of redirect_stream, to be printed to stdout 325 | """ 326 | global redirect_stream 327 | global return_values 328 | 329 | contents = redirect_stream.getvalue() 330 | if not contents: 331 | return # Nothing to send 332 | 333 | redirect_stream = StringIO() # New stream, delete old one 334 | 335 | old_return_values = return_values # Save to restore after 336 | try: 337 | return_values = 0 # Need to return the string, not a handle 338 | sys.stdout = write_stream 339 | write_stream.write("p") 340 | send_value(contents) 341 | finally: 342 | return_values = old_return_values 343 | sys.stdout = redirect_stream 344 | 345 | def return_error(err): 346 | """ 347 | Send an error message 348 | """ 349 | global return_values 350 | 351 | return_stdout() # Send stdout if any 352 | 353 | old_return_values = return_values # Save to restore after 354 | try: 355 | return_values = 0 # Need to return the error, not a handle 356 | sys.stdout = write_stream 357 | write_stream.write("e") 358 | send_value(str(err)) 359 | finally: 360 | return_values = old_return_values 361 | sys.stdout = redirect_stream 362 | 363 | def return_value(value): 364 | """ 365 | Send a value to stdout 366 | """ 367 | if isinstance(value, Exception): 368 | return return_error(value) 369 | 370 | return_stdout() # Send stdout if any 371 | 372 | # Mark response as a returned value 373 | try: 374 | sys.stdout = write_stream 375 | write_stream.write("r") 376 | send_value(value) 377 | finally: 378 | sys.stdout = redirect_stream 379 | 380 | 381 | def py_eval(command, eval_globals, eval_locals): 382 | """ 383 | Perform eval, but do not pass locals if we are in python 3. 384 | """ 385 | if is_py2: # Python 3 386 | return eval(command, eval_globals, eval_locals) 387 | # Python 3 388 | return eval(command, eval_globals) 389 | 390 | def py_exec(command, exec_globals, exec_locals): 391 | """ 392 | Perform exec, but do not pass locals if we are in python 3. 393 | """ 394 | if is_py2: # Python 3 395 | return exec(command, exec_globals, exec_locals) 396 | # Python 3 397 | return exec(command, exec_globals) 398 | 399 | def message_dispatch_loop(): 400 | """ 401 | Wait for a message, dispatch on the type of message. 402 | Message types are determined by the first character: 403 | 404 | e Evaluate an expression (expects string) 405 | x Execute a statement (expects string) 406 | q Quit 407 | r Return value from lisp (expects value) 408 | f Function call 409 | a Asynchronous function call 410 | R Retrieve value from asynchronous call 411 | s Set variable(s) 412 | """ 413 | global return_values # Controls whether values or handles are returned 414 | 415 | while True: 416 | try: 417 | # Read command type 418 | cmd_type = sys.stdin.read(1) 419 | if eval_globals["_py4cl_numpy_is_loaded"]: 420 | try: 421 | delete_numpy_pickle_arrays() 422 | except: 423 | pass 424 | 425 | if cmd_type == "e": # Evaluate an expression 426 | result = py_eval(recv_string(), eval_globals, eval_locals) 427 | return_value(result) 428 | 429 | elif cmd_type == "f" or cmd_type == "a": # Function call 430 | # Get a tuple (function, allargs) 431 | fn_name, allargs = recv_value() 432 | 433 | # Split positional arguments and keywords 434 | args = [] 435 | kwargs = {} 436 | if allargs: 437 | it = iter(allargs) # Use iterator so we can skip values 438 | for arg in it: 439 | if isinstance(arg, Symbol): 440 | # A keyword. Take the next value 441 | kwargs[ str(arg)[1:] ] = next(it) 442 | continue 443 | args.append(arg) 444 | 445 | # Get the function object. Using eval to handle cases like "math.sqrt" or lambda functions 446 | if callable(fn_name): 447 | function = fn_name # Already callable 448 | else: 449 | function = py_eval(fn_name, eval_globals, eval_locals) 450 | if cmd_type == "f": 451 | # Run function then return value 452 | return_value( function(*args, **kwargs) ) 453 | else: 454 | # Asynchronous 455 | 456 | # Get a handle, and send back to caller. 457 | # The handle can be used to fetch 458 | # the result using an "R" message. 459 | 460 | handle = next(async_handle) 461 | return_value(handle) 462 | 463 | try: 464 | # Run function, store result 465 | async_results[handle] = function(*args, **kwargs) 466 | except Exception as e: 467 | # Catching error here so it can 468 | # be stored as the return value 469 | async_results[handle] = e 470 | 471 | elif cmd_type == "O": # Return only handles 472 | return_values += 1 473 | 474 | elif cmd_type == "o": # Return values when possible (default) 475 | return_values -= 1 476 | 477 | elif cmd_type == "q": # Quit 478 | sys.exit(0) 479 | 480 | elif cmd_type == "R": 481 | # Request value using handle 482 | handle = recv_value() 483 | return_value( async_results.pop(handle) ) 484 | 485 | elif cmd_type == "r": # Return value from Lisp function 486 | return recv_value() 487 | 488 | elif cmd_type == "s": 489 | # Set variables. Should have the form 490 | # ( ("var1" value1) ("var2" value2) ...) 491 | setlist = recv_value() 492 | if is_py2: 493 | for name, value in setlist: 494 | eval_locals[name] = value 495 | else: 496 | for name, value in setlist: 497 | eval_globals[name] = value 498 | # Need to send something back to acknowlege 499 | return_value(True) 500 | 501 | elif cmd_type == "v": 502 | # Version info 503 | return_value(tuple(sys.version_info)) 504 | 505 | elif cmd_type == "x": # Execute a statement 506 | py_exec(recv_string(), eval_globals, eval_locals) 507 | return_value(None) 508 | 509 | else: 510 | return_error("Unknown message type '{0}'".format(cmd_type)) 511 | 512 | except KeyboardInterrupt as e: 513 | return_value(None) 514 | except Exception as e: 515 | return_error(e) 516 | 517 | 518 | # Store for python objects which can't be translated to Lisp objects 519 | python_objects = {} 520 | python_handle = itertools.count(0) # Running counter 521 | 522 | # Make callback function accessible to evaluation 523 | eval_globals["_py4cl_LispCallbackObject"] = LispCallbackObject 524 | eval_globals["_py4cl_Symbol"] = Symbol 525 | eval_globals["_py4cl_UnknownLispObject"] = UnknownLispObject 526 | eval_globals["_py4cl_objects"] = python_objects 527 | eval_globals["_py4cl_generator"] = generator 528 | # These store the environment used when eval'ing strings from Lisp 529 | # - particularly for numpy pickling 530 | eval_globals["_py4cl_config"] = config 531 | eval_globals["_py4cl_load_config"] = load_config 532 | try: 533 | # NumPy is used for Lisp -> Python conversion of multidimensional arrays 534 | eval_globals["_py4cl_numpy"] = numpy 535 | eval_globals["_py4cl_load_pickled_ndarray"] \ 536 | = load_pickled_ndarray 537 | except: 538 | pass 539 | 540 | # Handle fractions (RATIO type) 541 | # Lisp will pass strings containing "_py4cl_fraction(n,d)" 542 | # where n and d are integers. 543 | try: 544 | import fractions 545 | eval_globals["_py4cl_fraction"] = fractions.Fraction 546 | 547 | # Turn a Fraction into a Lisp RATIO 548 | lispifiers[fractions.Fraction] = str 549 | except: 550 | # In python2, ensure that fractions are converted to floats 551 | eval_globals["_py4cl_fraction"] = lambda a,b : float(a)/b 552 | 553 | async_results = {} # Store for function results. Might be Exception 554 | async_handle = itertools.count(0) # Running counter 555 | 556 | # Main loop 557 | message_dispatch_loop() 558 | 559 | 560 | 561 | -------------------------------------------------------------------------------- /src/callpython.lisp: -------------------------------------------------------------------------------- 1 | (in-package :py4cl) 2 | 3 | (define-condition python-error (error) 4 | ((text :initarg :text :reader text)) 5 | (:report (lambda (condition stream) 6 | (format stream "Python error: ~a" (text condition))))) 7 | 8 | (defun dispatch-reply (stream value) 9 | (write-char #\r stream) 10 | (stream-write-value value stream) 11 | (force-output stream)) 12 | 13 | (defun dispatch-messages (process) 14 | "Read response from python, loop to handle any callbacks" 15 | (let ((read-stream (uiop:process-info-output process)) 16 | (write-stream (uiop:process-info-input process))) 17 | (loop 18 | (case (read-char read-stream) ; First character is type of message 19 | ;; Returned value 20 | (#\r (return-from dispatch-messages 21 | (stream-read-value read-stream))) 22 | ;; Error 23 | (#\e (error 'python-error 24 | :text (stream-read-string read-stream))) 25 | 26 | ;; Delete object. This is called when an UnknownLispObject is deleted 27 | (#\d (free-handle (stream-read-value read-stream))) 28 | 29 | ;; Slot read 30 | (#\s (destructuring-bind (handle slot-name) (stream-read-value read-stream) 31 | (let ((object (lisp-object handle))) 32 | ;; User must register a function to handle slot access 33 | (dispatch-reply write-stream 34 | (restart-case 35 | (python-getattr object slot-name) 36 | ;; Provide some restarts for missing handler or missing slot 37 | (return-nil () nil) 38 | (return-zero () 0) 39 | (enter-value (return-value) 40 | :report "Provide a value to return" 41 | :interactive (lambda () 42 | (format t "Enter a value to return: ") 43 | (list (read))) 44 | return-value)))))) 45 | 46 | ;; Slot write 47 | (#\S (destructuring-bind (handle slot-name slot-value) (stream-read-value read-stream) 48 | (let ((object (lisp-object handle))) 49 | ;; User must register a function to handle slot write 50 | (python-setattr object slot-name slot-value) 51 | (dispatch-reply write-stream nil)))) 52 | 53 | ;; Callback. Value returned is a list, containing the function ID then the args 54 | (#\c 55 | (let ((call-value (stream-read-value read-stream))) 56 | (let ((return-value (apply (lisp-object (first call-value)) (second call-value)))) 57 | ;; Send a reply 58 | (dispatch-reply write-stream return-value)))) 59 | 60 | ;; Print stdout 61 | (#\p 62 | (let ((print-string (stream-read-value read-stream))) 63 | (princ print-string))) 64 | 65 | (otherwise (error "Unhandled message type")))))) 66 | 67 | (defun python-eval* (cmd-char &rest args) 68 | "Internal function, which converts ARGS into a string to be evaluated 69 | This handles both EVAL and EXEC calls with CMD-CHAR being different 70 | in the two cases. 71 | 72 | Anything in ARGS which is not a string is passed through PYTHONIZE 73 | " 74 | (python-start-if-not-alive) 75 | (let ((stream (uiop:process-info-input *python*)) 76 | (str (apply #'concatenate 'string (loop for val in args 77 | collecting (if (typep val 'string) 78 | val 79 | (pythonize val)))))) 80 | ;; Write "x" if exec, otherwise "e" 81 | (write-char cmd-char stream) 82 | (stream-write-string str stream) 83 | (force-output stream) 84 | ;; Wait for response from Python 85 | (dispatch-messages *python*))) 86 | 87 | (defun python-eval (&rest args) 88 | "Evaluate an expression in python, returning the result 89 | Arguments ARGS can be strings, or other objects. Anything which 90 | is not a string is converted to a python value 91 | 92 | Examples: 93 | 94 | (python-eval \"[i**2 for i in range(\" 4 \")]\") => #(0 1 4 9) 95 | 96 | (let ((a 10) (b 2)) 97 | (py4cl:python-eval a "*" b)) => 20 98 | " 99 | (delete-freed-python-objects) 100 | (apply #'python-eval* #\e args)) 101 | 102 | (defun (setf python-eval) (value &rest args) 103 | "Set an expression to a value. Just adds \"=\" and the value 104 | to the end of the expression. Note that the result is evaluated 105 | with exec rather than eval. 106 | 107 | Examples: 108 | 109 | (setf (python-eval \"a\") 2) ; python \"a=2\" 110 | " 111 | (apply #'python-eval* #\x (append args (list "=" (py4cl::pythonize value)))) 112 | value) 113 | 114 | (defun python-exec (&rest args) 115 | "Execute (using exec) an expression in python. 116 | This is used for statements rather than expressions. 117 | 118 | " 119 | (delete-freed-python-objects) 120 | (apply #'python-eval* #\x args)) 121 | 122 | (defun python-call (fun-name &rest args) 123 | "Call a python function, given the function name as a string 124 | and additional arguments. Keywords are converted to keyword arguments." 125 | (python-start-if-not-alive) 126 | (let ((stream (uiop:process-info-input *python*))) 127 | ;; Write "f" to indicate function call 128 | (write-char #\f stream) 129 | (stream-write-value (list fun-name args) stream) 130 | (force-output stream)) 131 | (dispatch-messages *python*)) 132 | 133 | (defun python-call-async (fun-name &rest args) 134 | "Call a python function asynchronously. 135 | Returns a lambda which when called returns the result." 136 | (python-start-if-not-alive) 137 | 138 | (let* ((process *python*) 139 | (stream (uiop:process-info-input process))) 140 | 141 | ;; Write "a" to indicate asynchronous function call 142 | (write-char #\a stream) 143 | (stream-write-value (list fun-name args) stream) 144 | (force-output stream) 145 | 146 | (let ((handle (dispatch-messages process)) 147 | value) 148 | (lambda () 149 | (if handle 150 | ;; Retrieve the value from python 151 | (progn 152 | (write-char #\R stream) 153 | (stream-write-value handle stream) 154 | (force-output stream) 155 | (setf handle nil 156 | value (dispatch-messages process))) 157 | ;; If no handle then already have the value 158 | value))))) 159 | 160 | (defun python-method (obj method-name &rest args) 161 | "Call a given method on an object OBJ. METHOD-NAME can be a 162 | symbol (converted to lower case) or a string. 163 | 164 | Examples: 165 | 166 | (python-method \"hello {0}\" 'format \"world\") 167 | ; => \"hello world\" 168 | 169 | (python-method '(1 2 3) '__len__) 170 | ; => 3 171 | " 172 | (python-start-if-not-alive) 173 | (py4cl:python-eval 174 | (py4cl::pythonize obj) 175 | (format nil ".~(~a~)" method-name) 176 | (if args 177 | (py4cl::pythonize args) 178 | "()"))) 179 | 180 | (defun python-generator (function stop-value) 181 | (python-call "_py4cl_generator" function stop-value)) 182 | 183 | (defun function-args (args) 184 | "Internal function, intended to be called by the CHAIN macro. 185 | Converts function arguments to a list of strings and (pythonize ) 186 | function calls. Handles keywords and insertion of commas. 187 | Returns a list which can be passed to PYTHON-EVAL. 188 | 189 | Examples: 190 | 191 | (py4cl::function-args '(1 :test 2)) 192 | => ((PY4CL::PYTHONIZE 1) \",\" \"test\" \"=\" (PY4CL::PYTHONIZE 2)) 193 | " 194 | (if (not args) 195 | '("") 196 | (if (keywordp (first args)) 197 | (append 198 | (list (string-downcase (first args)) 199 | "=" 200 | `(pythonize ,(second args))) 201 | (if (cddr args) 202 | (append '(",") (function-args (cddr args))))) 203 | 204 | (append 205 | (list `(pythonize ,(first args))) 206 | (if (rest args) 207 | (append '(",") (function-args (rest args)))))))) 208 | 209 | (defmacro chain (target &rest chain) 210 | "Chain method calls, member access, and indexing operations 211 | on objects. The operations in CHAIN are applied in order from 212 | first to last to the TARGET object. 213 | 214 | TARGET can be 215 | cons -- a python function to call, returning an object to operate on 216 | otherwise -- a value, to be converted to a python value 217 | 218 | CHAIN can consist of 219 | cons -- a method to call 220 | symbol -- a member data variable 221 | otherwise -- a value put between [] brackets to access an index 222 | 223 | Keywords inside python function calls are converted to python keywords. 224 | 225 | Functions can be specified using a symbol or a string. If a symbol is used 226 | then it is converted to python using STRING-DOWNCASE. 227 | 228 | Examples: 229 | 230 | (chain \"hello {0}\" (format \"world\") (capitalize)) 231 | => python: \"hello {0}\".format(\"world\").capitalize() 232 | => \"Hello world\" 233 | 234 | (chain (range 3) stop) 235 | => python: range(3).stop 236 | => 3 237 | 238 | (chain \"hello\" 4) 239 | => python: \"hello\"[4] 240 | => \"o\" 241 | " 242 | (python-start-if-not-alive) 243 | `(py4cl:python-eval 244 | ;; TARGET 245 | ,@(if (consp target) 246 | ;; A list -> python function call 247 | `(,(let ((func (first target))) ; The function name 248 | (if (stringp func) 249 | func ; Leave string unmodified 250 | (string-downcase func))) ; Otherwise convert to lower-case string 251 | "(" 252 | ,@(function-args (rest target)) 253 | ")") 254 | ;; A value 255 | (list (list 'py4cl::pythonize target))) 256 | ;; CHAIN 257 | ,@(loop for link in chain 258 | appending 259 | (cond 260 | ((consp link) 261 | ;; A list. Usually a method to call, but [] indicates __getitem__ 262 | (if (string= (first link) "[]") 263 | ;; Calling the __getitem__ method 264 | (list "[" (list 'py4cl::pythonize ; So that strings are escaped 265 | (if (cddr link) 266 | (append '(list) (rest link)) ; More than one -> wrap in list/tuple 267 | (cadr link))) ; Only one -> no tuple 268 | "]") 269 | ;; Calling a method 270 | `("." 271 | ,(let ((func (first link))) 272 | (if (stringp func) 273 | func ; Leave string unmodified 274 | (string-downcase func))) ; Otherwise convert to lower-case string 275 | "(" 276 | ,@(function-args (rest link)) 277 | ")"))) 278 | ((symbolp link) (list (format nil ".~(~a~)" link))) 279 | (t (list "[" (list 'py4cl::pythonize link) "]")))))) 280 | 281 | (defun python-setf (&rest args) 282 | "Set python variables in ARGS (\"var1\" value1 \"var2\" value2 ...) " 283 | ;; pairs converts a list (a b c d) into a list of pairs ((a b) (c d)) 284 | (labels ((pairs (items) 285 | (when items 286 | (unless (stringp (first items)) 287 | (error "Python variable names must be strings")) 288 | (unless (cdr items) 289 | (error "Expected an even number of inputs")) 290 | (cons (list (first items) (second items)) 291 | (pairs (cddr items)))))) 292 | 293 | (python-start-if-not-alive) 294 | (let ((stream (uiop:process-info-input *python*))) 295 | ;; Write "s" to indicate setting variables 296 | (write-char #\s stream) 297 | (stream-write-value (pairs args) stream) 298 | (force-output stream)) 299 | ;; Should get T returned, might be error 300 | (dispatch-messages *python*))) 301 | 302 | (defmacro remote-objects (&body body) 303 | "Ensures that all values returned by python functions 304 | and methods are kept in python, and only handles returned to lisp. 305 | This is useful if performing operations on large datasets." 306 | `(progn 307 | (python-start-if-not-alive) 308 | (let ((stream (uiop:process-info-input *python*))) 309 | ;; Turn on remote objects 310 | (write-char #\O stream) 311 | (force-output stream) 312 | (unwind-protect 313 | (progn ,@body) 314 | ;; Turn off remote objects 315 | (write-char #\o stream) 316 | (force-output stream))))) 317 | 318 | (defmacro remote-objects* (&body body) 319 | "Ensures that all values returned by python functions 320 | and methods are kept in python, and only handles returned to lisp. 321 | This is useful if performing operations on large datasets. 322 | 323 | This version evaluates the result, returning it as a lisp value if possible. 324 | " 325 | `(python-eval (remote-objects ,@body))) 326 | 327 | 328 | -------------------------------------------------------------------------------- /src/config.lisp: -------------------------------------------------------------------------------- 1 | (in-package :py4cl) 2 | 3 | ;; Particularly for numpy 4 | 5 | (defvar *config* () "Used for storing configuration at a centralized location.") 6 | ;; Refer initialize function to note which variables are included under *config* 7 | 8 | (defun take-input (prompt default) 9 | (format t prompt) 10 | (force-output) 11 | (let ((input (read-line))) 12 | (if (string= "" input) default input))) 13 | 14 | (defun initialize () 15 | "Intended to be called first upon installation. Sets up default python command, 16 | and numpy pickle file and lower bounds." 17 | (let ((pycmd (take-input "Provide the python binary to use (default python): " 18 | "python")) 19 | (numpy-pickle-location 20 | (take-input "~%PY4CL uses pickled files to transfer large arrays between lisp 21 | and python efficiently. These are expected to have sizes exceeding 100MB 22 | (this depends on the value of *NUMPY-PICKLE-LOWER-BOUND*). Therefore, choose an 23 | appropriate location (*NUMPY-PICKLE-LOCATION*) for storing these arrays on disk. 24 | 25 | Enter full file path for storage (default /tmp/_numpy_pickle.npy): " 26 | "/tmp/_numpy_pickle.npy")) 27 | (numpy-pickle-lower-bound 28 | (parse-integer 29 | (take-input "Enter lower bound for using pickling (default 100000): " 30 | "100000")))) 31 | (setq *config* ;; case conversion to and from symbols is handled by cl-json 32 | `((pycmd . ,pycmd) 33 | (numpy-pickle-location . ,numpy-pickle-location) 34 | (numpy-pickle-lower-bound . ,numpy-pickle-lower-bound))) 35 | ;; to avoid development overhead, we will not bring these variables "out" 36 | (save-config))) 37 | 38 | (defun save-config () 39 | (let ((config-path (concatenate 'string 40 | (directory-namestring py4cl/config:*base-directory*) 41 | ".config"))) 42 | 43 | (with-open-file (f config-path :direction :output :if-exists :supersede 44 | :if-does-not-exist :create) 45 | (cl-json:encode-json-alist *config* f)) 46 | (format t "Configuration is saved to ~D.~%" config-path))) 47 | 48 | (defun load-config () 49 | (let ((config-path (concatenate 'string 50 | (directory-namestring py4cl/config:*base-directory*) 51 | ".config")) 52 | (cl-json:*json-symbols-package* *package*)) 53 | (setq *config* (with-open-file (f config-path) 54 | (cl-json:decode-json f))))) 55 | 56 | (defun config-var (var) (cdr (assoc var *config*))) 57 | (defun (setf config-var) (new-value var) 58 | (setf (cdr (assoc var *config*)) new-value) 59 | ;; say, the user wants the python process to be project local 60 | (unless (eq var 'pycmd) (save-config)) 61 | (when (python-alive-p) (pycall "_py4cl_load_config"))) 62 | 63 | (defun py-cd (path) 64 | (pyexec "import os") 65 | (pycall "os.chdir" path)) 66 | -------------------------------------------------------------------------------- /src/do-after-load.lisp: -------------------------------------------------------------------------------- 1 | ;;; probably, ASDF should have a feature for doing this 2 | ;;; this file should be called after loading all the other files 3 | 4 | (in-package :py4cl) 5 | (let ((config-path (concatenate 'string 6 | (directory-namestring py4cl/config:*base-directory*) 7 | ".config")) 8 | (cl-json:*json-symbols-package* *package*)) 9 | (when (uiop:file-exists-p config-path) 10 | (load-config))) 11 | -------------------------------------------------------------------------------- /src/import-export.lisp: -------------------------------------------------------------------------------- 1 | ;;; Functions and macros for importing and exporting symbols to python 2 | 3 | (in-package :py4cl) 4 | 5 | (defmacro import-function (fun-name &key docstring 6 | (as (read-from-string fun-name)) 7 | from) 8 | "Define a function which calls python 9 | Example 10 | (py4cl:python-exec \"import math\") 11 | (py4cl:import-function \"math.sqrt\") 12 | (math.sqrt 42) 13 | -> 6.4807405 14 | 15 | Keywords: 16 | 17 | AS specifies the symbol to be used in Lisp. This can be a symbol 18 | or a string. If a string is given then it is read using READ-FROM-STRING. 19 | 20 | DOCSTRING is a string which becomes the function docstring 21 | 22 | FROM specifies a module to load the function from. This will cause the python 23 | module to be imported into the python session. 24 | " 25 | ;; Note: a string input is used, since python is case sensitive 26 | (unless (typep fun-name 'string) 27 | (error "Argument to IMPORT-FUNCTION must be a string")) 28 | 29 | (if from 30 | (progn 31 | ;; Ensure that python is running 32 | (python-start-if-not-alive) 33 | ;; import the function into python 34 | (python-exec "from " (string from) " import " fun-name))) 35 | 36 | ;; Input AS specifies the Lisp symbol, either as a string or a symbol 37 | (let ((fun-symbol (typecase as 38 | (string (read-from-string as)) 39 | (symbol as) 40 | (t (error "AS keyword must be string or symbol"))))) 41 | 42 | `(defun ,fun-symbol (&rest args) 43 | ,(or docstring "Python function") 44 | (apply #'python-call ,fun-name args)))) 45 | 46 | (defmacro import-module (module-name &key (as module-name as-supplied-p) (reload nil)) 47 | "Import a python module as a Lisp package. The module name should be 48 | a string. 49 | 50 | Example: 51 | (py4cl:import-module \"math\") 52 | (math:sqrt 4) ; => 2.0 53 | 54 | or using 55 | Keywords: 56 | AS specifies the name to be used for both the Lisp package and python module. 57 | It should be a string, and if not supplied then the module name is used. 58 | 59 | RELOAD specifies that the package should be deleted and reloaded. 60 | By default if the package already exists then a string is returned. 61 | " 62 | (unless (typep module-name 'string) 63 | (error "Argument to IMPORT-MODULE must be a string")) 64 | (unless (typep as 'string) 65 | (error "Keyword argument AS to IMPORT-MODULE must be a string")) 66 | 67 | ;; Check if the package already exists, and delete if reload is true 68 | ;; This is so that it is reloaded into python 69 | (let ((package-sym (read-from-string as))) 70 | (if (find-package package-sym) 71 | (if reload 72 | (delete-package package-sym) 73 | (return-from import-module "Package already exists.")))) 74 | 75 | ;; Ensure that python is running 76 | (python-start-if-not-alive) 77 | 78 | ;; Import the required module in python 79 | (if as-supplied-p 80 | (python-exec (concatenate 'string 81 | "import " module-name " as " as)) 82 | (python-exec (concatenate 'string 83 | "import " module-name))) 84 | 85 | ;; Also need to import the "inspect" module 86 | (python-exec "import inspect") 87 | 88 | ;; fn-names All callables whose names don't start with "_" 89 | (let* ((fn-names (python-eval (concatenate 'string 90 | "[name for name, fn in inspect.getmembers(" 91 | as 92 | ", callable) if name[0] != '_']"))) 93 | ;; Get the package name by passing through reader, rather than using STRING-UPCASE 94 | ;; so that the result reflects changes to the readtable 95 | ;; Setting *package* causes symbols to be interned by READ-FROM-STRING in this package 96 | ;; Note that the package doesn't use CL to avoid shadowing 97 | (*package* (make-package (string (read-from-string as)) 98 | :use '())) 99 | (fun-symbols (map 'list 100 | (lambda (fun-name) 101 | (read-from-string fun-name)) 102 | fn-names))) 103 | (import '(cl:nil)) ; So that missing docstring is handled 104 | `(progn 105 | ,(macroexpand `(defpackage ,(package-name *package*) 106 | (:use) 107 | (:export ,@fun-symbols))) 108 | ,@(loop for name across fn-names 109 | for fn-symbol = (read-from-string name) 110 | for fullname = (concatenate 'string as "." name) ; Include module prefix 111 | append `((import-function ,fullname :as ,fn-symbol 112 | :docstring ,(python-eval (concatenate 'string 113 | as "." name ".__doc__"))) 114 | (export ',fn-symbol ,*package*))) 115 | t))) 116 | 117 | (defun export-function (function python-name) 118 | "Makes a lisp FUNCTION available in python process as PYTHON-NAME" 119 | (python-exec (concatenate 'string 120 | python-name 121 | "=_py4cl_LispCallbackObject(" 122 | (write-to-string 123 | (object-handle function)) 124 | ")"))) 125 | 126 | -------------------------------------------------------------------------------- /src/lisp-classes.lisp: -------------------------------------------------------------------------------- 1 | (in-package :py4cl) 2 | 3 | (defgeneric python-getattr (object slot-name) 4 | (:documentation "Called when python accesses an object's slot (__getattr__)")) 5 | 6 | (defgeneric python-setattr (object slot-name value) 7 | (:documentation "Called when python sets an object's slot (__setattr__)")) 8 | -------------------------------------------------------------------------------- /src/package.lisp: -------------------------------------------------------------------------------- 1 | ;;;; package.lisp 2 | 3 | (defpackage #:py4cl 4 | (:use #:cl) 5 | (:export ; python-process 6 | #:*python-command* ; The executable to run (string) 7 | #:python-start 8 | #:python-stop 9 | #:python-alive-p 10 | #:python-start-if-not-alive 11 | #:python-version-info 12 | #:python-interrupt) 13 | (:export ; callpython 14 | #:python-error 15 | #:python-eval 16 | #:python-exec 17 | #:python-call 18 | #:python-call-async 19 | #:python-method 20 | #:python-generator 21 | #:chain 22 | #:python-setf 23 | #:remote-objects 24 | #:remote-objects*) 25 | (:export ; import-export 26 | #:import-function 27 | #:import-module 28 | #:export-function) 29 | (:export ; lisp-classes 30 | #:python-getattr 31 | #:python-setattr) 32 | (:export ; config 33 | #:*config* 34 | #:initialize 35 | #:save-config 36 | #:load-config 37 | #:config-var 38 | #:pycmd 39 | #:numpy-pickle-location 40 | #:numpy-pickle-lower-bound 41 | #:py-cd)) 42 | -------------------------------------------------------------------------------- /src/python-process.lisp: -------------------------------------------------------------------------------- 1 | ;;; Functions to start and stop python process 2 | 3 | (in-package :py4cl) 4 | 5 | (defvar *python-command* "python" 6 | "String, the Python executable to launch 7 | e.g. \"python\" or \"python3\"") 8 | 9 | (defvar *python* nil 10 | "Most recently started python subprocess") 11 | 12 | (defvar *current-python-process-id* 0 13 | "A number which changes when python is started. This 14 | is used to prevent garbage collection from deleting objects in the wrong 15 | python session") 16 | 17 | (defun python-start (&optional (command *python-command*)) 18 | "Start a new python subprocess 19 | This sets the global variable *python* to the process handle, 20 | in addition to returning it. 21 | COMMAND is a string with the python executable to launch e.g. \"python\" 22 | By default this is is set to *PYTHON-COMMAND* 23 | " 24 | (setf *python* 25 | (apply #'uiop:launch-program 26 | (concatenate 'string 27 | "exec " 28 | command ; Run python executable 29 | " " 30 | ;; Path *base-pathname* is defined in py4cl.asd 31 | ;; Calculate full path to python script 32 | (namestring (merge-pathnames #p"py4cl.py" py4cl/config:*base-directory*))) 33 | :input :stream :output :stream 34 | #+ccl '(:sharing :lock) #-ccl nil)) 35 | (incf *current-python-process-id*)) 36 | 37 | (defun python-alive-p (&optional (process *python*)) 38 | "Returns non-NIL if the python process is alive 39 | (e.g. SBCL -> T, CCL -> RUNNING). 40 | Optionally pass the process object returned by PYTHON-START" 41 | (and process 42 | (uiop:process-alive-p process))) 43 | 44 | (defun python-start-if-not-alive () 45 | "If no python process is running, tries to start it. 46 | If still not alive, raises a condition." 47 | (unless (python-alive-p) 48 | (python-start)) 49 | (unless (python-alive-p) 50 | (error "Could not start python process"))) 51 | 52 | ;; Function defined in writer.lisp, which clears an object store 53 | (declaim (ftype (function () t) clear-lisp-objects)) 54 | 55 | (defun python-stop (&optional (process *python*)) 56 | ;; If python is not running then return 57 | (unless (python-alive-p process) 58 | (return-from python-stop)) 59 | 60 | ;; First ask python subprocess to quit 61 | ;; Could give it a few seconds to close nicely 62 | (let ((stream (uiop:process-info-input process))) 63 | (write-char #\q stream)) 64 | ;; Close input, output streams 65 | (uiop:close-streams process) 66 | ;; Terminate 67 | (uiop:terminate-process process) 68 | ;; Mark as not alive 69 | (setf *python* nil) 70 | 71 | ;; Clear lisp objects 72 | (clear-lisp-objects)) 73 | 74 | (defvar *py4cl-tests* nil "Set nil for something like py4cl/tests::interrupt test, 75 | for unknown reasons.") 76 | (defun python-interrupt (&optional (process-info *python*)) 77 | (when (python-alive-p process-info) 78 | (uiop:run-program 79 | (concatenate 'string "/bin/kill -SIGINT -" 80 | (write-to-string (uiop:process-info-pid process-info))) 81 | :force-shell t) 82 | ;; something to do with running in separate threads! "deftest interrupt" 83 | (unless *py4cl-tests* (dispatch-messages process-info)))) 84 | 85 | (defun python-version-info () 86 | "Return a list, using the result of python's sys.version_info." 87 | (python-start-if-not-alive) 88 | (let ((stream (uiop:process-info-input *python*))) 89 | (write-char #\v stream) 90 | (force-output stream)) 91 | (dispatch-messages *python*)) 92 | -------------------------------------------------------------------------------- /src/reader.lisp: -------------------------------------------------------------------------------- 1 | ;;; Code to read from python process over a stream 2 | 3 | (in-package :py4cl) 4 | 5 | (defstruct python-object 6 | "A handle for a python object 7 | which couldn't be translated into a Lisp value. 8 | TYPE slot is the python type string 9 | HANDLE slot is a unique key used to refer to a value in python." 10 | (type "" :type string) 11 | handle) 12 | 13 | (defvar *freed-python-objects* nil 14 | "A list of handles to be freed. This is used because garbage collection may occur in parallel with the main thread.") 15 | 16 | (defun free-python-object (python-id handle) 17 | (push (list python-id handle) *freed-python-objects*)) 18 | 19 | (defun delete-freed-python-objects () 20 | ;; Remove (python-id handle) pairs from the list and process 21 | (loop for id-handle = (pop *freed-python-objects*) 22 | while id-handle 23 | do (let ((python-id (first id-handle)) 24 | (handle (second id-handle))) 25 | (if (and 26 | (python-alive-p) ; If not alive, python-exec will start python 27 | (= *current-python-process-id* python-id)) ; Python might have restarted 28 | ;; Call the internal function, to avoid infinite recursion or deadlock 29 | (python-eval* #\x " 30 | try: 31 | del _py4cl_objects[" handle "] 32 | except: 33 | pass")))) 34 | (delete-numpy-pickle-arrays)) 35 | 36 | (defun delete-numpy-pickle-arrays () 37 | "Delete pickled arrays, to free space." 38 | (loop :while (> *numpy-pickle-index* 0) 39 | :do (decf *numpy-pickle-index*) 40 | (uiop:delete-file-if-exists 41 | (concatenate 'string 42 | (config-var 'numpy-pickle-location) 43 | ".to." (write-to-string *numpy-pickle-index*))))) 44 | 45 | (defun make-python-object-finalize (&key (type "") handle) 46 | "Make a PYTHON-OBJECT struct with a finalizer. 47 | This deletes the object from the dict store in python. 48 | 49 | Uses trivial-garbage (public domain) 50 | " 51 | (tg:finalize 52 | (make-python-object :type type 53 | :handle handle) 54 | (let ((python-id *current-python-process-id*)) 55 | (lambda () ; This function is called when the python-object is garbage collected 56 | (ignore-errors 57 | ;; Put on a list to free later. Garbage collection may happen 58 | ;; in parallel with the main thread, which may be executing other commands. 59 | (free-python-object python-id handle)))))) 60 | 61 | (defun stream-read-string (stream) 62 | "Reads a string from a stream 63 | Expects a line containing the number of chars following 64 | e.g. '5~%hello' 65 | Returns the string or nil on error 66 | " 67 | (let ((nchars (parse-integer (read-line stream)))) 68 | (with-output-to-string (str) 69 | (loop for i from 1 to nchars do 70 | (write-char (read-char stream) str))))) 71 | 72 | (defun stream-read-value (stream) 73 | "Get a value from a stream 74 | Currently works by reading a string then using read-from-string 75 | " 76 | (let ((str (stream-read-string stream))) 77 | (multiple-value-bind (value count) 78 | (read-from-string str) 79 | ;; Check if all characters were used 80 | (unless (eql count (length str)) 81 | (error (concatenate 'string "unread characters in reading string \"" str "\""))) 82 | value))) 83 | -------------------------------------------------------------------------------- /src/writer.lisp: -------------------------------------------------------------------------------- 1 | ;;; Write data to python over a stream 2 | 3 | (in-package :py4cl) 4 | 5 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 6 | ;;; 7 | ;;; Object Handles 8 | 9 | (defvar *handle-counter* 0) 10 | 11 | (defvar *lisp-objects* (make-hash-table :test #'eql)) 12 | 13 | (defun clear-lisp-objects () 14 | "Clear the *lisp-objects* object store, allowing them to be GC'd" 15 | (setf *lisp-objects* (make-hash-table :test #'eql) 16 | *handle-counter* 0)) 17 | 18 | (defun free-handle (handle) 19 | "Remove an object with HANDLE from the hash table" 20 | (remhash handle *lisp-objects*)) 21 | 22 | (defun lisp-object (handle) 23 | "Get the lisp object corresponding to HANDLE" 24 | (or (gethash handle *lisp-objects*) 25 | (error "Invalid Handle."))) 26 | 27 | (defun object-handle (object) 28 | "Store OBJECT and return a handle" 29 | (let ((handle (incf *handle-counter*))) 30 | (setf (gethash handle *lisp-objects*) object) 31 | handle)) 32 | 33 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 34 | ;;; Convert objects to a form which python can eval 35 | 36 | (defgeneric pythonize (obj) 37 | (:documentation 38 | "Convert an object into a string which can be written to stream. 39 | Default implementation creates a handle to an unknown Lisp object.") 40 | (:method (obj) 41 | (concatenate 'string 42 | "_py4cl_UnknownLispObject(\"" 43 | (write-to-string 44 | (type-of obj)) 45 | "\", " 46 | (write-to-string 47 | (object-handle obj)) 48 | ")"))) 49 | 50 | (defmethod pythonize ((obj real)) 51 | "Write a real number. 52 | Note that python doesn't handle 'd','f', 's' or 'L' exponent markers" 53 | (substitute-if #\e (lambda (ch) 54 | (member ch '(#\d #\D #\f #\F #\s #\S #\l #\L))) 55 | (write-to-string obj))) 56 | 57 | (defmethod pythonize ((obj complex)) 58 | "Create string of the form \"(1+2j\". 59 | If imaginary part is negative the output is of form \"(1+-2j\" 60 | which is interpreted correctly by python (3.7.2)." 61 | (concatenate 'string 62 | "(" 63 | (write-to-string (realpart obj)) 64 | "+" 65 | (write-to-string (imagpart obj)) 66 | "j)")) 67 | 68 | (defvar *numpy-pickle-index* 0 69 | "Used for transferring multiple numpy-pickled arrays in one pyeval/exec/etc") 70 | ;; this is incremented by pythonize and reset to 0 at the beginning of 71 | ;; every pyeval*/pycall from delete-numpy-pickle-arrays in reader.lisp 72 | (defmethod pythonize ((obj array)) 73 | 74 | ;; Transfer large arrays via pickling 75 | (when (and (config-var 'numpy-pickle-lower-bound) 76 | (config-var 'numpy-pickle-location) 77 | (> (array-total-size obj) 78 | (config-var 'numpy-pickle-lower-bound)) 79 | (py4cl:python-eval "_py4cl_numpy_is_loaded")) 80 | (let ((filename (concatenate 'string 81 | (config-var 'numpy-pickle-location) 82 | ".to." (write-to-string *numpy-pickle-index*)))) 83 | (incf *numpy-pickle-index*) 84 | (numpy-file-format:store-array obj filename) 85 | (return-from pythonize 86 | (concatenate 'string "_py4cl_load_pickled_ndarray('" 87 | filename"')")))) 88 | 89 | ;; Handle case of empty array 90 | (if (= (array-total-size obj) 0) 91 | (return-from pythonize "[]")) 92 | 93 | ;; First convert the array to 1D [0,1,2,3,...] 94 | (let ((array1d (with-output-to-string (stream) 95 | (write-char #\[ stream) 96 | (princ (pythonize (row-major-aref obj 0)) stream) 97 | (do ((indx 1 (1+ indx))) 98 | ((>= indx (array-total-size obj))) 99 | (write-char #\, stream) 100 | (princ (pythonize (row-major-aref obj indx)) stream)) 101 | (write-char #\] stream)))) 102 | (if (= (array-rank obj) 1) 103 | ;; 1D array return as-is 104 | array1d 105 | ;; Multi-dimensional array. Call NumPy to resize 106 | (concatenate 'string 107 | "_py4cl_numpy.resize(" array1d ", " 108 | (pythonize (array-dimensions obj)) ")")))) 109 | 110 | (defmethod pythonize ((obj cons)) 111 | "Convert a list. This leaves a trailing comma so that python 112 | evals a list with a single element as a tuple 113 | " 114 | (with-output-to-string (stream) 115 | (write-char #\( stream) 116 | (dolist (val obj) 117 | (write-string (pythonize val) stream) 118 | (write-char #\, stream)) 119 | (write-char #\) stream))) 120 | 121 | (defmethod pythonize ((obj string)) 122 | (format nil (if (find #\newline obj) 123 | "\"\"~A\"\"" 124 | "~A") 125 | (write-to-string (coerce obj '(vector character)) 126 | :escape t :readably t))) 127 | 128 | (defmethod pythonize ((obj symbol)) 129 | "Handle symbols. Need to handle NIL, 130 | converting it to Python None, and convert T to True." 131 | (if obj 132 | (if (eq obj t) 133 | "True" 134 | (concatenate 'string 135 | "_py4cl_Symbol(':" (string-downcase (string obj)) "')")) 136 | "None")) 137 | 138 | (defmethod pythonize ((obj hash-table)) 139 | "Convert hash-table to python map. 140 | Produces a string {key1:value1, key2:value2,}" 141 | (concatenate 'string 142 | "{" 143 | (apply #'concatenate 'string 144 | (loop for key being the hash-keys of obj 145 | using (hash-value value) 146 | appending (list (pythonize key) ":" (pythonize value) ","))) 147 | "}")) 148 | 149 | (defmethod pythonize ((obj function)) 150 | "Handle a function by converting to a callback object 151 | The lisp function is stored in the same object store as other objects." 152 | (concatenate 'string 153 | "_py4cl_LispCallbackObject(" 154 | (write-to-string 155 | (object-handle obj)) 156 | ")")) 157 | 158 | (defmethod pythonize ((obj python-object)) 159 | "A handle for a python object, stored in a dict in Python" 160 | (concatenate 'string 161 | "_py4cl_objects[" 162 | (write-to-string (python-object-handle obj)) 163 | "]")) 164 | 165 | (defmethod pythonize ((obj ratio)) 166 | "Handle ratios, using Python's Fraction if available" 167 | (concatenate 'string 168 | "_py4cl_fraction(" 169 | (pythonize (numerator obj)) 170 | "," 171 | (pythonize (denominator obj)) 172 | ")")) 173 | 174 | (defun stream-write-string (str stream) 175 | "Write a string to a stream, putting the length first" 176 | ;; Convert the value to a string 177 | (princ (length str) stream) ; Header, so length of string is known to reader 178 | (terpri stream) 179 | (write-string str stream)) 180 | 181 | (defun stream-write-value (value stream) 182 | "Write a value to a stream, in a format which can be read 183 | by the python subprocess as the corresponding python type" 184 | (stream-write-string (pythonize value) stream)) 185 | -------------------------------------------------------------------------------- /tests/tests.lisp: -------------------------------------------------------------------------------- 1 | (py4cl:import-module "math" :reload t) 2 | (defpackage #:py4cl/tests 3 | (:use #:cl #:clunit) 4 | (:export #:run)) 5 | 6 | (in-package :py4cl/tests) 7 | 8 | (defsuite tests ()) 9 | 10 | ;; Tests which start and stop python 11 | (defsuite pytests (tests)) 12 | 13 | (deffixture pytests (@body) 14 | (py4cl:python-start) 15 | @body 16 | (py4cl:python-stop)) 17 | 18 | (defun run (&optional interactive?) 19 | "Run all the tests for py4cl." 20 | (run-suite 'tests :use-debugger interactive?)) 21 | 22 | ;; Start and stop Python, check that python-alive-p responds 23 | (deftest start-stop (tests) 24 | (assert-false (py4cl:python-alive-p)) 25 | (py4cl:python-start) 26 | (assert-true (py4cl:python-alive-p)) 27 | (py4cl:python-stop) 28 | (assert-false (py4cl:python-alive-p))) 29 | 30 | (deftest eval-integer (pytests) 31 | (let ((result (py4cl:python-eval "1 + 2 * 3"))) 32 | (assert-true (typep result 'integer)) 33 | (assert-equalp 7 result))) 34 | 35 | (deftest eval-malformed (pytests) 36 | (assert-condition py4cl:python-error 37 | (py4cl:python-eval "1 + "))) 38 | 39 | (deftest eval-real (pytests) 40 | (let ((result (py4cl:python-eval "1.3 + 2.2"))) 41 | (assert-true (typep result 'real)) 42 | (assert-equalp 3.5 result))) 43 | 44 | (deftest eval-vector (pytests) 45 | (let ((result (py4cl:python-eval "[i**2 for i in range(4)]"))) 46 | (assert-true (typep result 'array)) 47 | (assert-equalp #(0 1 4 9) result))) 48 | 49 | (deftest eval-list (pytests) 50 | (let ((result (py4cl:python-eval "(1,2,3)"))) 51 | (assert-true (typep result 'cons)) 52 | (assert-equalp '(1 2 3) result))) 53 | 54 | ;; Check passing strings, including quote characters which need to be escaped 55 | (deftest eval-string (pytests) 56 | (assert-equalp "say \"hello\" world" 57 | (py4cl:python-eval "'say \"hello\"' + ' world'"))) 58 | 59 | (deftest eval-string-newline (pytests) 60 | (let ((str "hello 61 | world")) 62 | (assert-equalp str (py4cl:python-eval (py4cl::pythonize str))))) 63 | 64 | (deftest pythonize-format-string (tests) 65 | (assert-equalp "\"foo\"" 66 | (py4cl::pythonize (format nil "foo")))) 67 | 68 | (deftest eval-format-string (pytests) 69 | (assert-equalp "foo" 70 | (py4cl:python-eval 71 | (py4cl::pythonize (format nil "foo"))))) 72 | 73 | ;; This tests whether outputs to stdout mess up the return stream 74 | (deftest eval-print (pytests) 75 | (unless (= 2 (first (py4cl:python-version-info))) 76 | ;; Should return the result of print, not the string printed 77 | (assert-equalp nil 78 | (py4cl:python-eval "print(\"hello\")") 79 | "This fails with python 2") 80 | 81 | ;; Should print the output to stdout 82 | (assert-equalp "hello world 83 | " 84 | (with-output-to-string (*standard-output*) 85 | (py4cl:chain (print "hello world")))) 86 | 87 | ;; Check that the output is cleared, by printing a shorter string 88 | (assert-equalp "testing 89 | " 90 | (with-output-to-string (*standard-output*) 91 | (py4cl:chain (print "testing")))))) 92 | 93 | (deftest eval-params (pytests) 94 | ;; Values are converted into python values 95 | (let ((a 4) 96 | (b 7)) 97 | (assert-equalp 11 98 | (py4cl:python-eval a "+" b))) 99 | 100 | ;; Arrays can also be passed 101 | (assert-equalp #2A((1 2) (3 4)) 102 | (py4cl:python-eval #2A((1 2) (3 4)))) 103 | 104 | (assert-equalp #2A((2 4) (6 8)) 105 | (py4cl:python-eval #2A((1 2) (3 4)) "*" 2)) 106 | 107 | (assert-equalp #3A(((2 4) (7 8)) ((8 5) (1 6))) 108 | (py4cl:python-eval #3A(((1 3) (6 7)) ((7 4) (0 5))) "+" 1)) 109 | 110 | ;; Test handling of real numbers in arrays 111 | (assert-equalp #(1.0 2.0) 112 | (py4cl:python-eval (vector 1.0 2.0))) 113 | 114 | ;; Test empty arrays 115 | (assert-equalp #() 116 | (py4cl:python-eval #())) 117 | 118 | ;; Unless the values are strings 119 | (let ((str "hello")) 120 | (assert-condition py4cl:python-error 121 | (py4cl:python-eval "len(" str ")")) ; "len(hello)" 122 | 123 | ;; To pass a string to python, run through pythonize: 124 | (assert-equalp 5 125 | (py4cl:python-eval "len(" (py4cl::pythonize str) ")")))) 126 | 127 | (deftest complex-values (pytests) 128 | ;; Single values 129 | (assert-equality #'= #C(1 2) 130 | (py4cl:python-eval #C(1 2))) 131 | (assert-equality #'= #C(1 -2) 132 | (py4cl:python-eval #C(1 -2))) 133 | (assert-equality #'= #C(-1 -2) 134 | (py4cl:python-eval #C(-1 -2))) 135 | 136 | ;; Expressions. Tested using multiply to catch things like 137 | ;; "1+2j * 2+3j -> 1+7j rather than (-4+7j) 138 | ;; Note: Python doesn't have complex integers, so all returned 139 | ;; values could be floats 140 | (assert-equality #'= #C(-4 7) 141 | (py4cl:python-eval #C(1 2) "*" #C(2 3))) 142 | (assert-equality #'= #C(4 7) 143 | (py4cl:python-eval #C(1 -2) "*" #C(-2 3))) 144 | 145 | ;; Lists of complex numbers 146 | (assert-equality #'= #C(6 9) 147 | (py4cl:python-call "sum" (list #C(1 2) #C(2 3) #C(3 4))))) 148 | 149 | (deftest exec-print (pytests) 150 | (unless (= 2 (first (py4cl:python-version-info))) 151 | ;; Python 3 152 | (assert-equalp nil 153 | (py4cl:python-exec "print(\"hello\")") 154 | "This fails with python 2"))) 155 | 156 | (deftest call-lambda-no-args (pytests) 157 | (assert-equalp 3 158 | (py4cl:python-call "lambda : 3"))) 159 | 160 | (deftest call-one-arg-int (pytests) 161 | (assert-equalp 42 162 | (py4cl:python-call "abs" -42))) 163 | 164 | (deftest call-one-arg-list (pytests) 165 | (assert-equalp 9 166 | (py4cl:python-call "sum" '(3 2 4)))) 167 | 168 | (deftest call-one-arg-string (pytests) 169 | (assert-equalp #("h" "e" "l" "l" "o") 170 | (py4cl:python-call "list" "hello"))) 171 | 172 | (deftest call-dotted-function (pytests) 173 | (py4cl:python-exec "import math") 174 | (assert-equalp (sqrt 42) 175 | (py4cl:python-call "math.sqrt" 42))) 176 | 177 | (deftest call-lambda-function (pytests) 178 | (assert-equalp 16 179 | (py4cl:python-call "lambda x: x*x" 4))) 180 | 181 | (deftest call-lambda-function-two-args (pytests) 182 | (assert-equalp 10 183 | (py4cl:python-call "lambda x, y: x*y - y" 3 5))) 184 | 185 | (deftest call-lambda-keywords (pytests) 186 | (assert-equalp -1 187 | (py4cl:python-call "lambda a=0, b=1: a-b" :b 2 :a 1)) 188 | (assert-equalp 1 189 | (py4cl:python-call "lambda a=0, b=1: a-b" :a 2 :b 1))) 190 | 191 | (deftest call-with-lambda-callback (pytests) 192 | ;; Define a function in python which calls its argument 193 | (py4cl:python-exec "runme = lambda f: f()") 194 | ;; Pass a lambda function to python-call 195 | (assert-equalp 42 196 | (py4cl:python-call "runme" (lambda () 42)))) 197 | 198 | (py4cl:import-function "sum") 199 | (deftest import-function-sum (pytests) 200 | (assert-equalp 6 201 | (sum '(2 1 3)))) 202 | 203 | (deftest call-return-numpy-types (pytests) 204 | (py4cl:python-exec "import numpy as np") 205 | (assert-equalp 42.0 206 | (py4cl:python-eval "np.float64(42.0)"))) 207 | 208 | ;; Simple callback function 209 | (defun test-func () 210 | 42) 211 | 212 | (deftest callback-no-args (pytests) 213 | (py4cl:export-function #'test-func "test") 214 | (assert-equalp 42 215 | (py4cl:python-eval "test()"))) 216 | 217 | ;; Even simpler function returning NIL 218 | (defun nil-func () 219 | nil) 220 | 221 | (deftest callback-no-args-return-nil (pytests) 222 | (py4cl:export-function #'nil-func "test_nil") 223 | (assert-equalp nil 224 | (py4cl:python-eval "test_nil()"))) 225 | 226 | ;; Python can't eval write-to-string's output "3.141592653589793d0" 227 | (deftest callback-return-double (pytests) 228 | (py4cl:export-function (lambda () pi) "test") 229 | (assert-equalp 3.1415927 230 | (py4cl:python-eval "test()"))) 231 | 232 | (deftest callback-one-arg (pytests) 233 | (py4cl:export-function (lambda (x) (* 2 x)) "double") 234 | (assert-equalp 4 235 | (py4cl:python-eval "double(2)"))) 236 | 237 | (deftest callback-two-args (pytests) 238 | (py4cl:export-function (lambda (x y) (/ x y)) "div") 239 | (assert-equalp 3 240 | (py4cl:python-eval "div(6, 2)"))) 241 | 242 | (deftest callback-many-args (pytests) 243 | (py4cl:export-function #'+ "add") 244 | (assert-equalp 15 245 | (py4cl:python-eval "add(2, 4, 6, 3)"))) 246 | 247 | (deftest callback-seq-arg (pytests) 248 | (py4cl:export-function #'reverse "reverse") 249 | (assert-equalp '(3 1 2 4) 250 | (py4cl:python-eval "reverse((4,2,1,3))")) 251 | (assert-equalp #(3 1 2 4) 252 | (py4cl:python-eval "reverse([4,2,1,3])"))) 253 | 254 | (deftest callback-keyword-arg (pytests) 255 | (py4cl:export-function (lambda (&key setting) setting) "test") 256 | (assert-equalp nil 257 | (py4cl:python-eval "test()")) 258 | (assert-equalp 42 259 | (py4cl:python-eval "test(setting=42)"))) 260 | 261 | 262 | ;; Call python during callback 263 | (deftest python-during-callback (pytests) 264 | (py4cl:export-function 265 | (lambda () (py4cl:python-eval "42")) 266 | "test") 267 | (assert-equalp 42 268 | (py4cl:python-eval "test()"))) 269 | 270 | ;; Hash-table support 271 | (deftest hash-table-empty (pytests) 272 | (assert-equalp "{}" 273 | (py4cl:python-call "str" (make-hash-table)))) 274 | 275 | (deftest hash-table-values (pytests) 276 | (let ((table (make-hash-table))) 277 | (setf (gethash "test" table) 3 278 | (gethash "more" table) 42) 279 | (assert-equalp 42 280 | (py4cl:python-call "lambda d: d[\"more\"]" table)) 281 | (assert-equalp 3 282 | (py4cl:python-call "lambda d: d[\"test\"]" table)) 283 | (assert-equalp 2 284 | (py4cl:python-call "len" table)))) 285 | 286 | (deftest hash-table-from-dict (pytests) 287 | ;; Simple keys 288 | (let ((table (py4cl:python-eval "{1:2, 2:3}"))) 289 | (assert-equalp 2 290 | (gethash 1 table)) 291 | (assert-equalp 3 292 | (gethash 2 table))) 293 | 294 | ;; Ensure values are being lispified 295 | (let ((table (py4cl:python-eval "{1:[1,2,3]}"))) 296 | (assert-equalp #(1 2 3) 297 | (gethash 1 table))) 298 | 299 | ;; Ensure keys are being lispified and string keys work 300 | (let ((table (py4cl:python-eval "{\"test\":42}"))) 301 | (assert-equalp 42 302 | (gethash "test" table)))) 303 | 304 | ;; Generators 305 | (deftest generator (pytests) 306 | (assert-equalp (ecase (car (py4cl:python-version-info)) 307 | (3 "") 308 | (2 "")) 309 | (slot-value (py4cl:python-generator #'identity 3) 'type)) 310 | (py4cl:python-exec " 311 | def foo(gen): 312 | return list(gen)") 313 | (assert-equalp #(1 2 3 4) 314 | (let ((gen (py4cl:python-generator (let ((x 0)) (lambda () (incf x))) 315 | 5))) 316 | (py4cl:python-call "foo" gen))) 317 | (assert-equalp #(#\h #\e #\l #\l #\o) 318 | (let ((gen (py4cl:python-generator (let ((str (make-string-input-stream "hello"))) 319 | (lambda () (read-char str nil))) 320 | nil))) 321 | (py4cl:python-call "foo" gen)))) 322 | 323 | ;; Asyncronous functions 324 | (deftest call-function-async (pytests) 325 | (let ((thunk (py4cl:python-call-async "str" 42))) 326 | ;; returns a function which when called returns the result 327 | (assert-equalp "42" 328 | (funcall thunk)) 329 | ;; And returns the same when called again 330 | (assert-equalp "42" 331 | (funcall thunk))) 332 | 333 | ;; Check if it handles errors 334 | (let ((thunk (py4cl:python-call-async "len"))) ; TypeError 335 | (assert-condition py4cl:python-error 336 | (funcall thunk))) 337 | 338 | ;; Check that values can be requested out of order 339 | (let ((thunk1 (py4cl:python-call-async "str" 23)) 340 | (thunk2 (py4cl:python-call-async "str" 12)) 341 | (thunk3 (py4cl:python-call-async "str" 7))) 342 | (assert-equalp "12" 343 | (funcall thunk2)) 344 | (assert-equalp "7" 345 | (funcall thunk3)) 346 | (assert-equalp "23" 347 | (funcall thunk1)))) 348 | 349 | (deftest python-objects (pytests) 350 | ;; Define a simple python class containing a value 351 | (py4cl:python-exec 352 | "class Test: 353 | pass 354 | 355 | a = Test() 356 | a.value = 42") 357 | 358 | ;; Check that the variable has been defined 359 | (assert-equalp 42 360 | (py4cl:python-eval "a.value")) 361 | 362 | ;; Implementation detail: No objects stored in python dict 363 | (assert-equalp 0 364 | (py4cl:python-eval "len(_py4cl_objects)")) 365 | 366 | ;; Evaluate and return a python object 367 | (let ((var (py4cl:python-eval "a"))) 368 | ;; Implementation detail: Type of returned object 369 | (assert-equalp 'PY4CL::PYTHON-OBJECT 370 | (type-of var)) 371 | 372 | ;; Implementation detail: Object is stored in a dictionary 373 | (assert-equalp 1 374 | (py4cl:python-eval "len(_py4cl_objects)")) 375 | 376 | ;; Can pass to eval to use dot accessor 377 | (assert-equalp 42 378 | (py4cl:python-eval var ".value")) 379 | 380 | ;; Can pass as argument to function 381 | (assert-equal 84 382 | (py4cl:python-call "lambda x : x.value * 2" var))) 383 | 384 | ;; Trigger a garbage collection so that VAR is finalized. 385 | ;; This should also delete the object in python 386 | (tg:gc :full t) 387 | 388 | ;; Implementation detail: dict object store should be empty 389 | ;; Note: This is dependent on the CL implementation. Trivial-garbage 390 | ;; doesn't seem to support ccl 391 | #-clozure (assert-equalp 0 392 | (py4cl::python-eval "len(_py4cl_objects)"))) 393 | 394 | (deftest python-del-objects (tests) 395 | ;; Check that finalizing objects doesn't start python 396 | (py4cl:python-start) 397 | (py4cl:python-exec 398 | "class Test: 399 | pass 400 | 401 | a = Test()") 402 | (let ((var (py4cl:python-eval "a"))) 403 | ;; Implementation detail: Type of returned object 404 | (assert-equalp 'PY4CL::PYTHON-OBJECT 405 | (type-of var)) 406 | 407 | (py4cl:python-stop) 408 | (assert-false (py4cl:python-alive-p))) 409 | 410 | ;; VAR out of scope. Make sure it's finalized 411 | (tg:gc :full t) 412 | 413 | (assert-false (py4cl:python-alive-p))) 414 | 415 | (deftest python-method (pytests) 416 | (assert-equalp 3 417 | (py4cl:python-method '(1 2 3) '__len__)) 418 | (assert-equalp "hello world" 419 | (py4cl:python-method "hello {0}" 'format "world"))) 420 | 421 | 422 | ;; Shorter more convenient slicing 423 | (py4cl:import-function "slice") 424 | 425 | (deftest chain (pytests) 426 | (assert-equalp "Hello world" 427 | (py4cl:chain "hello {0}" (format "world") (capitalize))) 428 | (assert-equalp "hello world" 429 | (let ((format-str "hello {0}") 430 | (argument "world")) 431 | (py4cl:chain format-str (format argument)))) 432 | (assert-equalp "result: 3" 433 | (py4cl:chain "result: {0}" (format (+ 1 2)))) 434 | (assert-equalp 3 435 | (py4cl:chain (slice 3) stop)) 436 | 437 | ;; Anything not a list or a symbol is put between [] brackets (__getitem__) 438 | (assert-equalp "o" 439 | (py4cl:chain "hello" 4)) 440 | 441 | ;; [] operator for indexing and slicing (alias for __getitem__) 442 | 443 | (assert-equalp "l" 444 | (py4cl:chain "hello" ([] 3))) 445 | (assert-equalp 3 446 | (py4cl:chain #2A((1 2) (3 4)) ([] 1 0))) 447 | (assert-equalp #(4 5) 448 | (py4cl:chain #2A((1 2 3) (4 5 6)) ([] 1 (slice 0 2)))) 449 | 450 | (let ((dict (py4cl:python-eval "{\"hello\":\"world\", \"ping\":\"pong\"}"))) 451 | (assert-equalp "world" 452 | (py4cl:chain dict "hello")) 453 | (assert-equalp "pong" 454 | (py4cl:chain dict ([] "ping"))))) 455 | 456 | (deftest chain-keywords (pytests) 457 | (py4cl:python-exec 458 | "def test_fn(arg, key=1): 459 | return arg * key") 460 | 461 | (assert-equalp 3 462 | (py4cl:chain (test_fn 3))) 463 | (assert-equalp 6 464 | (py4cl:chain (test_fn 3 :key 2))) 465 | 466 | (py4cl:python-exec 467 | "class testclass: 468 | def run(self, dummy = 1, value = 42): 469 | return value") 470 | 471 | (assert-equalp 42 472 | (py4cl:chain (testclass) (run))) 473 | 474 | (assert-equalp 31 475 | (py4cl:chain (testclass) (run :value 31)))) 476 | 477 | 478 | (deftest chain-strings (pytests) 479 | (py4cl:python-exec 480 | "class TestClass: 481 | def doThing(self, dummy = 1, value = 42): 482 | return value") 483 | 484 | (assert-equalp 42 485 | (py4cl:chain ("TestClass") ("doThing"))) 486 | 487 | (assert-equalp 31 488 | (py4cl:chain ("TestClass") ("doThing" :value 31)))) 489 | 490 | (deftest remote-objects (pytests) 491 | ;; REMOTE-OBJECTS returns a handle 492 | (assert-equalp 'py4cl::python-object 493 | (type-of (py4cl:remote-objects (py4cl:python-eval "1+2")))) 494 | 495 | ;; REMOTE-OBJECTS* returns a value 496 | (assert-equalp 3 497 | (py4cl:remote-objects* (py4cl:python-eval "1+2"))) 498 | 499 | (assert-equalp 3 500 | (py4cl:python-eval 501 | (py4cl:remote-objects (py4cl:python-eval "1+2")))) 502 | 503 | ;; Nested remote-object environments 504 | 505 | (assert-equalp 'py4cl::python-object 506 | (type-of (py4cl:remote-objects 507 | (py4cl:remote-objects (py4cl:python-eval "1+2")) 508 | (py4cl:python-eval "1+2"))))) 509 | 510 | (deftest call-callable-object (pytests) 511 | (assert-equalp 6 512 | (py4cl:python-call (py4cl:python-eval "lambda x : 2*x") 3))) 513 | 514 | (deftest setf-eval (pytests) 515 | (setf (py4cl:python-eval "test_value") 42) ; Set a variable 516 | (assert-equalp 42 517 | (py4cl:python-eval "test_value"))) 518 | 519 | (deftest setf-chain (pytests) 520 | (assert-equalp #(0 5 2 -1) 521 | (py4cl:remote-objects* 522 | (let ((list (py4cl:python-eval "[0, 1, 2, 3]"))) 523 | (setf (py4cl:chain list ([] 1)) 5 524 | (py4cl:chain list ([] -1)) -1) 525 | list))) 526 | 527 | (assert-equalp "world" 528 | (py4cl:remote-objects* 529 | (let ((dict (py4cl:python-eval "{}"))) 530 | (setf (py4cl:chain dict ([] "hello")) "world") 531 | (py4cl:chain dict ([] "hello"))))) 532 | 533 | ;; Define an empty class which can be modified 534 | (py4cl:python-exec " 535 | class testclass: 536 | pass") 537 | 538 | (let ((obj (py4cl:chain (testclass)))) 539 | (setf (py4cl:chain obj data_attrib) 21) 540 | (assert-equalp 21 541 | (py4cl:chain obj data_attrib)))) 542 | 543 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 544 | ;;; Passing unknown lisp objects to python 545 | 546 | (defstruct test-struct 547 | x y) 548 | 549 | (deftest lisp-structs (pytests) 550 | ;; Create a struct and pass to Python 551 | (let ((result (py4cl:python-call 552 | "lambda x: x" 553 | (make-test-struct :x 1 :y 2)))) 554 | 555 | ;; Check we got back the structure 556 | (assert-true (typep result 'test-struct)) 557 | (assert-equalp 1 558 | (test-struct-x result)) 559 | (assert-equalp 2 560 | (test-struct-y result)))) 561 | 562 | (defclass test-class () 563 | ((value :initarg :value) 564 | (thing :initarg :thing))) 565 | 566 | ;; Define a method to handle slot access from python 567 | (defmethod py4cl:python-getattr ((object test-class) slot-name) 568 | (cond 569 | ((string= slot-name "value") 570 | (slot-value object 'value)) 571 | ((string= slot-name "thing") 572 | (slot-value object 'thing)) 573 | ((string= slot-name "func") 574 | (lambda (arg) (* 2 arg))) 575 | (t (call-next-method)))) 576 | 577 | (deftest lisp-class-slots (pytests) 578 | (let ((object (make-instance 'test-class :thing 23 :value 42))) 579 | ;; Slot access 580 | (assert-equalp 23 581 | (py4cl:python-call "lambda x : x.thing" object)) 582 | (assert-equalp 42 583 | (py4cl:chain object value)) 584 | 585 | ;; Function (method) call 586 | (assert-equalp 42 587 | (py4cl:chain object (func 21)))) 588 | 589 | ;; The handler should work for other objects of the same class (class-of) 590 | (let ((object2 (make-instance 'test-class :thing "hello" :value 314))) 591 | (assert-equalp "hello" 592 | (py4cl:chain object2 thing)))) 593 | 594 | 595 | ;; Class inheriting from test-class 596 | (defclass child-class (test-class) 597 | ((other :initarg :other))) 598 | 599 | ;; Define method which passes to the next method if slot not recognised 600 | (defmethod py4cl:python-getattr ((object child-class) slot-name) 601 | (cond 602 | ((string= slot-name "other") 603 | (slot-value object 'other)) 604 | (t (call-next-method)))) 605 | 606 | (deftest lisp-class-inherit (pytests) 607 | (let ((object (make-instance 'child-class :thing 23 :value 42 :other 3))) 608 | (assert-equalp 23 609 | (py4cl:python-call "lambda x : x.thing" object)) 610 | (assert-equalp 42 611 | (py4cl:chain object value)) 612 | (assert-equalp 3 613 | (py4cl:chain object other)))) 614 | 615 | ;; Define a method to handle slot writing from python 616 | (defmethod py4cl:python-setattr ((object test-class) slot-name set-to-value) 617 | (cond 618 | ((string= slot-name "value") 619 | (setf (slot-value object 'value) set-to-value)) 620 | ((string= slot-name "thing") 621 | (setf (slot-value object 'thing) set-to-value)) 622 | (t (call-next-method)))) 623 | 624 | (deftest lisp-class-set-slots (pytests) 625 | (let ((object (make-instance 'test-class :thing 23 :value 42))) 626 | 627 | ;; Set value 628 | (py4cl:python-exec object ".thing = 3") 629 | 630 | (assert-equalp 3 (slot-value object 'thing)) 631 | (assert-equalp 3 (py4cl:python-eval object ".thing")) 632 | 633 | ;; Set again 634 | (setf (py4cl:chain object thing) 72) 635 | 636 | (assert-equalp 72 (slot-value object 'thing)) 637 | (assert-equalp 72 (py4cl:chain object thing)))) 638 | 639 | (deftest callback-in-remote-objects (pytests) 640 | ;; Callbacks send values to lisp in remote-objects environments 641 | (assert-equalp 6 642 | (py4cl:remote-objects* 643 | (py4cl:python-call (lambda (x y) (* x y)) 2 3)))) 644 | 645 | 646 | (deftest unicode-string-type (pytests) 647 | ;; Python 2 and python 3 handle unicode differently 648 | ;; This just catches the use of unicode type strings in python2 649 | ;; not the use of unicode characters 650 | (assert-equal "test unicode" 651 | (py4cl:python-eval "u'test unicode'")) 652 | (assert-equal 3 653 | (gethash "pizza" 654 | (py4cl:python-eval "{u'pizza': 3}")))) 655 | 656 | ;; ============================== PICKLE ======================================= 657 | 658 | (deftest transfer-multiple-arrays (pytests) 659 | (when (and (py4cl:config-var 'py4cl:numpy-pickle-location) 660 | (py4cl:config-var 'py4cl:numpy-pickle-lower-bound)) 661 | (let ((dimensions `((,(* 2 (py4cl:config-var 'py4cl:numpy-pickle-lower-bound))) 662 | (,(* 5 (py4cl:config-var 'py4cl:numpy-pickle-lower-bound)))))) 663 | (assert-equalp dimensions 664 | (mapcar #'array-dimensions 665 | (py4cl:python-eval 666 | (list (make-array (first dimensions) :element-type 'single-float) 667 | (make-array (second dimensions) :element-type 'single-float)))) 668 | "No bound or location for pickling.")))) 669 | 670 | (deftest transfer-without-pickle (pytests) 671 | (unless (and (py4cl:config-var 'py4cl:numpy-pickle-location) 672 | (py4cl:config-var 'py4cl:numpy-pickle-lower-bound)) 673 | (assert-equalp '(100000) 674 | (array-dimensions 675 | (py4cl:python-eval (make-array 100000 :element-type 'single-float))) 676 | "Pickle bound and location is present."))) 677 | 678 | 679 | ;; ==================== PROCESS-INTERRUPT ====================================== 680 | 681 | ;; Unable to test on CCL: 682 | ;; Stream # is private to # 683 | 684 | #-ccl (deftest interrupt (pytests) 685 | (unless (= 2 (first (py4cl:python-version-info))) 686 | (let ((py4cl::*py4cl-tests* t)) 687 | (py4cl:python-stop) 688 | (py4cl:python-exec " 689 | class Foo(): 690 | def foo(self): 691 | import time 692 | import sys 693 | sys.stdout.write('hello') 694 | sys.stdout.flush() 695 | time.sleep(5) 696 | return") 697 | (assert-equalp "hello" 698 | (let* ((return-value nil) 699 | (mon-thread (bt:make-thread 700 | (lambda () 701 | (setq return-value 702 | (with-output-to-string (*standard-output*) 703 | (py4cl:python-call "Foo().foo"))))))) 704 | (sleep 1) 705 | (py4cl:python-interrupt) 706 | (bt:join-thread mon-thread) 707 | return-value)) 708 | (assert-equalp "hello" 709 | (let* ((return-value nil) 710 | (mon-thread (bt:make-thread 711 | (lambda () 712 | (setq return-value 713 | (with-output-to-string (*standard-output*) 714 | (py4cl:python-method (py4cl:python-call "Foo") 715 | 'foo))))))) 716 | (sleep 1) 717 | (py4cl:python-interrupt) 718 | (bt:join-thread mon-thread) 719 | return-value)) 720 | 721 | ;; Check if no "residue" left 722 | 723 | (assert-equalp 5 (py4cl:python-eval 5))))) 724 | 725 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 726 | ;;; Rational numbers 727 | 728 | (deftest ratios (pytests) 729 | ;; Ratios survive a round trip 730 | (assert-equalp 1/2 731 | (py4cl:python-eval 1/2)) 732 | 733 | ;; Ratios (Fractions in python) can be manipulated 734 | (assert-equalp 1/4 735 | (py4cl:python-eval 1/2 "/" 2)) 736 | 737 | ;; Complex ratios not supported in python so converts to floats 738 | (assert-equality #'= #C(0.5 1.0) 739 | (py4cl:python-eval #C(1 2) "*" 1/2))) 740 | 741 | 742 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 743 | ;;; Scope in python exec 744 | 745 | (deftest python-exec-scope (pytests) 746 | ;; Local functions are retained in scope 747 | ;; This changed in python 3.x see e.g. https://stackoverflow.com/a/24734880 748 | (assert-equalp "10 749 | " 750 | (with-output-to-string (*standard-output*) 751 | (py4cl:python-exec " 752 | def foo(): 753 | return 5 754 | 755 | def bar(): 756 | return foo() + foo() 757 | 758 | print(bar())")))) 759 | --------------------------------------------------------------------------------