├── .gitignore ├── .travis.yml ├── LICENSE ├── README.rst ├── cl4py.png ├── cl4py ├── __init__.py ├── circularity.py ├── data.py ├── lisp.py ├── py.lisp ├── reader.py └── writer.py ├── setup.py └── test ├── sample-program.lisp ├── test_backtrace.py ├── test_for_readtable.py └── test_from_readme.py /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | dist/ 3 | *.egg-info/ 4 | *.egg 5 | *.py[cod] 6 | __pycache__/ 7 | *.so 8 | *~ 9 | .cache 10 | .mypy_cache 11 | .DS_Store 12 | *.fasl 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - "3.7" 5 | - "3.8" 6 | - "3.9" 7 | # Don't test on 3.10 8 | # - "nightly" # nightly build 9 | 10 | addons: 11 | apt: 12 | update: true 13 | 14 | before_install: 15 | - sudo apt-get -y install sbcl 16 | 17 | # command to install dependencies 18 | install: 19 | - pip install -e . 20 | 21 | # command to run tests 22 | script: 23 | - pytest 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Marco Heisig 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | cl4py - Common Lisp for Python 2 | ============================== 3 | 4 | The library cl4py (pronounce as *clappy*) allows Python programs to call 5 | Common Lisp libraries. Its official mascot is the cl4py-bird: 6 | 7 | .. image:: ./cl4py.png 8 | 9 | Motivation 10 | ---------- 11 | 12 | You are a Python programmer, but you want access to some of the powerful 13 | features of Lisp, for example to compile code at run time? Or you want to 14 | use some `awesome Lisp libraries `_? Or 15 | you are a Lisp programmer and want to show your work to your Python 16 | friends. In all these cases, cl4py is here to help you. 17 | 18 | Tutorial 19 | -------- 20 | 21 | You can start any number of Lisp subprocesses within Python, like this: 22 | 23 | .. code:: python 24 | 25 | >>> import cl4py 26 | >>> lisp = cl4py.Lisp() 27 | 28 | Of course, this requires you have some Lisp installed. If not, use 29 | something like ``apt install sbcl``, ``pacman -S sbcl`` or ``brew install 30 | sbcl`` to correct this deficiency. Once you have a running Lisp process, 31 | you can execute Lisp code on it: 32 | 33 | .. code:: python 34 | 35 | # In Lisp, numbers evaluate to themselves. 36 | >>> lisp.eval( 42 ) 37 | 42 38 | 39 | # ('+', 2, 3) is a short notation for cl4py.List(cl4py.Symbol('+'), 2, 3). 40 | # For convenience, whenever a Python tuple is converted to Lisp 41 | # data, any strings therein are automatically converted to Lisp symbols. 42 | >>> lisp.eval( ('+', 2, 3) ) 43 | 5 44 | 45 | # Nested expressions are allowed, too. 46 | >>> lisp.eval( ('/', ('*', 3, 5), 2) ) 47 | Fraction(15, 2) 48 | 49 | # Use cl4py.List instead of tuples to avoid the automatic conversion of 50 | # strings to symbols. 51 | >>> lisp.eval( cl4py.List(cl4py.Symbol('STRING='), 'foo', 'bar') ) 52 | () 53 | >>> lisp.eval( cl4py.List(cl4py.Symbol('STRING='), 'foo', 'foo') ) 54 | True 55 | 56 | # Here is how you can lookup a symbol's value: 57 | >>> lisp.eval(cl4py.Symbol('*PRINT-BASE*', 'COMMON-LISP')) 58 | 10 59 | 60 | # Of course you can also use Lisp macros: 61 | >>> lisp.eval( ('loop', 'for', 'i', 'below', 5, 'collect', 'i') ) 62 | List(0, 1, 2, 3, 4) 63 | 64 | >>> lisp.eval( ('with-output-to-string', ('stream',), 65 | ('princ', 12, 'stream'), 66 | ('princ', 34, 'stream')) ) 67 | '1234' 68 | 69 | A cl4py.Lisp object not only provides ``eval``, but also methods for 70 | looking up functions and packages: 71 | 72 | .. code:: python 73 | 74 | >>> add = lisp.function('+') 75 | >>> add(1, 2, 3, 4) 76 | 10 77 | 78 | >>> div = lisp.function('/') 79 | >>> div(2, 4) 80 | Fraction(1, 2) 81 | 82 | # Lisp packages are automatically converted to Python modules. 83 | >>> cl = lisp.find_package('CL') 84 | >>> cl.oddp(5) 85 | True 86 | 87 | >>> cl.cons(5, None) 88 | List(5) 89 | 90 | >>> cl.remove(5, [1, -5, 2, 7, 5, 9], key=cl.abs) 91 | [1, 2, 7, 9] 92 | 93 | # Higher-order functions work, too! 94 | >>> cl.mapcar(cl.constantly(4), (1, 2, 3)) 95 | List(4, 4, 4) 96 | 97 | # cl4py even supports macros and special forms as a thin 98 | # wrapper around lisp.eval. 99 | >>> cl.loop('repeat', 5, 'collect', 42) 100 | List(42, 42, 42, 42, 42) 101 | 102 | >>> cl.progn(5, 6, 7, ('+', 4, 4)) 103 | 8 104 | 105 | When converting Common Lisp packages to Python modules, we run into the 106 | problem that not every Common Lisp symbol name is a valid Python 107 | identifier. As a remedy, so we attempt to substitute problematic 108 | characters and symbols with something that Python can digest. Here you can 109 | see this substitution rules in action: 110 | 111 | .. code:: python 112 | 113 | # hyphens are turned into underscores 114 | >>> cl.type_of("foo") 115 | List(Symbol("SIMPLE-ARRAY", "COMMON-LISP"), Symbol("CHARACTER", "COMMON-LISP"), List(3)) 116 | 117 | # The functions +, -, *, /, 1+, and 1- are renamed to add, sub, 118 | # mul, div, inc, and dec, respectively. 119 | >>> cl.add(2,3,4,5) 120 | 14 121 | 122 | # Within a string, occurrences of -, *, +, <=, <, =, /=, >=, gt, and ~, 123 | # are replaced by _, O, X, le, lt, sim, ne, ge, ge, gt, and tilde, respectively. 124 | >>> cl.stringgt('baz', 'bar') 125 | 2 126 | 127 | # Earmuffs are stripped 128 | >>> cl.print_base 129 | 10 130 | 131 | # Constants are capitalized 132 | >>> cl.MOST_POSITIVE_DOUBLE_FLOAT 133 | 1.7976931348623157e+308 134 | 135 | The cl4py module provides a Cons class that mimics cons cells in Lisp. 136 | 137 | .. code:: python 138 | 139 | >>> lisp.eval( ('CONS', 1, 2) ) 140 | Cons(1, 2) 141 | 142 | >>> lst = lisp.eval( ('CONS', 1, ('CONS', 2, () )) ) 143 | List(1, 2) 144 | >>> lst.car 145 | 1 146 | >>> lst.cdr 147 | List(2) # an abbreviation for Cons(2, ()) 148 | 149 | # cl4py Conses are iterable! 150 | >>> list(lst) 151 | [1, 2] 152 | >>> sum(lst) 153 | 3 154 | 155 | # cl4py also supports dotted and circular lists. 156 | >>> lisp.eval( ('CONS', 1, ('CONS', 2, 3 )) ) 157 | DottedList(1, 2, 3) 158 | 159 | >>> twos = cl.cons(2,2) 160 | >>> twos.cdr = twos 161 | >>> twos 162 | DottedList(2, ...) 163 | 164 | >>> cl.mapcar(lisp.function('+'), (1, 2, 3, 4), twos) 165 | List(3, 4, 5, 6) 166 | 167 | 168 | Frequently Asked Problems 169 | ------------------------- 170 | 171 | Why does my Lisp subprocess complain about ``Package QL does not exist``. 172 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 173 | 174 | By default, cl4py starts a Lisp subprocess with ``sbcl --script``. This 175 | means, that the Lisp process will ignore any user initialization files, 176 | including the Quicklisp setup. However, we provide an extra option for 177 | installing and loading Quicklisp automatically: ``quicklisp=True`` 178 | 179 | 180 | .. code:: python 181 | 182 | >>> lisp = cl4py.Lisp(quicklisp=True); 183 | >>> ql = lisp.find_package('QL') 184 | >>> ql.quickload('YOUR-SYSTEM') 185 | 186 | 187 | Related Projects 188 | ---------------- 189 | 190 | - `burgled-batteries `_ 191 | - A bridge between Python and Lisp. The goal is that Lisp programs can 192 | use Python libraries, which is in some sense the opposite of 193 | cl4py. Furthermore it relies on the less portable mechanism of FFI 194 | calls. 195 | - `CLAUDE `_ 196 | - An earlier attempt to access Lisp libraries from Python. The key 197 | difference is that cl4py does not run Lisp directly in the host 198 | process. This makes cl4py more portable, but complicates the exchange of 199 | data. 200 | - `cl-python `_ 201 | - A much heavier solution than cl4py --- let's simply implement Python 202 | in Lisp! An amazing project. However, cl-python cannot access foreign 203 | libraries, e.g., NumPy. And people are probably hesitant to migrate away 204 | from CPython. 205 | - `Hy `_ 206 | - Python, but with Lisp syntax. This project is certainly a great way to 207 | get started with Lisp. It allows you to study the advantages of Lisp's 208 | seemingly weird syntax, without leaving the comfortable Python 209 | ecosystem. Once you understand the advantages of Lisp, you will doubly 210 | appreciate cl4py for your projects. 211 | - `py4cl `_ 212 | - A library that allows Common Lisp code to access Python libraries. It 213 | is basically the inverse of cl4py. 214 | -------------------------------------------------------------------------------- /cl4py.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcoheisig/cl4py/53a690ebf297d1db5680189c3ebd58f06d859633/cl4py.png -------------------------------------------------------------------------------- /cl4py/__init__.py: -------------------------------------------------------------------------------- 1 | from .data import List, DottedList, Quote, Cons, Symbol, Keyword 2 | from .lisp import Lisp 3 | 4 | -------------------------------------------------------------------------------- /cl4py/circularity.py: -------------------------------------------------------------------------------- 1 | import io 2 | import numpy 3 | from .data import * 4 | 5 | 6 | class SharpsignSharpsign: 7 | def __init__(self, label): 8 | self.label = label 9 | 10 | def __repr__(self): 11 | return "#{}#".format(self.label) 12 | 13 | 14 | class SharpsignEquals: 15 | def __init__(self, label, obj): 16 | self.label = label 17 | self.obj = obj 18 | 19 | def __repr__(self): 20 | return "#{}={}".format(self.label, self.obj) 21 | 22 | 23 | def decircularize(obj, readtable): 24 | """Return a structure that is similar to OBJ, but where each circularity 25 | has been replaced by appropriate SharpsignEquals and SharpsignSharpsign 26 | instances. 27 | """ 28 | # Phase 1: Scan the data and number all circular objects. 29 | table = {} 30 | n = 1 31 | def scan(obj): 32 | nonlocal n 33 | atom = not (isinstance(obj, Cons) or 34 | isinstance(obj, list) or 35 | (isinstance(obj, tuple) and len(obj) > 0) or 36 | isinstance(obj, dict)) 37 | if atom: 38 | return 39 | key = id(obj) 40 | if key in table: 41 | if table[key] == 0: 42 | table[key] = n 43 | n += 1 44 | return 45 | else: 46 | table[key] = 0 47 | if isinstance(obj, Cons): 48 | scan(obj.car) 49 | scan(obj.cdr) 50 | elif isinstance(obj, list): 51 | for elt in obj: 52 | scan(elt) 53 | elif isinstance(obj, tuple): 54 | for elt in obj: 55 | scan(elt) 56 | elif isinstance(obj, numpy.ndarray): 57 | if obj.dtype.hasobject: 58 | for elt in numpy.nditer(obj): 59 | scan(elt) 60 | elif isinstance(obj, dict): 61 | for key, val in obj.items(): 62 | scan(key) 63 | scan(val) 64 | scan(obj) 65 | # Phase 2: Create a copy of data, where all references have been 66 | # replaced by SharpsignEquals or SharpsignSharpsign objects. 67 | def copy(obj): 68 | key = id(obj) 69 | # No need to copy atoms. 70 | if not key in table: 71 | return obj 72 | n = table[key] 73 | if n < 0: 74 | # We have a circular reference. We use the sign of the 75 | # object's number to distinguish the first visit from 76 | # consecutive ones. 77 | return SharpsignSharpsign(abs(n)) 78 | else: 79 | if n > 0: 80 | table[key] = -table[key] 81 | if isinstance(obj, Cons): 82 | result = Cons(copy(obj.car), copy(obj.cdr)) 83 | elif isinstance(obj, list): 84 | result = list(copy(elt) for elt in obj) 85 | elif isinstance(obj, numpy.ndarray): 86 | if obj.dtype.hasobject: 87 | result = np.vectorize(copy)(obj) 88 | else: 89 | result = obj 90 | elif isinstance(obj, tuple): 91 | # Convert strings to List data to make tuples a shorthand 92 | # notation for Lisp data. 93 | result = List(*(symbol_from_str(elt, readtable) 94 | if isinstance(elt, str) 95 | else copy(elt) 96 | for elt in obj)) 97 | elif isinstance(obj, dict): 98 | result = {} 99 | for key, val in obj.items(): 100 | result[copy(key)] = copy(val) 101 | if n > 0: 102 | return SharpsignEquals(n, result) 103 | else: 104 | return result 105 | return copy(obj) 106 | 107 | 108 | def symbol_from_str(string, readtable): 109 | stream = io.StringIO(string) 110 | token = readtable.read(stream) 111 | try: 112 | readtable.read(stream) 113 | raise RuntimeError('The string "' + string + '" contains more than one token.') 114 | except EOFError: 115 | pass 116 | return token 117 | -------------------------------------------------------------------------------- /cl4py/data.py: -------------------------------------------------------------------------------- 1 | '''Correspondence of Python types and Lisp types in cl4py: 2 | 3 | | Python | | Lisp | 4 | |--------------------+-----+--------------------------------------| 5 | | True | <-> | T | 6 | | () | <-> | NIL | 7 | | None | --> | NIL | 8 | | int | <-> | integer | 9 | | float | <-> | double-float | 10 | | float | <-- | single-float | 11 | | complex | <-> | (complex *) | 12 | | list | <-> | simple-vector | 13 | | tuple | --> | list | 14 | | dict | <-> | hash-table | 15 | | str | <-> | string | 16 | | cl4py.Cons | <-> | cons | 17 | | cl4py.Symbol | <-> | symbol | 18 | | cl4py.LispWrapper | <-> | #N? handle | 19 | | fractions.Fraction | <-> | ratio | 20 | | numpy.array | <-> | array | 21 | 22 | One peculiarity is that when a Python tuple is converted to lisp list, all 23 | strings occurring therein are interpreted as Lisp symbols. This way, 24 | Python tuples can be used as a somewhat elegant notation for S-expressions. 25 | 26 | ''' 27 | import reprlib 28 | 29 | class LispObject: 30 | pass 31 | 32 | 33 | class Stream(LispObject): 34 | def __init__(self, textstream, debug=False): 35 | self.stream = textstream 36 | self.old = None 37 | self.new = None 38 | self.debug = debug 39 | def read_char(self, eof_error=True): 40 | if self.new == None: 41 | c = self.stream.read(1) 42 | if eof_error and not c: raise EOFError() 43 | if self.debug: print(c,end='') 44 | else: 45 | c = self.new 46 | self.old, self.new = c, None 47 | return c 48 | def unread_char(self): 49 | if self.old: 50 | self.old, self.new = None, self.old 51 | else: 52 | raise RuntimeError('Duplicate unread_char.') 53 | 54 | 55 | python_name_translations = { 56 | '+' : 'add', 57 | '*' : 'mul', 58 | '-' : 'sub', 59 | '/' : 'div', 60 | '1+' : 'inc', 61 | '1-' : 'dec', 62 | } 63 | 64 | 65 | python_name_substitutions = { 66 | '-' : '_', 67 | '*' : 'O', 68 | '+' : 'X', 69 | '<=' : 'le', 70 | '<' : 'lt', 71 | '/=' : 'ne', 72 | '>=' : 'ge', 73 | '>' : 'gt', 74 | '=' : 'sim', 75 | '~' : 'tilde', 76 | } 77 | 78 | 79 | def python_name(name: str): 80 | # Use explicit translations for certain names. 81 | if name in python_name_translations: 82 | return python_name_translations[name] 83 | else: 84 | # Strip earmuffs. 85 | if ((len(name) > 2) and 86 | (name[0] in '*+') and 87 | (name[0] == name[-1]) and 88 | (name[0] != name[1])): # Don't strip earmuffs off *** or +++ 89 | name = name[1:-1] 90 | # Substitute problematic characters. 91 | for (old, new) in python_name_substitutions.items(): 92 | name = name.replace(old, new) 93 | return name.lower() 94 | 95 | 96 | class Symbol(LispObject): 97 | def __init__(self, name: str, package=None): 98 | self.name = name 99 | self.package = package 100 | 101 | def __repr__(self): 102 | if self.package: 103 | return 'Symbol("{}", "{}")'.format(self.name, self.package) 104 | else: 105 | return 'Symbol("{}")'.format(self.name) 106 | 107 | def __str__(self): 108 | return "{}:{}".format(self.package, self.name) 109 | 110 | def __hash__(self): 111 | return hash((self.name, self.package)) 112 | 113 | def __eq__(self, other): 114 | if isinstance(other, Symbol): 115 | return (self.name, self.package) == (other.name, other.package) 116 | else: 117 | return False 118 | 119 | @property 120 | def python_name(self): 121 | return python_name(self.name) 122 | 123 | 124 | class Keyword(Symbol): 125 | def __init__(self, name): 126 | super(Keyword, self).__init__(name, 'KEYWORD') 127 | 128 | def __repr__(self): 129 | return 'Keyword("{}")'.format(self.name) 130 | 131 | 132 | class Package(LispObject, type(reprlib)): 133 | def __getattribute__(self, name): 134 | attr = super().__getattribute__(name) 135 | if hasattr(attr, '__get__'): 136 | return attr.__get__(name) 137 | else: 138 | return attr 139 | 140 | def __setattr__(self, name, value): 141 | attr = super().__getattribute__(name) 142 | if hasattr(attr, '__set__'): 143 | return attr.__set__(name, value) 144 | else: 145 | raise AttributeError() 146 | 147 | 148 | class Cons (LispObject): 149 | def __init__(self, car, cdr): 150 | self.car = car 151 | self.cdr = cdr 152 | 153 | @reprlib.recursive_repr("...") 154 | def __repr__(self): 155 | datum = self 156 | car = datum.car 157 | cdr = datum.cdr 158 | rcar = repr(car) 159 | rcdr = repr(cdr) 160 | if null(cdr): 161 | return "List(" + rcar + ")" 162 | elif rcdr.startswith("DottedList("): 163 | return "DottedList(" + rcar + ", " + rcdr[11:] 164 | elif rcdr.startswith("List("): 165 | return "List(" + rcar + ", " + rcdr[5:] 166 | else: 167 | return "DottedList(" + rcar + ", " + rcdr + ")" 168 | 169 | def __iter__(self): 170 | return ListIterator(self) 171 | 172 | def __getitem__(self, index): 173 | cons = self 174 | for _ in range(index): 175 | if not cons.cdr: 176 | raise RuntimeError('{} is too short for index {}.'.format(self,index)) 177 | cons = cons.cdr 178 | return cons.car 179 | 180 | def __setitem__(self, index, value): 181 | cons = self 182 | for _ in range(index): 183 | if not cons.cdr: 184 | raise RuntimeError('{} is too short for index {}.'.format(self,index)) 185 | cons = cons.cdr 186 | cons.car = value 187 | 188 | @property 189 | def python_name(self): 190 | if self.car == Symbol('COMMON-LISP', 'SETF'): 191 | return 'set_' + python_name(self.cdr.car) 192 | else: 193 | raise RuntimeError('Not a function name: {}'.format(self)) 194 | 195 | def __eq__(self, other) -> bool: 196 | if isinstance(other, Cons): 197 | return self.car == other.car and self.cdr == other.cdr 198 | else: 199 | return False 200 | 201 | 202 | class ListIterator: 203 | def __init__(self, elt): 204 | self.elt = elt 205 | 206 | def __iter__(self): 207 | return self 208 | 209 | def __next__(self): 210 | if isinstance(self.elt, Cons): 211 | value = self.elt.car 212 | self.elt = self.elt.cdr 213 | return value 214 | else: 215 | raise StopIteration 216 | 217 | 218 | def List(*args): 219 | head = () 220 | for arg in args[::-1]: 221 | head = Cons(arg, head) 222 | return head 223 | 224 | 225 | def DottedList(*args): 226 | head = args[-1] if args else () 227 | for arg in args[-2::-1]: 228 | head = Cons(arg, head) 229 | return head 230 | 231 | 232 | def Quote(arg): 233 | return List(Symbol('QUOTE', 'CL'), arg) 234 | 235 | 236 | def car(arg): 237 | if isinstance(arg, Cons): 238 | return arg.car 239 | elif null(arg): 240 | return () 241 | else: 242 | raise RuntimeError('Cannot take the CAR of ' + str(arg) + '.') 243 | 244 | 245 | def cdr(arg): 246 | if isinstance(arg, Cons): 247 | return arg.cdr 248 | elif null(arg): 249 | return () 250 | else: 251 | raise RuntimeError('Cannot take the CDR of ' + str(arg) + '.') 252 | 253 | 254 | def null(arg): 255 | if arg == (): 256 | return True 257 | if (isinstance(arg,Symbol) 258 | and arg.name == "NIL" 259 | and (arg.package == "COMMON-LISP" or 260 | arg.package == "CL")): 261 | return True 262 | else: 263 | return False 264 | 265 | 266 | class LispWrapper (LispObject): 267 | def __init__(self, lisp, handle): 268 | self.lisp = lisp 269 | self.handle = handle 270 | 271 | def __del__(self): 272 | self.lisp.to_free.append(self.handle) 273 | 274 | def __call__(self, *args, **kwargs): 275 | restAndKeys = [ Quote(arg) for arg in args ] 276 | for key, value in kwargs.items(): 277 | restAndKeys.append(Keyword(key.upper())) 278 | restAndKeys.append(Quote(value)) 279 | return self.lisp.eval(List(Symbol('FUNCALL', 'CL'), Quote(self), *restAndKeys)) 280 | 281 | 282 | class LispMacro (LispObject): 283 | def __init__(self, lisp, symbol): 284 | self.lisp = lisp 285 | self.symbol = symbol 286 | 287 | def __call__(self, *args): 288 | return self.lisp.eval((self.symbol, *args)) 289 | 290 | 291 | class LispVariable (LispObject): 292 | def __init__(self, lisp, symbol): 293 | self.lisp = lisp 294 | self.symbol = symbol 295 | 296 | def __get__(self, obj, objtype=None): 297 | return self.lisp.eval(self.symbol) 298 | 299 | def __set__(self, obj, value): 300 | return self.lisp.eval((Symbol('SETQ', 'CL'), self.symbol, Quote(value))) 301 | -------------------------------------------------------------------------------- /cl4py/lisp.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import io 3 | import os.path 4 | from urllib import request 5 | import tempfile 6 | from pkg_resources import resource_filename 7 | from collections import deque 8 | from .data import LispWrapper, Cons, Symbol, Quote 9 | from .reader import Readtable 10 | from .writer import lispify 11 | 12 | _DEFAULT_COMMAND = ('sbcl', '--script') 13 | 14 | 15 | class Lisp: 16 | debug: bool 17 | _backtrace: bool 18 | 19 | def __init__(self, cmd=_DEFAULT_COMMAND, quicklisp=False, debug=False, 20 | backtrace=True): 21 | command = list(cmd) 22 | p = subprocess.Popen(command + [resource_filename(__name__, 'py.lisp')], 23 | stdin = subprocess.PIPE, 24 | stdout = subprocess.PIPE, 25 | stderr = subprocess.PIPE, 26 | shell = False) 27 | self.process = p 28 | self.stdin = io.TextIOWrapper(p.stdin, write_through=True, 29 | line_buffering=1, 30 | encoding='utf-8') 31 | self.stdout = io.TextIOWrapper(p.stdout, encoding='utf-8') 32 | # The name of the current package. 33 | self.package = "COMMON-LISP-USER" 34 | # Each Lisp process has its own readtable. 35 | self.readtable = Readtable(self) 36 | # The classes dict maps from symbols to python classes. 37 | self.classes = {} 38 | # Whenever the reader encounters a Lisp object whose class is not 39 | # yet known, it stores it in this {class_name : instances} dict. 40 | # This allows us to patch these instances later. 41 | self.unpatched_instances = {} 42 | # If debug is true, cl4py will print plenty of debug information. 43 | self.debug = debug 44 | # Pending objects to free 45 | self.to_free = deque() 46 | 47 | # Collect ASDF -- we'll need it for UIOP later 48 | self.function('CL:REQUIRE')(Symbol("ASDF", "KEYWORD")) 49 | 50 | # Finally, check whether the user wants quicklisp to be available. 51 | self.quicklisp = quicklisp 52 | if quicklisp: 53 | install_and_load_quicklisp(self) 54 | self._backtrace = backtrace 55 | self.eval( ('defparameter', 'cl4py::*backtrace*', backtrace) ) 56 | 57 | 58 | 59 | @property 60 | def backtrace(self) -> bool: 61 | return self._backtrace 62 | 63 | 64 | @backtrace.setter 65 | def backtrace(self, value: bool) -> bool: 66 | self.eval ( ('setf', 'cl4py::*backtrace*', Quote(value))) 67 | self._backtrace = value 68 | return self._backtrace 69 | 70 | 71 | def __del__(self): 72 | alive = self.process.poll() == None 73 | if alive: 74 | self.stdin.write('(cl4py:quit)\n') 75 | self.process.wait() 76 | 77 | 78 | def eval(self, expr): 79 | sexp = lispify(self, expr) 80 | if self.debug: print(sexp) # pylint: disable=multiple-statements 81 | to_free = [self.to_free.popleft() for _ in range(len(self.to_free))] 82 | if to_free: 83 | if self.debug: print('deleting handles', to_free) # pylint: disable=multiple-statements 84 | free_exp = ' '.join('#{}!'.format(handle) for handle in to_free) 85 | # On the Lisp side, #N! is read as a comment, so a PROGN is not needed here. 86 | sexp = free_exp + ' ' + sexp 87 | self.stdin.write(sexp + '\n') 88 | pkg = self.readtable.read(self.stdout) 89 | val = self.readtable.read(self.stdout) 90 | err = self.readtable.read(self.stdout) 91 | msg = self.readtable.read(self.stdout) 92 | # Update the current package. 93 | self.package = pkg 94 | # Write the Lisp output to the Python output. 95 | print(msg,end='') 96 | # If there is an error, raise it. 97 | if isinstance(err, Cons): 98 | condition = err.car 99 | msg = err.cdr.car if err.cdr else "" 100 | def init(self): 101 | RuntimeError.__init__(self, msg) 102 | raise type(str(condition), (RuntimeError,), 103 | {'__init__': init})() 104 | # Now, check whether there are any unpatched instances. If so, 105 | # figure out their class definitions and patch them accordingly. 106 | items = list(self.unpatched_instances.items()) 107 | self.unpatched_instances.clear() 108 | for (cls_name, instances) in items: 109 | cls = type(cls_name.python_name, (LispWrapper,), {}) 110 | self.classes[cls_name] = cls 111 | alist = self.function('cl4py:class-information')(cls_name) 112 | for cons in alist: 113 | add_member_function(cls, cons.car, cons.cdr) 114 | for instance in instances: 115 | instance.__class__ = cls 116 | # Finally, return the resulting values. 117 | if val == (): 118 | return None 119 | elif val.cdr == (): 120 | return val.car 121 | else: 122 | return tuple(val) 123 | 124 | 125 | def find_package(self, name): 126 | return self.function('CL:FIND-PACKAGE')(name) 127 | 128 | 129 | def function(self, name): 130 | return self.eval( ('CL:FUNCTION', name) ) 131 | 132 | 133 | def add_member_function(cls, name, gf): 134 | method_name = name.python_name 135 | setattr(cls, method_name, lambda self, *args: gf(self, *args)) 136 | 137 | 138 | def install_and_load_quicklisp(lisp): 139 | quicklisp_setup = os.path.expanduser('~/quicklisp/setup.lisp') 140 | if os.path.isfile(quicklisp_setup): 141 | lisp.function('cl:load')(quicklisp_setup) 142 | else: 143 | install_quicklisp(lisp) 144 | 145 | 146 | def install_quicklisp(lisp): 147 | url = 'https://beta.quicklisp.org/quicklisp.lisp' 148 | with tempfile.NamedTemporaryFile(prefix='quicklisp-', suffix='.lisp') as tmp: 149 | with request.urlopen(url) as u: 150 | tmp.write(u.read()) 151 | lisp.function('cl:load')(tmp.name) 152 | print('Installing Quicklisp...') 153 | lisp.eval( ('quicklisp-quickstart:install',) ) 154 | -------------------------------------------------------------------------------- /cl4py/py.lisp: -------------------------------------------------------------------------------- 1 | (defpackage #:cl4py 2 | (:use #:common-lisp) 3 | (:import-from 4 | #+abcl #:mop 5 | #+allegro #:mop 6 | #+clisp #:clos 7 | #+clozure #:ccl 8 | #+cmu #:clos-mop 9 | #+ecl #:clos 10 | #+clasp #:clos 11 | #+lispworks #:clos 12 | #+mcl #:ccl 13 | #+sbcl #:sb-mop 14 | #+scl #:clos 15 | #+mezzano #:mezzano.clos 16 | 17 | #:compute-class-precedence-list 18 | #:specializer-direct-methods 19 | #:method-specializers 20 | #:method-generic-function 21 | #:generic-function-name) 22 | (:export 23 | #:cl4py 24 | #:quit 25 | #:class-information 26 | #:dtype-from-type 27 | #:dtype-from-code 28 | #:dtype-endianness 29 | #:dtype-type 30 | #:dtype-code 31 | #:dtype-size)) 32 | 33 | (in-package #:cl4py) 34 | 35 | ;;; A boolean, indicating whether we should also send a detailed backtrace 36 | ;;; to Python whenever something goes wrong on the Lisp side. This feature 37 | ;;; depends on UIOP and is therefore disabled by default. 38 | (defvar *backtrace* nil) 39 | 40 | ;;; Welcome to the Lisp side of cl4py. Basically, this is just a REPL that 41 | ;;; reads expressions from the Python side and prints results back to 42 | ;;; Python. 43 | 44 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 45 | ;;; 46 | ;;; Object Handles 47 | ;;; 48 | ;;; One challenge is that not all objects in Lisp can be written 49 | ;;; readably. As a pragmatic workaround, these objects are replaced by 50 | ;;; handles, by means of the #n? and #n! reader macros. The Python side is 51 | ;;; responsible for declaring when a handle may be deleted. 52 | 53 | (defvar *handle-counter* 0) 54 | 55 | (defvar *foreign-objects* (make-hash-table :test #'eql)) 56 | 57 | (defun free-handle (handle) 58 | (remhash handle *foreign-objects*)) 59 | 60 | (defun handle-object (handle) 61 | (or (gethash handle *foreign-objects*) 62 | (error "Invalid Handle."))) 63 | 64 | (defun object-handle (object) 65 | (let ((handle (incf *handle-counter*))) 66 | (setf (gethash handle *foreign-objects*) object) 67 | handle)) 68 | 69 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 70 | ;;; 71 | ;;; Reader Macros 72 | 73 | ;;; The #n! reader macro declares that a handle can be freed. 74 | (defun sharpsign-exclamation-mark (s c n) 75 | (declare (ignore s c)) 76 | (free-handle n) 77 | (values)) 78 | 79 | ;;; The #n? reader macro retrieves the object corresponding to the supplied 80 | ;;; handle. 81 | (defun sharpsign-question-mark (s c n) 82 | (declare (ignore s c)) 83 | (handle-object n)) 84 | 85 | ;;; The #N reader macro is used to retrieve NumPy arrays. For performance 86 | ;;; reasons, those arrays are not communicated as text, but in a binary 87 | ;;; format via the file system. 88 | (defun sharpsign-n (s c n) 89 | (declare (ignore c n)) 90 | (let* ((file (read s)) 91 | (array (load-array file))) 92 | (delete-file file) 93 | array)) 94 | 95 | ;;; We introduce a curly bracket notation to send hash tables. 96 | (defun left-curly-bracket (stream char) 97 | (declare (ignore char)) 98 | (let ((items (read-delimited-list #\} stream t)) 99 | (table (make-hash-table :test #'equal))) 100 | (loop for (key value) on items by #'cddr do 101 | (setf (gethash key table) value)) 102 | table)) 103 | 104 | (define-condition unmatched-closing-curly-bracket 105 | (reader-error) 106 | () 107 | (:report 108 | (lambda (condition stream) 109 | (format stream "Unmatched closing curly bracket on ~S." 110 | (stream-error-stream condition))))) 111 | 112 | (defun right-curly-bracket (stream char) 113 | (declare (ignore char)) 114 | (error 'unmatched-closing-curly-bracket 115 | :stream stream)) 116 | 117 | (defvar *cl4py-readtable* 118 | (let ((r (copy-readtable))) 119 | (set-dispatch-macro-character #\# #\! 'sharpsign-exclamation-mark r) 120 | (set-dispatch-macro-character #\# #\? 'sharpsign-question-mark r) 121 | (set-dispatch-macro-character #\# #\N 'sharpsign-n r) 122 | (set-macro-character #\{ 'left-curly-bracket nil r) 123 | (set-macro-character #\} 'right-curly-bracket nil r) 124 | (values r))) 125 | 126 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 127 | ;;; 128 | ;;; Printing for Python 129 | ;;; 130 | ;;; Not all Lisp objects can be communicated to Python. Most notably, 131 | ;;; functions and other non-builtin objects. Instead, we walk all objects 132 | ;;; before sending them to Python and replace occurrences of non 133 | ;;; serializable objects with reference handles. 134 | ;;; 135 | ;;; The printed structure is scanned first, such that circular structure 136 | ;;; can be printed correctly using #N= and #N#. 137 | 138 | ;;; Each entry in the *pyprint-table* is either the value T, meaning the 139 | ;;; object has been visited once, or its ID (an integer), meaning the 140 | ;;; object has been scanned multiple times. When writing, a positive ID 141 | ;;; means that the object should be written as #N=OBJECT, while a negative 142 | ;;; ID means that the object should be written as #N#. 143 | 144 | (defvar *pyprint-table*) 145 | 146 | (defvar *pyprint-counter*) 147 | 148 | ;; We use this dummy package during printing, to have each symbol written 149 | ;; with its full package prefix. 150 | (defpackage #:cl4py-empty-package (:use)) 151 | 152 | (defgeneric pyprint-scan (object)) 153 | 154 | (defgeneric pyprint-write (object stream)) 155 | 156 | (defun pyprint (object &optional (stream *standard-output*)) 157 | (let ((*pyprint-table* (make-hash-table :test #'eql)) 158 | (*pyprint-counter* 0)) 159 | (pyprint-scan object) 160 | (with-standard-io-syntax 161 | (let ((*package* (find-package '#:cl4py-empty-package))) 162 | (pyprint-write object stream))) 163 | (terpri stream) 164 | object)) 165 | 166 | (defmethod pyprint-scan :around ((object t)) 167 | (unless (or (symbolp object) 168 | (numberp object) 169 | (characterp object)) 170 | (multiple-value-bind (value present-p) 171 | (gethash object *pyprint-table*) 172 | (cond ((not present-p) 173 | (setf (gethash object *pyprint-table*) t) 174 | (call-next-method)) 175 | ((eq value t) 176 | (setf (gethash object *pyprint-table*) 177 | (incf *pyprint-counter*)))) 178 | (values)))) 179 | 180 | (defmethod pyprint-scan ((object t)) 181 | (declare (ignore object))) 182 | 183 | (defmethod pyprint-scan ((cons cons)) 184 | (pyprint-scan (car cons)) 185 | (pyprint-scan (cdr cons))) 186 | 187 | (defmethod pyprint-scan ((sequence sequence)) 188 | (map nil #'pyprint-scan sequence)) 189 | 190 | (defmethod pyprint-scan ((array array)) 191 | (loop for index below (array-total-size array) do 192 | (pyprint-scan (row-major-aref array index)))) 193 | 194 | (defmethod pyprint-scan ((hash-table hash-table)) 195 | (when (eq (hash-table-test hash-table) 'equal) 196 | (maphash 197 | (lambda (key value) 198 | (pyprint-scan key) 199 | (pyprint-scan value)) 200 | hash-table))) 201 | 202 | (defconstant +syntax-tag+ 0) 203 | (defconstant +function-tag+ 1) 204 | (defconstant +constant-tag+ 2) 205 | (defconstant +variable-tag+ 3) 206 | 207 | (defun package-contents-alist (package) 208 | (let ((alist '())) 209 | (loop for symbol being each external-symbol of package do 210 | (when (fboundp symbol) 211 | (if (or (macro-function symbol) 212 | (special-operator-p symbol)) 213 | (push (list +syntax-tag+ symbol) 214 | alist) 215 | (push (list +function-tag+ symbol (symbol-function symbol)) 216 | alist))) 217 | (when (boundp symbol) 218 | (if (constantp symbol) 219 | (push (list +constant-tag+ symbol (symbol-value symbol)) 220 | alist) 221 | (push (list +variable-tag+ symbol) 222 | alist)))) 223 | alist)) 224 | 225 | (defmethod pyprint-scan ((package package)) 226 | (pyprint-scan (package-name package)) 227 | (mapc #'pyprint-scan (package-contents-alist package))) 228 | 229 | (defmethod pyprint-write :around ((object t) stream) 230 | (let ((id (gethash object *pyprint-table*))) 231 | (if (integerp id) 232 | (cond ((plusp id) 233 | (setf (gethash object *pyprint-table*) (- id)) 234 | (format stream "#~D=" id) 235 | (call-next-method)) 236 | ((minusp id) 237 | (format stream "#~D#" (- id)))) 238 | (call-next-method)))) 239 | 240 | (defmethod pyprint-write ((object t) stream) 241 | (format stream "#~D?~S" 242 | (object-handle object) 243 | (class-name (class-of object)))) 244 | 245 | (defmethod pyprint-write ((number number) stream) 246 | (write number :stream stream)) 247 | 248 | (defmethod pyprint-write ((symbol symbol) stream) 249 | (write symbol :stream stream)) 250 | 251 | (defmethod pyprint-write ((string string) stream) 252 | (write-char #\" stream) 253 | (loop for char across string do 254 | (when (member char '(#\" #\\)) 255 | (write-char #\\ stream)) 256 | (write-char char stream)) 257 | (write-char #\" stream)) 258 | 259 | (defmethod pyprint-write ((character character) stream) 260 | (write character :stream stream)) 261 | 262 | (defmethod pyprint-write ((pathname pathname) stream) 263 | (pyprint-write (namestring (truename pathname)) stream)) 264 | 265 | (defun specializer-direct-member-functions (specializer) 266 | (loop for method in (specializer-direct-methods specializer) 267 | for max below 100 268 | when (and (eq (first (method-specializers method)) specializer) 269 | (method-generic-function method)) 270 | collect (method-generic-function method))) 271 | 272 | (defun class-member-functions (class) 273 | (remove-duplicates 274 | (mapcan #'specializer-direct-member-functions 275 | (remove (find-class 't) (compute-class-precedence-list class))))) 276 | 277 | (defmethod pyprint-write ((package package) stream) 278 | (write-string "#M" stream) 279 | (pyprint-write 280 | (cons (package-name package) 281 | (package-contents-alist package)) 282 | stream)) 283 | 284 | (defmethod pyprint-write ((cons cons) stream) 285 | (write-string "(" stream) 286 | (loop for car = (car cons) 287 | for cdr = (cdr cons) do 288 | (pyprint-write car stream) 289 | (write-string " " stream) 290 | (cond ((null cdr) 291 | (loop-finish)) 292 | ((or (atom cdr) 293 | (integerp (gethash cdr *pyprint-table*))) 294 | (write-string " . " stream) 295 | (pyprint-write cdr stream) 296 | (loop-finish)) 297 | (t 298 | (setf cons cdr)))) 299 | (write-string ")" stream)) 300 | 301 | (defmethod pyprint-write ((vector vector) stream) 302 | (cond ((simple-vector-p vector) 303 | (write-string "#(" stream) 304 | (loop for elt across vector do 305 | (pyprint-write elt stream) 306 | (write-char #\space stream)) 307 | (write-string ")" stream)) 308 | (t 309 | (call-next-method)))) 310 | 311 | (defmethod pyprint-write ((hash-table hash-table) stream) 312 | (cond ((eql (hash-table-test hash-table) 'equal) 313 | (write-string "{" stream) 314 | (maphash 315 | (lambda (key value) 316 | (pyprint-write key stream) 317 | (write-char #\space stream) 318 | (pyprint-write value stream) 319 | (write-char #\space stream)) 320 | hash-table) 321 | (write-string "}" stream)) 322 | (t 323 | (call-next-method)))) 324 | 325 | (defun array-contents (array) 326 | (labels ((contents (dimensions index) 327 | (if (null dimensions) 328 | (row-major-aref array index) 329 | (let* ((dimension (car dimensions)) 330 | (dimensions (cdr dimensions)) 331 | (count (reduce #'* dimensions))) 332 | (loop for i below dimension 333 | collect (contents dimensions index) 334 | do (incf index count)))))) 335 | (contents (array-dimensions array) 0))) 336 | 337 | (defmethod pyprint-write ((array array) stream) 338 | (let ((dtype (ignore-errors (dtype-from-type (array-element-type array))))) 339 | (cond ((or (not dtype) 340 | (eq (dtype-type dtype) t)) 341 | ;; Case 1 - General Arrays. 342 | (write-char #\# stream) 343 | (write (array-rank array) :stream stream) 344 | (write-char #\A stream) 345 | (pyprint-write (array-contents array) stream)) 346 | (t 347 | (let ((path (format nil "/tmp/cl4py-array-~D.npy" (random most-positive-fixnum)))) 348 | (store-array array path) 349 | (write-char #\# stream) 350 | (write-char #\N stream) 351 | (pyprint-write path stream)))))) 352 | 353 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 354 | ;;; 355 | ;;; Fast Array Serialization 356 | ;;; 357 | ;;; The following code adds support for transmitting arrays using the Numpy 358 | ;;; file format. It contains a copy of the systems 'ieee-floats' and 359 | ;;; 'numpy-file-format'. Legalese: 360 | 361 | ;;; Copyright (c) 2006 Marijn Haverbeke 362 | ;;; 363 | ;;; This software is provided 'as-is', without any express or implied 364 | ;;; warranty. In no event will the authors be held liable for any 365 | ;;; damages arising from the use of this software. 366 | ;;; 367 | ;;; Permission is granted to anyone to use this software for any 368 | ;;; purpose, including commercial applications, and to alter it and 369 | ;;; redistribute it freely, subject to the following restrictions: 370 | ;;; 371 | ;;; 1. The origin of this software must not be misrepresented; you must 372 | ;;; not claim that you wrote the original software. If you use this 373 | ;;; software in a product, an acknowledgment in the product 374 | ;;; documentation would be appreciated but is not required. 375 | ;;; 376 | ;;; 2. Altered source versions must be plainly marked as such, and must 377 | ;;; not be misrepresented as being the original software. 378 | ;;; 379 | ;;; 3. This notice may not be removed or altered from any source 380 | ;;; distribution. 381 | 382 | (eval-when (:compile-toplevel :load-toplevel :execute) 383 | (defmacro make-float-converters (encoder-name 384 | decoder-name 385 | exponent-bits 386 | significand-bits 387 | support-nan-and-infinity-p) 388 | (let* ((total-bits (+ 1 exponent-bits significand-bits)) 389 | (exponent-offset (1- (expt 2 (1- exponent-bits)))) ; (A) 390 | (sign-part `(ldb (byte 1 ,(1- total-bits)) bits)) 391 | (exponent-part `(ldb (byte ,exponent-bits ,significand-bits) bits)) 392 | (significand-part `(ldb (byte ,significand-bits 0) bits)) 393 | (nan support-nan-and-infinity-p) 394 | (max-exponent (1- (expt 2 exponent-bits)))) ; (B) 395 | `(progn 396 | (defun ,encoder-name (float) 397 | ,@(unless nan `((declare (type float float)))) 398 | (multiple-value-bind (sign significand exponent) 399 | (cond ,@(when nan `(((eq float :not-a-number) 400 | (values 0 1 ,max-exponent)) 401 | ((eq float :positive-infinity) 402 | (values 0 0 ,max-exponent)) 403 | ((eq float :negative-infinity) 404 | (values 1 0 ,max-exponent)))) 405 | (t 406 | (multiple-value-bind (significand exponent sign) (decode-float float) 407 | (let ((exponent (if (= 0 significand) 408 | exponent 409 | (+ (1- exponent) ,exponent-offset))) 410 | (sign (if (= sign 1.0) 0 1))) 411 | (unless (< exponent ,(expt 2 exponent-bits)) 412 | (error "Floating point overflow when encoding ~A." float)) 413 | (if (<= exponent 0) ; (C) 414 | (values sign (ash (round (* ,(expt 2 significand-bits) significand)) exponent) 0) 415 | (values sign (round (* ,(expt 2 significand-bits) (1- (* significand 2)))) exponent)))))) 416 | (let ((bits 0)) 417 | (declare (type (unsigned-byte ,total-bits) bits)) 418 | (setf ,sign-part sign 419 | ,exponent-part exponent 420 | ,significand-part significand) 421 | bits))) 422 | 423 | (defun ,decoder-name (bits) 424 | (declare (type (unsigned-byte ,total-bits) bits)) 425 | (let* ((sign ,sign-part) 426 | (exponent ,exponent-part) 427 | (significand ,significand-part)) 428 | ,@(when nan `((when (= exponent ,max-exponent) 429 | (return-from ,decoder-name 430 | (cond ((not (zerop significand)) :not-a-number) 431 | ((zerop sign) :positive-infinity) 432 | (t :negative-infinity)))))) 433 | (if (zerop exponent) ; (D) 434 | (setf exponent 1) 435 | (setf (ldb (byte 1 ,significand-bits) significand) 1)) 436 | (let ((float-significand (float significand ,(if (> total-bits 32) 1.0d0 1.0f0)))) 437 | (scale-float (if (zerop sign) float-significand (- float-significand)) 438 | (- exponent ,(+ exponent-offset significand-bits)))))))))) ; (E) 439 | 440 | ;; And instances of the above for the common forms of floats. 441 | (declaim (inline encode-float32 decode-float32 encode-float64 decode-float64)) 442 | (make-float-converters encode-float32 decode-float32 8 23 nil) 443 | (make-float-converters encode-float64 decode-float64 11 52 nil) 444 | 445 | (defconstant +endianness+ 446 | #+(and sbcl little-endian) :little-endian 447 | #+(and sbcl big-endian) :big-endian 448 | #+(and ccl little-endian-target) :little-endian 449 | #+(and ccl big-endian-target) :big-endian 450 | #+(and clisp) (if sys::*big-endian* :big-endian :little-endian) 451 | ;; Otherwise, we make an educated guess. 452 | #+(not (or sbcl ccl clisp)) :little-endian) 453 | 454 | (defgeneric dtype-endianness (dtype)) 455 | 456 | (defgeneric dtype-type (dtype)) 457 | 458 | (defgeneric dtype-code (dtype)) 459 | 460 | (defgeneric dtype-size (dtype)) 461 | 462 | (defparameter *dtypes* '()) 463 | 464 | (defclass dtype () 465 | ((%type :initarg :type :reader dtype-type) 466 | (%code :initarg :code :reader dtype-code) 467 | (%size :initarg :size :reader dtype-size) 468 | (%endianness :initarg :endianness :reader dtype-endianness))) 469 | 470 | (defmethod print-object ((dtype dtype) stream) 471 | (print-unreadable-object (dtype stream :type t) 472 | (prin1 (dtype-code dtype) stream))) 473 | 474 | (defun dtype-from-code (code) 475 | (or (find code *dtypes* :key #'dtype-code :test #'string=) 476 | (error "Cannot find dtype for the code ~S." code))) 477 | 478 | (defun dtype-from-type (type) 479 | (or (find-if 480 | (lambda (dtype) 481 | (and (eq (dtype-endianness dtype) +endianness+) 482 | (subtypep type (dtype-type dtype)))) 483 | *dtypes*) 484 | (error "Cannot find dtype for type ~S." type))) 485 | 486 | (defun define-dtype (code type size &optional (endianness +endianness+)) 487 | (let ((dtype (make-instance 'dtype 488 | :code code 489 | :type type 490 | :size size 491 | :endianness endianness))) 492 | (pushnew dtype *dtypes* :key #'dtype-code :test #'string=) 493 | dtype)) 494 | 495 | (defun define-multibyte-dtype (code type size) 496 | (define-dtype (concatenate 'string "<" code) type size :little-endian) 497 | (define-dtype (concatenate 'string ">" code) type size :big-endian) 498 | (define-dtype code type size +endianness+) 499 | (define-dtype (concatenate 'string "|" code) type size) 500 | (define-dtype (concatenate 'string "=" code) type size +endianness+)) 501 | 502 | (define-dtype "O" 't 64) 503 | (define-dtype "?" 'bit 1) 504 | (define-dtype "b" '(unsigned-byte 8) 8) 505 | (define-multibyte-dtype "i1" '(signed-byte 8) 8) 506 | (define-multibyte-dtype "i2" '(signed-byte 16) 16) 507 | (define-multibyte-dtype "i4" '(signed-byte 32) 32) 508 | (define-multibyte-dtype "i8" '(signed-byte 64) 64) 509 | (define-multibyte-dtype "u1" '(unsigned-byte 8) 8) 510 | (define-multibyte-dtype "u2" '(unsigned-byte 16) 16) 511 | (define-multibyte-dtype "u4" '(unsigned-byte 32) 32) 512 | (define-multibyte-dtype "u8" '(unsigned-byte 64) 64) 513 | (define-multibyte-dtype "f4" 'single-float 32) 514 | (define-multibyte-dtype "f8" 'double-float 64) 515 | (define-multibyte-dtype "c8" '(complex single-float) 64) 516 | (define-multibyte-dtype "c16" '(complex double-float) 128) 517 | 518 | ;; Finally, let's sort *dtypes* such that type queries always find the most 519 | ;; specific entry first. 520 | (setf *dtypes* (stable-sort *dtypes* #'subtypep :key #'dtype-type)) 521 | 522 | (defun read-python-object (stream &optional (skip #\,) (stop nil)) 523 | (loop for c = (read-char stream) do 524 | (case c 525 | ((#\space #\tab) (values)) 526 | ((#\' #\") (return (read-python-string c stream))) 527 | (#\( (return (read-python-tuple stream))) 528 | (#\[ (return (read-python-list stream))) 529 | (#\{ (return (read-python-dict stream))) 530 | ((#\T #\F) 531 | (unread-char c stream) 532 | (return (read-python-boolean stream))) 533 | (otherwise 534 | (cond ((eql c skip) 535 | (return (read-python-object stream nil stop))) 536 | ((eql c stop) 537 | (return stop)) 538 | ((digit-char-p c) 539 | (unread-char c stream) 540 | (return (read-python-integer stream))) 541 | (t 542 | (error "Invalid character: ~S" c))))))) 543 | 544 | (defun read-python-string (delimiter stream) 545 | (coerce 546 | (loop for c = (read-char stream) 547 | while (char/= c delimiter) 548 | collect c) 549 | 'string)) 550 | 551 | (defun read-python-integer (stream) 552 | (let ((result 0)) 553 | (loop for c = (read-char stream) do 554 | (let ((weight (digit-char-p c))) 555 | (if (null weight) 556 | (progn 557 | (unread-char c stream) 558 | (loop-finish)) 559 | (setf result (+ (* result 10) weight))))) 560 | result)) 561 | 562 | (defun read-python-boolean (stream) 563 | (flet ((skip (string) 564 | (loop for c across string do 565 | (assert (char= (read-char stream) c))))) 566 | (ecase (read-char stream) 567 | (#\T (skip "rue") t) 568 | (#\F (skip "alse") nil)))) 569 | 570 | (defun read-python-tuple (stream) 571 | (loop for object = (read-python-object stream nil #\)) 572 | then (read-python-object stream #\, #\)) 573 | until (eql object #\)) 574 | collect object)) 575 | 576 | (defun read-python-list (stream) 577 | (coerce 578 | (loop for object = (read-python-object stream nil #\]) 579 | then (read-python-object stream #\, #\]) 580 | until (eql object #\]) 581 | collect object) 582 | 'vector)) 583 | 584 | (defun read-python-dict (stream) 585 | (let ((dict (make-hash-table :test #'equal))) 586 | (loop 587 | (let ((key (read-python-object stream #\, #\}))) 588 | (when (eql key #\}) 589 | (return dict)) 590 | (setf (gethash key dict) 591 | (read-python-object stream #\:)))))) 592 | 593 | (defun read-python-object-from-string (string) 594 | (with-input-from-string (stream string) 595 | (read-python-object stream))) 596 | 597 | (defun load-array-metadata (filename) 598 | (with-open-file (stream filename :direction :input :element-type '(unsigned-byte 8)) 599 | ;; The first 6 bytes are a magic string: exactly \x93NUMPY. 600 | (unless (and (eql (read-byte stream) #x93) 601 | (eql (read-byte stream) 78) ; N 602 | (eql (read-byte stream) 85) ; U 603 | (eql (read-byte stream) 77) ; M 604 | (eql (read-byte stream) 80) ; P 605 | (eql (read-byte stream) 89)) ; Y 606 | (error "Not a Numpy file.")) 607 | (let* (;; The next 1 byte is an unsigned byte: the major version number 608 | ;; of the file format, e.g. \x01. 609 | (major-version (read-byte stream)) 610 | ;; The next 1 byte is an unsigned byte: the minor version number 611 | ;; of the file format, e.g. \x00. 612 | (minor-version (read-byte stream)) 613 | (header-len 614 | (if (= major-version 1) 615 | ;; Version 1.0: The next 2 bytes form a little-endian 616 | ;; unsigned int: the length of the header data HEADER_LEN. 617 | (logior (ash (read-byte stream) 0) 618 | (ash (read-byte stream) 8)) 619 | ;; Version 2.0: The next 4 bytes form a little-endian 620 | ;; unsigned int: the length of the header data HEADER_LEN. 621 | (logior (ash (read-byte stream) 0) 622 | (ash (read-byte stream) 8) 623 | (ash (read-byte stream) 16) 624 | (ash (read-byte stream) 24))))) 625 | (declare (ignore minor-version)) 626 | ;; The next HEADER_LEN bytes form the header data describing the 627 | ;; array’s format. It is an ASCII string which contains a Python 628 | ;; literal expression of a dictionary. It is terminated by a newline 629 | ;; (\n) and padded with spaces (\x20) to make the total of len(magic 630 | ;; string) + 2 + len(length) + HEADER_LEN be evenly divisible by 64 631 | ;; for alignment purposes. 632 | (let ((dict (read-python-object-from-string 633 | (let ((buffer (make-string header-len :element-type 'base-char))) 634 | (loop for index from 0 below header-len do 635 | (setf (schar buffer index) (code-char (read-byte stream)))) 636 | buffer)))) 637 | (values 638 | (gethash "shape" dict) 639 | (dtype-from-code (gethash "descr" dict)) 640 | (gethash "fortran_order" dict) 641 | (* 8 (+ header-len (if (= 1 major-version) 10 12)))))))) 642 | 643 | (defun load-array (filename) 644 | ;; We actually open the file twice, once to read the metadata - one byte 645 | ;; at a time, and once to read the array contents with a suitable element 646 | ;; type (e.g. (unsigned-byte 32) for single precision floating-point 647 | ;; numbers). 648 | (multiple-value-bind (dimensions dtype fortran-order header-bits) 649 | (load-array-metadata filename) 650 | (let* ((element-type (dtype-type dtype)) 651 | (array (make-array dimensions :element-type element-type)) 652 | (total-size (array-total-size array)) 653 | (chunk-size (if (subtypep element-type 'complex) 654 | (/ (dtype-size dtype) 2) 655 | (dtype-size dtype))) 656 | (stream-element-type 657 | (if (typep array '(array (signed-byte *))) 658 | `(signed-byte ,chunk-size) 659 | `(unsigned-byte ,chunk-size)))) 660 | (unless (not fortran-order) 661 | (error "Reading arrays in Fortran order is not yet supported.")) 662 | (unless (eq (dtype-endianness dtype) +endianness+) 663 | (error "Endianness conversion is not yet supported.")) 664 | ;; TODO Respect fortran-order and endianness. 665 | (with-open-file (stream filename :element-type stream-element-type) 666 | ;; Skip the header. 667 | (loop repeat (/ header-bits chunk-size) do (read-byte stream)) 668 | (etypecase array 669 | ((simple-array single-float) 670 | (loop for index below total-size do 671 | (setf (row-major-aref array index) 672 | (decode-float32 (read-byte stream))))) 673 | ((simple-array double-float) 674 | (loop for index below total-size do 675 | (setf (row-major-aref array index) 676 | (decode-float64 (read-byte stream))))) 677 | ((simple-array (complex single-float)) 678 | (loop for index below total-size do 679 | (setf (row-major-aref array index) 680 | (complex 681 | (decode-float32 (read-byte stream)) 682 | (decode-float32 (read-byte stream)))))) 683 | ((simple-array (complex double-float)) 684 | (loop for index below total-size do 685 | (setf (row-major-aref array index) 686 | (complex 687 | (decode-float64 (read-byte stream)) 688 | (decode-float64 (read-byte stream)))))) 689 | ((simple-array *) 690 | (loop for index below total-size do 691 | (setf (row-major-aref array index) 692 | (read-byte stream)))))) 693 | array))) 694 | 695 | (defun array-metadata-string (array) 696 | (with-output-to-string (stream nil :element-type 'base-char) 697 | (format stream "{'descr': '~A', ~ 698 | 'fortran_order': ~:[False~;True~], ~ 699 | 'shape': (~{~D,~^ ~}), }" 700 | (dtype-code (dtype-from-type (array-element-type array))) 701 | nil 702 | (array-dimensions array)))) 703 | 704 | (defun store-array (array filename) 705 | ;; We open the file twice - once with a stream element type of 706 | ;; (unsigned-byte 8) to write the header, and once with a stream element 707 | ;; type suitable for writing the array content. 708 | (let* ((dtype (dtype-from-type (array-element-type array))) 709 | (metadata (array-metadata-string array)) 710 | (metadata-length (- (* 64 (ceiling (+ 10 (length metadata)) 64)) 10))) 711 | (with-open-file (stream filename :direction :output 712 | :element-type '(unsigned-byte 8) 713 | :if-exists :supersede) 714 | (write-sequence #(#x93 78 85 77 80 89) stream) ; The magic string. 715 | (write-byte 1 stream) ; Major version. 716 | (write-byte 0 stream) ; Minor version. 717 | ;; Write the length of the metadata string (2 bytes, little endian). 718 | (write-byte (ldb (byte 8 0) metadata-length) stream) 719 | (write-byte (ldb (byte 8 8) metadata-length) stream) 720 | ;; Write the metadata string. 721 | (loop for char across metadata do 722 | (write-byte (char-code char) stream)) 723 | ;; Pad the header with spaces for 64 byte alignment. 724 | (loop repeat (- metadata-length (length metadata) 1) do 725 | (write-byte (char-code #\space) stream)) 726 | (write-byte (char-code #\newline) stream)) ; Finish with a newline. 727 | ;; Now, open the file a second time to write the array contents. 728 | (let* ((chunk-size (if (subtypep (array-element-type array) 'complex) 729 | (/ (dtype-size dtype) 2) 730 | (dtype-size dtype))) 731 | (stream-element-type 732 | (if (typep array '(array (signed-byte *))) 733 | `(signed-byte ,chunk-size) 734 | `(unsigned-byte ,chunk-size))) 735 | (total-size (array-total-size array))) 736 | (with-open-file (stream filename :direction :output 737 | :element-type stream-element-type 738 | :if-exists :append) 739 | (etypecase array 740 | ((simple-array single-float) 741 | (loop for index below total-size do 742 | (write-byte (encode-float32 (row-major-aref array index)) stream))) 743 | ((simple-array double-float) 744 | (loop for index below total-size do 745 | (write-byte (encode-float64 (row-major-aref array index)) stream))) 746 | ((simple-array (complex single-float)) 747 | (loop for index below total-size do 748 | (let ((c (row-major-aref array index))) 749 | (write-byte (encode-float32 (realpart c)) stream) 750 | (write-byte (encode-float32 (imagpart c)) stream)))) 751 | ((simple-array (complex double-float)) 752 | (loop for index below total-size do 753 | (let ((c (row-major-aref array index))) 754 | (write-byte (encode-float64 (realpart c)) stream) 755 | (write-byte (encode-float64 (imagpart c)) stream)))) 756 | ((simple-array *) 757 | (loop for index below total-size do 758 | (write-byte (row-major-aref array index) stream)))))))) 759 | 760 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 761 | ;;; 762 | ;;; Miscellaneous 763 | 764 | (defgeneric condition-string (condition)) 765 | 766 | (defmethod condition-string ((condition condition)) 767 | (with-output-to-string (stream) 768 | (terpri stream) 769 | (describe condition stream))) 770 | 771 | (defmethod condition-string ((simple-condition simple-condition)) 772 | (apply #'format nil 773 | (simple-condition-format-control simple-condition) 774 | (simple-condition-format-arguments simple-condition))) 775 | 776 | (defun class-information (class-name) 777 | (let ((class (find-class class-name nil)) 778 | (alist '())) 779 | (loop for member-function in (class-member-functions class) do 780 | (let ((name (generic-function-name member-function))) 781 | (when (or (symbolp name) 782 | (and (consp name) 783 | (eq (car name) 'setf) 784 | (symbolp (cadr name)) 785 | (null (cddr name)))) 786 | (push (cons name member-function) alist)))) 787 | alist)) 788 | 789 | (defun maybe-funcall (package name &rest args) 790 | (let ((package (find-package package))) 791 | (when (packagep package) 792 | (let ((symbol (find-symbol name package))) 793 | (when (fboundp symbol) 794 | (apply symbol args)))))) 795 | 796 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 797 | ;;; 798 | ;;; The cl4py REPL 799 | 800 | (defun quit () 801 | (maybe-funcall "UIOP" "QUIT") 802 | (maybe-funcall "CL-USER" "QUIT")) 803 | 804 | (defun cl4py (&rest args) 805 | (declare (ignore args)) 806 | (let* ((python (make-two-way-stream *standard-input* *standard-output*)) 807 | (lisp-output (make-string-output-stream)) 808 | (*standard-output* lisp-output) 809 | (*trace-output* lisp-output)) 810 | (flet ((read-python () 811 | (let ((*readtable* *cl4py-readtable*)) 812 | (read python)))) 813 | (loop 814 | (multiple-value-bind (value condition) 815 | (handler-case (values (multiple-value-list (eval (read-python))) nil) 816 | (reader-error (c) 817 | (clear-input python) 818 | (values '() c)) 819 | (serious-condition (c) 820 | (values '() c))) 821 | (let ((*read-eval* nil) 822 | (*print-circle* t)) 823 | ;; First, write the name of the current package. 824 | (pyprint (package-name *package*) python) 825 | ;; Second, write the obtained value. 826 | (pyprint value python) 827 | ;; Third, write the obtained condition, or NIL. 828 | (if condition 829 | (pyprint 830 | (list (class-name (class-of condition)) 831 | (if *backtrace* 832 | (concatenate 833 | 'string 834 | (condition-string condition) 835 | (with-output-to-string (stream) 836 | (maybe-funcall 837 | "UIOP" "PRINT-CONDITION-BACKTRACE" 838 | condition :stream stream)))) 839 | (condition-string condition)) 840 | python) 841 | (pyprint nil python)) 842 | ;; Fourth, write the output that has been obtained so far. 843 | (finish-output python) 844 | (pyprint (get-output-stream-string lisp-output) python))))))) 845 | 846 | ;;; Finally, launch the REPL. 847 | (cl4py) 848 | -------------------------------------------------------------------------------- /cl4py/reader.py: -------------------------------------------------------------------------------- 1 | import re 2 | import os 3 | import numpy 4 | import importlib.machinery 5 | import importlib.util 6 | from fractions import Fraction 7 | from enum import Enum 8 | from .data import * 9 | 10 | # An implementation of the Common Lisp reader algorithm, with the following 11 | # simplifications and changes: 12 | # 13 | # 1. Whitespace is never preserved. 14 | # 2. READ always assumes EOF error to be true. 15 | # 3. READTABLE-CASE is always :UPCASE. 16 | # 4. *READ-EVAL* is always false. 17 | # 5. *READ-BASE* is always 10. 18 | # 6. *READ-DEFAULT-FORMAT* is always SINGLE-FLOAT. 19 | # 7. There are no invalid characters. 20 | # 8. The input is assumed to be well formed. 21 | 22 | integer_regex = re.compile(r"[+-]?[0-9]+\.?") 23 | ratio_regex = re.compile(r"([+-]?[0-9]+)/([0-9]+)") 24 | float_regex = re.compile(r"([+-]?[0-9]+(?:\.[0-9]+)?)(?:([eEsSfFdDlL])([+-]?[0-9]+))?") 25 | symbol_regex = re.compile(r"(?:([^:]*?)(::?))?([^:]+)") 26 | 27 | SyntaxType = Enum('SyntaxType', 28 | ['CONSTITUENT', 29 | 'TERMINATING_MACRO_CHARACTER', 30 | 'NON_TERMINATING_MACRO_CHARACTER', 31 | 'SINGLE_ESCAPE', 32 | 'INVALID', 33 | 'MULTIPLE_ESCAPE', 34 | 'WHITESPACE']) 35 | 36 | 37 | class Readtable: 38 | def __init__(self, lisp): 39 | self.lisp = lisp 40 | self.macro_characters = {} 41 | # The variable tables is a stack of dicts, where one dict is pushed 42 | # for each non-recursive call to read. These dicts are used to 43 | # resolve circular references. 44 | self.tables = [] 45 | self.set_macro_character('(', left_parenthesis) 46 | self.set_macro_character(')', right_parenthesis) 47 | self.set_macro_character('{', left_curly_bracket) 48 | self.set_macro_character('}', right_curly_bracket) 49 | self.set_macro_character("'", single_quote) 50 | self.set_macro_character('"', double_quote) 51 | self.set_macro_character('#', sharpsign) 52 | self.set_macro_character(';', semicolon) 53 | self.set_dispatch_macro_character('#', '\\', sharpsign_backslash) 54 | self.set_dispatch_macro_character('#', "'", sharpsign_single_quote) 55 | self.set_dispatch_macro_character('#', '(', sharpsign_left_parenthesis) 56 | self.set_dispatch_macro_character('#', '?', sharpsign_questionmark) 57 | self.set_dispatch_macro_character('#', 'A', sharpsign_a) 58 | self.set_dispatch_macro_character('#', 'C', sharpsign_c) 59 | self.set_dispatch_macro_character('#', 'M', sharpsign_m) 60 | self.set_dispatch_macro_character('#', 'N', sharpsign_n) 61 | self.set_dispatch_macro_character('#', '=', sharpsign_equal) 62 | self.set_dispatch_macro_character('#', '#', sharpsign_sharpsign) 63 | 64 | 65 | def get_macro_character(self, char): 66 | return self.macro_characters[char] 67 | 68 | 69 | def set_macro_character(self, char, fn): 70 | self.macro_characters[char] = fn 71 | 72 | 73 | def get_dispatch_macro_character(self, dchar, schar): 74 | return self.macro_characters[(dchar, schar)] 75 | 76 | 77 | def set_dispatch_macro_character(self, dchar, schar, f): 78 | self.macro_characters[(dchar, schar)] = f 79 | 80 | 81 | def syntax_type(self, c): 82 | if c.isspace(): 83 | return SyntaxType.WHITESPACE 84 | elif c == '\\': 85 | return SyntaxType.SINGLE_ESCAPE 86 | elif c == '#': 87 | return SyntaxType.NON_TERMINATING_MACRO_CHARACTER 88 | elif c == '|': 89 | return SyntaxType.MULTIPLE_ESCAPE 90 | elif c in '"\'(),;`{}[]': 91 | return SyntaxType.TERMINATING_MACRO_CHARACTER 92 | else: 93 | return SyntaxType.CONSTITUENT 94 | 95 | 96 | def read(self, stream, recursive=False): 97 | if not isinstance(stream, Stream): 98 | stream = Stream(stream, debug=self.lisp.debug) 99 | if recursive: 100 | return self.read_aux(stream) 101 | else: 102 | self.tables.append({}) 103 | try: 104 | return self.read_aux(stream) 105 | finally: 106 | self.tables.pop() 107 | 108 | 109 | def read_aux(self, stream): 110 | while True: 111 | # 1. read one character 112 | x = stream.read_char() 113 | syntax_type = self.syntax_type(x) 114 | # 3. whitespace 115 | if syntax_type == SyntaxType.WHITESPACE: 116 | continue 117 | # 4. macro characters 118 | elif (syntax_type == SyntaxType.TERMINATING_MACRO_CHARACTER or 119 | syntax_type == SyntaxType.NON_TERMINATING_MACRO_CHARACTER): 120 | value = self.get_macro_character(x)(self, stream, x) 121 | if value is None: 122 | continue 123 | else: 124 | return value 125 | # 5. single escape character 126 | elif syntax_type == SyntaxType.SINGLE_ESCAPE: 127 | token = [stream.read_char()] 128 | escape = False 129 | # 6. multiple escape character 130 | elif syntax_type == SyntaxType.MULTIPLE_ESCAPE: 131 | token = [] 132 | escape = True 133 | # 7. constituent character 134 | else: 135 | token = [x.upper()] 136 | escape = False 137 | 138 | while True: 139 | y = stream.read_char(False) 140 | if not y: break 141 | syntax_type = self.syntax_type(y) 142 | if not escape: 143 | # 8. even number of multiple escape characters 144 | if syntax_type == SyntaxType.SINGLE_ESCAPE: 145 | token.append(stream.read_char()) 146 | elif syntax_type == SyntaxType.MULTIPLE_ESCAPE: 147 | escape = True 148 | elif syntax_type == SyntaxType.TERMINATING_MACRO_CHARACTER: 149 | stream.unread_char() 150 | break 151 | elif syntax_type == SyntaxType.WHITESPACE: 152 | stream.unread_char() 153 | break 154 | else: 155 | token.append(y.upper()) 156 | else: 157 | # 9. odd number of multiple escape characters 158 | if syntax_type == SyntaxType.SINGLE_ESCAPE: 159 | token.append(stream.read_char()) 160 | elif syntax_type == SyntaxType.MULTIPLE_ESCAPE: 161 | escape = False 162 | else: 163 | token.append(y) 164 | # 10. 165 | return self.parse(''.join(token)) 166 | 167 | 168 | def parse(self, token): 169 | # integer 170 | m = re.fullmatch(integer_regex, token) 171 | if m: 172 | return int(m.group(0)) 173 | # ratio 174 | m = re.fullmatch(ratio_regex, token) 175 | if m: 176 | return Fraction(int(m.group(1)), int(m.group(2))) 177 | # float 178 | m = re.fullmatch(float_regex, token) 179 | if m: 180 | base = m.group(1) 181 | exponent_marker = m.group(2) 182 | exponent = m.group(3) or "0" 183 | if not exponent_marker: 184 | return numpy.float32(base + 'e' + exponent) 185 | elif exponent_marker in 'sS': 186 | return numpy.float16(base + 'e' + exponent) 187 | elif exponent_marker in 'eEfF': 188 | return numpy.float32(base + 'e' + exponent) 189 | elif exponent_marker in 'dD': 190 | return numpy.float64(base + 'e' + exponent) 191 | elif exponent_marker in 'lL': 192 | return numpy.longdouble(base + 'e' + exponent) 193 | # symbol 194 | m = re.fullmatch(symbol_regex, token) 195 | if m: 196 | package = m.group(1) 197 | delimiter = m.group(2) 198 | name = m.group(3) 199 | if not package: 200 | if delimiter: 201 | return Keyword(name) 202 | else: 203 | return Symbol(name, self.lisp.package) 204 | else: 205 | if package in ['CL', 'COMMON-LISP']: 206 | if name == 'T': return True 207 | if name == 'NIL': return () 208 | return Symbol(name, package) 209 | raise RuntimeError('Failed to parse token "' + token + '".') 210 | 211 | 212 | 213 | def read_delimited_list(self, delim, stream, recursive): 214 | def skip_whitespace(): 215 | while True: 216 | x = stream.read_char() 217 | if self.syntax_type(x) != SyntaxType.WHITESPACE: 218 | stream.unread_char() 219 | break 220 | 221 | head = Cons((), ()) 222 | tail = head 223 | while True: 224 | skip_whitespace() 225 | x = stream.read_char() 226 | if x == delim: 227 | return head.cdr 228 | elif x == '.': 229 | tail.cdr = self.read_aux(stream) 230 | else: 231 | stream.unread_char() 232 | cons = Cons(self.read_aux(stream), ()) 233 | tail.cdr = cons 234 | tail = cons 235 | 236 | 237 | def left_parenthesis(r, s, c): 238 | return r.read_delimited_list(')', s, True) 239 | 240 | 241 | def right_parenthesis(r, s, c): 242 | raise RuntimeError('Unmatched closing parenthesis.') 243 | 244 | 245 | def left_curly_bracket(r, s, c): 246 | table = {} 247 | data = r.read_delimited_list('}', s, True) 248 | while data: 249 | key = car(data) 250 | rest = cdr(data) 251 | if null(rest): 252 | raise RuntimeError('Odd number of hash table data.') 253 | value = car(rest) 254 | table[key] = value 255 | data = cdr(rest) 256 | return table 257 | 258 | 259 | def right_curly_bracket(r, s, c): 260 | raise RuntimeError('Unmatched closing curly bracket.') 261 | 262 | 263 | def single_quote(r, s, c): 264 | return Cons("COMMON-LISP:QUOTE", Cons(r.read_aux(s), None)) 265 | 266 | 267 | def double_quote(r, s, c): 268 | result = '' 269 | while True: 270 | c = s.read_char() 271 | if c == '"': 272 | return result 273 | elif c == '\\': 274 | result += s.read_char() 275 | else: 276 | result += c 277 | 278 | 279 | def semicolon(r, s, c): 280 | while s.read_char() != '\n': 281 | pass 282 | 283 | 284 | def sharpsign(r, s, c): 285 | digits = '' 286 | while True: 287 | c = s.read_char() 288 | if c.isdigit(): 289 | digits += c 290 | else: 291 | c = c.upper() 292 | break 293 | n = int(digits) if digits else 0 294 | return r.get_dispatch_macro_character('#', c)(r, s, c, n) 295 | 296 | 297 | character_names = { 298 | 'NEWLINE' : '\x0A', 299 | 'SPACE' : '\x20', 300 | 'RUBOUT' : '\x7F', 301 | 'PAGE' : '\x0C', 302 | 'TAB' : '\x09', 303 | 'BACKSPACE' : '\x08', 304 | 'RETURN' : '\x0D', 305 | 'LINEFEED' : '\x0A', 306 | } 307 | 308 | 309 | def sharpsign_backslash(r, s, c, n): 310 | token = [s.read_char()] 311 | while True: 312 | c = s.read_char() 313 | if c.isalpha(): 314 | token.append(c) 315 | else: 316 | s.unread_char() 317 | break 318 | if len(token) == 1: 319 | return token[0] 320 | else: 321 | key = ''.join(token).upper() 322 | if key in character_names: 323 | return character_names[key] 324 | else: 325 | raise RuntimeError('Not a valid character name: {}'.format(key)) 326 | 327 | 328 | def sharpsign_single_quote(r, s, c, n): 329 | return List('CL:FUNCTION', r.read_aux(s)) 330 | 331 | 332 | def sharpsign_left_parenthesis(r, s, c, n): 333 | l = r.read_delimited_list(")", s, True) 334 | if not l: 335 | return [] 336 | else: 337 | return list(l) 338 | 339 | 340 | def sharpsign_questionmark(r, s, c, n): 341 | cls_name = r.read_aux(s) 342 | lisp = r.lisp 343 | cls = lisp.classes.get(cls_name) 344 | if cls: 345 | return cls(lisp, n) 346 | else: 347 | obj = LispWrapper(lisp, n) 348 | lst = lisp.unpatched_instances.setdefault(cls_name, []) 349 | lst.append(obj) 350 | return obj 351 | 352 | 353 | def sharpsign_a(r, s, c, n): 354 | L = r.read_aux(s) 355 | def listify(L, n): 356 | if n == 0: 357 | return L 358 | elif n == 1: 359 | return list(L) 360 | else: 361 | return [listify(l,n-1) for l in L] 362 | return numpy.array(listify(L, n)) 363 | 364 | 365 | def sharpsign_c(r, s, c, n): 366 | (real, imag) = list(r.read_aux(s)) 367 | return complex(real, imag) 368 | 369 | 370 | SYNTAX_TAG = 0 371 | FUNCTION_TAG = 1 372 | CONSTANT_TAG = 2 373 | VARIABLE_TAG = 3 374 | 375 | 376 | def sharpsign_m(r, s, c, n): 377 | data = r.read_aux(s) 378 | pkgname, alist = data.car, data.cdr 379 | spec = importlib.machinery.ModuleSpec(pkgname, None) 380 | module = importlib.util.module_from_spec(spec) 381 | module.__class__ = Package 382 | # Now register the package attributes. 383 | for cons in alist: 384 | tag = cons.car 385 | symbol = cons.cdr.car 386 | def register(key, value): 387 | module.__dict__[key] = value 388 | if symbol == True: 389 | register("T", True) 390 | elif symbol == (): 391 | register("NIL", ()) 392 | elif tag == SYNTAX_TAG: 393 | register(symbol.python_name, LispMacro(r.lisp, symbol)) 394 | elif tag == FUNCTION_TAG: 395 | register(symbol.python_name, cons.cdr.cdr.car) 396 | elif tag == CONSTANT_TAG: 397 | register(symbol.python_name.upper(), cons.cdr.cdr.car) 398 | elif tag == VARIABLE_TAG: 399 | register(symbol.python_name, LispVariable(r.lisp, symbol)) 400 | else: 401 | raise RuntimeError('Not a valid tag: {}'.format(tag)) 402 | return module 403 | 404 | 405 | def sharpsign_equal(r, s, c, n): 406 | value = r.read_aux(s) 407 | r.tables[-1][n] = value 408 | return value 409 | 410 | 411 | def sharpsign_sharpsign(r, s, c, n): 412 | return r.tables[-1][n] 413 | 414 | 415 | def sharpsign_n(r, s, c, n): 416 | f = r.read_aux(s) 417 | A = numpy.load(f) 418 | os.remove(f) 419 | return A 420 | 421 | -------------------------------------------------------------------------------- /cl4py/writer.py: -------------------------------------------------------------------------------- 1 | import re 2 | import numpy 3 | import tempfile 4 | import random 5 | from fractions import Fraction 6 | from .data import * 7 | from .circularity import * 8 | 9 | def lispify(lisp, obj): 10 | return lispify_datum(decircularize(obj, lisp.readtable)) 11 | 12 | 13 | def lispify_datum(obj): 14 | lispifier = lispifiers.get(type(obj)) 15 | if lispifier: 16 | return lispifier(obj) 17 | elif isinstance(obj, LispWrapper): 18 | return "#{}?".format(obj.handle) 19 | else: 20 | raise RuntimeError("Cannot lispify {}.".format(obj)) 21 | 22 | 23 | def lispify_ndarray(A): 24 | if not A.dtype.hasobject: 25 | return lispify_specialized_ndarray(A) 26 | 27 | def rec(A): 28 | if not getattr(A, 'ndim'): 29 | return lispify_datum(A) 30 | if A.ndim == 0: 31 | return " " + lispify_datum(A.item()) 32 | else: 33 | return "(" + " ".join(rec(a) for a in A) + ")" 34 | return "#{}A".format(A.ndim) + rec(A) 35 | 36 | 37 | def lispify_specialized_ndarray(A): 38 | r = random.randrange(2**63-1) 39 | tmp = tempfile.gettempdir() + '/cl4py-array-{}.npy'.format(r) 40 | numpy.save(tmp, A) 41 | return '#N"{}"'.format(tmp) 42 | 43 | 44 | def lispify_dict(d): 45 | s = "{" 46 | for key, value in d.items(): 47 | s += lispify_datum(key) + " " + lispify_datum(value) + " " 48 | return s + "}" 49 | 50 | 51 | def lispify_str(s): 52 | def escape(s): 53 | return s.translate(str.maketrans({'"':'\\"', '\\':'\\\\'})) 54 | return '"' + escape(s) + '"' 55 | 56 | 57 | def lispify_tuple(x): 58 | if len(x) == 0: 59 | return "NIL" 60 | else: 61 | # This should never happen, because decircularize implicitly 62 | # converts tuples to cl4py Lists. 63 | raise RuntimeError('Cannot lispify non-empty tuple.') 64 | 65 | 66 | def lispify_Cons(x): 67 | datum = x 68 | content = "" 69 | while isinstance(datum, Cons): 70 | content += lispify_datum(datum.car) + " " 71 | datum = datum.cdr 72 | if not null(datum): 73 | content += " . " + lispify_datum(datum) 74 | return "(" + content + ")" 75 | 76 | 77 | def lispify_Symbol(x): 78 | if not x.package: 79 | return "|" + x.name + "|" 80 | else: 81 | return "|" + x.package + "|::|" + x.name + "|" 82 | 83 | 84 | def lispify_Complex(x): 85 | return "#C(" + lispify_datum(x.real) + " " + lispify_datum(x.imag) + ")" 86 | 87 | 88 | def lispify_float16(x): 89 | return '{:E}'.format(x).replace('E', 'S') 90 | 91 | 92 | def lispify_float32(x): 93 | return '{:E}'.format(x) 94 | 95 | 96 | def lispify_float64(x): 97 | return '{:E}'.format(x).replace('E', 'D') 98 | 99 | 100 | def lispify_longdouble(x): 101 | return '{:E}'.format(x).replace('E', 'L') 102 | 103 | 104 | lispifiers = { 105 | # Built-in objects. 106 | bool : lambda x: "T" if x else "NIL", 107 | type(None) : lambda x: "NIL", 108 | int : str, 109 | float : lispify_float64, 110 | complex : lispify_Complex, 111 | list : lambda x: "#(" + " ".join(lispify_datum(elt) for elt in x) + ")", 112 | Fraction : str, 113 | tuple : lispify_tuple, 114 | str : lispify_str, 115 | dict : lispify_dict, 116 | # cl4py objects. 117 | Cons : lispify_Cons, 118 | Symbol : lispify_Symbol, 119 | Keyword : lispify_Symbol, 120 | SharpsignEquals : lambda x: "#" + str(x.label) + "=" + lispify_datum(x.obj), 121 | SharpsignSharpsign : lambda x: "#" + str(x.label) + "#", 122 | # Numpy objects. 123 | numpy.ndarray : lispify_ndarray, 124 | numpy.str_ : lispify_str, 125 | numpy.int8 : str, 126 | numpy.int16 : str, 127 | numpy.int32 : str, 128 | numpy.int64 : str, 129 | numpy.uint8 : str, 130 | numpy.uint16 : str, 131 | numpy.uint32 : str, 132 | numpy.uint64 : str, 133 | numpy.float16 : lispify_float16, 134 | numpy.float32 : lispify_float32, 135 | numpy.float64 : lispify_float64, 136 | numpy.longdouble : lispify_longdouble, 137 | numpy.complex64 : lispify_Complex, 138 | numpy.complex128 : lispify_Complex, 139 | } 140 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """The cl4py setup. 2 | 3 | See: 4 | https://github.com/marcoheisig/cl4py 5 | """ 6 | 7 | from setuptools import setup, find_packages 8 | from codecs import open 9 | from os import path 10 | 11 | def readme(): 12 | with open('README.rst') as f: 13 | return f.read() 14 | 15 | setup( 16 | name='cl4py', 17 | version='1.8.1', 18 | description='Common Lisp for Python', 19 | long_description=readme(), 20 | long_description_content_type='text/x-rst', 21 | license='MIT', 22 | url='https://github.com/marcoheisig/cl4py', 23 | author='Marco Heisig', 24 | author_email='marco.heisig@fau.de', 25 | install_requires=['numpy'], 26 | extras_require={}, 27 | keywords='foreign functions FFI', 28 | classifiers=[ 29 | 'Development Status :: 5 - Production/Stable', 30 | 'Intended Audience :: Developers', 31 | 'License :: OSI Approved :: MIT License', 32 | 'Programming Language :: Python :: 3', 33 | 'Programming Language :: Python :: 3.4', 34 | 'Programming Language :: Python :: 3.5', 35 | 'Programming Language :: Python :: 3.6', 36 | 'Programming Language :: Python :: 3.7', 37 | 'Programming Language :: Lisp' , 38 | ], 39 | packages=find_packages(exclude=['contrib', 'docs', 'test']), 40 | include_package_data = True, 41 | package_data={'': ['*.lisp']}, 42 | ) 43 | -------------------------------------------------------------------------------- /test/sample-program.lisp: -------------------------------------------------------------------------------- 1 | (in-package :common-lisp-user) 2 | 3 | (defun foo-{a7lkj9lakj} () 4 | nil) 5 | 6 | (defun make-error () 7 | (error "Artificial error")) 8 | 9 | (defun make-type-error () 10 | (error 'type-error :datum 'fake-variable :expected-type t) 11 | ) 12 | 13 | (export '(make-error make-type-error)) 14 | -------------------------------------------------------------------------------- /test/test_backtrace.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import pytest 4 | import cl4py 5 | from cl4py import List, Symbol 6 | 7 | # FIXME: should have test for quicklisp, but that would require a test 8 | # install of QL... 9 | 10 | @pytest.fixture() 11 | def stock_lisp(): 12 | return cl4py.Lisp() 13 | 14 | @pytest.fixture() 15 | def backtrace_lisp(): 16 | return cl4py.Lisp(backtrace=True) 17 | 18 | def load_sample_program(lisp_obj: cl4py.Lisp) -> None: 19 | cl = lisp_obj.find_package("COMMON-LISP") 20 | retval = cl.compile_file( 21 | os.path.join(os.path.dirname(__file__), "sample-program.lisp") 22 | ) 23 | cl.load(retval[0]) 24 | 25 | def test_backtrace_param(): 26 | lisp = cl4py.Lisp(backtrace=True) 27 | assert lisp.eval( Symbol("*BACKTRACE*", "CL4PY") ) 28 | assert lisp.backtrace 29 | lisp = cl4py.Lisp(backtrace=False) 30 | assert not lisp.eval( Symbol("*BACKTRACE*", "CL4PY")) 31 | assert not lisp.backtrace 32 | 33 | def test_backtrace_setting(): 34 | lisp = cl4py.Lisp(backtrace=True) 35 | assert lisp.eval( Symbol("*BACKTRACE*", "CL4PY") ) 36 | assert lisp.backtrace 37 | lisp.backtrace = False 38 | assert not lisp.eval( Symbol("*BACKTRACE*", "CL4PY")) 39 | assert not lisp.backtrace 40 | 41 | lisp = cl4py.Lisp(backtrace=False) 42 | assert not lisp.eval( Symbol("*BACKTRACE*", "CL4PY") ) 43 | assert not lisp.backtrace 44 | lisp.backtrace = True 45 | assert lisp.eval( Symbol("*BACKTRACE*", "CL4PY")) 46 | assert lisp.backtrace 47 | 48 | def test_produce_backtrace_type_error(stock_lisp, backtrace_lisp): 49 | with pytest.raises(RuntimeError): 50 | load_sample_program(stock_lisp) 51 | stock_lisp.find_package("COMMON-LISP-USER").make_type_error() 52 | 53 | lisp = cl4py.Lisp(backtrace=True, quicklisp=True) 54 | load_sample_program(lisp) 55 | try: 56 | lisp.find_package("COMMON-LISP-USER").make_type_error() 57 | except RuntimeError as e: 58 | msg = e.args[0] 59 | backtrace_re = re.compile('Backtrace', re.MULTILINE) 60 | assert re.search(backtrace_re, msg) 61 | else: 62 | pytest.fail("Should have seen a RuntimeError") 63 | 64 | def test_produce_backtrace_simple_error(stock_lisp, backtrace_lisp): 65 | with pytest.raises(RuntimeError): 66 | load_sample_program(stock_lisp) 67 | stock_lisp.find_package("COMMON-LISP-USER").make_error() 68 | 69 | 70 | load_sample_program(backtrace_lisp) 71 | try: 72 | backtrace_lisp.find_package("COMMON-LISP-USER").make_error() 73 | except RuntimeError as e: 74 | msg = e.args[0] 75 | backtrace_re = re.compile('Backtrace', re.MULTILINE) 76 | assert re.search(backtrace_re, msg) 77 | else: 78 | pytest.fail("Should have seen a RuntimeError") 79 | -------------------------------------------------------------------------------- /test/test_for_readtable.py: -------------------------------------------------------------------------------- 1 | from pytest import fixture 2 | import cl4py 3 | import os 4 | 5 | # pytest forces violation of this pylint rule 6 | # pylint: disable=redefined-outer-name 7 | 8 | 9 | @fixture(scope="module") 10 | def lisp(): 11 | return cl4py.Lisp() 12 | 13 | 14 | @fixture(scope="module") 15 | def cl(lisp): 16 | return lisp.function("find-package")("CL") 17 | 18 | 19 | # This test verifies issue underlying MR #9 20 | def test_readtable_problem(cl): 21 | retval = cl.compile_file( 22 | os.path.join(os.path.dirname(__file__), "sample-program.lisp") 23 | ) 24 | outfile = os.path.join(os.path.dirname(__file__), "sample-program.fasl") 25 | try: 26 | assert retval[0] == outfile 27 | assert os.path.exists(retval[0]) 28 | assert retval[1] == () 29 | assert retval[2] == () 30 | finally: 31 | cleanup(outfile) 32 | cleanup(outfile) 33 | 34 | def cleanup(outfile): 35 | if os.path.exists(outfile): 36 | try: 37 | os.remove(outfile) 38 | except: # pylint: disable=bare-except 39 | pass 40 | -------------------------------------------------------------------------------- /test/test_from_readme.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | import fractions 4 | 5 | from pytest import fixture 6 | import cl4py 7 | from cl4py import List, Symbol 8 | 9 | # pytest forces violation of this pylint rule 10 | # pylint: disable=redefined-outer-name 11 | 12 | @fixture(scope="module") 13 | def lisp(): 14 | return cl4py.Lisp() 15 | 16 | 17 | @fixture(scope="module") 18 | def cl(lisp): 19 | return lisp.find_package("CL") 20 | 21 | 22 | def test_startup(lisp): 23 | assert isinstance(lisp, cl4py.Lisp) 24 | 25 | 26 | def test_examples(lisp): 27 | assert lisp.eval( 42 ) == 42 28 | assert lisp.eval(("+", 2, 3)) == 5 29 | assert lisp.eval( ('/', ('*', 3, 5), 2) ) == fractions.Fraction(15, 2) 30 | assert lisp.eval( cl4py.List(cl4py.Symbol('STRING='), 'foo', 'bar') ) == () 31 | assert lisp.eval( cl4py.List(cl4py.Symbol('STRING='), 'foo', 'foo') ) == True 32 | assert lisp.eval(cl4py.Symbol('*PRINT-BASE*', 'COMMON-LISP')) == 10 33 | assert lisp.eval( ('loop', 'for', 'i', 'below', 5, 'collect', 'i') ) == cl4py.List(0, 1, 2, 3, 4) 34 | assert lisp.eval( ('with-output-to-string', ('stream',), 35 | ('princ', 12, 'stream'), 36 | ('princ', 34, 'stream')) ) == '1234' 37 | 38 | 39 | def test_finding_functions(lisp, cl): 40 | add = lisp.function("+") 41 | assert add(1, 2, 3, 4) == 10 42 | div = lisp.function("/") 43 | assert div(2, 4) == fractions.Fraction(1, 2) 44 | assert cl.oddp(5) 45 | assert cl.cons(5, None) == cl4py.List(5) 46 | assert cl.remove(5, [1, -5, 2, 7, 5, 9], key=cl.abs) == [1, 2, 7, 9] 47 | assert cl.mapcar(cl.constantly(4), (1, 2, 3)) == cl4py.List(4, 4, 4) 48 | assert cl.loop('repeat', 5, 'collect', 42) == List(42, 42, 42, 42, 42) 49 | assert cl.progn(5, 6, 7, ('+', 4, 4)) == 8 50 | 51 | 52 | def test_pythony_names(cl): 53 | assert cl.type_of("foo") == List( 54 | Symbol("SIMPLE-ARRAY", "COMMON-LISP"), 55 | Symbol("CHARACTER", "COMMON-LISP"), 56 | List(3), 57 | ) 58 | assert cl.add(2,3,4,5) == 14 59 | assert cl.stringgt('baz', 'bar') == 2 60 | assert cl.print_base == 10 61 | assert cl.MOST_POSITIVE_DOUBLE_FLOAT == 1.7976931348623157e+308 62 | 63 | 64 | def test_conses(cl, lisp): 65 | assert lisp.eval(("CONS", 1, 2)) == cl4py.Cons(1, 2) 66 | lst = lisp.eval(("CONS", 1, ("CONS", 2, ()))) 67 | assert lst == cl4py.List(1, 2) 68 | assert lst.car == 1 69 | assert lst.cdr == cl4py.List(2) 70 | assert list(lst) == [1, 2] 71 | assert sum(lst) == 3 72 | assert lisp.eval( ('CONS', 1, ('CONS', 2, 3 )) ) == cl4py.DottedList(1, 2, 3) 73 | twos = cl.cons(2, 2) 74 | twos.cdr = twos 75 | assert cl.mapcar(lisp.function("+"), (1, 2, 3, 4), twos) == List(3, 4, 5, 6) 76 | 77 | 78 | def test_error(cl, lisp): 79 | retval = cl.compile_file( 80 | os.path.join(os.path.dirname(__file__), "sample-program.lisp") 81 | ) 82 | cl.load(retval[0]) 83 | with pytest.raises(RuntimeError): 84 | lisp.eval( ("CL-USER::MAKE-ERROR", )) 85 | --------------------------------------------------------------------------------