├── .gitignore ├── .gitmodules ├── README.md ├── compile_to_py.py ├── cpython.py └── cpython_static.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | /.idea 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "cparser"] 2 | path = cparser 3 | url = https://github.com/albertz/PyCParser.git 4 | branch = master 5 | [submodule "CPython"] 6 | path = CPython 7 | url = https://github.com/albertz/CPython.git 8 | branch = master 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | PyCPython 2 | ========= 3 | 4 | Idea: Use [PyCParser](https://github.com/albertz/PyCParser) to parse and interpret CPython. :) 5 | 6 | Status so far: 7 | 8 | $ ./cpython.py -V 9 | ... 10 | Python 2.7.1 11 | 12 | Yea! 13 | 14 | # Compatibility 15 | 16 | PyPy, CPython 2.7 (so it can sort of host itself). 17 | 18 | The C data structures itself are compatible with CPython, 19 | so in theory, you can even load C extensions and it should work. 20 | 21 | # Why 22 | 23 | Just for fun. 24 | 25 | (Maybe, to make it in any way serious: [here](https://mail.python.org/pipermail/pypy-dev/2012-January/009048.html)) 26 | 27 | # Details 28 | 29 | See [PyCParser](https://github.com/albertz/PyCParser) for more. 30 | -------------------------------------------------------------------------------- /compile_to_py.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # PyCPython - interpret CPython in Python 3 | # by Albert Zeyer, 2011 4 | # code under BSD 2-Clause License 5 | 6 | from __future__ import print_function 7 | 8 | import sys 9 | import os 10 | import argparse 11 | from cpython import CPythonState, MyDir 12 | import cparser 13 | import cparser.interpreter 14 | from cparser.py_demo_unparse import Unparser 15 | import time 16 | import ast 17 | import ctypes 18 | 19 | 20 | # See State.CBuiltinTypes. 21 | builtin_ctypes_name_map = { 22 | ("void", "*"): "c_void_p", 23 | ("char",): "c_byte", 24 | ("unsigned", "char"): "c_ubyte", 25 | ("short",): "c_short", 26 | ("unsigned", "short"): "c_ushort", 27 | ("int",): "c_int", 28 | ("signed",): "c_int", 29 | ("unsigned", "int"): "c_uint", 30 | ("unsigned",): "c_uint", 31 | ("long",): "c_long", 32 | ("unsigned", "long"): "c_ulong", 33 | ("long", "long"): "c_longlong", 34 | ("unsigned", "long", "long"): "c_ulonglong", 35 | ("float",): "c_float", 36 | ("double",): "c_double", 37 | ("long", "double"): "c_longdouble", 38 | } 39 | 40 | 41 | def builtin_ctypes_name(s): 42 | # See State.CBuiltinTypes. 43 | assert isinstance(s, tuple) 44 | t = builtin_ctypes_name_map[s] 45 | assert hasattr(ctypes, t) 46 | return t 47 | 48 | 49 | def stdint_ctypes_name(s): 50 | # See State.StdIntTypes. 51 | assert isinstance(s, str) 52 | if s in ["ptrdiff_t", "intptr_t"]: 53 | s = "c_long" 54 | elif s == "FILE": 55 | s = "c_int" 56 | else: 57 | s = "c_%s" % s.replace("_t", "") 58 | assert hasattr(ctypes, s) 59 | return s 60 | 61 | 62 | def set_name_for_typedeffed_struct(obj, state): 63 | assert isinstance(obj, cparser.CTypedef) 64 | assert obj.name 65 | assert isinstance(state, cparser.State) 66 | if not isinstance(obj.type, (cparser.CStruct, cparser.CUnion)): 67 | return False 68 | if not obj.type.name: 69 | obj.type.name = "_anonymous_%s" % obj.name 70 | return True 71 | if isinstance(obj.type, cparser.CStruct): 72 | if obj.type.name in state.structs: 73 | return False 74 | # So that we will find it next time. Not sure why this can even happen. 75 | #state.structs[obj.type.name] = obj.type 76 | elif isinstance(obj.type, cparser.CUnion): 77 | if obj.type.name in state.unions: 78 | return False 79 | # See above. 80 | #state.unions[obj.type.name] = obj.type 81 | return True 82 | 83 | 84 | def fix_name(obj): 85 | assert obj.name 86 | if obj.name.startswith("__"): 87 | obj.name = "_M_%s" % obj.name[2:] 88 | 89 | 90 | class CodeGen: 91 | 92 | def __init__(self, f, state, interpreter): 93 | self.f = f 94 | self.state = state 95 | self.interpreter = interpreter 96 | self.structs = {} 97 | self.unions = {} 98 | self.delayed_structs = [] 99 | self._py_in_structs = False 100 | self._py_in_unions = False 101 | self._py_in_delayed = False 102 | self._py_in_globals = False 103 | self._py_globals = {} 104 | self._get_py_type_stack = [] 105 | self._anonymous_name_counter = 0 106 | 107 | def write_header(self): 108 | f = self.f 109 | f.write("#!/usr/bin/env python\n") 110 | f.write("# PyCPython - interpret CPython in Python\n") 111 | f.write("# Statically compiled CPython.\n\n") 112 | f.write("import sys\n") 113 | f.write("import better_exchook\n") 114 | f.write("import cparser\n") 115 | f.write("import cparser.interpreter\n") 116 | f.write("import cparser.cparser_utils\n") 117 | f.write("import ctypes\n") 118 | f.write("\n") 119 | f.write("better_exchook.install()\n") 120 | f.write("cparser.cparser_utils.setup_Structure_debug_helper()\n") 121 | f.write("intp = cparser.interpreter.Interpreter()\n") 122 | f.write("intp.setupStatic()\n") 123 | f.write("helpers = intp.helpers\n") 124 | f.write("ctypes_wrapped = intp.ctypes_wrapped\n") 125 | f.write("\n\n") 126 | 127 | def write_structs(self): 128 | self._py_in_structs = True 129 | self._write_structs("struct") 130 | self._py_in_structs = False 131 | 132 | def write_unions(self): 133 | self._py_in_unions = True 134 | self._write_structs("union") 135 | self._py_in_unions = False 136 | 137 | def fix_names(self): 138 | for content in self.state.contentlist: 139 | if isinstance(content, cparser.CTypedef): 140 | set_name_for_typedeffed_struct(content, self.state) 141 | if isinstance(content, ( 142 | cparser.CStruct, cparser.CUnion, cparser.CFunc, cparser.CTypedef, cparser.CVarDecl)): 143 | assert content.name 144 | fix_name(content) 145 | 146 | def _get_anonymous_name_counter(self): 147 | self._anonymous_name_counter += 1 148 | return self._anonymous_name_counter 149 | 150 | def _get_anonymous_name(self): 151 | return "_anonymous_%i" % self._get_anonymous_name_counter() 152 | 153 | def _write_structs(self, base_type): 154 | f = self.f 155 | f.write("class %ss:\n" % base_type) 156 | f.write(" pass\n") 157 | f.write("\n") 158 | cparse_base_type = {"struct": cparser.CStruct, "union": cparser.CUnion}[base_type] 159 | last_log_time = time.time() 160 | for i, content in enumerate(self.state.contentlist): 161 | if time.time() - last_log_time > 2.0: 162 | last_log_time = time.time() 163 | perc_compl = 100.0 * i / len(self.state.contentlist) 164 | cur_content_s = "%s %s" % (content.__class__.__name__, 165 | (getattr(content, "name", None) or "")) 166 | cur_file_s = getattr(content, "defPos", "") 167 | print("Compile %ss... (%.0f%%) (%s) (%s)" % (base_type, perc_compl, cur_content_s, cur_file_s)) 168 | if isinstance(content, cparser.CTypedef): 169 | content = content.type 170 | if not isinstance(content, cparse_base_type): 171 | continue 172 | if cparser.isExternDecl(content): 173 | content = self.state.getResolvedDecl(content) 174 | if cparser.isExternDecl(content): 175 | # Dummy placeholder. 176 | self._write_struct_dummy_extern(content) 177 | continue 178 | else: 179 | # We have a full declaration available. 180 | # It will be written later when we come to it in this loop. 181 | continue # we will write it later 182 | self._try_write_struct(content) 183 | f.write("\n\n") 184 | 185 | class WriteStructException(Exception): pass 186 | class NoBody(WriteStructException): pass 187 | class RecursiveConstruction(WriteStructException): pass 188 | class IncompleteStructCannotCompleteHere(WriteStructException): pass 189 | 190 | def _write_struct(self, content): 191 | assert content.name 192 | if content.body is None: 193 | raise self.NoBody() 194 | if getattr(content, "_comppy_constructing", None): 195 | raise self.RecursiveConstruction() 196 | content._comppy_constructing = True 197 | base_type = {cparser.CStruct: "struct", cparser.CUnion: "union"}[type(content)] 198 | ctype_base = {"struct": "ctypes.Structure", "union": "ctypes.Union"}[base_type] 199 | struct_dict = {"struct": self.structs, "union": self.unions}[base_type] 200 | # see _getCTypeStruct for reference 201 | fields = [] 202 | try: 203 | for c in content.body.contentlist: 204 | if not isinstance(c, cparser.CVarDecl): continue 205 | t = self.get_py_type(c.type) 206 | if c.arrayargs: 207 | if len(c.arrayargs) != 1: raise Exception(str(c) + " has too many array args") 208 | n = c.arrayargs[0].value 209 | t = "%s * %i" % (t, n) 210 | if hasattr(c, "bitsize"): 211 | fields.append("(%r, %s, %s)" % (str(c.name), t, c.bitsize)) 212 | else: 213 | fields.append("(%r, %s)" % (str(c.name), t)) 214 | finally: 215 | content._comppy_constructing = False 216 | f = self.f 217 | f.write("class _class_%s_%s(%s):\n" % (base_type, content.name, ctype_base)) 218 | f.write(" _fields_ = [\n") 219 | f.write(" %s]\n" % (",\n" + " " * 8).join(fields)) 220 | f.write("%ss.%s = _class_%s_%s\n" % (base_type, content.name, base_type, content.name)) 221 | f.write("del _class_%s_%s\n" % (base_type, content.name)) 222 | f.write("\n") 223 | content._comppy_written = True 224 | assert content.name not in struct_dict 225 | struct_dict[content.name] = content 226 | 227 | def _write_struct_dummy_extern(self, content): 228 | base_type = {cparser.CStruct: "struct", cparser.CUnion: "union"}[type(content)] 229 | self.f.write("%ss.%s = ctypes_wrapped.c_int # Dummy extern declaration\n" % (base_type, content.name)) 230 | struct_dict = {"struct": self.structs, "union": self.unions}[base_type] 231 | assert content.name 232 | assert content.name not in struct_dict 233 | struct_dict[content.name] = content 234 | 235 | def _try_write_struct(self, content): 236 | assert content.name 237 | assert content.body is not None 238 | base_type = {cparser.CStruct: "struct", cparser.CUnion: "union"}[type(content)] 239 | struct_dict = {"struct": self.structs, "union": self.unions}[base_type] 240 | if getattr(content, "_comppy_written", None): 241 | assert struct_dict[content.name] is content 242 | return 243 | try: 244 | self._write_struct(content) 245 | except self.WriteStructException: 246 | if content not in self.delayed_structs: 247 | assert content.name not in struct_dict 248 | self.delayed_structs.append(content) 249 | content._delayed_write = True 250 | 251 | def _check_local_struct_type(self, t): 252 | if not t.name: 253 | t.name = "_local_" + self._get_anonymous_name() 254 | if t.body is None: 255 | t2 = self.state.getResolvedDecl(t) 256 | if t2 is not None: 257 | assert t2.name == t.name 258 | t = t2 259 | base_type = {cparser.CStruct: "struct", cparser.CUnion: "union"}[type(t)] 260 | struct_dict = {"struct": self.structs, "union": self.unions}[base_type] 261 | if t.name not in struct_dict: 262 | if {"struct": self._py_in_structs, "union": self._py_in_unions}[base_type]: 263 | self._write_struct(t) 264 | elif self._py_in_delayed: 265 | self._write_delayed_struct(t) 266 | else: 267 | raise self.IncompleteStructCannotCompleteHere() 268 | 269 | def _write_delayed_struct(self, content, indent=""): 270 | # See cparser._getCTypeStruct for reference. 271 | f = self.f 272 | assert content.name 273 | base_type = {cparser.CStruct: "struct", cparser.CUnion: "union"}[type(content)] 274 | struct_dict = getattr(self, "%ss" % base_type) 275 | assert content.name not in struct_dict 276 | ctype_base = {"struct": "ctypes.Structure", "union": "ctypes.Union"}[base_type] 277 | if not getattr(content, "_wrote_header", False): 278 | f.write("%sclass _class_%s_%s(%s): pass\n" % (indent, base_type, content.name, ctype_base)) 279 | f.write("%s%ss.%s = _class_%s_%s\n" % (indent, base_type, content.name, base_type, content.name)) 280 | content._wrote_header = True 281 | if content.body is None: 282 | return 283 | 284 | parent_type = None 285 | type_stack = list(self._get_py_type_stack) 286 | if type_stack: 287 | # type_stack[-1], or getResolvedDecl of it, that is the current content. 288 | type_stack.pop() 289 | # Take next non-typedef type. 290 | while type_stack: 291 | parent_type = type_stack.pop() 292 | if not isinstance(parent_type, cparser.CTypedef): 293 | break 294 | 295 | if getattr(content, "_comppy_constructing", None): 296 | if parent_type is not None: 297 | # If the parent referred to us as a pointer, it's fine, 298 | # we can use our incomplete type and don't need to construct it now. 299 | if cparser.isPointerType(parent_type, alsoFuncPtr=True, alsoArray=False): 300 | return 301 | # Otherwise, we must construct it now. 302 | content._comppy_constructing.append(parent_type) 303 | if len(content._comppy_constructing) > 2: 304 | # We got called more than once. This is an infinite loop. 305 | raise self.RecursiveConstruction( 306 | "The parent types when we were called: %s" % (content._comppy_constructing,)) 307 | else: 308 | content._comppy_constructing = [parent_type] 309 | fields = [] 310 | try: 311 | for c in content.body.contentlist: 312 | if not isinstance(c, cparser.CVarDecl): continue 313 | t = self.get_py_type(c.type) 314 | if c.arrayargs: 315 | if len(c.arrayargs) != 1: raise Exception(str(c) + " has too many array args") 316 | n = c.arrayargs[0].value 317 | t = "%s * %i" % (t, n) 318 | if hasattr(c, "bitsize"): 319 | fields.append("(%r, %s, %s)" % (str(c.name), t, c.bitsize)) 320 | else: 321 | fields.append("(%r, %s)" % (str(c.name), t)) 322 | finally: 323 | content._comppy_constructing.pop() 324 | 325 | # finalize the type 326 | if content.name not in struct_dict: 327 | struct_dict[content.name] = content 328 | f.write("%s_class_%s_%s._fields_ = [\n%s %s]\n" % ( 329 | indent, base_type, content.name, indent, 330 | (",\n%s " % indent).join(fields))) 331 | f.write("%sdel _class_%s_%s\n" % (indent, base_type, content.name)) 332 | f.write("\n") 333 | 334 | def write_delayed_structs(self): 335 | self._py_in_delayed = True 336 | f = self.f 337 | for t in self.delayed_structs: 338 | base_type = {cparser.CStruct: "struct", cparser.CUnion: "union"}[type(t)] 339 | struct_dict = getattr(self, "%ss" % base_type) 340 | if t.name in struct_dict: continue # could be written meanwhile 341 | self._write_delayed_struct(t) 342 | del self.delayed_structs[:] 343 | f.write("\n\n") 344 | self._py_in_delayed = False 345 | 346 | def get_py_type(self, t): 347 | self._get_py_type_stack.append(t) 348 | try: 349 | return self._get_py_type(t) 350 | finally: 351 | self._get_py_type_stack.pop() 352 | 353 | def _get_py_type(self, t): 354 | if isinstance(t, cparser.CTypedef): 355 | if self._py_in_globals: 356 | assert t.name, "typedef target typedef must have name" 357 | return t.name 358 | return self.get_py_type(t.type) 359 | elif isinstance(t, (cparser.CStruct, cparser.CUnion)): 360 | base_type = {cparser.CStruct: "struct", cparser.CUnion: "union"}[type(t)] 361 | self._check_local_struct_type(t) 362 | assert t.name, "struct must have name, should have been assigned earlier also to anonymous structs" 363 | return "%ss.%s" % (base_type, t.name) 364 | elif isinstance(t, cparser.CBuiltinType): 365 | return "ctypes_wrapped.%s" % builtin_ctypes_name(t.builtinType) 366 | elif isinstance(t, cparser.CStdIntType): 367 | return "ctypes_wrapped.%s" % stdint_ctypes_name(t.name) 368 | elif isinstance(t, cparser.CEnum): 369 | int_type_name = t.getMinCIntType() 370 | return "ctypes_wrapped.%s" % stdint_ctypes_name(int_type_name) 371 | elif isinstance(t, cparser.CPointerType): 372 | if cparser.isVoidPtrType(t): 373 | return "ctypes_wrapped.c_void_p" 374 | else: 375 | return "ctypes.POINTER(%s)" % self.get_py_type(t.pointerOf) 376 | elif isinstance(t, cparser.CArrayType): 377 | if not t.arrayLen: 378 | return "ctypes.POINTER(%s)" % self.get_py_type(t.arrayOf) 379 | l = cparser.getConstValue(self.state, t.arrayLen) 380 | if l is None: 381 | l = 1 382 | return "%s * %i" % (self.get_py_type(t.arrayOf), l) 383 | elif isinstance(t, cparser.CFuncPointerDecl): 384 | if isinstance(t.type, cparser.CPointerType): 385 | # https://bugs.python.org/issue5710 386 | restype = "ctypes_wrapped.c_void_p" 387 | elif t.type == cparser.CBuiltinType(("void",)): 388 | restype = "None" 389 | else: 390 | restype = self.get_py_type(t.type) 391 | return "ctypes.CFUNCTYPE(%s)" % ", ".join([restype] + [self.get_py_type(a) for a in t.args]) 392 | elif isinstance(t, cparser.CFuncArgDecl): 393 | return self.get_py_type(t.type) 394 | raise Exception("unexpected type: %s" % type(t)) 395 | 396 | def _fixup_global_g_inner(self, a): 397 | def maybe_replace(value): 398 | # search for attrib access like "g.something" 399 | if (isinstance(value, ast.Attribute) 400 | and isinstance(value.value, ast.Name) 401 | and value.value.id == "g"): # found one 402 | assert value.attr in self._py_globals 403 | if isinstance(self._py_globals[value.attr], cparser.CFunc): 404 | # overwrite this with "something.__func__" to resolve the staticmethod 405 | return ast.Attribute( 406 | value=ast.Name(id=value.attr, ctx=ast.Load()), 407 | attr="__func__", 408 | ctx=ast.Load() 409 | ) 410 | # overwrite this with just "something" 411 | return ast.Name(id=value.attr, ctx=ast.Load()) 412 | return value 413 | for node in ast.walk(a): 414 | for fieldname, value in ast.iter_fields(node): 415 | if isinstance(value, ast.AST): 416 | setattr(node, fieldname, maybe_replace(value)) 417 | elif isinstance(value, list): 418 | for i in range(len(value)): 419 | value[i] = maybe_replace(value[i]) 420 | elif isinstance(value, tuple): 421 | setattr(node, fieldname, tuple(map(maybe_replace, value))) 422 | 423 | def write_globals(self): 424 | self._py_in_globals = True 425 | f = self.f 426 | f.write("class g:\n") 427 | last_log_time = time.time() 428 | count = count_incomplete = 0 429 | for i, content in enumerate(self.state.contentlist): 430 | if time.time() - last_log_time > 2.0: 431 | last_log_time = time.time() 432 | perc_compl = 100.0 * i / len(self.state.contentlist) 433 | cur_content_s = "%s %s" % (content.__class__.__name__, 434 | (getattr(content, "name", None) or "")) 435 | cur_file_s = getattr(content, "defPos", "") 436 | print("Compile... (%.0f%%) (%s) (%s)" % (perc_compl, cur_content_s, cur_file_s)) 437 | if isinstance(content, (cparser.CStruct, cparser.CUnion)): 438 | continue # Handled in the other loops. 439 | try: 440 | if cparser.isExternDecl(content): 441 | content = self.state.getResolvedDecl(content) 442 | if cparser.isExternDecl(content): 443 | # We will write some dummy placeholder. 444 | count_incomplete += 1 445 | else: 446 | # We have a full declaration available. 447 | continue # we will write it later 448 | count += 1 449 | if content.name: 450 | fix_name(content) 451 | if content.name in self._py_globals: 452 | print("Error (ignored): %r defined twice, earlier as %r, now as %r" % ( 453 | content.name, self._py_globals[content.name], content)) 454 | continue 455 | self._py_globals[content.name] = content 456 | else: 457 | continue 458 | if isinstance(content, cparser.CFunc): 459 | funcEnv = self.interpreter._translateFuncToPyAst(content, noBodyMode="code-with-exception") 460 | pyAst = funcEnv.astNode 461 | assert isinstance(pyAst, ast.FunctionDef) 462 | pyAst.decorator_list.append(ast.Name(id="staticmethod", ctx=ast.Load())) 463 | Unparser(pyAst, indent=1, file=f) 464 | f.write("\n") 465 | elif isinstance(content, (cparser.CStruct, cparser.CUnion)): 466 | pass # Handled in the other loops. 467 | elif isinstance(content, cparser.CTypedef): 468 | f.write(" %s = %s\n" % (content.name, self.get_py_type(content.type))) 469 | elif isinstance(content, cparser.CVarDecl): 470 | # See cparser.interpreter.GlobalScope.getVar() for reference. 471 | decl_type, bodyAst, bodyType = \ 472 | self.interpreter.globalScope._getDeclTypeBodyAstAndType(content) 473 | pyEmptyAst = self.interpreter.globalScope._getEmptyValueAst(decl_type) 474 | self._fixup_global_g_inner(pyEmptyAst) 475 | f.write(" %s = " % content.name) 476 | Unparser(pyEmptyAst, file=f) 477 | f.write("\n") 478 | bodyValueAst = self.interpreter.globalScope._getVarBodyValueAst( 479 | content, decl_type, bodyAst, bodyType) 480 | if bodyValueAst is not None: 481 | self._fixup_global_g_inner(bodyValueAst) 482 | f.write(" helpers.assign(%s, " % content.name) 483 | Unparser(bodyValueAst, file=f) 484 | f.write(")\n") 485 | elif isinstance(content, cparser.CEnum): 486 | int_type_name = content.getMinCIntType() 487 | f.write(" %s = ctypes_wrapped.%s\n" % (content.name, stdint_ctypes_name(int_type_name))) 488 | else: 489 | raise Exception("unexpected content type: %s" % type(content)) 490 | except Exception as exc: 491 | print("!!! Exception while compiling %r" % content) 492 | if content.name: 493 | f.write(" %s = 'Compile exception ' %r\n" % (content.name, str(exc))) 494 | sys.excepthook(*sys.exc_info()) 495 | # We continue... 496 | f.write("\n\n") 497 | self._py_in_globals = False 498 | 499 | def _new_wrapped_value_callback(self, name, value): 500 | assert self._py_in_globals 501 | assert isinstance(value, cparser.CWrapValue) 502 | if issubclass(value.value, (ctypes.Structure, ctypes.Union)): 503 | # Created via cparser._getCTypeStruct(). 504 | assert isinstance(value.value._py, (cparser.CStruct, cparser.CUnion)) 505 | t = value.value._py 506 | base_type = {cparser.CStruct: "struct", cparser.CUnion: "union"}[type(t)] 507 | assert not t.name 508 | t.name = "_local_" + self._get_anonymous_name() 509 | self._write_delayed_struct(t, indent=" ") 510 | self.f.write(" values.%s = cparser.CWrapValue(%ss.%s)\n" % (name, base_type, t.name)) 511 | else: 512 | self.f.write(" values.%s = None # TODO CWrapValue(value=%r, decl=%r, name=%r)\n" % ( 513 | name, value.value, value.decl, value.name)) 514 | 515 | def write_values(self): 516 | f = self.f 517 | f.write("class values:\n") 518 | 519 | handled_list = set() 520 | def maybe_add_wrap_value(container_name, var_name, var): 521 | if not isinstance(var, cparser.CWrapValue): return 522 | wrap_name = self.interpreter.wrappedValues.get_value(var) 523 | handled_list.add(wrap_name) 524 | f.write(" %s = intp.stateStructs[0].%s[%r]\n" % (wrap_name, container_name, var_name)) 525 | 526 | # These are added by globalincludewrappers. 527 | for varname, var in sorted(self.state.vars.items()): 528 | maybe_add_wrap_value("vars", varname, var) 529 | for varname, var in sorted(self.state.funcs.items()): 530 | maybe_add_wrap_value("funcs", varname, var) 531 | remaining = self.interpreter.wrappedValues.list.difference(handled_list) 532 | assert not remaining 533 | f.write("\n\n") 534 | 535 | if self._new_wrapped_value_callback not in self.interpreter.wrappedValues.callbacks_register_new: 536 | self.interpreter.wrappedValues.callbacks_register_new.append(self._new_wrapped_value_callback) 537 | 538 | def write_footer(self): 539 | f = self.f 540 | f.write("if __name__ == '__main__':\n") 541 | f.write(" g.Py_Main(ctypes_wrapped.c_int(len(sys.argv)),\n" 542 | " (ctypes.POINTER(ctypes_wrapped.c_char) * (len(sys.argv) + 1))(\n" 543 | " *[ctypes.cast(intp._make_string(arg), ctypes.POINTER(ctypes_wrapped.c_char))\n" 544 | " for arg in sys.argv]))\n") 545 | f.write("\n") 546 | 547 | 548 | def main(argv): 549 | state = CPythonState() 550 | 551 | out_fn = MyDir + "/cpython_static.py" 552 | print("Compile CPython to %s." % os.path.basename(out_fn)) 553 | 554 | print("Parsing CPython...", end="") 555 | try: 556 | state.parse_cpython() 557 | except Exception: 558 | print("!!! Exception while parsing. Should not happen. Cannot recover. Please report this bug.") 559 | print("The parser currently is here:", state.curPosAsStr()) 560 | raise 561 | if state._errors: 562 | print("finished, parse errors:") 563 | for m in state._errors: 564 | print(m) 565 | else: 566 | print("finished, no parse errors.") 567 | 568 | interpreter = cparser.interpreter.Interpreter() 569 | interpreter.register(state) 570 | 571 | print("Compile...") 572 | f = open(out_fn, "w") 573 | code_gen = CodeGen(f, state, interpreter) 574 | code_gen.write_header() 575 | code_gen.fix_names() 576 | code_gen.write_structs() 577 | code_gen.write_unions() 578 | code_gen.write_delayed_structs() 579 | code_gen.write_values() 580 | code_gen.write_globals() 581 | code_gen.write_footer() 582 | f.close() 583 | 584 | print("Done.") 585 | 586 | 587 | if __name__ == "__main__": 588 | main(sys.argv) 589 | -------------------------------------------------------------------------------- /cpython.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # PyCPython - interpret CPython in Python 3 | # by Albert Zeyer, 2011 4 | # code under BSD 2-Clause License 5 | 6 | from __future__ import print_function 7 | 8 | import argparse 9 | import os 10 | import sys 11 | 12 | if __name__ == '__main__': 13 | MyDir = os.path.dirname(sys.argv[0]) 14 | else: 15 | MyDir = "." 16 | 17 | CPythonDir = MyDir + "/CPython" 18 | 19 | import cparser 20 | import cparser.interpreter 21 | 22 | 23 | class CPythonState(cparser.State): 24 | 25 | def __init__(self): 26 | super(CPythonState, self).__init__() 27 | self.autoSetupSystemMacros() 28 | self.autoSetupGlobalIncludeWrappers() 29 | 30 | def findIncludeFullFilename(self, filename, local): 31 | fullfn = CPythonDir + "/Include/" + filename 32 | if os.path.exists(fullfn): return fullfn 33 | return super(CPythonState, self).findIncludeFullFilename(filename, local) 34 | 35 | def readLocalInclude(self, filename): 36 | #print " ", filename, "..." 37 | if filename == "pyconfig.h": 38 | def reader(): 39 | # see CPython/pyconfig.h.in for reference 40 | import ctypes 41 | sizeofMacro = lambda t: cparser.Macro(rightside=str(ctypes.sizeof(t))) 42 | self.macros["SIZEOF_SHORT"] = sizeofMacro(ctypes.c_short) 43 | self.macros["SIZEOF_INT"] = sizeofMacro(ctypes.c_int) 44 | self.macros["SIZEOF_LONG"] = sizeofMacro(ctypes.c_long) 45 | self.macros["SIZEOF_LONG_LONG"] = sizeofMacro(ctypes.c_longlong) 46 | self.macros["SIZEOF_DOUBLE"] = sizeofMacro(ctypes.c_double) 47 | self.macros["SIZEOF_FLOAT"] = sizeofMacro(ctypes.c_float) 48 | self.macros["SIZEOF_VOID_P"] = sizeofMacro(ctypes.c_void_p) 49 | self.macros["SIZEOF_SIZE_T"] = sizeofMacro(ctypes.c_size_t) 50 | self.macros["SIZEOF_UINTPTR_T"] = sizeofMacro(ctypes.POINTER(ctypes.c_uint)) 51 | self.macros["SIZEOF_PTHREAD_T"] = self.macros["SIZEOF_LONG"] 52 | self.macros["SIZEOF_WCHAR_T"] = sizeofMacro(ctypes.c_wchar) 53 | self.macros["SIZEOF_PID_T"] = self.macros["SIZEOF_INT"] 54 | self.macros["SIZEOF_TIME_T"] = self.macros["SIZEOF_LONG"] 55 | self.macros["SIZEOF__BOOL"] = cparser.Macro(rightside="1") 56 | self.macros["HAVE_SIGNAL_H"] = cparser.Macro(rightside="1") 57 | self.macros["HAVE_STDARG_PROTOTYPES"] = cparser.Macro(rightside="1") 58 | self.macros["HAVE_STD_ATOMIC"] = cparser.Macro(rightside="1") 59 | self.macros["HAVE_WCHAR_H"] = cparser.Macro(rightside="1") 60 | self.macros["_POSIX_THREADS"] = cparser.Macro(rightside="1") 61 | # _GNU_SOURCE, _POSIX_C_SOURCE or so? 62 | return 63 | yield None # make it a generator 64 | return reader(), None 65 | return super(CPythonState, self).readLocalInclude(filename) 66 | 67 | def parse_cpython(self): 68 | # We keep all in the same state, i.e. the same static space. 69 | # This also means that we don't reset macro definitions. This speeds up header includes. 70 | # Usually this is not a problem. 71 | self.macros["Py_BUILD_CORE"] = cparser.Macro(rightside="1") # Makefile 72 | self.macros["Py_BUILD_CORE_BUILTIN"] = cparser.Macro(rightside="1") # Makefile 73 | cparser.parse(CPythonDir + "/Modules/main.c", self) # Py_Main 74 | self.macros["FAST_LOOPS"] = cparser.Macro(rightside="0") # not sure where this would come from 75 | cparser.parse(CPythonDir + "/Python/ceval.c", self) # PyEval_EvalFrameEx etc 76 | del self.macros["EMPTY"] # will be redefined later 77 | cparser.parse(CPythonDir + "/Python/getopt.c", self) # _PyOS_GetOpt 78 | cparser.parse(CPythonDir + "/Python/pythonrun.c", self) # Py_Initialize 79 | cparser.parse(CPythonDir + "/Python/pystate.c", self) # PyInterpreterState_New 80 | cparser.parse(CPythonDir + "/Python/sysmodule.c", self) # PySys_ResetWarnOptions 81 | cparser.parse(CPythonDir + "/Python/random.c", self) # _PyRandom_Init 82 | cparser.parse(CPythonDir + "/Objects/object.c", self) # _Py_ReadyTypes etc 83 | cparser.parse(CPythonDir + "/Objects/typeobject.c", self) # PyType_Ready 84 | cparser.parse(CPythonDir + "/Objects/tupleobject.c", self) # PyTuple_New 85 | del self.macros["Return"] # will be used differently 86 | # We need these macro hacks because dictobject.c will use the same vars. 87 | self.macros["length_hint_doc"] = cparser.Macro(rightside="length_hint_doc__dict") 88 | self.macros["numfree"] = cparser.Macro(rightside="numfree__dict") 89 | self.macros["free_list"] = cparser.Macro(rightside="free_list__dict") 90 | cparser.parse(CPythonDir + "/Objects/dictobject.c", self) # PyDict_New 91 | # We need this macro hack because stringobject.c will use the same var. 92 | self.macros["sizeof__doc__"] = cparser.Macro(rightside="sizeof__doc__str") 93 | cparser.parse(CPythonDir + "/Objects/stringobject.c", self) # PyString_FromString 94 | cparser.parse(CPythonDir + "/Objects/obmalloc.c", self) # PyObject_Free 95 | cparser.parse(CPythonDir + "/Modules/gcmodule.c", self) # _PyObject_GC_NewVar 96 | cparser.parse(CPythonDir + "/Objects/descrobject.c", self) # PyDescr_NewWrapper 97 | # We need these macro hacks because methodobject.c will use the same vars. 98 | self.macros["numfree"] = cparser.Macro(rightside="numfree__methodobj") 99 | self.macros["free_list"] = cparser.Macro(rightside="free_list__methodobj") 100 | cparser.parse(CPythonDir + "/Objects/methodobject.c", self) # PyCFunction_NewEx 101 | # We need these macro hacks because methodobject.c will use the same vars. 102 | self.macros["numfree"] = cparser.Macro(rightside="numfree__list") 103 | self.macros["free_list"] = cparser.Macro(rightside="free_list__list") 104 | self.macros["sizeof_doc"] = cparser.Macro(rightside="sizeof_doc__list") 105 | self.macros["length_hint_doc"] = cparser.Macro(rightside="length_hint_doc__list") 106 | self.macros["index_doc"] = cparser.Macro(rightside="index_doc__list") 107 | self.macros["count_doc"] = cparser.Macro(rightside="count__list") 108 | cparser.parse(CPythonDir + "/Objects/listobject.c", self) # PyList_New 109 | cparser.parse(CPythonDir + "/Objects/abstract.c", self) # PySequence_List 110 | cparser.parse(CPythonDir + "/Python/modsupport.c", self) # Py_BuildValue 111 | 112 | 113 | def init_faulthandler(sigusr1_chain=False): 114 | """ 115 | :param bool sigusr1_chain: whether the default SIGUSR1 handler should also be called. 116 | """ 117 | try: 118 | import faulthandler 119 | except ImportError as e: 120 | print("faulthandler import error. %s" % e) 121 | return 122 | # Only enable if not yet enabled -- otherwise, leave it in its current state. 123 | if not faulthandler.is_enabled(): 124 | faulthandler.enable() 125 | if os.name != 'nt': 126 | import signal 127 | # This will print a backtrace on SIGUSR1. 128 | # Note that this also works when Python is hanging, 129 | # i.e. in cases where register_sigusr1_print_backtrace() will not work. 130 | faulthandler.register(signal.SIGUSR1, all_threads=True, chain=sigusr1_chain) 131 | 132 | 133 | def register_sigusr1_print_backtrace(): 134 | if os.name == "nt": 135 | return 136 | 137 | def sigusr1_handler(sig, frame): 138 | print("--- SIGUSR1 handler") 139 | better_exchook.print_tb(tb=frame) 140 | 141 | import signal 142 | signal.signal(signal.SIGUSR1, sigusr1_handler) 143 | 144 | 145 | def main(argv): 146 | argparser = argparse.ArgumentParser( 147 | usage="%s [PyCPython options, see below] [CPython options, see via --help]" % argv[0], 148 | description="Emulate CPython by interpreting its C code via PyCParser.", 149 | epilog="All other options are passed on to CPython. Use --help to see them.", 150 | add_help=False # we will add our own 151 | ) 152 | argparser.add_argument( 153 | '--pycpython-help', action='help', help='show this help message and exit') 154 | argparser.add_argument( 155 | '--dump-python', action='store', nargs=1, 156 | help="Dumps the converted Python code of the specified function, e.g. Py_Main.") 157 | argparser.add_argument( 158 | '--verbose-jit', action='store_true', 159 | help="Prints what functions and global vars we are going to translate.") 160 | args_ns, argv_rest = argparser.parse_known_args(argv[1:]) 161 | argv = argv[:1] + argv_rest 162 | print("PyCPython -", argparser.description,) 163 | print("(use --pycpython-help for help)") 164 | 165 | state = CPythonState() 166 | 167 | print("Parsing CPython...", end="") 168 | state.parse_cpython() 169 | 170 | if state._errors: 171 | print("finished, parse errors:") 172 | for m in state._errors: 173 | print(m) 174 | else: 175 | print("finished, no parse errors.") 176 | 177 | interpreter = cparser.interpreter.Interpreter() 178 | interpreter.register(state) 179 | 180 | if args_ns.dump_python: 181 | for fn in args_ns.dump_python: 182 | print() 183 | print("PyAST of %s:" % fn) 184 | interpreter.dumpFunc(fn) 185 | sys.exit() 186 | 187 | if args_ns.verbose_jit: 188 | interpreter.debug_print_getFunc = True 189 | interpreter.debug_print_getVar = True 190 | 191 | args = ("Py_Main", len(argv), argv + [None]) 192 | print("Run", args, ":") 193 | interpreter.runFunc(*args) 194 | 195 | 196 | if __name__ == '__main__': 197 | import better_exchook 198 | better_exchook.install() 199 | register_sigusr1_print_backtrace() 200 | init_faulthandler(sigusr1_chain=True) 201 | main(sys.argv) 202 | --------------------------------------------------------------------------------