├── LiveAPI_MakeDoc.py
├── ReadMe.txt
├── _Sample_Output
    ├── Live.css
    └── Live.xml
├── _Support
    ├── __init__.py
    ├── dis.py
    ├── inspect.py
    ├── linecache.py
    ├── opcode.py
    ├── token.py
    └── tokenize.py
└── __init__.py


/LiveAPI_MakeDoc.py:
--------------------------------------------------------------------------------
  1 | # http://remotescripts.blogspot.com
  2 | 
  3 | """
  4 | Copyright (C) 2011 Hanz Petrov <hanz.petrov@gmail.com>
  5 | MIDI Remote Script for generating Live API documentation,
  6 | based on realtime inspection of the Live module.
  7 | Writes two files to the userhome directory - Live.xml and Live.css.
  8 | 
  9 | Inspired in part by the following Live API exploration modules:
 10 | dumpXML by Nathan Ramella http://code.google.com/p/liveapi/source/browse/trunk/docs/Ableton+Live+API/makedocs
 11 | and LiveAPIGen by Patrick Mueller http://muellerware.org
 12 | 
 13 | Parts of the describe methods are based on "describe" by Anand, found at:
 14 | http://code.activestate.com/recipes/553262-list-classes-methods-and-functions-in-a-module/
 15 | 
 16 | This program is free software: you can redistribute it and/or modify
 17 | it under the terms of the GNU General Public License as published by
 18 | the Free Software Foundation, either version 3 of the License, or
 19 | (at your option) any later version.
 20 | 
 21 | This program is distributed in the hope that it will be useful,
 22 | but WITHOUT ANY WARRANTY; without even the implied warranty of
 23 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 24 | GNU General Public License for more details.
 25 | 
 26 | You should have received a copy of the GNU General Public License
 27 | along with this program.  If not, see <http://www.gnu.org/licenses/>.
 28 | """
 29 | 
 30 | import Live
 31 | import os, sys, types
 32 | from _Support import inspect
 33 | from _Framework.ControlSurface import ControlSurface
 34 | 
 35 | class APIMakeDoc(ControlSurface):
 36 | 
 37 |     def __init__(self, c_instance):
 38 |         ControlSurface.__init__(self, c_instance) 
 39 |         module = Live
 40 |         outfilename = (str(module.__name__) + ".xml")
 41 |         outfilename = (os.path.join(os.path.expanduser('~'), outfilename))
 42 |         cssfilename = "Live.css"
 43 |         cssfilename = (os.path.join(os.path.expanduser('~'), cssfilename))
 44 |         make_doc(module, outfilename, cssfilename)
 45 | 
 46 | 
 47 |     def disconnect(self):
 48 |         ControlSurface.disconnect(self)
 49 | 
 50 | 
 51 | def make_doc(module, outfilename, cssfilename):
 52 |     if outfilename != None:
 53 |         stdout_old = sys.stdout
 54 | 
 55 |         outputfile = open(cssfilename, 'w')
 56 |         sys.stdout = outputfile
 57 |         print(css)
 58 |         outputfile.close()
 59 | 
 60 |         outputfile = open(outfilename, 'w')
 61 |         sys.stdout = outputfile
 62 | 
 63 |         print ('<?xml-stylesheet type="text/css" href="Live.css"?>') # set stylesheet to Live.css
 64 |         print ('<Live>')
 65 |         app = Live.Application.get_application() # get a handle to the App
 66 |         maj = app.get_major_version() # get the major version from the App
 67 |         min = app.get_minor_version() # get the minor version from the App
 68 |         bug = app.get_bugfix_version() # get the bugfix version from the App            
 69 |         print ('Live API version ' + str(maj) + "." + str(min) + "." + str(bug)) # main title
 70 |         print('<Doc>\t%s</Doc>\n' % header)
 71 |         print('<Doc>\t%s</Doc>\n' % disclaimer)
 72 | 
 73 |         describe_module(module)
 74 | 
 75 |         print ("</Live>")
 76 |         outputfile.close()
 77 |         sys.stdout = stdout_old
 78 |             
 79 |             
 80 | def get_doc(obj):
 81 |     """ Get object's doc string and remove \n's and clean up <'s and >'s for XML compatibility"""
 82 | 
 83 |     doc = False
 84 |     if obj.__doc__ != None:
 85 |         doc = (obj.__doc__).replace("\n", "") #remove newlines from Live API docstings, for wrapped display
 86 |         doc = doc.replace("   ", "") #Strip chunks of whitespace from docstrings, for wrapped display
 87 |         doc = doc.replace("<", "&lt;") #replace XML reserved characters 
 88 |         doc = doc.replace(">", "&gt;")
 89 |         doc = doc.replace("&", "&amp;")
 90 |     return doc
 91 | 
 92 | 
 93 | def print_obj_info(description, obj, name = None):
 94 |     """ Print object's descriptor and name on one line, and docstring (if any) on the next """
 95 |     
 96 |     if hasattr(obj, '__name__'):
 97 |         name_str = obj.__name__
 98 |     else:
 99 |         name_str = name        
100 | 
101 |     if len(LINE) != 0:
102 |         LINE.append("." + name_str)
103 |         if inspect.ismethod(obj) or inspect.isbuiltin(obj): 
104 |             LINE[-1] += "()"
105 |     else:
106 |         LINE.append(name_str)
107 |     line_str = ""
108 |     for item in LINE:
109 |         line_str += item
110 |     print ('<%s>%s<Description>%s</Description></%s>\n' % (description, line_str, description, description))
111 | 
112 |     if hasattr(obj, '__doc__'):
113 |         if obj.__doc__ != None:
114 |             print('<Doc>\t%s</Doc>\n' % get_doc(obj))
115 | 
116 | 
117 | def describe_obj(descr, obj):
118 |     """ Describe object passed as argument, and identify as Class, Method, Property, Value, or Built-In """
119 | 
120 |     if obj.__name__ == "<unnamed Boost.Python function>" or obj.__name__.startswith('__'): #filter out descriptors
121 |         return
122 |     if (obj.__name__ == ("type")) or (obj.__name__ == ("class")): #filter out non-subclass type types 
123 |         return
124 |     print_obj_info(descr, obj)
125 |     if inspect.ismethod(obj) or inspect.isbuiltin(obj): #go no further for these objects
126 |         LINE.pop()
127 |         return
128 |     else:
129 |         try: 
130 |             members = inspect.getmembers(obj)
131 |             for (name, member) in members:
132 |                 if inspect.isbuiltin(member):                    
133 |                     describe_obj("Built-In", member)
134 |             for (name, member) in members:
135 |                 if str(type(member)) == "<type 'property'>":
136 |                     print_obj_info("Property", member, name)
137 |                     LINE.pop()
138 |             for (name, member) in members:
139 |                 if inspect.ismethod(member):
140 |                     describe_obj("Method", member)                    
141 |             for (name, member) in members:
142 |                 if (str(type(member)).startswith( "<class" )):
143 |                     print_obj_info("Value", member, name)
144 |                     LINE.pop()
145 |             for (name, member) in members:
146 |                 if str(type(member)) == "<type 'object'>" or (str(type(member)) == "<type 'type'>" and not repr(obj).startswith("<class '")): #filter out unwanted types
147 |                     continue
148 |                 if inspect.isclass(member) and str(type(member)) == "<type 'type'>":
149 |                     describe_obj("Sub-Class", member)   
150 |             for (name, member) in members:
151 |                 if str(type(member)) == "<type 'object'>" or (str(type(member)) == "<type 'type'>" and not repr(obj).startswith("<class '")): #filter out unwanted types
152 |                     continue
153 |                 if inspect.isclass(member) and not str(type(member)) == "<type 'type'>":
154 |                     describe_obj("Class", member)
155 |             LINE.pop()
156 |         except: 
157 |             return
158 | 
159 | 
160 | def describe_module(module):
161 |     """ Describe the module object passed as argument
162 |     including its root classes and functions """
163 | 
164 |     print_obj_info("Module", module)
165 | 
166 |     for name in dir(module): #do the built-ins first
167 |         obj = getattr(module, name)
168 |         if inspect.isbuiltin(obj):
169 |             describe_obj("Built-In", obj)
170 | 
171 |     for name in dir(module): #then the rest
172 |         obj = getattr(module, name)            
173 |         if inspect.isclass(obj):
174 |             describe_obj("Class", obj)
175 |         elif (inspect.ismethod(obj) or inspect.isfunction(obj)):
176 |             describe_obj("Method", obj)
177 |         elif inspect.ismodule(obj):
178 |             describe_module(obj)
179 |     LINE.pop()            
180 |             
181 |             
182 | LINE = []
183 |             
184 | header = """Unofficial Live API documentation generated by the "API_MakeDoc" MIDI Remote Script.   
185 |             <requirement xmlns:html="http://www.w3.org/1999/xhtml">
186 |             <html:a href="http://remotescripts.blogspot.com">http://remotescripts.blogspot.com</html:a></requirement>
187 |             """
188 | disclaimer  = """This is unofficial documentation. Please do not contact Ableton with questions or problems relating to the use of this documentation."""
189 | 
190 | css = """/* Style Sheet for formatting XML output of Live API Inspector */
191 |     Live
192 |     {
193 |     background: #f8f8f8;
194 |     display: block;
195 |     margin-bottom: 10px;
196 |     margin-left: 20px;
197 |     margin-top: 10px;
198 |     padding: 4px;
199 |     font-family: "Lucida Sans Unicode", "Lucida Sans", "Lucida Grande", Verdana, sans-serif;
200 |     font-weight: bold;
201 |     color: #000000;
202 |     font-size: 10pt;
203 |     } 
204 | 
205 |     Module, Class, Sub-Class
206 |     {
207 |     display: block;
208 |     margin-bottom: 5px;
209 |     margin-top: 10px;
210 |     margin-left: -5px;
211 |     padding-left: 5px;
212 |     padding-top: 4px;
213 |     padding-bottom: 4px;
214 |     background: silver;
215 |     font-size: 12pt;
216 |     background-color: #DDD;
217 |     border: solid 1px #AAA;
218 |     color: #333;
219 |     }
220 | 
221 |     Module
222 |     {
223 |     display: block;
224 |     color: #000;
225 |     background-color: #CCC;
226 |     }
227 | 
228 |     Description
229 |     {
230 |     display: inline;
231 |     margin-left: 5px;
232 |     color: #000000;
233 |     font-family: Arial, Helvetica, sans-serif;
234 |     font-style: italic;
235 |     font-weight: normal;
236 |     font-size: 9pt;
237 |     }
238 | 
239 |     Doc
240 |     {
241 |     display: block;
242 |     color: #408080;
243 |     margin-left: 20pt;
244 |     font-family: Arial, Helvetica, sans-serif;
245 |     font-style: italic;
246 |     font-weight: normal;
247 |     font-size: 9pt;
248 |     }
249 |     Method 
250 |     {
251 |     display: block;
252 |     margin-top: 10px;
253 |     color: #000080;
254 |     }
255 |     Built-In 
256 |     {		
257 |     display: block;
258 |     margin-top: 10px;
259 |     color: #081798;
260 |     }
261 |     Property 
262 |     {
263 |     display: block;
264 |     margin-top: 10px;
265 |     color: #0000AF;
266 |     }
267 |     Value 
268 |     {
269 |     display: block;
270 |     margin-top: 10px;
271 |     color: #1C5A8D;
272 |     }
273 |     """


--------------------------------------------------------------------------------
/ReadMe.txt:
--------------------------------------------------------------------------------
 1 | 
 2 | http://remotescripts.blogspot.com
 3 | 
 4 | ----------------------
 5 | API_MakeDoc revision 0
 6 | ----------------------
 7 | 
 8 | This set of Python scripts will generate Live API documentation in XML format, based on realtime inspection of the Live module. Compatible with Live 8.1 & higher, on PC or mac. 
 9 | 
10 | To run, first drop the API_LiveDoc folder into Ableton's MIDI Remote Scripts directory:
11 | 
12 | MAC OS X -> /Applications/Live 8.x OS X/Live.app/Contents/App-Resources/MIDI Remote Scripts/
13 | PC -> C:\Program Files\Ableton\Live 8.x\Resources\MIDI Remote Scripts\ 
14 | 
15 | Next, select "API_MakeDoc" from the MIDI Preferences dialog, then reset immediately to "None" (there is no need to close the dialog first). Two files will be written to the userhome directory: Live.xml and Live.css (sample output is included in the _Sample_Output directory). To locate the userhome directory:
16 | 
17 | MAC OS X -> click the Home icon in the Finder toolbar
18 | PC -> C:\Documents and Settings\username\
19 | 
20 | Open Live.xml in a web browser to view the API documentation.
21 | 
22 | ----------------
23 | Revision History
24 | ----------------
25 | 
26 | 2011-03-20 
27 | * Initial API_MakeDoc release
28 | 
29 | Visit http://remotescripts.blogspot.com for the latest updates and more information
30 | 
31 | 
32 | ----------
33 | DISCLAIMER
34 | ----------
35 | 
36 | THESE FILES ARE PROVIDED AS-IS, WITHOUT ANY WARRANTY, EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO FITNESS FOR ANY PARTICULAR PURPOSE.


--------------------------------------------------------------------------------
/_Sample_Output/Live.css:
--------------------------------------------------------------------------------
 1 | /* Style Sheet for formatting XML output of Live API Inspector */
 2 |     Live
 3 |     {
 4 |     background: #f8f8f8;
 5 |     display: block;
 6 |     margin-bottom: 10px;
 7 |     margin-left: 20px;
 8 |     margin-top: 10px;
 9 |     padding: 4px;
10 |     font-family: "Lucida Sans Unicode", "Lucida Sans", "Lucida Grande", Verdana, sans-serif;
11 |     font-weight: bold;
12 |     color: #000000;
13 |     font-size: 10pt;
14 |     } 
15 | 
16 |     Module, Class, Sub-Class
17 |     {
18 |     display: block;
19 |     margin-bottom: 5px;
20 |     margin-top: 10px;
21 |     margin-left: -5px;
22 |     padding-left: 5px;
23 |     padding-top: 4px;
24 |     padding-bottom: 4px;
25 |     background: silver;
26 |     font-size: 12pt;
27 |     background-color: #DDD;
28 |     border: solid 1px #AAA;
29 |     color: #333;
30 |     }
31 | 
32 |     Module
33 |     {
34 |     display: block;
35 |     color: #000;
36 |     background-color: #CCC;
37 |     }
38 | 
39 |     Description
40 |     {
41 |     display: inline;
42 |     margin-left: 5px;
43 |     color: #000000;
44 |     font-family: Arial, Helvetica, sans-serif;
45 |     font-style: italic;
46 |     font-weight: normal;
47 |     font-size: 9pt;
48 |     }
49 | 
50 |     Doc
51 |     {
52 |     display: block;
53 |     color: #408080;
54 |     margin-left: 20pt;
55 |     font-family: Arial, Helvetica, sans-serif;
56 |     font-style: italic;
57 |     font-weight: normal;
58 |     font-size: 9pt;
59 |     }
60 |     Method 
61 |     {
62 |     display: block;
63 |     margin-top: 10px;
64 |     color: #000080;
65 |     }
66 |     Built-In 
67 |     {		
68 |     display: block;
69 |     margin-top: 10px;
70 |     color: #081798;
71 |     }
72 |     Property 
73 |     {
74 |     display: block;
75 |     margin-top: 10px;
76 |     color: #0000AF;
77 |     }
78 |     Value 
79 |     {
80 |     display: block;
81 |     margin-top: 10px;
82 |     color: #1C5A8D;
83 |     }
84 |     
85 | 


--------------------------------------------------------------------------------
/_Support/__init__.py:
--------------------------------------------------------------------------------
1 | 
2 | def create_instance(c_instance):
3 |     """ Dummy function, this is needed so that the folder
4 |       is recognised as a module
5 |   """
6 |     return 0
7 | 
8 | 


--------------------------------------------------------------------------------
/_Support/dis.py:
--------------------------------------------------------------------------------
  1 | """Disassembler of Python byte code into mnemonics."""
  2 | 
  3 | import sys
  4 | import types
  5 | 
  6 | from opcode import *
  7 | from opcode import __all__ as _opcodes_all
  8 | 
  9 | __all__ = ["dis","disassemble","distb","disco"] + _opcodes_all
 10 | del _opcodes_all
 11 | 
 12 | def dis(x=None):
 13 |     """Disassemble classes, methods, functions, or code.
 14 | 
 15 |     With no argument, disassemble the last traceback.
 16 | 
 17 |     """
 18 |     if x is None:
 19 |         distb()
 20 |         return
 21 |     if type(x) is types.InstanceType:
 22 |         x = x.__class__
 23 |     if hasattr(x, 'im_func'):
 24 |         x = x.im_func
 25 |     if hasattr(x, 'func_code'):
 26 |         x = x.func_code
 27 |     if hasattr(x, '__dict__'):
 28 |         items = x.__dict__.items()
 29 |         items.sort()
 30 |         for name, x1 in items:
 31 |             if type(x1) in (types.MethodType,
 32 |                             types.FunctionType,
 33 |                             types.CodeType,
 34 |                             types.ClassType):
 35 |                 print "Disassembly of %s:" % name
 36 |                 try:
 37 |                     dis(x1)
 38 |                 except TypeError, msg:
 39 |                     print "Sorry:", msg
 40 |                 print
 41 |     elif hasattr(x, 'co_code'):
 42 |         disassemble(x)
 43 |     elif isinstance(x, str):
 44 |         disassemble_string(x)
 45 |     else:
 46 |         raise TypeError, \
 47 |               "don't know how to disassemble %s objects" % \
 48 |               type(x).__name__
 49 | 
 50 | def distb(tb=None):
 51 |     """Disassemble a traceback (default: last traceback)."""
 52 |     if tb is None:
 53 |         try:
 54 |             tb = sys.last_traceback
 55 |         except AttributeError:
 56 |             raise RuntimeError, "no last traceback to disassemble"
 57 |         while tb.tb_next: tb = tb.tb_next
 58 |     disassemble(tb.tb_frame.f_code, tb.tb_lasti)
 59 | 
 60 | def disassemble(co, lasti=-1):
 61 |     """Disassemble a code object."""
 62 |     code = co.co_code
 63 |     labels = findlabels(code)
 64 |     linestarts = dict(findlinestarts(co))
 65 |     n = len(code)
 66 |     i = 0
 67 |     extended_arg = 0
 68 |     free = None
 69 |     while i < n:
 70 |         c = code[i]
 71 |         op = ord(c)
 72 |         if i in linestarts:
 73 |             if i > 0:
 74 |                 print
 75 |             print "%3d" % linestarts[i],
 76 |         else:
 77 |             print '   ',
 78 | 
 79 |         if i == lasti: print '-->',
 80 |         else: print '   ',
 81 |         if i in labels: print '>>',
 82 |         else: print '  ',
 83 |         print repr(i).rjust(4),
 84 |         print opname[op].ljust(20),
 85 |         i = i+1
 86 |         if op >= HAVE_ARGUMENT:
 87 |             oparg = ord(code[i]) + ord(code[i+1])*256 + extended_arg
 88 |             extended_arg = 0
 89 |             i = i+2
 90 |             if op == EXTENDED_ARG:
 91 |                 extended_arg = oparg*65536L
 92 |             print repr(oparg).rjust(5),
 93 |             if op in hasconst:
 94 |                 print '(' + repr(co.co_consts[oparg]) + ')',
 95 |             elif op in hasname:
 96 |                 print '(' + co.co_names[oparg] + ')',
 97 |             elif op in hasjrel:
 98 |                 print '(to ' + repr(i + oparg) + ')',
 99 |             elif op in haslocal:
100 |                 print '(' + co.co_varnames[oparg] + ')',
101 |             elif op in hascompare:
102 |                 print '(' + cmp_op[oparg] + ')',
103 |             elif op in hasfree:
104 |                 if free is None:
105 |                     free = co.co_cellvars + co.co_freevars
106 |                 print '(' + free[oparg] + ')',
107 |         print
108 | 
109 | def disassemble_string(code, lasti=-1, varnames=None, names=None,
110 |                        constants=None):
111 |     labels = findlabels(code)
112 |     n = len(code)
113 |     i = 0
114 |     while i < n:
115 |         c = code[i]
116 |         op = ord(c)
117 |         if i == lasti: print '-->',
118 |         else: print '   ',
119 |         if i in labels: print '>>',
120 |         else: print '  ',
121 |         print repr(i).rjust(4),
122 |         print opname[op].ljust(15),
123 |         i = i+1
124 |         if op >= HAVE_ARGUMENT:
125 |             oparg = ord(code[i]) + ord(code[i+1])*256
126 |             i = i+2
127 |             print repr(oparg).rjust(5),
128 |             if op in hasconst:
129 |                 if constants:
130 |                     print '(' + repr(constants[oparg]) + ')',
131 |                 else:
132 |                     print '(%d)'%oparg,
133 |             elif op in hasname:
134 |                 if names is not None:
135 |                     print '(' + names[oparg] + ')',
136 |                 else:
137 |                     print '(%d)'%oparg,
138 |             elif op in hasjrel:
139 |                 print '(to ' + repr(i + oparg) + ')',
140 |             elif op in haslocal:
141 |                 if varnames:
142 |                     print '(' + varnames[oparg] + ')',
143 |                 else:
144 |                     print '(%d)' % oparg,
145 |             elif op in hascompare:
146 |                 print '(' + cmp_op[oparg] + ')',
147 |         print
148 | 
149 | disco = disassemble                     # XXX For backwards compatibility
150 | 
151 | def findlabels(code):
152 |     """Detect all offsets in a byte code which are jump targets.
153 | 
154 |     Return the list of offsets.
155 | 
156 |     """
157 |     labels = []
158 |     n = len(code)
159 |     i = 0
160 |     while i < n:
161 |         c = code[i]
162 |         op = ord(c)
163 |         i = i+1
164 |         if op >= HAVE_ARGUMENT:
165 |             oparg = ord(code[i]) + ord(code[i+1])*256
166 |             i = i+2
167 |             label = -1
168 |             if op in hasjrel:
169 |                 label = i+oparg
170 |             elif op in hasjabs:
171 |                 label = oparg
172 |             if label >= 0:
173 |                 if label not in labels:
174 |                     labels.append(label)
175 |     return labels
176 | 
177 | def findlinestarts(code):
178 |     """Find the offsets in a byte code which are start of lines in the source.
179 | 
180 |     Generate pairs (offset, lineno) as described in Python/compile.c.
181 | 
182 |     """
183 |     byte_increments = [ord(c) for c in code.co_lnotab[0::2]]
184 |     line_increments = [ord(c) for c in code.co_lnotab[1::2]]
185 | 
186 |     lastlineno = None
187 |     lineno = code.co_firstlineno
188 |     addr = 0
189 |     for byte_incr, line_incr in zip(byte_increments, line_increments):
190 |         if byte_incr:
191 |             if lineno != lastlineno:
192 |                 yield (addr, lineno)
193 |                 lastlineno = lineno
194 |             addr += byte_incr
195 |         lineno += line_incr
196 |     if lineno != lastlineno:
197 |         yield (addr, lineno)
198 | 
199 | def _test():
200 |     """Simple test program to disassemble a file."""
201 |     if sys.argv[1:]:
202 |         if sys.argv[2:]:
203 |             sys.stderr.write("usage: python dis.py [-|file]\n")
204 |             sys.exit(2)
205 |         fn = sys.argv[1]
206 |         if not fn or fn == "-":
207 |             fn = None
208 |     else:
209 |         fn = None
210 |     if fn is None:
211 |         f = sys.stdin
212 |     else:
213 |         f = open(fn)
214 |     source = f.read()
215 |     if fn is not None:
216 |         f.close()
217 |     else:
218 |         fn = "<stdin>"
219 |     code = compile(source, fn, "exec")
220 |     dis(code)
221 | 
222 | if __name__ == "__main__":
223 |     _test()
224 | 


--------------------------------------------------------------------------------
/_Support/inspect.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: iso-8859-1 -*-
  2 | """Get useful information from live Python objects.
  3 | 
  4 | This module encapsulates the interface provided by the internal special
  5 | attributes (func_*, co_*, im_*, tb_*, etc.) in a friendlier fashion.
  6 | It also provides some help for examining source code and class layout.
  7 | 
  8 | Here are some of the useful functions provided by this module:
  9 | 
 10 |     ismodule(), isclass(), ismethod(), isfunction(), istraceback(),
 11 |         isframe(), iscode(), isbuiltin(), isroutine() - check object types
 12 |     getmembers() - get members of an object that satisfy a given condition
 13 | 
 14 |     getfile(), getsourcefile(), getsource() - find an object's source code
 15 |     getdoc(), getcomments() - get documentation on an object
 16 |     getmodule() - determine the module that an object came from
 17 |     getclasstree() - arrange classes so as to represent their hierarchy
 18 | 
 19 |     getargspec(), getargvalues() - get info about function arguments
 20 |     formatargspec(), formatargvalues() - format an argument spec
 21 |     getouterframes(), getinnerframes() - get info about frames
 22 |     currentframe() - get the current stack frame
 23 |     stack(), trace() - get info about frames on the stack or in a traceback
 24 | """
 25 | 
 26 | # This module is in the public domain.  No warranties.
 27 | 
 28 | __author__ = 'Ka-Ping Yee <ping@lfw.org>'
 29 | __date__ = '1 Jan 2001'
 30 | 
 31 | import sys, os, types, string, re, dis, imp, tokenize, linecache
 32 | from operator import attrgetter
 33 | 
 34 | # ----------------------------------------------------------- type-checking
 35 | def ismodule(object):
 36 |     """Return true if the object is a module.
 37 | 
 38 |     Module objects provide these attributes:
 39 |         __doc__         documentation string
 40 |         __file__        filename (missing for built-in modules)"""
 41 |     return isinstance(object, types.ModuleType)
 42 | 
 43 | def isclass(object):
 44 |     """Return true if the object is a class.
 45 | 
 46 |     Class objects provide these attributes:
 47 |         __doc__         documentation string
 48 |         __module__      name of module in which this class was defined"""
 49 |     return isinstance(object, types.ClassType) or hasattr(object, '__bases__')
 50 | 
 51 | def ismethod(object):
 52 |     """Return true if the object is an instance method.
 53 | 
 54 |     Instance method objects provide these attributes:
 55 |         __doc__         documentation string
 56 |         __name__        name with which this method was defined
 57 |         im_class        class object in which this method belongs
 58 |         im_func         function object containing implementation of method
 59 |         im_self         instance to which this method is bound, or None"""
 60 |     return isinstance(object, types.MethodType)
 61 | 
 62 | def ismethoddescriptor(object):
 63 |     """Return true if the object is a method descriptor.
 64 | 
 65 |     But not if ismethod() or isclass() or isfunction() are true.
 66 | 
 67 |     This is new in Python 2.2, and, for example, is true of int.__add__.
 68 |     An object passing this test has a __get__ attribute but not a __set__
 69 |     attribute, but beyond that the set of attributes varies.  __name__ is
 70 |     usually sensible, and __doc__ often is.
 71 | 
 72 |     Methods implemented via descriptors that also pass one of the other
 73 |     tests return false from the ismethoddescriptor() test, simply because
 74 |     the other tests promise more -- you can, e.g., count on having the
 75 |     im_func attribute (etc) when an object passes ismethod()."""
 76 |     return (hasattr(object, "__get__")
 77 |             and not hasattr(object, "__set__") # else it's a data descriptor
 78 |             and not ismethod(object)           # mutual exclusion
 79 |             and not isfunction(object)
 80 |             and not isclass(object))
 81 | 
 82 | def isdatadescriptor(object):
 83 |     """Return true if the object is a data descriptor.
 84 | 
 85 |     Data descriptors have both a __get__ and a __set__ attribute.  Examples are
 86 |     properties (defined in Python) and getsets and members (defined in C).
 87 |     Typically, data descriptors will also have __name__ and __doc__ attributes
 88 |     (properties, getsets, and members have both of these attributes), but this
 89 |     is not guaranteed."""
 90 |     return (hasattr(object, "__set__") and hasattr(object, "__get__"))
 91 | 
 92 | if hasattr(types, 'MemberDescriptorType'):
 93 |     # CPython and equivalent
 94 |     def ismemberdescriptor(object):
 95 |         """Return true if the object is a member descriptor.
 96 | 
 97 |         Member descriptors are specialized descriptors defined in extension
 98 |         modules."""
 99 |         return isinstance(object, types.MemberDescriptorType)
100 | else:
101 |     # Other implementations
102 |     def ismemberdescriptor(object):
103 |         """Return true if the object is a member descriptor.
104 | 
105 |         Member descriptors are specialized descriptors defined in extension
106 |         modules."""
107 |         return False
108 | 
109 | if hasattr(types, 'GetSetDescriptorType'):
110 |     # CPython and equivalent
111 |     def isgetsetdescriptor(object):
112 |         """Return true if the object is a getset descriptor.
113 | 
114 |         getset descriptors are specialized descriptors defined in extension
115 |         modules."""
116 |         return isinstance(object, types.GetSetDescriptorType)
117 | else:
118 |     # Other implementations
119 |     def isgetsetdescriptor(object):
120 |         """Return true if the object is a getset descriptor.
121 | 
122 |         getset descriptors are specialized descriptors defined in extension
123 |         modules."""
124 |         return False
125 | 
126 | def isfunction(object):
127 |     """Return true if the object is a user-defined function.
128 | 
129 |     Function objects provide these attributes:
130 |         __doc__         documentation string
131 |         __name__        name with which this function was defined
132 |         func_code       code object containing compiled function bytecode
133 |         func_defaults   tuple of any default values for arguments
134 |         func_doc        (same as __doc__)
135 |         func_globals    global namespace in which this function was defined
136 |         func_name       (same as __name__)"""
137 |     return isinstance(object, types.FunctionType)
138 | 
139 | def istraceback(object):
140 |     """Return true if the object is a traceback.
141 | 
142 |     Traceback objects provide these attributes:
143 |         tb_frame        frame object at this level
144 |         tb_lasti        index of last attempted instruction in bytecode
145 |         tb_lineno       current line number in Python source code
146 |         tb_next         next inner traceback object (called by this level)"""
147 |     return isinstance(object, types.TracebackType)
148 | 
149 | def isframe(object):
150 |     """Return true if the object is a frame object.
151 | 
152 |     Frame objects provide these attributes:
153 |         f_back          next outer frame object (this frame's caller)
154 |         f_builtins      built-in namespace seen by this frame
155 |         f_code          code object being executed in this frame
156 |         f_exc_traceback traceback if raised in this frame, or None
157 |         f_exc_type      exception type if raised in this frame, or None
158 |         f_exc_value     exception value if raised in this frame, or None
159 |         f_globals       global namespace seen by this frame
160 |         f_lasti         index of last attempted instruction in bytecode
161 |         f_lineno        current line number in Python source code
162 |         f_locals        local namespace seen by this frame
163 |         f_restricted    0 or 1 if frame is in restricted execution mode
164 |         f_trace         tracing function for this frame, or None"""
165 |     return isinstance(object, types.FrameType)
166 | 
167 | def iscode(object):
168 |     """Return true if the object is a code object.
169 | 
170 |     Code objects provide these attributes:
171 |         co_argcount     number of arguments (not including * or ** args)
172 |         co_code         string of raw compiled bytecode
173 |         co_consts       tuple of constants used in the bytecode
174 |         co_filename     name of file in which this code object was created
175 |         co_firstlineno  number of first line in Python source code
176 |         co_flags        bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg
177 |         co_lnotab       encoded mapping of line numbers to bytecode indices
178 |         co_name         name with which this code object was defined
179 |         co_names        tuple of names of local variables
180 |         co_nlocals      number of local variables
181 |         co_stacksize    virtual machine stack space required
182 |         co_varnames     tuple of names of arguments and local variables"""
183 |     return isinstance(object, types.CodeType)
184 | 
185 | def isbuiltin(object):
186 |     """Return true if the object is a built-in function or method.
187 | 
188 |     Built-in functions and methods provide these attributes:
189 |         __doc__         documentation string
190 |         __name__        original name of this function or method
191 |         __self__        instance to which a method is bound, or None"""
192 |     return isinstance(object, types.BuiltinFunctionType)
193 | 
194 | def isroutine(object):
195 |     """Return true if the object is any kind of function or method."""
196 |     return (isbuiltin(object)
197 |             or isfunction(object)
198 |             or ismethod(object)
199 |             or ismethoddescriptor(object))
200 | 
201 | def getmembers(object, predicate=None):
202 |     """Return all members of an object as (name, value) pairs sorted by name.
203 |     Optionally, only return members that satisfy a given predicate."""
204 |     results = []
205 |     for key in dir(object):
206 |         value = getattr(object, key)
207 |         if not predicate or predicate(value):
208 |             results.append((key, value))
209 |     results.sort()
210 |     return results
211 | 
212 | def classify_class_attrs(cls):
213 |     """Return list of attribute-descriptor tuples.
214 | 
215 |     For each name in dir(cls), the return list contains a 4-tuple
216 |     with these elements:
217 | 
218 |         0. The name (a string).
219 | 
220 |         1. The kind of attribute this is, one of these strings:
221 |                'class method'    created via classmethod()
222 |                'static method'   created via staticmethod()
223 |                'property'        created via property()
224 |                'method'          any other flavor of method
225 |                'data'            not a method
226 | 
227 |         2. The class which defined this attribute (a class).
228 | 
229 |         3. The object as obtained directly from the defining class's
230 |            __dict__, not via getattr.  This is especially important for
231 |            data attributes:  C.data is just a data object, but
232 |            C.__dict__['data'] may be a data descriptor with additional
233 |            info, like a __doc__ string.
234 |     """
235 | 
236 |     mro = getmro(cls)
237 |     names = dir(cls)
238 |     result = []
239 |     for name in names:
240 |         # Get the object associated with the name.
241 |         # Getting an obj from the __dict__ sometimes reveals more than
242 |         # using getattr.  Static and class methods are dramatic examples.
243 |         if name in cls.__dict__:
244 |             obj = cls.__dict__[name]
245 |         else:
246 |             obj = getattr(cls, name)
247 | 
248 |         # Figure out where it was defined.
249 |         homecls = getattr(obj, "__objclass__", None)
250 |         if homecls is None:
251 |             # search the dicts.
252 |             for base in mro:
253 |                 if name in base.__dict__:
254 |                     homecls = base
255 |                     break
256 | 
257 |         # Get the object again, in order to get it from the defining
258 |         # __dict__ instead of via getattr (if possible).
259 |         if homecls is not None and name in homecls.__dict__:
260 |             obj = homecls.__dict__[name]
261 | 
262 |         # Also get the object via getattr.
263 |         obj_via_getattr = getattr(cls, name)
264 | 
265 |         # Classify the object.
266 |         if isinstance(obj, staticmethod):
267 |             kind = "static method"
268 |         elif isinstance(obj, classmethod):
269 |             kind = "class method"
270 |         elif isinstance(obj, property):
271 |             kind = "property"
272 |         elif (ismethod(obj_via_getattr) or
273 |               ismethoddescriptor(obj_via_getattr)):
274 |             kind = "method"
275 |         else:
276 |             kind = "data"
277 | 
278 |         result.append((name, kind, homecls, obj))
279 | 
280 |     return result
281 | 
282 | # ----------------------------------------------------------- class helpers
283 | def _searchbases(cls, accum):
284 |     # Simulate the "classic class" search order.
285 |     if cls in accum:
286 |         return
287 |     accum.append(cls)
288 |     for base in cls.__bases__:
289 |         _searchbases(base, accum)
290 | 
291 | def getmro(cls):
292 |     "Return tuple of base classes (including cls) in method resolution order."
293 |     if hasattr(cls, "__mro__"):
294 |         return cls.__mro__
295 |     else:
296 |         result = []
297 |         _searchbases(cls, result)
298 |         return tuple(result)
299 | 
300 | # -------------------------------------------------- source code extraction
301 | def indentsize(line):
302 |     """Return the indent size, in spaces, at the start of a line of text."""
303 |     expline = string.expandtabs(line)
304 |     return len(expline) - len(string.lstrip(expline))
305 | 
306 | def getdoc(object):
307 |     """Get the documentation string for an object.
308 | 
309 |     All tabs are expanded to spaces.  To clean up docstrings that are
310 |     indented to line up with blocks of code, any whitespace than can be
311 |     uniformly removed from the second line onwards is removed."""
312 |     try:
313 |         doc = object.__doc__
314 |     except AttributeError:
315 |         return None
316 |     if not isinstance(doc, types.StringTypes):
317 |         return None
318 |     try:
319 |         lines = string.split(string.expandtabs(doc), '\n')
320 |     except UnicodeError:
321 |         return None
322 |     else:
323 |         # Find minimum indentation of any non-blank lines after first line.
324 |         margin = sys.maxint
325 |         for line in lines[1:]:
326 |             content = len(string.lstrip(line))
327 |             if content:
328 |                 indent = len(line) - content
329 |                 margin = min(margin, indent)
330 |         # Remove indentation.
331 |         if lines:
332 |             lines[0] = lines[0].lstrip()
333 |         if margin < sys.maxint:
334 |             for i in range(1, len(lines)): lines[i] = lines[i][margin:]
335 |         # Remove any trailing or leading blank lines.
336 |         while lines and not lines[-1]:
337 |             lines.pop()
338 |         while lines and not lines[0]:
339 |             lines.pop(0)
340 |         return string.join(lines, '\n')
341 | 
342 | def getfile(object):
343 |     """Work out which source or compiled file an object was defined in."""
344 |     if ismodule(object):
345 |         if hasattr(object, '__file__'):
346 |             return object.__file__
347 |         raise TypeError('arg is a built-in module')
348 |     if isclass(object):
349 |         object = sys.modules.get(object.__module__)
350 |         if hasattr(object, '__file__'):
351 |             return object.__file__
352 |         raise TypeError('arg is a built-in class')
353 |     if ismethod(object):
354 |         object = object.im_func
355 |     if isfunction(object):
356 |         object = object.func_code
357 |     if istraceback(object):
358 |         object = object.tb_frame
359 |     if isframe(object):
360 |         object = object.f_code
361 |     if iscode(object):
362 |         return object.co_filename
363 |     raise TypeError('arg is not a module, class, method, '
364 |                     'function, traceback, frame, or code object')
365 | 
366 | def getmoduleinfo(path):
367 |     """Get the module name, suffix, mode, and module type for a given file."""
368 |     filename = os.path.basename(path)
369 |     suffixes = map(lambda (suffix, mode, mtype):
370 |                    (-len(suffix), suffix, mode, mtype), imp.get_suffixes())
371 |     suffixes.sort() # try longest suffixes first, in case they overlap
372 |     for neglen, suffix, mode, mtype in suffixes:
373 |         if filename[neglen:] == suffix:
374 |             return filename[:neglen], suffix, mode, mtype
375 | 
376 | def getmodulename(path):
377 |     """Return the module name for a given file, or None."""
378 |     info = getmoduleinfo(path)
379 |     if info: return info[0]
380 | 
381 | def getsourcefile(object):
382 |     """Return the Python source file an object was defined in, if it exists."""
383 |     filename = getfile(object)
384 |     if string.lower(filename[-4:]) in ('.pyc', '.pyo'):
385 |         filename = filename[:-4] + '.py'
386 |     for suffix, mode, kind in imp.get_suffixes():
387 |         if 'b' in mode and string.lower(filename[-len(suffix):]) == suffix:
388 |             # Looks like a binary file.  We want to only return a text file.
389 |             return None
390 |     if os.path.exists(filename):
391 |         return filename
392 |     # only return a non-existent filename if the module has a PEP 302 loader
393 |     if hasattr(getmodule(object, filename), '__loader__'):
394 |         return filename
395 | 
396 | def getabsfile(object, _filename=None):
397 |     """Return an absolute path to the source or compiled file for an object.
398 | 
399 |     The idea is for each object to have a unique origin, so this routine
400 |     normalizes the result as much as possible."""
401 |     if _filename is None:
402 |         _filename = getsourcefile(object) or getfile(object)
403 |     return os.path.normcase(os.path.abspath(_filename))
404 | 
405 | modulesbyfile = {}
406 | _filesbymodname = {}
407 | 
408 | def getmodule(object, _filename=None):
409 |     """Return the module an object was defined in, or None if not found."""
410 |     if ismodule(object):
411 |         return object
412 |     if hasattr(object, '__module__'):
413 |         return sys.modules.get(object.__module__)
414 |     # Try the filename to modulename cache
415 |     if _filename is not None and _filename in modulesbyfile:
416 |         return sys.modules.get(modulesbyfile[_filename])
417 |     # Try the cache again with the absolute file name
418 |     try:
419 |         file = getabsfile(object, _filename)
420 |     except TypeError:
421 |         return None
422 |     if file in modulesbyfile:
423 |         return sys.modules.get(modulesbyfile[file])
424 |     # Update the filename to module name cache and check yet again
425 |     # Copy sys.modules in order to cope with changes while iterating
426 |     for modname, module in sys.modules.items():
427 |         if ismodule(module) and hasattr(module, '__file__'):
428 |             f = module.__file__
429 |             if f == _filesbymodname.get(modname, None):
430 |                 # Have already mapped this module, so skip it
431 |                 continue
432 |             _filesbymodname[modname] = f
433 |             f = getabsfile(module)
434 |             # Always map to the name the module knows itself by
435 |             modulesbyfile[f] = modulesbyfile[
436 |                 os.path.realpath(f)] = module.__name__
437 |     if file in modulesbyfile:
438 |         return sys.modules.get(modulesbyfile[file])
439 |     # Check the main module
440 |     main = sys.modules['__main__']
441 |     if not hasattr(object, '__name__'):
442 |         return None
443 |     if hasattr(main, object.__name__):
444 |         mainobject = getattr(main, object.__name__)
445 |         if mainobject is object:
446 |             return main
447 |     # Check builtins
448 |     builtin = sys.modules['__builtin__']
449 |     if hasattr(builtin, object.__name__):
450 |         builtinobject = getattr(builtin, object.__name__)
451 |         if builtinobject is object:
452 |             return builtin
453 | 
454 | def findsource(object):
455 |     """Return the entire source file and starting line number for an object.
456 | 
457 |     The argument may be a module, class, method, function, traceback, frame,
458 |     or code object.  The source code is returned as a list of all the lines
459 |     in the file and the line number indexes a line in that list.  An IOError
460 |     is raised if the source code cannot be retrieved."""
461 |     file = getsourcefile(object) or getfile(object)
462 |     module = getmodule(object, file)
463 |     if module:
464 |         lines = linecache.getlines(file, module.__dict__)
465 |     else:
466 |         lines = linecache.getlines(file)
467 |     if not lines:
468 |         raise IOError('could not get source code')
469 | 
470 |     if ismodule(object):
471 |         return lines, 0
472 | 
473 |     if isclass(object):
474 |         name = object.__name__
475 |         pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
476 |         # make some effort to find the best matching class definition:
477 |         # use the one with the least indentation, which is the one
478 |         # that's most probably not inside a function definition.
479 |         candidates = []
480 |         for i in range(len(lines)):
481 |             match = pat.match(lines[i])
482 |             if match:
483 |                 # if it's at toplevel, it's already the best one
484 |                 if lines[i][0] == 'c':
485 |                     return lines, i
486 |                 # else add whitespace to candidate list
487 |                 candidates.append((match.group(1), i))
488 |         if candidates:
489 |             # this will sort by whitespace, and by line number,
490 |             # less whitespace first
491 |             candidates.sort()
492 |             return lines, candidates[0][1]
493 |         else:
494 |             raise IOError('could not find class definition')
495 | 
496 |     if ismethod(object):
497 |         object = object.im_func
498 |     if isfunction(object):
499 |         object = object.func_code
500 |     if istraceback(object):
501 |         object = object.tb_frame
502 |     if isframe(object):
503 |         object = object.f_code
504 |     if iscode(object):
505 |         if not hasattr(object, 'co_firstlineno'):
506 |             raise IOError('could not find function definition')
507 |         lnum = object.co_firstlineno - 1
508 |         pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
509 |         while lnum > 0:
510 |             if pat.match(lines[lnum]): break
511 |             lnum = lnum - 1
512 |         return lines, lnum
513 |     raise IOError('could not find code object')
514 | 
515 | def getcomments(object):
516 |     """Get lines of comments immediately preceding an object's source code.
517 | 
518 |     Returns None when source can't be found.
519 |     """
520 |     try:
521 |         lines, lnum = findsource(object)
522 |     except (IOError, TypeError):
523 |         return None
524 | 
525 |     if ismodule(object):
526 |         # Look for a comment block at the top of the file.
527 |         start = 0
528 |         if lines and lines[0][:2] == '#!': start = 1
529 |         while start < len(lines) and string.strip(lines[start]) in ('', '#'):
530 |             start = start + 1
531 |         if start < len(lines) and lines[start][:1] == '#':
532 |             comments = []
533 |             end = start
534 |             while end < len(lines) and lines[end][:1] == '#':
535 |                 comments.append(string.expandtabs(lines[end]))
536 |                 end = end + 1
537 |             return string.join(comments, '')
538 | 
539 |     # Look for a preceding block of comments at the same indentation.
540 |     elif lnum > 0:
541 |         indent = indentsize(lines[lnum])
542 |         end = lnum - 1
543 |         if end >= 0 and string.lstrip(lines[end])[:1] == '#' and \
544 |             indentsize(lines[end]) == indent:
545 |             comments = [string.lstrip(string.expandtabs(lines[end]))]
546 |             if end > 0:
547 |                 end = end - 1
548 |                 comment = string.lstrip(string.expandtabs(lines[end]))
549 |                 while comment[:1] == '#' and indentsize(lines[end]) == indent:
550 |                     comments[:0] = [comment]
551 |                     end = end - 1
552 |                     if end < 0: break
553 |                     comment = string.lstrip(string.expandtabs(lines[end]))
554 |             while comments and string.strip(comments[0]) == '#':
555 |                 comments[:1] = []
556 |             while comments and string.strip(comments[-1]) == '#':
557 |                 comments[-1:] = []
558 |             return string.join(comments, '')
559 | 
560 | class EndOfBlock(Exception): pass
561 | 
562 | class BlockFinder:
563 |     """Provide a tokeneater() method to detect the end of a code block."""
564 |     def __init__(self):
565 |         self.indent = 0
566 |         self.islambda = False
567 |         self.started = False
568 |         self.passline = False
569 |         self.last = 1
570 | 
571 |     def tokeneater(self, type, token, (srow, scol), (erow, ecol), line):
572 |         if not self.started:
573 |             # look for the first "def", "class" or "lambda"
574 |             if token in ("def", "class", "lambda"):
575 |                 if token == "lambda":
576 |                     self.islambda = True
577 |                 self.started = True
578 |             self.passline = True    # skip to the end of the line
579 |         elif type == tokenize.NEWLINE:
580 |             self.passline = False   # stop skipping when a NEWLINE is seen
581 |             self.last = srow
582 |             if self.islambda:       # lambdas always end at the first NEWLINE
583 |                 raise EndOfBlock
584 |         elif self.passline:
585 |             pass
586 |         elif type == tokenize.INDENT:
587 |             self.indent = self.indent + 1
588 |             self.passline = True
589 |         elif type == tokenize.DEDENT:
590 |             self.indent = self.indent - 1
591 |             # the end of matching indent/dedent pairs end a block
592 |             # (note that this only works for "def"/"class" blocks,
593 |             #  not e.g. for "if: else:" or "try: finally:" blocks)
594 |             if self.indent <= 0:
595 |                 raise EndOfBlock
596 |         elif self.indent == 0 and type not in (tokenize.COMMENT, tokenize.NL):
597 |             # any other token on the same indentation level end the previous
598 |             # block as well, except the pseudo-tokens COMMENT and NL.
599 |             raise EndOfBlock
600 | 
601 | def getblock(lines):
602 |     """Extract the block of code at the top of the given list of lines."""
603 |     blockfinder = BlockFinder()
604 |     try:
605 |         tokenize.tokenize(iter(lines).next, blockfinder.tokeneater)
606 |     except (EndOfBlock, IndentationError):
607 |         pass
608 |     return lines[:blockfinder.last]
609 | 
610 | def getsourcelines(object):
611 |     """Return a list of source lines and starting line number for an object.
612 | 
613 |     The argument may be a module, class, method, function, traceback, frame,
614 |     or code object.  The source code is returned as a list of the lines
615 |     corresponding to the object and the line number indicates where in the
616 |     original source file the first line of code was found.  An IOError is
617 |     raised if the source code cannot be retrieved."""
618 |     lines, lnum = findsource(object)
619 | 
620 |     if ismodule(object): return lines, 0
621 |     else: return getblock(lines[lnum:]), lnum + 1
622 | 
623 | def getsource(object):
624 |     """Return the text of the source code for an object.
625 | 
626 |     The argument may be a module, class, method, function, traceback, frame,
627 |     or code object.  The source code is returned as a single string.  An
628 |     IOError is raised if the source code cannot be retrieved."""
629 |     lines, lnum = getsourcelines(object)
630 |     return string.join(lines, '')
631 | 
632 | # --------------------------------------------------- class tree extraction
633 | def walktree(classes, children, parent):
634 |     """Recursive helper function for getclasstree()."""
635 |     results = []
636 |     classes.sort(key=attrgetter('__module__', '__name__'))
637 |     for c in classes:
638 |         results.append((c, c.__bases__))
639 |         if c in children:
640 |             results.append(walktree(children[c], children, c))
641 |     return results
642 | 
643 | def getclasstree(classes, unique=0):
644 |     """Arrange the given list of classes into a hierarchy of nested lists.
645 | 
646 |     Where a nested list appears, it contains classes derived from the class
647 |     whose entry immediately precedes the list.  Each entry is a 2-tuple
648 |     containing a class and a tuple of its base classes.  If the 'unique'
649 |     argument is true, exactly one entry appears in the returned structure
650 |     for each class in the given list.  Otherwise, classes using multiple
651 |     inheritance and their descendants will appear multiple times."""
652 |     children = {}
653 |     roots = []
654 |     for c in classes:
655 |         if c.__bases__:
656 |             for parent in c.__bases__:
657 |                 if not parent in children:
658 |                     children[parent] = []
659 |                 children[parent].append(c)
660 |                 if unique and parent in classes: break
661 |         elif c not in roots:
662 |             roots.append(c)
663 |     for parent in children:
664 |         if parent not in classes:
665 |             roots.append(parent)
666 |     return walktree(roots, children, None)
667 | 
668 | # ------------------------------------------------ argument list extraction
669 | # These constants are from Python's compile.h.
670 | CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS = 1, 2, 4, 8
671 | 
672 | def getargs(co):
673 |     """Get information about the arguments accepted by a code object.
674 | 
675 |     Three things are returned: (args, varargs, varkw), where 'args' is
676 |     a list of argument names (possibly containing nested lists), and
677 |     'varargs' and 'varkw' are the names of the * and ** arguments or None."""
678 | 
679 |     if not iscode(co):
680 |         raise TypeError('arg is not a code object')
681 | 
682 |     code = co.co_code
683 |     nargs = co.co_argcount
684 |     names = co.co_varnames
685 |     args = list(names[:nargs])
686 |     step = 0
687 | 
688 |     # The following acrobatics are for anonymous (tuple) arguments.
689 |     for i in range(nargs):
690 |         if args[i][:1] in ('', '.'):
691 |             stack, remain, count = [], [], []
692 |             while step < len(code):
693 |                 op = ord(code[step])
694 |                 step = step + 1
695 |                 if op >= dis.HAVE_ARGUMENT:
696 |                     opname = dis.opname[op]
697 |                     value = ord(code[step]) + ord(code[step+1])*256
698 |                     step = step + 2
699 |                     if opname in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):
700 |                         remain.append(value)
701 |                         count.append(value)
702 |                     elif opname == 'STORE_FAST':
703 |                         stack.append(names[value])
704 | 
705 |                         # Special case for sublists of length 1: def foo((bar))
706 |                         # doesn't generate the UNPACK_TUPLE bytecode, so if
707 |                         # `remain` is empty here, we have such a sublist.
708 |                         if not remain:
709 |                             stack[0] = [stack[0]]
710 |                             break
711 |                         else:
712 |                             remain[-1] = remain[-1] - 1
713 |                             while remain[-1] == 0:
714 |                                 remain.pop()
715 |                                 size = count.pop()
716 |                                 stack[-size:] = [stack[-size:]]
717 |                                 if not remain: break
718 |                                 remain[-1] = remain[-1] - 1
719 |                             if not remain: break
720 |             args[i] = stack[0]
721 | 
722 |     varargs = None
723 |     if co.co_flags & CO_VARARGS:
724 |         varargs = co.co_varnames[nargs]
725 |         nargs = nargs + 1
726 |     varkw = None
727 |     if co.co_flags & CO_VARKEYWORDS:
728 |         varkw = co.co_varnames[nargs]
729 |     return args, varargs, varkw
730 | 
731 | def getargspec(func):
732 |     """Get the names and default values of a function's arguments.
733 | 
734 |     A tuple of four things is returned: (args, varargs, varkw, defaults).
735 |     'args' is a list of the argument names (it may contain nested lists).
736 |     'varargs' and 'varkw' are the names of the * and ** arguments or None.
737 |     'defaults' is an n-tuple of the default values of the last n arguments.
738 |     """
739 | 
740 |     if ismethod(func):
741 |         func = func.im_func
742 |     if not isfunction(func):
743 |         raise TypeError('arg is not a Python function')
744 |     args, varargs, varkw = getargs(func.func_code)
745 |     return args, varargs, varkw, func.func_defaults
746 | 
747 | def getargvalues(frame):
748 |     """Get information about arguments passed into a particular frame.
749 | 
750 |     A tuple of four things is returned: (args, varargs, varkw, locals).
751 |     'args' is a list of the argument names (it may contain nested lists).
752 |     'varargs' and 'varkw' are the names of the * and ** arguments or None.
753 |     'locals' is the locals dictionary of the given frame."""
754 |     args, varargs, varkw = getargs(frame.f_code)
755 |     return args, varargs, varkw, frame.f_locals
756 | 
757 | def joinseq(seq):
758 |     if len(seq) == 1:
759 |         return '(' + seq[0] + ',)'
760 |     else:
761 |         return '(' + string.join(seq, ', ') + ')'
762 | 
763 | def strseq(object, convert, join=joinseq):
764 |     """Recursively walk a sequence, stringifying each element."""
765 |     if type(object) in (list, tuple):
766 |         return join(map(lambda o, c=convert, j=join: strseq(o, c, j), object))
767 |     else:
768 |         return convert(object)
769 | 
770 | def formatargspec(args, varargs=None, varkw=None, defaults=None,
771 |                   formatarg=str,
772 |                   formatvarargs=lambda name: '*' + name,
773 |                   formatvarkw=lambda name: '**' + name,
774 |                   formatvalue=lambda value: '=' + repr(value),
775 |                   join=joinseq):
776 |     """Format an argument spec from the 4 values returned by getargspec.
777 | 
778 |     The first four arguments are (args, varargs, varkw, defaults).  The
779 |     other four arguments are the corresponding optional formatting functions
780 |     that are called to turn names and values into strings.  The ninth
781 |     argument is an optional function to format the sequence of arguments."""
782 |     specs = []
783 |     if defaults:
784 |         firstdefault = len(args) - len(defaults)
785 |     for i in range(len(args)):
786 |         spec = strseq(args[i], formatarg, join)
787 |         if defaults and i >= firstdefault:
788 |             spec = spec + formatvalue(defaults[i - firstdefault])
789 |         specs.append(spec)
790 |     if varargs is not None:
791 |         specs.append(formatvarargs(varargs))
792 |     if varkw is not None:
793 |         specs.append(formatvarkw(varkw))
794 |     return '(' + string.join(specs, ', ') + ')'
795 | 
796 | def formatargvalues(args, varargs, varkw, locals,
797 |                     formatarg=str,
798 |                     formatvarargs=lambda name: '*' + name,
799 |                     formatvarkw=lambda name: '**' + name,
800 |                     formatvalue=lambda value: '=' + repr(value),
801 |                     join=joinseq):
802 |     """Format an argument spec from the 4 values returned by getargvalues.
803 | 
804 |     The first four arguments are (args, varargs, varkw, locals).  The
805 |     next four arguments are the corresponding optional formatting functions
806 |     that are called to turn names and values into strings.  The ninth
807 |     argument is an optional function to format the sequence of arguments."""
808 |     def convert(name, locals=locals,
809 |                 formatarg=formatarg, formatvalue=formatvalue):
810 |         return formatarg(name) + formatvalue(locals[name])
811 |     specs = []
812 |     for i in range(len(args)):
813 |         specs.append(strseq(args[i], convert, join))
814 |     if varargs:
815 |         specs.append(formatvarargs(varargs) + formatvalue(locals[varargs]))
816 |     if varkw:
817 |         specs.append(formatvarkw(varkw) + formatvalue(locals[varkw]))
818 |     return '(' + string.join(specs, ', ') + ')'
819 | 
820 | # -------------------------------------------------- stack frame extraction
821 | def getframeinfo(frame, context=1):
822 |     """Get information about a frame or traceback object.
823 | 
824 |     A tuple of five things is returned: the filename, the line number of
825 |     the current line, the function name, a list of lines of context from
826 |     the source code, and the index of the current line within that list.
827 |     The optional second argument specifies the number of lines of context
828 |     to return, which are centered around the current line."""
829 |     if istraceback(frame):
830 |         lineno = frame.tb_lineno
831 |         frame = frame.tb_frame
832 |     else:
833 |         lineno = frame.f_lineno
834 |     if not isframe(frame):
835 |         raise TypeError('arg is not a frame or traceback object')
836 | 
837 |     filename = getsourcefile(frame) or getfile(frame)
838 |     if context > 0:
839 |         start = lineno - 1 - context//2
840 |         try:
841 |             lines, lnum = findsource(frame)
842 |         except IOError:
843 |             lines = index = None
844 |         else:
845 |             start = max(start, 1)
846 |             start = max(0, min(start, len(lines) - context))
847 |             lines = lines[start:start+context]
848 |             index = lineno - 1 - start
849 |     else:
850 |         lines = index = None
851 | 
852 |     return (filename, lineno, frame.f_code.co_name, lines, index)
853 | 
854 | def getlineno(frame):
855 |     """Get the line number from a frame object, allowing for optimization."""
856 |     # FrameType.f_lineno is now a descriptor that grovels co_lnotab
857 |     return frame.f_lineno
858 | 
859 | def getouterframes(frame, context=1):
860 |     """Get a list of records for a frame and all higher (calling) frames.
861 | 
862 |     Each record contains a frame object, filename, line number, function
863 |     name, a list of lines of context, and index within the context."""
864 |     framelist = []
865 |     while frame:
866 |         framelist.append((frame,) + getframeinfo(frame, context))
867 |         frame = frame.f_back
868 |     return framelist
869 | 
870 | def getinnerframes(tb, context=1):
871 |     """Get a list of records for a traceback's frame and all lower frames.
872 | 
873 |     Each record contains a frame object, filename, line number, function
874 |     name, a list of lines of context, and index within the context."""
875 |     framelist = []
876 |     while tb:
877 |         framelist.append((tb.tb_frame,) + getframeinfo(tb, context))
878 |         tb = tb.tb_next
879 |     return framelist
880 | 
881 | currentframe = sys._getframe
882 | 
883 | def stack(context=1):
884 |     """Return a list of records for the stack above the caller's frame."""
885 |     return getouterframes(sys._getframe(1), context)
886 | 
887 | def trace(context=1):
888 |     """Return a list of records for the stack below the current exception."""
889 |     return getinnerframes(sys.exc_info()[2], context)
890 | 


--------------------------------------------------------------------------------
/_Support/linecache.py:
--------------------------------------------------------------------------------
  1 | """Cache lines from files.
  2 | 
  3 | This is intended to read lines from modules imported -- hence if a filename
  4 | is not found, it will look down the module search path for a file by
  5 | that name.
  6 | """
  7 | 
  8 | import sys
  9 | import os
 10 | 
 11 | __all__ = ["getline", "clearcache", "checkcache"]
 12 | 
 13 | def getline(filename, lineno, module_globals=None):
 14 |     lines = getlines(filename, module_globals)
 15 |     if 1 <= lineno <= len(lines):
 16 |         return lines[lineno-1]
 17 |     else:
 18 |         return ''
 19 | 
 20 | 
 21 | # The cache
 22 | 
 23 | cache = {} # The cache
 24 | 
 25 | 
 26 | def clearcache():
 27 |     """Clear the cache entirely."""
 28 | 
 29 |     global cache
 30 |     cache = {}
 31 | 
 32 | 
 33 | def getlines(filename, module_globals=None):
 34 |     """Get the lines for a file from the cache.
 35 |     Update the cache if it doesn't contain an entry for this file already."""
 36 | 
 37 |     if filename in cache:
 38 |         return cache[filename][2]
 39 |     else:
 40 |         return updatecache(filename, module_globals)
 41 | 
 42 | 
 43 | def checkcache(filename=None):
 44 |     """Discard cache entries that are out of date.
 45 |     (This is not checked upon each call!)"""
 46 | 
 47 |     if filename is None:
 48 |         filenames = cache.keys()
 49 |     else:
 50 |         if filename in cache:
 51 |             filenames = [filename]
 52 |         else:
 53 |             return
 54 | 
 55 |     for filename in filenames:
 56 |         size, mtime, lines, fullname = cache[filename]
 57 |         if mtime is None:
 58 |             continue   # no-op for files loaded via a __loader__
 59 |         try:
 60 |             stat = os.stat(fullname)
 61 |         except os.error:
 62 |             del cache[filename]
 63 |             continue
 64 |         if size != stat.st_size or mtime != stat.st_mtime:
 65 |             del cache[filename]
 66 | 
 67 | 
 68 | def updatecache(filename, module_globals=None):
 69 |     """Update a cache entry and return its list of lines.
 70 |     If something's wrong, print a message, discard the cache entry,
 71 |     and return an empty list."""
 72 | 
 73 |     if filename in cache:
 74 |         del cache[filename]
 75 |     if not filename or filename[0] + filename[-1] == '<>':
 76 |         return []
 77 | 
 78 |     fullname = filename
 79 |     try:
 80 |         stat = os.stat(fullname)
 81 |     except os.error, msg:
 82 |         basename = os.path.split(filename)[1]
 83 | 
 84 |         # Try for a __loader__, if available
 85 |         if module_globals and '__loader__' in module_globals:
 86 |             name = module_globals.get('__name__')
 87 |             loader = module_globals['__loader__']
 88 |             get_source = getattr(loader, 'get_source', None)
 89 | 
 90 |             if name and get_source:
 91 |                 if basename.startswith(name.split('.')[-1]+'.'):
 92 |                     try:
 93 |                         data = get_source(name)
 94 |                     except (ImportError, IOError):
 95 |                         pass
 96 |                     else:
 97 |                         if data is None:
 98 |                             # No luck, the PEP302 loader cannot find the source
 99 |                             # for this module.
100 |                             return []
101 |                         cache[filename] = (
102 |                             len(data), None,
103 |                             [line+'\n' for line in data.splitlines()], fullname
104 |                         )
105 |                         return cache[filename][2]
106 | 
107 |         # Try looking through the module search path.
108 | 
109 |         for dirname in sys.path:
110 |             # When using imputil, sys.path may contain things other than
111 |             # strings; ignore them when it happens.
112 |             try:
113 |                 fullname = os.path.join(dirname, basename)
114 |             except (TypeError, AttributeError):
115 |                 # Not sufficiently string-like to do anything useful with.
116 |                 pass
117 |             else:
118 |                 try:
119 |                     stat = os.stat(fullname)
120 |                     break
121 |                 except os.error:
122 |                     pass
123 |         else:
124 |             # No luck
125 | ##          print '*** Cannot stat', filename, ':', msg
126 |             return []
127 |     try:
128 |         fp = open(fullname, 'rU')
129 |         lines = fp.readlines()
130 |         fp.close()
131 |     except IOError, msg:
132 | ##      print '*** Cannot open', fullname, ':', msg
133 |         return []
134 |     size, mtime = stat.st_size, stat.st_mtime
135 |     cache[filename] = size, mtime, lines, fullname
136 |     return lines
137 | 


--------------------------------------------------------------------------------
/_Support/opcode.py:
--------------------------------------------------------------------------------
  1 | 
  2 | """
  3 | opcode module - potentially shared between dis and other modules which
  4 | operate on bytecodes (e.g. peephole optimizers).
  5 | """
  6 | 
  7 | __all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs",
  8 |            "haslocal", "hascompare", "hasfree", "opname", "opmap",
  9 |            "HAVE_ARGUMENT", "EXTENDED_ARG"]
 10 | 
 11 | cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is',
 12 |         'is not', 'exception match', 'BAD')
 13 | 
 14 | hasconst = []
 15 | hasname = []
 16 | hasjrel = []
 17 | hasjabs = []
 18 | haslocal = []
 19 | hascompare = []
 20 | hasfree = []
 21 | 
 22 | opmap = {}
 23 | opname = [''] * 256
 24 | for op in range(256): opname[op] = '<%r>' % (op,)
 25 | del op
 26 | 
 27 | def def_op(name, op):
 28 |     opname[op] = name
 29 |     opmap[name] = op
 30 | 
 31 | def name_op(name, op):
 32 |     def_op(name, op)
 33 |     hasname.append(op)
 34 | 
 35 | def jrel_op(name, op):
 36 |     def_op(name, op)
 37 |     hasjrel.append(op)
 38 | 
 39 | def jabs_op(name, op):
 40 |     def_op(name, op)
 41 |     hasjabs.append(op)
 42 | 
 43 | # Instruction opcodes for compiled code
 44 | # Blank lines correspond to available opcodes
 45 | 
 46 | def_op('STOP_CODE', 0)
 47 | def_op('POP_TOP', 1)
 48 | def_op('ROT_TWO', 2)
 49 | def_op('ROT_THREE', 3)
 50 | def_op('DUP_TOP', 4)
 51 | def_op('ROT_FOUR', 5)
 52 | 
 53 | def_op('NOP', 9)
 54 | def_op('UNARY_POSITIVE', 10)
 55 | def_op('UNARY_NEGATIVE', 11)
 56 | def_op('UNARY_NOT', 12)
 57 | def_op('UNARY_CONVERT', 13)
 58 | 
 59 | def_op('UNARY_INVERT', 15)
 60 | 
 61 | def_op('LIST_APPEND', 18)
 62 | def_op('BINARY_POWER', 19)
 63 | def_op('BINARY_MULTIPLY', 20)
 64 | def_op('BINARY_DIVIDE', 21)
 65 | def_op('BINARY_MODULO', 22)
 66 | def_op('BINARY_ADD', 23)
 67 | def_op('BINARY_SUBTRACT', 24)
 68 | def_op('BINARY_SUBSCR', 25)
 69 | def_op('BINARY_FLOOR_DIVIDE', 26)
 70 | def_op('BINARY_TRUE_DIVIDE', 27)
 71 | def_op('INPLACE_FLOOR_DIVIDE', 28)
 72 | def_op('INPLACE_TRUE_DIVIDE', 29)
 73 | def_op('SLICE+0', 30)
 74 | def_op('SLICE+1', 31)
 75 | def_op('SLICE+2', 32)
 76 | def_op('SLICE+3', 33)
 77 | 
 78 | def_op('STORE_SLICE+0', 40)
 79 | def_op('STORE_SLICE+1', 41)
 80 | def_op('STORE_SLICE+2', 42)
 81 | def_op('STORE_SLICE+3', 43)
 82 | 
 83 | def_op('DELETE_SLICE+0', 50)
 84 | def_op('DELETE_SLICE+1', 51)
 85 | def_op('DELETE_SLICE+2', 52)
 86 | def_op('DELETE_SLICE+3', 53)
 87 | 
 88 | def_op('INPLACE_ADD', 55)
 89 | def_op('INPLACE_SUBTRACT', 56)
 90 | def_op('INPLACE_MULTIPLY', 57)
 91 | def_op('INPLACE_DIVIDE', 58)
 92 | def_op('INPLACE_MODULO', 59)
 93 | def_op('STORE_SUBSCR', 60)
 94 | def_op('DELETE_SUBSCR', 61)
 95 | def_op('BINARY_LSHIFT', 62)
 96 | def_op('BINARY_RSHIFT', 63)
 97 | def_op('BINARY_AND', 64)
 98 | def_op('BINARY_XOR', 65)
 99 | def_op('BINARY_OR', 66)
100 | def_op('INPLACE_POWER', 67)
101 | def_op('GET_ITER', 68)
102 | 
103 | def_op('PRINT_EXPR', 70)
104 | def_op('PRINT_ITEM', 71)
105 | def_op('PRINT_NEWLINE', 72)
106 | def_op('PRINT_ITEM_TO', 73)
107 | def_op('PRINT_NEWLINE_TO', 74)
108 | def_op('INPLACE_LSHIFT', 75)
109 | def_op('INPLACE_RSHIFT', 76)
110 | def_op('INPLACE_AND', 77)
111 | def_op('INPLACE_XOR', 78)
112 | def_op('INPLACE_OR', 79)
113 | def_op('BREAK_LOOP', 80)
114 | def_op('WITH_CLEANUP', 81)
115 | def_op('LOAD_LOCALS', 82)
116 | def_op('RETURN_VALUE', 83)
117 | def_op('IMPORT_STAR', 84)
118 | def_op('EXEC_STMT', 85)
119 | def_op('YIELD_VALUE', 86)
120 | def_op('POP_BLOCK', 87)
121 | def_op('END_FINALLY', 88)
122 | def_op('BUILD_CLASS', 89)
123 | 
124 | HAVE_ARGUMENT = 90              # Opcodes from here have an argument:
125 | 
126 | name_op('STORE_NAME', 90)       # Index in name list
127 | name_op('DELETE_NAME', 91)      # ""
128 | def_op('UNPACK_SEQUENCE', 92)   # Number of tuple items
129 | jrel_op('FOR_ITER', 93)
130 | 
131 | name_op('STORE_ATTR', 95)       # Index in name list
132 | name_op('DELETE_ATTR', 96)      # ""
133 | name_op('STORE_GLOBAL', 97)     # ""
134 | name_op('DELETE_GLOBAL', 98)    # ""
135 | def_op('DUP_TOPX', 99)          # number of items to duplicate
136 | def_op('LOAD_CONST', 100)       # Index in const list
137 | hasconst.append(100)
138 | name_op('LOAD_NAME', 101)       # Index in name list
139 | def_op('BUILD_TUPLE', 102)      # Number of tuple items
140 | def_op('BUILD_LIST', 103)       # Number of list items
141 | def_op('BUILD_MAP', 104)        # Always zero for now
142 | name_op('LOAD_ATTR', 105)       # Index in name list
143 | def_op('COMPARE_OP', 106)       # Comparison operator
144 | hascompare.append(106)
145 | name_op('IMPORT_NAME', 107)     # Index in name list
146 | name_op('IMPORT_FROM', 108)     # Index in name list
147 | 
148 | jrel_op('JUMP_FORWARD', 110)    # Number of bytes to skip
149 | jrel_op('JUMP_IF_FALSE', 111)   # ""
150 | jrel_op('JUMP_IF_TRUE', 112)    # ""
151 | jabs_op('JUMP_ABSOLUTE', 113)   # Target byte offset from beginning of code
152 | 
153 | name_op('LOAD_GLOBAL', 116)     # Index in name list
154 | 
155 | jabs_op('CONTINUE_LOOP', 119)   # Target address
156 | jrel_op('SETUP_LOOP', 120)      # Distance to target address
157 | jrel_op('SETUP_EXCEPT', 121)    # ""
158 | jrel_op('SETUP_FINALLY', 122)   # ""
159 | 
160 | def_op('LOAD_FAST', 124)        # Local variable number
161 | haslocal.append(124)
162 | def_op('STORE_FAST', 125)       # Local variable number
163 | haslocal.append(125)
164 | def_op('DELETE_FAST', 126)      # Local variable number
165 | haslocal.append(126)
166 | 
167 | def_op('RAISE_VARARGS', 130)    # Number of raise arguments (1, 2, or 3)
168 | def_op('CALL_FUNCTION', 131)    # #args + (#kwargs << 8)
169 | def_op('MAKE_FUNCTION', 132)    # Number of args with default values
170 | def_op('BUILD_SLICE', 133)      # Number of items
171 | def_op('MAKE_CLOSURE', 134)
172 | def_op('LOAD_CLOSURE', 135)
173 | hasfree.append(135)
174 | def_op('LOAD_DEREF', 136)
175 | hasfree.append(136)
176 | def_op('STORE_DEREF', 137)
177 | hasfree.append(137)
178 | 
179 | def_op('CALL_FUNCTION_VAR', 140)     # #args + (#kwargs << 8)
180 | def_op('CALL_FUNCTION_KW', 141)      # #args + (#kwargs << 8)
181 | def_op('CALL_FUNCTION_VAR_KW', 142)  # #args + (#kwargs << 8)
182 | def_op('EXTENDED_ARG', 143)
183 | EXTENDED_ARG = 143
184 | 
185 | del def_op, name_op, jrel_op, jabs_op
186 | 


--------------------------------------------------------------------------------
/_Support/token.py:
--------------------------------------------------------------------------------
  1 | #! /usr/bin/env python
  2 | 
  3 | """Token constants (from "token.h")."""
  4 | 
  5 | #  This file is automatically generated; please don't muck it up!
  6 | #
  7 | #  To update the symbols in this file, 'cd' to the top directory of
  8 | #  the python source tree after building the interpreter and run:
  9 | #
 10 | #    python Lib/token.py
 11 | 
 12 | #--start constants--
 13 | ENDMARKER = 0
 14 | NAME = 1
 15 | NUMBER = 2
 16 | STRING = 3
 17 | NEWLINE = 4
 18 | INDENT = 5
 19 | DEDENT = 6
 20 | LPAR = 7
 21 | RPAR = 8
 22 | LSQB = 9
 23 | RSQB = 10
 24 | COLON = 11
 25 | COMMA = 12
 26 | SEMI = 13
 27 | PLUS = 14
 28 | MINUS = 15
 29 | STAR = 16
 30 | SLASH = 17
 31 | VBAR = 18
 32 | AMPER = 19
 33 | LESS = 20
 34 | GREATER = 21
 35 | EQUAL = 22
 36 | DOT = 23
 37 | PERCENT = 24
 38 | BACKQUOTE = 25
 39 | LBRACE = 26
 40 | RBRACE = 27
 41 | EQEQUAL = 28
 42 | NOTEQUAL = 29
 43 | LESSEQUAL = 30
 44 | GREATEREQUAL = 31
 45 | TILDE = 32
 46 | CIRCUMFLEX = 33
 47 | LEFTSHIFT = 34
 48 | RIGHTSHIFT = 35
 49 | DOUBLESTAR = 36
 50 | PLUSEQUAL = 37
 51 | MINEQUAL = 38
 52 | STAREQUAL = 39
 53 | SLASHEQUAL = 40
 54 | PERCENTEQUAL = 41
 55 | AMPEREQUAL = 42
 56 | VBAREQUAL = 43
 57 | CIRCUMFLEXEQUAL = 44
 58 | LEFTSHIFTEQUAL = 45
 59 | RIGHTSHIFTEQUAL = 46
 60 | DOUBLESTAREQUAL = 47
 61 | DOUBLESLASH = 48
 62 | DOUBLESLASHEQUAL = 49
 63 | AT = 50
 64 | OP = 51
 65 | ERRORTOKEN = 52
 66 | N_TOKENS = 53
 67 | NT_OFFSET = 256
 68 | #--end constants--
 69 | 
 70 | tok_name = {}
 71 | for _name, _value in globals().items():
 72 |     if type(_value) is type(0):
 73 |         tok_name[_value] = _name
 74 | 
 75 | 
 76 | def ISTERMINAL(x):
 77 |     return x < NT_OFFSET
 78 | 
 79 | def ISNONTERMINAL(x):
 80 |     return x >= NT_OFFSET
 81 | 
 82 | def ISEOF(x):
 83 |     return x == ENDMARKER
 84 | 
 85 | 
 86 | def main():
 87 |     import re
 88 |     import sys
 89 |     args = sys.argv[1:]
 90 |     inFileName = args and args[0] or "Include/token.h"
 91 |     outFileName = "Lib/token.py"
 92 |     if len(args) > 1:
 93 |         outFileName = args[1]
 94 |     try:
 95 |         fp = open(inFileName)
 96 |     except IOError, err:
 97 |         sys.stdout.write("I/O error: %s\n" % str(err))
 98 |         sys.exit(1)
 99 |     lines = fp.read().split("\n")
100 |     fp.close()
101 |     prog = re.compile(
102 |         "#define[ \t][ \t]*([A-Z0-9][A-Z0-9_]*)[ \t][ \t]*([0-9][0-9]*)",
103 |         re.IGNORECASE)
104 |     tokens = {}
105 |     for line in lines:
106 |         match = prog.match(line)
107 |         if match:
108 |             name, val = match.group(1, 2)
109 |             val = int(val)
110 |             tokens[val] = name          # reverse so we can sort them...
111 |     keys = tokens.keys()
112 |     keys.sort()
113 |     # load the output skeleton from the target:
114 |     try:
115 |         fp = open(outFileName)
116 |     except IOError, err:
117 |         sys.stderr.write("I/O error: %s\n" % str(err))
118 |         sys.exit(2)
119 |     format = fp.read().split("\n")
120 |     fp.close()
121 |     try:
122 |         start = format.index("#--start constants--") + 1
123 |         end = format.index("#--end constants--")
124 |     except ValueError:
125 |         sys.stderr.write("target does not contain format markers")
126 |         sys.exit(3)
127 |     lines = []
128 |     for val in keys:
129 |         lines.append("%s = %d" % (tokens[val], val))
130 |     format[start:end] = lines
131 |     try:
132 |         fp = open(outFileName, 'w')
133 |     except IOError, err:
134 |         sys.stderr.write("I/O error: %s\n" % str(err))
135 |         sys.exit(4)
136 |     fp.write("\n".join(format))
137 |     fp.close()
138 | 
139 | 
140 | if __name__ == "__main__":
141 |     main()
142 | 


--------------------------------------------------------------------------------
/_Support/tokenize.py:
--------------------------------------------------------------------------------
  1 | """Tokenization help for Python programs.
  2 | 
  3 | generate_tokens(readline) is a generator that breaks a stream of
  4 | text into Python tokens.  It accepts a readline-like method which is called
  5 | repeatedly to get the next line of input (or "" for EOF).  It generates
  6 | 5-tuples with these members:
  7 | 
  8 |     the token type (see token.py)
  9 |     the token (a string)
 10 |     the starting (row, column) indices of the token (a 2-tuple of ints)
 11 |     the ending (row, column) indices of the token (a 2-tuple of ints)
 12 |     the original line (string)
 13 | 
 14 | It is designed to match the working of the Python tokenizer exactly, except
 15 | that it produces COMMENT tokens for comments and gives type OP for all
 16 | operators
 17 | 
 18 | Older entry points
 19 |     tokenize_loop(readline, tokeneater)
 20 |     tokenize(readline, tokeneater=printtoken)
 21 | are the same, except instead of generating tokens, tokeneater is a callback
 22 | function to which the 5 fields described above are passed as 5 arguments,
 23 | each time a new token is found."""
 24 | 
 25 | __author__ = 'Ka-Ping Yee <ping@lfw.org>'
 26 | __credits__ = \
 27 |     'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro'
 28 | 
 29 | import string, re
 30 | from token import *
 31 | 
 32 | import token
 33 | __all__ = [x for x in dir(token) if x[0] != '_'] + ["COMMENT", "tokenize",
 34 |            "generate_tokens", "NL", "untokenize"]
 35 | del x
 36 | del token
 37 | 
 38 | COMMENT = N_TOKENS
 39 | tok_name[COMMENT] = 'COMMENT'
 40 | NL = N_TOKENS + 1
 41 | tok_name[NL] = 'NL'
 42 | N_TOKENS += 2
 43 | 
 44 | def group(*choices): return '(' + '|'.join(choices) + ')'
 45 | def any(*choices): return group(*choices) + '*'
 46 | def maybe(*choices): return group(*choices) + '?'
 47 | 
 48 | Whitespace = r'[ \f\t]*'
 49 | Comment = r'#[^\r\n]*'
 50 | Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
 51 | Name = r'[a-zA-Z_]\w*'
 52 | 
 53 | Hexnumber = r'0[xX][\da-fA-F]*[lL]?'
 54 | Octnumber = r'0[0-7]*[lL]?'
 55 | Decnumber = r'[1-9]\d*[lL]?'
 56 | Intnumber = group(Hexnumber, Octnumber, Decnumber)
 57 | Exponent = r'[eE][-+]?\d+'
 58 | Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
 59 | Expfloat = r'\d+' + Exponent
 60 | Floatnumber = group(Pointfloat, Expfloat)
 61 | Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
 62 | Number = group(Imagnumber, Floatnumber, Intnumber)
 63 | 
 64 | # Tail end of ' string.
 65 | Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
 66 | # Tail end of " string.
 67 | Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
 68 | # Tail end of ''' string.
 69 | Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
 70 | # Tail end of """ string.
 71 | Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
 72 | Triple = group("[uU]?[rR]?'''", '[uU]?[rR]?"""')
 73 | # Single-line ' or " string.
 74 | String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
 75 |                r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
 76 | 
 77 | # Because of leftmost-then-longest match semantics, be sure to put the
 78 | # longest operators first (e.g., if = came before ==, == would get
 79 | # recognized as two instances of =).
 80 | Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
 81 |                  r"//=?",
 82 |                  r"[+\-*/%&|^=<>]=?",
 83 |                  r"~")
 84 | 
 85 | Bracket = '[][(){}]'
 86 | Special = group(r'\r?\n', r'[:;.,`@]')
 87 | Funny = group(Operator, Bracket, Special)
 88 | 
 89 | PlainToken = group(Number, Funny, String, Name)
 90 | Token = Ignore + PlainToken
 91 | 
 92 | # First (or only) line of ' or " string.
 93 | ContStr = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
 94 |                 group("'", r'\\\r?\n'),
 95 |                 r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
 96 |                 group('"', r'\\\r?\n'))
 97 | PseudoExtras = group(r'\\\r?\n', Comment, Triple)
 98 | PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
 99 | 
100 | tokenprog, pseudoprog, single3prog, double3prog = map(
101 |     re.compile, (Token, PseudoToken, Single3, Double3))
102 | endprogs = {"'": re.compile(Single), '"': re.compile(Double),
103 |             "'''": single3prog, '"""': double3prog,
104 |             "r'''": single3prog, 'r"""': double3prog,
105 |             "u'''": single3prog, 'u"""': double3prog,
106 |             "ur'''": single3prog, 'ur"""': double3prog,
107 |             "R'''": single3prog, 'R"""': double3prog,
108 |             "U'''": single3prog, 'U"""': double3prog,
109 |             "uR'''": single3prog, 'uR"""': double3prog,
110 |             "Ur'''": single3prog, 'Ur"""': double3prog,
111 |             "UR'''": single3prog, 'UR"""': double3prog,
112 |             'r': None, 'R': None, 'u': None, 'U': None}
113 | 
114 | triple_quoted = {}
115 | for t in ("'''", '"""',
116 |           "r'''", 'r"""', "R'''", 'R"""',
117 |           "u'''", 'u"""', "U'''", 'U"""',
118 |           "ur'''", 'ur"""', "Ur'''", 'Ur"""',
119 |           "uR'''", 'uR"""', "UR'''", 'UR"""'):
120 |     triple_quoted[t] = t
121 | single_quoted = {}
122 | for t in ("'", '"',
123 |           "r'", 'r"', "R'", 'R"',
124 |           "u'", 'u"', "U'", 'U"',
125 |           "ur'", 'ur"', "Ur'", 'Ur"',
126 |           "uR'", 'uR"', "UR'", 'UR"' ):
127 |     single_quoted[t] = t
128 | 
129 | tabsize = 8
130 | 
131 | class TokenError(Exception): pass
132 | 
133 | class StopTokenizing(Exception): pass
134 | 
135 | def printtoken(type, token, (srow, scol), (erow, ecol), line): # for testing
136 |     print "%d,%d-%d,%d:\t%s\t%s" % \
137 |         (srow, scol, erow, ecol, tok_name[type], repr(token))
138 | 
139 | def tokenize(readline, tokeneater=printtoken):
140 |     """
141 |     The tokenize() function accepts two parameters: one representing the
142 |     input stream, and one providing an output mechanism for tokenize().
143 | 
144 |     The first parameter, readline, must be a callable object which provides
145 |     the same interface as the readline() method of built-in file objects.
146 |     Each call to the function should return one line of input as a string.
147 | 
148 |     The second parameter, tokeneater, must also be a callable object. It is
149 |     called once for each token, with five arguments, corresponding to the
150 |     tuples generated by generate_tokens().
151 |     """
152 |     try:
153 |         tokenize_loop(readline, tokeneater)
154 |     except StopTokenizing:
155 |         pass
156 | 
157 | # backwards compatible interface
158 | def tokenize_loop(readline, tokeneater):
159 |     for token_info in generate_tokens(readline):
160 |         tokeneater(*token_info)
161 | 
162 | 
163 | def untokenize(iterable):
164 |     """Transform tokens back into Python source code.
165 | 
166 |     Each element returned by the iterable must be a token sequence
167 |     with at least two elements, a token number and token value.
168 | 
169 |     Round-trip invariant:
170 |         # Output text will tokenize the back to the input
171 |         t1 = [tok[:2] for tok in generate_tokens(f.readline)]
172 |         newcode = untokenize(t1)
173 |         readline = iter(newcode.splitlines(1)).next
174 |         t2 = [tok[:2] for tokin generate_tokens(readline)]
175 |         assert t1 == t2
176 |     """
177 | 
178 |     startline = False
179 |     indents = []
180 |     toks = []
181 |     toks_append = toks.append
182 |     for tok in iterable:
183 |         toknum, tokval = tok[:2]
184 | 
185 |         if toknum in (NAME, NUMBER):
186 |             tokval += ' '
187 | 
188 |         if toknum == INDENT:
189 |             indents.append(tokval)
190 |             continue
191 |         elif toknum == DEDENT:
192 |             indents.pop()
193 |             continue
194 |         elif toknum in (NEWLINE, COMMENT, NL):
195 |             startline = True
196 |         elif startline and indents:
197 |             toks_append(indents[-1])
198 |             startline = False
199 |         toks_append(tokval)
200 |     return ''.join(toks)
201 | 
202 | 
203 | def generate_tokens(readline):
204 |     """
205 |     The generate_tokens() generator requires one argment, readline, which
206 |     must be a callable object which provides the same interface as the
207 |     readline() method of built-in file objects. Each call to the function
208 |     should return one line of input as a string.  Alternately, readline
209 |     can be a callable function terminating with StopIteration:
210 |         readline = open(myfile).next    # Example of alternate readline
211 | 
212 |     The generator produces 5-tuples with these members: the token type; the
213 |     token string; a 2-tuple (srow, scol) of ints specifying the row and
214 |     column where the token begins in the source; a 2-tuple (erow, ecol) of
215 |     ints specifying the row and column where the token ends in the source;
216 |     and the line on which the token was found. The line passed is the
217 |     logical line; continuation lines are included.
218 |     """
219 |     lnum = parenlev = continued = 0
220 |     namechars, numchars = string.ascii_letters + '_', '0123456789'
221 |     contstr, needcont = '', 0
222 |     contline = None
223 |     indents = [0]
224 | 
225 |     while 1:                                   # loop over lines in stream
226 |         try:
227 |             line = readline()
228 |         except StopIteration:
229 |             line = ''
230 |         lnum = lnum + 1
231 |         pos, max = 0, len(line)
232 | 
233 |         if contstr:                            # continued string
234 |             if not line:
235 |                 raise TokenError, ("EOF in multi-line string", strstart)
236 |             endmatch = endprog.match(line)
237 |             if endmatch:
238 |                 pos = end = endmatch.end(0)
239 |                 yield (STRING, contstr + line[:end],
240 |                            strstart, (lnum, end), contline + line)
241 |                 contstr, needcont = '', 0
242 |                 contline = None
243 |             elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
244 |                 yield (ERRORTOKEN, contstr + line,
245 |                            strstart, (lnum, len(line)), contline)
246 |                 contstr = ''
247 |                 contline = None
248 |                 continue
249 |             else:
250 |                 contstr = contstr + line
251 |                 contline = contline + line
252 |                 continue
253 | 
254 |         elif parenlev == 0 and not continued:  # new statement
255 |             if not line: break
256 |             column = 0
257 |             while pos < max:                   # measure leading whitespace
258 |                 if line[pos] == ' ': column = column + 1
259 |                 elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize
260 |                 elif line[pos] == '\f': column = 0
261 |                 else: break
262 |                 pos = pos + 1
263 |             if pos == max: break
264 | 
265 |             if line[pos] in '#\r\n':           # skip comments or blank lines
266 |                 yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
267 |                            (lnum, pos), (lnum, len(line)), line)
268 |                 continue
269 | 
270 |             if column > indents[-1]:           # count indents or dedents
271 |                 indents.append(column)
272 |                 yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
273 |             while column < indents[-1]:
274 |                 if column not in indents:
275 |                     raise IndentationError(
276 |                         "unindent does not match any outer indentation level",
277 |                         ("<tokenize>", lnum, pos, line))
278 |                 indents = indents[:-1]
279 |                 yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
280 | 
281 |         else:                                  # continued statement
282 |             if not line:
283 |                 raise TokenError, ("EOF in multi-line statement", (lnum, 0))
284 |             continued = 0
285 | 
286 |         while pos < max:
287 |             pseudomatch = pseudoprog.match(line, pos)
288 |             if pseudomatch:                                # scan for tokens
289 |                 start, end = pseudomatch.span(1)
290 |                 spos, epos, pos = (lnum, start), (lnum, end), end
291 |                 token, initial = line[start:end], line[start]
292 | 
293 |                 if initial in numchars or \
294 |                    (initial == '.' and token != '.'):      # ordinary number
295 |                     yield (NUMBER, token, spos, epos, line)
296 |                 elif initial in '\r\n':
297 |                     yield (parenlev > 0 and NL or NEWLINE,
298 |                                token, spos, epos, line)
299 |                 elif initial == '#':
300 |                     yield (COMMENT, token, spos, epos, line)
301 |                 elif token in triple_quoted:
302 |                     endprog = endprogs[token]
303 |                     endmatch = endprog.match(line, pos)
304 |                     if endmatch:                           # all on one line
305 |                         pos = endmatch.end(0)
306 |                         token = line[start:pos]
307 |                         yield (STRING, token, spos, (lnum, pos), line)
308 |                     else:
309 |                         strstart = (lnum, start)           # multiple lines
310 |                         contstr = line[start:]
311 |                         contline = line
312 |                         break
313 |                 elif initial in single_quoted or \
314 |                     token[:2] in single_quoted or \
315 |                     token[:3] in single_quoted:
316 |                     if token[-1] == '\n':                  # continued string
317 |                         strstart = (lnum, start)
318 |                         endprog = (endprogs[initial] or endprogs[token[1]] or
319 |                                    endprogs[token[2]])
320 |                         contstr, needcont = line[start:], 1
321 |                         contline = line
322 |                         break
323 |                     else:                                  # ordinary string
324 |                         yield (STRING, token, spos, epos, line)
325 |                 elif initial in namechars:                 # ordinary name
326 |                     yield (NAME, token, spos, epos, line)
327 |                 elif initial == '\\':                      # continued stmt
328 |                     continued = 1
329 |                 else:
330 |                     if initial in '([{': parenlev = parenlev + 1
331 |                     elif initial in ')]}': parenlev = parenlev - 1
332 |                     yield (OP, token, spos, epos, line)
333 |             else:
334 |                 yield (ERRORTOKEN, line[pos],
335 |                            (lnum, pos), (lnum, pos+1), line)
336 |                 pos = pos + 1
337 | 
338 |     for indent in indents[1:]:                 # pop remaining indent levels
339 |         yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
340 |     yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
341 | 
342 | if __name__ == '__main__':                     # testing
343 |     import sys
344 |     if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline)
345 |     else: tokenize(sys.stdin.readline)
346 | 


--------------------------------------------------------------------------------
/__init__.py:
--------------------------------------------------------------------------------
1 | # http://remotescripts.blogspot.com
2 | 
3 | from LiveAPI_MakeDoc import APIMakeDoc
4 | 
5 | def create_instance(c_instance):
6 |     return APIMakeDoc(c_instance)
7 | 


--------------------------------------------------------------------------------