├── .gitignore ├── Pycraft └── pycraft.py ├── README.md ├── aync_sched.py ├── config_parser.py ├── dancing_links.py ├── disjoint_set.py ├── emojify ├── emoji │ ├── application_osx_terminal.png │ ├── arrow_rotate_anticlockwise.png │ ├── book_addresses.png │ ├── cake.png │ ├── cd.png │ ├── clock_.png │ ├── controller.png │ ├── cup.png │ ├── ee_1.png │ ├── ee_2.png │ ├── ee_22.png │ ├── ee_23.png │ ├── ee_24.png │ ├── ee_25.png │ ├── ee_26.png │ ├── ee_27.png │ ├── ee_28.png │ ├── ee_29.png │ ├── ee_3.png │ ├── ee_30.png │ ├── ee_31.png │ ├── ee_32.png │ ├── ee_33.png │ ├── ee_34.png │ ├── ee_35.png │ ├── ee_4.png │ ├── ee_5.png │ ├── ee_6.png │ ├── ee_7.png │ ├── group.png │ ├── magnifier.png │ ├── page_white_edit.png │ ├── palette.png │ ├── resultset_next.png │ └── ruby.png └── emojify.py ├── fileinput.py ├── go_repl.py ├── html_template.py ├── http_server ├── README.md ├── config.json └── web_server.py ├── image_crawler ├── README.md ├── config.ini ├── crawler.py └── requirements.txt ├── lisp.py ├── memento.py ├── mock_server ├── data.db ├── manage.py └── mock_server │ ├── __init__.py │ ├── models.py │ ├── settings.py │ ├── templates │ └── index.html │ ├── urls.py │ ├── views.py │ └── wsgi.py ├── online_judge ├── README.md ├── html │ └── page.html ├── judge.py ├── problem.py ├── problemset │ ├── __init__.py │ ├── problem_fib.py │ ├── problem_hello.py │ ├── problem_panlindrome.py │ ├── problem_reverselist.py │ └── problem_sort.py ├── utils.py └── web.py ├── open_todo.py ├── patch_module.py ├── quine.py ├── rpc.py ├── timeit.py └── web_terminal ├── index.html └── web_terminal.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc -------------------------------------------------------------------------------- /Pycraft/pycraft.py: -------------------------------------------------------------------------------- 1 | # My clone of Minecraft using python and Panda3D 2 | from panda3d.core import GeomVertexFormat, GeomVertexData 3 | from panda3d.core import Geom, GeomTriangles, GeomVertexWriter 4 | from panda3d.core import Texture, GeomNode 5 | from panda3d.core import LVector3, Vec3 6 | from direct.showbase.ShowBase import ShowBase 7 | import random, math 8 | 9 | 10 | # helper functions to draw a primitive cube 11 | def normalized(*args): 12 | myVec = LVector3(*args) 13 | myVec.normalize() 14 | return myVec 15 | 16 | def makeSquare(x1, y1, z1, x2, y2, z2): 17 | format = GeomVertexFormat.getV3n3cpt2() 18 | vdata = GeomVertexData('square', format, Geom.UHDynamic) 19 | 20 | vertex = GeomVertexWriter(vdata, 'vertex') 21 | normal = GeomVertexWriter(vdata, 'normal') 22 | # color = GeomVertexWriter(vdata, 'color') 23 | texcoord = GeomVertexWriter(vdata, 'texcoord') 24 | 25 | # make sure we draw the sqaure in the right plane 26 | if x1 != x2: 27 | vertex.addData3(x1, y1, z1) 28 | vertex.addData3(x2, y1, z1) 29 | vertex.addData3(x2, y2, z2) 30 | vertex.addData3(x1, y2, z2) 31 | 32 | normal.addData3(normalized(2 * x1 - 1, 2 * y1 - 1, 2 * z1 - 1)) 33 | normal.addData3(normalized(2 * x2 - 1, 2 * y1 - 1, 2 * z1 - 1)) 34 | normal.addData3(normalized(2 * x2 - 1, 2 * y2 - 1, 2 * z2 - 1)) 35 | normal.addData3(normalized(2 * x1 - 1, 2 * y2 - 1, 2 * z2 - 1)) 36 | 37 | else: 38 | vertex.addData3(x1, y1, z1) 39 | vertex.addData3(x2, y2, z1) 40 | vertex.addData3(x2, y2, z2) 41 | vertex.addData3(x1, y1, z2) 42 | 43 | normal.addData3(normalized(2 * x1 - 1, 2 * y1 - 1, 2 * z1 - 1)) 44 | normal.addData3(normalized(2 * x2 - 1, 2 * y2 - 1, 2 * z1 - 1)) 45 | normal.addData3(normalized(2 * x2 - 1, 2 * y2 - 1, 2 * z2 - 1)) 46 | normal.addData3(normalized(2 * x1 - 1, 2 * y1 - 1, 2 * z2 - 1)) 47 | 48 | # adding different colors to the vertex for visibility 49 | # color.addData4f(1.0, 0.0, 0.0, 1.0) 50 | # color.addData4f(0.0, 1.0, 0.0, 1.0) 51 | # color.addData4f(0.0, 0.0, 1.0, 1.0) 52 | # color.addData4f(1.0, 0.0, 1.0, 1.0) 53 | 54 | texcoord.addData2f(0.0, 1.0) 55 | texcoord.addData2f(0.0, 0.0) 56 | texcoord.addData2f(1.0, 0.0) 57 | texcoord.addData2f(1.0, 1.0) 58 | 59 | # Quads aren't directly supported by the Geom interface 60 | # you might be interested in the CardMaker class if you are 61 | # interested in rectangle though 62 | tris = GeomTriangles(Geom.UHDynamic) 63 | tris.addVertices(0, 1, 3) 64 | tris.addVertices(1, 2, 3) 65 | 66 | square = Geom(vdata) 67 | square.addPrimitive(tris) 68 | return square 69 | 70 | def makeCube(): 71 | square0 = makeSquare(-1, -1, -1, 1, -1, 1) 72 | square1 = makeSquare(-1, 1, -1, 1, 1, 1) 73 | square2 = makeSquare(-1, 1, 1, 1, -1, 1) 74 | square3 = makeSquare(-1, 1, -1, 1, -1, -1) 75 | square4 = makeSquare(-1, -1, -1, -1, 1, 1) 76 | square5 = makeSquare(1, -1, -1, 1, 1, 1) 77 | snode = GeomNode('square') 78 | snode.addGeom(square0) 79 | snode.addGeom(square1) 80 | snode.addGeom(square2) 81 | snode.addGeom(square3) 82 | snode.addGeom(square4) 83 | snode.addGeom(square5) 84 | return snode 85 | 86 | WORLD_SIZE = 128 87 | SECTOR_SIZE = 8 88 | SECTOR_RADIUS = 3 89 | NEIGHBOURS = [ 90 | ( 0, 1, 0), 91 | ( 0,-1, 0), 92 | (-1, 0, 0), 93 | ( 1, 0, 0), 94 | ( 0, 0, 1), 95 | ( 0, 0,-1), 96 | ] 97 | MOVE_SPEED = 0.5 98 | ROTATE_SPEED = 0.15 99 | GRAVITY = 0.2 100 | 101 | def position_to_sector(pos): 102 | x, y, z = pos 103 | xx, yy = int(round(x))/SECTOR_SIZE, int(round(y))/SECTOR_SIZE 104 | return xx, yy 105 | 106 | class Game(ShowBase): 107 | textures = ["STONE", "BRICK", "SAND", "GRASS"] 108 | def __init__(self): 109 | ShowBase.__init__(self) 110 | self.world = {} 111 | self.block_nodes = {} 112 | self.sectors = set() 113 | self.sector_blocks = {} 114 | self.position = (0,0,1) 115 | self.lookat = [0, 0] # horizontal and vertial camera direction 116 | 117 | self.flying = False 118 | self.vspeed = 0 119 | self.move_direction= [0,0] 120 | # mouse positions 121 | self.mx = None 122 | self.my = None 123 | 124 | self.generate_initial_world() 125 | self.refresh_sectors() 126 | self.update_camera() 127 | self.register_handlers() 128 | self.add_tasks() 129 | 130 | def register_handlers(self): 131 | self.accept("w", self.move_forward) 132 | self.accept("s", self.move_backward) 133 | self.accept("a", self.move_left) 134 | self.accept("d", self.move_right) 135 | self.accept("w-up", self.stop_move_forward) 136 | self.accept("s-up", self.stop_move_backward) 137 | self.accept("a-up", self.stop_move_left) 138 | self.accept("d-up", self.stop_move_right) 139 | self.accept("mouse1", self.erase_block) 140 | self.accept("mouse2", self.create_block) 141 | self.accept("tab", self.toggle_flying) 142 | self.accept(" ", self.jump) 143 | 144 | def move_forward(self): 145 | self.move_direction[0]+=1 146 | 147 | def move_backward(self): 148 | self.move_direction[0]-=1 149 | 150 | def move_left(self): 151 | self.move_direction[1]-=1 152 | 153 | def move_right(self): 154 | self.move_direction[1]+=1 155 | 156 | def stop_move_forward(self): 157 | self.move_direction[0]-=1 158 | 159 | def stop_move_backward(self): 160 | self.move_direction[0]+=1 161 | 162 | def stop_move_left(self): 163 | self.move_direction[1]+=1 164 | 165 | def stop_move_right(self): 166 | self.move_direction[1]-=1 167 | 168 | def erase_block(self): 169 | # find the block looked at 170 | # remove the block from world and sector_blocks 171 | # remove the node for the block 172 | pass 173 | 174 | def create_block(self): 175 | pass 176 | 177 | def toggle_flying(self): 178 | self.flying = not self.flying 179 | 180 | def jump(self): 181 | if self.vspeed == 0: 182 | self.vspeed = self.MOVE_SPEED 183 | 184 | def add_tasks(self): 185 | self.taskMgr.add(self.move_task, "move_task") 186 | self.taskMgr.add(self.mouse_task, "mouse_task") 187 | 188 | def detect_collision(self, pos): 189 | return pos 190 | 191 | def move_task(self, task): 192 | # first check if moving in any direction 193 | if any(self.move_direction): 194 | radian = math.atan2(self.move_direction[1], self.move_direction[0]) 195 | degree = math.degrees(radian) 196 | move_z = math.radians(self.lookat[1]) 197 | move_x = math.radians(self.lookat[0]+degree) 198 | 199 | if self.flying: 200 | if self.move_direction[1]: 201 | vspeed = 0.0 202 | hspeed = MOVE_SPEED 203 | else: 204 | vspeed = MOVE_SPEED * math.sin(move_z) 205 | hspeed = MOVE_SPEED * math.cos(move_z) 206 | if self.move_direction[0] <0: 207 | hspeed = -hspeed 208 | else: 209 | hspeed = MOVE_SPEED 210 | vspeed = 0 211 | x_speed = hspeed * math.cos(move_x) 212 | y_speed = hspeed * math.sin(move_x) 213 | newpos = self.position[0]+x_speed, self.position[1]+y_speed, self.position[2]+hspeed 214 | self.position = self.detect_collision(newpos) 215 | # TODO: if cross sector, render sectors 216 | # second check if dropping when not flying 217 | if not self.flying: 218 | newpos = self.position[0], self.position[1], self.position[2]+self.vspeed 219 | self.position = self.detect_collision(newpos) 220 | self.vspeed -= GRAVITY 221 | 222 | def mouse_task(self, task): 223 | mw = self.mouseWatcherNode 224 | if mw.hasMouse(): 225 | mpos = mw.getMouse() # get the mouse position 226 | if self.mx is None or self.my is None: 227 | self.mx = mpos.getX() 228 | self.my = mpos.getY() 229 | else: 230 | dx = mpos.getX() - self.mx 231 | dy = mpos.getY() - self.my 232 | self.lookat[0] += ROTATE_SPEED*dx 233 | self.lookat[1] += ROTATE_SPEED*dy 234 | if self.lookat[1] > 90.0: 235 | self.lookat[1] = 90 236 | elif self.lookat[1]<-90.0: 237 | self.lookat[1] = -90 238 | self.update_camera() 239 | 240 | def add_block(self, pos, tex_idx): 241 | self.world[pos] = tex_idx 242 | sector = position_to_sector(pos) 243 | self.sector_blocks.setdefault(sector, []).append(pos) 244 | 245 | def generate_initial_world(self): 246 | for x in range(-WORLD_SIZE, WORLD_SIZE+1): 247 | for y in range(-WORLD_SIZE, WORLD_SIZE+1): 248 | self.add_block((x, y, -2), 0) # a layer of stones 249 | self.add_block((x, y, -1), 3) # a layer of grass 250 | 251 | for _ in range(20): 252 | # generate random hills 253 | hillx = random.randint(-WORLD_SIZE, WORLD_SIZE) 254 | hilly = random.randint(-WORLD_SIZE, WORLD_SIZE) 255 | size = random.randint(2, 6) 256 | height = random.randint(3, 8) 257 | texture = random.randint(0,4) 258 | for i in range(height): 259 | for m in range(hillx-size, hillx+size): 260 | for n in range(hilly-size, hilly+size): 261 | self.add_block((m,n,i), texture) 262 | size -= 1 263 | if size <= 0: break 264 | 265 | def update_camera(self): 266 | x, y, z = self.position 267 | self.camera.setPos(x, y, z) 268 | self.camera.setHpr(Vec3(self.lookat[0], self.lookat[1], 0)) 269 | 270 | def show_block(self, x, y, z, texture): 271 | cube = makeCube() 272 | node = self.render.attachNewNode(cube) 273 | node.setPos(x, y, z) 274 | node.setTexture(texture) 275 | return node 276 | 277 | def hide_block(self, pos): 278 | if pos in self.block_nodes: 279 | self.block_nodes[pos].removeNode() 280 | del self.block_nodes[pos] 281 | 282 | def exposed(self, pos): 283 | x, y, z = pos 284 | for dx, dy, dz in NEIGHBOURS: 285 | if (x+dx, y+dy, z+dz) not in self.world: 286 | return True 287 | return False 288 | 289 | def refresh_sectors(self): 290 | sx, sy = position_to_sector(self.position) 291 | new_sectors = set() 292 | for i in range(sx-SECTOR_RADIUS, sx+SECTOR_RADIUS+1): 293 | for j in range(sy-SECTOR_RADIUS, sy+SECTOR_RADIUS+1): 294 | if (i-sx)**2 + (j-sy)**2 > SECTOR_RADIUS**2: continue 295 | new_sectors.add((i, j)) 296 | added = new_sectors - self.sectors 297 | deleted = self.sectors - new_sectors 298 | for sector in added: 299 | blocks = self.sector_blocks[sector] 300 | for block in blocks: 301 | if not self.exposed(block): continue 302 | x, y, z = block 303 | tex_idx = self.world[block] 304 | texture = self.get_texture(tex_idx) 305 | node = self.show_block(x, y, z, texture) 306 | self.block_nodes[block] = node 307 | for sector in deleted: 308 | blocks = self.sector_blocks[sector] 309 | for block in blocks: 310 | self.hide_block(block) 311 | self.sectors = new_sectors 312 | 313 | def get_texture(self, idx): 314 | myTexture = self.loader.loadTexture("texture.png") 315 | return myTexture 316 | 317 | def load_textures(self): 318 | pass 319 | 320 | game = Game() 321 | game.run() 322 | 323 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Python Gems # 2 | ---------- 3 | **A collection of python scripts that solve interesting problems.** 4 | 5 | - async_sched.py - *A asynchronous scheduler implemented using coroutines, in principle similar to Tornado's ioloop* 6 | 7 | - config_parser.py - *My implementation of python standard library's ConfigParser module* 8 | 9 | - dancing_links.py - *My implementation of Dr. Knuth's dancing links algorithm, with a demo to solve N-Queen problem* 10 | 11 | - disjoint_set.py - *Disjoint set is a very important data structure, this is my naive implementation* 12 | 13 | - fileinput.py - *My implementation of python standard library's fileinput module* 14 | 15 | - go_repl.py - *A REPL for golang, support executing Go statements with instant feedback* 16 | 17 | - html_template.py - *A simple html template engine, supporting similar syntax as Django template language* 18 | 19 | - lisp.py - *A Lisp parser implemented in python, inspired by Peter Novig's essay* 20 | 21 | - memento.py - *Very elegant memento design pattern impl, copied from activestate recipes* 22 | 23 | - patch_module.py - *Patch python modules lazily, only when they are imported* 24 | 25 | - quine.py - *A python script to print itself* 26 | 27 | - rpc.py - *Simplistic RPC for python* 28 | 29 | - timeit.py - *My partial implementation of standard library's timeit module* 30 | 31 | - emojify - *Render an image with emoji's based on the colors in original image* 32 | 33 | - web_terminal - *A remote console from a web browser* 34 | 35 | - online_judge - *A OJ system like leetcode, with a small problemset, supporting only python solutions* 36 | 37 | - image_crawler - *A web image crawler written based on Tornado* 38 | 39 | - http_server - *A basic http server supporting static files/wsgi apps/proxying* 40 | -------------------------------------------------------------------------------- /aync_sched.py: -------------------------------------------------------------------------------- 1 | # This is an asynchronous task scheduler based on coroutines 2 | import socket 3 | import select 4 | from collections import deque 5 | 6 | class YieldPoint: 7 | def yield_task(self, task): 8 | pass 9 | def resume_task(self, task): 10 | pass 11 | 12 | class Scheduler: 13 | def __init__(self): 14 | self.task_cnt = 0 15 | self.tasks = deque() 16 | self.write_wait_tasks = {} 17 | self.read_wait_tasks = {} 18 | 19 | def wait_for_write(self, fileno, event, task): 20 | self.write_wait_tasks[fileno] = (event, task) 21 | 22 | def wait_for_read(self, fileno, event, task): 23 | self.read_wait_tasks[fileno] = (event, task) 24 | 25 | def new_task(self, task): 26 | self.tasks.append((task, None)) 27 | self.task_cnt += 1 28 | print "%d tasks"%self.task_cnt 29 | 30 | def add_task_back(self, task, data): 31 | self.tasks.append((task, data)) 32 | 33 | def _poll(self): 34 | r, w, x = select.select(self.read_wait_tasks, self.write_wait_tasks, []) 35 | for r_id in r: 36 | e, task = self.read_wait_tasks.pop(r_id) 37 | e.resume_task(task) 38 | for w_id in w: 39 | e, task = self.write_wait_tasks.pop(w_id) 40 | e.resume_task(task) 41 | 42 | def run(self): 43 | while self.task_cnt: 44 | if not self.tasks: 45 | self._poll() 46 | task, data = self.tasks.popleft() 47 | try: 48 | event = task.send(data) 49 | if not isinstance(event, YieldPoint): 50 | raise Exception("Task must yield YieldPoint") 51 | event.yield_task(task) 52 | except StopIteration: 53 | self.task_cnt -= 1 54 | print "%d tasks"%self.task_cnt 55 | 56 | # A echo server is implemented as an example 57 | sched = Scheduler() 58 | class ListenYieldPoint(YieldPoint): 59 | def __init__(self, sock): 60 | self.sock = sock 61 | def yield_task(self, task): 62 | sched.wait_for_read(self.sock, self, task) 63 | def resume_task(self, task): 64 | s, _ = self.sock.accept() 65 | sched.add_task_back(task, s) 66 | 67 | class RecvYieldPoint(YieldPoint): 68 | def __init__(self, sock): 69 | self.sock = sock 70 | def yield_task(self, task): 71 | sched.wait_for_read(self.sock, self, task) 72 | def resume_task(self, task): 73 | data = self.sock.recv(128) 74 | sched.add_task_back(task, data) 75 | 76 | class SendYieldPoint(YieldPoint): 77 | def __init__(self, sock, data): 78 | self.sock = sock 79 | self.data = data 80 | def yield_task(self, task): 81 | sched.wait_for_write(self.sock, self, task) 82 | def resume_task(self, task): 83 | sent = self.sock.send(self.data) 84 | sched.add_task_back(task, sent) 85 | 86 | def listener(cnt=5): 87 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 88 | sock.bind(("127.0.0.1", 5555)) 89 | i = 0 90 | while i' is used). 105 | """ 106 | secs = _parse_config_file(fp) 107 | self._sections.update(secs) 108 | 109 | def parse_item(self, section, key, val): 110 | # Now support only 1 level of interpolation 111 | if not _is_format_str(val): 112 | return val 113 | else: 114 | # first try default section 115 | try: 116 | newval = val%self._defaults 117 | except KeyError: 118 | try: 119 | newval = val%self._sections[section] 120 | except KeyError: 121 | newval = val 122 | return newval 123 | 124 | def get(self, section, option, raw=False, vars=None): 125 | if not self.has_section(section): 126 | return None 127 | sec = self._sections[section] 128 | if option not in sec: 129 | return None 130 | return self.parse_item(section, option, sec[option]) 131 | 132 | def getint(self, section, option): 133 | return int(self.get(section, option)) 134 | 135 | def getfloat(self, section, option): 136 | return float(self.get(section, option)) 137 | 138 | def getboolean(self, section, option): 139 | val = self.get(section, option).strip() 140 | if val.lower() in ["0", "false", "no", "off"]: 141 | return False 142 | elif val.lower() in ["1", "true", "yes", "on"]: 143 | return True 144 | else: 145 | raise ValueError("Invalid boolean option") 146 | 147 | def items(self, section, raw=False, vars=None): 148 | try: 149 | d = self._sections[section] 150 | except KeyError: 151 | d = {} 152 | d.update(self._defaults) 153 | return d.items() 154 | 155 | def remove_section(self, section): 156 | if self.has_section(section): 157 | del self._sections[section] 158 | 159 | def remove_option(self, section, option): 160 | if self.has_option(section, option): 161 | del self._sections[section][option] 162 | 163 | def set(self, section, option, value): 164 | if self.has_section(section): 165 | self._sections[section][option] = value 166 | 167 | def write(self, fp): 168 | #write the configuration state in .ini format 169 | pass -------------------------------------------------------------------------------- /dancing_links.py: -------------------------------------------------------------------------------- 1 | # The implementation of Dr Knuth's dancing links/algorithm X 2 | # https://www.ocf.berkeley.edu/~jchu/publicportal/sudoku/0011047.pdf 3 | # And demonstrate a N-Queens solver with dancing links 4 | # TODO: will add a soduku solver in the future 5 | 6 | # Both Node and HeaderNode have pointers in 4 directions 7 | # next, prev, up, down 8 | class Node: 9 | # instance of Node has a candidate attribute 10 | def __init__(self, candidate): 11 | # The candidate is actually index in candidates list 12 | self.candidate = candidate 13 | class HeaderNode: 14 | # Header node has a constraint 15 | def __init__(self, constraint): 16 | self.constraint = constraint 17 | 18 | class DancingLinks(object): 19 | def __init__(self, candidates, constraints, optional, check_func): 20 | self.candidates = candidates 21 | self.constraints = constraints 22 | self.optional = optional 23 | self.check = check_func 24 | # self.rows = [] 25 | self.head = None 26 | # to hold complete results and partial result 27 | self.results = [] 28 | self.partial = [] 29 | 30 | def build_links(self): 31 | # first build the header row 32 | self.head = HeaderNode(None) 33 | cursor = self.head 34 | for constraint in self.constraints: 35 | header = HeaderNode(constraint) 36 | cursor.next = header 37 | header.prev = cursor 38 | # single node loop in vertical direction 39 | header.up = header 40 | header.down = header 41 | cursor = header 42 | cursor.next = self.head 43 | self.head.prev = cursor 44 | 45 | # Now build the rows 46 | for i, candidate in enumerate(self.candidates): 47 | rowhead = None 48 | current = None 49 | cursor = self.head.next 50 | while cursor!=self.head: 51 | if self.check(candidate, cursor.constraint): 52 | # print candidate, cursor.constraint 53 | node = Node(i) 54 | # build left/right links 55 | if not rowhead: 56 | rowhead = current = node 57 | else: 58 | current.next = node 59 | node.prev = current 60 | current = node 61 | # build up/down links 62 | temp = cursor.up 63 | cursor.up = node 64 | node.down = cursor 65 | node.up = temp 66 | temp.down = node 67 | # go to next constraint 68 | cursor = cursor.next 69 | # close the row loop 70 | if current: 71 | current.next = rowhead 72 | rowhead.prev = current 73 | # self.rows.append(rowhead) 74 | 75 | # run algorithm x to find all exact matches 76 | def algorithm_x(self): 77 | # if constraint list is empty, current partial is a solution 78 | empty = (self.head.next == self.head) 79 | # very delicate situation: all constraints left are optional constraints, with no row satisfying each of them 80 | if not empty: 81 | all_empty_optional = True 82 | col = self.head.next 83 | while col!=self.head: 84 | if col.constraint not in self.optional or col.down != col: 85 | all_empty_optional = False 86 | break 87 | col = col.next 88 | 89 | if empty or all_empty_optional: 90 | result = sorted(self.partial) 91 | if result not in self.results: 92 | self.results.append(result) 93 | else: 94 | col = self.head.next 95 | if col.down == col: 96 | if col.constraint in self.optional: 97 | col = col.next 98 | else: 99 | # if non-optional constraint column is empty -> deadend, backtrack 100 | return 101 | # Pick this col to start 102 | row = col.down 103 | while row!=col: 104 | # Add this row to partial result 105 | self.partial.append(row.candidate) 106 | # Cover this row 107 | self.cover_row(row) 108 | # Recurse 109 | self.algorithm_x() 110 | # Uncover picked row 111 | self.uncover_row(row) 112 | # Pop picked row 113 | self.partial.pop() 114 | # Back track to next row 115 | row = row.down 116 | 117 | def cover_row(self, r): 118 | rr = r 119 | self.cover_column(r) 120 | r = r.next 121 | while r!=rr: 122 | self.cover_column(r) 123 | r = r.next 124 | 125 | def uncover_row(self, r): 126 | rr = r 127 | r = r.prev 128 | while r!=rr: 129 | self.uncover_column(r) 130 | r = r.prev 131 | self.uncover_column(r) 132 | 133 | def cover_column(self, c): 134 | # First find the column header 135 | while not isinstance(c, HeaderNode): 136 | c = c.up 137 | # Remove the header from header row 138 | # The dancing links! 139 | c.next.prev = c.prev 140 | c.prev.next = c.next 141 | 142 | # Remove the rows up down 143 | h = c 144 | c = c.down 145 | while c!=h: 146 | r = c 147 | cell = c.next 148 | while cell!=r: 149 | cell.up.down = cell.down 150 | cell.down.up = cell.up 151 | cell = cell.next 152 | c = c.down 153 | 154 | def uncover_column(self, c): 155 | # First find the column header 156 | while not isinstance(c, HeaderNode): 157 | c = c.up 158 | # Put the header node back into header row 159 | c.prev.next = c 160 | c.next.prev = c 161 | # Restore the rows, bottom up 162 | h = c 163 | c = c.up 164 | while c!=h: 165 | r = c 166 | cell = c.next 167 | while cell!=r: 168 | cell.up.down = cell 169 | cell.down.up = cell 170 | cell = cell.next 171 | c = c.up 172 | 173 | def get_results(self): 174 | # convert the result index list into actual candidates list 175 | return [map(lambda x: self.candidates[x], result) for result in self.results] 176 | 177 | def solve_N_queens(n): 178 | candidates = [(x, y) for x in range(n) for y in range(n)] 179 | constraints = [] 180 | optional = [] 181 | for i in range(n): 182 | # Every row should have one and only one queen 183 | constraints.append(('row', i)) 184 | for i in range(n): 185 | # Every column should have one and only one queen 186 | constraints.append(('col', i)) 187 | 188 | # Diagnal constraints are optional, very hard-to-find bug 189 | for i in range(n*2-1): 190 | # diagnal 191 | constraints.append(('diag', i)) 192 | optional.append(('diag', i)) 193 | for i in range(n*2-1): 194 | constraints.append(('rdiag', i)) 195 | optional.append(('rdiag', i)) 196 | 197 | def checker(candidate, constraint): 198 | t, val = constraint 199 | if t=='row': 200 | return candidate[0]==val 201 | if t=='col': 202 | return candidate[1]==val 203 | if t=='diag': 204 | return (candidate[0]+candidate[1])==val 205 | else: 206 | return (n-1-candidate[0]+candidate[1])==val 207 | 208 | dl = DancingLinks(candidates, constraints, optional, checker) 209 | dl.build_links() 210 | dl.algorithm_x() 211 | results = dl.get_results() 212 | 213 | for result in results: 214 | print "+++++++++" 215 | for i in range(n): 216 | s = "" 217 | for j in range(n): 218 | if (i, j) in result: 219 | s+="1" 220 | else: 221 | s+="0" 222 | print s 223 | print "+++++++++" 224 | print "%d results found for N-Queen"%len(results) 225 | 226 | def main(): 227 | solve_N_queens(10) 228 | 229 | if __name__ == "__main__": 230 | main() 231 | -------------------------------------------------------------------------------- /disjoint_set.py: -------------------------------------------------------------------------------- 1 | class DisjointSet(object): 2 | def __init__(self): 3 | self.leader = {} # maps a member to the group's leader 4 | self.group = {} # maps a group leader to the group (which is a set) 5 | 6 | def add(self, a, b): 7 | leadera = self.leader.get(a) 8 | leaderb = self.leader.get(b) 9 | if leadera is not None: 10 | if leaderb is not None: 11 | if leadera == leaderb: return # nothing to do 12 | groupa = self.group[leadera] 13 | groupb = self.group[leaderb] 14 | if len(groupa) < len(groupb): 15 | a, leadera, groupa, b, leaderb, groupb = b, leaderb, groupb, a, leadera, groupa 16 | groupa |= groupb 17 | del self.group[leaderb] 18 | for k in groupb: 19 | self.leader[k] = leadera 20 | else: 21 | self.group[leadera].add(b) 22 | self.leader[b] = leadera 23 | else: 24 | if leaderb is not None: 25 | self.group[leaderb].add(a) 26 | self.leader[a] = leaderb 27 | else: 28 | self.leader[a] = self.leader[b] = a 29 | self.group[a] = set([a, b]) 30 | 31 | mylist=""" 32 | specimen|3 33 | sample 34 | prototype 35 | example 36 | sample|3 37 | prototype 38 | example 39 | specimen 40 | prototype|3 41 | example 42 | specimen 43 | sample 44 | example|3 45 | specimen 46 | sample 47 | prototype 48 | prototype|1 49 | illustration 50 | specimen|1 51 | cat 52 | happy|2 53 | glad 54 | cheerful 55 | """ 56 | ds = DisjointSet() 57 | for line in mylist.strip().splitlines(): 58 | if '|' in line: 59 | node, _ = line.split('|') 60 | else: 61 | ds.add(node, line) 62 | 63 | for _,g in ds.group.items(): 64 | print g 65 | 66 | -------------------------------------------------------------------------------- /emojify/emoji/application_osx_terminal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/application_osx_terminal.png -------------------------------------------------------------------------------- /emojify/emoji/arrow_rotate_anticlockwise.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/arrow_rotate_anticlockwise.png -------------------------------------------------------------------------------- /emojify/emoji/book_addresses.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/book_addresses.png -------------------------------------------------------------------------------- /emojify/emoji/cake.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/cake.png -------------------------------------------------------------------------------- /emojify/emoji/cd.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/cd.png -------------------------------------------------------------------------------- /emojify/emoji/clock_.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/clock_.png -------------------------------------------------------------------------------- /emojify/emoji/controller.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/controller.png -------------------------------------------------------------------------------- /emojify/emoji/cup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/cup.png -------------------------------------------------------------------------------- /emojify/emoji/ee_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_1.png -------------------------------------------------------------------------------- /emojify/emoji/ee_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_2.png -------------------------------------------------------------------------------- /emojify/emoji/ee_22.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_22.png -------------------------------------------------------------------------------- /emojify/emoji/ee_23.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_23.png -------------------------------------------------------------------------------- /emojify/emoji/ee_24.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_24.png -------------------------------------------------------------------------------- /emojify/emoji/ee_25.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_25.png -------------------------------------------------------------------------------- /emojify/emoji/ee_26.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_26.png -------------------------------------------------------------------------------- /emojify/emoji/ee_27.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_27.png -------------------------------------------------------------------------------- /emojify/emoji/ee_28.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_28.png -------------------------------------------------------------------------------- /emojify/emoji/ee_29.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_29.png -------------------------------------------------------------------------------- /emojify/emoji/ee_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_3.png -------------------------------------------------------------------------------- /emojify/emoji/ee_30.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_30.png -------------------------------------------------------------------------------- /emojify/emoji/ee_31.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_31.png -------------------------------------------------------------------------------- /emojify/emoji/ee_32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_32.png -------------------------------------------------------------------------------- /emojify/emoji/ee_33.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_33.png -------------------------------------------------------------------------------- /emojify/emoji/ee_34.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_34.png -------------------------------------------------------------------------------- /emojify/emoji/ee_35.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_35.png -------------------------------------------------------------------------------- /emojify/emoji/ee_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_4.png -------------------------------------------------------------------------------- /emojify/emoji/ee_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_5.png -------------------------------------------------------------------------------- /emojify/emoji/ee_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_6.png -------------------------------------------------------------------------------- /emojify/emoji/ee_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ee_7.png -------------------------------------------------------------------------------- /emojify/emoji/group.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/group.png -------------------------------------------------------------------------------- /emojify/emoji/magnifier.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/magnifier.png -------------------------------------------------------------------------------- /emojify/emoji/page_white_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/page_white_edit.png -------------------------------------------------------------------------------- /emojify/emoji/palette.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/palette.png -------------------------------------------------------------------------------- /emojify/emoji/resultset_next.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/resultset_next.png -------------------------------------------------------------------------------- /emojify/emoji/ruby.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/emojify/emoji/ruby.png -------------------------------------------------------------------------------- /emojify/emojify.py: -------------------------------------------------------------------------------- 1 | # To make an emojified image given a regular image 2 | # inspired by this news feed: 3 | # http://news.dzwww.com/shehuixinwen/201509/t20150903_13011163.htm 4 | import os 5 | import sys 6 | from PIL import Image, ImageDraw 7 | 8 | EMOJI_SIZE = 32 9 | emojis = [] 10 | avg_colors = [] 11 | 12 | def load_emojis(): 13 | # load the emoji files 14 | script_path = os.path.dirname(os.path.realpath(__file__)) 15 | emoji_path = os.path.join(script_path, "emoji") 16 | fnames = os.listdir(emoji_path) 17 | for fname in fnames: 18 | fullname = os.path.join(emoji_path, fname) 19 | _, ext = os.path.splitext(fname) 20 | # make sure it is emoji png 21 | if os.path.isfile(fullname) and ext.lower() == ".png": 22 | im = Image.open(fullname) 23 | rgbim = im.convert("RGBA") 24 | w,h = im.size 25 | # compute the center average 26 | r, g, b = 0,0,0 27 | cnt = 0 28 | for i in range(w/8, 7*w/8): 29 | for j in range(h/8, 7*h/8): 30 | cnt += 1 31 | pr, pg, pb,_ = rgbim.getpixel((i,j)) 32 | r += pr 33 | g += pg 34 | b += pb 35 | # put the emoji and colors into containers 36 | emoji_color = r/cnt, g/cnt, b/cnt 37 | emojis.append(rgbim) 38 | avg_colors.append(emoji_color) 39 | print "Emojis loaded ..." 40 | 41 | # experiment with different distance functions 42 | def distance1(c1, c2): 43 | r, g, b = c1 44 | tr, tg, tb = c2 45 | return abs(r-tr) + abs(g-tg) +abs(b-tb) 46 | 47 | def distance2(c1, c2): 48 | # http://www.compuphase.com/cmetric.htm 49 | r, g, b = c1 50 | tr, tg, tb = c2 51 | return 3*(r-tr)**2 + 4*(g-tg)**2 + 2*(b-tb)**2 52 | 53 | # find the right emoji to replace the pixel(s) 54 | def find_emoji(color): 55 | min_distance = sys.maxint 56 | index = -1 57 | for i in range(len(avg_colors)): 58 | distance = distance2(color, avg_colors[i]) 59 | if distance < min_distance: 60 | min_distance = distance 61 | index = i 62 | assert index >= 0 63 | #print "found emoji %d"%index 64 | return emojis[index] 65 | 66 | def convert_image(image_path): 67 | try: 68 | img = Image.open(image_path) 69 | except IOError: 70 | print "Fail to open file %s"%image_path 71 | return 72 | img.convert("RGB") 73 | w, h = img.size 74 | 75 | # For large images, we convert pixel blocks 76 | # Consider 512 as the threshold 77 | unit = w/128 + 1 78 | print "Replacing %d x %d squares with emojis"%(unit, unit) 79 | neww, newh = w/unit, h/unit 80 | new_im = Image.new("RGBA", (neww*EMOJI_SIZE, newh*EMOJI_SIZE)) 81 | # canvas = ImageDraw.Draw(new_im) 82 | print "Starting to draw emojis ..." 83 | for x in range(neww): 84 | for y in range(newh): 85 | startx, starty = unit*x, unit*y 86 | rr, gg, bb = 0,0,0 87 | for xx in range(startx, startx+unit): 88 | for yy in range(starty, starty+unit): 89 | _r, _g, _b = img.getpixel((xx, yy)) 90 | rr += _r 91 | gg += _g 92 | bb += _b 93 | avg = rr/(unit*unit), gg/(unit*unit), bb/(unit*unit) 94 | #print avg 95 | emoji = find_emoji(avg) 96 | 97 | canvas_x, canvas_y = x*EMOJI_SIZE, y*EMOJI_SIZE 98 | new_im.paste(emoji, box=(canvas_x, canvas_y, 99 | canvas_x+EMOJI_SIZE, canvas_y+EMOJI_SIZE), mask=emoji) 100 | print "Finished drawing emojified image ..." 101 | 102 | new_im.show() 103 | target_path = os.path.join(os.path.dirname(image_path), "emojified.jpg") 104 | new_im.save(target_path) 105 | print "Image saved here: %s"%target_path 106 | 107 | def main(): 108 | if len(sys.argv)<2: 109 | print "Please specify the image file to convert." 110 | raise SystemExit 111 | fname = sys.argv[1] 112 | load_emojis() 113 | convert_image(fname) 114 | 115 | if __name__ == "__main__": 116 | main() 117 | 118 | 119 | 120 | -------------------------------------------------------------------------------- /fileinput.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is a simplistic implementation of fileinput module 3 | """ 4 | import sys 5 | 6 | class FileInput(object): 7 | def __init__(self, files=None): 8 | if not files: 9 | self._files = sys.argv[1:] 10 | if not self._files: 11 | self._files = ("_", ) 12 | elif isinstance(files, list): 13 | self._files = tuple(files) 14 | elif isinstance(files, basestring): 15 | self._files = (files, ) 16 | else: 17 | raise ValueError("Invalid input") 18 | 19 | self._file = None 20 | self._fileindex = 0 21 | self._lineindex = 0 22 | self._buffer = None 23 | self._filename = None 24 | self._is_stdin = False 25 | 26 | def __iter__(self): 27 | return self 28 | 29 | def next(self): 30 | line = self.get_next_line() 31 | if line: 32 | return line 33 | else: 34 | raise StopIteration 35 | 36 | def get_next_line(self): 37 | if not self._buffer: 38 | if not self.load_next_file(): 39 | return None 40 | try: 41 | line = self._buffer[self._lineindex] 42 | except IndexError: 43 | if self.load_next_file(): 44 | return self.get_next_line() 45 | else: 46 | return None 47 | else: 48 | self._lineindex += 1 49 | return line 50 | 51 | def load_next_file(self): 52 | if not self._files: 53 | return False 54 | _filename = self._files[0] 55 | self._files = self._files[1:] 56 | self._fileindex += 1 57 | 58 | if _filename == "_": 59 | self._filename = "" 60 | self._file = sys.stdin 61 | self._is_stdin = True 62 | else: 63 | self._filename = _filename 64 | if self._file and not self._is_stdin: 65 | self._file.close() 66 | try: 67 | self._file = open(_filename, "r") 68 | except: 69 | return self.load_next_file() 70 | self._lineindex = 0 71 | self._buffer = self._file.readlines() 72 | if not self._buffer: 73 | return load_next_file() 74 | return True 75 | 76 | def lineno(self): 77 | return self._lineindex 78 | 79 | def fileno(self): 80 | return self._fileindex 81 | 82 | def filename(self): 83 | return self._filename 84 | 85 | if __name__ == "__main__": 86 | f = FileInput() 87 | f = FileInput("../test.js") 88 | f = FileInput(["../test.js", "../weixin.txt", "foobar.txt"]) 89 | for line in f: 90 | print "%s(%d):%d\t%s"%(f.filename(), f.fileno(), f.lineno(), line) 91 | 92 | 93 | -------------------------------------------------------------------------------- /go_repl.py: -------------------------------------------------------------------------------- 1 | """A very very simple go REPL""" 2 | import subprocess 3 | import tempfile 4 | import re 5 | import operator 6 | 7 | src_template = """ 8 | package main 9 | {imports} 10 | {variables} 11 | {types} 12 | {funcs} 13 | 14 | func main(){{ 15 | {mains} 16 | }} 17 | """ 18 | 19 | class ParsingError(Exception): 20 | def __init__(self, msg): 21 | super(ParsingError, self).__init__(self, msg) 22 | self.msg = msg 23 | 24 | class Session(object): 25 | PROMPT = ">>> " 26 | CONT = "... " 27 | metachars = "()[]{}`\"'" 28 | pairs = { 29 | "(":")", 30 | "[":"]", 31 | "{":"}", 32 | "`":"`", 33 | "\"":"\"", 34 | "'":"'" 35 | } 36 | def __init__(self): 37 | self._reset() 38 | 39 | def _reset(self): 40 | self.imports = [] 41 | self.variables = [] 42 | self.types = [] 43 | self.funcs = [] 44 | self.mains = [] 45 | self.old_stdout = None 46 | self.stdout = None 47 | self._reset_buffer() 48 | 49 | def _reset_buffer(self): 50 | self.pair_stack = [] 51 | self.statement = "" 52 | self.current_line = "" 53 | self.is_complete = True 54 | 55 | def handle_imports(self, stmt): 56 | """ Maitain a list of pair [module name, used] in self.imports 57 | only used imports are placed in src for execution 58 | """ 59 | # First parse the "import" statement, supporting only 60 | # import "abc" and import ("abc" "cde"), 61 | # ignoring import _ "abc" and alias for now 62 | modules = re.findall(r"\"[^\s\"]+\"", stmt) 63 | 64 | for new_module in modules: 65 | if new_module in map(operator.itemgetter(0), self.imports): 66 | # not complain about duplicate imports 67 | continue 68 | self.imports.append([new_module, False]) 69 | 70 | # Before rendering source code, filter out the unused imports 71 | def update_import_used_states(self, snippets): 72 | for pair in self.imports: 73 | full_name = pair[0][1:-1] 74 | if r'/' in full_name: 75 | module_name = full_name.split(r'/')[-1] 76 | else: 77 | module_name = full_name 78 | 79 | # For the record, packages don't have to be used like 'module.' 80 | # But we will deal with the most common case only 81 | used = False 82 | for snippet in snippets: 83 | if module_name+"." in snippet: 84 | used = True 85 | break 86 | pair[1] = used 87 | 88 | def handle_statement(self): 89 | _target = self.statement.strip() 90 | if _target.startswith("!"): 91 | self.handle_command(_target) 92 | elif _target.startswith("import "): 93 | self.handle_imports(_target) 94 | elif "var " in _target or "const " in _target: 95 | # By default, we put all vars and consts in global scope 96 | # So that funcs can use them 97 | self.variables.append(_target) 98 | elif "type " in _target: 99 | self.types.append(_target) 100 | elif _target.startswith("func"): 101 | self.funcs.append(_target) 102 | else: 103 | self.mains.append(_target) 104 | if ":=" not in _target: 105 | self.run() 106 | 107 | def check_if_complete(self): 108 | # Very simplistic syntax check to determine: 109 | # 1 .If current statement is complete (all pairs closed) 110 | # 2. If there is syntax error in current line, raise 111 | # Checks: 112 | # 1. () [] {} "" '' `` should all be in pairs 113 | # 2. unless they are within "" or `` 114 | # 3. [] "" '' have to be closed in the same line 115 | for c in self.current_line: 116 | if c not in self.metachars: 117 | continue 118 | if self.pair_stack: 119 | stacktop = self.pair_stack[-1] 120 | # First check if c is closing something 121 | if c in self.pairs.values(): 122 | if c == self.pairs[stacktop]: 123 | del self.pair_stack[-1] 124 | continue 125 | elif c in "}])" and stacktop not in ["`", "\""]: 126 | raise ParsingError("'%s' closed at wrong placse"%c) 127 | # Check if need to push to stack 128 | if stacktop in ["`", "\""]: 129 | continue 130 | self.pair_stack.append(c) 131 | elif c in self.pairs: 132 | self.pair_stack.append(c) 133 | else: 134 | raise ParsingError("%s appears at wrong place"%c) 135 | 136 | self.is_complete = not self.pair_stack 137 | if self.is_complete: 138 | return True 139 | else: 140 | if self.pair_stack[-1] in "[\"'": 141 | raise ParsingError("%s have to be closed in the same line"%self.pair_stack[-1]) 142 | 143 | def generate_imports(self): 144 | # Generate simple import "abc" statements 145 | used_imports = filter(operator.itemgetter(1), self.imports) 146 | return "\n".join(["import "+name for name, _ in used_imports]) 147 | 148 | def render_src(self): 149 | _v = '\n'.join(self.variables) 150 | _t = '\n'.join(self.types) 151 | _f = '\n'.join(self.funcs) 152 | _m = '\n'.join(self.mains) 153 | self.update_import_used_states([_v, _t, _f, _m]) 154 | _i = self.generate_imports() 155 | return src_template.format( 156 | imports = _i, 157 | variables = _v, 158 | types = _t, 159 | funcs = _f, 160 | mains = _m 161 | ) 162 | 163 | def run(self): 164 | # generate the program source 165 | program = self.render_src() 166 | # write to temparory file 167 | f = tempfile.NamedTemporaryFile(suffix=".go", delete=False) 168 | f.write(program) 169 | f.flush() 170 | f.close() 171 | # Now go run 172 | gorun = subprocess.Popen(["go", "run", f.name], 173 | stdout=subprocess.PIPE, 174 | stderr=subprocess.PIPE) 175 | stdout, stderr = gorun.communicate() 176 | if stdout: 177 | self.old_stdout = self.stdout 178 | self.stdout = stdout 179 | # Only show the diff 180 | if not self.old_stdout: 181 | print self.stdout 182 | else: 183 | index = self.stdout.find(self.old_stdout) 184 | if index >=0: 185 | print self.stdout[index+len(self.old_stdout):] 186 | else: 187 | print self.stdout 188 | if stderr: 189 | # If error happens, remove the last statement in mains 190 | print stderr 191 | del self.mains[-1] 192 | 193 | def handle_command(self, command): 194 | command = command.strip()[1:] 195 | if command == "clear": 196 | self._reset() 197 | print "Starting afresh ..." 198 | elif command == "help": 199 | self.show_help() 200 | elif command == "exit": 201 | raise SystemExit() 202 | elif command == "src": 203 | print self.render_src() 204 | elif command == "history": 205 | if self.mains: 206 | for i, stmt in enumerate(self.mains): 207 | print i+1, "\t", stmt 208 | print "Use !del command to delete a statement from source" 209 | elif command.startswith("del"): 210 | segments = command.split() 211 | if len(segments)<2: 212 | print "!del should be followed by one or more index numbers" 213 | try: 214 | indice = reversed(sorted([int(seg) for seg in segments[1:]])) 215 | except: 216 | print "Some index numbers are invalid" 217 | else: 218 | for idx in indice: 219 | if idx > len(self.mains) or idx <= 0: 220 | print "Some index numbers are out of range" 221 | break 222 | del self.mains[idx-1] 223 | else: 224 | print "Invalid command." 225 | 226 | def show_help(self): 227 | print "+++++++++COMMANDS++++++++++" 228 | print "!help\t\tPrint this help" 229 | print "!exit\t\tExit this REPL" 230 | print "!src\t\tPrint the current source code" 231 | print "!clear\t\tClear defined vars/funcs/types and start afresh" 232 | print "!history\tShow statement you entered (not including var/func definitions)" 233 | print "!del \tDelete the statement from source code" 234 | print "+++++++++++++++++++++++++++" 235 | 236 | def main_loop(self): 237 | self.show_help() 238 | print "Enter your Go statements Or command(start with !):" 239 | while True: 240 | line = raw_input(self.PROMPT) 241 | self.statement = line 242 | self.current_line = line 243 | try: 244 | self.check_if_complete() 245 | except ParsingError as e: 246 | print "[ERROR] - "+e.msg 247 | self._reset_buffer() 248 | continue 249 | if self.is_complete: 250 | self.handle_statement() 251 | else: 252 | try: 253 | while True: 254 | line = raw_input(self.CONT) 255 | if not line and self.is_complete: 256 | break 257 | self.current_line = line 258 | self.statement += "\n" + line 259 | self.check_if_complete() 260 | except ParsingError as e: 261 | print "[ERROR] "+e.msg 262 | self._reset_buffer() 263 | continue 264 | else: 265 | self.handle_statement() 266 | 267 | 268 | def main(): 269 | Session().main_loop() 270 | 271 | if __name__== "__main__": 272 | main() 273 | 274 | 275 | 276 | -------------------------------------------------------------------------------- /html_template.py: -------------------------------------------------------------------------------- 1 | """ 2 | An attempt to implement a HTML template engine like Django template/Jinja2 3 | Supporting basic constructs: 4 | 1. variable {{variable}} 5 | 2. loop {% for val in vals %}... {% endfor %} 6 | 3. conditional {% if exp %} {% else %} {% endif %} 7 | 4. function call {% call funtion_name(arguments) %} 8 | """ 9 | import re, collections 10 | 11 | class Context(object): 12 | def __init__(self, context): 13 | self.context_stack = [context] 14 | 15 | def __contains__(self, val): 16 | for ctx in self.context_stack: 17 | if val in ctx: 18 | return True 19 | return False 20 | 21 | def push_context(self, context): 22 | self.context_stack.append(context) 23 | 24 | def pop_context(self): 25 | self.context_stack.pop() 26 | 27 | def __getitem__(self, varname): 28 | for c in reversed(self.context_stack): 29 | if varname in c: 30 | return c[varname] 31 | return None 32 | 33 | class Template(object): 34 | SEG_REGEX = re.compile(r"({{.*?}}|{%.*?%})") 35 | LOOP_REGEX = re.compile(r"for[\s+]([^\s]*)[\s+]in[\s+](.*)$") 36 | COND_REGEX = re.compile(r"if[\s+](.*)") 37 | CALL_REGEX = re.compile(r"call[\s+]([a-zA-Z0-9_]+)\((.*)\)") 38 | 39 | def __init__(self, template_str): 40 | self.template = template_str 41 | 42 | def _tokenize(self): 43 | self.tokens = self.SEG_REGEX.split(self.template) 44 | 45 | def compile(self): 46 | self._tokenize() 47 | self.nodes = [] 48 | while self.tokens: 49 | node = self._get_next_node() 50 | self.nodes.append(node) 51 | 52 | def _get_next_node(self): 53 | if not self.tokens: 54 | raise Exception("The template string is invalid") 55 | token = self.tokens.pop(0) 56 | if token.startswith("{{"): 57 | # this is a var 58 | varname = token[2:-2].strip() 59 | varnode = VarNode(varname) 60 | return varnode 61 | elif token.startswith("{%"): 62 | # this is a block 63 | inner = token[2:-2].strip() 64 | # is it a loop node? 65 | loop_match = self.LOOP_REGEX.match(inner) 66 | if loop_match: 67 | loop_var = loop_match.group(1) 68 | loop_list = loop_match.group(2) 69 | loopnode = LoopNode(loop_var, loop_list) 70 | children = [] 71 | while True: 72 | node = self._get_next_node() 73 | if isinstance(node, EndLoopNode): 74 | break 75 | children.append(node) 76 | loopnode.children = children 77 | return loopnode 78 | # is it a conditional node? 79 | cond_match = self.COND_REGEX.match(inner) 80 | if cond_match: 81 | condnode = CondNode(cond_match.group(1)) 82 | truenodes = [] 83 | falsenodes = [] 84 | in_list = truenodes 85 | while True: 86 | node = self._get_next_node() 87 | if isinstance(node, ElseNode): 88 | in_list = falsenodes 89 | continue 90 | elif isinstance(node, EndIfNode): 91 | break 92 | in_list.append(node) 93 | condnode.true_children = truenodes 94 | condnode.false_children = falsenodes 95 | return condnode 96 | # is it a function call 97 | call_match = self.CALL_REGEX.match(inner) 98 | if call_match: 99 | fname = call_match.group(1) 100 | arguments = [arg.strip() for arg in call_match.group(2).split(",")] 101 | callnode = CallNode(fname, arguments) 102 | return callnode 103 | # Other sentinel nodes: ELSE ENDIF ENDFOR 104 | if inner.lower()=="else": 105 | return ElseNode() 106 | if inner.lower()=="endif": 107 | return EndIfNode() 108 | if inner.lower()=="endfor": 109 | return EndLoopNode() 110 | else: 111 | return TextNode(token) 112 | 113 | def render(self, context): 114 | result_str = "" 115 | for node in self.nodes: 116 | result_str += node.render(context) 117 | return result_str 118 | 119 | class Node(object): 120 | pass 121 | 122 | class TextNode(Node): 123 | def __init__(self, text): 124 | self.text = text 125 | def render(self, context): 126 | return self.text 127 | 128 | class VarNode(Node): 129 | def __init__(self, var): 130 | self.var = var 131 | 132 | def render(self, context): 133 | if self.var not in context: 134 | raise Exception("Var %s missing from context"%self.var) 135 | return str(context[self.var]) 136 | 137 | class LoopNode(Node): 138 | def __init__(self, loop_var, loop_list): 139 | self.loop_var = loop_var 140 | self.loop_list = loop_list 141 | self.children = [] 142 | 143 | def render(self, context): 144 | result_str = "" 145 | if self.loop_list in context: 146 | _list = context[self.loop_list] 147 | else: 148 | _list = eval(self.loop_list,{},context) 149 | if not isinstance(_list, collections.Iterable): 150 | raise Exception("Var %s missing from context, or is not an iterable"%self.loop_list) 151 | 152 | # build in_loop context 153 | local_context = { 154 | "it":0, 155 | self.loop_var: None 156 | } 157 | context.push_context(local_context) 158 | for i, x in enumerate(_list): 159 | local_context["it"] = i 160 | local_context[self.loop_var] = x 161 | for child in self.children: 162 | result_str += child.render(context) 163 | 164 | context.pop_context() 165 | return result_str 166 | 167 | class CondNode(Node): 168 | def __init__(self, exp): 169 | self.exp = exp 170 | self.true_children = [] 171 | self.false_children = [] 172 | 173 | def render(self, context): 174 | cond = eval(self.exp, {}, context) 175 | result_str = "" 176 | if cond: 177 | children = self.true_children 178 | else: 179 | children = self.false_children 180 | for child in children: 181 | result_str += child.render(context) 182 | return result_str 183 | 184 | class CallNode(Node): 185 | def __init__(self, fname, args): 186 | self.fname = fname 187 | self.args = args 188 | def render(self, context): 189 | if self.fname not in context or not callable(context[self.fname]): 190 | raise Exception("Var %s missing from context, or not callable"%self.fname) 191 | arguments = [eval(arg, {}, context) for arg in self.args] 192 | result = context[self.fname](*arguments) 193 | return str(result) 194 | 195 | # Sentinels 196 | class EndLoopNode(Node): 197 | pass 198 | 199 | class ElseNode(Node): 200 | pass 201 | 202 | class EndIfNode(Node): 203 | pass 204 | 205 | 206 | def render_string_template(template_str, context): 207 | tpl = Template(template_str) 208 | tpl.compile() 209 | 210 | ctx = Context(context) 211 | return tpl.render(ctx) 212 | 213 | def test(): 214 | rendered = render_string_template('{% for item in items %}
{{it}}: {{item}}
{% endfor %}', {"items":["abc", "xyz"]}) 215 | print rendered 216 | rendered = render_string_template('{% for i in [1, 2, 3] %}
{{name}}-{{it}}-{{i}}
{% endfor %}', {"name":"Jon Snow"}) 217 | print rendered 218 | rendered = render_string_template('{% if num > 5 %}
more than 5
{% endif %}', {"num": 6}) 219 | print rendered 220 | rendered = render_string_template('{% if num > 5 %}
more than 5
{% else %}
less than 5
{% endif %}', {"num":2}) 221 | print rendered 222 | def pow(m=2, e=2): 223 | return m ** e 224 | rendered = render_string_template('{% call pow(2, x) %}', {"pow":pow, "x": 4}) 225 | print rendered 226 | 227 | if __name__=="__main__": 228 | test() 229 | 230 | 231 | -------------------------------------------------------------------------------- /http_server/README.md: -------------------------------------------------------------------------------- 1 | # Web Server # 2 | 3 | A simple web server in python, that supports serving: 4 | 5 | - Static files 6 | - WSGI application 7 | - Proxy to another web service 8 | - Running server in multi-thread/multi-process mode 9 | 10 | ## Configuration file ## 11 | 12 | { 13 | "server":{ 14 | "ip": "127.0.0.1", 15 | "port": 8000, 16 | "mode": "thread" 17 | }, 18 | "routes":{ 19 | "/app": { 20 | "type": "wsgi", 21 | "application": "D:/Workspace/Example/wsgi.py" 22 | }, 23 | "/app/static": { 24 | "type": "static", 25 | "dir": "D:/workspace/static/" 26 | }, 27 | "/app/proxy": { 28 | "type": "proxy", 29 | "proxyurl": "http://www.qq.com/" 30 | } 31 | } 32 | } 33 | 34 | After setting config.json, just run: 35 | 36 | python web_server.py 37 | 38 | *Note: this is just a coding practice, for learning about HTTP/wsgi, so don't consider using it for production.* 39 | -------------------------------------------------------------------------------- /http_server/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "server":{ 3 | "ip": "127.0.0.1", 4 | "port": 8000, 5 | "mode": "thread" 6 | }, 7 | "routes":{ 8 | "/app": { 9 | "type": "wsgi", 10 | "application": "D:/Workspace/QR/Vad/Vad/wsgi.py" 11 | }, 12 | "/app/static": { 13 | "type": "static", 14 | "dir": "D:/workspace/python-gems/" 15 | }, 16 | "/app/proxy": { 17 | "type": "proxy", 18 | "proxyurl": "http://www.qq.com/" 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /http_server/web_server.py: -------------------------------------------------------------------------------- 1 | import SocketServer, socket 2 | import mimetools, mimetypes 3 | import os 4 | import imp 5 | import urlparse 6 | import time 7 | import shutil 8 | import json 9 | import sys 10 | import urllib 11 | import requests 12 | import copy 13 | try: 14 | import cStringIO as StringIO 15 | except: 16 | import StringIO 17 | 18 | # templates 19 | error_tpl = u""" 20 | 21 | 22 | Error %d 23 | 24 | 25 |

%d

26 |

%s

27 | 28 | 29 | """ 30 | 31 | listing_tpl = u""" 32 | 33 | 34 | %s 35 | 36 | 37 |

%s

38 | %s 39 | 40 | 41 | """ 42 | # Exceptions 43 | class WSGIFileNotFound(Exception): 44 | "Raised when wsgi file is not found in file system" 45 | class WSGIInvalid(Exception): 46 | "Raised when wsgi file is not a valid python module, or application doesn't exist" 47 | class StaticDirNotValid(Exception): 48 | "Raised when static dir is not found or is not a directory" 49 | class DuplicatePath(Exception): 50 | "Raised when defining duplicate path in configuration file" 51 | 52 | # A mux to route HTTP path to correct handler 53 | class Mux(object): 54 | def __init__(self): 55 | self.dict = {} 56 | self.sortedkeys = [] 57 | 58 | def register_handler(self, path, handler): 59 | # register a virutal path to a handler 60 | if path in self.dict: 61 | raise DuplicatePath() 62 | self.dict[path] = handler 63 | idx = -1 64 | for i, key in enumerate(self.sortedkeys): 65 | if path.startswith(key): 66 | idx = i 67 | break 68 | if idx < 0: 69 | self.sortedkeys.append(path) 70 | else: 71 | self.sortedkeys.insert(idx, path) 72 | 73 | def get_handler(self, path): 74 | for key in self.sortedkeys: 75 | if path.startswith(key): 76 | return self.dict[key] 77 | return None 78 | 79 | # global mux object 80 | mux = Mux() 81 | 82 | # the main server 83 | class WebServer(object): 84 | # configuration is a json file 85 | def _read_config(self): 86 | try: 87 | f = open("config.json", 'r') 88 | except: 89 | add_error_log("Fail to read config file, exiting now ...") 90 | raise SystemExit() 91 | try: 92 | config = json.load(f) 93 | except ValueError: 94 | add_error_log("Fail to parse config file, exiting now...") 95 | raise SystemExit() 96 | try: 97 | self._address = config['server']['ip'] 98 | self._port = config['server']["port"] 99 | self._mode = config['server']['mode'] 100 | except KeyError: 101 | add_error_log("Missing server basic configuration, using defaults...") 102 | self._routes = config['routes'] 103 | 104 | def __init__(self): 105 | # defaults 106 | self._address = "127.0.0.1" 107 | self._port = 80 108 | self._mode = "thread" 109 | # load from config 110 | self._read_config() 111 | # initialize the mux 112 | for path in self._routes: 113 | d = self._routes[path] 114 | if d['type'] == "static": 115 | try: 116 | handler = StaticHandler(path, d['dir']) 117 | except StaticDirNotValid: 118 | add_error_log("Static directory in config file not valid, exiting...") 119 | raise SystemExit() 120 | elif d['type'] == "wsgi": 121 | try: 122 | handler = WSGIHandler(path, d['application']) 123 | except WSGIInvalid, WSGIFileNotFound: 124 | handler = None 125 | add_error_log("WSGI file invalid, ignoring path %s"%path) 126 | elif d['type'] =="proxy": 127 | handler = ProxyHandler(path, d['proxyurl']) 128 | else: 129 | add_error_log("Unsupported path definition: %s"%path) 130 | # link the handler and self 131 | handler.server = self 132 | try: 133 | mux.register_handler(path, handler) 134 | except DuplicatePath: 135 | add_error_log("Config file contains duplicate path definition, exiting...") 136 | raise SystemExit() 137 | 138 | def start(self): 139 | address = (self._address, self._port) 140 | if self._mode == "thread": 141 | server = SocketServer.ThreadingTCPServer(address, HTTPServerHandler) 142 | else: 143 | server = SocketServer.ForkingTCPServer(address, HTTPServerHandler) 144 | host, port = server.socket.getsockname()[:2] 145 | self.server_name = socket.getfqdn(host) 146 | self.server_port = port 147 | server.serve_forever() 148 | 149 | # implementation of handlers, each handler class should implement a handle_request(serv) method 150 | # For now, static handler only accept GET requests 151 | class StaticHandler(object): 152 | def __init__(self, virtual_path, static_dir): 153 | self.virtual_path = virtual_path 154 | self.static_dir = static_dir 155 | if not os.path.exists(static_dir) or not os.path.isdir(static_dir): 156 | raise StaticDirNotValid 157 | 158 | def handle_request(self, serv): 159 | if serv.verb.lower() != "get": 160 | serv.send_error_response(400, "Unsupported HTTP Method") 161 | # get the file system real path for the file/dir 162 | parsed = urlparse.urlparse(serv.path) 163 | relative_path = parsed.path[len(self.virtual_path):] 164 | unquoted_path = urllib.unquote(relative_path) 165 | real_path = self.static_dir + unquoted_path.decode(sys.getfilesystemencoding()) 166 | 167 | if not os.path.exists(real_path): 168 | serv.send_error_response(404, "File/directory not found.") 169 | return 170 | # handle differently for dir and file 171 | if os.path.isdir(real_path): 172 | # get the file listing 173 | listing = os.listdir(real_path) 174 | if not relative_path.endswith("/"): 175 | relative_path = relative_path + "/" 176 | 177 | # first try index.html if exists 178 | index_files = ["index.html", "index.htm"] 179 | for index in index_files: 180 | if index in listing: 181 | serv.send_response_line(302, "Redirected") 182 | index_path = os.path.join(self.virtual_path + relative_path, index) 183 | serv.send_header("Location", index_path) 184 | serv.end_headers() 185 | return 186 | 187 | # index.html not present, generate the listing html 188 | # Send the response line and headers 189 | serv.send_response_line(200, "OK") 190 | serv.send_header("Content-Type", "text/html;charset=%s"%sys.getfilesystemencoding()) 191 | serv.send_header("Connection", "close") 192 | # Construct HTML 193 | listing_str = "" 194 | if relative_path != "/": 195 | # if not root, add parent directory link 196 | parent_path = "/".join(relative_path.split("/")[:-2]) 197 | href = self.virtual_path + parent_path 198 | line = u"..
"%href 199 | listing_str += line 200 | # construct the file list 201 | for item in listing: 202 | if isinstance(item, unicode): 203 | # decode to unicode 204 | stritem = item.encode(sys.getfilesystemencoding()) 205 | unicodeitem = item 206 | else: 207 | stritem = item 208 | unicodeitem = item.decode(sys.getfilesystemencoding()) 209 | # urllib.quote must be given str, not unicode 210 | quoted_item = urllib.quote(stritem) 211 | # construct url with quoted file name 212 | href = os.path.join(self.virtual_path+relative_path, quoted_item) 213 | snippet = u"%s
"%(href, unicodeitem) 214 | listing_str += snippet 215 | 216 | display_path = self.virtual_path+relative_path 217 | listing_html = listing_tpl%(display_path, display_path, listing_str) 218 | listing_html = listing_html.encode(sys.getfilesystemencoding()) 219 | serv.send_header("Content-Length", len(listing_html)) 220 | serv.end_headers() 221 | serv.wfile.write(listing_html) 222 | else: 223 | try: 224 | f = open(real_path, "rb") 225 | except: 226 | serv.send_error_response(404, "File not found") 227 | return 228 | serv.send_response_line(200, "OK") 229 | _, ext = os.path.splitext(real_path) 230 | # ignore case of extension 231 | ext = ext.lower() 232 | # make a guess based on mimetypes 233 | content_type = mimetypes.types_map.get(ext, '') 234 | if not content_type: 235 | # default to text/html 236 | content_type = "text/html" 237 | serv.send_header("Content-Type", content_type) 238 | # content-length and last-modified 239 | stat = os.fstat(f.fileno()) 240 | serv.send_header("Content-Length", str(stat.st_size)) 241 | serv.send_header("Last-Modified", timestamp_to_string(stat.st_mtime)) 242 | serv.send_header("Connection", "close") 243 | serv.end_headers() 244 | # now copy the file over 245 | shutil.copyfileobj(f, serv.wfile) 246 | 247 | class ProxyHandler(object): 248 | def __init__(self, virtual_path, proxyurl): 249 | self.virtual_path = virtual_path 250 | if proxyurl.endswith("/"): 251 | self.proxyurl = proxyurl[:-1] 252 | else: 253 | self.proxyurl = proxyurl 254 | 255 | def handle_request(self, serv): 256 | parsed = urlparse.urlparse(serv.path) 257 | real_path = parsed.path[len(self.virtual_path):] 258 | if not real_path.startswith("/"): 259 | real_path= "/" + real_path 260 | real_url = self.proxyurl + real_path 261 | 262 | headers = copy.copy(serv.headers) 263 | del headers["Cookie"] 264 | headers['Host'] = urlparse.urlparse(self.proxyurl).netloc 265 | 266 | # Read the body of request 267 | body = "" 268 | if 'Content-Length' in headers and headers["Content-Length"]: 269 | body = serv.rfile.read(int(headers["Content-Length"])) 270 | print body 271 | # use requests to do the dirty work 272 | try: 273 | response = requests.request(serv.verb.upper(), real_url, 274 | params=parsed.query, headers=headers, data=body, 275 | allow_redirects=False) 276 | except Exception, e: 277 | add_error_log("Error when sending request to proxyurl:"+ str(e)) 278 | import traceback 279 | print traceback.format_exc() 280 | raise e 281 | # just translate 282 | serv.send_response_line(response.status_code, response.reason) 283 | for header in response.headers: 284 | # encoding has been handled by requests 285 | if header.lower()=="content-encoding" or header.lower()=="transfer-encoding": 286 | continue 287 | serv.send_header(header, response.headers[header]) 288 | serv.send_header("Content-Length", len(response.content)) 289 | 290 | serv.end_headers() 291 | 292 | serv.wfile.write(response.content) 293 | 294 | 295 | class WSGIHandler(object): 296 | def __init__(self, virtual_path, app_path): 297 | self.virtual_path = virtual_path 298 | self.load_application(app_path) 299 | 300 | def load_application(self, app_path): 301 | if not os.path.exists(app_path): 302 | raise WSGIFileNotFound() 303 | path, filename = os.path.split(app_path) 304 | modulename, ext = os.path.splitext(filename) 305 | # Add the path to sys.path 306 | sys.path.append(path) 307 | try: 308 | if ext.lower() == ".py" or ext.lower() == ".wsgi": 309 | m = imp.load_source(modulename, app_path) 310 | else: 311 | m = imp.load_compiled(modulename, app_path) 312 | except Exception as e: 313 | add_error_log(str(e)) 314 | raise WSGIInvalid() 315 | else: 316 | if not hasattr(m, "application"): 317 | add_error_log("Wsgi application not found") 318 | raise WSGIInvalid() 319 | self.app = m.application 320 | if not callable(self.app): 321 | add_error_log("Wsgi application not callable") 322 | raise WSGIInvalid() 323 | 324 | def get_headers_environ(self, serv): 325 | headers_environ = {} 326 | for key in serv.headers: 327 | val = serv.headers[key] 328 | if "-" in key: 329 | key = key.replace("-", "_") 330 | key = "HTTP_" + key.upper() 331 | if key not in headers_environ: 332 | headers_environ[key] = val 333 | else: 334 | headers_environ[key] += "," + val 335 | return headers_environ 336 | 337 | def prepare_environ(self, serv): 338 | parsed = urlparse.urlparse(serv.path) 339 | real_path = parsed.path[len(self.virtual_path):] 340 | if not real_path.startswith("/"): 341 | real_path= "/" + real_path 342 | environ = { 343 | "REQUEST_METHOD": serv.verb, 344 | "SCRIPT_NAME": self.virtual_path, 345 | "PATH_INFO": real_path, 346 | "QUERY_STRING": parsed.query, 347 | "CONTENT_TYPE": serv.headers.get("Content-Type", ""), 348 | "CONTENT_LENGTH": serv.headers.get("Content-Length", ""), 349 | "SERVER_NAME": self.server.server_name, 350 | "SERVER_PORT": self.server.server_port, 351 | "SERVER_PROTOCOL": "HTTP/1.1", 352 | "wsgi.input": serv.rfile, 353 | "wsgi.errors": serv.error, 354 | "wsgi.version": (1,0), 355 | "wsgi.run_once": False, 356 | "wsgi.url_scheme": "http", 357 | "wsgi.multithread": self.server._mode=="thread", 358 | "wsgi.multiprocess": self.server._mode=="process", 359 | } 360 | environ.update(self.get_headers_environ(serv)) 361 | return environ 362 | 363 | def handle_request(self, serv): 364 | # environ 365 | environ = self.prepare_environ(serv) 366 | # start_response 367 | def start_response(status, response_headers): 368 | serv.send_status_line(status) 369 | for k, v in response_headers: 370 | serv.send_header(k, v) 371 | serv.end_headers() 372 | # Get response lines 373 | try: 374 | response_chucks = self.app(environ, start_response) 375 | except Exception as e: 376 | add_error_log("* ERROR IN WSGI APP *" + str(e)) 377 | raise e 378 | else: 379 | for chuck in response_chucks: 380 | serv.wfile.write(chuck) 381 | 382 | # This is the handler entry point, dispatching requests to different handlers with the help of mux 383 | class HTTPServerHandler(SocketServer.StreamRequestHandler): 384 | def __init__(self, request, client_addr, server): 385 | self.error = StringIO.StringIO() 386 | SocketServer.StreamRequestHandler.__init__(self, request, client_addr, server) 387 | 388 | # Should read the request from self.rfile 389 | # and write the response to self.wfile 390 | def handle_one_request(self): 391 | try: 392 | # read the first line from request 393 | request_line = self.rfile.readline() 394 | words = request_line.strip().split() 395 | if len(words) != 3: 396 | self.send_error_response(400, "Invalid HTTP request") 397 | return 398 | self.verb, self.path, _ = words 399 | add_access_log(self.verb+" "+self.path) 400 | # read the header lines 401 | self.headers = mimetools.Message(self.rfile, 0) 402 | connection_type = self.headers.get("Connection", "") 403 | if connection_type == "close": 404 | self.close_connection = True 405 | elif connection_type == "keep-alive": 406 | self.close_connection = False 407 | 408 | # delegate body handling to mux 409 | handler = mux.get_handler(self.path) 410 | if not handler: 411 | self.send_error_response(404, "File Not Found") 412 | return 413 | handler.handle_request(self) 414 | if not self.wfile.closed: 415 | self.wfile.flush() 416 | self.wfile.close() 417 | 418 | # If the request handler write some error messages, record them in log 419 | if self.error.tell(): 420 | self.error.seek(0) 421 | add_error_log(self.error.read()) 422 | except Exception, e: 423 | add_error_log(str(e)) 424 | self.close_connection = True 425 | 426 | def handle(self): 427 | self.close_connection = True 428 | self.handle_one_request() 429 | # supporting keep-alive 430 | while not self.close_connection: 431 | self.handle_one_request() 432 | 433 | def send_response_line(self, code, explanation): 434 | self.send_status_line("%d %s"%(code, explanation)) 435 | 436 | def send_status_line(self, status): 437 | response_line = "HTTP/1.1 %s\r\n"%status 438 | self.wfile.write(response_line) 439 | self.send_header("Server", "Neo's HTTP Server") 440 | 441 | def send_header(self, name, value): 442 | self.wfile.write("%s: %s\r\n"%(name, value)) 443 | if name.lower() == "connection": 444 | if value.lower() == "close": 445 | self.close_connection = True 446 | elif value.lower() == "keep-alive": 447 | self.close_connection = False 448 | 449 | def end_headers(self): 450 | self.wfile.write("\r\n") 451 | 452 | def send_error_response(self, code, explanation): 453 | self.send_response_line(code, explanation) 454 | self.send_header("Content-type", "text/html") 455 | self.send_header("Connection", "close") 456 | self.end_headers() 457 | message_body = error_tpl%(code, code, explanation) 458 | self.wfile.write(message_body) 459 | if not self.wfile.closed: 460 | self.wfile.flush() 461 | 462 | # helper functions 463 | def timestamp_to_string(timestamp=None): 464 | """Return the current date and time formatted for a message header.""" 465 | weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] 466 | monthname = [None, 467 | 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 468 | 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] 469 | if timestamp is None: 470 | timestamp = time.time() 471 | year, month, day, hh, mm, ss, wd, y, z = time.gmtime(timestamp) 472 | s = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( 473 | weekdayname[wd], 474 | day, monthname[month], year, 475 | hh, mm, ss) 476 | return s 477 | 478 | def add_error_log(entry): 479 | print "[ERROR] - " + entry 480 | 481 | def add_access_log(entry): 482 | print "[REQUEST] - " + entry 483 | 484 | # The driver 485 | def main(): 486 | server = WebServer() 487 | server.start() 488 | 489 | if __name__ == "__main__": 490 | main() 491 | -------------------------------------------------------------------------------- /image_crawler/README.md: -------------------------------------------------------------------------------- 1 | # YAPIC - Yet another python image crawler # 2 | 3 | Just execute `python crawler.py http://start-url-to-crawl` 4 | 5 | Specify options in `config.ini`: 6 | 7 | ; start url for crawler 8 | starturl=http://pic.kdslife.com/ 9 | 10 | ; regexes for links and image urls 11 | linkregex=http://pic.kdslife.com/content_.*.html 12 | imgregex=http://img.club.pchome.net/.*.jpg 13 | 14 | ; integer>=1, larger politeness means slower crawling 15 | ; but also less likely to be denied service 16 | politeness=3 17 | 18 | ; the directory to store the downloaded images 19 | imgdir=E:/kds/ 20 | 21 | ; the min size of images that you want to download 22 | minwidth=200 23 | minheight=200 24 | -------------------------------------------------------------------------------- /image_crawler/config.ini: -------------------------------------------------------------------------------- 1 | [global] 2 | ; start url for crawler 3 | starturl=http://pic.kdslife.com/ 4 | 5 | ; regexes for links and image urls 6 | linkregex=http://pic.kdslife.com/content_.*.html 7 | imgregex=http://img.club.pchome.net/.*.jpg 8 | 9 | ; integer>=1, larger politeness means slower crawling, but also less likely to be denied service 10 | politeness=3 11 | 12 | ; the directory to store the downloaded images 13 | imgdir=E:/kds/ 14 | 15 | ; the min size of images that you want to download 16 | minwidth=200 17 | minheight=200 18 | -------------------------------------------------------------------------------- /image_crawler/crawler.py: -------------------------------------------------------------------------------- 1 | # This is an asynchronous image crawler written in python 2 | from tornado.httpclient import AsyncHTTPClient 3 | from tornado.queues import Queue 4 | from tornado.locks import Semaphore 5 | from tornado.ioloop import IOLoop 6 | from tornado.web import Application, RequestHandler 7 | from tornado.httpserver import HTTPServer 8 | from tornado import gen 9 | 10 | import random 11 | import bs4 12 | import urlparse 13 | import PIL.Image 14 | import ConfigParser 15 | import sys 16 | import time 17 | import re 18 | import os.path 19 | 20 | # data structures 21 | links = Queue() 22 | imageurls = Queue() 23 | visited_links = set() 24 | downloaded_images = set() 25 | link_failures = [] 26 | download_failures = [] 27 | img_counter = 0 28 | 29 | class WebHandler(RequestHandler): 30 | def get(self): 31 | response_str = "

Visited links: %d
"%len(visited_links) 32 | response_str += "Downloaded images: %d

"%len(downloaded_images) 33 | response_str += "
" 34 | response_str += "VISITED:
" 35 | for link in visited_links: 36 | response_str += link+"
" 37 | response_str += "DOWNLOADED:
" 38 | for img in downloaded_images: 39 | response_str += img+"
" 40 | self.write(response_str) 41 | 42 | class Crawler(object): 43 | def _init_defaults(self): 44 | self.start_link = None 45 | self.link_priority = 2 46 | self.img_priority = 8 47 | self.politeness = 2 48 | self.workers_limit = 10 # allow at most 10 concurrent workers 49 | self.link_regex = re.compile("^http://.*") 50 | self.img_regex = re.compile(".*") 51 | self.fname_digits = 4 52 | self.min_width = 200 53 | self.min_height = 200 54 | self.img_dir = "E:/tmp/" 55 | self.idle_wait_loops = 100 56 | self.port = 8888 57 | 58 | def _load_config(self): 59 | parser = ConfigParser.ConfigParser() 60 | parser.read("config.ini") 61 | 62 | if parser.has_option("global", "starturl"): 63 | starturl = parser.get("global", "starturl") 64 | self.start_link = starturl 65 | 66 | if parser.has_option("global", "linkregex"): 67 | self.link_regex = re.compile(parser.get("global", "linkregex")) 68 | if parser.has_option("global", "imgregex"): 69 | self.img_regex = re.compile(parser.get("global", "imgregex")) 70 | 71 | if parser.has_option("global", "politeness"): 72 | politeness = parser.getint("global", "politeness") 73 | if politeness <=0: 74 | print "politeness must be a positive integer" 75 | raise SystemExit() 76 | self.politeness = politeness 77 | if parser.has_option("global", "imgdir"): 78 | imgdir = parser.get("global", "imgdir") 79 | if not os.path.exists(imgdir) or not os.path.isdir(imgdir): 80 | print "invalid imgdir configuration" 81 | raise SystemExit() 82 | if not imgdir.endswith("/"): 83 | imgdir+="/" 84 | self.img_dir = imgdir 85 | 86 | if parser.has_option("global", "minwidth"): 87 | width = parser.getint("global", "minwidth") 88 | self.min_width = width 89 | if parser.has_option("global", "minheight"): 90 | height = parser.getint("global", "minheight") 91 | self.min_height = height 92 | 93 | def __init__(self, start_link=None): 94 | self._init_defaults() 95 | # Now load the config file to override defaults 96 | self._load_config() 97 | 98 | if start_link: 99 | self.start_link = start_link 100 | if not self.start_link: 101 | raise SystemExit("No start link is provided, exiting now...") 102 | links.put(self.start_link) 103 | self.semaphore = Semaphore(self.workers_limit) 104 | 105 | @gen.coroutine 106 | def run(self): 107 | # First start an debug server 108 | app = Application([(r"/", WebHandler)]) 109 | server = HTTPServer(app) 110 | server.listen(self.port) 111 | 112 | idle_loops = 0 113 | while True: 114 | if imageurls.qsize()==0 and links.qsize()==0: 115 | print "Both link and image queues are empty now" 116 | idle_loops += 1 117 | if idle_loops == self.idle_wait_loops: 118 | break 119 | else: 120 | idle_loops = 0 # clear the idle loop counter 121 | if imageurls.qsize()==0: 122 | self.handle_links() 123 | elif links.qsize()==0: 124 | self.handle_imageurls() 125 | else: 126 | choices = [0]*self.link_priority +[1]*self.img_priority 127 | choice = random.choice(choices) 128 | if choice: 129 | self.handle_imageurls() 130 | else: 131 | self.handle_links() 132 | yield gen.sleep(0.1 * self.politeness) 133 | # Wait for all link handlers 134 | links.join() 135 | # Handling imageurls if generated by the last few links 136 | while imageurls.qsize(): 137 | self.handle_imageurls() 138 | imageurls.join() 139 | 140 | @gen.coroutine 141 | def handle_links(self): 142 | yield self.semaphore.acquire() 143 | newlink = yield links.get() 144 | 145 | # Make sure we haven't visited this one 146 | if newlink in visited_links: 147 | self.semaphore.release() 148 | raise gen.Return() 149 | visited_links.add(newlink) 150 | 151 | # use async client to fetch this url 152 | client = AsyncHTTPClient() 153 | tries = 3 # Give it 3 chances before putting it in failure 154 | while tries: 155 | response = yield client.fetch(newlink) 156 | if response.code==200: 157 | break 158 | tries -= 1 159 | 160 | # release the semaphore 161 | self.semaphore.release() 162 | if response.code!=200: 163 | link_failures.append(newlink) 164 | print "[FAILURE] - %s"%newlink 165 | raise gen.Return() 166 | 167 | # TODO: replace this with a report api 168 | print "[VISITED] - %s"%newlink 169 | 170 | # parse url to get the base url 171 | components = urlparse.urlparse(newlink) 172 | baseurl = components[0]+"://"+components[1] 173 | path = components[2] 174 | 175 | # parse the html with bs 176 | soup = bs4.BeautifulSoup(response.body) 177 | # extract valid links and put into links 178 | a_tags = soup.find_all("a") 179 | for tag in a_tags: 180 | if "href" not in tag.attrs: 181 | continue 182 | href = tag['href'] 183 | if href.startswith("#"): 184 | continue 185 | if href.startswith("/"): # relative 186 | href = baseurl+href 187 | else: 188 | if not path.endswith("/"): 189 | path = path[:path.rfind("/")+1] 190 | href = baseurl+"/"+path+href 191 | if not self.link_regex.match(href): 192 | continue 193 | if href in visited_links: 194 | continue 195 | links.put(href) 196 | print "NEWLINK:", href 197 | 198 | # extract imgs and put into imageurls 199 | img_tags = soup.find_all("img") 200 | for tag in img_tags: 201 | if "src" not in tag.attrs: 202 | continue 203 | src = tag['src'] 204 | if src.startswith("/"): # relative 205 | src = baseurl+src 206 | if not self.img_regex.match(src): 207 | continue 208 | if src in downloaded_images: 209 | continue 210 | imageurls.put(src) 211 | print "NEW IMAGE:", src 212 | 213 | # now the task is done 214 | links.task_done() 215 | 216 | @gen.coroutine 217 | def handle_imageurls(self): 218 | yield self.semaphore.acquire() 219 | imgurl = yield imageurls.get() 220 | 221 | if imgurl in downloaded_images: 222 | self.semaphore.release() 223 | raise gen.Return() 224 | # mark the image as downloaded 225 | downloaded_images.add(imgurl) 226 | 227 | # use async client to fetch this url 228 | client = AsyncHTTPClient() 229 | tries = 3 # Give it 3 chances before putting it in failure 230 | while tries: 231 | response = yield client.fetch(imgurl) 232 | if response.code==200: 233 | break 234 | tries -= 1 235 | # Download is finished, release semaphore 236 | self.semaphore.release() 237 | 238 | if response.code!=200: 239 | download_failures.append(imgurl) 240 | print "[FAILURE] - %s"%imgurl 241 | raise gen.Return() 242 | 243 | # TODO: replace this with a report api 244 | print "[DOWNLOADED] - %s"%imgurl 245 | 246 | # Read the file content 247 | img = PIL.Image.open(response.buffer) 248 | w, h = img.size 249 | if w 1: 278 | crawler = Crawler(sys.argv[1]) 279 | else: 280 | crawler = Crawler() 281 | 282 | IOLoop.current().run_sync(crawler.run) 283 | 284 | # TODO: replace with reporting api calls 285 | print "++++++++++++++++++++++++++++" 286 | print "%d links visited."%len(visited_links) 287 | print "%d images downloaded"%len(downloaded_images) 288 | print "Link failures:", link_failures 289 | print "Image download failures:", download_failures 290 | print "++++++++++++++++++++++++++++" 291 | 292 | if __name__ == "__main__": 293 | main() 294 | 295 | -------------------------------------------------------------------------------- /image_crawler/requirements.txt: -------------------------------------------------------------------------------- 1 | tornado>=4.2 2 | PIL 3 | bs4 -------------------------------------------------------------------------------- /lisp.py: -------------------------------------------------------------------------------- 1 | # This is a LISP parser implemented in python, 2 | # written after reading Peter Novig's lis.py essay http://www.norvig.com/lispy.html 3 | def tokenize(program): 4 | return program.replace('(', '( ').replace(')', ' )').split() 5 | 6 | def parse(program): 7 | tokens = tokenize(program) 8 | return get_next(tokens) 9 | 10 | Symbol = str 11 | 12 | def atom(token): 13 | try: 14 | return int(token) 15 | except: 16 | try: 17 | return float(token) 18 | except: 19 | return Symbol(token) 20 | 21 | def get_next(tokens): 22 | token = tokens.pop(0) 23 | if token =='(': 24 | parsed = [] 25 | while True: 26 | token = tokens[0] 27 | if token==')': 28 | tokens.pop(0) 29 | break 30 | else: 31 | parsed.append(get_next(tokens)) 32 | return parsed 33 | elif token==")": 34 | raise Exception("Syntax Error") 35 | else: 36 | return atom(token) 37 | 38 | class Env(dict): 39 | def __init__(self, parent={}): 40 | dict.__init__(self) 41 | self.parent = parent 42 | def __getitem__(self, name): 43 | if name in self: 44 | return dict.__getitem__(self, name) 45 | if name in self.parent: 46 | return self.parent[name] 47 | raise KeyError 48 | 49 | global_env = Env() 50 | import operator as op 51 | global_env.update({ 52 | '+':op.add, '-':op.sub, '*':op.mul, '/':op.div, 53 | '>':op.gt, '<':op.lt, '>=':op.ge, '<=':op.le, '=':op.eq, 54 | 'abs': abs, 55 | }) 56 | 57 | class Proc(object): 58 | def __init__(self, args, exp, env): 59 | self.args = args 60 | self.exp = exp 61 | self.env = Env(env) 62 | def __call__(self, *arguments): 63 | for n, v in zip(self.args, arguments): 64 | self.env[n] = v 65 | return eval(self.exp, self.env) 66 | 67 | def eval(exp, env = global_env): 68 | if isinstance(exp, basestring): 69 | return env[exp] 70 | elif isinstance(exp, (int, float)): 71 | return exp 72 | elif isinstance(exp, list): 73 | head = exp[0] 74 | if head=="define": 75 | _, name, val = exp 76 | env[name] = eval(val, env) 77 | elif head=="set!": 78 | _, name, val = exp 79 | if name not in env: 80 | raise Exception("Undefined") 81 | env[name] = eval(val, env) 82 | elif head=="quote": 83 | return exp[1] 84 | elif head=="if": 85 | test = eval(exp[1], env) 86 | if test: 87 | return eval(exp[2], env) 88 | else: 89 | return eval(exp[3], env) 90 | elif head=="lambda": 91 | return Proc(exp[1], exp[2], env) 92 | else: 93 | proc = env[head] 94 | return proc(*[eval(arg, env) for arg in exp[1:]]) 95 | 96 | def repl(): 97 | while True: 98 | program = raw_input("LISP > ") 99 | val = eval(parse(program)) 100 | if val: 101 | print val 102 | 103 | repl() 104 | 105 | -------------------------------------------------------------------------------- /memento.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """http://code.activestate.com/recipes/413838-memento-closure/""" 5 | 6 | from copy import copy, deepcopy 7 | 8 | 9 | def memento(obj, deep=False): 10 | state = copy(obj.__dict__) if deep else deepcopy(obj.__dict__) 11 | 12 | def restore(): 13 | obj.__dict__.clear() 14 | obj.__dict__.update(state) 15 | 16 | return restore 17 | 18 | 19 | class Transaction: 20 | """A transaction guard. 21 | This is, in fact, just syntactic sugar around a memento closure. 22 | """ 23 | deep = False 24 | states = [] 25 | 26 | def __init__(self, *targets): 27 | self.targets = targets 28 | self.commit() 29 | 30 | def commit(self): 31 | self.states = [memento(target, self.deep) for target in self.targets] 32 | 33 | def rollback(self): 34 | for a_state in self.states: 35 | a_state() 36 | 37 | 38 | class Transactional(object): 39 | """Adds transactional semantics to methods. Methods decorated with 40 | @Transactional will rollback to entry-state upon exceptions. 41 | """ 42 | 43 | def __init__(self, method): 44 | self.method = method 45 | 46 | def __get__(self, obj, T): 47 | def transaction(*args, **kwargs): 48 | state = memento(obj) 49 | try: 50 | return self.method(obj, *args, **kwargs) 51 | except Exception as e: 52 | state() 53 | raise e 54 | 55 | return transaction 56 | 57 | 58 | class NumObj(object): 59 | def __init__(self, value): 60 | self.value = value 61 | 62 | def __repr__(self): 63 | return '<%s: %r>' % (self.__class__.__name__, self.value) 64 | 65 | def increment(self): 66 | self.value += 1 67 | 68 | @Transactional 69 | def do_stuff(self): 70 | self.value = '1111' # <- invalid value 71 | self.increment() # <- will fail and rollback 72 | 73 | 74 | if __name__ == '__main__': 75 | num_obj = NumObj(-1) 76 | print(num_obj) 77 | 78 | a_transaction = Transaction(num_obj) 79 | try: 80 | for i in range(3): 81 | num_obj.increment() 82 | print(num_obj) 83 | a_transaction.commit() 84 | print('-- committed') 85 | 86 | for i in range(3): 87 | num_obj.increment() 88 | print(num_obj) 89 | num_obj.value += 'x' # will fail 90 | print(num_obj) 91 | except Exception as e: 92 | a_transaction.rollback() 93 | print('-- rolled back') 94 | print(num_obj) 95 | 96 | print('-- now doing stuff ...') 97 | try: 98 | num_obj.do_stuff() 99 | except Exception as e: 100 | print('-> doing stuff failed!') 101 | import sys 102 | import traceback 103 | 104 | traceback.print_exc(file=sys.stdout) 105 | print(num_obj) 106 | 107 | 108 | ### OUTPUT ### 109 | # 110 | # 111 | # 112 | # 113 | # -- committed 114 | # 115 | # 116 | # 117 | # -- rolled back 118 | # 119 | # -- now doing stuff ... 120 | # -> doing stuff failed! 121 | # Traceback (most recent call last): 122 | # File "memento.py", line 97, in 123 | # num_obj.do_stuff() 124 | # File "memento.py", line 52, in transaction 125 | # raise e 126 | # File "memento.py", line 49, in transaction 127 | # return self.method(obj, *args, **kwargs) 128 | # File "memento.py", line 70, in do_stuff 129 | # self.increment() # <- will fail and rollback 130 | # File "memento.py", line 65, in increment 131 | # self.value += 1 132 | # TypeError: Can't convert 'int' object to str implicitly 133 | # 134 | -------------------------------------------------------------------------------- /mock_server/data.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/mock_server/data.db -------------------------------------------------------------------------------- /mock_server/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mock_server.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /mock_server/mock_server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RealHacker/python-gems/9124eecb231ad7479fda5e21e9a9dc2639986c34/mock_server/mock_server/__init__.py -------------------------------------------------------------------------------- /mock_server/mock_server/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | class URL(models.Model): 4 | name = models.CharField(max_length=64) 5 | pattern = models.CharField(max_length=256) 6 | 7 | 8 | class Condition(models.Model): 9 | METHOD_CHOICES = ( 10 | ("GET", "GET"), 11 | ("POST", "POST"), 12 | ("HEAD", "HEAD"), 13 | ("DELETE", "DELETE"), 14 | ("PUT", "PUT") 15 | ) 16 | url = models.ForeignKey(URL) 17 | method = models.CharField(max_length=32, choices=METHOD_CHOICES) 18 | state_filter = models.CharField(max_length=32, null=True) 19 | query_filter = models.CharField(max_length=128, null=True) 20 | 21 | class Response(models.Model): 22 | RESPONSE_TYPES = ( 23 | (1, "TEMPLATE"), 24 | (2, "JSON"), 25 | ) 26 | condition = models.ForeignKey(Condition) 27 | is_inactive = models.BooleanField() 28 | response_type = models.IntegerField(choices=RESPONSE_TYPES) 29 | data = models.TextField() 30 | status_code = models.IntegerField(default=200) 31 | tpl_name = models.CharField(null=True, max_length=128) 32 | tpl_type = models.CharField(null=True, max_length=32) 33 | redirect_url = models.CharField(null=True, max_length=128) 34 | 35 | class GlobalState(models.Model): 36 | name = models.CharField(max_length=32, unique=True) 37 | value = models.CharField(max_length=32) 38 | statetype = models.CharField(max_length=32) 39 | choices = models.CharField(max_length=256) 40 | 41 | class Proxy(models.Model): 42 | isOn = models.BooleanField() 43 | proxy_server = models.CharField(max_length=128) 44 | 45 | class AccessRecord(models.Model): 46 | condition = models.ForeignKey(Condition) 47 | fullurl = models.CharField(max_length=256) 48 | timestamp = models.DateTimeField(auto_now_add=True) 49 | 50 | 51 | -------------------------------------------------------------------------------- /mock_server/mock_server/settings.py: -------------------------------------------------------------------------------- 1 | # Django settings for mock_server project. 2 | 3 | DEBUG = True 4 | TEMPLATE_DEBUG = DEBUG 5 | 6 | ADMINS = ( 7 | # ('Your Name', 'your_email@example.com'), 8 | ) 9 | 10 | MANAGERS = ADMINS 11 | 12 | DATABASES = { 13 | 'default': { 14 | 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. 15 | 'NAME': 'data.db', # Or path to database file if using sqlite3. 16 | # The following settings are not used with sqlite3: 17 | 'USER': '', 18 | 'PASSWORD': '', 19 | 'HOST': '', # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP. 20 | 'PORT': '', # Set to empty string for default. 21 | } 22 | } 23 | 24 | # Hosts/domain names that are valid for this site; required if DEBUG is False 25 | # See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts 26 | ALLOWED_HOSTS = [] 27 | 28 | # Local time zone for this installation. Choices can be found here: 29 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 30 | # although not all choices may be available on all operating systems. 31 | # In a Windows environment this must be set to your system time zone. 32 | TIME_ZONE = 'America/Chicago' 33 | 34 | # Language code for this installation. All choices can be found here: 35 | # http://www.i18nguy.com/unicode/language-identifiers.html 36 | LANGUAGE_CODE = 'en-us' 37 | 38 | SITE_ID = 1 39 | 40 | # If you set this to False, Django will make some optimizations so as not 41 | # to load the internationalization machinery. 42 | USE_I18N = True 43 | 44 | # If you set this to False, Django will not format dates, numbers and 45 | # calendars according to the current locale. 46 | USE_L10N = True 47 | 48 | # If you set this to False, Django will not use timezone-aware datetimes. 49 | USE_TZ = True 50 | 51 | # Absolute filesystem path to the directory that will hold user-uploaded files. 52 | # Example: "/var/www/example.com/media/" 53 | MEDIA_ROOT = '' 54 | 55 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 56 | # trailing slash. 57 | # Examples: "http://example.com/media/", "http://media.example.com/" 58 | MEDIA_URL = '' 59 | 60 | # Absolute path to the directory static files should be collected to. 61 | # Don't put anything in this directory yourself; store your static files 62 | # in apps' "static/" subdirectories and in STATICFILES_DIRS. 63 | # Example: "/var/www/example.com/static/" 64 | STATIC_ROOT = '' 65 | 66 | # URL prefix for static files. 67 | # Example: "http://example.com/static/", "http://static.example.com/" 68 | STATIC_URL = '/static/' 69 | 70 | # Additional locations of static files 71 | STATICFILES_DIRS = ( 72 | # Put strings here, like "/home/html/static" or "C:/www/django/static". 73 | # Always use forward slashes, even on Windows. 74 | # Don't forget to use absolute paths, not relative paths. 75 | ) 76 | 77 | # List of finder classes that know how to find static files in 78 | # various locations. 79 | STATICFILES_FINDERS = ( 80 | 'django.contrib.staticfiles.finders.FileSystemFinder', 81 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 82 | # 'django.contrib.staticfiles.finders.DefaultStorageFinder', 83 | ) 84 | 85 | # Make this unique, and don't share it with anybody. 86 | SECRET_KEY = '$8u!g&pv!n@iaxkz!p$($*)j!_szx@$lo=u7=^#^(x+cyp8=q3' 87 | 88 | # List of callables that know how to import templates from various sources. 89 | TEMPLATE_LOADERS = ( 90 | 'django.template.loaders.filesystem.Loader', 91 | 'django.template.loaders.app_directories.Loader', 92 | # 'django.template.loaders.eggs.Loader', 93 | ) 94 | 95 | MIDDLEWARE_CLASSES = ( 96 | 'django.middleware.common.CommonMiddleware', 97 | 'django.contrib.sessions.middleware.SessionMiddleware', 98 | # 'django.middleware.csrf.CsrfViewMiddleware', 99 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 100 | 'django.contrib.messages.middleware.MessageMiddleware', 101 | # Uncomment the next line for simple clickjacking protection: 102 | # 'django.middleware.clickjacking.XFrameOptionsMiddleware', 103 | ) 104 | 105 | ROOT_URLCONF = 'mock_server.urls' 106 | 107 | # Python dotted path to the WSGI application used by Django's runserver. 108 | WSGI_APPLICATION = 'mock_server.wsgi.application' 109 | 110 | TEMPLATE_DIRS = ( 111 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 112 | # Always use forward slashes, even on Windows. 113 | # Don't forget to use absolute paths, not relative paths. 114 | ) 115 | 116 | INSTALLED_APPS = ( 117 | 'django.contrib.auth', 118 | 'django.contrib.contenttypes', 119 | 'django.contrib.sessions', 120 | 'django.contrib.sites', 121 | 'django.contrib.messages', 122 | 'django.contrib.staticfiles', 123 | 'mock_server', 124 | # Uncomment the next line to enable the admin: 125 | # 'django.contrib.admin', 126 | # Uncomment the next line to enable admin documentation: 127 | # 'django.contrib.admindocs', 128 | ) 129 | 130 | # A sample logging configuration. The only tangible logging 131 | # performed by this configuration is to send an email to 132 | # the site admins on every HTTP 500 error when DEBUG=False. 133 | # See http://docs.djangoproject.com/en/dev/topics/logging for 134 | # more details on how to customize your logging configuration. 135 | LOGGING = { 136 | 'version': 1, 137 | 'disable_existing_loggers': False, 138 | 'filters': { 139 | 'require_debug_false': { 140 | '()': 'django.utils.log.RequireDebugFalse' 141 | } 142 | }, 143 | 'handlers': { 144 | 'mail_admins': { 145 | 'level': 'ERROR', 146 | 'filters': ['require_debug_false'], 147 | 'class': 'django.utils.log.AdminEmailHandler' 148 | } 149 | }, 150 | 'loggers': { 151 | 'django.request': { 152 | 'handlers': ['mail_admins'], 153 | 'level': 'ERROR', 154 | 'propagate': True, 155 | }, 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /mock_server/mock_server/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Mock Server 8 | 9 | 10 | 11 | 12 | 30 | 31 | 32 |
33 |
34 | 35 |
36 | 37 | 38 | 39 | 40 | 41 | 359 | 360 | -------------------------------------------------------------------------------- /mock_server/mock_server/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | 3 | urlpatterns = patterns('', 4 | # The web page for the SPA 5 | url(r'^mock/$', 'mock_server.views.index', name='home'), 6 | # Global settings for state / proxy 7 | url(r'^mock/states/$', 'mock_server.views.states', name='states'), 8 | url(r'^mock/proxy/$', 'mock_server.views.proxy', name='proxy'), 9 | # URL settings 10 | url(r'^mock/urls/$', 'mock_server.views.urls'), 11 | url(r'^mock/conditions/$', 'mock_server.views.conditions'), 12 | url(r'^mock/responses/$', 'mock_server.views.responses'), 13 | url(r'^.*$', 'mock_server.views.urlhandler'), 14 | ) 15 | 16 | -------------------------------------------------------------------------------- /mock_server/mock_server/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | from models import * 3 | from django.http import HttpResponse, HttpResponseRedirect 4 | import json 5 | import re 6 | import urlparse 7 | 8 | tpl_404 =""" 9 | 10 | 11 | 404 not found 12 | 13 | 14 |

404 - Not Found

15 |

%s

16 | 17 | 18 | """ 19 | def mock_server_error(msg): 20 | json_msg = json.dumps({"msg": msg}) 21 | return HttpResponse(json_msg, status=500, content_type="application/json") 22 | 23 | def mock_server_success(): 24 | return HttpResponse("OK", status=200) 25 | 26 | def mock_server_404(msg): 27 | page_404 = tpl_404%msg 28 | return HttpResponse(page_404, status=404) 29 | 30 | def index(request): 31 | """ The main page for application """ 32 | return render(request, "index.html") 33 | 34 | def states(request): 35 | # CRUD for global states 36 | if request.method=="GET": 37 | states = GlobalState.objects.values() 38 | states_json = json.dumps(states) 39 | return HttpResponse(states_json, content_type="application/json") 40 | else: 41 | try: 42 | payload = json.loads(request.body) 43 | except: 44 | return mock_server_error("Fail to unmarshal json string") 45 | if request.method=="POST": 46 | # add a new state 47 | required = ["name", "value", "type"] 48 | for field in required: 49 | if field not in payload: 50 | return mock_server_error("Lacking required field: %s"%field) 51 | if payload['type']=="boolean": 52 | if payload['value'] not in ['true', 'false']: 53 | return mock_server_error("Invalid value for boolean state") 54 | 55 | new_state = GlobalState(name=payload['name'], 56 | value=payload['value'], statetype=payload['type']) 57 | new_state.save() 58 | return mock_server_success() 59 | elif request.method=="PUT": 60 | required = ["name", "value"] 61 | for field in required: 62 | if field not in payload: 63 | return mock_server_error("Lacking required field: %s"%field) 64 | try: 65 | state = GlobalState.objects.get(name=payload['name']) 66 | except: 67 | return mock_server_error("Fail to get global state: %s"%payload['name']) 68 | 69 | if state.statetype=="boolean": 70 | if payload['value'] not in ['true', 'false']: 71 | return mock_server_error("Invalid value for boolean state") 72 | state.value = payload['value'] 73 | state.save() 74 | return mock_server_success() 75 | elif request.method=="DELETE": 76 | if "name" not in payload: 77 | return mock_server_error("Lacking required field: name") 78 | try: 79 | state = GlobalState.objects.get(name=payload['name']) 80 | except: 81 | return mock_server_error("Fail to get global state: %s"%payload['name']) 82 | state.delete() 83 | return mock_server_success() 84 | else: 85 | return mock_server_error("HTTP method not supported.") 86 | 87 | def proxy(request): 88 | proxy = Proxy.objects.all() 89 | if request.method=="GET": 90 | result = { 91 | "isOn": False, 92 | "proxyserver": None 93 | } 94 | if proxy and proxy[0].isOn: 95 | result["isOn"]=True 96 | result["proxyserver"]=proxy.proxy_server 97 | result_json = json.dumps(result) 98 | return HttpResponse(result_json, content_type="application/json") 99 | elif request.method=="POST": 100 | try: 101 | payload = json.loads(request.body) 102 | except: 103 | return mock_server_error("Fail to unmarshal json string") 104 | if "isOn" not in payload: 105 | return mock_server_error("Lacking required field: isOn") 106 | if payload['isOn']=="true" and "proxyserver" not in payload: 107 | return mock_server_error("Lacking required field: proxyserver") 108 | if not proxy: 109 | p = Proxy() 110 | else: 111 | p = proxy[0] 112 | proxy.isOn = payload['isOn'] 113 | proxy.proxy_server = payload['proxyserver'] 114 | proxy.save() 115 | return mock_server_success() 116 | else: 117 | return mock_server_error("HTTP method not supported.") 118 | 119 | def urls(request): 120 | if request.method=="GET": 121 | urls = list(URL.objects.values()) 122 | urls_json = json.dumps(urls) 123 | return HttpResponse(urls_json, content_type="application/json") 124 | elif request.method=="POST": 125 | try: 126 | payload = json.loads(request.body) 127 | except: 128 | return mock_server_error("Fail to unmarshal json string") 129 | if "name" not in payload or "pattern" not in payload: 130 | return mock_server_error("Lacking required field") 131 | try: 132 | re.compile(payload['pattern']) 133 | except: 134 | return mock_server_error("invalid regular expression") 135 | url = URL(name=payload['name'], pattern=payload['pattern']) 136 | url.save() 137 | return mock_server_success() 138 | elif request.method=="DELETE": 139 | try: 140 | payload = json.loads(request.body) 141 | except: 142 | return mock_server_error("Fail to unmarshal json string") 143 | if "id" not in payload: 144 | return mock_server_error("Lacking required field:id") 145 | try: 146 | url = URL.objects.get(id=int(payload['id'])) 147 | except: 148 | return mock_server_error("URL not found") 149 | url.delete() 150 | return mock_server_success() 151 | else: 152 | return mock_server_error("HTTP method not supported.") 153 | 154 | def conditions(request): 155 | if request.method=="GET": 156 | if "urlid" not in request.GET: 157 | return mock_server_error("Lacking required field: urlid") 158 | conditions = list(Condition.objects.filter(url_id=int(request.GET['urlid'])).values()) 159 | json_conditions = json.dumps(conditions) 160 | return HttpResponse(json_conditions, content_type="application/json") 161 | elif request.method=="POST": 162 | # add a new condition, not supporting modification 163 | try: 164 | payload = json.loads(request.body) 165 | except: 166 | return mock_server_error("Fail to unmarshal json string") 167 | if "urlid" not in payload or "method" not in payload: 168 | return mock_server_error("Lacking required fields") 169 | try: 170 | url = URL.objects.get(id=int(payload['urlid'])) 171 | except: 172 | return mock_server_error("URL not found") 173 | condition = Condition(url=url, method=payload['method']) 174 | if "state_filter" in payload: 175 | condition.state_filter = payload['state_filter'] 176 | if "query_filter" in payload: 177 | condition.query_filter = payload['query_filter'] 178 | condition.save() 179 | return mock_server_success() 180 | elif request.method=="DELETE": 181 | try: 182 | payload = json.loads(request.body) 183 | except: 184 | return mock_server_error("Fail to unmarshal json string") 185 | if "id" not in payload: 186 | return mock_server_error("Lacking required field: id") 187 | try: 188 | condition = Condition.objects.get(id=int(payload['id'])) 189 | except: 190 | return mock_server_error("Condition not found") 191 | condition.delete() 192 | return mock_server_success() 193 | else: 194 | return mock_server_error("HTTP method not supported.") 195 | 196 | def responses(request): 197 | if request.method=="GET": 198 | if "conditionid" not in request.GET: 199 | return mock_server_error("Lacking required field: conditionid") 200 | try: 201 | response = Response.objects.get(condition_id=int(request.GET['conditionid'], is_inactive=False)) 202 | except: 203 | return mock_server_error("Response not found") 204 | json_response = json.dumps({ 205 | "id": response.id, 206 | "condition_id": response.condition_id, 207 | "response_type": response.response_type, 208 | "data": response.data, 209 | "status_code": response.status_code, 210 | "tpl_name": response.tpl_name, 211 | "tpl_type": response.tpl_type, 212 | "redirect_url": response.redirect_url, 213 | }) 214 | return HttpResponse(json_response, content_type="application/json") 215 | elif request.method=="POST": 216 | try: 217 | payload = json.loads(request.body) 218 | except: 219 | return mock_server_error("Fail to unmarshal json string") 220 | if "conditionid" not in payload or "response_type" not in payload: 221 | return mock_server_error("Lacking required field: conditionid") 222 | try: 223 | condition = Condition.objects.get(id=int(payload['conditionid'])) 224 | except: 225 | return mock_server_error("Condition not found") 226 | try: 227 | response = Response.objects.get(condition=condition) 228 | except: 229 | response = Response(condition=condition) 230 | response.response_type = payload['response_type'] 231 | 232 | if "status_code" in payload: 233 | response.status_code = payload['status_code'] 234 | if payload['status_code']/100==3: # redirect 235 | if "redirect_url" not in payload: 236 | return mock_server_error("Lacking required field: redirect_url") 237 | response.redirect_url = payload['redirect_url'] 238 | return mock_server_success() 239 | if "data" not in payload: 240 | return mock_server_error("Lacking required field: data") 241 | response.data = payload['data'] 242 | 243 | if payload['response_type']==1: 244 | if "tpl_name" not in payload or "tpl_type" not in payload: 245 | return mock_server_error("Lacking required field: tpl_*") 246 | response.tpl_name = payload['tpl_name'] 247 | response.tpl_type = payload['tpl_type'] 248 | return mock_server_success() 249 | 250 | # Handle the mocks 251 | def urlhandler(request): 252 | path = request.path 253 | # First try to match the path to a URL pattern 254 | urls = URL.objects.all() 255 | found = False 256 | for url in urls: 257 | mo = re.match(url.pattern, path) 258 | if mo: 259 | if mo.group(0)==path or (path[-1]=="/" and mo.group(0)==path[:-1]): 260 | found = True 261 | break 262 | elif url.pattern.endswith("/"): 263 | mo = re.match(url.pattern[:-1], path) 264 | if mo and mo.group(0)==path: 265 | found = True 266 | break 267 | if not found: 268 | return mock_server_404("URL pattern not found") 269 | 270 | found = False 271 | # find matching condition 272 | states = GlobalState.objects.all() 273 | for condition in url.condition_set: 274 | # match method 275 | method_match = (condition.method == request.method) 276 | # match the global states 277 | if not condition.state_filter: 278 | state_match = True 279 | else: 280 | state_conds = condition.state_filter.split(",").strip() 281 | names = [cond.split("=")[0] for cond in state_conds] 282 | dicts = {cond.split("=")[0]:cond.split("=")[1] for cond in state_conds} 283 | state_match = True 284 | for state in states: 285 | if state.name not in names: continue 286 | if state.value != dicts[state.name]: 287 | state_match = False 288 | break 289 | # match the query 290 | if request.method!="GET" or not condition.query_filter: 291 | query_match = True 292 | else: 293 | query_match = True 294 | qs = urlparse.parse_qs(condition.query_filter) 295 | for key in qs: 296 | if key not in request.GET: 297 | query_match = False 298 | break 299 | if request.GET[key] not in qs[key]: 300 | query_match = False 301 | break 302 | if method_match and state_match and query_match: 303 | found = True 304 | break 305 | if not found: 306 | return mock_server_404("Request Condition not found") 307 | 308 | try: 309 | response = Response.objects.get(condition=condition) 310 | except: 311 | return mock_server_404("Response not define for this condition") 312 | 313 | # record the request in database 314 | record = AccessRecord(condition=condition, fullurl=path) 315 | record.save() 316 | 317 | # deliver the response 318 | if response.status_code/100==3 and response.redirect_url: 319 | return HttpResponseRedirect(response.redirect_url, status_code=response.status_code) 320 | 321 | status_code = response.status_code if response.status_code else 200 322 | 323 | if response.response_type==2: # json 324 | return HttpResponse(response.data, content_type="application/json", status_code=status_code) 325 | 326 | elif response.response_type==1: 327 | data = json.loads(response.data) 328 | if response.tpl_type=="django": 329 | return render(request, response.tpl_name, data, status_code=status_code) 330 | # jinja here 331 | else: 332 | return mock_server_error("Template engine not supported") 333 | 334 | 335 | 336 | -------------------------------------------------------------------------------- /mock_server/mock_server/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for mock_server project. 3 | 4 | This module contains the WSGI application used by Django's development server 5 | and any production WSGI deployments. It should expose a module-level variable 6 | named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover 7 | this application via the ``WSGI_APPLICATION`` setting. 8 | 9 | Usually you will have the standard Django WSGI application here, but it also 10 | might make sense to replace the whole Django WSGI application with a custom one 11 | that later delegates to the Django one. For example, you could introduce WSGI 12 | middleware here, or combine a Django application with an application of another 13 | framework. 14 | 15 | """ 16 | import os 17 | 18 | # We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks 19 | # if running multiple sites in the same mod_wsgi process. To fix this, use 20 | # mod_wsgi daemon mode with each site in its own daemon process, or use 21 | # os.environ["DJANGO_SETTINGS_MODULE"] = "mock_server.settings" 22 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mock_server.settings") 23 | 24 | # This application object is used by any WSGI server configured to use this 25 | # file. This includes Django's development server, if the WSGI_APPLICATION 26 | # setting points here. 27 | from django.core.wsgi import get_wsgi_application 28 | application = get_wsgi_application() 29 | 30 | # Apply WSGI middleware here. 31 | # from helloworld.wsgi import HelloWorldApplication 32 | # application = HelloWorldApplication(application) 33 | -------------------------------------------------------------------------------- /online_judge/README.md: -------------------------------------------------------------------------------- 1 | # Lootcode # 2 | 3 | ## Visit online version [HERE](http://lootcode.sinaapp.com) ## 4 | 5 | Lootcode is my implementation of an online judge (OJ) system, inspired by the very well-known **Leetcode** website among programmers. It is not supposed to be feature-complete, or even bug-free, and I have no intention to make it a public service. It is just my personal python project, when trying to solve the problems involved with submitting code to execute remotely on a server. 6 | 7 | ## Features ## 8 | 9 | - Problem definition with a python Class, including problem metadata, test cases, solution validation code 10 | - Only python code submission is supported 11 | - A very simple sandbox to restrict exploit code 12 | - TLE (Time Limit Exceeded) support 13 | - 2 run modes - run submitted code inline or in another thread 14 | - 5 trivial problems for demo purposes 15 | - TBD: memory exceeded support 16 | 17 | Feel free to fork the code if you are interested, also any comments/suggestions are [welcome](mailto:real-hacker@126.com). 18 | 19 | ## My leetcode solutions in Python ## 20 | [https://github.com/RealHacker/leetcode-solutions](https://github.com/RealHacker/leetcode-solutions) 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /online_judge/html/page.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | {{title}} | LootCode OJ 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 126 | 127 | 128 | 133 |
134 |
135 |
136 | 141 | Lootcode - 撸代码 142 |
143 |
144 | 159 | 175 |
176 |
177 |
178 | 179 | 180 |
181 |
182 |
183 |
184 |
185 |

{{title}}

186 | 198 |
199 |
200 |
201 |
202 | 203 |

{% raw description %}

204 | {% autoescape xhtml_escape %} 205 | 230 | 231 |
232 |
233 |
234 |
235 |
236 |
237 |
238 |
239 |
240 |
241 | 242 | 243 |
244 | 301 | 381 | 440 | 441 | 518 |
519 | 520 |
521 |
522 |

Page Source Copied Shamelessly From LeetCode

523 |

Copyleft © 2015 NeoWang

524 |
525 | 526 | 527 | 531 | 532 | 665 | 666 | 667 | 668 | 669 | -------------------------------------------------------------------------------- /online_judge/judge.py: -------------------------------------------------------------------------------- 1 | """ 2 | The Judge class is responsible for executing the submitted 3 | source code on test cases, and return final result: pass/fail 4 | """ 5 | from problem import Problem, SolutionError 6 | import signal, os 7 | import threading 8 | 9 | TIMEOUT = 2 10 | class TimeoutError(Exception): 11 | "Raised when the execution time out" 12 | 13 | def timeout_handler(signum, frame): 14 | raise TimeoutError("Time Exceeded when executing your solution") 15 | 16 | class Judge(object): 17 | def __init__(self, problem_id, src, mode="inline"): 18 | problems = Problem.load_problemset() 19 | self.src = src 20 | self.problem = None 21 | for problem in problems: 22 | if problem._id == problem_id: 23 | self.problem = problem 24 | self.error = None 25 | if mode=="inline": 26 | self.executor = self.inline_execute 27 | else: # mode == "thread" 28 | self.executor = self.thread_execute 29 | 30 | def run_tests(self): 31 | if not self.problem: 32 | return False, "Problem not found." 33 | for raw_in, raw_out in self.problem.tests: 34 | # get input ready 35 | _input = self.problem.prepare_input(raw_in) 36 | # execute this test case 37 | try: 38 | ret, _output = self.executor(_input) 39 | except Exception as e: # A guard for uncaught exception in execution 40 | return False, str(e) 41 | else: 42 | if not ret: 43 | return ret, _output 44 | else: 45 | # Execution is successful, validate if result is correct 46 | try: 47 | correct = self.problem.validate_result(raw_in, _input, raw_out, _output) 48 | except Exception, e: 49 | return False, str(e) 50 | if not correct: 51 | error_str = "Result incorrect for input %s, your output: %s"%(raw_in, self.problem.get_output_str(_output)) 52 | return False, error_str 53 | # Now all tests passed 54 | return True, None 55 | 56 | def inline_execute(self, _in): 57 | context = {} 58 | try: 59 | code = compile(self.src, "", "exec") 60 | except SyntaxError, e: 61 | return False, str(e) 62 | try: 63 | exec code in context 64 | except Exception as e: 65 | return False, str(e) 66 | if "Solution" not in context or \ 67 | not callable(context['Solution']): 68 | return False, "Solution class not found in solution code." 69 | 70 | solution = context['Solution']() 71 | try: 72 | method = getattr(solution, self.problem.method_name) 73 | except AttributeError as e: 74 | return False, "Method %s not found"%self.problem.method_name 75 | 76 | # use alarm to monitor TLE, only *nix system support timeout alarm 77 | if hasattr(signal, "SIGALRM"): 78 | signal.signal(signal.SIGALRM, timeout_handler) 79 | signal.alarm(1) 80 | 81 | sb = Sandbox() 82 | sb.enter_sandbox() 83 | try: 84 | if isinstance(_in, tuple): 85 | _out = method(*_in) 86 | else: 87 | _out = method(_in) 88 | except TimeoutError as e: 89 | return False, "Time limit exceeded when executing your code" 90 | except Exception as e: 91 | return False, str(e) 92 | finally: 93 | sb.exit_sandbox() 94 | if hasattr(signal, "alarm"): 95 | signal.alarm(0) # cancel the alarm 96 | 97 | return True, _out 98 | 99 | def thread_execute(self, _in): 100 | # these 2 variables will be set by the thread 101 | _result = [] 102 | _exception = [] 103 | 104 | def runner(): 105 | context = {} 106 | try: 107 | code = compile(self.src, "", "exec") 108 | except SyntaxError, e: 109 | _exception.append(e) 110 | raise 111 | try: 112 | exec code in context 113 | except Exception as e: 114 | _exception.append(e) 115 | raise 116 | 117 | if "Solution" not in context or \ 118 | not callable(context['Solution']): 119 | e = SolutionError("Solution class not found in solution code.") 120 | _exception.append(e) 121 | raise e 122 | 123 | 124 | solution = context['Solution']() 125 | try: 126 | method = getattr(solution, self.problem.method_name) 127 | except AttributeError as e: 128 | _exception.append(e) 129 | raise 130 | 131 | sandbox = Sandbox() 132 | sandbox.enter_sandbox() 133 | try: 134 | if isinstance(_in, tuple): 135 | _out = method(*_in) 136 | else: 137 | _out = method(_in) 138 | except Exception as e: 139 | _exception.append(e) 140 | sandbox.exit_sandbox() 141 | raise 142 | finally: 143 | sandbox.exit_sandbox() 144 | 145 | _result.append(_out) 146 | 147 | t = threading.Thread(target=runner) 148 | t.start() 149 | t.join(TIMEOUT) 150 | if t.isAlive(): 151 | return False, "Time limit exceeded when executing your code" 152 | elif _exception: 153 | return False, str(_exception[0]) 154 | else: 155 | return True, _result[0] 156 | 157 | 158 | class SecurityError(Exception): 159 | "Raised when the execution violates sandbox restriction" 160 | 161 | class SafeModule: 162 | def __getattr__(self, attrname): 163 | return Sandbox.jail 164 | 165 | # This sandbox is inspired by this link: https://isisblogs.poly.edu/2012/10/26/escaping-python-sandboxes/ 166 | class Sandbox(object): 167 | @staticmethod 168 | def jail(*args): 169 | raise SecurityError("You are trying to violate sandbox security") 170 | 171 | def __init__(self): 172 | self._builtins = {} 173 | self.os_module = None 174 | self.subprocess_module = None 175 | 176 | def enter_sandbox(self): 177 | UNSAFE = [ 178 | 'file', 179 | 'execfile', 180 | 'compile', 181 | 'reload', 182 | 'eval', 183 | 'input' 184 | ] 185 | import __builtin__ 186 | for func in UNSAFE: 187 | self._builtins[func] = __builtin__.__dict__[func] 188 | __builtin__.__dict__[func] = self.jail 189 | 190 | import sys 191 | import os 192 | import subprocess 193 | self.os_module = sys.modules['os'] 194 | self.subprocess_module = sys.modules['subprocess'] 195 | sys.modules['os'] = SafeModule() 196 | sys.modules['subprocess'] = SafeModule() 197 | 198 | def exit_sandbox(self): 199 | import __builtin__ 200 | __builtin__.__dict__.update(self._builtins) 201 | import sys 202 | sys.modules['os'] = self.os_module 203 | sys.modules['subprocess'] = self.subprocess_module 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | -------------------------------------------------------------------------------- /online_judge/problem.py: -------------------------------------------------------------------------------- 1 | """ 2 | Every Problem should inherit from Problem class. 3 | The rationale for using python class to define problems 4 | as opposed to persistent data (JSON file, DB) is: 5 | 1. Some problems require the program to modify data structures in place, 6 | so you need code to validate its correctness. 7 | 2. For some problems, input/expected output may need customized serialization 8 | and deserialization code. 9 | 3. Sometimes the only way to validate a program is to call a correct solver 10 | and compare results. 11 | """ 12 | import operator 13 | 14 | class Problem(object): 15 | """ 16 | Each Problem subclass should define these attributes: 17 | _id: the identifier 18 | title: the title to describe the problem 19 | description: 20 | method_name: the method name within Solution 21 | tests: a list of test cases, each test case is 22 | a tuple (input, expected_output) 23 | expected_output can be None if no output is expeced 24 | """ 25 | def prepare_input(self, raw_in): 26 | # Override this when the test input needs preparation 27 | # (like deserialization) before passing to program 28 | return raw_in 29 | 30 | def validate_result(self, raw_in, real_in, expected_out, real_out): 31 | # The validation depends on the problem: 32 | # 1. Validate real_out == expected_out (may need deserialization) 33 | # 2. Validate read_in is modified correctly 34 | # 3. May need a real solver to run, and compare results 35 | # Here is the default validation, override if needed 36 | return expected_out == real_out 37 | 38 | def get_output_str(self, real_out): 39 | # Get a printable string for program output 40 | # used when the output is incorrect 41 | return str(real_out) 42 | 43 | @classmethod 44 | def load_problemset(cls): 45 | import problemset 46 | problem_classes = cls.__subclasses__() 47 | problems = [problem_cls() for problem_cls in problem_classes] 48 | problems.sort(key=operator.attrgetter("_id")) 49 | return problems 50 | 51 | @classmethod 52 | def get_problem_by_id(cls, _id): 53 | import problemset 54 | problem = None 55 | for pcls in cls.__subclasses__(): 56 | if pcls._id == _id: 57 | problem = pcls() 58 | break 59 | return problem 60 | 61 | 62 | 63 | class SolutionError(Exception): 64 | """ 65 | Solution Error with an string argument to show reason 66 | """ 67 | -------------------------------------------------------------------------------- /online_judge/problemset/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import os, sys 3 | 4 | cur_path = os.path.dirname(os.path.realpath(__file__)) 5 | parentdir = os.path.dirname(cur_path) 6 | sys.path.insert(0,parentdir) 7 | 8 | modules = ["."+os.path.splitext(module_file)[0] for module_file in os.listdir(cur_path)] 9 | for module in modules: 10 | if "__init__" not in module: 11 | importlib.import_module(module, __package__) 12 | -------------------------------------------------------------------------------- /online_judge/problemset/problem_fib.py: -------------------------------------------------------------------------------- 1 | from problem import Problem 2 | 3 | class FibProblem(Problem): 4 | _id = 2 5 | title = "Fibonacci numbers" 6 | description = "Given integer n, return the first n Fibonacci numbers in a list" 7 | method_name = "fib" 8 | args = ("n",) 9 | tests = [ 10 | (1, [1]), (2, [1,1]), (5, [1,1, 2,3,5]), 11 | (28, [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597, 2584, 4181 12 | , 6765, 10946, 17711, 28657, 46368, 75025, 121393, 196418, 317811]) 13 | ] 14 | -------------------------------------------------------------------------------- /online_judge/problemset/problem_hello.py: -------------------------------------------------------------------------------- 1 | from problem import Problem 2 | 3 | class HelloProblem(Problem): 4 | """ 5 | Awefully simple problem, no need to override any method 6 | """ 7 | _id = 1 8 | title = "Hello Problem" 9 | description = "Just return 'Hello ' followed by the name passed in." 10 | "I know, it's silly, just to prove everything works" 11 | method_name = "hello" 12 | args = ("name",) 13 | tests = [("Foo", "Hello Foo"), 14 | ("Bar", "Hello Bar")] 15 | 16 | -------------------------------------------------------------------------------- /online_judge/problemset/problem_panlindrome.py: -------------------------------------------------------------------------------- 1 | from problem import Problem 2 | 3 | 4 | class PalindromeProblem(Problem): 5 | _id = 5 6 | title = "Palindrome Or Not" 7 | description = "A palindrome is a word, phrase, number,\ 8 | or other sequence of characters which reads the same backward or forward.\ 9 | return True if the supplied word is palindrome, else return False" 10 | method_name = "is_palindrome" 11 | args = ("word",) 12 | tests = [ 13 | ("abcdcba", True), 14 | ("a", True), 15 | ("ABCba", False), 16 | ("aaaaa", True), 17 | ("aabaaa", False), 18 | ] 19 | -------------------------------------------------------------------------------- /online_judge/problemset/problem_reverselist.py: -------------------------------------------------------------------------------- 1 | from problem import Problem, SolutionError 2 | from utils import LISTNODE_DEF, make_linked_list, cmp_linked_list, dump_linked_list 3 | 4 | class ReverseListProblem(Problem): 5 | _id = 3 6 | title = "Reverse Linked List" 7 | description = "Given a linked list, reverse it and return the head node."+\ 8 | "
"+ LISTNODE_DEF 9 | method_name = "reverse_linked_list" 10 | args = ("head",) 11 | tests = [ 12 | ([1], [1]), 13 | ([1, 2, 3], [3, 2, 1]), 14 | ([1, 2, 2, 4, 4, 3], [3, 4, 4, 2, 2, 1]), 15 | (range(1, 1000), range(999, 0, -1)) 16 | ] 17 | 18 | def prepare_input(self, raw_in): 19 | # make a linked list to send it in 20 | return make_linked_list(raw_in) 21 | 22 | def validate_result(self, raw_in, real_in, expected_out, real_out): 23 | cmp_result = cmp_linked_list(real_out, expected_out) 24 | # make sure the solution doesn't use new nodes 25 | if real_in: 26 | p = real_out 27 | while p.next: 28 | p = p.next 29 | if p != real_in: 30 | raise SolutionError, "You instantiated new nodes!" 31 | return cmp_result 32 | 33 | def get_output_str(self, real_out): 34 | return str(dump_linked_list(real_out)) 35 | -------------------------------------------------------------------------------- /online_judge/problemset/problem_sort.py: -------------------------------------------------------------------------------- 1 | from problem import Problem 2 | 3 | 4 | class SortProblem(Problem): 5 | _id = 4 6 | title = "Sort Numbers Array" 7 | description = "Given an array of integers, sort it ascendingly and return the resulting list." 8 | method_name = "sort_array" 9 | args = ("nums",) 10 | tests = [ 11 | ([1], [1]), 12 | ([1, 3, 2,4, 5], [1,2, 3,4,5]), 13 | ([5, 5, 5, 5, 5, 1], [1, 5, 5, 5, 5, 5]), 14 | (range(999, 0, -1), range(1, 1000)) 15 | ] 16 | -------------------------------------------------------------------------------- /online_judge/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module defines a few data structures and utility functions 3 | for use in problem solution and solver 4 | """ 5 | 6 | LISTNODE_DEF = """ 7 | 
 8 | # Definition for singly-linked list node.
 9 | # class ListNode:
10 | #     def __init__(self, x):
11 | #         self.val = x
12 | #         self.next = None
13 |
14 | """ 15 | 16 | class ListNode(object): 17 | """ 18 | ListNode has a next pointer and a val field. 19 | """ 20 | def __init__(self, x): 21 | self.val = x 22 | self.next = None 23 | 24 | def make_linked_list(numbers): 25 | next = None 26 | node = None 27 | for n in reversed(numbers): 28 | node = ListNode(n) 29 | node.next = next 30 | next = node 31 | return node 32 | 33 | def dump_linked_list(llist): 34 | numbers = [] 35 | while llist: 36 | numbers.append(llist.val) 37 | llist = llist.next 38 | return numbers 39 | 40 | def cmp_linked_list(llist, numbers): 41 | node = llist 42 | for number in numbers: 43 | if not node or node.val != number: 44 | return False 45 | node = node.next 46 | return not node 47 | 48 | -------------------------------------------------------------------------------- /online_judge/web.py: -------------------------------------------------------------------------------- 1 | import tornado 2 | import tornado.web 3 | import tornado.template 4 | from problem import Problem 5 | from judge import Judge 6 | import json 7 | 8 | class ProblemHandler(tornado.web.RequestHandler): 9 | def get(self, pid=1): 10 | p = Problem.get_problem_by_id(int(pid)) 11 | if not p: 12 | # TODO: a 404 page 13 | self.write("404 - Problem not found.") 14 | else: 15 | loader = tornado.template.Loader("html") 16 | html = loader.load("page.html").generate( 17 | _id=p._id, 18 | title=p.title, 19 | description= p.description, 20 | method_name = p.method_name, 21 | args = p.args 22 | ) 23 | self.write(html) 24 | 25 | def post(self): 26 | try: 27 | payload = json.loads(self.request.body) 28 | except ValueError: 29 | raise 30 | if "id" not in payload or 'src' not in payload: 31 | raise Exception("Argument missing") 32 | pid = int(payload['id']) 33 | src = payload['src'] 34 | judge = Judge(pid, src, mode="inline") 35 | ok, msg = judge.run_tests() 36 | 37 | if not ok: 38 | ret = json.dumps({"pass":False, "msg": msg}) 39 | else: 40 | ret = json.dumps({"pass":True}) 41 | self.set_header("content-type", "application/json") 42 | self.write(ret) 43 | 44 | import tornado.wsgi 45 | application = tornado.wsgi.WSGIApplication([ 46 | (r"/", ProblemHandler), 47 | ]) 48 | #import sae 49 | #application = sae.create_wsgi_app(app) 50 | 51 | def debug(): 52 | application = tornado.web.Application([ 53 | (r"/", ProblemHandler), 54 | (r"/([0-9]+)/", ProblemHandler), 55 | ]) 56 | application.listen(8888) 57 | tornado.ioloop.IOLoop.current().start() 58 | 59 | if __name__ == "__main__": 60 | debug() 61 | -------------------------------------------------------------------------------- /open_todo.py: -------------------------------------------------------------------------------- 1 | from datetime import date 2 | import os 3 | 4 | # A simple script to open a new markdown todo file every day. 5 | 6 | year = date.today().year 7 | month = date.today().strftime('%b').lower() 8 | day = date.today().day 9 | 10 | filename = os.path.join(str(year) + '_' + month, str(day) + '.md') 11 | try: 12 | os.mkdir(os.path.abspath(str(year) + '_' + month)) 13 | except: 14 | pass 15 | file = open(filename, 'a') 16 | file.close() 17 | 18 | # replace this line with your way of opening markown files 19 | os.system('start typora ' + filename) 20 | 21 | # you could also just update a symbolic link and reference that link while opening your editor of choice 22 | -------------------------------------------------------------------------------- /patch_module.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | import importlib 3 | import sys 4 | 5 | class ModulePatchRegister: 6 | register = defaultdict(list) 7 | @classmethod 8 | def register_patch(cls, mod_name, func): 9 | cls.register[mod_name].append(func) 10 | 11 | @classmethod 12 | def is_module_patched(cls, name): 13 | return name in cls.register 14 | 15 | @classmethod 16 | def get_module_patches(cls, name): 17 | return cls.register[name] 18 | 19 | class PatchMetaPathFinder: 20 | def __init__(self): 21 | self.skip = set() 22 | 23 | def find_module(self, name, path): 24 | if name in self.skip: 25 | return None 26 | self.skip.add(name) 27 | return PatchModuleLoader(self) 28 | 29 | class PatchModuleLoader: 30 | def __init__(self, finder): 31 | self._finder = finder 32 | 33 | def load_module(self, name): 34 | mod = importlib.import_module(name) 35 | if ModulePatchRegister.is_module_patched(name): 36 | for patch in ModulePatchRegister.get_module_patches(name): 37 | patch(mod) 38 | self._finder.skip.remove(name) 39 | return mod 40 | 41 | sys.meta_path.insert(0, PatchMetaPathFinder()) 42 | 43 | def when_importing(modname): 44 | def decorated(func): 45 | if modname in sys.modules: 46 | func(sys.modules[modname]) 47 | else: 48 | ModulePatchRegister.register_patch(modname, func) 49 | return decorated 50 | 51 | 52 | # For demo purpose 53 | @when_importing("threading") 54 | def warn(mod): 55 | print "Warning, you are entering dangerous territory!" 56 | 57 | @when_importing("math") 58 | def new_math(mod): 59 | def new_abs(num): 60 | return num if num<0 else -num 61 | mod.abs = new_abs 62 | -------------------------------------------------------------------------------- /quine.py: -------------------------------------------------------------------------------- 1 | s = 's = %r\nprint(s%%s)' 2 | print(s%s) 3 | -------------------------------------------------------------------------------- /rpc.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is a RPC server/client implementation using socket 3 | * It authenticate the client with a secret shared by client/server 4 | * It uses JSON to serialize the function call payload 5 | """ 6 | import socket 7 | import random, string 8 | import hmac 9 | import json 10 | import threading 11 | 12 | secret = "SECRET" 13 | 14 | class RPCHandler: 15 | def __init__(self, secret): 16 | self._secret = secret 17 | self._register = {} 18 | 19 | def register_func(self, func): 20 | self._register[func.__name__] = func 21 | 22 | def handle_call(self, sock): 23 | keymsg = ''.join([random.choice(string.lowercase) for i in range(8)]) 24 | sock.sendall(keymsg) 25 | hash = hmac.new(self._secret, keymsg) 26 | digest = hash.digest() 27 | response = sock.recv(512) 28 | if response != digest: 29 | sock.sendall("Authentication Failed!") 30 | sock.close() 31 | else: 32 | sock.sendall("Authenticated!") 33 | try: 34 | while True: 35 | req = sock.recv(512) 36 | d = json.loads(req) 37 | funcname = d["name"] 38 | args = d["args"] 39 | kwargs = d["kwargs"] 40 | print "Client calling %s(%s, %s)"%(funcname, args, kwargs) 41 | try: 42 | ret = self._register[funcname](*args, **kwargs) 43 | sock.sendall(json.dumps({"ret": ret})) 44 | except Exception as e: 45 | sock.sendall(json.dumps({"exception": str(e)})) 46 | except EOFError: 47 | print "Closing RPC Handler..." 48 | 49 | handler = RPCHandler(secret) 50 | 51 | class RPCServer: 52 | def __init__(self, address): 53 | self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 54 | self._sock.bind(address) 55 | 56 | def serve_forever(self): 57 | self._sock.listen(0) 58 | while True: 59 | client_sock,_ = self._sock.accept() 60 | thread = threading.Thread(target=handler.handle_call, args=(client_sock, )) 61 | thread.daemon = True 62 | thread.start() 63 | 64 | class RPCProxy(object): 65 | def __init__(self, address, secret): 66 | self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 67 | self._sock.connect(address) 68 | msg = self._sock.recv(512) 69 | h = hmac.new(secret, msg) 70 | self._sock.sendall(h.digest()) 71 | print self._sock.recv(512) 72 | 73 | def __getattr__(self, name): 74 | def proxy_func(*args, **kwargs): 75 | payload = { 76 | "name": name, 77 | "args": args, 78 | "kwargs": kwargs 79 | } 80 | self._sock.sendall(json.dumps(payload)) 81 | result = json.loads(self._sock.recv(512)) 82 | if "exception" in result: 83 | raise Exception(result['exception']) 84 | else: 85 | return result['ret'] 86 | if name.startswith("_"): 87 | return super(RPCProxy, self).__getattr__(name) 88 | else: 89 | return proxy_func 90 | 91 | -------------------------------------------------------------------------------- /timeit.py: -------------------------------------------------------------------------------- 1 | # This is an implementation of timeit module, as a practice 2 | import itertools 3 | import gc 4 | 5 | _template = """ 6 | def run_timer(it, _timer): 7 | %s 8 | _start = _timer.time() 9 | for _i in it: 10 | %s 11 | _end = _timer.time() 12 | return _end - _start 13 | """ 14 | 15 | def get_runner(f, _setup): 16 | def run_timer(it, _timer): 17 | _setup() 18 | _start = _timer.time() 19 | for _i in it: 20 | f() 21 | _end = _timer.time() 22 | return _end - _start 23 | return run_timer 24 | 25 | class Timer(object): 26 | def __init__(self, target="pass", setup="pass"): 27 | self._target = target 28 | self._setup = setup 29 | ns = {} 30 | if isinstance(target, basestring): 31 | if isinstance(setup, basestring): 32 | src = _template%(setup, target) 33 | elif callable(setup): 34 | src = _template%("_setup()", target) 35 | ns["_setup"] = setup 36 | else: 37 | raise ValueError("Needs a callable or python statement") 38 | # print src 39 | code = compile(src, "_dummy_", "exec") 40 | exec code in globals(), ns 41 | self.runner = ns['run_timer'] 42 | elif callable(target): 43 | if isinstance(setup, basestring): 44 | def temp(): 45 | exec setup in globals(), ns 46 | self.runner = get_runner(target, temp) 47 | elif callable(setup): 48 | self.runner = get_runner(target, setup) 49 | else: 50 | raise ValueError("target must be statement for callable") 51 | 52 | def timeit(self, times): 53 | import time 54 | it = itertools.repeat(None, times) 55 | gc_on = gc.isenabled() 56 | gc.disable() 57 | try: 58 | duration = self.runner(it, time) 59 | finally: 60 | if gc_on: 61 | gc.enable() 62 | return duration 63 | 64 | def repeat(self, cycles, times): 65 | t = [] 66 | for i in xrange(cycles): 67 | tt = self.timeit(times) 68 | t.append(tt) 69 | return t 70 | 71 | def timeit(target, setup, times): 72 | timer = Timer(target, setup) 73 | return timer.timeit(times) 74 | 75 | def repeat(target, setup, times, cycles): 76 | timer = Timer(target, setup) 77 | return timer.repeat(cycles, times) 78 | 79 | 80 | def main(): 81 | stmt1 = "a = 2+4" 82 | stmt2 = "test(1)" 83 | setup2 = "test = lambda x:x*2" 84 | def test_func(): 85 | global x 86 | x = x*2 87 | 88 | timer = Timer(stmt1) 89 | print stmt1, timer.timeit(100000) 90 | 91 | timer = Timer(stmt2, setup2) 92 | print stmt2, timer.timeit(10000) 93 | 94 | timer = Timer(test_func, "global x;x=1") 95 | print "func1", timer.timeit(10000) 96 | 97 | print "func2", timeit("i,j=m, n", "m=1;n=2", 10000) 98 | 99 | if __name__=="__main__": 100 | main() 101 | 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /web_terminal/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 22 | Python web terminal 23 | 24 | 25 | 26 |
27 |

Welcome to python terminal, please enter your command:

28 | 29 | 30 |
31 | 53 | 54 | -------------------------------------------------------------------------------- /web_terminal/web_terminal.py: -------------------------------------------------------------------------------- 1 | import BaseHTTPServer 2 | import json 3 | import subprocess 4 | 5 | page404 = """ 6 | 7 | 8 | PAGE NOT FOUND 9 | 10 | 11 |

404 - Page not found

12 | 13 | 14 | """ 15 | 16 | class TerminalHandler(BaseHTTPServer.BaseHTTPRequestHandler): 17 | def do_GET(self): 18 | if self.path=="/": 19 | with open("index.html", 'r') as f: 20 | pagestr = f.read() 21 | self.send_response(200) 22 | self.send_header("Content-type", "text/html; charset=utf-8") 23 | self.send_header("Content-length", str(len(pagestr))) 24 | self.end_headers() 25 | self.wfile.write(pagestr) 26 | else: 27 | self.send_error(404, "File not found") 28 | # self.wfile.write(page404) 29 | 30 | def do_POST(self): 31 | if self.path=="/cmd": 32 | body = self.rfile.read(int(self.headers['Content-Length'])) 33 | print body 34 | try: 35 | request = json.loads(body) 36 | except: 37 | self.send_error(500, "command format error") 38 | else: 39 | cmdline = request.get("cmd", None) 40 | runner = CommandRunner(cmdline) 41 | output = runner.get_output() 42 | ret = json.dumps({"result":str(output)}) 43 | self.send_response(200) 44 | self.send_header("Content-type", "application/json; charset=utf-8") 45 | self.end_headers() 46 | self.wfile.write(ret) 47 | else: 48 | self.send_error(404, "File not found") 49 | 50 | class CommandRunner(object): 51 | def __init__(self, cmd): 52 | self._cmds = cmd.split() 53 | 54 | def get_output(self): 55 | print "COMMANDS:", self._cmds 56 | return subprocess.check_output(self._cmds, shell=True) 57 | 58 | server = BaseHTTPServer.HTTPServer(("127.0.0.1", 8000), TerminalHandler) 59 | server.serve_forever() 60 | --------------------------------------------------------------------------------