├── .hgignore ├── README.txt ├── remoteexec.py ├── setup.py └── test.py /.hgignore: -------------------------------------------------------------------------------- 1 | ^dist/ 2 | ^MANIFEST$ 3 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | Remote Exec lets you ship python code to a remote machine and run it 2 | there, all without installing anything other than the standard Python 3 | interpreter on the server. 4 | 5 | It connects to the remote host using SSH, sends the python files you 6 | specify, compiles them on the server, and passes control to the 7 | specified main function. 8 | 9 | Additionally, the client arranges for stdin/stdout on the server side 10 | to be connected to a network socket on the client side, so that you 11 | can communicate with the uploaded server binary as if you'd connected 12 | to it normally. 13 | 14 | What this lets you do is ensure that the server end of your software 15 | is never out of date, since it'll always be in sync with the client 16 | code you're running. 17 | 18 | The original idea came from Avery Pennarun's awesome sshuttle 19 | (http://github.com/apenwarr/sshuttle), which uses this great hack to 20 | function with any server that has Python installed (i.e. all of 21 | them). 22 | 23 | His original version smushed all files into a single namespace, 24 | however while I was factoring it out of sshuttle he came up with the 25 | compile/eval/__dict__.update hack that allows reconstruction of the 26 | original module structure on the server, neatly avoiding the need for 27 | the skip_imports hack sshuttle uses. I took it from there and 28 | implemented it as a reusable library. 29 | -------------------------------------------------------------------------------- /remoteexec.py: -------------------------------------------------------------------------------- 1 | def _load(verbose): 2 | import imp 3 | import sys 4 | import zlib 5 | def _log(s): 6 | sys.stderr.write(s+'\n') 7 | sys.stderr.flush() 8 | if verbose: 9 | log = _log 10 | else: 11 | log = lambda _: None 12 | log('Assembler started') 13 | files = {} 14 | decomp = zlib.decompressobj() 15 | while True: 16 | name = sys.stdin.readline().strip() 17 | if not name: 18 | break 19 | log('Reading module ' + name) 20 | n = int(sys.stdin.readline()) 21 | files[name] = decomp.decompress(sys.stdin.read(n)) 22 | log('Read module %s (%d compressed, %d decompressed)' 23 | % (name, n, len(files[name]))) 24 | log('Finished reading modules, starting compilation') 25 | while files: 26 | l = len(files) 27 | for name in list(files.keys()): 28 | log('Compiling ' + name) 29 | code = compile(files[name], name, 'exec') 30 | d = {} 31 | try: 32 | eval(code, d, d) 33 | except ImportError: 34 | log("Can't compile %s yet, needs other modules" % name) 35 | continue 36 | mod = imp.new_module(name) 37 | mod.__dict__.update(d) 38 | sys.modules[name] = mod 39 | del files[name] 40 | log('Compiled and loaded ' + name) 41 | if len(files) == l: 42 | _log("Infinite compile loop, you're probably missing an import") 43 | exit(1) 44 | del sys.modules[__name__].__dict__['_load'] 45 | sys.stdout.flush() 46 | sys.stderr.flush() 47 | module, func = sys.stdin.readline().strip().rsplit('.', 2) 48 | log('All code loaded, sending sync string') 49 | sys.stdout.write('\0REINDEERFLOTILLA0101') 50 | sys.stdout.flush() 51 | log('Sync sent, handing off to %s.%s()' % (module, func)) 52 | sys.modules[module].__dict__[func]() 53 | # END ASSEMBLER 54 | # The above is the stage2 assembler that gets run on the remote 55 | # system. To ensure that syntax errors and exceptions have useful line 56 | # numbers, keep it at the top of the file. 57 | 58 | import os 59 | import os.path 60 | import socket 61 | import subprocess 62 | import zlib 63 | 64 | # Stage 1 assembler, compiles and executes stage2. 65 | _STAGE1 = ''' 66 | import sys; 67 | exec compile(sys.stdin.read(%d), "assembler.py", "exec") 68 | ''' 69 | _SYNC_STRING = 'REINDEERFLOTILLA0101' 70 | 71 | def _readfile(filename): 72 | f = open(filename) 73 | try: 74 | return f.read() 75 | finally: 76 | f.close() 77 | 78 | def _pack(filenames, literal_modules, main_func): 79 | out = [] 80 | compress = zlib.compressobj(9) 81 | for filename in filenames: 82 | _, basename = os.path.split(filename) 83 | assert basename[-3:] == '.py' 84 | source = compress.compress(_readfile(filename)) 85 | source += compress.flush(zlib.Z_SYNC_FLUSH) 86 | out.append('%s\n%d\n%s' % (basename[:-3], len(source), source)) 87 | for name, source in literal_modules.iteritems(): 88 | source = compress.compress(source) 89 | source += compress.flush(zlib.Z_SYNC_FLUSH) 90 | out.append('%s\n%d\n%s' % (name, len(source), source)) 91 | out.append('\n%s\n' % main_func) 92 | return ''.join(out) 93 | 94 | def _get_assembler(verbose=False): 95 | filename = __file__ 96 | if filename.endswith('.pyc'): 97 | filename = filename[:-1] 98 | source = _readfile(filename) 99 | assembler = source.split('# END ASSEMBLER\n')[0] 100 | return '%s\n_load(%s)\n' % (assembler, verbose) 101 | 102 | # style guide 103 | 104 | class Fatal(Exception): 105 | pass 106 | 107 | def _sync(p, s): 108 | z = 'x' 109 | while z and z != '\0': 110 | z = s.recv(1) 111 | sync = s.recv(len(_SYNC_STRING)) 112 | 113 | ret = p.poll() 114 | if ret: 115 | raise Fatal('server died with error code %d' % ret) 116 | 117 | if sync != _SYNC_STRING: 118 | raise Fatal('expected sync string %s, got %s' % (_SYNC_STRING, sync)) 119 | 120 | def remote_exec(hostname=None, user=None, port=22, 121 | ssh_cmd=None, module_filenames=None, 122 | literal_modules=None, main_func=None, 123 | verbose_load=False): 124 | if not ssh_cmd: 125 | if user: 126 | user = user + '@' 127 | else: 128 | user = '' 129 | ssh_cmd = ['ssh', '-p', str(port), '%s%s' % (user, hostname)] 130 | main = _pack(module_filenames or [], 131 | literal_modules or {}, 132 | main_func or 'main.main') 133 | stage2 = _get_assembler(verbose_load) 134 | stage1 = _STAGE1 % len(stage2) 135 | pycmd = ("P=python2; $P -V 2>/dev/null || P=python; " 136 | "exec \"$P\" -c '%s'") % stage1 137 | cmd = ssh_cmd + ['--', pycmd] 138 | 139 | (s1,s2) = socket.socketpair() 140 | sla,slb = os.dup(s1.fileno()), os.dup(s1.fileno()) 141 | s1.close() 142 | def setup(): 143 | s2.close() 144 | p = subprocess.Popen(cmd, stdin=sla, stdout=slb, 145 | preexec_fn=setup, close_fds=True) 146 | try: 147 | os.close(sla) 148 | os.close(slb) 149 | s2.sendall(stage2) 150 | s2.sendall(main) 151 | _sync(p, s2) 152 | return p, s2 153 | except: 154 | if not p.poll(): 155 | os.kill(p.pid, 9) 156 | p.wait() 157 | raise 158 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from distutils.core import setup 4 | 5 | setup( 6 | name = 'remoteexec', 7 | version = '1.0.0', 8 | author = 'David Anderson', 9 | py_modules = ['remoteexec'], 10 | author_email = 'dave@natulte.net', 11 | license = 'Apache License, Version 2.0', 12 | url = 'http://bitbucket.org/danderson/py-remoteexec', 13 | classifiers = [ 14 | 'Development Status :: 5 - Production/Stable', 15 | 'Intended Audience :: Developers', 16 | 'License :: OSI Approved :: Apache Software License', 17 | 'Operating System :: OS Independent', 18 | 'Programming Language :: Python :: 2', 19 | 'Programming Language :: Python', 20 | 'Topic :: Software Development :: Libraries :: Python Modules', 21 | ]) 22 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import socket 4 | import sys 5 | 6 | mods = { 7 | 'test1': '\n'.join([ 8 | 'import test', 9 | 'import test2', 10 | 'def print_stuff():', 11 | ' print "Hello from test1"', 12 | ' print test2.hello', 13 | ' print test.mods', 14 | ' test.hostname()', 15 | ]), 16 | 'test2': '\n'.join([ 17 | 'hello = "Hello from test2"', 18 | ]), 19 | } 20 | 21 | def hostname(): 22 | print socket.gethostname() 23 | 24 | if __name__ == '__main__': 25 | import remoteexec 26 | p, s = remoteexec.remote_exec( 27 | hostname=sys.argv[1], 28 | module_filenames=['test.py', 'remoteexec.py'], 29 | literal_modules=mods, 30 | main_func='test1.print_stuff', 31 | verbose_load=True) 32 | f = s.makefile('r') 33 | s.close() 34 | print f.read() 35 | --------------------------------------------------------------------------------