├── sshspawner ├── __init__.py └── sshspawner.py ├── requirements.txt ├── .gitignore ├── version.py ├── jupyterhub_config.py ├── scripts └── get_port.py ├── README.md ├── LICENSE └── setup.py /sshspawner/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | asyncssh 2 | jupyterhub 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.sublime-* 2 | *.pyc 3 | *.egg-info 4 | __pycache__/ 5 | .DS_Store 6 | -------------------------------------------------------------------------------- /version.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Jupyter Development Team. 2 | # Distributed under the terms of the Modified BSD License. 3 | 4 | version_info = ( 5 | 0, 6 | 1, 7 | 0, 8 | ) 9 | __version__ = '.'.join(map(str, version_info[:3])) 10 | 11 | if len(version_info) > 3: 12 | __version__ = '%s-%s' % (__version__, version_info[3]) 13 | -------------------------------------------------------------------------------- /jupyterhub_config.py: -------------------------------------------------------------------------------- 1 | # Sample ConfigurationS 2 | c.JupyterHub.spawner_class = 'sshspawner.sshspawner.SSHSpawner' 3 | 4 | # The remote host to spawn notebooks on 5 | c.SSHSpawner.remote_hosts = ['cori19-224.nersc.gov'] 6 | c.SSHSpawner.remote_port = '2222' 7 | c.SSHSpawner.ssh_command = 'ssh' 8 | 9 | # The system path for the remote SSH session. Must have the jupyter-singleuser and python executables 10 | c.SSHSpawner.path = '/global/common/cori/software/python/3.5-anaconda/bin:/global/common/cori/das/jupyterhub/:/usr/common/usg/bin:/usr/bin:/bin:/usr/bin/X11:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin' 11 | 12 | # The command to return an unused port on the target system. See scripts/get_port.py for an example 13 | c.SSHSpawner.remote_port_command = '/usr/bin/python /global/common/cori/das/jupyterhub/get_port.py' 14 | 15 | -------------------------------------------------------------------------------- /scripts/get_port.py: -------------------------------------------------------------------------------- 1 | 2 | import argparse 3 | import socket 4 | 5 | def main(): 6 | args = parse_arguments() 7 | if args.ip: 8 | print("{} {}".format(port(), ip())) 9 | else: 10 | print(port()) 11 | 12 | def parse_arguments(): 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument("--ip", "-i", 15 | help="Include IP address in output", 16 | action="store_true") 17 | return parser.parse_args() 18 | 19 | def port(): 20 | s = socket.socket() 21 | s.bind(('', 0)) 22 | port = s.getsockname()[1] 23 | s.close() 24 | return port 25 | 26 | def ip(address=("8.8.8.8", 80)): 27 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 28 | s.connect(address) 29 | ip = s.getsockname()[0] 30 | s.close() 31 | return ip 32 | 33 | if __name__ == "__main__": 34 | main() 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # sshspawner 3 | 4 | The *sshspawner* enables JupyterHub to spawn single-user notebook servers on remote hosts over SSH. 5 | We provide this package as a reference implementation only, the authors offer no general user support. 6 | 7 | ## Features 8 | 9 | * Supports SSH key-based authentication 10 | * Pool of remote hosts for spawning notebook servers 11 | * Extensible custom load-balacing for remote host pool 12 | * Remote-side scripting to return IP and port 13 | 14 | ## Requirements 15 | 16 | * Python 3 17 | * [JupyterHub](http://jupyter.org/install) 18 | * [AsyncSSH](https://asyncssh.readthedocs.io/en/latest/#installation) 19 | 20 | ## Installation 21 | 22 | ``` 23 | python3 setup.py install 24 | ``` 25 | 26 | Install [scripts/get_port.py](scripts/get_port.py) on remote host and set correct path for `c.SSHSpawner.remote_port_command` in [jupyterhub_config.py](jupyterhub_config.py) 27 | 28 | ## Configuration 29 | 30 | See [jupyterhub_config.py](jupyterhub_config.py) for a sample configuration. 31 | Adjust values for your installation. 32 | 33 | ## License 34 | 35 | All code is licensed under the terms of the revised BSD license. 36 | 37 | ## Resources 38 | 39 | - [Documentation for JupyterHub](https://jupyterhub.readthedocs.io) 40 | - [Documentation for Project Jupyter](https://jupyter.readthedocs.io/en/latest/index.html) | [PDF](https://media.readthedocs.org/pdf/jupyter/latest/jupyter.pdf) 41 | - [Project Jupyter website](https://jupyter.org) 42 | 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, The Regents of the University of California, 2 | through Lawrence Berkeley National Laboratory. All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of the University of California, Lawrence Berkeley National 15 | Laboratory, U.S. Dept. of Energy nor the names of its contributors may be 16 | used to endorse or promote products derived from this software without 17 | specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | # 4 | # Copyright (c) Juptyer Development Team. 5 | # Distributed under the terms of the Modified BSD License. 6 | # 7 | # ---------------------------------------------------------------------------- 8 | # Minimal Python version sanity check (from IPython/Jupyterhub) 9 | # ---------------------------------------------------------------------------- 10 | from __future__ import print_function 11 | 12 | import os 13 | import sys 14 | 15 | from distutils.core import setup 16 | 17 | pjoin = os.path.join 18 | here = os.path.abspath(os.path.dirname(__file__)) 19 | 20 | # Get the current package version. 21 | version_ns = {} 22 | with open(pjoin(here, 'version.py')) as f: 23 | exec(f.read(), {}, version_ns) 24 | 25 | setup_args = dict( 26 | name='sshspawner', 27 | packages=['sshspawner'], 28 | version=version_ns['__version__'], 29 | description="""SSH Spawner: A custom spawner for Jupyterhub to spawn 30 | notebooks over SSH""", 31 | long_description="""Spawn Jupyter notebooks on a remote node over SSH. Supports 32 | SSH Key based authentication.""", 33 | author="Shane Canon, Shreyas Cholia, William Krinsman, Kelly L. Rowland, Rollin Thomas", 34 | author_email="scanon@lbl.gov, scholia@lbl.gov, krinsman@berkeley.edu, kellyrowland@lbl.gov, rcthomas@lbl.gov", 35 | url="http://www.nersc.gov", 36 | license="BSD", 37 | platforms="Linux, Mac OS X", 38 | keywords=['Interactive', 'Interpreter', 'Shell', 'Web'], 39 | classifiers=[ 40 | 'Intended Audience :: Developers', 41 | 'Intended Audience :: System Administrators', 42 | 'Intended Audience :: Science/Research', 43 | 'License :: OSI Approved :: BSD License', 44 | 'Programming Language :: Python', 45 | 'Programming Language :: Python :: 3', 46 | ], 47 | ) 48 | 49 | # setuptools requirements 50 | if 'setuptools' in sys.modules: 51 | setup_args['install_requires'] = install_requires = [] 52 | with open('requirements.txt') as f: 53 | for line in f.readlines(): 54 | req = line.strip() 55 | if not req or req.startswith(('-e', '#')): 56 | continue 57 | install_requires.append(req) 58 | 59 | 60 | def main(): 61 | setup(**setup_args) 62 | 63 | if __name__ == '__main__': 64 | main() 65 | -------------------------------------------------------------------------------- /sshspawner/sshspawner.py: -------------------------------------------------------------------------------- 1 | import asyncio, asyncssh 2 | import os 3 | from textwrap import dedent 4 | import warnings 5 | import random 6 | import pwd 7 | import shutil 8 | from tempfile import TemporaryDirectory 9 | 10 | from traitlets import Bool, Unicode, Integer, List, observe, default 11 | from jupyterhub.spawner import Spawner 12 | 13 | 14 | class SSHSpawner(Spawner): 15 | 16 | # http://traitlets.readthedocs.io/en/stable/migration.html#separation-of-metadata-and-keyword-arguments-in-traittype-contructors 17 | # config is an unrecognized keyword 18 | 19 | remote_hosts = List(trait=Unicode(), 20 | help="Possible remote hosts from which to choose remote_host.", 21 | config=True) 22 | 23 | # Removed 'config=True' tag. 24 | # Any user configureation of remote_host is redundant. 25 | # The spawner now chooses the value of remote_host. 26 | remote_host = Unicode("remote_host", 27 | help="SSH remote host to spawn sessions on") 28 | 29 | # This is a external remote IP, let the server listen on all interfaces if we want 30 | remote_ip = Unicode("remote_ip", 31 | help="IP on remote side") 32 | 33 | remote_port = Unicode("22", 34 | help="SSH remote port number", 35 | config=True) 36 | 37 | ssh_command = Unicode("/usr/bin/ssh", 38 | help="Actual SSH command", 39 | config=True) 40 | 41 | path = Unicode("/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin", 42 | help="Default PATH (should include jupyter and python)", 43 | config=True) 44 | 45 | # The get_port.py script is in scripts/get_port.py 46 | # FIXME See if we avoid having to deploy a script on remote side? 47 | # For instance, we could just install sshspawner on the remote side 48 | # as a package and have it put get_port.py in the right place. 49 | # If we were fancy it could be configurable so it could be restricted 50 | # to specific ports. 51 | remote_port_command = Unicode("/usr/bin/python /usr/local/bin/get_port.py", 52 | help="Command to return unused port on remote host", 53 | config=True) 54 | 55 | # FIXME Fix help, what happens when not set? 56 | hub_api_url = Unicode("", 57 | help=dedent("""If set, Spawner will configure the containers to use 58 | the specified URL to connect the hub api. This is useful when the 59 | hub_api is bound to listen on all ports or is running inside of a 60 | container."""), 61 | config=True) 62 | 63 | ssh_keyfile = Unicode("~/.ssh/id_rsa", 64 | help=dedent("""Key file used to authenticate hub with remote host. 65 | 66 | `~` will be expanded to the user's home directory and `{username}` 67 | will be expanded to the user's username"""), 68 | config=True) 69 | 70 | pid = Integer(0, 71 | help=dedent("""Process ID of single-user server process spawned for 72 | current user.""")) 73 | 74 | resource_path = Unicode(".jupyterhub-resources", 75 | help=dedent("""The base path where all necessary resources are 76 | placed. Generally left relative so that resources are placed into 77 | this base directory in the user's home directory."""), 78 | config=True) 79 | 80 | def load_state(self, state): 81 | """Restore state about ssh-spawned server after a hub restart. 82 | 83 | The ssh-spawned processes need IP and the process id.""" 84 | super().load_state(state) 85 | if "pid" in state: 86 | self.pid = state["pid"] 87 | if "remote_ip" in state: 88 | self.remote_ip = state["remote_ip"] 89 | 90 | def get_state(self): 91 | """Save state needed to restore this spawner instance after hub restore. 92 | 93 | The ssh-spawned processes need IP and the process id.""" 94 | state = super().get_state() 95 | if self.pid: 96 | state["pid"] = self.pid 97 | if self.remote_ip: 98 | state["remote_ip"] = self.remote_ip 99 | return state 100 | 101 | def clear_state(self): 102 | """Clear stored state about this spawner (ip, pid)""" 103 | super().clear_state() 104 | self.remote_ip = "remote_ip" 105 | self.pid = 0 106 | 107 | async def start(self): 108 | """Start single-user server on remote host.""" 109 | 110 | username = self.user.name 111 | kf = self.ssh_keyfile.format(username=username) 112 | cf = kf + "-cert.pub" 113 | k = asyncssh.read_private_key(kf) 114 | c = asyncssh.read_certificate(cf) 115 | 116 | self.remote_host = self.choose_remote_host() 117 | 118 | self.remote_ip, port = await self.remote_random_port() 119 | if self.remote_ip is None or port is None or port == 0: 120 | return False 121 | self.remote_port = str(port) 122 | cmd = [] 123 | 124 | cmd.extend(self.cmd) 125 | cmd.extend(self.get_args()) 126 | 127 | if self.user.settings["internal_ssl"]: 128 | with TemporaryDirectory() as td: 129 | local_resource_path = td 130 | 131 | self.cert_paths = self.stage_certs( 132 | self.cert_paths, 133 | local_resource_path 134 | ) 135 | 136 | # create resource path dir in user's home on remote 137 | async with asyncssh.connect(self.remote_ip, username=username,client_keys=[(k,c)],known_hosts=None) as conn: 138 | mkdir_cmd = "mkdir -p {path} 2>/dev/null".format(path=self.resource_path) 139 | result = await conn.run(mkdir_cmd) 140 | 141 | # copy files 142 | files = [os.path.join(local_resource_path, f) for f in os.listdir(local_resource_path)] 143 | async with asyncssh.connect(self.remote_ip, username=username,client_keys=[(k,c)],known_hosts=None) as conn: 144 | await asyncssh.scp(files, (conn, self.resource_path)) 145 | 146 | if self.hub_api_url != "": 147 | old = "--hub-api-url={}".format(self.hub.api_url) 148 | new = "--hub-api-url={}".format(self.hub_api_url) 149 | for index, value in enumerate(cmd): 150 | if value == old: 151 | cmd[index] = new 152 | for index, value in enumerate(cmd): 153 | if value[0:6] == '--port': 154 | cmd[index] = '--port=%d' % (port) 155 | 156 | remote_cmd = ' '.join(cmd) 157 | 158 | self.pid = await self.exec_notebook(remote_cmd) 159 | 160 | self.log.debug("Starting User: {}, PID: {}".format(self.user.name, self.pid)) 161 | 162 | if self.pid < 0: 163 | return None 164 | 165 | return (self.remote_ip, port) 166 | 167 | async def poll(self): 168 | """Poll ssh-spawned process to see if it is still running. 169 | 170 | If it is still running return None. If it is not running return exit 171 | code of the process if we have access to it, or 0 otherwise.""" 172 | 173 | if not self.pid: 174 | # no pid, not running 175 | self.clear_state() 176 | return 0 177 | 178 | # send signal 0 to check if PID exists 179 | alive = await self.remote_signal(0) 180 | self.log.debug("Polling returned {}".format(alive)) 181 | 182 | if not alive: 183 | self.clear_state() 184 | return 0 185 | else: 186 | return None 187 | 188 | async def stop(self, now=False): 189 | """Stop single-user server process for the current user.""" 190 | alive = await self.remote_signal(15) 191 | self.clear_state() 192 | 193 | def get_remote_user(self, username): 194 | """Map JupyterHub username to remote username.""" 195 | return username 196 | 197 | def choose_remote_host(self): 198 | """ 199 | Given the list of possible nodes from which to choose, make the choice of which should be the remote host. 200 | """ 201 | remote_host = random.choice(self.remote_hosts) 202 | return remote_host 203 | 204 | @observe('remote_host') 205 | def _log_remote_host(self, change): 206 | self.log.debug("Remote host was set to %s." % self.remote_host) 207 | 208 | @observe('remote_ip') 209 | def _log_remote_ip(self, change): 210 | self.log.debug("Remote IP was set to %s." % self.remote_ip) 211 | 212 | # FIXME this needs to now return IP and port too 213 | async def remote_random_port(self): 214 | """Select unoccupied port on the remote host and return it. 215 | 216 | If this fails for some reason return `None`.""" 217 | 218 | username = self.get_remote_user(self.user.name) 219 | kf = self.ssh_keyfile.format(username=username) 220 | cf = kf + "-cert.pub" 221 | k = asyncssh.read_private_key(kf) 222 | c = asyncssh.read_certificate(cf) 223 | 224 | # this needs to be done against remote_host, first time we're calling up 225 | async with asyncssh.connect(self.remote_host,username=username,client_keys=[(k,c)],known_hosts=None) as conn: 226 | result = await conn.run(self.remote_port_command) 227 | stdout = result.stdout 228 | stderr = result.stderr 229 | retcode = result.exit_status 230 | 231 | if stdout != b"": 232 | ip, port = stdout.split() 233 | port = int(port) 234 | self.log.debug("ip={} port={}".format(ip, port)) 235 | else: 236 | ip, port = None, None 237 | self.log.error("Failed to get a remote port") 238 | self.log.error("STDERR={}".format(stderr)) 239 | self.log.debug("EXITSTATUS={}".format(retcode)) 240 | return (ip, port) 241 | 242 | # FIXME add docstring 243 | async def exec_notebook(self, command): 244 | """TBD""" 245 | 246 | env = super(SSHSpawner, self).get_env() 247 | env['JUPYTERHUB_API_URL'] = self.hub_api_url 248 | if self.path: 249 | env['PATH'] = self.path 250 | username = self.get_remote_user(self.user.name) 251 | kf = self.ssh_keyfile.format(username=username) 252 | cf = kf + "-cert.pub" 253 | k = asyncssh.read_private_key(kf) 254 | c = asyncssh.read_certificate(cf) 255 | bash_script_str = "#!/bin/bash\n" 256 | 257 | for item in env.items(): 258 | # item is a (key, value) tuple 259 | # command = ('export %s=%s;' % item) + command 260 | bash_script_str += 'export %s=%s\n' % item 261 | bash_script_str += 'unset XDG_RUNTIME_DIR\n' 262 | 263 | bash_script_str += 'touch .jupyter.log\n' 264 | bash_script_str += 'chmod 600 .jupyter.log\n' 265 | bash_script_str += '%s < /dev/null >> .jupyter.log 2>&1 & pid=$!\n' % command 266 | bash_script_str += 'echo $pid\n' 267 | 268 | run_script = "/tmp/{}_run.sh".format(self.user.name) 269 | with open(run_script, "w") as f: 270 | f.write(bash_script_str) 271 | if not os.path.isfile(run_script): 272 | raise Exception("The file " + run_script + "was not created.") 273 | else: 274 | with open(run_script, "r") as f: 275 | self.log.debug(run_script + " was written as:\n" + f.read()) 276 | 277 | async with asyncssh.connect(self.remote_ip, username=username,client_keys=[(k,c)],known_hosts=None) as conn: 278 | result = await conn.run("bash -s", stdin=run_script) 279 | stdout = result.stdout 280 | stderr = result.stderr 281 | retcode = result.exit_status 282 | 283 | self.log.debug("exec_notebook status={}".format(retcode)) 284 | if stdout != b'': 285 | pid = int(stdout) 286 | else: 287 | return -1 288 | 289 | return pid 290 | 291 | async def remote_signal(self, sig): 292 | """Signal on the remote host.""" 293 | 294 | username = self.get_remote_user(self.user.name) 295 | kf = self.ssh_keyfile.format(username=username) 296 | cf = kf + "-cert.pub" 297 | k = asyncssh.read_private_key(kf) 298 | c = asyncssh.read_certificate(cf) 299 | 300 | command = "kill -s %s %d < /dev/null" % (sig, self.pid) 301 | 302 | async with asyncssh.connect(self.remote_ip, username=username,client_keys=[(k,c)],known_hosts=None) as conn: 303 | result = await conn.run(command) 304 | stdout = result.stdout 305 | stderr = result.stderr 306 | retcode = result.exit_status 307 | self.log.debug("command: {} returned {} --- {} --- {}".format(command, stdout, stderr, retcode)) 308 | return (retcode == 0) 309 | 310 | def stage_certs(self, paths, dest): 311 | shutil.move(paths['keyfile'], dest) 312 | shutil.move(paths['certfile'], dest) 313 | shutil.copy(paths['cafile'], dest) 314 | 315 | key_base_name = os.path.basename(paths['keyfile']) 316 | cert_base_name = os.path.basename(paths['certfile']) 317 | ca_base_name = os.path.basename(paths['cafile']) 318 | 319 | key = os.path.join(self.resource_path, key_base_name) 320 | cert = os.path.join(self.resource_path, cert_base_name) 321 | ca = os.path.join(self.resource_path, ca_base_name) 322 | 323 | return { 324 | "keyfile": key, 325 | "certfile": cert, 326 | "cafile": ca, 327 | } 328 | --------------------------------------------------------------------------------