├── rk ├── ssh │ ├── __init__.py │ ├── forward.py │ └── tunnel.py ├── __init__.py ├── resources │ └── img │ │ ├── logo-32x32.png │ │ └── logo-64x64.png ├── config │ ├── kernels.json │ ├── rk.ini │ ├── argparse.txt │ └── messages.txt └── rk.py ├── img ├── quickstart_0001_728px.png └── user_guide-_kernels_dict_0001_728px.png ├── .gitignore ├── MANIFEST.in ├── fabfile.py ├── LICENSE.txt ├── setup.py ├── README.rst └── scripts └── rkscript /rk/ssh/__init__.py: -------------------------------------------------------------------------------- 1 | from rk.ssh.tunnel import * 2 | -------------------------------------------------------------------------------- /rk/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __version__ = '0.3b1' 4 | -------------------------------------------------------------------------------- /img/quickstart_0001_728px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/korniichuk/rk/HEAD/img/quickstart_0001_728px.png -------------------------------------------------------------------------------- /rk/resources/img/logo-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/korniichuk/rk/HEAD/rk/resources/img/logo-32x32.png -------------------------------------------------------------------------------- /rk/resources/img/logo-64x64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/korniichuk/rk/HEAD/rk/resources/img/logo-64x64.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.gz 3 | *.pyc 4 | *.tar 5 | *.tgz 6 | *.zip 7 | .pypirc 8 | /*.egg-info/* 9 | /dist/* 10 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE.txt README.rst 2 | recursive-include img *.png 3 | recursive-include rk *.ini *.json *.png *.py *.txt 4 | -------------------------------------------------------------------------------- /img/user_guide-_kernels_dict_0001_728px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/korniichuk/rk/HEAD/img/user_guide-_kernels_dict_0001_728px.png -------------------------------------------------------------------------------- /rk/config/kernels.json: -------------------------------------------------------------------------------- 1 | { 2 | "template": { 3 | "display_name": "Template", 4 | "interpreter": "python", 5 | "language": "python", 6 | "remote_host": "remote_username@remote_host" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /rk/config/rk.ini: -------------------------------------------------------------------------------- 1 | config_argparse_rel_path = "config/argparse.txt" 2 | config_kernels_rel_path = "config/kernels.json" 3 | config_messages_rel_path = "config/messages.txt" 4 | connection_file = "{connection_file}" 5 | display_name = "Template" 6 | img_location = "resources/img" 7 | interpreter = "python" 8 | kernel_name = "template" 9 | kernels_location = "/usr/local/share/jupyter/kernels" 10 | language = "python" 11 | logo_name_srt = "logo-{0}x{0}.png" 12 | remote_host = "remote_username@remote_host" 13 | rk_log_location = "/tmp/rk/log" 14 | script = "rkscript" 15 | -------------------------------------------------------------------------------- /rk/config/argparse.txt: -------------------------------------------------------------------------------- 1 | _parser 2 | The remote jupyter kernel/kernels administration utility 3 | _subparsers 4 | subcommands 5 | _parser_install 6 | install remote jupyter kernel/kernels 7 | _parser_install_all 8 | install all remote jupyter kernels from kernels dict 9 | _parser_install_template 10 | install template of remote kernel 11 | _parser_list 12 | show list of remote jupyter kernels from kernels dict 13 | _parser_ssh 14 | setup SSH for auto login without a password 15 | _parser_uninstall 16 | uninstall remote jupyter kernel/kernels 17 | _parser_uninstall_all 18 | uninstall all jupyter kernels from kernels location 19 | _parser_uninstall_template 20 | uninstall template of remote kernel 21 | -------------------------------------------------------------------------------- /fabfile.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python2 2 | # -*- coding: utf-8 -*- 3 | 4 | """The remote jupyter kernel/kernels administration utility fabric file""" 5 | 6 | from fabric.api import local 7 | 8 | def git(): 9 | """Setup Git""" 10 | 11 | local("git remote rm origin") 12 | local("git remote add origin https://korniichuk@github.com/korniichuk/rk.git") 13 | local("git remote add bitbucket https://korniichuk@bitbucket.org/korniichuk/rk.git") 14 | 15 | def live(): 16 | """Upload package to PyPI Live""" 17 | 18 | local("python setup.py register -r pypi") 19 | local("python setup.py sdist --format=zip,gztar upload -r pypi") 20 | 21 | def test(): 22 | """Upload package to PyPI Test""" 23 | 24 | local("python setup.py register -r pypitest") 25 | local("python setup.py sdist --format=zip,gztar upload -r pypitest") 26 | -------------------------------------------------------------------------------- /rk/config/messages.txt: -------------------------------------------------------------------------------- 1 | _ask_remote_host 2 | Enter REMOTE_HOST (with optional REMOTE_USERNAME: REMOTE_USERNAME@REMOTE_HOST): 3 | _delete 4 | KERNEL_NAME '%s' already exists. Delete files and continue? [y/n] 5 | _delete_template 6 | Template of remote kernel already exists. Delete files and continue? [y/n] 7 | _error_ArgumentsNumber 8 | Error: The rkscript takes exactly %s arguments (%s given). 9 | _error_NoKernel 10 | Error: KERNEL_NAME '%s' not found. 11 | _error_NoKernels 12 | Error: KERNEL_NAME: '%s' not found. 13 | _error_NoRoot 14 | Error: To do that you need a superuser (root) privileges. 15 | _error_NoTemplate 16 | Error: Template of remote kernel not found. 17 | _error_Oops 18 | Error: %s. 19 | _installed 20 | installed '%s' remote jupyter kernel 21 | _installed_template 22 | installed template of remote kernel 23 | _uninstalled 24 | uninstalled '%s'remote jupyter kernel 25 | _uninstalled_all 26 | uninstalled '%s' jupyter kernel 27 | _uninstalled_all_multiple 28 | uninstalled '%s' jupyter kernels 29 | _uninstalled_all_zero 30 | jupyter kernels not found in kernels location 31 | _uninstalled_template 32 | uninstalled template of remote kernel 33 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from os.path import dirname, join 4 | from setuptools import setup 5 | 6 | setup( 7 | author = "Ruslan Korniichuk", 8 | author_email = "ruslan.korniichuk@gmail.com", 9 | classifiers = [ 10 | "Development Status :: 4 - Beta", 11 | "Environment :: Console", 12 | "Intended Audience :: Developers", 13 | "Intended Audience :: Information Technology", 14 | "Intended Audience :: Science/Research", 15 | "Intended Audience :: System Administrators", 16 | "License :: Public Domain", 17 | "Operating System :: POSIX :: Linux", 18 | "Programming Language :: Python", 19 | "Programming Language :: Python :: 2", 20 | "Programming Language :: Python :: 2.7", 21 | "Programming Language :: Python :: 2 :: Only", 22 | "Topic :: Scientific/Engineering", 23 | "Topic :: System :: Systems Administration", 24 | "Topic :: Utilities" 25 | ], 26 | description = "The remote jupyter kernel/kernels administration utility", 27 | download_url = "https://github.com/korniichuk/rk/archive/0.3.zip", 28 | entry_points = { 29 | 'console_scripts': 'rk = rk.rk:main' 30 | }, 31 | include_package_data = True, 32 | install_requires = [ 33 | "configobj", 34 | "execnet", 35 | "paramiko" 36 | ], 37 | keywords = ["ipython", "jupyter", "remote kernel", "rk", "python2"], 38 | license = "Public Domain", 39 | long_description = open(join(dirname(__file__), "README.rst")).read(), 40 | name = "rk", 41 | packages = ["rk"], 42 | platforms = ["Linux"], 43 | scripts=['scripts/rkscript'], 44 | url = "https://github.com/korniichuk/rk", 45 | version = "0.3b1", 46 | zip_safe = True 47 | ) 48 | -------------------------------------------------------------------------------- /rk/ssh/forward.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is adapted from a paramiko demo, and thus licensed under LGPL 2.1. 3 | # Original Copyright (C) 2003-2007 Robey Pointer 4 | # Edits Copyright (C) 2010 The IPython Team 5 | # 6 | # Paramiko is free software; you can redistribute it and/or modify it under the 7 | # terms of the GNU Lesser General Public License as published by the Free 8 | # Software Foundation; either version 2.1 of the License, or (at your option) 9 | # any later version. 10 | # 11 | # Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY 12 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 13 | # A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 14 | # details. 15 | # 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with Paramiko; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA. 19 | 20 | """ 21 | Sample script showing how to do local port forwarding over paramiko. 22 | 23 | This script connects to the requested SSH server and sets up local port 24 | forwarding (the openssh -L option) from a local port through a tunneled 25 | connection to a destination reachable from the SSH server machine. 26 | """ 27 | 28 | from __future__ import print_function 29 | 30 | import logging 31 | import select 32 | try: # Python 3 33 | import socketserver 34 | except ImportError: # Python 2 35 | import SocketServer as socketserver 36 | 37 | logger = logging.getLogger('ssh') 38 | 39 | class ForwardServer (socketserver.ThreadingTCPServer): 40 | daemon_threads = True 41 | allow_reuse_address = True 42 | 43 | 44 | class Handler (socketserver.BaseRequestHandler): 45 | 46 | def handle(self): 47 | try: 48 | chan = self.ssh_transport.open_channel('direct-tcpip', 49 | (self.chain_host, self.chain_port), 50 | self.request.getpeername()) 51 | except Exception as e: 52 | logger.debug('Incoming request to %s:%d failed: %s' % (self.chain_host, 53 | self.chain_port, 54 | repr(e))) 55 | return 56 | if chan is None: 57 | logger.debug('Incoming request to %s:%d was rejected by the SSH server.' % 58 | (self.chain_host, self.chain_port)) 59 | return 60 | 61 | logger.debug('Connected! Tunnel open %r -> %r -> %r' % (self.request.getpeername(), 62 | chan.getpeername(), (self.chain_host, self.chain_port))) 63 | while True: 64 | r, w, x = select.select([self.request, chan], [], []) 65 | if self.request in r: 66 | data = self.request.recv(1024) 67 | if len(data) == 0: 68 | break 69 | chan.send(data) 70 | if chan in r: 71 | data = chan.recv(1024) 72 | if len(data) == 0: 73 | break 74 | self.request.send(data) 75 | chan.close() 76 | self.request.close() 77 | logger.debug('Tunnel closed ') 78 | 79 | 80 | def forward_tunnel(local_port, remote_host, remote_port, transport): 81 | # this is a little convoluted, but lets me configure things for the Handler 82 | # object. (SocketServer doesn't give Handlers any way to access the outer 83 | # server normally.) 84 | class SubHander (Handler): 85 | chain_host = remote_host 86 | chain_port = remote_port 87 | ssh_transport = transport 88 | ForwardServer(('127.0.0.1', local_port), SubHander).serve_forever() 89 | 90 | 91 | __all__ = ['forward_tunnel'] 92 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. contents:: Table of contents 2 | :depth: 2 3 | 4 | Installation 5 | ============ 6 | Install the rk from PyPI 7 | ------------------------ 8 | :: 9 | 10 | $ sudo pip install rk 11 | 12 | Install the rk from GitHub 13 | -------------------------- 14 | :: 15 | 16 | $ sudo pip install git+git://github.com/korniichuk/rk#egg=rk 17 | 18 | Upgrade the rk from PyPI 19 | ------------------------ 20 | :: 21 | 22 | $ sudo pip install -U rk 23 | 24 | or:: 25 | 26 | $ sudo pip install --upgrade rk 27 | 28 | .. important:: The rk set to dafault the `kernels dict`_ ``kernels.json``. Save the current `kernels dict`_ to home dir before upgrade, and replace default `kernels dict`_ file after. 29 | 30 | Uninstall the rk 31 | ---------------- 32 | :: 33 | 34 | $ sudo pip uninstall rk 35 | 36 | Development installation 37 | ======================== 38 | :: 39 | 40 | $ git clone git://github.com/korniichuk/rk.git 41 | $ cd rk 42 | $ sudo pip install . 43 | 44 | Quickstart 45 | ========== 46 | 47 | .. image:: ./img/quickstart_0001_728px.png 48 | :alt: quickstart [youtube video] 49 | :target: https://youtu.be/joEIPZJUB94 50 | 51 | **First**, make sure that you can login to a remote machine without entering password. The most basic form of the command is:: 52 | 53 | $ ssh REMOTE_HOST 54 | 55 | If your username is different on a remote machine, you can specify it by using this syntax:: 56 | 57 | $ ssh REMOTE_USERNAME@REMOTE_HOST 58 | 59 | Example:: 60 | 61 | $ ssh albert@192.168.0.1 62 | 63 | .. note:: You can `setup SSH for auto login without a password`_ like this: ``$ rk ssh``. 64 | 65 | **Second**, install a template of a remote jupyter kernel to `kernels location`_:: 66 | 67 | $ rk install-template 68 | 69 | **Third**, change the ``kernel.json`` file:: 70 | 71 | $ sudo gedit /usr/local/share/jupyter/kernels/template/kernel.json 72 | 73 | The ``kernel.json`` file looks like this:: 74 | 75 | { 76 | "argv": [ 77 | "rkscript", 78 | "python", 79 | "{connection_file}", 80 | "remote_username@remote_host" 81 | ], 82 | "display_name": "Template", 83 | "language": "python" 84 | } 85 | 86 | For a python2 remote jupyter kernel just change ``remote_username@remote_host``. For example from ``remote_username@remote_host`` to ``albert@192.168.0.1``. 87 | 88 | **Fourth**, launch jupyter notebook and check your new remote juputer kernel:: 89 | 90 | $ jupyter notebook 91 | 92 | or:: 93 | 94 | $ ipython notebook 95 | 96 | Choose: ``Files -> New -> Template``. 97 | 98 | User guide 99 | ========== 100 | Help 101 | ---- 102 | The standard output for –help:: 103 | 104 | $ rk -h 105 | 106 | or:: 107 | 108 | $ rk --help 109 | 110 | For information on using subcommand "SUBCOMMAND", do:: 111 | 112 | $ rk SUBCOMMAND -h 113 | 114 | or:: 115 | 116 | $ rk SUBCOMMAND --help 117 | 118 | Example:: 119 | 120 | $ rk install -h 121 | 122 | Version 123 | ------- 124 | The standard output for –version:: 125 | 126 | $ rk -v 127 | 128 | or:: 129 | 130 | $ rk --version 131 | 132 | Kernels dict 133 | ------------ 134 | 135 | .. image:: ./img/user_guide-_kernels_dict_0001_728px.png 136 | :alt: user guide: kernels dict [youtube video] 137 | :target: https://youtu.be/czh3K4xjVD4 138 | 139 | Open ``kernels.json`` file:: 140 | 141 | $ sudo gedit /usr/local/lib/python2.7/dist-packages/rk/config/kernels.json 142 | 143 | The ``kernels.json`` file looks like this:: 144 | 145 | { 146 | "template": { 147 | "display_name": "Template", 148 | "interpreter": "python", 149 | "language": "python", 150 | "remote_host": "remote_username@remote_host" 151 | } 152 | } 153 | 154 | Where: 155 | 156 | * ``template`` -- the name of a remote jupyter kernel, 157 | 158 | * ``display_name`` -- a kernel’s name as it should be displayed in the UI. Unlike the kernel name used in the API, this can contain arbitrary unicode characters [1]_, 159 | * ``interpreter`` -- an entry point or an absolute path to language interpreter on a remote machine, 160 | * ``language`` -- a name of the language of a kernel. When loading notebooks, if no matching kernelspec key (may differ across machines) is found, a kernel with a matching language will be used. This allows a notebook written on any python or julia kernel to be properly associated with the user's python or julia kernel, even if they aren’t listed under the same name as the author’s [1]_, 161 | * ``remote_host`` -- just a remote host or, if your username is different on a remote machine, use this syntax: remote username AT remote host. 162 | 163 | .. note:: For checking absolute path to language interpreter on a remote machine use a `which `_ Unix command. For example, for the python3 language on a remote machine: ``$ which python3``. 164 | 165 | Change ``kernels.json`` file and add info about your remote jupyter kernels, for example like this:: 166 | 167 | { 168 | "albert2": { 169 | "display_name": "Albert Python 2", 170 | "interpreter": "python2", 171 | "language": "python", 172 | "remote_host": "albert@192.168.0.1" 173 | }, 174 | "albert3": { 175 | "display_name": "Albert Python 3", 176 | "interpreter": "python3", 177 | "language": "python", 178 | "remote_host": "albert@192.168.0.1" 179 | } 180 | } 181 | 182 | Where: 183 | 184 | * ``albert2``, ``albert3`` -- the names of a remote jupyter kernels, 185 | 186 | * ``Albert Python 2``, ``Albert Python 3`` -- the display names for the UI, 187 | * ``python2``, ``python3`` -- entry points on a remote machine, 188 | * ``python`` -- the name of the language of a remote jupyter kernel, 189 | * ``albert`` -- the remote username on a remote machine, not similar with a username on a local machine, 190 | * ``192.168.0.1`` -- the remote host. 191 | 192 | Kernels location 193 | ---------------- 194 | Jupyter support the system and the user `kernels locations `_: 195 | 196 | +----------+--------------------------------------------------------------------+ 197 | | |kernels location | 198 | +==========+====================================================================+ 199 | |**system**|``/usr/local/share/jupyter/kernels``; ``/usr/share/jupyter/kernels``| 200 | +----------+--------------------------------------------------------------------+ 201 | |**user** |``~/.ipython/kernels`` | 202 | +----------+--------------------------------------------------------------------+ 203 | 204 | The default kernels location in the rk: ``/usr/local/share/jupyter/kernels``. 205 | 206 | Change the default kernels location:: 207 | 208 | $ sudo gedit /usr/local/lib/python2.7/dist-packages/rk/config/rk.ini 209 | 210 | .. important:: The user kernels location takes priority over the system kernels locations. 211 | 212 | Show list of remote jupyter kernels from kernels dict 213 | ----------------------------------------------------- 214 | :: 215 | 216 | $ rk list 217 | 218 | Install a remote jupyter kernel/kernels from kernels dict to kernels location 219 | ----------------------------------------------------------------------------- 220 | :: 221 | 222 | $ rk install KERNEL_NAME [KERNEL_NAME ...] 223 | 224 | Where: 225 | 226 | * ``KERNEL_NAME`` -- a name of a remote jupyter kernel in the `kernels dict`_ ``kernels.json``. 227 | 228 | Example:: 229 | 230 | $ rk install albert2 231 | $ rk install albert2 albert3 232 | 233 | Install a template of a remote jupyter kernel to kernels location 234 | ----------------------------------------------------------------- 235 | :: 236 | 237 | $ rk install-template 238 | 239 | .. important:: After this subcommand open the ``kernel.json`` file and change values of dict: ``$ sudo gedit /usr/local/share/jupyter/kernels/template/kernel.json``. 240 | 241 | Install all remote jupyter kernels from kernels dict to kernels location 242 | ------------------------------------------------------------------------ 243 | :: 244 | 245 | $ rk install-all 246 | 247 | Uninstall a remote jupyter kernel/kernels from kernels location 248 | --------------------------------------------------------------- 249 | :: 250 | 251 | $ rk uninstall KERNEL_NAME [KERNEL_NAME ...] 252 | 253 | Where: 254 | 255 | * KERNEL_NAME -- a name of installed remote jupyter kernel. 256 | 257 | Example:: 258 | 259 | $ rk uninstall albert2 260 | $ rk uninstall albert2 albert3 261 | 262 | Uninstall a template of a remote jupyter kernel from kernels location 263 | --------------------------------------------------------------------- 264 | :: 265 | 266 | $ rk uninstall-template 267 | 268 | Uninstall all jupyter kernels from kernels location 269 | --------------------------------------------------- 270 | :: 271 | 272 | $ rk uninstall-all 273 | 274 | Setup SSH for auto login without a password 275 | ------------------------------------------- 276 | :: 277 | 278 | $ rk ssh 279 | 280 | If you are familiar with `ssh-keygen `_, `ssh-copy-id `_ and `ssh-add `_, this code also setup SSH for auto login without a password [2]_:: 281 | 282 | $ ssh-keygen -t rsa -b 4096 -N '' -f ~/.ssh/id_rsa 283 | $ ssh-copy-id REMOTE_HOST 284 | $ eval "$(ssh-agent -s)" 285 | $ ssh-add ~/.ssh/id_rsa 286 | 287 | .. note:: If your username is different on a remote machine, you can specify it by using this syntax: ``$ ssh-copy-id REMOTE_USERNAME@REMOTE_HOST``. 288 | 289 | Log files 290 | --------- 291 | The default log files location in the rk: ``/tmp/rk/log``. The name of rk log file, for working remote jupyter kernel, look like this: ``bree@192.168.0.1_1879-03-14_11.30.00.txt``. And the log file looks like this:: 292 | 293 | date: 1879-03-14 Friday 294 | time: 11:30:00 295 | 296 | usernames: bree<->albert 297 | remote host: 192.168.0.1 298 | 299 | stdin ports: 37654<->58933 300 | hb ports: 53538<->59782 301 | iopub ports: 45330<->51989 302 | shell ports: 36523<->36107 303 | control ports: 50090<->53633 304 | 305 | pids: 16965<->20944 306 | 307 | .. note:: Change the default log files location: ``$ sudo gedit /usr/local/lib/python2.7/dist-packages/rk/config/rk.ini``. 308 | 309 | The paramiko log file is available in a local connection file directory. The name of paramiko log file, for working remote jupyter kernel, look like this: ``paramiko-843664c7-798d-4a9e-979c-22d0dc4a6bd5.txt``. 310 | 311 | History 312 | ======= 313 | Legend 314 | ------ 315 | 316 | * **added** 317 | * corrected 318 | * *removed* 319 | 320 | rk 0.3 321 | ------ 322 | 323 | * bug in the rk and in the rkscript: an initial component of ``~`` or ``~user`` is not replaced in a paths. 324 | * bug in the rk: a superuser (root) privileges required for the user kernels location ``~/.ipython/kernels``. 325 | * **setup SSH for auto login without a password with a "ssh" subcommand.** 326 | * error in the rkscript: list index out of range. 327 | * **info about working remote jupyter kernel in rk log file.** 328 | * **paramiko log file in a local connection file dir.** 329 | * error in the rkscript: no handlers could be found for logger "paramiko.transport". 330 | * local port forwarding in the rkscript via paramiko, not via pexpect. 331 | 332 | rk 0.2 333 | ------ 334 | 335 | * **uninstall all jupyter kernels from kernels location with a "uninstall-all" subcommand.** 336 | * **uninstall a remote jupyter kernel/kernels from kernels location with a "uninstall" subcommand.** 337 | * **install a remote jupyter kernel/kernels from kernels dict to kernels location with a "install" subcommand.** 338 | * **install all remote jupyter kernels from kernels dict to kernels location with a "install-all" subcommand.** 339 | * **show list of remote jupyter kernels from kernels dict with a "list" subcommand.** 340 | 341 | .. rubric:: Footnotes 342 | 343 | .. [1] http://ipython.org/ipython-doc/dev/development/kernels.html#kernel-specs 344 | .. [2] https://help.github.com/articles/generating-ssh-keys/ 345 | -------------------------------------------------------------------------------- /scripts/rkscript: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Remote jupyter kernel via SSH 5 | Make sure that you can login to a remote machine without entering password. 6 | 7 | """ 8 | 9 | from datetime import datetime 10 | from errno import EACCES, ENOTDIR 11 | from getpass import getuser 12 | from json import load 13 | from os import chmod, getcwd, getpid, makedirs, remove 14 | from os.path import dirname, exists, expanduser, isfile, join, split 15 | from site import getsitepackages 16 | from sys import argv 17 | 18 | from configobj import ConfigObj 19 | from execnet import makegateway 20 | from paramiko.util import log_to_file 21 | 22 | from rk.ssh import paramiko_tunnel 23 | 24 | arguments_number = 3 # interpreter, local_connection_file, 25 | # remote_username_at_remote_host 26 | messages = {} # Strings for output 27 | week = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 28 | 'Sunday'] 29 | 30 | module_name = "rk" 31 | module_location = join(getsitepackages()[0], module_name) 32 | config_rk_abs_path = join(module_location, "config/rk.ini") 33 | config = ConfigObj(config_rk_abs_path) 34 | 35 | def create_directory(directory_name, mode=0o777): 36 | """Recursive directory creation function 37 | os.chmod work only for last directory 38 | 39 | """ 40 | 41 | try: 42 | makedirs(directory_name, mode) 43 | except Exception as exception: 44 | error_code = exception.errno 45 | if error_code == EACCES: # 13 (Python3 PermissionError) 46 | print(messages["_error_NoRoot"]) 47 | exit(1) 48 | elif error_code == ENOTDIR: # 20 (Python3 NotADirectoryError) 49 | path = directory_name 50 | while path != '/': 51 | if isfile(path): 52 | try: 53 | remove(path) 54 | except Exception as exception: # Python3 PermissionError 55 | error_code = exception.errno 56 | if error_code == EACCES: # 13 57 | print(messages["_error_NoRoot"]) 58 | exit(1) 59 | else: 60 | print(messages["_error_Oops"] % 61 | strerror(error_code)) 62 | exit(1) 63 | path = dirname(path) 64 | try: 65 | makedirs(directory_name, mode) 66 | except Exception as exception: # Python3 PermissionError 67 | error_code = exception.errno 68 | if error_code == EACCES: # 13 69 | print(messages["_error_NoRoot"]) 70 | exit(1) 71 | else: 72 | print(messages["_error_Oops"] % strerror(error_code)) 73 | exit(1) 74 | else: 75 | print(messages["_error_Oops"] % strerror(error_code)) 76 | exit(1) 77 | 78 | def get_date_time(): 79 | """Get yyyy-mm-dd_hh.mm.ss""" 80 | 81 | def normalize(element): 82 | """Add '0' from front""" 83 | 84 | if len(element) == 1: 85 | element = '0' + element 86 | return element 87 | 88 | now = datetime.now() 89 | year = str(now.year) 90 | month = normalize(str(now.month)) 91 | day = normalize(str(now.day)) 92 | hour = normalize(str(now.hour)) 93 | minute = normalize(str(now.minute)) 94 | second = normalize(str(now.second)) 95 | date = year + '-' + month + '-' + day 96 | time = hour + '.' + minute + '.' + second 97 | date_time = date + '_' + time 98 | return date_time 99 | 100 | def create_messages(): 101 | """Create "messages" dictionary""" 102 | 103 | config_messages_rel_path = config["config_messages_rel_path"] 104 | config_messages_abs_path = join(module_location, config_messages_rel_path) 105 | with open(config_messages_abs_path, 'r') as f: 106 | messages_list = f.read().splitlines() 107 | for i in range(0, len(messages_list), 2): 108 | messages[messages_list[i]] = messages_list[i+1] 109 | 110 | create_messages() 111 | argv_len = len(argv) - 1 # argv[0]: is the script name 112 | if argv_len == arguments_number: 113 | interpreter = argv[1] # An entry point or an absolute path 114 | # to language interpreter on a remote machine 115 | local_connection_file = argv[2] # Absolute path of a local connection file 116 | remote_username_at_remote_host = argv[3] # Just a remote host or, 117 | # if your username is different on a remote machine, 118 | # use this syntax: remote username AT remote host. 119 | else: 120 | print(messages["_error_ArgumentsNumber"] % (arguments_number, argv_len)) 121 | exit(1) 122 | local_username = getuser() 123 | if '@' in remote_username_at_remote_host: 124 | remote_username, remote_host = remote_username_at_remote_host.split('@') 125 | if local_username != remote_username: 126 | # Local username is NOT the same as a remote username 127 | remote_connection_file = local_connection_file.replace(local_username, 128 | remote_username) 129 | else: 130 | # Local username is the same as a remote username 131 | remote_connection_file = local_connection_file 132 | remote_username_at_remote_host = remote_host 133 | else: 134 | # Local username is the same as a remote username 135 | remote_connection_file = local_connection_file 136 | remote_username = local_username 137 | remote_host = remote_username_at_remote_host 138 | # Load a connection file 139 | with open(local_connection_file, 'r') as f: 140 | cfg = load(f) 141 | # GET a current working directory of a process 142 | cwd = getcwd() 143 | # Launch a kernel process on a remote machine 144 | gw = makegateway("ssh=%s//python=%s" % (remote_username_at_remote_host, 145 | interpreter)) 146 | ch = gw.remote_exec(""" 147 | import socket 148 | from json import dumps 149 | from os import chdir, getcwd, getpid, remove 150 | from os.path import exists, expanduser, isdir, isfile, join, split 151 | from struct import pack 152 | 153 | try: 154 | from ipykernel.kernelapp import launch_new_instance 155 | except ImportError: 156 | from IPython.kernel.zmq.kernelapp import launch_new_instance 157 | 158 | remote_connection_file = "%s" 159 | cfg = %s 160 | last_cwd = "%s" 161 | remote_ports = {} 162 | 163 | ports = [k for k,v in cfg.items() if k.endswith("_port")] 164 | # Select random ports 165 | for port in ports: 166 | sock = socket.socket() 167 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, pack("ii", 0, 0)) 168 | sock.bind(('', 0)) # Random free port from 1024 to 65535 169 | sock_name = sock.getsockname()[1] 170 | remote_ports[port] = sock_name 171 | cfg[port] = sock_name 172 | sock.close() 173 | channel.send(remote_ports) 174 | remote_pid = getpid() 175 | channel.send(remote_pid) 176 | if not exists(remote_connection_file): 177 | dir_name, file_name = split(remote_connection_file) 178 | if exists(dir_name) and isdir(dir_name): 179 | # Write a connection file 180 | with open(remote_connection_file, 'w') as f: 181 | f.write(dumps(cfg)) 182 | else: 183 | default_j4_dir_name = "/run/user/1000/jupyter" 184 | if ((default_j4_dir_name != dir_name) and 185 | exists(default_j4_dir_name) and 186 | isdir(default_j4_dir_name)): 187 | remote_connection_file = join(default_j4_dir_name, file_name) 188 | # Write a connection file to jupyter 4 "j4" default dir 189 | with open(remote_connection_file, 'w') as f: 190 | f.write(dumps(cfg)) 191 | else: 192 | path = "~/.ipython/profile_default/security" 193 | default_j3_dir_name = (expanduser(path)) 194 | if ((default_j3_dir_name != dir_name) and 195 | exists(default_j3_dir_name) and 196 | isdir(default_j3_dir_name)): 197 | remote_connection_file = join(default_j3_dir_name, 198 | file_name) 199 | # Write a connection file to jupyter 3 "j3" default dir 200 | with open(remote_connection_file, 'w') as f: 201 | f.write(dumps(cfg)) 202 | else: 203 | cwd = getcwd() 204 | remote_connection_file = join(cwd, file_name) 205 | # Write a connection file to cwd 206 | with open(remote_connection_file, 'w') as f: 207 | f.write(dumps(cfg)) 208 | # SET a current working directory of a process 209 | if exists(last_cwd) and isdir(last_cwd): 210 | chdir(last_cwd) 211 | launch_new_instance(["-f", remote_connection_file]) 212 | # Delete a connection file 213 | if exists(remote_connection_file) and isfile(remote_connection_file): 214 | remove(remote_connection_file) 215 | """ % (remote_connection_file, cfg, cwd)) 216 | # Local and remote ports dicts 217 | local_ports = {k: v for k,v in cfg.items() if k.endswith("_port")} 218 | remote_ports = ch.receive() 219 | # Local and remote PIDs 220 | local_pid = getpid() 221 | remote_pid = ch.receive() 222 | # Create paramiko log file 223 | paramiko_log_location, paramiko_log_file_name = split(local_connection_file) 224 | paramiko_log_file_name = paramiko_log_file_name.replace("kernel", "paramiko") 225 | paramiko_log_file_name = paramiko_log_file_name.replace(".json", ".txt") 226 | paramiko_log_abs_path = join(paramiko_log_location, paramiko_log_file_name) 227 | log_to_file(paramiko_log_abs_path) 228 | # Redirect localhost:local_port to remote_host:remote_port 229 | for k,v in local_ports.items(): 230 | paramiko_tunnel(v, remote_ports[k], remote_username_at_remote_host) 231 | # Create rk log file 232 | date_time = get_date_time() 233 | date, time = date_time.replace('.', ':').split('_') 234 | date = date + ' ' + week[datetime.weekday(datetime.now())] 235 | rk_log_file_name = "%s@%s_%s.txt" % (local_username, remote_host, date_time) 236 | rk_log_location = config["rk_log_location"] 237 | if '~' in rk_log_location: 238 | rk_log_location = expanduser(rk_log_location) 239 | rk_log_abs_path = join(rk_log_location, rk_log_file_name) 240 | if exists(rk_log_location) and isfile(rk_log_location): 241 | try: 242 | remove(rk_log_location) 243 | except Exception as exception: # Python3 PermissionError 244 | error_code = exception.errno 245 | if error_code == EACCES: # 13 246 | print(messages["_error_NoRoot"]) 247 | exit(1) 248 | else: 249 | print(messages["_error_Oops"] % strerror(error_code)) 250 | exit(1) 251 | if not exists(rk_log_location): 252 | create_directory(rk_log_location, 0o777) 253 | path = rk_log_location 254 | while path != '/': 255 | try: 256 | chmod(path, 0o777) 257 | except OSError: 258 | break 259 | path = dirname(path) 260 | try: 261 | with open(rk_log_abs_path, 'w') as f: 262 | f.write("date: %s\n" % date) 263 | f.write("time: %s\n" % time) 264 | f.write("\n") 265 | if local_username == remote_username: 266 | f.write("usernames: %s\n" % local_username) 267 | else: 268 | f.write("usernames: %s<->%s\n" % (local_username, remote_username)) 269 | f.write("remote host: %s\n" % remote_host) 270 | f.write("\n") 271 | for k,v in local_ports.items(): 272 | f.write("%ss: %s<->%s\n" % (k.replace('_', ' '), v, 273 | remote_ports[k])) 274 | f.write("\n") 275 | f.write("pids: %s<->%s\n" % (local_pid, remote_pid)) 276 | except Exception as exception: 277 | error_code = exception.errno 278 | if error_code == EACCES: # 13 (Python3 PermissionError) 279 | print(messages["_error_NoRoot"]) 280 | exit(1) 281 | else: 282 | print(messages["_error_Oops"] % strerror(error_code)) 283 | exit(1) 284 | # Waits for closing, i.e. remote_exec() finish 285 | ch.waitclose() 286 | # Delete paramiko log file 287 | if exists(paramiko_log_abs_path) and isfile(paramiko_log_abs_path): 288 | try: 289 | remove(paramiko_log_abs_path) 290 | except Exception as exception: 291 | error_code = exception.errno 292 | if error_code == EACCES: # 13 (Python3 PermissionError) 293 | print(messages["_error_NoRoot"]) 294 | exit(1) 295 | else: 296 | print(messages["_error_Oops"] % strerror(error_code)) 297 | exit(1) 298 | # Delete rk log file 299 | if exists(rk_log_abs_path) and isfile(rk_log_abs_path): 300 | try: 301 | remove(rk_log_abs_path) 302 | except Exception as exception: 303 | error_code = exception.errno 304 | if error_code == EACCES: # 13 (Python3 PermissionError) 305 | print(messages["_error_NoRoot"]) 306 | exit(1) 307 | else: 308 | print(messages["_error_Oops"] % strerror(error_code)) 309 | exit(1) 310 | -------------------------------------------------------------------------------- /rk/ssh/tunnel.py: -------------------------------------------------------------------------------- 1 | """Basic ssh tunnel utilities, and convenience functions for tunneling 2 | zeromq connections. 3 | """ 4 | 5 | # Copyright (C) 2010-2011 IPython Development Team 6 | # Copyright (C) 2011- PyZMQ Developers 7 | # 8 | # Redistributed from IPython under the terms of the BSD License. 9 | 10 | 11 | from __future__ import print_function 12 | 13 | import atexit 14 | import os 15 | import signal 16 | import socket 17 | import sys 18 | import warnings 19 | from getpass import getpass, getuser 20 | from multiprocessing import Process 21 | 22 | try: 23 | with warnings.catch_warnings(): 24 | warnings.simplefilter('ignore', DeprecationWarning) 25 | import paramiko 26 | SSHException = paramiko.ssh_exception.SSHException 27 | except ImportError: 28 | paramiko = None 29 | class SSHException(Exception): 30 | pass 31 | else: 32 | from .forward import forward_tunnel 33 | 34 | try: 35 | import pexpect 36 | except ImportError: 37 | pexpect = None 38 | 39 | 40 | _random_ports = set() 41 | 42 | def select_random_ports(n): 43 | """Selects and return n random ports that are available.""" 44 | ports = [] 45 | for i in range(n): 46 | sock = socket.socket() 47 | sock.bind(('', 0)) 48 | while sock.getsockname()[1] in _random_ports: 49 | sock.close() 50 | sock = socket.socket() 51 | sock.bind(('', 0)) 52 | ports.append(sock) 53 | for i, sock in enumerate(ports): 54 | port = sock.getsockname()[1] 55 | sock.close() 56 | ports[i] = port 57 | _random_ports.add(port) 58 | return ports 59 | 60 | 61 | #----------------------------------------------------------------------------- 62 | # Check for passwordless login 63 | #----------------------------------------------------------------------------- 64 | 65 | def try_passwordless_ssh(server, keyfile, paramiko=None): 66 | """Attempt to make an ssh connection without a password. 67 | This is mainly used for requiring password input only once 68 | when many tunnels may be connected to the same server. 69 | 70 | If paramiko is None, the default for the platform is chosen. 71 | """ 72 | if paramiko is None: 73 | paramiko = sys.platform == 'win32' 74 | if not paramiko: 75 | f = _try_passwordless_openssh 76 | else: 77 | f = _try_passwordless_paramiko 78 | return f(server, keyfile) 79 | 80 | def _try_passwordless_openssh(server, keyfile): 81 | """Try passwordless login with shell ssh command.""" 82 | if pexpect is None: 83 | raise ImportError("pexpect unavailable, use paramiko") 84 | cmd = 'ssh -f '+ server 85 | if keyfile: 86 | cmd += ' -i ' + keyfile 87 | cmd += ' exit' 88 | 89 | # pop SSH_ASKPASS from env 90 | env = os.environ.copy() 91 | env.pop('SSH_ASKPASS', None) 92 | 93 | ssh_newkey = 'Are you sure you want to continue connecting' 94 | p = pexpect.spawn(cmd, env=env) 95 | while True: 96 | try: 97 | i = p.expect([ssh_newkey, '[Pp]assword:'], timeout=.1) 98 | if i==0: 99 | raise SSHException('The authenticity of the host can\'t be established.') 100 | except pexpect.TIMEOUT: 101 | continue 102 | except pexpect.EOF: 103 | return True 104 | else: 105 | return False 106 | 107 | def _try_passwordless_paramiko(server, keyfile): 108 | """Try passwordless login with paramiko.""" 109 | if paramiko is None: 110 | msg = "Paramiko unavaliable, " 111 | if sys.platform == 'win32': 112 | msg += "Paramiko is required for ssh tunneled connections on Windows." 113 | else: 114 | msg += "use OpenSSH." 115 | raise ImportError(msg) 116 | username, server, port = _split_server(server) 117 | client = paramiko.SSHClient() 118 | client.load_system_host_keys() 119 | #client.set_missing_host_key_policy(paramiko.WarningPolicy()) 120 | client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 121 | try: 122 | client.connect(server, port, username=username, key_filename=keyfile, 123 | look_for_keys=True) 124 | except paramiko.AuthenticationException: 125 | return False 126 | else: 127 | client.close() 128 | return True 129 | 130 | 131 | def tunnel_connection(socket, addr, server, keyfile=None, password=None, paramiko=None, timeout=60): 132 | """Connect a socket to an address via an ssh tunnel. 133 | 134 | This is a wrapper for socket.connect(addr), when addr is not accessible 135 | from the local machine. It simply creates an ssh tunnel using the remaining args, 136 | and calls socket.connect('tcp://localhost:lport') where lport is the randomly 137 | selected local port of the tunnel. 138 | 139 | """ 140 | new_url, tunnel = open_tunnel(addr, server, keyfile=keyfile, password=password, paramiko=paramiko, timeout=timeout) 141 | socket.connect(new_url) 142 | return tunnel 143 | 144 | 145 | def open_tunnel(addr, server, keyfile=None, password=None, paramiko=None, timeout=60): 146 | """Open a tunneled connection from a 0MQ url. 147 | 148 | For use inside tunnel_connection. 149 | 150 | Returns 151 | ------- 152 | 153 | (url, tunnel) : (str, object) 154 | The 0MQ url that has been forwarded, and the tunnel object 155 | """ 156 | 157 | lport = select_random_ports(1)[0] 158 | transport, addr = addr.split('://') 159 | ip,rport = addr.split(':') 160 | rport = int(rport) 161 | if paramiko is None: 162 | paramiko = sys.platform == 'win32' 163 | if paramiko: 164 | tunnelf = paramiko_tunnel 165 | else: 166 | tunnelf = openssh_tunnel 167 | 168 | tunnel = tunnelf(lport, rport, server, remoteip=ip, keyfile=keyfile, password=password, timeout=timeout) 169 | return 'tcp://127.0.0.1:%i'%lport, tunnel 170 | 171 | def openssh_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=60): 172 | """Create an ssh tunnel using command-line ssh that connects port lport 173 | on this machine to localhost:rport on server. The tunnel 174 | will automatically close when not in use, remaining open 175 | for a minimum of timeout seconds for an initial connection. 176 | 177 | This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`, 178 | as seen from `server`. 179 | 180 | keyfile and password may be specified, but ssh config is checked for defaults. 181 | 182 | Parameters 183 | ---------- 184 | 185 | lport : int 186 | local port for connecting to the tunnel from this machine. 187 | rport : int 188 | port on the remote machine to connect to. 189 | server : str 190 | The ssh server to connect to. The full ssh server string will be parsed. 191 | user@server:port 192 | remoteip : str [Default: 127.0.0.1] 193 | The remote ip, specifying the destination of the tunnel. 194 | Default is localhost, which means that the tunnel would redirect 195 | localhost:lport on this machine to localhost:rport on the *server*. 196 | 197 | keyfile : str; path to public key file 198 | This specifies a key to be used in ssh login, default None. 199 | Regular default ssh keys will be used without specifying this argument. 200 | password : str; 201 | Your ssh password to the ssh server. Note that if this is left None, 202 | you will be prompted for it if passwordless key based login is unavailable. 203 | timeout : int [default: 60] 204 | The time (in seconds) after which no activity will result in the tunnel 205 | closing. This prevents orphaned tunnels from running forever. 206 | """ 207 | if pexpect is None: 208 | raise ImportError("pexpect unavailable, use paramiko_tunnel") 209 | ssh="ssh " 210 | if keyfile: 211 | ssh += "-i " + keyfile 212 | 213 | if ':' in server: 214 | server, port = server.split(':') 215 | ssh += " -p %s" % port 216 | 217 | cmd = "%s -O check %s" % (ssh, server) 218 | (output, exitstatus) = pexpect.run(cmd, withexitstatus=True) 219 | if not exitstatus: 220 | pid = int(output[output.find("(pid=")+5:output.find(")")]) 221 | cmd = "%s -O forward -L 127.0.0.1:%i:%s:%i %s" % ( 222 | ssh, lport, remoteip, rport, server) 223 | (output, exitstatus) = pexpect.run(cmd, withexitstatus=True) 224 | if not exitstatus: 225 | atexit.register(_stop_tunnel, cmd.replace("-O forward", "-O cancel", 1)) 226 | return pid 227 | cmd = "%s -f -S none -L 127.0.0.1:%i:%s:%i %s sleep %i" % ( 228 | ssh, lport, remoteip, rport, server, timeout) 229 | 230 | # pop SSH_ASKPASS from env 231 | env = os.environ.copy() 232 | env.pop('SSH_ASKPASS', None) 233 | 234 | ssh_newkey = 'Are you sure you want to continue connecting' 235 | tunnel = pexpect.spawn(cmd, env=env) 236 | failed = False 237 | while True: 238 | try: 239 | i = tunnel.expect([ssh_newkey, '[Pp]assword:'], timeout=.1) 240 | if i==0: 241 | raise SSHException('The authenticity of the host can\'t be established.') 242 | except pexpect.TIMEOUT: 243 | continue 244 | except pexpect.EOF: 245 | if tunnel.exitstatus: 246 | print(tunnel.exitstatus) 247 | print(tunnel.before) 248 | print(tunnel.after) 249 | raise RuntimeError("tunnel '%s' failed to start"%(cmd)) 250 | else: 251 | return tunnel.pid 252 | else: 253 | if failed: 254 | print("Password rejected, try again") 255 | password=None 256 | if password is None: 257 | password = getpass("%s's password: "%(server)) 258 | tunnel.sendline(password) 259 | failed = True 260 | 261 | def _stop_tunnel(cmd): 262 | pexpect.run(cmd) 263 | 264 | def _split_server(server): 265 | if '@' in server: 266 | username,server = server.split('@', 1) 267 | else: 268 | username = getuser() 269 | if ':' in server: 270 | server, port = server.split(':') 271 | port = int(port) 272 | else: 273 | port = 22 274 | return username, server, port 275 | 276 | def paramiko_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=60): 277 | """launch a tunner with paramiko in a subprocess. This should only be used 278 | when shell ssh is unavailable (e.g. Windows). 279 | 280 | This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`, 281 | as seen from `server`. 282 | 283 | If you are familiar with ssh tunnels, this creates the tunnel: 284 | 285 | ssh server -L localhost:lport:remoteip:rport 286 | 287 | keyfile and password may be specified, but ssh config is checked for defaults. 288 | 289 | 290 | Parameters 291 | ---------- 292 | 293 | lport : int 294 | local port for connecting to the tunnel from this machine. 295 | rport : int 296 | port on the remote machine to connect to. 297 | server : str 298 | The ssh server to connect to. The full ssh server string will be parsed. 299 | user@server:port 300 | remoteip : str [Default: 127.0.0.1] 301 | The remote ip, specifying the destination of the tunnel. 302 | Default is localhost, which means that the tunnel would redirect 303 | localhost:lport on this machine to localhost:rport on the *server*. 304 | 305 | keyfile : str; path to public key file 306 | This specifies a key to be used in ssh login, default None. 307 | Regular default ssh keys will be used without specifying this argument. 308 | password : str; 309 | Your ssh password to the ssh server. Note that if this is left None, 310 | you will be prompted for it if passwordless key based login is unavailable. 311 | timeout : int [default: 60] 312 | The time (in seconds) after which no activity will result in the tunnel 313 | closing. This prevents orphaned tunnels from running forever. 314 | 315 | """ 316 | if paramiko is None: 317 | raise ImportError("Paramiko not available") 318 | 319 | if password is None: 320 | if not _try_passwordless_paramiko(server, keyfile): 321 | password = getpass("%s's password: "%(server)) 322 | 323 | p = Process(target=_paramiko_tunnel, 324 | args=(lport, rport, server, remoteip), 325 | kwargs=dict(keyfile=keyfile, password=password)) 326 | p.daemon=False 327 | p.start() 328 | atexit.register(_shutdown_process, p) 329 | return p 330 | 331 | def _shutdown_process(p): 332 | if p.is_alive(): 333 | p.terminate() 334 | 335 | def _paramiko_tunnel(lport, rport, server, remoteip, keyfile=None, password=None): 336 | """Function for actually starting a paramiko tunnel, to be passed 337 | to multiprocessing.Process(target=this), and not called directly. 338 | """ 339 | username, server, port = _split_server(server) 340 | client = paramiko.SSHClient() 341 | client.load_system_host_keys() 342 | #client.set_missing_host_key_policy(paramiko.WarningPolicy()) 343 | client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 344 | 345 | try: 346 | client.connect(server, port, username=username, key_filename=keyfile, 347 | look_for_keys=True, password=password) 348 | # except paramiko.AuthenticationException: 349 | # if password is None: 350 | # password = getpass("%s@%s's password: "%(username, server)) 351 | # client.connect(server, port, username=username, password=password) 352 | # else: 353 | # raise 354 | except Exception as e: 355 | print('*** Failed to connect to %s:%d: %r' % (server, port, e)) 356 | sys.exit(1) 357 | 358 | # Don't let SIGINT kill the tunnel subprocess 359 | signal.signal(signal.SIGINT, signal.SIG_IGN) 360 | 361 | try: 362 | forward_tunnel(lport, remoteip, rport, client.get_transport()) 363 | except KeyboardInterrupt: 364 | print('SIGINT: Port forwarding stopped cleanly') 365 | sys.exit(0) 366 | except Exception as e: 367 | print("Port forwarding stopped uncleanly: %s"%e) 368 | sys.exit(255) 369 | 370 | if sys.platform == 'win32': 371 | ssh_tunnel = paramiko_tunnel 372 | else: 373 | ssh_tunnel = openssh_tunnel 374 | 375 | 376 | __all__ = ['tunnel_connection', 'ssh_tunnel', 'openssh_tunnel', 'paramiko_tunnel', 'try_passwordless_ssh'] 377 | 378 | 379 | -------------------------------------------------------------------------------- /rk/rk.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from argparse import ArgumentParser 4 | from errno import EACCES, ENOTDIR 5 | from getpass import getuser 6 | from json import dumps, load 7 | from os import link, listdir, makedirs, remove, strerror 8 | from os.path import dirname, exists, expanduser, isdir, isfile, join 9 | from shutil import copyfile, rmtree 10 | from subprocess import call 11 | from sys import argv, exit 12 | 13 | from configobj import ConfigObj 14 | 15 | module_location = dirname(__file__) 16 | config_rk_abs_path = join(module_location, "config/rk.ini") 17 | config = ConfigObj(config_rk_abs_path) 18 | 19 | argparse = {} # Strings for -h --help 20 | messages = {} # Strings for output 21 | 22 | def create_dictionaries(): 23 | """Create "argparse" and "messages" dictionaries""" 24 | 25 | config_argparse_rel_path = config["config_argparse_rel_path"] 26 | config_argparse_abs_path = join(module_location, config_argparse_rel_path) 27 | config_messages_rel_path = config["config_messages_rel_path"] 28 | config_messages_abs_path = join(module_location, config_messages_rel_path) 29 | with open(config_argparse_abs_path, 'r') as f: 30 | argparse_list = f.read().splitlines() 31 | for i in range(0, len(argparse_list), 2): 32 | argparse[argparse_list[i]] = argparse_list[i+1] 33 | with open(config_messages_abs_path, 'r') as f: 34 | messages_list = f.read().splitlines() 35 | for i in range(0, len(messages_list), 2): 36 | messages[messages_list[i]] = messages_list[i+1] 37 | 38 | def install_all(args): 39 | """Install all remote jupyter kernels from kernels dict""" 40 | 41 | config_kernels_rel_path = config["config_kernels_rel_path"] 42 | config_kernels_abs_path = join(module_location, config_kernels_rel_path) 43 | # Load kernels.json file 44 | with open(config_kernels_abs_path, 'r') as f: 45 | kernels_dict = load(f) 46 | # Create kernels list from kernels dict 47 | kernels_list = [k for k in kernels_dict.keys()] 48 | # Sort kernels list 49 | kernels_list.sort() 50 | # Install remote jupyter kernels 51 | args.kernel_names = kernels_list 52 | install_kernel(args) 53 | 54 | def install_kernel(args): 55 | """Install remote jupyter kernel/kernels""" 56 | 57 | def copy_logos(img_location, logo_name_srt, destination): 58 | """Copy logos""" 59 | 60 | for size in ["32", "64"]: 61 | logo_abs_path_str = join(join(module_location, img_location), 62 | logo_name_srt) 63 | logo_abs_path = logo_abs_path_str.format(size) 64 | logo_name = logo_name_srt.format(size) 65 | if exists(logo_abs_path) and isfile(logo_abs_path): 66 | try: 67 | link(logo_abs_path, join(destination, logo_name)) 68 | except Exception: 69 | try: 70 | copyfile(logo_abs_path, join(destination, logo_name)) 71 | except Exception as exception: # Python3 PermissionError 72 | error_code = exception.errno 73 | if error_code == EACCES: # 13 74 | print(messages["_error_NoRoot"]) 75 | exit(1) 76 | else: 77 | print(messages["_error_Oops"] % 78 | strerror(error_code)) 79 | exit(1) 80 | 81 | def create_directory(directory_name, mode=0o777): 82 | """Recursive directory creation function 83 | os.chmod work only for last directory 84 | 85 | """ 86 | 87 | try: 88 | makedirs(directory_name, mode) 89 | except Exception as exception: 90 | error_code = exception.errno 91 | if error_code == EACCES: # 13 (Python3 PermissionError) 92 | print(messages["_error_NoRoot"]) 93 | exit(1) 94 | elif error_code == ENOTDIR: # 20 (Python3 NotADirectoryError) 95 | path = directory_name 96 | while path != '/': 97 | if isfile(path): 98 | try: 99 | remove(path) 100 | except Exception as exception: # Python3 101 | # PermissionError 102 | error_code = exception.errno 103 | if error_code == EACCES: # 13 104 | print(messages["_error_NoRoot"]) 105 | exit(1) 106 | else: 107 | print(messages["_error_Oops"] % 108 | strerror(error_code)) 109 | exit(1) 110 | path = dirname(path) 111 | try: 112 | makedirs(directory_name, mode) 113 | except Exception as exception: # Python3 PermissionError 114 | error_code = exception.errno 115 | if error_code == EACCES: # 13 116 | print(messages["_error_NoRoot"]) 117 | exit(1) 118 | else: 119 | print(messages["_error_Oops"] % strerror(error_code)) 120 | exit(1) 121 | else: 122 | print(messages["_error_Oops"] % strerror(error_code)) 123 | exit(1) 124 | 125 | def create_kernel_json_file(display_name, language, script, interpreter, 126 | connection_file, remote_host, destination): 127 | """Create kernel.json file""" 128 | 129 | kernel_dict = {"argv": [], "display_name": display_name, 130 | "language": language} 131 | kernel_dict["argv"].append(script) 132 | kernel_dict["argv"].append(interpreter) 133 | kernel_dict["argv"].append(connection_file) 134 | kernel_dict["argv"].append(remote_host) 135 | try: 136 | with open(join(destination, "kernel.json"), 'w') as f: 137 | f.write(dumps(kernel_dict, indent=1, sort_keys=True)) 138 | except Exception as exception: # Python3 PermissionError 139 | error_code = exception.errno 140 | if error_code == EACCES: # 13 141 | print(messages["_error_NoRoot"]) 142 | exit(1) 143 | else: 144 | print(messages["_error_Oops"] % strerror(error_code)) 145 | exit(1) 146 | 147 | kernels_location = config["kernels_location"] 148 | if '~' in kernels_location: 149 | kernels_location = expanduser(kernels_location) 150 | img_location = config["img_location"] 151 | logo_name_srt = config["logo_name_srt"] 152 | script = config["script"] 153 | connection_file = config["connection_file"] 154 | config_kernels_rel_path = config["config_kernels_rel_path"] 155 | config_kernels_abs_path = join(module_location, 156 | config_kernels_rel_path) 157 | kernel_names = args.kernel_names 158 | if kernel_names == None: 159 | # Install template of remote kernel 160 | kernel_name = config["kernel_name"] 161 | display_name = config["display_name"] 162 | language = config["language"] 163 | interpreter = config["interpreter"] 164 | remote_host = config["remote_host"] 165 | kernel_abs_path = join(kernels_location, kernel_name) 166 | if exists(kernel_abs_path) and isfile(kernel_abs_path): 167 | try: 168 | remove(kernel_abs_path) 169 | except Exception as exception: # Python3 PermissionError 170 | error_code = exception.errno 171 | if error_code == EACCES: # 13 172 | print(messages["_error_NoRoot"]) 173 | exit(1) 174 | else: 175 | print(messages["_error_Oops"] % strerror(error_code)) 176 | exit(1) 177 | if not exists(kernel_abs_path): 178 | # Create directory 179 | create_directory(kernel_abs_path, 0o755) 180 | # Copy logos 181 | copy_logos(img_location, logo_name_srt, kernel_abs_path) 182 | # Create kernel.json 183 | create_kernel_json_file(display_name, language, script, 184 | interpreter, connection_file, 185 | remote_host, kernel_abs_path) 186 | print(messages["_installed_template"]) 187 | else: 188 | print(messages["_delete_template"]) 189 | answer = raw_input() 190 | answer_lower = answer.lower() 191 | if ((answer_lower == 'y') or (answer_lower == 'yes') or 192 | (answer_lower == 'yep')): 193 | uninstall_kernel(args) 194 | install_kernel(args) 195 | else: 196 | # Install kernel/kernels 197 | # Load kernels.json file 198 | with open(config_kernels_abs_path, 'r') as f: 199 | kernels_dict = load(f) 200 | # Check kernel_names list/ 201 | no_kernel_names = [] 202 | for kernel_name in kernel_names: 203 | if kernel_name not in kernels_dict: 204 | no_kernel_names.append(kernel_name) 205 | if len(no_kernel_names) != 0: 206 | if len(no_kernel_names) == 1: 207 | print(messages["_error_NoKernel"] % no_kernel_names[0]) 208 | else: 209 | print(messages["_error_NoKernels"] % 210 | '\' \''.join(no_kernel_names)) 211 | exit(1) 212 | # /Check kernel_names list 213 | for kernel_name in kernel_names: 214 | display_name = kernels_dict[kernel_name]["display_name"] 215 | language = kernels_dict[kernel_name]["language"] 216 | interpreter = kernels_dict[kernel_name]["interpreter"] 217 | remote_host = kernels_dict[kernel_name]["remote_host"] 218 | kernel_abs_path = join(kernels_location, kernel_name) 219 | if exists(kernel_abs_path) and isfile(kernel_abs_path): 220 | try: 221 | remove(kernel_abs_path) 222 | except Exception as exception: # Python3 PermissionError 223 | error_code = exception.errno 224 | if error_code == EACCES: # 13 225 | print(messages["_error_NoRoot"]) 226 | exit(1) 227 | else: 228 | print(messages["_error_Oops"] % strerror(error_code)) 229 | exit(1) 230 | if not exists(kernel_abs_path): 231 | # Create directory 232 | create_directory(kernel_abs_path, 0o755) 233 | # Copy logos 234 | copy_logos(img_location, logo_name_srt, kernel_abs_path) 235 | # Create kernel.json 236 | create_kernel_json_file(display_name, language, script, 237 | interpreter, connection_file, 238 | remote_host, kernel_abs_path) 239 | print(messages["_installed"] % kernel_name) 240 | else: 241 | print(messages["_delete"] % kernel_name) 242 | answer = raw_input() 243 | answer_lower = answer.lower() 244 | if ((answer_lower == 'y') or (answer_lower == 'yes') or 245 | (answer_lower == 'yep')): 246 | args.kernel_names = [kernel_name] 247 | uninstall_kernel(args) 248 | install_kernel(args) 249 | 250 | def main(): 251 | """Main function""" 252 | 253 | create_dictionaries() 254 | args = parse_command_line_args() 255 | args.function_name(args) 256 | 257 | def parse_command_line_args(): 258 | """Parse command line arguments""" 259 | 260 | # Create top parser 261 | parser = ArgumentParser(prog="rk", description=argparse["_parser"], 262 | add_help=True) 263 | parser.add_argument("-v", "--version", action="version", 264 | version="rk 0.3b1") 265 | # Create subparsers for the top parser 266 | subparsers = parser.add_subparsers(title=argparse["_subparsers"]) 267 | # Create the parser for the "list" subcommand 268 | parser_list = subparsers.add_parser("list", 269 | description=argparse["_parser_list"], 270 | help=argparse["_parser_list"]) 271 | parser_list.set_defaults(function_name=show_kernels_list) 272 | # Create the parser for the "install" subcommand 273 | parser_install = subparsers.add_parser("install", 274 | description=argparse["_parser_install"], 275 | help=argparse["_parser_install"]) 276 | parser_install.add_argument("kernel_names", action="store", nargs='+', 277 | metavar="KERNEL_NAME") 278 | parser_install.set_defaults(function_name=install_kernel) 279 | # Create the parser for the "install-template" subcommand 280 | parser_install_template = subparsers.add_parser("install-template", 281 | description=argparse["_parser_install_template"], 282 | help=argparse["_parser_install_template"]) 283 | parser_install_template.set_defaults(function_name=install_kernel, 284 | kernel_names=None) 285 | # Create the parser for the "install-all" subcommand 286 | parser_install_all = subparsers.add_parser("install-all", 287 | description=argparse["_parser_install_all"], 288 | help=argparse["_parser_install_all"]) 289 | parser_install_all.set_defaults(function_name=install_all) 290 | # Create the parser for the "uninstall" subcommand 291 | parser_uninstall= subparsers.add_parser("uninstall", 292 | description=argparse["_parser_uninstall"], 293 | help=argparse["_parser_uninstall"]) 294 | parser_uninstall.add_argument("kernel_names", action="store", nargs='+', 295 | metavar="KERNEL_NAME") 296 | parser_uninstall.set_defaults(function_name=uninstall_kernel) 297 | # Create the parser for the "uninstall-template" subcommand 298 | parser_uninstall_template = subparsers.add_parser("uninstall-template", 299 | description=argparse["_parser_uninstall_template"], 300 | help=argparse["_parser_uninstall_template"]) 301 | parser_uninstall_template.set_defaults(function_name=uninstall_kernel, 302 | kernel_names=None) 303 | # Create the parser for the "uninstall-all" subcommand 304 | parser_uninstall_all = subparsers.add_parser("uninstall-all", 305 | description=argparse["_parser_uninstall_all"], 306 | help=argparse["_parser_uninstall_all"]) 307 | parser_uninstall_all.set_defaults(function_name=uninstall_all) 308 | # Create the parser for the "ssh" subcommand 309 | parser_list = subparsers.add_parser("ssh", 310 | description=argparse["_parser_ssh"], 311 | help=argparse["_parser_ssh"]) 312 | parser_list.set_defaults(function_name=setup_ssh_auto_login) 313 | if len(argv) == 1: 314 | parser.print_help() 315 | exit(0) # Clean exit without any errors/problems 316 | return parser.parse_args() 317 | 318 | def setup_ssh_auto_login(args): 319 | """Setup SSH for auto login without a password""" 320 | 321 | keys_location = "~/.ssh" 322 | pri_key_paths = ["~/.ssh/id_dsa", "~/.ssh/id_ecdsa", "~/.ssh/id_ed25519", 323 | "~/.ssh/id_rsa"] 324 | 325 | # Check current keys 326 | total_keys_flag = False 327 | pri_key_flag = False 328 | pub_key_flag = False 329 | for pri_key_path in pri_key_paths: 330 | pri_key_abs_path = expanduser(pri_key_path) 331 | if exists(pri_key_abs_path) and isfile(pri_key_abs_path): 332 | pri_key_flag = True 333 | pub_key_abs_path = pri_key_abs_path + ".pub" 334 | if exists(pub_key_abs_path) and isfile(pub_key_abs_path): 335 | pub_key_flag = True 336 | if (pri_key_flag == True) and (pub_key_flag == True): 337 | total_keys_flag = True 338 | break 339 | else: 340 | pri_key_flag = False 341 | pub_key_flag = False 342 | if total_keys_flag == False: 343 | # Check keys dir 344 | keys_dir = expanduser(keys_location) 345 | if not exists(keys_dir): 346 | # Create keys dir 347 | makedirs(keys_dir) 348 | # Create a public and a private keys using the ssh-keygen command 349 | call("ssh-keygen -t rsa -b 4096 -N '' -f ~/.ssh/id_rsa", shell=True) 350 | # Ask about a remote machine 351 | print(messages["_ask_remote_host"]) 352 | remote_username_at_remote_host = raw_input() 353 | if '@' in remote_username_at_remote_host: 354 | l_username = getuser() 355 | r_username, r_host = remote_username_at_remote_host.split('@') 356 | if l_username == r_username: 357 | # Local username is the same as a remote username 358 | remote_username_at_remote_host = r_host 359 | # Copy a public key to a remote machine using the ssh-copy-id command 360 | call("ssh-copy-id %s" % remote_username_at_remote_host, shell=True) 361 | # Ensure ssh-agent is enabled 362 | call("eval \"$(ssh-agent -s)\"", shell=True) 363 | # Adds private key identities to the authentication agent 364 | call("ssh-add ~/.ssh/id_rsa", shell=True) 365 | 366 | def show_kernels_list(args): 367 | """Show list of remote jupyter kernels from kernels dict""" 368 | 369 | config_kernels_rel_path = config["config_kernels_rel_path"] 370 | config_kernels_abs_path = join(module_location, config_kernels_rel_path) 371 | # Load kernels.json file 372 | with open(config_kernels_abs_path, 'r') as f: 373 | kernels_dict = load(f) 374 | # Create kernels list from kernels dict 375 | kernels_list = [k for k in kernels_dict.keys()] 376 | # Sort kernels list 377 | kernels_list.sort() 378 | # Print kernels list 379 | for kernel in kernels_list: 380 | print("%s (display name: \"%s\")" % (kernel, 381 | kernels_dict[kernel]["display_name"])) 382 | 383 | def uninstall_all(args): 384 | """Uninstall all jupyter kernels from kernels location""" 385 | 386 | kernels_location = config["kernels_location"] 387 | if '~' in kernels_location: 388 | kernels_location = expanduser(kernels_location) 389 | kernel_names = [] 390 | for element in listdir(kernels_location): 391 | element_abs_path = join(kernels_location, element) 392 | if isdir(element_abs_path): 393 | try: 394 | rmtree(element_abs_path) 395 | except Exception as exception: # Python3 PermissionError 396 | error_code = exception.errno 397 | if error_code == EACCES: # 13 398 | print(messages["_error_NoRoot"]) 399 | exit(1) 400 | else: 401 | print(messages["_error_Oops"] % strerror(error_code)) 402 | exit(1) 403 | kernel_names.append(element) 404 | kernel_names.sort() 405 | if len(kernel_names) == 0: 406 | print(messages["_uninstalled_all_zero"]) 407 | elif len(kernel_names) == 1: 408 | print(messages["_uninstalled_all"] % kernel_names[0]) 409 | else: 410 | print(messages["_uninstalled_all_multiple"] % 411 | '\' \''.join(kernel_names)) 412 | 413 | def uninstall_kernel(args): 414 | """Uninstall remote jupyter kernel/kernels""" 415 | 416 | kernels_location = config["kernels_location"] 417 | if '~' in kernels_location: 418 | kernels_location = expanduser(kernels_location) 419 | kernel_names = args.kernel_names 420 | if kernel_names == None: 421 | # Uninstall template of remote kernel 422 | kernel_name = config["kernel_name"] 423 | kernel_abs_path = join(kernels_location, kernel_name) 424 | if exists(kernel_abs_path): 425 | if isdir(kernel_abs_path): 426 | try: 427 | rmtree(kernel_abs_path) 428 | except Exception as exception: # Python3 PermissionError 429 | error_code = exception.errno 430 | if error_code == EACCES: # 13 431 | print(messages["_error_NoRoot"]) 432 | exit(1) 433 | else: 434 | print(messages["_error_Oops"] % strerror(error_code)) 435 | exit(1) 436 | elif isfile(kernel_abs_path): 437 | try: 438 | remove(kernel_abs_path) 439 | except Exception as exception: # Python3 PermissionError 440 | error_code = exception.errno 441 | if error_code == EACCES: # 13 442 | print(messages["_error_NoRoot"]) 443 | exit(1) 444 | else: 445 | print(messages["_error_Oops"] % strerror(error_code)) 446 | exit(1) 447 | print(messages["_uninstalled_template"]) 448 | else: 449 | print(messages["_error_NoTemplate"]) 450 | exit(1) 451 | else: 452 | # Uninstall kernel/kernels 453 | # Check kernel_names list/ 454 | no_kernel_names = [] 455 | for kernel_name in kernel_names: 456 | kernel_abs_path = join(kernels_location, kernel_name) 457 | if not exists(kernel_abs_path): 458 | no_kernel_names.append(kernel_name) 459 | if len(no_kernel_names) != 0: 460 | if len(no_kernel_names) == 1: 461 | print(messages["_error_NoKernel"] % kernel_name) 462 | else: 463 | print(messages["_error_NoKernels"] % 464 | '\' \''.join(no_kernel_names)) 465 | exit(1) 466 | # /Check kernel_names list 467 | for kernel_name in kernel_names: 468 | kernel_abs_path = join(kernels_location, kernel_name) 469 | if isdir(kernel_abs_path): 470 | try: 471 | rmtree(kernel_abs_path) 472 | except Exception as exception: # Python3 PermissionError 473 | error_code = exception.errno 474 | if error_code == EACCES: # 13 475 | print(messages["_error_NoRoot"]) 476 | exit(1) 477 | else: 478 | print(messages["_error_Oops"] % strerror(error_code)) 479 | exit(1) 480 | elif isfile(kernel_abs_path): 481 | try: 482 | remove(kernel_abs_path) 483 | except Exception as exception: # Python3 PermissionError 484 | error_code = exception.errno 485 | if error_code == EACCES: # 13 486 | print(messages["_error_NoRoot"]) 487 | exit(1) 488 | else: 489 | print(messages["_error_Oops"] % strerror(error_code)) 490 | exit(1) 491 | print(messages["_uninstalled"] % kernel_name) 492 | --------------------------------------------------------------------------------