├── .gitignore ├── CHANGELOG ├── LICENSE ├── MANIFEST.in ├── README.rst ├── setup.cfg ├── setup.py ├── userdocker.py └── userdocker ├── __init__.py ├── config ├── __init__.py └── default.py ├── helpers ├── __init__.py ├── cmd.py ├── exceptions.py ├── execute.py ├── logger.py ├── nvidia.py └── parser.py ├── parser.py ├── subcommands ├── __init__.py ├── attach.py ├── dockviz.py ├── images.py ├── ps.py ├── pull.py ├── run.py └── version.py └── userdocker.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | Version 2.1.0 2 | ============= 3 | 4 | This is a minor release. 5 | 6 | New features: 7 | ------------- 8 | 9 | - NV_ALLOW_OWN_GPU_REUSE allows users to run further containers on GPUs they 10 | already use. Will only happen when explicitly requesting a GPU via NV_GPU, not 11 | when implicitly assigning a new GPU. 12 | - ARGS_ALWAYS and ARGS_AVAILABLE allow assignments in simple string args making 13 | exactly this `--arg=val` combination mandatory/available to the user. 14 | 15 | Minor improvements: 16 | ------------------- 17 | 18 | - Print warnings in case no config.py is found, #2 19 | - Enhanced docs for mounting /etc/{passwd,group} for uid to username mappings in 20 | containers, #5 21 | - Help now lists available mounts and allowed images 22 | 23 | Bug fixes: 24 | ---------- 25 | 26 | - Removed allow_abbrev to allow flag combinations, #3 27 | - Fixed parsing dups bug in ARGS_AVAILABLE and ARGS_ALWAYS, #4 28 | - GPU assignment is now saved in USERDOCKER_NV_GPU container env var for later 29 | use as docker's inspect was less stable than expected ("HostConfig" seems to 30 | have disappeared) 31 | 32 | 33 | 34 | Version 2.0.0 35 | ============= 36 | 37 | This is a major release. 38 | 39 | Backwards incompatibilities: 40 | ---------------------------- 41 | 42 | - VOLUME_MOUNT config vars could previously include {USER} and {HOME} strings. 43 | This is now replaced by user_name and user_home python variables that can be 44 | used throughout all config files (and in all settings, not only mounts). 45 | - HOME env var in containers is set to user_home by default now. 46 | - Executor nvidia-docker now includes configurable limits to NV_GPU env var. 47 | Before the default was to always make all GPUs available (see new config 48 | options below). 49 | - PRIVILEGED dropped (use ARGS_AVAILABLE, but let me know how this is useful 50 | with userdocker!) 51 | - ALLOWED_PUBLISH_PORTS_ALL dropped (use ARGS_AVAILABLE) 52 | 53 | New features: 54 | ------------- 55 | 56 | - Config files now support python variables: uid, gid, user_name, group_name and 57 | user_home. 58 | - userdocker version command will include userdocker version information. 59 | - Use of DOCKER_HOST env var will raise an ERROR. 60 | - Users can now re-attach to their previously started containers (in case of 61 | connection loss for example). 62 | - ALLOWED_PORT_MAPPINGS config var to allow configurable explicit user port 63 | publishing (docker run -p). Defaults to selectable container ports that are 64 | mappable to random host port (like in -P), but user can decide if host local 65 | or world accessible. 66 | - Improved support for nvidia-docker's NV_GPU env var, which is now checked 67 | against admin config options: 68 | 69 | - User available GPU(s) configurable, including defaults in case NV_GPU isn't 70 | specified (NV_ALLOWED_GPUS, NV_MAX_GPU_COUNT_RESERVATION, 71 | NV_DEFAULT_GPU_COUNT_RESERVATION) 72 | - Optional exclusive GPU usage per container via the 73 | NV_EXCLUSIVE_CONTAINER_GPU_RESERVATION config option. 74 | - Optional exclusion of otherwise busy GPUs from being available to user 75 | containers (NV_GPU_UNAVAILABLE_ABOVE_MEMORY_USED) 76 | - Users can list GPUs that are used by other containers 77 | (userdocker ps --gpu-used) 78 | - Users can list free GPUs (userdocker ps --gpu-free) 79 | 80 | Bug fixes: 81 | ---------- 82 | 83 | - Error message in case of missing executable improved. 84 | 85 | 86 | 87 | Version 1.0.1 88 | ============= 89 | 90 | This is a patch release. 91 | 92 | Bug fixes: 93 | ---------- 94 | 95 | - Mitigation for nvidia-docker "--" commandline bug. 96 | 97 | 98 | 99 | Version 1.0.0 100 | ============= 101 | 102 | Initial release. 103 | 104 | Features: 105 | --------- 106 | 107 | - Similar commandline interface as ``docker ...`` called ``userdocker ...`` 108 | - Support for several docker commands / plugins (docker, nvidia-docker) 109 | - Fine granular configurability for admins in ``/etc/userdocker/`` allows to: 110 | 111 | - restrict runnable images if desired (allows admin reviews) 112 | - restrict run to locally available images 113 | - restrict available mount points (or enforce them, or default mount) 114 | - probe mounts (to make sure nfs automounts don't make docker sad) 115 | - enforce non-root user in container (same uid:gid as on host) 116 | - enforce dropping caps 117 | - enforce environment vars 118 | - enforce docker args 119 | - restrict port publishing 120 | - explicitly white-list available args to user 121 | 122 | - System wide config + overrides for individual groups, gids, users, uids. 123 | - Easy extensibility for further subcommands and args. 124 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Jörn Hees 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include CHANGELOG 3 | include LICENSE 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Userdocker 2 | ========== 3 | 4 | Userdocker is a wrapper that allows admins to grant restricted docker 5 | commandline access to users. 6 | 7 | .. note:: 8 | 9 | Userdocker is currently in BETA state. Despite our ongoing efforts to test 10 | on our local infrastructure, further testing, reviewing and feedback are 11 | very welcome. Use with caution and watch the GitHub repo for issues and 12 | new releases! 13 | 14 | 15 | Userdocker is aimed towards scientific high performance computing and cluster 16 | setups, as they exist in most universities or research groups. Often, such 17 | scientific computations have peculiar dependencies that are difficult to satisfy 18 | across linux distributions (and drive admins crazy ;) ). 19 | 20 | In theory such use-cases could largely benefit from docker, as it would allow 21 | users to easily define environments themselves and run them basically without 22 | negative performance impact, as they run directly on the host's kernel. In 23 | reality however granting docker commandline access to users effectively makes 24 | them root equivalent on the host (root in container, volume mount...), making 25 | this prohibitive for cluster computing. 26 | 27 | Userdocker solves this problem by wrapping the docker command and just making 28 | the safe parts available to users. Admins can decide what they consider safe 29 | (with sane defaults). The userdocker command largely follows the docker 30 | commandline syntax, so users can use it as an in-place replacement for the 31 | docker command. 32 | 33 | Feedback / bugreports / contributions welcome: 34 | 35 | https://github.com/joernhees/userdocker 36 | 37 | 38 | Sample Usage: 39 | ============= 40 | 41 | .. code-block:: bash 42 | 43 | # command line help (including subcommands the user is allowed to execute) 44 | sudo userdocker -h 45 | 46 | # (docker images) list images (and useful tree visualization) 47 | sudo userdocker images 48 | sudo userdocker dockviz 49 | 50 | # (docker run) run a debian image with user (read-only) mounted home 51 | sudo userdocker run -it --rm -v $HOME:$HOME:ro debian bash 52 | 53 | # (docker attach) re-attach to own container after connection loss 54 | sudo userdocker attach 438c7648e76b 55 | 56 | # (docker ps) list running containers 57 | sudo userdocker ps 58 | 59 | # (docker pull / load) pull or load 60 | sudo userdocker pull debian 61 | sudo userdocker load < image.tar.gz 62 | 63 | # (nvidia-docker) extensions for nvidia GPU support 64 | alias nvidia-userdocker='userdocker --executor=nvidia-docker' 65 | NV_GPU=1,3,7 nvidia-userdocker run -it --rm nvcr.io/nvidia/tensorflow 66 | userdocker ps --gpu-used 67 | userdocker ps --gpu-free 68 | 69 | Features: 70 | ========= 71 | 72 | - Similar commandline interface as ``docker ...`` called ``userdocker ...`` 73 | - Support for several docker commands / plugins (docker, nvidia-docker) 74 | - Fine granular configurability for admins in ``/etc/userdocker/`` allows to: 75 | 76 | - restrict runnable images if desired (allows admin reviews) 77 | - restrict run to locally available images 78 | - restrict available mount points (or enforce them, or default mount) 79 | - probe mounts (to make sure nfs automounts don't make docker sad) 80 | - enforce non-root user in container (same uid:gid as on host) 81 | - enforce dropping caps 82 | - enforce environment vars 83 | - enforce docker args 84 | - restrict port publishing 85 | - explicitly white-list available args to user 86 | - restrict allowed GPU access / reservations via ``NV_GPU`` 87 | 88 | - System wide config + overrides for individual groups, gids, users, uids. 89 | - Easy extensibility for further subcommands and args. 90 | 91 | 92 | Installation: 93 | ============= 94 | 95 | The installation of userdocker works in three steps: 96 | 97 | 98 | 1. Install package: 99 | ------------------- 100 | 101 | First make sure that docker is installed: 102 | 103 | .. code-block:: bash 104 | 105 | sudo docker version 106 | 107 | Afterwards, as userdocker is written in python3 and available as python package: 108 | 109 | .. code-block:: bash 110 | 111 | sudo pip3 install userdocker 112 | 113 | This will give you a ``userdocker`` command that you can test with: 114 | 115 | .. code-block:: bash 116 | 117 | userdocker -h 118 | 119 | The above is the preferable way of installation of the latest stable release. 120 | 121 | If you want to try the current master (stable dev): 122 | 123 | .. code-block:: bash 124 | 125 | sudo pip3 install -U https://github.com/joernhees/userdocker/archive/master.tar.gz 126 | 127 | Alternatively (and to contribute), you can clone this repo and execute: 128 | 129 | .. code-block:: bash 130 | 131 | sudo python3 setup.py install 132 | 133 | 134 | 2. Configuration: 135 | ----------------- 136 | 137 | Copy the default config to ``/etc/userdocker/config.py``, then edit the file. 138 | The config contains tons of comments and explanations to help you make the right 139 | decisions for your scenario. 140 | 141 | .. code-block:: bash 142 | 143 | sudo cp /etc/userdocker/default.py /etc/userdocker/config.py 144 | 145 | 146 | 3. Allowing users to run ``sudo userdocker``: 147 | --------------------------------------------- 148 | 149 | You should now allow the users in question to run ``sudo userdocker``. This is 150 | basically done by adding a ``/etc/sudoers.d/userdocker`` file. If you want to 151 | grant this permission to all users in group ``users``, add the following 152 | two lines: 153 | 154 | :: 155 | 156 | Defaults env_keep += "NV_GPU" 157 | %users ALL=(root) NOPASSWD: /usr/local/bin/userdocker 158 | 159 | The first is strongly recommended in case you want to allow users to use nvidia 160 | GPUs from within docker containers via nvidia-docker (see EXECUTORS in config). 161 | Without it they cannot pass the NV_GPU environment variable to the userdocker 162 | (and thereby nvidia-docker) command to select their desired GPU(s). 163 | 164 | 165 | FAQ: 166 | ==== 167 | 168 | Why sudo? 169 | --------- 170 | 171 | Because it supports logging and is in general a lot more configurable than the 172 | alternatives. For example if you only want to make ``userdocker`` available on 173 | some nodes in your cluster, you can use the Host\_List field: 174 | 175 | :: 176 | 177 | %users node1,node2,node4=(root) /usr/local/bin/userdocker 178 | 179 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal=0 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from os import path 3 | from setuptools import find_packages 4 | from setuptools import setup 5 | 6 | from userdocker import __doc__ 7 | from userdocker import __version__ 8 | 9 | here = path.abspath(path.dirname(__file__)) 10 | 11 | with open(path.join(here, 'README.rst'), encoding='utf-8') as f: 12 | long_description = f.read() 13 | 14 | setup( 15 | name='userdocker', 16 | version=__version__, 17 | description=__doc__.strip().splitlines()[0].strip(), 18 | long_description=long_description, 19 | url='https://github.com/joernhees/userdocker', 20 | author='Jörn Hees', 21 | author_email='dev+userdocker@joernhees.de', 22 | license='MIT', 23 | classifiers=[ 24 | 'Development Status :: 4 - Beta', 25 | 'Environment :: Console', 26 | 'Intended Audience :: Developers', 27 | 'Intended Audience :: Science/Research', 28 | 'Intended Audience :: System Administrators', 29 | 'License :: OSI Approved :: MIT License', 30 | 'Operating System :: POSIX :: Linux', 31 | 'Programming Language :: Python :: 3.4', 32 | 'Programming Language :: Python :: 3.5', 33 | 'Programming Language :: Python :: 3.6', 34 | 'Topic :: Scientific/Engineering', 35 | 'Topic :: Security', 36 | 'Topic :: Software Development', 37 | 'Topic :: System', 38 | 'Topic :: System :: Clustering', 39 | 'Topic :: System :: Distributed Computing', 40 | 'Topic :: System :: Emulators', 41 | 'Topic :: System :: Operating System', 42 | 'Topic :: System :: Systems Administration', 43 | 'Topic :: System :: Systems Administration :: Authentication/Directory', 44 | 'Topic :: Utilities', 45 | ], 46 | keywords='docker user limit admin hpc cluster computing permissions', 47 | packages=find_packages(), 48 | include_package_data=True, 49 | data_files=[('/etc/userdocker/', ['userdocker/config/default.py'])], 50 | entry_points={'console_scripts': ['userdocker=userdocker.userdocker:main']}, 51 | zip_safe=True, 52 | ) 53 | -------------------------------------------------------------------------------- /userdocker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | if __name__ == '__main__': 5 | from userdocker.userdocker import main 6 | main() 7 | -------------------------------------------------------------------------------- /userdocker/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | userdocker allows admins to grant restricted docker command access to users. 5 | 6 | Feedback welcome: 7 | https://github.com/joernhees/userdocker 8 | """ 9 | 10 | __version__ = '2.1.0-dev2' 11 | -------------------------------------------------------------------------------- /userdocker/config/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from glob import glob 4 | import grp 5 | import os 6 | import pwd 7 | 8 | 9 | uid = os.getuid() 10 | uid = int(os.getenv('SUDO_UID', uid)) 11 | gid = os.getgid() 12 | gid = int(os.getenv('SUDO_GID', gid)) 13 | user_pwd = pwd.getpwuid(uid) 14 | user_name = user_pwd.pw_name 15 | user_home = user_pwd.pw_dir 16 | group_name = grp.getgrgid(gid).gr_name 17 | group_names = [] 18 | gids = [] 19 | for _g in grp.getgrall(): 20 | if user_name in _g.gr_mem: 21 | group_names.append(_g.gr_name) 22 | gids.append(_g.gr_gid) 23 | del user_pwd 24 | 25 | 26 | # see default.py for explanation on config load order 27 | from .default import * 28 | configs_loaded = ['default'] 29 | _cd = '/etc/userdocker/' 30 | _cfns = ( 31 | glob(_cd + 'config.py') 32 | + sorted([ 33 | _cfn for _gn in group_names 34 | for _cfn in glob(_cd + 'group/config_[0-9][0-9]_%s.py' % _gn)]) 35 | + sorted([ 36 | _cfn for _gid in gids 37 | for _cfn in glob(_cd + 'gid/config_[0-9][0-9]_%d.py' % _gid)]) 38 | + glob(_cd + 'user/config_%s.py' % user_name) 39 | + glob(_cd + 'uid/config_%d.py' % uid) 40 | ) 41 | for _cfn in _cfns: 42 | with open(_cfn) as _cf: 43 | exec(_cf.read()) 44 | configs_loaded.append(_cfn) 45 | 46 | 47 | # helpers to show final config 48 | def items(): 49 | masked = ( 50 | # imports: 51 | 'default', 'glob', 'grp', 'os', 'pwd', 52 | # methods: 53 | 'items', 'path', 54 | ) 55 | for _var, _val in sorted(globals().items()): 56 | if not _var.startswith('_') and _var not in masked: 57 | yield _var, _val 58 | 59 | 60 | def path(): 61 | return os.path.dirname(os.path.realpath(__file__)) 62 | -------------------------------------------------------------------------------- /userdocker/config/default.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | ################################################################################ 4 | # WARNING: 5 | # /etc/userdocker/default.py will not be loaded at runtime, but is meant as a 6 | # template! It will be overwritten by future installs/updates! 7 | # Copy this file to /etc/docker/config.py, then edit! 8 | # 9 | # The following (python) variables may be used throughout config files (see 10 | # VOLUME_MOUNTS_DEFAULT or ENV_VARS for an example): 11 | from . import uid, gid, user_name, group_name, user_home 12 | # (if you like, you can remove the line without any effect) 13 | # 14 | # The order of config files is (if existing): 15 | # - package defaults 16 | # - /etc/userdocker/config.py 17 | # - /etc/userdocker/group/config__.py (see note below) 18 | # - /etc/userdocker/gid/config__.py (see note below) 19 | # - /etc/userdocker/user/config_.py 20 | # - /etc/userdocker/uid/config_.py 21 | # 22 | # All config files are executed in place, allowing later config files to 23 | # override or modify previous ones. The above might sound complicated, but just 24 | # start with a /etc/userdocker/config.py and then define exceptions later. 25 | # 26 | # As a user can be in several groups, the group configs include a 2 digit prio. 27 | # On execution, we will get all groups for the user, collect the corresponding 28 | # config files matching those groups if they exist and load all collected 29 | # config files sorted ascending by prio. Lowest prio is 00 (loaded first), 30 | # highest prio is 99 (loaded last). 31 | # Afterwards the same is done for gid config files. 32 | # 33 | # Example: if a user is in groups "adm" and "udocker" and an admin created the 34 | # config files config_99_adm.py and config_50_udocker.py, then the execution 35 | # order is: 36 | # - package defaults 37 | # - /etc/userdocker/config.py 38 | # - /etc/userdocker/group/config_50_udocker.py 39 | # - /etc/userdocker/group/config_99_adm.py 40 | # - ... 41 | # This means that config_99_adm.py can grant users in group adm a lot more 42 | # permissions (reliably), even if they are also in the udocker group. 43 | ################################################################################ 44 | 45 | # Admin default value for userdocker log level: 46 | # ['DEBUG', 'INFO', 'WARNING', 'ERROR'] 47 | LOGLVL = 'INFO' 48 | 49 | # Available executors for users. 50 | EXECUTORS = { 51 | 'docker': '/usr/bin/docker', 52 | 'nvidia-docker': '/usr/bin/nvidia-docker', 53 | } 54 | EXECUTOR_DEFAULT = 'docker' 55 | 56 | # The following allows you to specify which docker top level commands a user can 57 | # run at all (still restricted by the following settings): 58 | ALLOWED_SUBCOMMANDS = [ 59 | 'attach', # allows users to re-attach to _their_ containers 60 | 'dockviz', # tree visualization of images 61 | 'images', 62 | 'load', # see RUN_PULL as well 63 | 'ps', 64 | 'pull', # see RUN_PULL as well 65 | 'run', 66 | 'version', 67 | ] 68 | 69 | # Arguments that you want to enforce on the user: 70 | # Do not include args that are handled below (e.g. run -v, -p, --cap-drop)! 71 | # The following arguments will always be injected for the corresponding command: 72 | ARGS_ALWAYS = { 73 | 'run': [ 74 | # '-t', 75 | # '-i', 76 | '--rm', 77 | # '--shm-size=1g', 78 | ], 79 | } 80 | 81 | # The following arguments are available to the user for the 82 | # given command. 83 | # Do not include args that are handled below (e.g. run -v, -p, --cap-drop)! 84 | # Aliases are supported as tuples/lists below, but not in ARGS_ALWAYS. 85 | # Simple string args can contain assignments (e.g., '--arg=val'). The user can 86 | # then use --arg=val. 87 | ARGS_AVAILABLE = { 88 | 'attach': [ 89 | '--no-stdin', 90 | ], 91 | 'images': [ 92 | ('-a', '--all'), 93 | '--digests', 94 | '--no-trunc', 95 | ('-q', '--quiet'), 96 | ], 97 | 'load': [ 98 | ('-q', '--quiet'), 99 | ], 100 | 'ps': [ 101 | ('-a', '--all'), 102 | ('-l', '--latest'), 103 | ('-s', '--size'), 104 | ('-q', '--quiet'), 105 | '--no-trunc', 106 | ], 107 | 'pull': [ 108 | ('-a', '--all-tags'), 109 | ], 110 | 'run': [ 111 | ('-t', '--tty'), 112 | ('-i', '--interactive'), 113 | '--read-only', 114 | # users can map all exposed container ports to random free host ports: 115 | ('-P', '--publish-all'), 116 | # '--shm-size=16g', # allows users to set shared mem size 117 | ], 118 | } 119 | 120 | 121 | # Volume mounts: 122 | # - VOLUME_MOUNTS_ALWAYS will be mounted whether the user wants it or not 123 | # - VOLUME_MOUNTS_AVAILABLE will only be mounted if the user explicitly 124 | # specifies them 125 | # - VOLUME_MOUNTS_DEFAULT will be added unless the user specifies the 126 | # --no-default-mounts option 127 | # - The user can mount any of the above explicitly with "-v", redundancy is ok 128 | # - You (admin) can specify whatever usually comes after the "-v" arg in 129 | # "host_path:container_path[:flags]" form. If container_path is empty, the 130 | # user may select one. If you don't specify a flag, the user can append a 131 | # "ro" to guard herself (for any allowed mount). 132 | # Example: 133 | # VOLUME_MOUNTS_AVAILABLE = ['/scratch:/scratch', '/data:/input:ro', '/foo:'] 134 | VOLUME_MOUNTS_ALWAYS = [] 135 | VOLUME_MOUNTS_AVAILABLE = [] 136 | VOLUME_MOUNTS_DEFAULT = [ 137 | # Mount /etc/{passwd,groups} for correct uid,gid display in containers. 138 | # Not enabling this has mostly cosmetic effects. All mappings to the host 139 | # file system are via uid:gid. 140 | # WARNING: not enabled by default, as systems using ldap usually do not 141 | # offer the necessary information in these files, but via getent. 142 | # '/etc/passwd:/etc/passwd:ro', 143 | # '/etc/group:/etc/group:ro', 144 | 145 | # default mount user's home 146 | user_home + ':' + user_home, 147 | ] 148 | 149 | # This setting issues a listdir for used host dirs in mounts. 150 | # Useful for server-side auto-mounts. 151 | PROBE_USED_MOUNTS = True 152 | 153 | 154 | # User is allowed to run an image if any of the following regexps match it 155 | ALLOWED_IMAGE_REGEXPS = [ 156 | '^[A-Za-z_].*', 157 | ] 158 | 159 | # Normally docker run automatically pulls images that aren't available locally. 160 | # This possible values here are ['default', 'never', 'always'] 161 | # 'never' will restrict to locally available images (see ALLOWED_COMMANDS and 162 | # restrict load command if desired!). 163 | RUN_PULL = 'default' 164 | 165 | 166 | # If set, the user will run with his uid and gid in the container. 167 | # Changing this to false is probably a really bad idea, especially when combined 168 | # with any mounts. 169 | USER_IN_CONTAINER = True 170 | 171 | # The following allows to drop / grant capabilities of all containers. 172 | # By default we drop all 173 | CAPS_DROP = ['ALL'] 174 | CAPS_ADD = [] 175 | 176 | # User ability to map ports explicitly: 177 | # Unlike the probably safe `-P` run arg (which maps all exposed container ports 178 | # to random free host ports (world accessible)), giving users explicit control 179 | # over port mappings is probably not the best idea, as they are likely to 180 | # collide on frequently used ports such as 5000 or 8080. 181 | # Also it might unintentionally allow them to bind ports in the root range if 182 | # you are not careful. 183 | # For flexibility, we allow you to specify regexps that each -p arg is matched 184 | # against one by one. Only if each of the user's -p args matches at least one of 185 | # these, the docker run command is executed. If the list is empty, the -p 186 | # argument is neither allowed nor shown to the user. 187 | ALLOWED_PORT_MAPPINGS = [ 188 | # useful defaults: most similar to -P, but allows users to select 189 | # ports instead of mapping all exposed publicly. Also might allow them to 190 | # bind them local to host only: 191 | r'^127\.0\.0\.1::[0-9]+$', # local access from host (via random free port) 192 | '^[0-9]+$', # public access (via random free host port) 193 | 194 | # more examples: 195 | 196 | # allow `-p 127.0.0.1:5000-6000:80`, so user can map container 80 to 197 | # random host port in range of 5000-6000 that is only accessible from host: 198 | # r'^127\.0\.0\.1:5000-6000:80$' 199 | 200 | # allow `-p 8080:80`, so user can map container 80 to host 8080 (if free): 201 | # '^8080:80$' # probably useful in user-specific configs 202 | 203 | # allow all (probably bad idea!): 204 | # '^.*$', # allows all, probably bad idea! 205 | ] 206 | 207 | # Environment vars to set for the container: 208 | ENV_VARS = [ 209 | # sets HOME env var to user's home 210 | 'HOME=' + user_home, 211 | ] 212 | ENV_VARS_EXT = { 213 | 'nvidia-docker': [ 214 | 'NCCL_TOPOLOGY=CUBEMESH', 215 | ] 216 | } 217 | 218 | 219 | # nvidia docker specific settings 220 | # The following settings allow to restrict the way users can use nvidia GPUs 221 | # in their container. 222 | # - NV_ALLOWED_GPUS allows you to only make a subset of GPUs available to users. 223 | # Setting this to [] or None will result in nvidia-docker to always fail as no 224 | # GPUs are available. If that's desired, you should instead remove the 225 | # nvidia-docker executor above, as that's less confusing for users! 226 | # - NV_DEFAULT_GPU_COUNT_RESERVATION allows to specify how many GPUs are passed 227 | # to a container if the user does not specify the NV_GPU env var. This is 228 | # different from the nvidia-docker defaults, which would by default allow 229 | # access to all GPUs! 230 | # - NV_MAX_GPU_COUNT_RESERVATION allows you to limit the amount of GPUs made 231 | # available to a single container. Setting this to -1 means no limit. 232 | # - GPUs on which more than NV_GPU_UNAVAILABLE_ABOVE_MEMORY_USED MB of memory is 233 | # used will be marked as unavailable. This setting is userdocker independent. 234 | # Setting this to -1 results in GPUs always being regarded as available. 235 | # - If NV_EXCLUSIVE_GPU_RESERVATION is set, any GPUs already used in any other 236 | # container are regarded as unavailable for this container. 237 | # - NV_ALLOW_OWN_GPU_REUSE allows users to run multiple containers on GPUs they 238 | # already use. This only happens when explicitly setting NV_GPU. 239 | NVIDIA_SMI = '/usr/bin/nvidia-smi' # path to nvidia-smi 240 | NV_ALLOWED_GPUS = 'ALL' # otherwise a list like [1, 3]. [] for none. 241 | NV_DEFAULT_GPU_COUNT_RESERVATION = 1 242 | NV_MAX_GPU_COUNT_RESERVATION = -1 243 | NV_GPU_UNAVAILABLE_ABOVE_MEMORY_USED = 0 244 | NV_EXCLUSIVE_CONTAINER_GPU_RESERVATION = True 245 | NV_ALLOW_OWN_GPU_REUSE = True 246 | -------------------------------------------------------------------------------- /userdocker/helpers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joernhees/userdocker/57b276952b262cdd3da82f7c002374bf42e07fc9/userdocker/helpers/__init__.py -------------------------------------------------------------------------------- /userdocker/helpers/cmd.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from ..config import ARGS_ALWAYS 4 | from .logger import logger 5 | 6 | 7 | def init_cmd(args): 8 | cmd = [args.executor_path, args.subcommand] \ 9 | + ARGS_ALWAYS.get(args.subcommand, []) 10 | logger.debug("patch_through_args: %s", args.patch_through_args) 11 | for pt_arg in args.patch_through_args: 12 | if pt_arg not in cmd: 13 | cmd.append(pt_arg) 14 | return cmd 15 | -------------------------------------------------------------------------------- /userdocker/helpers/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | class UserDockerException(Exception): 5 | pass 6 | -------------------------------------------------------------------------------- /userdocker/helpers/execute.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import logging 4 | import os 5 | from shlex import quote 6 | import subprocess 7 | import sys 8 | 9 | from .exceptions import UserDockerException 10 | from .logger import logger 11 | 12 | 13 | def exec_cmd(cmd, dry_run=False, return_status=True, loglvl=logging.INFO): 14 | logger.log( 15 | loglvl, 16 | '%s command: %s', 17 | 'would execute' if dry_run else 'executing', 18 | ' '.join([quote(c) for c in cmd]) 19 | ) 20 | logger.debug('internal repr: %s', cmd) 21 | 22 | if dry_run: 23 | return 0 24 | 25 | if not os.path.exists(cmd[0]): 26 | raise UserDockerException( 27 | "ERROR: can't find executable: %s" % cmd[0] 28 | ) 29 | 30 | try: 31 | if return_status: 32 | ret = subprocess.check_call(cmd) 33 | else: 34 | ret = subprocess.check_output(cmd, universal_newlines=True) 35 | return ret 36 | except subprocess.CalledProcessError as e: 37 | ret = e.returncode 38 | sys.exit(ret) 39 | 40 | 41 | def exit_exec_cmd(cmd, dry_run=False): 42 | sys.exit(exec_cmd(cmd, dry_run=dry_run)) 43 | -------------------------------------------------------------------------------- /userdocker/helpers/logger.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import logging 4 | 5 | from .. import config 6 | 7 | logger = logging.getLogger('userdocker') 8 | 9 | 10 | def logger_setup(args): 11 | logging.basicConfig() 12 | logging.root.setLevel(args.loglvl) 13 | if logger.isEnabledFor(logging.DEBUG): 14 | logger.debug('configs loaded: %s', config.configs_loaded) 15 | if len(config.configs_loaded) <= 1: 16 | logger.warning( 17 | 'No config found, using defaults! You should copy\n%s\nto\n%s\n' 18 | 'and adapt the setting to your needs!', 19 | config.path(), '/etc/userdocker/config.py' 20 | ) 21 | logger.debug('resulting config:') 22 | for _var, _val in config.items(): 23 | logger.debug(" %s = %r", _var, _val) 24 | -------------------------------------------------------------------------------- /userdocker/helpers/nvidia.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import json 4 | import logging 5 | import re 6 | from collections import defaultdict 7 | from operator import itemgetter 8 | 9 | from ..config import uid 10 | from ..config import NVIDIA_SMI 11 | from ..config import NV_ALLOWED_GPUS 12 | from ..config import NV_EXCLUSIVE_CONTAINER_GPU_RESERVATION 13 | from ..config import NV_GPU_UNAVAILABLE_ABOVE_MEMORY_USED 14 | from .logger import logger 15 | from .execute import exec_cmd 16 | 17 | 18 | def container_find_userdocker_user_uid_gpus(container_env): 19 | pairs = [var.partition('=') for var in container_env] 20 | users = [v for k, _, v in pairs if k == 'USERDOCKER_USER'] 21 | uids = [v for k, _, v in pairs if k == 'USERDOCKER_UID'] 22 | gpus = [v for k, _, v in pairs if k == 'USERDOCKER_NV_GPU'] 23 | if gpus: 24 | gpus = [int(g) for g in gpus[0].split(',')] 25 | return users[0] if users else '', int(uids[0]) if uids else None, gpus 26 | 27 | 28 | def nvidia_get_gpus_used_by_containers(docker): 29 | running_containers = exec_cmd( 30 | [docker, 'ps', '-q'], 31 | return_status=False, 32 | loglvl=logging.DEBUG, 33 | ).split() 34 | gpu_used_by_containers = defaultdict(list) 35 | if not running_containers: 36 | return gpu_used_by_containers 37 | gpu_used_by_containers_str = exec_cmd( 38 | [ 39 | docker, 'inspect', '--format', 40 | '[{{json .Name}}, {{json .Id}}, {{json .Config.Env}}]' 41 | ] + running_containers, 42 | return_status=False, 43 | loglvl=logging.DEBUG, 44 | ) 45 | logger.debug('gpu_used_by_containers_str: %s', gpu_used_by_containers_str) 46 | for line in gpu_used_by_containers_str.splitlines(): 47 | container_name, container, container_env = json.loads(line) 48 | container_user, container_uid, gpus = \ 49 | container_find_userdocker_user_uid_gpus(container_env) 50 | for gpu_id in gpus: 51 | gpu_used_by_containers[gpu_id].append( 52 | (container, container_name, container_user, container_uid) 53 | ) 54 | logger.debug( 55 | 'gpu %d used by container: %s, name: %s, user: %s, uid: %s', 56 | gpu_id, container, container_name, container_user, container_uid 57 | ) 58 | return gpu_used_by_containers 59 | 60 | 61 | def nvidia_get_available_gpus(docker, nvidia_smi=NVIDIA_SMI): 62 | if not NV_ALLOWED_GPUS: 63 | return [] 64 | 65 | gpu_mem_used_str = exec_cmd( 66 | [nvidia_smi, 67 | '--query-gpu=index,memory.used,utilization.gpu', 68 | '--format=csv'], 69 | return_status=False, 70 | loglvl=logging.DEBUG, 71 | ) 72 | logger.debug('gpu usage:\n%s', gpu_mem_used_str) 73 | gpu_mem_used = {} 74 | for line in gpu_mem_used_str.splitlines()[1:]: # skip header 75 | gpu, mem_used, gpu_utilization = line.split(', ') 76 | gpu = int(gpu) 77 | mem_used = int(mem_used.split(' MiB')[0]) 78 | gpu_mem_used[gpu] = mem_used 79 | 80 | gpus_used_by_containers = nvidia_get_gpus_used_by_containers(docker) 81 | gpus_used_by_own_containers = [ 82 | gpu for gpu, info in gpus_used_by_containers.items() 83 | if any(i[3] == uid for i in info) 84 | ] 85 | 86 | # get available gpus asc by mem used and reservation counts 87 | mem_limit = NV_GPU_UNAVAILABLE_ABOVE_MEMORY_USED 88 | mem_res_gpu = [ 89 | (m, len(gpus_used_by_containers.get(gpu, [])), gpu) 90 | for gpu, m in gpu_mem_used.items() 91 | ] 92 | available_gpus = [ 93 | g for m, r, g in sorted(mem_res_gpu) if mem_limit < 0 or m <= mem_limit 94 | ] 95 | if NV_ALLOWED_GPUS != 'ALL': 96 | available_gpus = [g for g in available_gpus if g in NV_ALLOWED_GPUS] 97 | logger.debug( 98 | 'available GPUs after mem and allowance filtering: %r', available_gpus) 99 | 100 | if NV_EXCLUSIVE_CONTAINER_GPU_RESERVATION: 101 | available_gpus = [ 102 | gpu for gpu in available_gpus 103 | if gpu not in gpus_used_by_containers 104 | ] 105 | 106 | return available_gpus, gpus_used_by_own_containers 107 | -------------------------------------------------------------------------------- /userdocker/helpers/parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import sys 4 | import argparse 5 | 6 | from ..config import ARGS_ALWAYS 7 | from ..config import ARGS_AVAILABLE 8 | 9 | 10 | class _PatchThroughAssignmentAction(argparse._AppendAction): 11 | """Action that appends not only the value but the option=value to dest. 12 | 13 | Useful for patch through args with assignment like: --shm-size=1g. 14 | """ 15 | 16 | def __call__(self, parser, namespace, values, option_string=None): 17 | super(_PatchThroughAssignmentAction, self).__call__( 18 | parser, namespace, 19 | values=option_string + '=' + values, 20 | option_string=option_string) 21 | 22 | 23 | def init_subcommand_parser(parent_parser, scmd): 24 | parser = parent_parser.add_parser( 25 | scmd, 26 | help='Lets a user run "docker %s ..." command' % scmd, 27 | ) 28 | parser.set_defaults( 29 | patch_through_args=[], 30 | ) 31 | 32 | # patch args through 33 | _args_seen = [] 34 | for args in ARGS_AVAILABLE.get(scmd, []) + ARGS_ALWAYS.get(scmd, []): 35 | if isinstance(args, str): 36 | # just a single arg as string 37 | args = [args] 38 | elif isinstance(args, (list, tuple)): 39 | # aliases as list or tuple 40 | args = list(args) 41 | else: 42 | raise NotImplementedError( 43 | "Cannot understand admin defined ARG %s for command %s" % ( 44 | args, scmd)) 45 | 46 | # remove dups (e.g. from being in AVAILABLE and ALWAYS) 47 | args = [arg for arg in args if arg not in _args_seen] 48 | _args_seen.extend(args) 49 | if not args: 50 | continue 51 | 52 | for arg in args: 53 | # make sure each arg starts with - and doesn't contain ' ' 54 | if not arg.startswith('-') or ' ' in arg: 55 | raise NotImplementedError( 56 | "Cannot understand admin defined ARG %s for command %s" % ( 57 | arg, scmd)) 58 | if '=' in arg: 59 | if len(args) != 1: 60 | raise NotImplementedError( 61 | "Only supports single string args with values: " 62 | "%s for command %s" % (arg, scmd)) 63 | 64 | h = "see docker help" 65 | if set(args) & set(ARGS_ALWAYS.get(scmd, [])): 66 | h += ' (enforced by admin)' 67 | 68 | s = args[0] 69 | if '=' in s: 70 | arg, val = s.split('=') 71 | args = [arg] 72 | kwds = { 73 | "help": h, 74 | "action": _PatchThroughAssignmentAction, 75 | "choices": [val], 76 | "dest": "patch_through_args", 77 | } 78 | else: 79 | kwds = { 80 | "help": h, 81 | "action": "append_const", 82 | "const": s, 83 | "dest": "patch_through_args", 84 | } 85 | parser.add_argument(*args, **kwds) 86 | 87 | return parser 88 | -------------------------------------------------------------------------------- /userdocker/parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import argparse 4 | import logging 5 | import sys 6 | 7 | from . import __doc__ 8 | from . import __version__ 9 | from .config import ALLOWED_SUBCOMMANDS 10 | from .config import EXECUTOR_DEFAULT 11 | from .config import EXECUTORS 12 | from .config import LOGLVL 13 | from .helpers.parser import init_subcommand_parser 14 | 15 | # dispatch specific specific_parsers to those defined in subcommands package 16 | from .subcommands import specific_parsers 17 | 18 | 19 | def parse_args(): 20 | parser = argparse.ArgumentParser( 21 | description=__doc__.strip(), 22 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 23 | ) 24 | 25 | parser.add_argument( 26 | "--version", 27 | action="version", 28 | version='%(prog)s ' + __version__ 29 | ) 30 | 31 | debug_group = parser.add_mutually_exclusive_group() 32 | debug_group.add_argument( 33 | "-q", "--quiet", 34 | help="silences userdocker output below WARNING severity", 35 | action="store_const", 36 | dest="loglvl", 37 | const=logging.WARNING, 38 | default=LOGLVL, 39 | ) 40 | debug_group.add_argument( 41 | "--debug", 42 | help="debug and config output for invoked docker command", 43 | action="store_const", 44 | dest="loglvl", 45 | const=logging.DEBUG, 46 | default=LOGLVL, 47 | ) 48 | 49 | parser.add_argument( 50 | "-n", "--dry-run", 51 | help="doesn't actually invoke the docker command", 52 | action="store_true", 53 | ) 54 | 55 | parser.add_argument( 56 | "--executor", 57 | help="prints the invoked docker commandline", 58 | default=EXECUTOR_DEFAULT, 59 | choices=EXECUTORS, 60 | ) 61 | 62 | # individual commands will be sub-specific_parsers 63 | subparsers = parser.add_subparsers(dest="subcommand") 64 | subparsers.required = True 65 | 66 | for scmd in ALLOWED_SUBCOMMANDS: 67 | specific_parser = specific_parsers.get(scmd) 68 | if specific_parser: 69 | specific_parser(subparsers) 70 | else: 71 | init_subcommand_parser(subparsers, scmd) 72 | 73 | args = parser.parse_args() 74 | args.executor_path = EXECUTORS[args.executor] 75 | return args 76 | -------------------------------------------------------------------------------- /userdocker/subcommands/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .attach import * 4 | from .dockviz import * 5 | from .images import * 6 | from .ps import * 7 | from .pull import * 8 | from .run import * 9 | from .version import * 10 | 11 | SPECIFIC_PARSER_PREFIX = 'parser_' 12 | specific_parsers = { 13 | _var.split(SPECIFIC_PARSER_PREFIX)[1]: _val 14 | for _var, _val in globals().items() 15 | if _var.startswith(SPECIFIC_PARSER_PREFIX) 16 | } 17 | 18 | SPECIFIC_CMD_EXECUTOR_PREFIX = 'exec_cmd_' 19 | specific_command_executors = { 20 | _var.split(SPECIFIC_CMD_EXECUTOR_PREFIX)[1]: _val 21 | for _var, _val in globals().items() 22 | if _var.startswith(SPECIFIC_CMD_EXECUTOR_PREFIX) 23 | } 24 | 25 | __all__ = [specific_parsers, specific_command_executors] 26 | -------------------------------------------------------------------------------- /userdocker/subcommands/attach.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import json 4 | import logging 5 | 6 | from ..config import uid 7 | from ..helpers.cmd import init_cmd 8 | from ..helpers.exceptions import UserDockerException 9 | from ..helpers.execute import exec_cmd 10 | from ..helpers.execute import exit_exec_cmd 11 | from ..helpers.logger import logger 12 | from ..helpers.parser import init_subcommand_parser 13 | 14 | 15 | def parser_attach(parser): 16 | sub_parser = init_subcommand_parser(parser, 'attach') 17 | 18 | sub_parser.add_argument( 19 | "--detach-keys", 20 | help="Override the key sequence for detaching a container", 21 | ) 22 | 23 | sub_parser.add_argument( 24 | "container", 25 | help="container's ID or name to attach to" 26 | ) 27 | 28 | 29 | def exec_cmd_attach(args): 30 | cmd = init_cmd(args) 31 | 32 | if args.detach_keys: 33 | cmd += ['--detach-keys', args.detach_keys] 34 | 35 | container = args.container 36 | cmd += [container] 37 | 38 | # check if we're allowed to attach to container (if it's ours) 39 | container_env = exec_cmd( 40 | [ 41 | args.executor_path, 'inspect', 42 | '--format', '{{json .Config.Env}}', 43 | container 44 | ], 45 | return_status=False, 46 | loglvl=logging.DEBUG, 47 | ) 48 | if not container_env: 49 | raise UserDockerException( 50 | 'ERROR: could not find container %s' % container 51 | ) 52 | userdocker_uid_env = [ 53 | env for env in json.loads(container_env) 54 | if env.startswith('USERDOCKER_UID') 55 | ] 56 | if not userdocker_uid_env: 57 | raise UserDockerException( 58 | 'ERROR: could not find USERDOCKER_UID env var in container %s' 59 | % container 60 | ) 61 | userdocker_uid = int(userdocker_uid_env[0].split('USERDOCKER_UID=')[1]) 62 | logger.debug( 63 | "Container %s was started by user id %d", container, userdocker_uid) 64 | if uid != userdocker_uid: 65 | raise UserDockerException( 66 | 'ERROR: container %s was started by user id %d, but you are %d. ' 67 | 'Permission denied!' % (container, userdocker_uid, uid) 68 | ) 69 | 70 | exit_exec_cmd(cmd, dry_run=args.dry_run) 71 | -------------------------------------------------------------------------------- /userdocker/subcommands/dockviz.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from ..helpers.execute import exit_exec_cmd 4 | 5 | 6 | def exec_cmd_dockviz(args): 7 | # just run dockviz 8 | cmd = [ 9 | args.executor_path, "run", "-it", "--rm", 10 | "-v", "/var/run/docker.sock:/var/run/docker.sock", 11 | "nate/dockviz", "images", "--tree" 12 | ] 13 | exit_exec_cmd(cmd, dry_run=args.dry_run) 14 | -------------------------------------------------------------------------------- /userdocker/subcommands/images.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from ..helpers.cmd import init_cmd 4 | from ..helpers.execute import exit_exec_cmd 5 | from ..helpers.parser import init_subcommand_parser 6 | 7 | 8 | def parser_images(parser): 9 | sub_parser = init_subcommand_parser(parser, 'images') 10 | 11 | sub_parser.add_argument( 12 | "repo_tag", 13 | help="optional repo[:tag] to restrict output", 14 | nargs='?', 15 | ) 16 | 17 | 18 | def exec_cmd_images(args): 19 | cmd = init_cmd(args) 20 | if args.repo_tag: 21 | cmd.append("--") 22 | cmd.append(args.repo_tag) 23 | exit_exec_cmd(cmd, dry_run=args.dry_run) 24 | -------------------------------------------------------------------------------- /userdocker/subcommands/ps.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from ..helpers.cmd import init_cmd 3 | from ..helpers.execute import exit_exec_cmd 4 | from ..helpers.nvidia import nvidia_get_available_gpus 5 | from ..helpers.nvidia import nvidia_get_gpus_used_by_containers 6 | from ..helpers.parser import init_subcommand_parser 7 | 8 | 9 | def parser_ps(parser): 10 | sub_parser = init_subcommand_parser(parser, 'ps') 11 | 12 | arg_group = sub_parser.add_mutually_exclusive_group() 13 | arg_group.add_argument( 14 | "--gpu-used", 15 | help="show GPUs used by nvidia-docker containers", 16 | action="store_true", 17 | ) 18 | 19 | arg_group.add_argument( 20 | "--gpu-used-mine", 21 | help="show GPUs reserved by your containers", 22 | action="store_true", 23 | ) 24 | 25 | arg_group.add_argument( 26 | "--gpu-free", 27 | help="show allowed and free GPUs (asc by MB mem used)", 28 | action="store_true", 29 | ) 30 | 31 | 32 | def exec_cmd_ps(args): 33 | if not args.gpu_used and not args.gpu_free and not args.gpu_used_mine: 34 | exit_exec_cmd(init_cmd(args), dry_run=args.dry_run) 35 | 36 | if args.gpu_used: 37 | gpus_used = nvidia_get_gpus_used_by_containers(args.executor_path) 38 | if gpus_used: 39 | print("\t".join(("GPU", "Container", "ContainerName", "User"))) 40 | for i, l in sorted(gpus_used.items()): 41 | for container, container_name, user, _ in sorted(l): 42 | print("\t".join((str(i), container, container_name, user))) 43 | elif args.gpu_used_mine: 44 | available_gpus, own_gpus = nvidia_get_available_gpus(args.executor_path) 45 | for gpu in own_gpus: 46 | print(gpu) 47 | elif args.gpu_free: 48 | available_gpus, own_gpus = nvidia_get_available_gpus(args.executor_path) 49 | for gpu in available_gpus: 50 | print(gpu) 51 | -------------------------------------------------------------------------------- /userdocker/subcommands/pull.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from ..helpers.cmd import init_cmd 4 | from ..helpers.execute import exit_exec_cmd 5 | from ..helpers.parser import init_subcommand_parser 6 | 7 | 8 | def parser_pull(parser): 9 | sub_parser = init_subcommand_parser(parser, 'pull') 10 | 11 | sub_parser.add_argument( 12 | "name_tag_digest", 13 | help="NAME[:TAG|@DIGEST] to pull", 14 | ) 15 | 16 | 17 | def exec_cmd_pull(args): 18 | cmd = init_cmd(args) 19 | if args.name_tag_digest: 20 | cmd.append("--") 21 | cmd.append(args.name_tag_digest) 22 | exit_exec_cmd(cmd, dry_run=args.dry_run) 23 | -------------------------------------------------------------------------------- /userdocker/subcommands/run.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import argparse 4 | import logging 5 | import os 6 | import re 7 | 8 | from .. import __version__ 9 | from ..config import ALLOWED_IMAGE_REGEXPS 10 | from ..config import ALLOWED_PORT_MAPPINGS 11 | from ..config import CAPS_ADD 12 | from ..config import CAPS_DROP 13 | from ..config import ENV_VARS 14 | from ..config import ENV_VARS_EXT 15 | from ..config import NV_ALLOW_OWN_GPU_REUSE 16 | from ..config import NV_ALLOWED_GPUS 17 | from ..config import NV_DEFAULT_GPU_COUNT_RESERVATION 18 | from ..config import NV_MAX_GPU_COUNT_RESERVATION 19 | from ..config import PROBE_USED_MOUNTS 20 | from ..config import RUN_PULL 21 | from ..config import USER_IN_CONTAINER 22 | from ..config import VOLUME_MOUNTS_ALWAYS 23 | from ..config import VOLUME_MOUNTS_AVAILABLE 24 | from ..config import VOLUME_MOUNTS_DEFAULT 25 | from ..config import gid 26 | from ..config import uid 27 | from ..config import user_name 28 | from ..helpers.cmd import init_cmd 29 | from ..helpers.exceptions import UserDockerException 30 | from ..helpers.execute import exec_cmd 31 | from ..helpers.execute import exit_exec_cmd 32 | from ..helpers.logger import logger 33 | from ..helpers.nvidia import nvidia_get_available_gpus 34 | from ..helpers.parser import init_subcommand_parser 35 | 36 | 37 | def parser_run(parser): 38 | sub_parser = init_subcommand_parser(parser, 'run') 39 | 40 | sub_parser.add_argument( 41 | "--no-default-mounts", 42 | help="does not automatically add default mounts", 43 | action="store_true", 44 | ) 45 | 46 | mounts_help = [] 47 | if VOLUME_MOUNTS_ALWAYS: 48 | mounts_help += ['Admin enforced: %s.' % ', '.join(VOLUME_MOUNTS_ALWAYS)] 49 | if VOLUME_MOUNTS_DEFAULT: 50 | mounts_help += ['Default: %s.' % ', '.join(VOLUME_MOUNTS_DEFAULT)] 51 | if VOLUME_MOUNTS_AVAILABLE: 52 | mounts_help += ['Available: %s.' % ', '.join(VOLUME_MOUNTS_AVAILABLE)] 53 | if mounts_help: 54 | sub_parser.add_argument( 55 | "-v", "--volume", 56 | help="user specified volume mounts (can be given multiple times). " 57 | "%s" % " ".join(mounts_help), 58 | action="append", 59 | dest="volumes", 60 | default=[], 61 | ) 62 | 63 | sub_parser.add_argument( 64 | "--entrypoint", 65 | help="Overwrite the default ENTRYPOINT of the image", 66 | ) 67 | 68 | sub_parser.add_argument( 69 | "-w", "--workdir", 70 | help="Working directory inside the container", 71 | ) 72 | 73 | if ALLOWED_PORT_MAPPINGS: 74 | sub_parser.add_argument( 75 | "-p", "--publish", 76 | help="Publish a container's ports to the host (see docker help). " 77 | "Allowed: " + ', '.join(ALLOWED_PORT_MAPPINGS), 78 | action="append", 79 | dest="port_mappings", 80 | default=[], 81 | ) 82 | 83 | sub_parser.add_argument( 84 | "image", 85 | help="the image to run. Allowed: " + ', '.join(ALLOWED_IMAGE_REGEXPS), 86 | ) 87 | 88 | sub_parser.add_argument( 89 | "image_args", 90 | help="arguments passed to the image", 91 | nargs=argparse.REMAINDER 92 | ) 93 | 94 | 95 | def prepare_nvidia_docker_run(args): 96 | # mainly handles GPU arbitration via ENV var for nvidia-docker 97 | # note that these are ENV vars for the command, not the container 98 | 99 | if os.getenv('NV_HOST'): 100 | raise UserDockerException('ERROR: NV_HOST env var not supported yet') 101 | 102 | # check if allowed 103 | if not NV_ALLOWED_GPUS: 104 | raise UserDockerException( 105 | "ERROR: No GPUs available due to admin setting." 106 | ) 107 | 108 | nv_gpus = os.getenv('NV_GPU', '') 109 | if nv_gpus: 110 | # the user has set NV_GPU, just check if it's ok 111 | nv_gpus = [g.strip() for g in nv_gpus.split(',')] 112 | try: 113 | nv_gpus = [int(gpu) for gpu in nv_gpus] 114 | except ValueError as e: 115 | raise UserDockerException( 116 | "ERROR: Can't parse NV_GPU, use index notation: %s" % e 117 | ) 118 | 119 | if not ( 120 | NV_ALLOWED_GPUS == 'ALL' 121 | or all(gpu in NV_ALLOWED_GPUS for gpu in nv_gpus)): 122 | raise UserDockerException( 123 | "ERROR: Access to at least one specified NV_GPU denied by " 124 | "admin. Available GPUs: %r" % (NV_ALLOWED_GPUS,) 125 | ) 126 | 127 | # check if in bounds (and MAX >= 0) 128 | if 0 <= NV_MAX_GPU_COUNT_RESERVATION < len(nv_gpus): 129 | raise UserDockerException( 130 | "ERROR: Number of requested GPUs > %d (admin limit)" % ( 131 | NV_MAX_GPU_COUNT_RESERVATION,) 132 | ) 133 | 134 | # check if available 135 | gpus_available, own_gpus = nvidia_get_available_gpus(args.executor_path) 136 | if NV_ALLOW_OWN_GPU_REUSE: 137 | gpus_available.extend(own_gpus) 138 | for g in nv_gpus: 139 | if g not in gpus_available: 140 | msg = ( 141 | 'ERROR: GPU %d is currently not available!\nUse:\n' 142 | '"sudo userdocker ps --gpu-free" to find available GPUs.\n' 143 | '"sudo userdocker ps --gpu-used" and "nvidia-smi" to see ' 144 | 'status.' % g 145 | ) 146 | if NV_ALLOW_OWN_GPU_REUSE and own_gpus: 147 | msg += '\n"sudo userdocker ps --gpu-used-mine to show own' \ 148 | '(reusable) GPUs.' 149 | raise UserDockerException(msg) 150 | else: 151 | # NV_GPU wasn't set, use admin defaults, tell user 152 | gpu_default = NV_DEFAULT_GPU_COUNT_RESERVATION 153 | logger.info( 154 | "NV_GPU environment variable not set, trying to acquire admin " 155 | "default of %d GPUs" % gpu_default 156 | ) 157 | gpus_available, own_gpus = nvidia_get_available_gpus(args.executor_path) 158 | gpus = gpus_available[:gpu_default] 159 | if len(gpus) < gpu_default: 160 | msg = ( 161 | 'Could not find %d available GPU(s)!\nUse:\n' 162 | '"sudo userdocker ps --gpu-used" and "nvidia-smi" to see ' 163 | 'status.' % gpu_default 164 | ) 165 | if NV_ALLOW_OWN_GPU_REUSE and own_gpus: 166 | msg += '\n You can set NV_GPU to reuse a GPU you have already' \ 167 | ' reserved.' 168 | raise UserDockerException(msg) 169 | gpu_env = ",".join([str(g) for g in gpus]) 170 | logger.info("Setting NV_GPU=%s" % gpu_env) 171 | os.environ['NV_GPU'] = gpu_env 172 | 173 | 174 | def exec_cmd_run(args): 175 | cmd = init_cmd(args) 176 | 177 | # check port mappings 178 | for pm in getattr(args, 'port_mappings', []): 179 | for pm_pattern in ALLOWED_PORT_MAPPINGS: 180 | if re.match(pm_pattern, pm): 181 | cmd += ['-p', pm] 182 | break 183 | else: 184 | raise UserDockerException( 185 | "ERROR: given port mapping not allowed: %s" % pm 186 | ) 187 | 188 | 189 | # check mounts 190 | mounts = [] 191 | mounts_available = \ 192 | VOLUME_MOUNTS_ALWAYS + VOLUME_MOUNTS_DEFAULT + VOLUME_MOUNTS_AVAILABLE 193 | 194 | mounts += VOLUME_MOUNTS_ALWAYS 195 | 196 | if not args.no_default_mounts: 197 | mounts += VOLUME_MOUNTS_DEFAULT 198 | 199 | for user_mount in getattr(args, 'volumes', []): 200 | if user_mount in mounts: 201 | continue 202 | if user_mount in mounts_available: 203 | mounts += [user_mount] 204 | continue 205 | 206 | # literal matches didn't work, check if the user appended a 'ro' flag 207 | if len(user_mount.split(':')) == 3: 208 | host_path, container_path, flag = user_mount.split(':') 209 | if flag == 'ro': 210 | st = ':'.join([host_path, container_path]) 211 | if st in mounts: 212 | # upgrade mount to include ro flag 213 | idx = mounts.index(st) 214 | mounts[idx] = user_mount 215 | continue 216 | if st in mounts_available: 217 | mounts += [user_mount] 218 | continue 219 | 220 | # allow potential unspecified container_path mounts 221 | host_path = user_mount.split(':')[0] + ':' 222 | if host_path in mounts_available: 223 | mounts += [user_mount] 224 | continue 225 | 226 | raise UserDockerException( 227 | "ERROR: given mount not allowed: %s" % user_mount 228 | ) 229 | 230 | mount_host_paths = [m.split(':')[0] for m in mounts] 231 | for ms in mount_host_paths: 232 | if not os.path.exists(ms): 233 | raise UserDockerException( 234 | "ERROR: mount can't be found: %s" % ms 235 | ) 236 | if PROBE_USED_MOUNTS and os.path.isdir(ms): 237 | os.listdir(ms) 238 | 239 | for mount in mounts: 240 | if ':' not in mount: 241 | raise UserDockerException( 242 | "ERROR: anonymous mounts currently not supported: %s" % mount 243 | ) 244 | cmd += ["-v", mount] 245 | 246 | 247 | if args.executor == 'nvidia-docker': 248 | prepare_nvidia_docker_run(args) 249 | 250 | env_vars = ENV_VARS + ENV_VARS_EXT.get(args.executor, []) 251 | env_vars += [ 252 | "USERDOCKER=%s" % __version__, 253 | "USERDOCKER_USER=%s" % user_name, 254 | "USERDOCKER_UID=%d" % uid, 255 | ] 256 | if args.executor == 'nvidia-docker': 257 | # remember which GPU was assigned to the container for ps --gpu-used 258 | env_vars += [ 259 | "USERDOCKER_NV_GPU=%s" % os.environ['NV_GPU'] 260 | ] 261 | for env_var in env_vars: 262 | cmd += ['-e', env_var] 263 | 264 | 265 | if USER_IN_CONTAINER: 266 | cmd += ["-u", "%d:%d" % (uid, gid)] 267 | 268 | for cap_drop in CAPS_DROP: 269 | cmd += ["--cap-drop=%s" % cap_drop] 270 | for cap_add in CAPS_ADD: 271 | cmd += ["--cap-add=%s" % cap_add] 272 | 273 | if args.workdir: 274 | cmd += ["-w", args.workdir] 275 | if args.entrypoint: 276 | cmd += ["--entrypoint", args.entrypoint] 277 | 278 | # additional injection protection, deactivated for now due to nvidia-docker 279 | # unability to handle this 280 | # cmd.append("--") 281 | 282 | img = args.image 283 | if ":" not in img and "@" not in img: 284 | # user didn't explicitly set a tag or digest, append ":latest" 285 | img += ":latest" 286 | 287 | if ALLOWED_IMAGE_REGEXPS: 288 | for air in ALLOWED_IMAGE_REGEXPS: 289 | if re.match(air, img): 290 | break 291 | else: 292 | raise UserDockerException( 293 | "ERROR: image %s not in allowed image regexps: %s" % ( 294 | img, ALLOWED_IMAGE_REGEXPS)) 295 | 296 | # pull image? 297 | if RUN_PULL == "default": 298 | # just let `docker run` do its thing 299 | pass 300 | elif RUN_PULL == "always": 301 | # pull image 302 | exec_cmd( 303 | [args.executor_path, 'pull', img], 304 | dry_run=args.dry_run, 305 | loglvl=logging.DEBUG, 306 | ) 307 | elif RUN_PULL == "never": 308 | # check if image is available locally 309 | tmp = exec_cmd( 310 | [args.executor_path, 'images', '-q', img], 311 | return_status=False, 312 | loglvl=logging.DEBUG, 313 | ) 314 | if not tmp: 315 | raise UserDockerException( 316 | "ERROR: you can only use locally available images, but %s could" 317 | " not be found locally" % img 318 | ) 319 | else: 320 | raise UserDockerException( 321 | "ERROR: RUN_PULL config variable not expected range, contact admin" 322 | ) 323 | 324 | cmd.append(img) 325 | cmd.extend(args.image_args) 326 | 327 | exit_exec_cmd(cmd, dry_run=args.dry_run) 328 | -------------------------------------------------------------------------------- /userdocker/subcommands/version.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .. import __version__ 4 | from ..helpers.cmd import init_cmd 5 | from ..helpers.execute import exit_exec_cmd 6 | 7 | 8 | def exec_cmd_version(args): 9 | print("Userdocker Version: %s\n" % __version__) 10 | exit_exec_cmd(init_cmd(args), dry_run=args.dry_run) 11 | -------------------------------------------------------------------------------- /userdocker/userdocker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import logging 4 | import os 5 | import sys 6 | 7 | from .helpers.logger import logger 8 | from .helpers.logger import logger_setup 9 | 10 | from .helpers.cmd import init_cmd 11 | from .helpers.exceptions import UserDockerException 12 | from .helpers.execute import exit_exec_cmd 13 | from .parser import parse_args 14 | from .subcommands import specific_command_executors 15 | 16 | 17 | if not os.getenv('SUDO_UID'): 18 | logging.basicConfig() 19 | logger.warning("%s should be executed via sudo", sys.argv[0]) 20 | 21 | 22 | def prepare_and_exec_cmd(args): 23 | scmd = args.subcommand 24 | if scmd in specific_command_executors: 25 | specific_command_executors[scmd](args) 26 | else: 27 | exit_exec_cmd(init_cmd(args), dry_run=args.dry_run) 28 | 29 | 30 | def parse_and_exec_cmd(): 31 | if os.getenv('DOCKER_HOST'): 32 | raise UserDockerException( 33 | 'ERROR: DOCKER_HOST env var not supported yet' 34 | ) 35 | args = parse_args() 36 | logger_setup(args) 37 | prepare_and_exec_cmd(args) 38 | 39 | 40 | def main(): 41 | try: 42 | parse_and_exec_cmd() 43 | except UserDockerException as e: 44 | print(e, file=sys.stderr) 45 | sys.exit(1) 46 | --------------------------------------------------------------------------------