├── .gitignore ├── CREDITS.md ├── LICENSE ├── README.md ├── python ├── .gitignore ├── Dockerfile ├── Dockerfile.test ├── LICENSE ├── MANIFEST.in ├── Makefile ├── Makefile.pypi ├── entrypoint-devel.sh ├── nvidia_deepops │ ├── __init__.py │ ├── cli.py │ ├── docker │ │ ├── __init__.py │ │ ├── client │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── dockercli.py │ │ │ └── dockerpy.py │ │ └── registry │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── dgxregistry.py │ │ │ ├── dockregistry.py │ │ │ └── ngcregistry.py │ ├── progress.py │ └── utils.py ├── requirements.txt ├── requirements_dev.txt ├── setup.cfg ├── setup.py └── tests │ ├── __init__.py │ └── test_nvidia_deepops.py └── replicator ├── .gitignore ├── Dockerfile ├── Dockerfile.test ├── MANIFEST.in ├── Makefile ├── Makefile.pypi ├── README.md ├── entrypoint-devel.sh ├── ngc_replicator ├── __init__.py ├── ngc_replicator.py └── replicator_pb2.py ├── requirements.txt ├── requirements_dev.txt ├── scripts └── docker-utils ├── setup.cfg ├── setup.py ├── tests ├── __init__.py └── test_ngc_replicator.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | .Dockerfile 2 | -------------------------------------------------------------------------------- /CREDITS.md: -------------------------------------------------------------------------------- 1 | 2 | ## docker-reg-tool 3 | 4 | A copy of the `docker_reg_tool` script is included as `docker-utils` 5 | 6 | https://github.com/byrnedo/docker-reg-tool 7 | 8 | The MIT License (MIT) 9 | 10 | Copyright (c) 2016 Donal Byrne 11 | 12 | Permission is hereby granted, free of charge, to any person obtaining a copy 13 | of this software and associated documentation files (the "Software"), to deal 14 | in the Software without restriction, including without limitation the rights 15 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 | copies of the Software, and to permit persons to whom the Software is 17 | furnished to do so, subject to the following conditions: 18 | 19 | The above copyright notice and this permission notice shall be included in all 20 | copies or substantial portions of the Software. 21 | 22 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28 | SOFTWARE. 29 | 30 | 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, NVIDIA Corporation 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NGC Replicator 2 | 3 | Clones nvcr.io using the either DGX (compute.nvidia.com) or NGC (ngc.nvidia.com) 4 | API keys. 5 | 6 | The replicator will make an offline clone of the NGC/DGX container registry. 7 | In its current form, the replicator will download every CUDA container image as 8 | well as each Deep Learning framework image in the NVIDIA project. 9 | 10 | Tarfiles will be saved in `/output` inside the container, so be sure to volume 11 | mount that directory. In the following example, we will collect our images in 12 | `/tmp` on the host. 13 | 14 | Use `--min-version` to limit the number of versions to download. In the example 15 | below, we will only clone versions `17.10` and later DL framework images. 16 | 17 | ``` 18 | docker run --rm -it -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/output \ 19 | deepops/replicator --project=nvidia --min-version=17.12 \ 20 | --api-key= 21 | ``` 22 | 23 | You can also filter on specific images. 24 | If you want to filter only on image names containing the strings "tensorflow", 25 | "pytorch", and "tensorrt", you would simply add `--image` for each option, e.g. 26 | 27 | ``` 28 | docker run --rm -it -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/output \ 29 | deepops/replicator --project=nvidia --min-version=17.12 \ 30 | --image=tensorflow --image=pytorch --image=tensorrt \ 31 | --dry-run \ 32 | --api-key= 33 | ``` 34 | 35 | Note: the `--dry-run` option lets you see what will happen without committing 36 | to a lengthy download. 37 | 38 | By default, the `--image` flag does a substring match in order to ensure you match 39 | all images that may be desired. 40 | Sometimes, however, you only want to download a specific image with no substring 41 | matching. 42 | In this case, you can add the `--strict-name-match` flag, e.g. 43 | 44 | ``` 45 | docker run --rm -it -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/output \ 46 | deepops/replicator --project=nvidia --min-version=17.12 \ 47 | --image=tensorflow \ 48 | --strict-name-match \ 49 | --dry-run \ 50 | --api-key= 51 | ``` 52 | 53 | Note: a `state.yml` file will be created the output directory. This saved state will be used to 54 | avoid pulling images that were previously pulled. If you wish to repull and save an image, just 55 | delete the entry in `state.yml` corresponding to the `image_name` and `tag` you wish to refresh. 56 | 57 | ## Kubernetes Deployment 58 | 59 | If you don't already have a `deepops` namespace, create one now. 60 | 61 | ``` 62 | kubectl create namespace deepops 63 | ``` 64 | 65 | Next, create a secret with your NGC API Key 66 | 67 | ``` 68 | kubectl -n deepops create secret generic ngc-secret 69 | --from-literal=apikey= 70 | ``` 71 | 72 | Next, create a persistent volume claim that will life outside the lifecycle of the CronJob. If 73 | you are using [DeepOps](https://github.com/nvidia/deepops) you can use a Rook/Ceph PVC similar 74 | to: 75 | 76 | ``` 77 | --- 78 | apiVersion: v1 79 | kind: PersistentVolumeClaim 80 | metadata: 81 | name: ngc-replicator-pvc 82 | namespace: deepops 83 | labels: 84 | app: ngc-replicator 85 | spec: 86 | storageClassName: rook-raid0-retain # <== Replace with your StorageClass 87 | accessModes: 88 | - ReadWriteOnce 89 | resources: 90 | requests: 91 | storage: 32Mi 92 | ``` 93 | 94 | Finally, create a `CronJob` that executes the replicator on a schedule. This 95 | eample run the replicator every hour. Note: This example used 96 | [Rook](https://rook.io) block storage to provide a persistent volume to hold the 97 | `state.yml` between executions. This ensures you will only download new 98 | container images. For more details, see our [DeepOps 99 | project](https://github.com/nvidia/deepops). 100 | 101 | ``` 102 | --- 103 | apiVersion: v1 104 | kind: ConfigMap 105 | metadata: 106 | name: replicator-config 107 | namespace: deepops 108 | data: 109 | ngc-update.sh: | 110 | #!/bin/bash 111 | ngc_replicator \ 112 | --project=nvidia \ 113 | --min-version=$(date +"%y.%m" -d "1 month ago") \ 114 | --py-version=py3 \ 115 | --image=tensorflow --image=pytorch --image=tensorrt \ 116 | --no-exporter \ 117 | --registry-url=registry.local # <== Replace with your local repo 118 | --- 119 | apiVersion: batch/v1beta1 120 | kind: CronJob 121 | metadata: 122 | name: ngc-replicator 123 | namespace: deepops 124 | labels: 125 | app: ngc-replicator 126 | spec: 127 | schedule: "0 4 * * *" 128 | jobTemplate: 129 | spec: 130 | template: 131 | spec: 132 | nodeSelector: 133 | node-role.kubernetes.io/master: "" 134 | containers: 135 | - name: replicator 136 | image: deepops/replicator 137 | imagePullPolicy: Always 138 | command: [ "/bin/sh", "-c", "/ngc-update/ngc-update.sh" ] 139 | env: 140 | - name: NGC_REPLICATOR_API_KEY 141 | valueFrom: 142 | secretKeyRef: 143 | name: ngc-secret 144 | key: apikey 145 | volumeMounts: 146 | - name: registry-config 147 | mountPath: /ngc-update 148 | - name: docker-socket 149 | mountPath: /var/run/docker.sock 150 | - name: ngc-replicator-storage 151 | mountPath: /output 152 | volumes: 153 | - name: registry-config 154 | configMap: 155 | name: replicator-config 156 | defaultMode: 0777 157 | - name: docker-socket 158 | hostPath: 159 | path: /var/run/docker.sock 160 | type: File 161 | - name: ngc-replicator-storage 162 | persistentVolumeClaim: 163 | claimName: ngc-replicator-pvc 164 | restartPolicy: Never 165 | ``` 166 | 167 | ## Developer Quickstart 168 | 169 | ``` 170 | make dev 171 | py.test 172 | ``` 173 | 174 | ## TODOs 175 | 176 | - [x] save markdown readmes for each image. these are not version controlled 177 | - [x] test local registry push service. coded, beta testing 178 | - [ ] add templater to workflow 179 | -------------------------------------------------------------------------------- /python/.gitignore: -------------------------------------------------------------------------------- 1 | /__init__.py 2 | .eggs/ 3 | *.egg-info/ 4 | tests/secrets.py 5 | 6 | -------------------------------------------------------------------------------- /python/Dockerfile: -------------------------------------------------------------------------------- 1 | RUN pip install --upgrade pip 2 | 3 | RUN apt update && apt install -y --no-install-recommends curl tar vim-tiny make sudo && \ 4 | rm -rf /var/cache/apt/* 5 | 6 | ENV DOCKER_CHANNEL stable 7 | ENV DOCKER_VERSION 17.12.0-ce 8 | RUN if ! curl -fL -o docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz"; then \ 9 | echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}'"; \ 10 | exit 1; \ 11 | fi; \ 12 | \ 13 | tar --extract \ 14 | --file docker.tgz \ 15 | --strip-components 1 \ 16 | --directory /usr/local/bin/ \ 17 | ; \ 18 | rm docker.tgz 19 | 20 | COPY ./requirements.txt /tmp/requirements.txt 21 | RUN pip install -r /tmp/requirements.txt && rm /tmp/requirements.txt 22 | 23 | ENV LC_ALL=C.UTF-8 24 | ENV LANG=C.UTF-8 25 | 26 | WORKDIR /source/nvidia_deepops 27 | COPY . . 28 | #RUN pip install -r requirements.txt 29 | RUN python setup.py install 30 | 31 | ENTRYPOINT [] 32 | CMD ["/bin/bash"] 33 | 34 | # DeepOps containers will have {{ cluster_config }} mapped to /opt/deepops as read-only. 35 | # Each container also owns {{ cluster_config }}/{{ container_name }} in which it has full 36 | # access -- this path is mapped to /data inside the container. 37 | RUN mkdir -p /data 38 | -------------------------------------------------------------------------------- /python/Dockerfile.test: -------------------------------------------------------------------------------- 1 | RUN pip install pytest 2 | CMD ["pytest"] 3 | -------------------------------------------------------------------------------- /python/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions 5 | are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of NVIDIA CORPORATION nor the names of its 12 | contributors may be used to endorse or promote products derived 13 | from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 16 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 18 | PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 19 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 20 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 22 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 23 | OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /python/MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include tests * 2 | recursive-exclude * __pycache__ 3 | recursive-exclude * *.py[co] 4 | -------------------------------------------------------------------------------- /python/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # * Redistributions of source code must retain the above copyright 7 | # notice, this list of conditions and the following disclaimer. 8 | # * Redistributions in binary form must reproduce the above copyright 9 | # notice, this list of conditions and the following disclaimer in the 10 | # documentation and/or other materials provided with the distribution. 11 | # * Neither the name of NVIDIA CORPORATION nor the names of its 12 | # contributors may be used to endorse or promote products derived 13 | # from this software without specific prior written permission. 14 | # 15 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 16 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 18 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 19 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 20 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 22 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 23 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | BASE_IMAGE ?= python:3.6-slim 28 | IMAGE_NAME ?= deepops_python 29 | RELEASE_VERSION=0.4.2 30 | RELEASE_IMAGE ?= deepops/python 31 | 32 | APT_PROXY ?= "false" 33 | APT_CACHE="RUN echo 'Acquire::HTTP::Proxy \"${APT_PROXY}\";' >> /etc/apt/apt.conf.d/01proxy" 34 | ifdef PYPI_PROXY 35 | PYPI_CACHE="RUN mkdir -p ~/.pip && echo [global] > ~/.pip/pip.conf && echo trusted-host = ${PYPI_PROXY} >> ~/.pip/pip.conf && echo index-url = http://${PYPI_PROXY}:3141/root/pypi/+simple/ >> ~/.pip/pip.conf" 36 | else 37 | PYPI_CACHE="" 38 | endif 39 | 40 | .PHONY: build tag push release clean distclean 41 | 42 | default: clean build 43 | 44 | build: 45 | @echo FROM ${BASE_IMAGE} > .Dockerfile 46 | @echo ${APT_CACHE} >> .Dockerfile 47 | @echo ${PYPI_CACHE} >> .Dockerfile 48 | @cat Dockerfile >> .Dockerfile 49 | @echo "RUN rm -f /etc/apt/apt.conf.d/01proxy" >> .Dockerfile 50 | @echo "RUN rm -rf ~/.pip" >> .Dockerfile 51 | docker build -t ${IMAGE_NAME} -f .Dockerfile . 52 | 53 | build-dev: build 54 | @echo FROM ${IMAGE_NAME} > .Dockerfile 55 | @echo ${APT_CACHE} >> .Dockerfile 56 | @echo ${PYPI_CACHE} >> .Dockerfile 57 | @cat Dockerfile.test >> .Dockerfile 58 | @echo "RUN rm -f /etc/apt/apt.conf.d/01proxy" >> .Dockerfile 59 | @echo "RUN rm -rf ~/.pip" >> .Dockerfile 60 | docker build -t ${IMAGE_NAME}:dev -f .Dockerfile . 61 | 62 | dev: build-dev 63 | docker run --rm -ti -v ${PWD}:/devel -v /var/run/docker.sock:/var/run/docker.sock \ 64 | --entrypoint=/devel/entrypoint-devel.sh --workdir=/devel ${IMAGE_NAME}:dev sh 65 | 66 | test: build-dev 67 | docker run --rm -ti -v /var/run/docker.sock:/var/run/docker.sock ${IMAGE_NAME}:dev 68 | 69 | tag: build 70 | docker tag ${IMAGE_NAME} ${RELEASE_IMAGE}:${RELEASE_VERSION} 71 | docker tag ${IMAGE_NAME} ${RELEASE_IMAGE}:latest 72 | 73 | push: tag 74 | docker push ${RELEASE_IMAGE}:${RELEASE_VERSION} 75 | docker push ${RELEASE_IMAGE}:latest 76 | 77 | release: push 78 | make -f Makefile.pypi dist 79 | 80 | clean: 81 | @rm -f .Dockerfile 2> /dev/null ||: 82 | @find . -name "__pycache__" | xargs rm -rf 83 | @docker rm -v `docker ps -a -q -f "status=exited"` 2> /dev/null ||: 84 | @docker rmi `docker images -q -f "dangling=true"` 2> /dev/null ||: 85 | 86 | distclean: clean 87 | @docker rmi ${BASE_IMAGE} 2> /dev/null ||: 88 | @docker rmi ${IMAGE_NAME} 2> /dev/null ||: 89 | @docker rmi ${RELEASE_IMAGE} 2> /dev/null ||: 90 | -------------------------------------------------------------------------------- /python/Makefile.pypi: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # * Redistributions of source code must retain the above copyright 7 | # notice, this list of conditions and the following disclaimer. 8 | # * Redistributions in binary form must reproduce the above copyright 9 | # notice, this list of conditions and the following disclaimer in the 10 | # documentation and/or other materials provided with the distribution. 11 | # * Neither the name of NVIDIA CORPORATION nor the names of its 12 | # contributors may be used to endorse or promote products derived 13 | # from this software without specific prior written permission. 14 | # 15 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 16 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 18 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 19 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 20 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 22 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 23 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | .PHONY: clean clean-test clean-pyc clean-build docs help 28 | .DEFAULT_GOAL := help 29 | define BROWSER_PYSCRIPT 30 | import os, webbrowser, sys 31 | try: 32 | from urllib import pathname2url 33 | except: 34 | from urllib.request import pathname2url 35 | 36 | webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) 37 | endef 38 | export BROWSER_PYSCRIPT 39 | 40 | define PRINT_HELP_PYSCRIPT 41 | import re, sys 42 | 43 | for line in sys.stdin: 44 | match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) 45 | if match: 46 | target, help = match.groups() 47 | print("%-20s %s" % (target, help)) 48 | endef 49 | export PRINT_HELP_PYSCRIPT 50 | BROWSER := python -c "$$BROWSER_PYSCRIPT" 51 | 52 | help: 53 | @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) 54 | 55 | clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts 56 | 57 | 58 | clean-build: ## remove build artifacts 59 | rm -fr build/ 60 | rm -fr dist/ 61 | rm -fr .eggs/ 62 | find . -name '*.egg-info' -exec rm -fr {} + 63 | find . -name '*.egg' -exec rm -f {} + 64 | 65 | clean-pyc: ## remove Python file artifacts 66 | find . -name '*.pyc' -exec rm -f {} + 67 | find . -name '*.pyo' -exec rm -f {} + 68 | find . -name '*~' -exec rm -f {} + 69 | find . -name '__pycache__' -exec rm -fr {} + 70 | 71 | clean-test: ## remove test and coverage artifacts 72 | rm -fr .tox/ 73 | rm -f .coverage 74 | rm -fr htmlcov/ 75 | 76 | lint: ## check style with flake8 77 | flake8 nvidia_deepops tests 78 | 79 | test: ## run tests quickly with the default Python 80 | py.test 81 | 82 | 83 | test-all: ## run tests on every Python version with tox 84 | tox 85 | 86 | coverage: ## check code coverage quickly with the default Python 87 | coverage run --source nvidia_deepops -m pytest 88 | coverage report -m 89 | coverage html 90 | $(BROWSER) htmlcov/index.html 91 | 92 | docs: ## generate Sphinx HTML documentation, including API docs 93 | rm -f docs/nvidia_deepops.rst 94 | rm -f docs/modules.rst 95 | sphinx-apidoc -o docs/ nvidia_deepops 96 | $(MAKE) -C docs clean 97 | $(MAKE) -C docs html 98 | $(BROWSER) docs/_build/html/index.html 99 | 100 | servedocs: docs ## compile the docs watching for changes 101 | watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . 102 | 103 | release: clean ## package and upload a release 104 | python setup.py sdist upload 105 | python setup.py bdist_wheel upload 106 | 107 | dist: clean ## builds source and wheel package 108 | python3 setup.py sdist 109 | python3 setup.py bdist_wheel 110 | ls -l dist 111 | 112 | install: clean ## install the package to the active Python's site-packages 113 | python3 setup.py install 114 | -------------------------------------------------------------------------------- /python/entrypoint-devel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | set -e 30 | cat <<'EOF' 31 | 32 | _ 33 | | | 34 | __| | ___ ___ _ __ ___ _ __ ___ 35 | / _` |/ _ \/ _ \ '_ \ / _ \| '_ \/ __| 36 | | (_| | __/ __/ |_) | (_) | |_) \__ \ 37 | \__,_|\___|\___| .__/ \___/| .__/|___/ 38 | | | | | 39 | |_| |_| 40 | 41 | Starting developer environment ... 42 | EOF 43 | 44 | pip install -e . 45 | 46 | if [[ $# -eq 0 ]]; then 47 | exec "/bin/bash" 48 | else 49 | exec "$@" 50 | fi 51 | -------------------------------------------------------------------------------- /python/nvidia_deepops/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | """Top-level package for NVIDIA DeepOps Python Library.""" 30 | 31 | __author__ = """Ryan Olson""" 32 | __email__ = 'rolson@nvidia.com' 33 | __version__ = '0.4.2' 34 | 35 | from nvidia_deepops.progress import Progress 36 | -------------------------------------------------------------------------------- /python/nvidia_deepops/cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | """Console script for nvidia_deepops.""" 30 | 31 | import click 32 | import hashlib 33 | import yaml 34 | 35 | from . import progress 36 | 37 | @click.command() 38 | def main(args=None): 39 | """Console script for nvidia_deepops.""" 40 | click.echo("cli coming soon...") 41 | 42 | 43 | @click.command() 44 | @click.option("--name", required=True) 45 | @click.option("--key", required=True) 46 | @click.option("--status", type=click.Choice(progress.STATES.values())) 47 | @click.option("--header") 48 | @click.option("--subtitle") 49 | @click.option("--fixed/--infinite", default=True) 50 | @click.option("--op", type=click.Choice(["create", "append", "update", "run"])) 51 | def progress_cli(name, key, status, header, subtitle, fixed, op): 52 | op = op or "run" 53 | with progress.load_state(name) as state: 54 | if fixed: 55 | state.set_fixed_progress() 56 | else: 57 | state.set_infinite_progress() 58 | if op == "create": 59 | state.steps.clear() 60 | if op == "create" or op == "append": 61 | state.add_step(key=key, status=status, header=header, subHeader=subtitle) 62 | elif op == "update": 63 | step = state.steps[key] 64 | status = status or step["status"] 65 | header = header or step["header"] 66 | subtitle = subtitle or step["subHeader"] 67 | state.update_step(key=key, status=status, header=header, subHeader=subtitle) 68 | state.post() 69 | elif op == "run": 70 | keys = list(state.steps.keys()) 71 | completed_keys = keys[0:keys.index(key)] 72 | for k in completed_keys: 73 | state.update_step(key=k, status="complete") 74 | state.update_step(key=key, status="running") 75 | click.echo("{op} Step: {key}\nHeader: {header}\nSubtitle: {subHeader}".format( 76 | op=op.title(), key=key, **state.steps[key]) 77 | ) 78 | state.post() 79 | else: 80 | raise RuntimeError("this shouldn't happen") 81 | 82 | 83 | if __name__ == "__main__": 84 | main() 85 | -------------------------------------------------------------------------------- /python/nvidia_deepops/docker/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | from .client import * 30 | from .registry import * 31 | -------------------------------------------------------------------------------- /python/nvidia_deepops/docker/client/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | from .base import * 30 | from .dockercli import * 31 | from .dockerpy import * 32 | -------------------------------------------------------------------------------- /python/nvidia_deepops/docker/client/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | import abc 30 | 31 | 32 | __all__ = ('BaseClient',) 33 | 34 | ABC = abc.ABCMeta('ABC', (object,), {}) # compatible with Python 2 *and* 3 35 | 36 | 37 | class BaseClient(ABC): 38 | 39 | @abc.abstractmethod 40 | def pull(self, url): 41 | raise NotImplementedError() 42 | 43 | @abc.abstractmethod 44 | def tag(self, src_url, dst_url): 45 | raise NotImplementedError() 46 | 47 | @abc.abstractmethod 48 | def push(self, url): 49 | raise NotImplementedError() 50 | 51 | @abc.abstractmethod 52 | def remove(self, url): 53 | raise NotImplementedError() 54 | 55 | -------------------------------------------------------------------------------- /python/nvidia_deepops/docker/client/dockercli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | import logging 30 | import os 31 | import shlex 32 | import subprocess 33 | import sys 34 | 35 | import docker 36 | 37 | from nvidia_deepops import utils 38 | from nvidia_deepops.docker.client.base import BaseClient 39 | 40 | __all__ = ('DockerClient',) 41 | 42 | log = utils.get_logger(__name__, level=logging.INFO) 43 | 44 | 45 | class DockerClient(BaseClient): 46 | 47 | def __init__(self): 48 | self.client = docker.from_env(timeout=600) 49 | 50 | def call(self, command, stdout=None, stderr=None, quiet=False): 51 | stdout = stdout or sys.stderr 52 | stderr = stderr or sys.stderr 53 | if quiet: 54 | stdout = subprocess.PIPE 55 | stderr = subprocess.PIPE 56 | log.debug(command) 57 | subprocess.check_call(shlex.split(command), stdout=stdout, 58 | stderr=stderr) 59 | 60 | def login(self, *, username, password, registry): 61 | self.call( 62 | "docker login -u {} -p {} {}".format(username, password, registry)) 63 | self.client.login(username=username, 64 | password=password, registry=registry) 65 | 66 | def get(self, *, url): 67 | try: 68 | return self.client.images.get(url) 69 | except docker.errors.ImageNotFound: 70 | return None 71 | 72 | def pull(self, url): 73 | self.call("docker pull %s" % url) 74 | return url 75 | 76 | def push(self, url): 77 | self.call("docker push %s" % url) 78 | return url 79 | 80 | def tag(self, src_url, dst_url): 81 | self.call("docker tag %s %s" % (src_url, dst_url)) 82 | return dst_url 83 | 84 | def remove(self, url): 85 | self.call("docker rmi %s" % url) 86 | return url 87 | 88 | def url2filename(self, url): 89 | return "docker_image_{}.tar".format(url).replace("/", "%%") 90 | 91 | def filename2url(self, filename): 92 | return os.path.basename(filename).replace("docker_image_", "")\ 93 | .replace(".tar", "").replace("%%", "/") 94 | 95 | def save(self, url, path=None): 96 | filename = self.url2filename(url) 97 | if path: 98 | filename = os.path.join(path, filename) 99 | self.call("docker save -o {} {}".format(filename, url)) 100 | return filename 101 | 102 | def image_exists(self, url): 103 | try: 104 | self.call("docker image inspect {}".format(url), quiet=True) 105 | return True 106 | except Exception: 107 | return False 108 | 109 | def load(self, filename, expected_url=None): 110 | url = expected_url or self.filename2url(filename) 111 | basename = os.path.basename(filename) 112 | if expected_url is None and not basename.startswith("docker_image_"): 113 | raise RuntimeError("Invalid filename") 114 | self.call("docker load -i %s" % filename) 115 | if not self.image_exists(url): 116 | log.error("expected url from %s is %s" % (filename, url)) 117 | raise RuntimeError("Image {} not found".format(url)) 118 | log.debug("loaded {} from {}".format(url, filename)) 119 | return url 120 | 121 | def build(self, *, target_image, dockerfile="Dockerfile"): 122 | self.call("docker build -f {} -t {} .".format(dockerfile, target_image)) -------------------------------------------------------------------------------- /python/nvidia_deepops/docker/client/dockerpy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | import logging 30 | import os 31 | 32 | import docker 33 | 34 | from nvidia_deepops import utils 35 | from nvidia_deepops.docker.client.base import BaseClient 36 | 37 | 38 | __all__ = ('DockerPy',) 39 | 40 | 41 | log = utils.get_logger(__name__, level=logging.INFO) 42 | 43 | 44 | class DockerPy(BaseClient): 45 | 46 | def __init__(self): 47 | self.client = docker.from_env(timeout=600) 48 | 49 | def login(self, *, username, password, registry): 50 | self.client.login(username=username, 51 | password=password, registry=registry) 52 | 53 | def get(self, *, url): 54 | try: 55 | return self.client.images.get(url) 56 | except docker.errors.ImageNotFound: 57 | return None 58 | 59 | def pull(self, url): 60 | log.debug("docker pull %s" % url) 61 | self.client.images.pull(url) 62 | 63 | def push(self, url): 64 | log.debug("docker push %s" % url) 65 | self.client.images.push(url) 66 | 67 | def tag(self, src_url, dst_url): 68 | log.debug("docker tag %s --> %s" % (src_url, dst_url)) 69 | image = self.client.images.get(src_url) 70 | image.tag(dst_url) 71 | 72 | def remove(self, url): 73 | log.debug("docker rmi %s" % url) 74 | self.client.images.remove(url) 75 | 76 | def url2filename(self, url): 77 | return "docker_image_{}.tar".format(url).replace("/", "%%") 78 | 79 | def filename2url(self, filename): 80 | return os.path.basename(filename).replace("docker_image_", "")\ 81 | .replace(".tar", "").replace("%%", "/") 82 | 83 | def save(self, url, path=None): 84 | filename = self.url2filename(url) 85 | if path: 86 | filename = os.path.join(path, filename) 87 | log.debug("saving %s --> %s" % (url, filename)) 88 | image = self.client.api.get_image(url) 89 | with open(filename, "wb") as tarfile: 90 | tarfile.write(image.data) 91 | return filename 92 | 93 | def load(self, filename): 94 | log.debug("loading image from %s" % filename) 95 | with open(filename, "rb") as file: 96 | self.client.images.load(file) 97 | basename = os.path.basename(filename) 98 | if basename.startswith("docker_image_"): 99 | url = self.filename2url(filename) 100 | log.debug("expected url from %s is %s" % (filename, url)) 101 | return url 102 | -------------------------------------------------------------------------------- /python/nvidia_deepops/docker/registry/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | from .base import * 30 | from .dockregistry import * 31 | from .dgxregistry import * 32 | from .ngcregistry import * 33 | -------------------------------------------------------------------------------- /python/nvidia_deepops/docker/registry/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | import abc 30 | 31 | 32 | __all__ = ('BaseRegistry',) 33 | 34 | 35 | ABC = abc.ABCMeta('ABC', (object,), {}) # compatible with Python 2 *and* 3 36 | 37 | 38 | class BaseRegistry(ABC): 39 | 40 | @abc.abstractmethod 41 | def get_image_names(self, project=None): 42 | raise NotImplementedError() 43 | 44 | @abc.abstractmethod 45 | def get_image_tags(self, image_name): 46 | raise NotImplementedError() 47 | 48 | @abc.abstractmethod 49 | def get_state(self, project=None, filter_fn=None): 50 | """ 51 | Returns a unique hash for each image and tag with the ability to filter 52 | on the project/prefix. 53 | 54 | :param str project: Filter images on the prefix, e.g. project="nvidia" 55 | filters all `nvidia/*` images 56 | :param filter_fn: Callable function that takes (name, tag, docker_id) 57 | kwargs and returns true/false. Ff the image should be included in 58 | the returned set. 59 | :return: dict of dicts 60 | { 61 | "image_name_A": { 62 | "tag_1": "dockerImageId_1", 63 | "tag_2": "dockerImageId_2", 64 | }, ... 65 | } 66 | 67 | """ 68 | raise NotImplementedError() 69 | 70 | def docker_url(self, name, tag): 71 | return "{}/{}:{}".format(self.url, name, tag) 72 | 73 | def get_images_and_tags(self, project=None): 74 | """ 75 | Returns a dict keyed on image_name with values as a list of tags names 76 | :param project: optional filter on image_name, e.g. project='nvidia' 77 | filters all 'nvidia/*' images 78 | :return: Dict key'd by image names. Dict val are lists of tags. Ex.: 79 | { 80 | "nvidia/pytorch": ["17.07"], 81 | "nvidia/tensorflow": ["17.07", "17.06"], 82 | } 83 | """ 84 | image_names = self.get_image_names(project=project) 85 | return {name: self.get_image_tags(name) for name in image_names} 86 | -------------------------------------------------------------------------------- /python/nvidia_deepops/docker/registry/dgxregistry.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | import collections 30 | import base64 31 | import logging 32 | 33 | import contexttimer 34 | import requests 35 | 36 | from nvidia_deepops import utils 37 | from nvidia_deepops.docker.registry.base import BaseRegistry 38 | 39 | 40 | log = utils.get_logger(__name__, level=logging.INFO) 41 | dev = utils.get_logger("devel", level=logging.ERROR) 42 | 43 | 44 | __all__ = ('DGXRegistry',) 45 | 46 | 47 | class DGXRegistry(BaseRegistry): 48 | 49 | def __init__(self, api_key, nvcr_url='nvcr.io', 50 | nvcr_api_url=None): 51 | self.api_key = api_key 52 | self.api_key_b64 = base64.b64encode(api_key.encode("utf-8"))\ 53 | .decode("utf-8") 54 | self.url = nvcr_url 55 | nvcr_api_url = 'https://compute.nvidia.com' if nvcr_api_url is None \ 56 | else nvcr_api_url 57 | self._nvcr_api_url = nvcr_api_url 58 | 59 | def _get(self, endpoint): 60 | dev.debug("GET %s" % self._api_url(endpoint)) 61 | with contexttimer.Timer() as timer: 62 | req = requests.get(self._api_url(endpoint), headers={ 63 | 'Authorization': 'APIKey {}'.format(self.api_key_b64), 64 | 'Accept': 'application/json', 65 | }) 66 | log.info("GET {} - took {} sec".format(self._api_url(endpoint), 67 | timer.elapsed)) 68 | req.raise_for_status() 69 | data = req.json() 70 | # dev.debug("GOT {}: {}".format(self._api_url(endpoint), 71 | # pprint.pformat(data, indent=4))) 72 | return data 73 | 74 | def _api_url(self, endpoint): 75 | return "{}/rest/api/v1/".format(self._nvcr_api_url) + endpoint 76 | 77 | def _get_repo_data(self, project=None): 78 | """ 79 | Returns a list of dictionaries containing top-level details for each 80 | image. 81 | 82 | :param project: optional project/namespace; filter on all `nvidia` or 83 | `nvidian_sas` projects 84 | :return: list of dicts with the following format: 85 | { 86 | "isReadOnly": true, 87 | "isPublic": true, 88 | "namespace": "nvidia", 89 | "name": "caffe2", 90 | "description": "## What is Caffe2?\n\nCaffe2 is a deep-learning 91 | framework ... " 92 | } 93 | """ 94 | def in_project(img): 95 | if project: 96 | return img["namespace"] == project 97 | return True 98 | 99 | def update(image): 100 | image["image_name"] = image["namespace"] + "/" + image["name"] 101 | return image 102 | data = self._get("repository?includePublic=true") 103 | return [update(image) for image in data["repositories"] 104 | if in_project(image)] 105 | 106 | def get_image_names(self, project=None, cache=None): 107 | """ 108 | Returns a list of image names optionally filtered on project. All 109 | names include the base project/namespace. 110 | 111 | :param project: optional filter, e.g. project="nvidia" filters all 112 | "nvidia/*" images 113 | :return: ["nvidia/caffe", "nvidia/cuda", ...] 114 | """ 115 | return [image["image_name"] 116 | for image in cache or self._get_repo_data(project=project)] 117 | 118 | def get_image_descriptions(self, project=None, cache=None): 119 | return {image['image_name']: image.get("description", "") 120 | for image in cache or self._get_repo_data(project=project)} 121 | 122 | def get_image_tags(self, image_name, cache=None): 123 | """ 124 | Returns only the list of tag names similar to how the v2 api behaves. 125 | 126 | :param image_name: should consist of `/`, e.g. 127 | `nvidia/caffe` 128 | :return: list of tag strings: ['17.07', '17.06', ... ] 129 | """ 130 | return [tag['name'] 131 | for tag in cache or self._get_image_data(image_name)] 132 | 133 | def _get_image_data(self, image_name): 134 | """ 135 | Returns tags and other attributes of interest for each version of 136 | `image_name` 137 | 138 | :param image_name: should consist of `/`, e.g. 139 | `nvidia/caffe` 140 | :return: list of dicts for each tag with the following format: 141 | { 142 | "dockerImageId": "9c496e628c7d64badd2b587d4c0a387b0db00...", 143 | "lastModified": "2017-03-27T18:48:21.000Z", 144 | "name": "17.03", 145 | "size": 1244439426 146 | } 147 | """ 148 | endpoint = "/".join(["repository", image_name]) 149 | return self._get(endpoint)['tags'] 150 | 151 | def get_state(self, project=None, filter_fn=None): 152 | names = self.get_image_names(project=project) 153 | state = collections.defaultdict(dict) 154 | for name in names: 155 | for tag in self._get_image_data(name): 156 | if filter_fn is not None and callable(filter_fn): 157 | if not filter_fn(name=name, tag=tag["name"], 158 | docker_id=tag["dockerImageId"]): 159 | continue 160 | state[name][tag["name"]] = { 161 | "docker_id": tag["dockerImageId"], 162 | "registry": "nvcr.io", 163 | } 164 | return state 165 | -------------------------------------------------------------------------------- /python/nvidia_deepops/docker/registry/dockregistry.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | import pprint 30 | import logging 31 | 32 | import contexttimer 33 | import requests 34 | from requests.auth import AuthBase, HTTPBasicAuth 35 | 36 | from nvidia_deepops import utils 37 | from nvidia_deepops.docker.registry.base import BaseRegistry 38 | 39 | 40 | __all__ = ('DockerRegistry',) 41 | 42 | 43 | log = utils.get_logger(__name__, level=logging.INFO) 44 | 45 | 46 | class RegistryError(Exception): 47 | def __init__(self, message, code=None, detail=None): 48 | super(RegistryError, self).__init__(message) 49 | self.code = code 50 | self.detail = detail 51 | 52 | @classmethod 53 | def from_data(cls, data): 54 | """ 55 | Encapsulate an error response in an exception 56 | """ 57 | errors = data.get('errors') 58 | if not errors or len(errors) == 0: 59 | return cls('Unknown error!') 60 | 61 | # For simplicity, we'll just include the first error. 62 | err = errors[0] 63 | return cls( 64 | message=err.get('message'), 65 | code=err.get('code'), 66 | detail=err.get('detail'), 67 | ) 68 | 69 | 70 | class BearerAuth(AuthBase): 71 | def __init__(self, token): 72 | self.token = token 73 | 74 | def __call__(self, req): 75 | req.headers['Authorization'] = 'Bearer {}'.format(self.token) 76 | return req 77 | 78 | 79 | class DockerRegistry(BaseRegistry): 80 | 81 | def __init__(self, *, url, username=None, password=None, verify_ssl=False): 82 | url = url.rstrip('/') 83 | if not (url.startswith('http://') or url.startswith('https://')): 84 | url = 'https://' + url 85 | self.url = url 86 | 87 | self.username = username 88 | self.password = password 89 | self.verify_ssl = verify_ssl 90 | self.auth = None 91 | 92 | def authenticate(self): 93 | """ 94 | Forcefully auth for testing 95 | """ 96 | r = requests.head(self.url + '/v2/', verify=self.verify_ssl) 97 | self._authenticate_for(r) 98 | 99 | def _authenticate_for(self, resp): 100 | """ 101 | Authenticate to satsify the unauthorized response 102 | """ 103 | # Get the auth. info from the headers 104 | scheme, params = resp.headers['Www-Authenticate'].split(None, 1) 105 | assert (scheme == 'Bearer') 106 | info = {k: v.strip('"') for k, v in (i.split('=') 107 | for i in params.split(','))} 108 | 109 | # Request a token from the auth server 110 | params = {k: v for k, v in info.items() if k in ('service', 'scope')} 111 | auth = HTTPBasicAuth(self.username, self.password) 112 | r2 = requests.get(info['realm'], params=params, 113 | auth=auth, verify=self.verify_ssl) 114 | 115 | if r2.status_code == 401: 116 | raise RuntimeError("Authentication Error") 117 | r2.raise_for_status() 118 | 119 | self.auth = BearerAuth(r2.json()['token']) 120 | 121 | def _get(self, endpoint): 122 | url = '{0}/v2/{1}'.format(self.url, endpoint) 123 | log.debug("GET {}".format(url)) 124 | 125 | # Try to use previous bearer token 126 | with contexttimer.Timer() as timer: 127 | r = requests.get(url, auth=self.auth, verify=self.verify_ssl) 128 | 129 | log.info("GET {} - took {} sec".format(url, timer.elapsed)) 130 | 131 | # If necessary, try to authenticate and try again 132 | if r.status_code == 401: 133 | self._authenticate_for(r) 134 | r = requests.get(url, auth=self.auth, verify=self.verify_ssl) 135 | 136 | data = r.json() 137 | 138 | if r.status_code != 200: 139 | raise RegistryError.from_data(data) 140 | 141 | log.debug("GOT {}: {}".format(url, pprint.pformat(data, indent=4))) 142 | return data 143 | 144 | def get_image_names(self, project=None): 145 | data = self._get('_catalog') 146 | return [image for image in data['repositories']] 147 | 148 | def get_image_tags(self, image_name): 149 | endpoint = '{name}/tags/list'.format(name=image_name) 150 | return self._get(endpoint)['tags'] 151 | 152 | def get_manifest(self, name, reference): 153 | data = self._get( 154 | '{name}/manifests/{reference}'.format(name=name, 155 | reference=reference)) 156 | pprint.pprint(data) 157 | -------------------------------------------------------------------------------- /python/nvidia_deepops/docker/registry/ngcregistry.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | import collections 30 | import base64 31 | import logging 32 | import pprint 33 | 34 | import contexttimer 35 | import requests 36 | 37 | from nvidia_deepops import utils 38 | from nvidia_deepops.docker.registry.base import BaseRegistry 39 | 40 | 41 | log = utils.get_logger(__name__, level=logging.INFO) 42 | dev = utils.get_logger("devel", level=logging.ERROR) 43 | 44 | 45 | __all__ = ('NGCRegistry',) 46 | 47 | 48 | class NGCRegistry(BaseRegistry): 49 | 50 | def __init__(self, api_key, nvcr_url='nvcr.io', 51 | nvcr_api_url=None, 52 | ngc_auth_url=None): 53 | self.api_key = api_key 54 | self.api_key_b64 = base64.b64encode( 55 | api_key.encode("utf-8")).decode("utf-8") 56 | self.url = nvcr_url 57 | 58 | nvcr_api_url = 'https://api.ngc.nvidia.com' if nvcr_api_url is None \ 59 | else nvcr_api_url 60 | self._nvcr_api_url = nvcr_api_url 61 | ngc_auth_url = 'https://authn.nvidia.com' if ngc_auth_url is None \ 62 | else ngc_auth_url 63 | self._ngc_auth_url = ngc_auth_url 64 | 65 | self._token = None 66 | self.orgs = None 67 | self.default_org = None 68 | self._authenticate_for(None) 69 | 70 | def _authenticate_for(self, resp): 71 | """ 72 | Authenticate to satsify the unauthorized response 73 | """ 74 | # Invalidate current bearer token 75 | self._token = None 76 | 77 | # Future-proofing the API so the response from the failed request could 78 | # be evaluated here 79 | 80 | # Request a token from the auth server 81 | req = requests.get( 82 | url="{}/token?scope=group/ngc".format(self._ngc_auth_url), 83 | headers={ 84 | 'Authorization': 'ApiKey {}'.format(self.api_key_b64), 85 | 'Accept': 'application/json', 86 | } 87 | ) 88 | 89 | # Raise error on failed request 90 | req.raise_for_status() 91 | 92 | # Set new Bearer Token 93 | self._token = req.json()['token'] 94 | 95 | # Unfortunately NGC requests require an org-name, even for requests 96 | # where the org-name is extra/un-needed information. 97 | # To handle this condition, we will get the list of orgs the user 98 | # belongs to 99 | if not self.orgs: 100 | log.debug("no org list - fetching that now") 101 | data = self._get("orgs") 102 | self.orgs = data['organizations'] 103 | self.default_org = self.orgs[0]['name'] 104 | log.debug("default_org: {}".format(self.default_org)) 105 | 106 | @property 107 | def token(self): 108 | if not self._token: 109 | self._authenticate_for(None) 110 | if not self._token: 111 | raise RuntimeError( 112 | "NGC Bearer token is not set; this is unexpected") 113 | return self._token 114 | 115 | def _get(self, endpoint): 116 | dev.debug("GET %s" % self._api_url(endpoint)) 117 | 118 | # try to user current bearer token; this could result in a 401 if the 119 | # token is expired 120 | with contexttimer.Timer() as timer: 121 | req = requests.get(self._api_url(endpoint), headers={ 122 | 'Authorization': 'Bearer {}'.format(self.token), 123 | 'Accept': 'application/json', 124 | }) 125 | log.info("GET {} - took {} sec".format(self._api_url(endpoint), 126 | timer.elapsed)) 127 | 128 | if req.status_code == 401: 129 | # re-authenticate and repeat the request - failure here is final 130 | self._authenticate_for(req) 131 | req = requests.get(self._api_url(endpoint), headers={ 132 | 'Authorization': 'Bearer {}'.format(self.token), 133 | 'Accept': 'application/json', 134 | }) 135 | 136 | req.raise_for_status() 137 | 138 | data = req.json() 139 | dev.debug("GOT {}: {}".format(self._api_url( 140 | endpoint), pprint.pformat(data, indent=4))) 141 | return data 142 | 143 | def _api_url(self, endpoint): 144 | return "{}/v2/".format(self._nvcr_api_url) + endpoint 145 | 146 | def _get_repo_data(self, project=None): 147 | """ 148 | Returns a list of dictionaries containing top-level details for each 149 | image. 150 | :param project: optional project/namespace; filter on all `nvidia` or 151 | `nvidian_sas` projects 152 | :return: list of dicts with the following format: 153 | { 154 | "requestStatus": { 155 | "statusCode": "SUCCESS", 156 | "requestId": "edbbaccf-f1f0-4107-b2ba-47bda0b4b308" 157 | }, 158 | "repositories": [ 159 | { 160 | "isReadOnly": true, 161 | "isPublic": true, 162 | "namespace": "nvidia", 163 | "name": "caffe", 164 | "description": "## What is NVCaffe?\n\nCaffe is a deep 165 | learning framework ...", 166 | }, 167 | { 168 | "isReadOnly": true, 169 | "isPublic": true, 170 | "namespace": "nvidia", 171 | "name": "caffe2", 172 | "description": "## What is Caffe2?\n\nCaffe2 is a 173 | deep-learning framework ...", 174 | }, 175 | ... 176 | ] 177 | } 178 | """ 179 | def in_project(img): 180 | if project: 181 | return img["namespace"] == project 182 | return True 183 | 184 | def update(image): 185 | image["image_name"] = image["namespace"] + "/" + image["name"] 186 | return image 187 | 188 | data = self._get( 189 | "org/{}/repos?include-teams=true&include-public=true" 190 | .format(self.default_org)) 191 | return [update(image) 192 | for image in data["repositories"] if in_project(image)] 193 | 194 | def get_image_names(self, project=None, cache=None): 195 | """ 196 | Returns a list of image names optionally filtered on project. All 197 | names include the base project/namespace. 198 | 199 | :param project: optional filter, e.g. project="nvidia" filters all 200 | "nvidia/*" images 201 | :return: ["nvidia/caffe", "nvidia/cuda", ...] 202 | """ 203 | return [image["image_name"] 204 | for image in cache or self._get_repo_data(project=project)] 205 | 206 | def get_image_descriptions(self, project=None, cache=None): 207 | return {image['image_name']: image["description"] 208 | for image in cache or self._get_repo_data(project=project)} 209 | 210 | def get_image_tags(self, image_name, cache=None): 211 | """ 212 | Returns only the list of tag names similar to how the v2 api behaves. 213 | 214 | :param image_name: should consist of `/`, e.g. 215 | `nvidia/caffe` 216 | :return: list of tag strings: ['17.07', '17.06', ... ] 217 | """ 218 | return [image['tag'] 219 | for image in cache or self._get_image_data(image_name)] 220 | 221 | def _get_image_data(self, image_name): 222 | """ 223 | Returns tags and other attributes of interest for each version of 224 | `image_name` 225 | 226 | :param image_name: should consist of `/`, e.g. 227 | `nvidia/caffe` 228 | :return: list of dicts for each tag with the following format: 229 | { 230 | "requestStatus": { 231 | "statusCode": "SUCCESS", 232 | "requestId": "49468dff-8cba-4dcf-a841-a8bd43495fb5" 233 | }, 234 | "images": [ 235 | { 236 | "updatedDate": "2017-12-04T05:56:41.1440512Z", 237 | "tag": "17.12", 238 | "user": {}, 239 | "size": 1350502380 240 | }, 241 | { 242 | "updatedDate": "2017-11-16T21:19:08.363176299Z", 243 | "tag": "17.11", 244 | "user": {}, 245 | "size": 1350349188 246 | }, 247 | ] 248 | } 249 | """ 250 | org_name, repo_name = image_name.split('/') 251 | endpoint = "org/{}/repos/{}/images".format(org_name, repo_name) 252 | return self._get(endpoint).get('images', []) 253 | 254 | def get_state(self, project=None, filter_fn=None): 255 | names = self.get_image_names(project=project) 256 | state = collections.defaultdict(dict) 257 | for name in names: 258 | image_data = self._get_image_data(name) 259 | for image in image_data: 260 | tag = image["tag"] 261 | docker_id = image["updatedDate"] 262 | if filter_fn is not None and callable(filter_fn): 263 | if not filter_fn(name=name, tag=tag, docker_id=docker_id): 264 | # if filter_fn is false, then the image is not added to 265 | # the state 266 | continue 267 | state[name][tag] = { 268 | "docker_id": docker_id, 269 | "registry": "nvcr.io", 270 | } 271 | return state 272 | -------------------------------------------------------------------------------- /python/nvidia_deepops/progress.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | import collections 30 | import hashlib 31 | import itertools 32 | import json 33 | import logging 34 | import os 35 | 36 | import requests 37 | import yaml 38 | 39 | from contextlib import contextmanager 40 | 41 | from . import utils 42 | 43 | log = utils.get_logger(__name__, level=logging.INFO) 44 | 45 | STATES = { 46 | "waiting": "waiting", 47 | "running": "running", 48 | "complete": "complete", 49 | "error": "error", 50 | } 51 | 52 | def filename(name, path=None): 53 | path = path or "/tmp" 54 | sha256 = hashlib.sha256() 55 | sha256.update(os.path.join("/tmp/{}".format(name)).encode("utf-8")) 56 | filename = sha256.hexdigest() 57 | return os.path.join(path, filename) 58 | 59 | @contextmanager 60 | def load_state(name, progress_uri=None, path=None): 61 | progress_uri = progress_uri or os.environ.get("DEEPOPS_WEBUI_PROGRESS_URI") 62 | _filename = filename(name, path=path) 63 | p = Progress(uri=progress_uri) 64 | if os.path.exists(_filename): 65 | p.read_prgress(_filename) 66 | yield p 67 | p.write_progress(_filename) 68 | 69 | 70 | class Progress: 71 | 72 | def __init__(self, *, uri=None, progress_length_unknown=False): 73 | self.uri = uri 74 | self.steps = collections.OrderedDict() 75 | self.progress_length_unknown = progress_length_unknown 76 | 77 | def add_step(self, *, key, status=None, header=None, subHeader=None): 78 | self.steps[key] = { 79 | "status": STATES.get(status, "waiting"), 80 | "header": header or key, 81 | "subHeader": subHeader or "" 82 | } 83 | 84 | def set_infinite_progress(self): 85 | self.progress_length_unknown = True 86 | 87 | def set_fixed_progress(self): 88 | self.progress_length_unknown = False 89 | 90 | def update_step(self, *, key, status, header=None, subHeader=None): 91 | step = self.steps[key] 92 | step["status"] = STATES[status] 93 | if header: 94 | step["header"] = header 95 | if subHeader: 96 | step["subHeader"] = subHeader 97 | 98 | def write_progress(self, path): 99 | ordered_data = { 100 | "keys": list(self.steps.keys()), 101 | "vals": list(self.steps.values()), 102 | "length_unknown": self.progress_length_unknown 103 | } 104 | with open(path, "w") as file: 105 | yaml.dump(ordered_data, file) 106 | 107 | def read_prgress(self, path): 108 | if not os.path.exists(path): 109 | raise RuntimeError("{} does not exist".format(path)) 110 | with open(path, "r") as file: 111 | ordered_data = yaml.load(file) 112 | if ordered_data is None: 113 | return 114 | steps = collections.OrderedDict() 115 | for key, val in zip(ordered_data["keys"], ordered_data["vals"]): 116 | steps[key] = val 117 | self.steps = steps 118 | self.progress_length_unknown = ordered_data["length_unknown"] 119 | 120 | 121 | @contextmanager 122 | def run_step(self, *, key, post_on_complete=True, progress_length_unknown=None): 123 | progress_length_unknown = progress_length_unknown or self.progress_length_unknown 124 | step = self.steps[key] 125 | step["status"] = STATES["running"] 126 | self.post(progress_length_unknown=progress_length_unknown) 127 | try: 128 | yield step 129 | step["status"] = STATES["complete"] 130 | except Exception as err: 131 | step["status"] = STATES["error"] 132 | step["subHeader"] = str(err) 133 | post_on_complete = True 134 | raise 135 | finally: 136 | if post_on_complete: 137 | self.post(progress_length_unknown=progress_length_unknown) 138 | 139 | 140 | def data(self, progress_length_unknown=False): 141 | progress_length_unknown = progress_length_unknown or self.progress_length_unknown 142 | steps = [v for _, v in self.steps.items()] 143 | return { 144 | "percent": -2 if progress_length_unknown else -1, 145 | "steps": steps 146 | } 147 | 148 | def post(self, progress_length_unknown=None): 149 | progress_length_unknown = progress_length_unknown or self.progress_length_unknown 150 | data = self.data(progress_length_unknown=progress_length_unknown) 151 | log.debug(data) 152 | if self.uri: 153 | try: 154 | r = requests.post(self.uri, json=data) 155 | r.raise_for_status() 156 | except Exception as err: 157 | log.warn("progress update failed with {}".format(str(err))) 158 | 159 | -------------------------------------------------------------------------------- /python/nvidia_deepops/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | import contextlib 30 | import logging 31 | import os 32 | import shlex 33 | import subprocess 34 | import sys 35 | 36 | 37 | def get_logger(name, level=None): 38 | level = level or logging.INFO 39 | log = logging.getLogger(name) 40 | log.setLevel(level) 41 | ch = logging.StreamHandler(sys.stdout) 42 | ch.setLevel(logging.DEBUG) 43 | formatter = logging.Formatter( 44 | '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s') 45 | ch.setFormatter(formatter) 46 | log.addHandler(ch) 47 | return log 48 | 49 | 50 | def execute(command, stdout=None, stderr=None): 51 | stdout = stdout or sys.stdout 52 | stderr = stderr or sys.stderr 53 | return subprocess.check_call(shlex.split(command), stdout=stdout, 54 | stderr=stderr) 55 | 56 | 57 | @contextlib.contextmanager 58 | def cd(path): 59 | old_dir = os.getcwd() 60 | os.chdir(path) 61 | try: 62 | yield 63 | finally: 64 | os.chdir(old_dir) -------------------------------------------------------------------------------- /python/requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2017.4.17 2 | chardet==3.0.4 3 | click==6.7 4 | docker==2.4.2 5 | docker-pycreds==0.2.1 6 | idna==2.5 7 | requests>=2.20.0 8 | six==1.10.0 9 | urllib3==1.26.5 10 | websocket-client==0.44.0 11 | contexttimer==0.3.3 12 | PyYAML==5.4 13 | Jinja2==2.11.3 14 | cookiecutter==1.6.0 15 | -------------------------------------------------------------------------------- /python/requirements_dev.txt: -------------------------------------------------------------------------------- 1 | pip==19.2 2 | bumpversion==0.5.3 3 | wheel==0.29.0 4 | watchdog==0.8.3 5 | flake8==2.6.0 6 | tox==2.3.1 7 | coverage==4.1 8 | Sphinx==1.4.8 9 | 10 | pytest==2.9.2 11 | pytest-runner==2.11.1 12 | -------------------------------------------------------------------------------- /python/setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.4.2 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | search = version='{current_version}' 8 | replace = version='{new_version}' 9 | 10 | [bumpversion:file:nvidia_deepops/__init__.py] 11 | search = __version__ = '{current_version}' 12 | replace = __version__ = '{new_version}' 13 | 14 | [bumpversion:file:Makefile] 15 | 16 | [bdist_wheel] 17 | universal = 1 18 | 19 | [flake8] 20 | exclude = docs 21 | 22 | [aliases] 23 | test = pytest 24 | 25 | [metadata] 26 | license_file = LICENSE 27 | -------------------------------------------------------------------------------- /python/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions 8 | # are met: 9 | # * Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # * Redistributions in binary form must reproduce the above copyright 12 | # notice, this list of conditions and the following disclaimer in the 13 | # documentation and/or other materials provided with the distribution. 14 | # * Neither the name of NVIDIA CORPORATION nor the names of its 15 | # contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 19 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 22 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 23 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 24 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 25 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 26 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | """The setup script.""" 31 | 32 | from setuptools import setup, find_packages 33 | 34 | requirements = [ 35 | 'Click>=6.0', 36 | 'docker', 37 | 'contexttimer', 38 | # TODO: put package requirements here 39 | ] 40 | 41 | setup_requirements = [ 42 | 'pytest-runner', 43 | # TODO(ryanolson): put setup requirements (distutils extensions, etc.) here 44 | ] 45 | 46 | test_requirements = [ 47 | 'pytest', 48 | # TODO: put package test requirements here 49 | ] 50 | 51 | setup( 52 | name='nvidia_deepops', 53 | version='0.4.2', 54 | description="Core Python library for DeepOps services", 55 | author="Ryan Olson", 56 | author_email='rolson@nvidia.com', 57 | # url='https://github.com/ryanolson/nvidia_deepops', 58 | packages=find_packages(exclude=['tests']), 59 | entry_points={ 60 | 'console_scripts': [ 61 | 'deepops-progress=nvidia_deepops.cli:progress_cli' 62 | ] 63 | }, 64 | include_package_data=True, 65 | install_requires=requirements, 66 | zip_safe=False, 67 | keywords='nvidia_deepops', 68 | classifiers=[ 69 | 'Development Status :: 2 - Pre-Alpha', 70 | 'Intended Audience :: Developers', 71 | 'Natural Language :: English', 72 | # "Programming Language :: Python :: 2", 73 | # 'Programming Language :: Python :: 2.7', 74 | 'Programming Language :: Python :: 3', 75 | # 'Programming Language :: Python :: 3.3', 76 | # 'Programming Language :: Python :: 3.4', 77 | 'Programming Language :: Python :: 3.5', 78 | 'Programming Language :: Python :: 3.6', 79 | ], 80 | test_suite='tests', 81 | tests_require=test_requirements, 82 | setup_requires=setup_requirements, 83 | ) 84 | -------------------------------------------------------------------------------- /python/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # * Neither the name of NVIDIA CORPORATION nor the names of its 14 | # contributors may be used to endorse or promote products derived 15 | # from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 18 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 25 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | """Unit test package for nvidia_deepops.""" 30 | -------------------------------------------------------------------------------- /python/tests/test_nvidia_deepops.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions 8 | # are met: 9 | # * Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # * Redistributions in binary form must reproduce the above copyright 12 | # notice, this list of conditions and the following disclaimer in the 13 | # documentation and/or other materials provided with the distribution. 14 | # * Neither the name of NVIDIA CORPORATION nor the names of its 15 | # contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 19 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 22 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 23 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 24 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 25 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 26 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | import collections 31 | import logging 32 | import os 33 | # import pprint 34 | 35 | import pytest 36 | 37 | import traceback 38 | 39 | from click.testing import CliRunner 40 | from docker.errors import APIError 41 | 42 | from nvidia_deepops import utils 43 | # from nvidia_deepops import cli 44 | from nvidia_deepops.docker import (BaseClient, DockerClient, registry) 45 | 46 | 47 | BaseRegistry = registry.BaseRegistry 48 | # DockerRegistry = registry.DockerRegistry 49 | DGXRegistry = registry.DGXRegistry 50 | NGCRegistry = registry.NGCRegistry 51 | 52 | 53 | dev = utils.get_logger(__name__, level=logging.DEBUG) 54 | 55 | try: 56 | from .secrets import ngcpassword, dgxpassword 57 | HAS_SECRETS = True 58 | except Exception: 59 | HAS_SECRETS = False 60 | 61 | secrets = pytest.mark.skipif(not HAS_SECRETS, reason="No secrets.py file found") 62 | 63 | @pytest.fixture 64 | def response(): 65 | """Sample pytest fixture. 66 | 67 | See more at: http://doc.pytest.org/en/latest/fixture.html 68 | """ 69 | # import requests 70 | # return requests.get('https://github.com/audreyr/cookiecutter-pypackage') 71 | 72 | 73 | def test_content(response): 74 | """Sample pytest test function with the pytest fixture as an argument.""" 75 | # from bs4 import BeautifulSoup 76 | # assert 'GitHub' in BeautifulSoup(response.content).title.string 77 | 78 | 79 | def test_command_line_interface(): 80 | """Test the CLI.""" 81 | runner = CliRunner() 82 | # result = runner.invoke(cli.main) 83 | # assert result.exit_code == 0 84 | # assert 'cloner.cli.main' in result.output 85 | # help_result = runner.invoke(cli.main, ['--help']) 86 | # assert help_result.exit_code == 0 87 | # assert '--help Show this message and exit.' in help_result.output 88 | 89 | 90 | class FakeClient(BaseClient): 91 | 92 | def __init__(self, registries=None, images=None): 93 | self.registries = registries or [] 94 | self.images = images or [] 95 | 96 | def registry_for_url(self, url): 97 | for reg in self.registries: 98 | if url.startswith(reg.url): 99 | return reg 100 | raise RuntimeError("registry not found for %s" % url) 101 | 102 | def url_to_name_and_tag(self, url, reg=None): 103 | dev.debug("url: %s" % url) 104 | reg = reg or self.registry_for_url(url) 105 | return reg.url_to_name_and_tag(url) 106 | 107 | def should_be_present(self, url): 108 | if url not in self.images: 109 | dev.debug(self.images) 110 | raise ValueError("client does not have an image named %s" % url) 111 | 112 | def pull(self, url): 113 | reg = self.registry_for_url(url) 114 | name, tag = self.url_to_name_and_tag(url, reg=reg) 115 | reg.should_be_present(name, tag=tag) 116 | self.images.append(url) 117 | self.should_be_present(url) 118 | 119 | def tag(self, src, dst): 120 | self.should_be_present(src) 121 | self.images.append(dst) 122 | 123 | def push(self, url): 124 | self.should_be_present(url) 125 | reg = self.registry_for_url(url) 126 | dev.debug("push %s" % url) 127 | name, tag = self.url_to_name_and_tag(url, reg=reg) 128 | reg.images[name].append(tag) 129 | 130 | def remove(self, url): 131 | self.should_be_present(url) 132 | self.images.remove(url) 133 | 134 | 135 | class FakeRegistry(BaseRegistry): 136 | 137 | def __init__(self, url, images=None): 138 | self.url = url 139 | self.images = collections.defaultdict(list) 140 | images = images or {} 141 | for name, tags in images.items(): 142 | self.images[name] = tags 143 | 144 | def docker_url(self, name, tag="latest"): 145 | return "{}/{}:{}".format(self.url, name, tag) 146 | 147 | def url_to_name_and_tag(self, url): 148 | name_tag = url.replace(self.url + "/", "").split(":") 149 | dev.debug("name_tag: %s" % name_tag) 150 | if len(name_tag) == 1: 151 | return name_tag, "latest" 152 | elif len(name_tag) == 2: 153 | return name_tag 154 | else: 155 | raise RuntimeError("bad name_tag") 156 | 157 | def should_be_present(self, url_or_name, tag=None): 158 | if url_or_name.startswith(self.url) and tag is None: 159 | name, tag = self.url_to_name_and_tag(url_or_name) 160 | else: 161 | name, tag = url_or_name, tag or "latest" 162 | if tag not in self.images[name]: 163 | dev.debug(self.images) 164 | raise ValueError("%s not found for %s" % (tag, name)) 165 | 166 | def get_image_tags(self, image_name): 167 | return self.images[image_name] 168 | 169 | def get_image_names(self, project=None): 170 | def predicate(name): 171 | if project: 172 | return name.startswith(project + "/") 173 | return True 174 | return [name for name in self.images.keys() if predicate(name)] 175 | 176 | def get_state(self, project=None, filter_fn=None): 177 | image_names = self.get_image_names(project=project) 178 | state = collections.defaultdict(dict) 179 | for name in image_names: 180 | for tag in self.images[name]: 181 | if filter_fn is not None and callable(filter_fn): 182 | if not filter_fn(name=name, tag=tag, docker_id=tag): 183 | continue 184 | state[name][tag] = tag 185 | return state 186 | 187 | 188 | def test_fakeregistry_docker_url(): 189 | fqdn = FakeRegistry("nvcr.io") 190 | assert fqdn.docker_url("nvidia/pytorch") == "nvcr.io/nvidia/pytorch:latest" 191 | assert fqdn.docker_url("nvidia/pytorch", "17.05") == \ 192 | "nvcr.io/nvidia/pytorch:17.05" 193 | 194 | fqdn = FakeRegistry("nvcr.io:5000") 195 | assert fqdn.docker_url("nvidia/pytorch") == \ 196 | "nvcr.io:5000/nvidia/pytorch:latest" 197 | assert fqdn.docker_url("nvidia/pytorch", "17.05") == \ 198 | "nvcr.io:5000/nvidia/pytorch:17.05" 199 | 200 | 201 | @pytest.fixture 202 | def nvcr(): 203 | return FakeRegistry("nvcr.io", images={ 204 | "nvidia/tensorflow": ["17.07", "17.06"], 205 | "nvidia/pytorch": ["17.07", "17.05"], 206 | "nvidia/cuda": ["8.0-devel", "9.0-devel"], 207 | "nvidian_sas/dgxbench": ["16.08"], 208 | "nvidian_sas/dgxdash": ["latest"], 209 | }) 210 | 211 | 212 | @pytest.fixture 213 | def locr(): 214 | return FakeRegistry("registry:5000", images={ 215 | "nvidia/pytorch": ["17.06", "17.05"], 216 | "nvidia/cuda": ["8.0-devel"], 217 | }) 218 | 219 | 220 | def test_get_state(nvcr): 221 | state = nvcr.get_state(project="nvidia") 222 | assert len(state.keys()) == 3 223 | assert len(state["nvidia/tensorflow"].keys()) == 2 224 | assert state["nvidia/cuda"]["9.0-devel"] == "9.0-devel" 225 | 226 | 227 | def test_get_state_filter(nvcr): 228 | def filter_on_tag(*, name, tag, docker_id): 229 | try: 230 | val = float(tag) 231 | except Exception: 232 | traceback.print_exc() 233 | return True 234 | return val >= 17.06 235 | 236 | state = nvcr.get_state(project="nvidia", filter_fn=filter_on_tag) 237 | assert len(state.keys()) == 3 238 | assert len(state["nvidia/tensorflow"].keys()) == 2 239 | assert len(state["nvidia/pytorch"].keys()) == 1 240 | 241 | 242 | def test_client_lifecycle(nvcr, locr): 243 | client = FakeClient(registries=[nvcr, locr]) 244 | # we should see an exception when the image is not in the registry 245 | with pytest.raises(Exception): 246 | client.pull(nvcr.docker_url("nvidia/pytorch", tag="17.06")) 247 | src = nvcr.docker_url("nvidia/pytorch", tag="17.07") 248 | dst = locr.docker_url("nvidia/pytorch", tag="17.07") 249 | with pytest.raises(Exception): 250 | locr.should_be_present(dst) 251 | client.pull(src) 252 | client.should_be_present(src) 253 | client.tag(src, dst) 254 | client.push(dst) 255 | locr.should_be_present(dst) 256 | client.remove(src) 257 | with pytest.raises(Exception): 258 | client.should_be_present(src) 259 | # client.delete_remote(src) 260 | # with pytest.raises(Exception): 261 | # nvcr.should_be_present(src) 262 | 263 | 264 | def test_pull_nonexistent_image(nvcr): 265 | client = FakeClient(registries=[nvcr]) 266 | with pytest.raises(Exception): 267 | client.pull(nvcr.docker_url("nvidia/tensorflow", "latest")) 268 | 269 | 270 | def test_push_nonexistent_image(locr): 271 | client = FakeClient(registries=[locr]) 272 | with pytest.raises(Exception): 273 | client.push(locr.docker_url("ryan/awesome")) 274 | 275 | 276 | def docker_client_pull_and_remove(client, url): 277 | client.pull(url) 278 | image = client.get(url=url) 279 | assert image is not None 280 | client.remove(url) 281 | with pytest.raises(APIError): 282 | client.client.images.get(url) 283 | assert client.get(url=url) is None 284 | 285 | 286 | @pytest.mark.remote 287 | @pytest.mark.dockerclient 288 | @pytest.mark.parametrize("image_name", [ 289 | "busybox:latest", 290 | "ubuntu:16.04", 291 | ]) 292 | def test_pull_and_remove_from_docker_hub(image_name): 293 | client = DockerClient() 294 | docker_client_pull_and_remove(client, image_name) 295 | 296 | 297 | @secrets 298 | @pytest.mark.nvcr 299 | @pytest.mark.remote 300 | @pytest.mark.dockerclient 301 | @pytest.mark.parametrize("image_name", [ 302 | "nvcr.io/nvsa_clone/busybox:latest", 303 | "nvcr.io/nvsa_clone/ubuntu:16.04", 304 | ]) 305 | def test_pull_and_remove_from_nvcr(image_name): 306 | client = DockerClient() 307 | client.login( 308 | username="$oauthtoken", 309 | password=dgxpassword, 310 | registry="nvcr.io/v2") 311 | docker_client_pull_and_remove(client, image_name) 312 | 313 | 314 | @secrets 315 | @pytest.mark.remote 316 | @pytest.mark.dockerregistry 317 | def test_get_state_dgx(): 318 | dgx_registry = DGXRegistry(dgxpassword) 319 | state = dgx_registry.get_state(project="nvidia") 320 | dev.debug(state) 321 | assert state["nvidia/cuda"]["8.0-cudnn5.1-devel-ubuntu14.04"] == \ 322 | "c61f351b591fbfca93b3c0fcc3bd0397e7f3c6c2c2f1880ded2fdc1e5f9edd9e" 323 | 324 | 325 | @secrets 326 | @pytest.mark.remote 327 | @pytest.mark.dockerregistry 328 | def test_get_state_ngc(): 329 | ngc_registry = NGCRegistry(ngcpassword) 330 | state = ngc_registry.get_state(project="nvidia") 331 | dev.debug(state) 332 | assert "9.0-cudnn7-devel-ubuntu16.04" in state["nvidia/cuda"] 333 | 334 | 335 | @secrets 336 | @pytest.mark.nvcr 337 | @pytest.mark.remote 338 | @pytest.mark.dockerregistry 339 | def test_dgx_registry_list(): 340 | dgx_registry = DGXRegistry(dgxpassword) 341 | images_and_tags = dgx_registry.get_images_and_tags(project="nvsa_clone") 342 | dev.debug(images_and_tags) 343 | assert "nvsa_clone/busybox" in images_and_tags 344 | assert "nvsa_clone/ubuntu" in images_and_tags 345 | assert "latest" in images_and_tags["nvsa_clone/busybox"] 346 | assert "16.04" in images_and_tags["nvsa_clone/ubuntu"] 347 | 348 | 349 | @secrets 350 | @pytest.mark.nvcr 351 | @pytest.mark.remote 352 | @pytest.mark.dockerregistry 353 | def test_ngc_registry_list(): 354 | ngc_registry = NGCRegistry(ngcpassword) 355 | images_and_tags = ngc_registry.get_images_and_tags(project="nvidia") 356 | dev.debug(images_and_tags) 357 | images = ["nvidia/tensorflow", "nvidia/pytorch", 358 | "nvidia/mxnet", "nvidia/tensorrt"] 359 | for image in images: 360 | assert image in images_and_tags 361 | assert "17.12" in images_and_tags[image] 362 | 363 | 364 | @secrets 365 | @pytest.mark.nvcr 366 | @pytest.mark.remote 367 | def test_dgx_markdowns(): 368 | dgx_registry = DGXRegistry(dgxpassword) 369 | markdowns = dgx_registry.get_image_descriptions(project="nvidia") 370 | dev.debug(markdowns) 371 | assert "nvidia/cuda" in markdowns 372 | 373 | 374 | @secrets 375 | @pytest.mark.nvcr 376 | @pytest.mark.remote 377 | def test_ngc_markdowns(): 378 | ngc_registry = NGCRegistry(ngcpassword) 379 | markdowns = ngc_registry.get_images_and_tags(project="nvidia") 380 | dev.debug(markdowns) 381 | images = ["nvidia/tensorflow", "nvidia/pytorch", 382 | "nvidia/mxnet", "nvidia/tensorrt"] 383 | for image in images: 384 | assert image in markdowns 385 | 386 | 387 | @pytest.mark.new 388 | @pytest.mark.remote 389 | @pytest.mark.parametrize("url", [ 390 | "busybox:latest", 391 | ]) 392 | def test_pull_and_save_and_remove(url): 393 | client = DockerClient() 394 | client.pull(url) 395 | filename = client.save(url) 396 | assert os.path.exists(filename) 397 | client.remove(url) 398 | assert client.get(url=url) is None 399 | read_url = client.load(filename) 400 | assert read_url == url 401 | assert client.get(url=url) is not None 402 | client.remove(url) 403 | os.unlink(filename) 404 | assert not os.path.exists(filename) 405 | -------------------------------------------------------------------------------- /replicator/.gitignore: -------------------------------------------------------------------------------- 1 | tests/secrets.py 2 | -------------------------------------------------------------------------------- /replicator/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE=deepops_python 2 | FROM $BASE_IMAGE AS singularity 3 | 4 | ARG APT_PROXY=false 5 | RUN echo "Acquire::HTTP::Proxy \"$APT_PROXY\";" >> /etc/apt/apt.conf.d/01proxy 6 | 7 | # Singularity build requirements 8 | RUN apt-get update -y && \ 9 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 10 | build-essential \ 11 | libssl-dev \ 12 | uuid-dev \ 13 | libgpgme11-dev \ 14 | wget \ 15 | git \ 16 | ca-certificates \ 17 | squashfs-tools \ 18 | gcc \ 19 | tzdata && \ 20 | rm -rf /var/lib/apt/lists/* 21 | 22 | # Go 23 | RUN cd /var/tmp && \ 24 | mkdir -p /tmp && wget -q -nc --no-check-certificate -P /tmp https://dl.google.com/go/go1.13.linux-amd64.tar.gz && \ 25 | mkdir -p /usr/local && tar -x -f /tmp/go1.13.linux-amd64.tar.gz -C /usr/local -z && \ 26 | rm -rf /tmp/go1.13.linux-amd64.tar.gz 27 | ENV GOPATH=/root/go \ 28 | PATH=/usr/local/go/bin:$PATH:/root/go/bin 29 | 30 | # Singularity 31 | RUN mkdir -p /tmp && wget -q -nc --no-check-certificate -P /tmp https://github.com/sylabs/singularity/releases/download/v3.7.4/singularity-3.7.4.tar.gz && \ 32 | mkdir -p $GOPATH/src/github.com/sylabs && tar -x -f /tmp/singularity-3.7.4.tar.gz -C $GOPATH/src/github.com/sylabs -z && \ 33 | cd $GOPATH/src/github.com/sylabs/singularity && \ 34 | go env -w GO111MODULE=off && \ 35 | go get -u -v github.com/golang/dep/cmd/dep && \ 36 | cd $GOPATH/src/github.com/sylabs/singularity && \ 37 | ./mconfig --prefix=/usr/local/singularity && \ 38 | cd builddir && \ 39 | make && \ 40 | make install && \ 41 | rm -rf /tmp/singularity-3.7.4.tar.gz 42 | 43 | FROM $BASE_IMAGE 44 | 45 | ARG APT_PROXY=false 46 | RUN echo "Acquire::HTTP::Proxy \"$APT_PROXY\";" >> /etc/apt/apt.conf.d/01proxy 47 | 48 | # Singularity 49 | RUN apt-get update -y && \ 50 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 51 | squashfs-tools && \ 52 | rm -rf /var/lib/apt/lists/* 53 | COPY --from=singularity /usr/local/singularity /usr/local/singularity 54 | ENV PATH=/usr/local/singularity/bin:$PATH 55 | 56 | COPY requirements.txt /tmp/requirements.txt 57 | RUN pip install -r /tmp/requirements.txt && rm -f /tmp/requirements.txt 58 | 59 | WORKDIR /source/ngc_replicator 60 | COPY . . 61 | RUN mkdir -p /output 62 | RUN python setup.py install 63 | ENTRYPOINT ["ngc_replicator"] 64 | 65 | RUN apt-get update && apt-get install -y --no-install-recommends \ 66 | jq && \ 67 | rm -rf /var/lib/apt/lists/* 68 | 69 | COPY scripts/docker-utils /usr/bin/docker-utils 70 | 71 | RUN rm -f /etc/apt/apt.conf.d/01proxy 72 | -------------------------------------------------------------------------------- /replicator/Dockerfile.test: -------------------------------------------------------------------------------- 1 | FROM replicator 2 | 3 | RUN pip install pytest 4 | ENTRYPOINT ["py.test"] 5 | -------------------------------------------------------------------------------- /replicator/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CONTRIBUTING.rst 2 | include HISTORY.rst 3 | include LICENSE 4 | include README.rst 5 | 6 | recursive-include tests * 7 | recursive-exclude * __pycache__ 8 | recursive-exclude * *.py[co] 9 | 10 | recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif 11 | -------------------------------------------------------------------------------- /replicator/Makefile: -------------------------------------------------------------------------------- 1 | BASE_IMAGE ?= deepops_python 2 | IMAGE_NAME ?= replicator 3 | RELEASE_VERSION=0.4.0 4 | RELEASE_IMAGE ?= deepops/replicator 5 | 6 | APT_PROXY ?= "false" 7 | 8 | .PHONY: build tag push release clean distclean 9 | 10 | default: build 11 | 12 | build: 13 | docker build --build-arg BASE_IMAGE=${BASE_IMAGE} --build-arg APT_PROXY=${APT_PROXY} -t ${IMAGE_NAME} -f Dockerfile . 14 | 15 | dev: build 16 | docker build ${CACHES} -t ${IMAGE_NAME}:dev -f Dockerfile.test . 17 | docker run --rm -ti -v ${PWD}:/devel -v /var/run/docker.sock:/var/run/docker.sock --net=host \ 18 | --entrypoint=/devel/entrypoint-devel.sh --workdir=/devel ${IMAGE_NAME}:dev sh 19 | 20 | test: build 21 | docker build ${CACHES} -t ${IMAGE_NAME}:dev -f Dockerfile.test . 22 | docker run --rm -ti -v /var/run/docker.sock:/var/run/docker.sock ${IMAGE_NAME}:dev 23 | 24 | tag: build 25 | docker tag ${IMAGE_NAME} ${RELEASE_IMAGE}:${RELEASE_VERSION} 26 | docker tag ${IMAGE_NAME} ${RELEASE_IMAGE}:latest 27 | 28 | push: tag 29 | docker push ${RELEASE_IMAGE}:${RELEASE_VERSION} 30 | docker push ${RELEASE_IMAGE}:latest 31 | 32 | release: push 33 | make -f Makefile.pypi dist 34 | 35 | clean: 36 | @rm -f .Dockerfile 2> /dev/null ||: 37 | @rm -f ./tests/__pycache__ 2> /dev/null ||: 38 | @docker rm -v `docker ps -a -q -f "status=exited"` 2> /dev/null ||: 39 | @docker rmi `docker images -q -f "dangling=true"` 2> /dev/null ||: 40 | 41 | distclean: clean 42 | @docker rmi ${BASE_IMAGE} 2> /dev/null ||: 43 | @docker rmi ${IMAGE_NAME} 2> /dev/null ||: 44 | @docker rmi ${RELEASE_IMAGE} 2> /dev/null ||: 45 | -------------------------------------------------------------------------------- /replicator/Makefile.pypi: -------------------------------------------------------------------------------- 1 | .PHONY: clean clean-test clean-pyc clean-build docs help 2 | .DEFAULT_GOAL := help 3 | define BROWSER_PYSCRIPT 4 | import os, webbrowser, sys 5 | try: 6 | from urllib import pathname2url 7 | except: 8 | from urllib.request import pathname2url 9 | 10 | webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) 11 | endef 12 | export BROWSER_PYSCRIPT 13 | 14 | define PRINT_HELP_PYSCRIPT 15 | import re, sys 16 | 17 | for line in sys.stdin: 18 | match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) 19 | if match: 20 | target, help = match.groups() 21 | print("%-20s %s" % (target, help)) 22 | endef 23 | export PRINT_HELP_PYSCRIPT 24 | BROWSER := python3 -c "$$BROWSER_PYSCRIPT" 25 | 26 | help: 27 | @python3 -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) 28 | 29 | clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts 30 | 31 | 32 | clean-build: ## remove build artifacts 33 | rm -fr build/ 34 | rm -fr dist/ 35 | rm -fr .eggs/ 36 | find . -name '*.egg-info' -exec rm -fr {} + 37 | find . -name '*.egg' -exec rm -f {} + 38 | 39 | clean-pyc: ## remove Python file artifacts 40 | find . -name '*.pyc' -exec rm -f {} + 41 | find . -name '*.pyo' -exec rm -f {} + 42 | find . -name '*~' -exec rm -f {} + 43 | find . -name '__pycache__' -exec rm -fr {} + 44 | 45 | clean-test: ## remove test and coverage artifacts 46 | rm -fr .tox/ 47 | rm -f .coverage 48 | rm -fr htmlcov/ 49 | 50 | lint: ## check style with flake8 51 | flake8 ngc_replicator tests 52 | 53 | test: ## run tests quickly with the default Python 54 | py.test 55 | 56 | 57 | test-all: ## run tests on every Python version with tox 58 | tox 59 | 60 | coverage: ## check code coverage quickly with the default Python 61 | coverage run --source ngc_replicator -m pytest 62 | coverage report -m 63 | coverage html 64 | $(BROWSER) htmlcov/index.html 65 | 66 | docs: ## generate Sphinx HTML documentation, including API docs 67 | rm -f docs/ngc_replicator.rst 68 | rm -f docs/modules.rst 69 | sphinx-apidoc -o docs/ ngc_replicator 70 | $(MAKE) -C docs clean 71 | $(MAKE) -C docs html 72 | $(BROWSER) docs/_build/html/index.html 73 | 74 | servedocs: docs ## compile the docs watching for changes 75 | watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . 76 | 77 | #release: clean ## package and upload a release 78 | # python3 setup.py sdist upload 79 | # python3 setup.py bdist_wheel upload 80 | 81 | dist: clean ## builds source and wheel package 82 | python3 setup.py sdist 83 | python3 setup.py bdist_wheel 84 | ls -l dist 85 | 86 | install: clean ## install the package to the active Python's site-packages 87 | python3 setup.py install 88 | -------------------------------------------------------------------------------- /replicator/README.md: -------------------------------------------------------------------------------- 1 | # NGC Replicator 2 | 3 | Clones nvcr.io using the either DGX (compute.nvidia.com) or NGC (ngc.nvidia.com) 4 | API keys. 5 | 6 | The replicator will make an offline clone of the NGC/DGX container registry. 7 | In its current form, the replicator will download every CUDA container image as 8 | well as each Deep Learning framework image in the NVIDIA project. 9 | 10 | Tarfiles will be saved in `/output` inside the container, so be sure to volume 11 | mount that directory. In the following example, we will collect our images in 12 | `/tmp` on the host. 13 | 14 | Use `--min-version` to limit the number of versions to download. In the example 15 | below, we will only clone versions `17.10` and later DL framework images. 16 | 17 | ``` 18 | docker run --rm -it -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/output \ 19 | deepops/replicator --project=nvidia --min-version=17.12 \ 20 | --api-key= 21 | ``` 22 | 23 | You can also filter on specific images. If you only wanted Tensorflow, PyTorch 24 | and TensorRT, you would simply add `--image` for each option, e.g. 25 | 26 | ``` 27 | docker run --rm -it -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/output \ 28 | deepops/replicator --project=nvidia --min-version=17.12 \ 29 | --image=tensorflow --image=pytorch --image=tensorrt \ 30 | --dry-run \ 31 | --api-key= 32 | ``` 33 | 34 | Note: the `--dry-run` option lets you see what will happen without committing 35 | to a lengthy download. 36 | 37 | Use `--singularity` to generate Singularity image files, e.g., 38 | 39 | ``` 40 | docker run --rm -it -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/output \ 41 | deepops/replicator --project=hpc --image=milc --image=namd \ 42 | --singularity --no-exporter \ 43 | --api-key= 44 | ``` 45 | 46 | Note: a `state.yml` file will be created the output directory. This saved state will be used to 47 | avoid pulling images that were previously pulled. If you wish to repull and save an image, just 48 | delete the entry in `state.yml` corresponding to the `image_name` and `tag` you wish to refresh. 49 | 50 | ## Kubernetes Deployment 51 | 52 | If you don't already have a `deepops` namespace, create one now. 53 | 54 | ``` 55 | kubectl create namespace deepops 56 | ``` 57 | 58 | Next, create a secret with your NGC API Key 59 | 60 | ``` 61 | kubectl -n deepops create secret generic ngc-secret 62 | --from-literal=apikey= 63 | ``` 64 | 65 | Next, create a persistent volume claim that will life outside the lifecycle of the CronJob. If 66 | you are using [DeepOps](https://github.com/nvidia/deepops) you can use a Rook/Ceph PVC similar 67 | to: 68 | 69 | ``` 70 | --- 71 | apiVersion: v1 72 | kind: PersistentVolumeClaim 73 | metadata: 74 | name: ngc-replicator-pvc 75 | namespace: deepops 76 | labels: 77 | app: ngc-replicator 78 | spec: 79 | storageClassName: rook-raid0-retain # <== Replace with your StorageClass 80 | accessModes: 81 | - ReadWriteOnce 82 | resources: 83 | requests: 84 | storage: 32Mi 85 | ``` 86 | 87 | Finally, create a `CronJob` that executes the replicator on a schedule. This 88 | eample run the replicator every hour. Note: This example used 89 | [Rook](https://rook.io) block storage to provide a persistent volume to hold the 90 | `state.yml` between executions. This ensures you will only download new 91 | container images. For more details, see our [DeepOps 92 | project](https://github.com/nvidia/deepops). 93 | 94 | ``` 95 | --- 96 | apiVersion: v1 97 | kind: ConfigMap 98 | metadata: 99 | name: replicator-config 100 | namespace: deepops 101 | data: 102 | ngc-update.sh: | 103 | #!/bin/bash 104 | ngc_replicator \ 105 | --project=nvidia \ 106 | --min-version=$(date +"%y.%m" -d "1 month ago") \ 107 | --py-version=py3 \ 108 | --image=tensorflow --image=pytorch --image=tensorrt \ 109 | --no-exporter \ 110 | --registry-url=registry.local # <== Replace with your local repo 111 | --- 112 | apiVersion: batch/v1beta1 113 | kind: CronJob 114 | metadata: 115 | name: ngc-replicator 116 | namespace: deepops 117 | labels: 118 | app: ngc-replicator 119 | spec: 120 | schedule: "0 4 * * *" 121 | jobTemplate: 122 | spec: 123 | template: 124 | spec: 125 | nodeSelector: 126 | node-role.kubernetes.io/master: "" 127 | containers: 128 | - name: replicator 129 | image: deepops/replicator 130 | imagePullPolicy: Always 131 | command: [ "/bin/sh", "-c", "/ngc-update/ngc-update.sh" ] 132 | env: 133 | - name: NGC_REPLICATOR_API_KEY 134 | valueFrom: 135 | secretKeyRef: 136 | name: ngc-secret 137 | key: apikey 138 | volumeMounts: 139 | - name: registry-config 140 | mountPath: /ngc-update 141 | - name: docker-socket 142 | mountPath: /var/run/docker.sock 143 | - name: ngc-replicator-storage 144 | mountPath: /output 145 | volumes: 146 | - name: registry-config 147 | configMap: 148 | name: replicator-config 149 | defaultMode: 0777 150 | - name: docker-socket 151 | hostPath: 152 | path: /var/run/docker.sock 153 | type: File 154 | - name: ngc-replicator-storage 155 | persistentVolumeClaim: 156 | claimName: ngc-replicator-pvc 157 | restartPolicy: Never 158 | ``` 159 | 160 | ## Developer Quickstart 161 | 162 | ``` 163 | make dev 164 | py.test 165 | ``` 166 | 167 | ## TODOs 168 | 169 | - [x] save markdown readmes for each image. these are not version controlled 170 | - [x] test local registry push service. coded, beta testing 171 | - [ ] add templater to workflow 172 | -------------------------------------------------------------------------------- /replicator/entrypoint-devel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | cat <<'EOF' 4 | _ _ _ 5 | | (_) | | 6 | _ __ __ _ ___ _ __ ___ _ __ | |_ ___ __ _| |_ ___ _ __ 7 | | '_ \ / _` |/ __| | '__/ _ \ '_ \| | |/ __/ _` | __/ _ \| '__| 8 | | | | | (_| | (__ | | | __/ |_) | | | (_| (_| | || (_) | | 9 | |_| |_|\__, |\___| |_| \___| .__/|_|_|\___\__,_|\__\___/|_| 10 | __/ | ______ | | 11 | |___/ |______| |_| 12 | 13 | Starting developer environment ... 14 | EOF 15 | 16 | pip install -e . 17 | 18 | if [ $# -eq 0 ]; then 19 | exec "/bin/bash" 20 | else 21 | exec "$@" 22 | fi 23 | -------------------------------------------------------------------------------- /replicator/ngc_replicator/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Top-level package for NGC Replicator.""" 4 | 5 | __author__ = """Ryan Olson""" 6 | __email__ = 'rolson@nvidia.com' 7 | __version__ = '0.4.0' 8 | -------------------------------------------------------------------------------- /replicator/ngc_replicator/ngc_replicator.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import collections 3 | import json 4 | import logging 5 | import os 6 | import pprint 7 | import re 8 | import time 9 | 10 | from concurrent import futures 11 | 12 | import click 13 | #import grpc 14 | import yaml 15 | 16 | from nvidia_deepops import Progress, utils 17 | from nvidia_deepops.docker import DockerClient, NGCRegistry, DGXRegistry 18 | 19 | from . import replicator_pb2 20 | #from . import replicator_pb2_grpc 21 | 22 | log = utils.get_logger(__name__, level=logging.INFO) 23 | 24 | _ONE_DAY_IN_SECONDS = 60 * 60 * 24 25 | 26 | 27 | class Replicator: 28 | 29 | def __init__(self, *, api_key, project, **optional_config): 30 | log.info("Initializing Replicator") 31 | self._config = optional_config 32 | self.project = project 33 | self.service = self.config("service") 34 | if len(api_key) == 40: 35 | self.nvcr = DGXRegistry(api_key) 36 | else: 37 | self.nvcr = NGCRegistry(api_key) 38 | self.nvcr_client = DockerClient() 39 | self.nvcr_client.login(username="$oauthtoken", password=api_key, registry="nvcr.io/v2") 40 | self.registry_client = None 41 | self.min_version = self.config("min_version") 42 | self.py_version = self.config("py_version") 43 | self.images = self.config("image") or [] 44 | self.progress = Progress(uri=self.config("progress_uri")) 45 | if self.config("registry_url"): 46 | self.registry_url = self.config("registry_url") 47 | self.registry_client = DockerClient() 48 | if self.config("registry_username") and self.config("registry_password"): 49 | self.registry_client.login(username=self.config("registry_username"), 50 | password=self.config("registry_password"), 51 | registry=self.config("registry_url")) 52 | self.output_path = self.config("output_path") or "/output" 53 | self.state_path = os.path.join(self.output_path, "state.yml") 54 | self.state = collections.defaultdict(dict) 55 | if os.path.exists(self.state_path): 56 | with open(self.state_path, "r") as file: 57 | tmp = yaml.load(file, Loader=yaml.UnsafeLoader) 58 | if tmp: 59 | for key, val in tmp.items(): 60 | self.state[key] = val 61 | self.export_to_tarfile = self.config("exporter") 62 | self.third_party_images = [] 63 | if self.config("external_images"): 64 | self.third_party_images.extend(self.read_external_images_file()) 65 | if self.export_to_tarfile: 66 | log.info("tarfiles will be saved to {}".format(self.output_path)) 67 | self.export_to_singularity = self.config("singularity") 68 | if self.export_to_singularity: 69 | log.info("singularity images will be saved to {}".format(self.output_path)) 70 | log.info("Replicator initialization complete") 71 | 72 | def read_external_images_file(self): 73 | with open(self.config("external_images"), "r") as file: 74 | data = yaml.load(file, Loader=yaml.UnsafeLoader) 75 | images = data.get("images", []) 76 | images = [replicator_pb2.DockerImage(name=image["name"], tag=image.get("tag", "latest")) for image in images] 77 | return images 78 | 79 | def config(self, key, default=None): 80 | return self._config.get(key, default) 81 | 82 | def save_state(self): 83 | with open(self.state_path, "w") as file: 84 | yaml.dump(self.state, file) 85 | 86 | def sync(self, project=None): 87 | log.info("Replicator Started") 88 | 89 | # pull images 90 | new_images = {image.name: image.tag for image in self.sync_images(project=project)} 91 | 92 | # pull image descriptions - new_images should be empty for dry runs 93 | self.progress.update_step(key="markdown", status="running") 94 | self.update_progress() 95 | descriptions = self.nvcr.get_image_descriptions(project=project) 96 | for image_name, _ in new_images.items(): 97 | markdown = os.path.join(self.output_path, "description_{}.md".format(image_name.replace('/', '%%'))) 98 | with open(markdown, "w") as out: 99 | out.write(descriptions.get(image_name, "")) 100 | self.progress.update_step(key="markdown", status="complete") 101 | self.update_progress() 102 | log.info("Replicator finished") 103 | 104 | def sync_images(self, project=None): 105 | project = project or self.project 106 | for image in self.images_to_download(project=project): 107 | if self.config("dry_run"): 108 | click.echo("[dry-run] clone_image({}, {}, {})".format(image.name, image.tag, image.docker_id)) 109 | continue 110 | log.info("Pulling {}:{}".format(image.name, image.tag)) 111 | self.clone_image(image.name, image.tag, image.docker_id) # independent 112 | self.state[image.name][image.tag] = image.docker_id # dep [clone] 113 | yield image 114 | self.save_state() 115 | 116 | def images_to_download(self, project=None): 117 | project = project or self.project 118 | 119 | self.progress.add_step(key="query", status="running", header="Getting list of Docker images to clone") 120 | self.update_progress(progress_length_unknown=True) 121 | 122 | # determine images and tags (and dockerImageIds) from the remote registry 123 | if self.config("strict_name_match"): 124 | filter_fn = self.filter_on_tag_strict if self.min_version or self.images else None 125 | else: 126 | filter_fn = self.filter_on_tag if self.min_version or self.images else None 127 | remote_state = self.nvcr.get_state(project=project, filter_fn=filter_fn) 128 | 129 | # determine which images need to be fetch for the local state to match the remote 130 | to_pull = self.missing_images(remote_state) 131 | 132 | # sort images into two buckets: cuda and not cuda 133 | cuda_images = { key: val for key, val in to_pull.items() if key.endswith("cuda") } 134 | other_images = { key: val for key, val in to_pull.items() if not key.endswith("cuda") } 135 | 136 | all_images = [image for image in self.images_from_state(cuda_images)] 137 | all_images.extend([image for image in self.images_from_state(other_images)]) 138 | 139 | if self.config("external_images"): 140 | all_images.extend(self.third_party_images) 141 | 142 | for image in all_images: 143 | self.progress.add_step(key="{}:{}".format(image.name, image.tag), 144 | header="Cloning {}:{}".format(image.name, image.tag), 145 | subHeader="Waiting to pull image") 146 | self.progress.add_step(key="markdown", header="Downloading NVIDIA Deep Learning READMEs") 147 | self.progress.update_step(key="query", status="complete") 148 | self.update_progress() 149 | 150 | for image in self.images_from_state(cuda_images): 151 | yield image 152 | 153 | for image in self.images_from_state(other_images): 154 | yield image 155 | 156 | if self.config("external_images"): 157 | for image in self.third_party_images: 158 | yield image 159 | 160 | def update_progress(self, progress_length_unknown=False): 161 | self.progress.post(progress_length_unknown=progress_length_unknown) 162 | 163 | @staticmethod 164 | def images_from_state(state): 165 | for image_name, tag_data in state.items(): 166 | for tag, docker_id in tag_data.items(): 167 | yield replicator_pb2.DockerImage(name=image_name, tag=tag, docker_id=docker_id.get("docker_id", "")) 168 | 169 | def clone_image(self, image_name, tag, docker_id): 170 | if docker_id: 171 | url = self.nvcr.docker_url(image_name, tag=tag) 172 | else: 173 | url = "{}:{}".format(image_name, tag) 174 | if self.export_to_tarfile: 175 | tarfile = self.nvcr_client.url2filename(url) 176 | if os.path.exists(tarfile): 177 | log.warning("{} exists; removing and rebuilding".format(tarfile)) 178 | os.remove(tarfile) 179 | log.info("cloning %s --> %s" % (url, tarfile)) 180 | self.progress.update_step(key="{}:{}".format(image_name, tag), status="running", subHeader="Pulling image from Registry") 181 | self.update_progress() 182 | self.nvcr_client.pull(url) 183 | self.progress.update_step(key="{}:{}".format(image_name, tag), status="running", subHeader="Saving image to tarfile") 184 | self.update_progress() 185 | self.nvcr_client.save(url, path=self.output_path) 186 | self.progress.update_step(key="{}:{}".format(image_name, tag), status="complete", subHeader="Saved {}".format(tarfile)) 187 | log.info("Saved image: %s --> %s" % (url, tarfile)) 188 | if self.export_to_singularity: 189 | sif = os.path.join(self.output_path, "{}.sif".format(url).replace("/", "_")) 190 | if os.path.exists(sif): 191 | log.warning("{} exists; removing and rebuilding".format(sif)) 192 | os.remove(sif) 193 | log.info("cloning %s --> %s" % (url, sif)) 194 | self.progress.update_step(key="{}:{}".format(image_name, tag), status="running", subHeader="Pulling image from Registry") 195 | self.update_progress() 196 | self.nvcr_client.pull(url) 197 | self.progress.update_step(key="{}:{}".format(image_name, tag), status="running", subHeader="Saving image to singularity image file") 198 | self.update_progress() 199 | utils.execute("singularity build {} docker-daemon://{}".format(sif, url)) 200 | self.progress.update_step(key="{}:{}".format(image_name, tag), status="complete", subHeader="Saved {}".format(sif)) 201 | log.info("Saved image: %s --> %s" % (url, sif)) 202 | if self.registry_client: 203 | push_url = "{}/{}:{}".format(self.registry_url, image_name, tag) 204 | self.nvcr_client.pull(url) 205 | self.registry_client.tag(url, push_url) 206 | self.registry_client.push(push_url) 207 | self.registry_client.remove(push_url) 208 | if not self.config("no_remove") and not image_name.endswith("cuda") and self.nvcr_client.get(url=url): 209 | try: 210 | self.nvcr_client.remove(url) 211 | except: 212 | log.warning("tried to remove docker image {}, but unexpectedly failed".format(url)) 213 | return image_name, tag, docker_id 214 | 215 | def filter_on_tag(self, *, name, tag, docker_id, strict_name_match=False): 216 | """ 217 | Filter function used by the `nvidia_deepops` library for selecting images. 218 | 219 | Return True if the name/tag/docker_id combo should be included for consideration. 220 | Return False and the image will be excluded from consideration, i.e. not cloned/replicated. 221 | """ 222 | if self.images: 223 | log.debug("filtering on images name, only allow {}".format(self.images)) 224 | found = False 225 | for image in self.images: 226 | if (not strict_name_match) and (image in name): 227 | log.debug("{} passes filter; matches {}".format(name, image)) 228 | found = True 229 | elif (strict_name_match) and image.strip() == (name.split('/')[-1]).strip(): 230 | log.debug("{} passes strict filter; matches {}".format(name, image)) 231 | found = True 232 | if not found: 233 | log.debug("{} fails filter by image name".format(name)) 234 | return False 235 | # if you are here, you have passed the name test 236 | # now, we check the version of the container by trying to extract the YY.MM details from the tag 237 | if self.py_version: 238 | if tag.find(self.py_version) == -1: 239 | log.debug("tag {} fails py_version {} filter".format(tag, self.py_version)) 240 | return False 241 | version_regex = re.compile(r"^(\d\d\.\d\d)") 242 | float_tag = version_regex.findall(tag) 243 | if float_tag and len(float_tag) == 1: 244 | try: 245 | # this is a bit ugly, but if for some reason the cast of float_tag[0] or min_verison fail 246 | # we fallback to safety and skip tag filtering 247 | val = float(float_tag[0]) 248 | lower_bound = float(self.min_version) 249 | if val < lower_bound: 250 | return False 251 | except Exception: 252 | pass 253 | # if you are here, you have passed the tag test 254 | return True 255 | 256 | def filter_on_tag_strict(self, *, name, tag, docker_id): 257 | return self.filter_on_tag(name=name, tag=tag, docker_id=docker_id, strict_name_match=True) 258 | 259 | def missing_images(self, remote): 260 | """ 261 | Generates a dict of dicts on a symmetric difference between remote/local which also includes 262 | any image/tag pair in both but with differing dockerImageIds. 263 | :param remote: `image_name:tag:docker_id` of remote content 264 | :param local: `image_name:tag:docker_id` of local content 265 | :return: `image_name:tag:docker_id` for each missing or different entry in remote but not in local 266 | """ 267 | to_pull = collections.defaultdict(dict) 268 | local = self.state 269 | 270 | # determine which images are not present 271 | image_names = set(remote.keys()) - set(local.keys()) 272 | for image_name in image_names: 273 | to_pull[image_name] = remote[image_name] 274 | 275 | # log.debug("remote image names: %s" % remote.keys()) 276 | # log.debug("local image names: %s" % local.keys()) 277 | log.debug("image names not present: %s" % to_pull.keys()) 278 | 279 | # determine which tags are not present 280 | for image_name, tag_data in remote.items(): 281 | tags = set(tag_data.keys()) - set(local[image_name].keys()) 282 | # log.debug("remote %s tags: %s" % (image_name, tag_data.keys())) 283 | # log.debug("local %s tags: %s" % (image_name, local[image_name].keys())) 284 | log.debug("tags not present for image {}: {}".format(image_name, tags)) 285 | for tag in tags: 286 | to_pull[image_name][tag] = remote[image_name][tag] 287 | 288 | # determine if any name/tag pairs have a different dockerImageId than previously seen 289 | # this handles the cases where someone push a new images and overwrites a name:tag image 290 | for image_name, tag_data in remote.items(): 291 | if image_name not in local: continue 292 | for tag, docker_id in tag_data.items(): 293 | if tag not in local[image_name]: continue 294 | if docker_id.get("docker_id") != local[image_name][tag]: 295 | log.debug("%s:%s changed on server" % (image_name, tag)) 296 | to_pull[image_name][tag] = docker_id 297 | 298 | log.info("images to be fetched: %s" % pprint.pformat(to_pull, indent=4)) 299 | return to_pull 300 | 301 | 302 | ## class ReplicatorService(replicator_pb2_grpc.ReplicatorServicer): 303 | ## 304 | ## def __init__(self, *, replicator): 305 | ## self.replicator = replicator 306 | ## self.replicator.service = True 307 | ## 308 | ## def StartReplication(self, request, context): 309 | ## project = request.org_name or self.replicator.project 310 | ## for image in self.replicator.sync_images(project=project): 311 | ## yield image 312 | ## 313 | ## def ListImages(self, request, context): 314 | ## project = request.org_name or self.replicator.project 315 | ## for image in self.replicator.images_to_download(project=project): 316 | ## yield image 317 | ## # images_and_tags = self.replicator.nvcr.get_images_and_tags(project=project) 318 | ## # for image_name, tags in images_and_tags.items(): 319 | ## # for tag in tags: 320 | ## # yield replicator_pb2.DockerImage(name=image_name, tag=tag) 321 | ## 322 | ## def DownloadedImages(self, request, context): 323 | ## for images in self.replicator.images_from_state(self.replicator.state): 324 | ## yield images 325 | 326 | 327 | @click.command() 328 | @click.option("--api-key", envvar="NGC_REPLICATOR_API_KEY") 329 | @click.option("--project", default="nvidia") 330 | @click.option("--output-path", default="/output") 331 | @click.option("--min-version") 332 | @click.option("--py-version") 333 | @click.option("--image", multiple=True) 334 | @click.option("--registry-url") 335 | @click.option("--registry-username") 336 | @click.option("--registry-password") 337 | @click.option("--dry-run", is_flag=True) 338 | @click.option("--service", is_flag=True) 339 | @click.option("--external-images") 340 | @click.option("--progress-uri") 341 | @click.option("--no-remove", is_flag=True) 342 | @click.option("--exporter/--no-exporter", default=True) 343 | @click.option("--templater/--no-templater", default=False) 344 | @click.option("--singularity/--no-singularity", default=False) 345 | @click.option("--strict-name-match/--no-strict-name-match", default=False) 346 | def main(**config): 347 | """ 348 | NGC Replication Service 349 | """ 350 | if config.get("api_key", None) is None: 351 | click.echo("API key required; use --api-key or NGC_REPLICATOR_API_KEY", err=True) 352 | raise click.Abort 353 | 354 | replicator = Replicator(**config) 355 | 356 | if replicator.service: 357 | # server = grpc.server(futures.ThreadPoolExecutor(max_workers=1)) 358 | # replicator_pb2_grpc.add_ReplicatorServicer_to_server( 359 | # ReplicatorService(replicator=replicator), server 360 | # ) 361 | # server.add_insecure_port('[::]:50051') 362 | # log.info("starting GRPC service on port 50051") 363 | # server.start() 364 | # try: 365 | # while True: 366 | # time.sleep(_ONE_DAY_IN_SECONDS) 367 | # except KeyboardInterrupt: 368 | # server.stop(0) 369 | raise NotImplementedError("GPRC Service has been depreciated") 370 | else: 371 | replicator.sync() 372 | 373 | 374 | if __name__ == "__main__": 375 | main(auto_envvar_prefix='NGC_REPLICATOR') 376 | -------------------------------------------------------------------------------- /replicator/ngc_replicator/replicator_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: replicator.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | from google.protobuf import descriptor_pb2 11 | # @@protoc_insertion_point(imports) 12 | 13 | _sym_db = _symbol_database.Default() 14 | 15 | 16 | 17 | 18 | DESCRIPTOR = _descriptor.FileDescriptor( 19 | name='replicator.proto', 20 | package='', 21 | syntax='proto3', 22 | serialized_pb=_b('\n\x10replicator.proto\"@\n\x07Request\x12\x10\n\x08org_name\x18\x01 \x01(\t\x12\x13\n\x0bmin_version\x18\x02 \x01(\t\x12\x0e\n\x06images\x18\x03 \x03(\t\"6\n\x11ReplicationStatus\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x10\n\x08progress\x18\x02 \x01(\x02\";\n\x0b\x44ockerImage\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03tag\x18\x02 \x01(\t\x12\x11\n\tdocker_id\x18\x03 \x01(\t2\x9c\x01\n\nReplicator\x12\x34\n\x10StartReplication\x12\x08.Request\x1a\x12.ReplicationStatus\"\x00\x30\x01\x12(\n\nListImages\x12\x08.Request\x1a\x0c.DockerImage\"\x00\x30\x01\x12.\n\x10\x44ownloadedImages\x12\x08.Request\x1a\x0c.DockerImage\"\x00\x30\x01\x62\x06proto3') 23 | ) 24 | 25 | 26 | 27 | 28 | _REQUEST = _descriptor.Descriptor( 29 | name='Request', 30 | full_name='Request', 31 | filename=None, 32 | file=DESCRIPTOR, 33 | containing_type=None, 34 | fields=[ 35 | _descriptor.FieldDescriptor( 36 | name='org_name', full_name='Request.org_name', index=0, 37 | number=1, type=9, cpp_type=9, label=1, 38 | has_default_value=False, default_value=_b("").decode('utf-8'), 39 | message_type=None, enum_type=None, containing_type=None, 40 | is_extension=False, extension_scope=None, 41 | options=None, file=DESCRIPTOR), 42 | _descriptor.FieldDescriptor( 43 | name='min_version', full_name='Request.min_version', index=1, 44 | number=2, type=9, cpp_type=9, label=1, 45 | has_default_value=False, default_value=_b("").decode('utf-8'), 46 | message_type=None, enum_type=None, containing_type=None, 47 | is_extension=False, extension_scope=None, 48 | options=None, file=DESCRIPTOR), 49 | _descriptor.FieldDescriptor( 50 | name='images', full_name='Request.images', index=2, 51 | number=3, type=9, cpp_type=9, label=3, 52 | has_default_value=False, default_value=[], 53 | message_type=None, enum_type=None, containing_type=None, 54 | is_extension=False, extension_scope=None, 55 | options=None, file=DESCRIPTOR), 56 | ], 57 | extensions=[ 58 | ], 59 | nested_types=[], 60 | enum_types=[ 61 | ], 62 | options=None, 63 | is_extendable=False, 64 | syntax='proto3', 65 | extension_ranges=[], 66 | oneofs=[ 67 | ], 68 | serialized_start=20, 69 | serialized_end=84, 70 | ) 71 | 72 | 73 | _REPLICATIONSTATUS = _descriptor.Descriptor( 74 | name='ReplicationStatus', 75 | full_name='ReplicationStatus', 76 | filename=None, 77 | file=DESCRIPTOR, 78 | containing_type=None, 79 | fields=[ 80 | _descriptor.FieldDescriptor( 81 | name='message', full_name='ReplicationStatus.message', index=0, 82 | number=1, type=9, cpp_type=9, label=1, 83 | has_default_value=False, default_value=_b("").decode('utf-8'), 84 | message_type=None, enum_type=None, containing_type=None, 85 | is_extension=False, extension_scope=None, 86 | options=None, file=DESCRIPTOR), 87 | _descriptor.FieldDescriptor( 88 | name='progress', full_name='ReplicationStatus.progress', index=1, 89 | number=2, type=2, cpp_type=6, label=1, 90 | has_default_value=False, default_value=float(0), 91 | message_type=None, enum_type=None, containing_type=None, 92 | is_extension=False, extension_scope=None, 93 | options=None, file=DESCRIPTOR), 94 | ], 95 | extensions=[ 96 | ], 97 | nested_types=[], 98 | enum_types=[ 99 | ], 100 | options=None, 101 | is_extendable=False, 102 | syntax='proto3', 103 | extension_ranges=[], 104 | oneofs=[ 105 | ], 106 | serialized_start=86, 107 | serialized_end=140, 108 | ) 109 | 110 | 111 | _DOCKERIMAGE = _descriptor.Descriptor( 112 | name='DockerImage', 113 | full_name='DockerImage', 114 | filename=None, 115 | file=DESCRIPTOR, 116 | containing_type=None, 117 | fields=[ 118 | _descriptor.FieldDescriptor( 119 | name='name', full_name='DockerImage.name', index=0, 120 | number=1, type=9, cpp_type=9, label=1, 121 | has_default_value=False, default_value=_b("").decode('utf-8'), 122 | message_type=None, enum_type=None, containing_type=None, 123 | is_extension=False, extension_scope=None, 124 | options=None, file=DESCRIPTOR), 125 | _descriptor.FieldDescriptor( 126 | name='tag', full_name='DockerImage.tag', index=1, 127 | number=2, type=9, cpp_type=9, label=1, 128 | has_default_value=False, default_value=_b("").decode('utf-8'), 129 | message_type=None, enum_type=None, containing_type=None, 130 | is_extension=False, extension_scope=None, 131 | options=None, file=DESCRIPTOR), 132 | _descriptor.FieldDescriptor( 133 | name='docker_id', full_name='DockerImage.docker_id', index=2, 134 | number=3, type=9, cpp_type=9, label=1, 135 | has_default_value=False, default_value=_b("").decode('utf-8'), 136 | message_type=None, enum_type=None, containing_type=None, 137 | is_extension=False, extension_scope=None, 138 | options=None, file=DESCRIPTOR), 139 | ], 140 | extensions=[ 141 | ], 142 | nested_types=[], 143 | enum_types=[ 144 | ], 145 | options=None, 146 | is_extendable=False, 147 | syntax='proto3', 148 | extension_ranges=[], 149 | oneofs=[ 150 | ], 151 | serialized_start=142, 152 | serialized_end=201, 153 | ) 154 | 155 | DESCRIPTOR.message_types_by_name['Request'] = _REQUEST 156 | DESCRIPTOR.message_types_by_name['ReplicationStatus'] = _REPLICATIONSTATUS 157 | DESCRIPTOR.message_types_by_name['DockerImage'] = _DOCKERIMAGE 158 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 159 | 160 | Request = _reflection.GeneratedProtocolMessageType('Request', (_message.Message,), dict( 161 | DESCRIPTOR = _REQUEST, 162 | __module__ = 'replicator_pb2' 163 | # @@protoc_insertion_point(class_scope:Request) 164 | )) 165 | _sym_db.RegisterMessage(Request) 166 | 167 | ReplicationStatus = _reflection.GeneratedProtocolMessageType('ReplicationStatus', (_message.Message,), dict( 168 | DESCRIPTOR = _REPLICATIONSTATUS, 169 | __module__ = 'replicator_pb2' 170 | # @@protoc_insertion_point(class_scope:ReplicationStatus) 171 | )) 172 | _sym_db.RegisterMessage(ReplicationStatus) 173 | 174 | DockerImage = _reflection.GeneratedProtocolMessageType('DockerImage', (_message.Message,), dict( 175 | DESCRIPTOR = _DOCKERIMAGE, 176 | __module__ = 'replicator_pb2' 177 | # @@protoc_insertion_point(class_scope:DockerImage) 178 | )) 179 | _sym_db.RegisterMessage(DockerImage) 180 | 181 | 182 | 183 | _REPLICATOR = _descriptor.ServiceDescriptor( 184 | name='Replicator', 185 | full_name='Replicator', 186 | file=DESCRIPTOR, 187 | index=0, 188 | options=None, 189 | serialized_start=204, 190 | serialized_end=360, 191 | methods=[ 192 | _descriptor.MethodDescriptor( 193 | name='StartReplication', 194 | full_name='Replicator.StartReplication', 195 | index=0, 196 | containing_service=None, 197 | input_type=_REQUEST, 198 | output_type=_REPLICATIONSTATUS, 199 | options=None, 200 | ), 201 | _descriptor.MethodDescriptor( 202 | name='ListImages', 203 | full_name='Replicator.ListImages', 204 | index=1, 205 | containing_service=None, 206 | input_type=_REQUEST, 207 | output_type=_DOCKERIMAGE, 208 | options=None, 209 | ), 210 | _descriptor.MethodDescriptor( 211 | name='DownloadedImages', 212 | full_name='Replicator.DownloadedImages', 213 | index=2, 214 | containing_service=None, 215 | input_type=_REQUEST, 216 | output_type=_DOCKERIMAGE, 217 | options=None, 218 | ), 219 | ]) 220 | _sym_db.RegisterServiceDescriptor(_REPLICATOR) 221 | 222 | DESCRIPTOR.services_by_name['Replicator'] = _REPLICATOR 223 | 224 | # @@protoc_insertion_point(module_scope) 225 | -------------------------------------------------------------------------------- /replicator/requirements.txt: -------------------------------------------------------------------------------- 1 | PyYAML==5.4 2 | protobuf==3.5.1 3 | -------------------------------------------------------------------------------- /replicator/requirements_dev.txt: -------------------------------------------------------------------------------- 1 | pip==19.2 2 | bumpversion==0.5.3 3 | wheel==0.29.0 4 | watchdog==0.8.3 5 | flake8==2.6.0 6 | tox==2.3.1 7 | coverage==4.1 8 | Sphinx==1.4.8 9 | 10 | pytest==2.9.2 11 | pytest-runner==2.11.1 12 | -------------------------------------------------------------------------------- /replicator/scripts/docker-utils: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | CMD=$0 5 | 6 | function isTruthy { 7 | local arg=$1 8 | 9 | if [[ "$arg" = true ]] || [[ "$arg" = "true" ]] || [[ "$arg" -eq 1 ]]; then 10 | echo "true" 11 | fi 12 | 13 | } 14 | 15 | if [[ ${TRACE:-""} ]] && [[ $(isTruthy "$TRACE") ]]; then 16 | set -x 17 | fi 18 | 19 | 20 | function usage { 21 | cat <&2 echo "ERROR: Must have protocol in registry url" && usage 60 | 61 | # remove the protocol 62 | REG="$(echo ${1/$PROTO/})" 63 | shift 64 | ACTION="$1" 65 | shift 66 | 67 | CREDS="" 68 | DOCKER_CONFIG="$HOME/.docker/config.json" 69 | 70 | if [[ -f "$DOCKER_CONFIG" ]]; then 71 | AUTH_INFO=$(jq -r '.auths."'$REG'".auth' < "$DOCKER_CONFIG") 72 | if [ "$AUTH_INFO" = "null" ]; then 73 | AUTH_INFO=$(jq -r '.auths."'$PROTO$REG'".auth' < "$DOCKER_CONFIG") 74 | # if [ "$AUTH_INFO" = "null" ]; then 75 | # echo "ERROR: Failed to retrieve credentials from $DOCKER_CONFIG for ${REG}!" 76 | # exit 4 77 | # fi 78 | fi 79 | CREDS="Authorization: Basic $AUTH_INFO" 80 | if [ "$AUTH_INFO" = "null" ]; then 81 | CREDS="" 82 | fi 83 | elif [[ ${BASIC_AUTH:-""} ]]; then 84 | CREDS="Authorization: Basic $(echo -n $BASIC_AUTH|base64)" 85 | fi 86 | 87 | SEC_FLAG="" 88 | 89 | if [[ ${INSECURE_REGISTRY:-""} ]] && [[ $(isTruthy "$INSECURE_REGISTRY") ]]; then 90 | SEC_FLAG="-k" 91 | fi 92 | 93 | function curlCmd { 94 | curl "$SEC_FLAG" --header "$CREDS" $* 95 | } 96 | 97 | 98 | 99 | case "$ACTION" in 100 | list) 101 | if [ $# -eq 1 ]; then 102 | repo=${1} 103 | if [ -n "$repo" ]; then 104 | curlCmd -s "$PROTO$REG/v2/$repo/tags/list" | jq -r '.tags|.[]' 105 | fi 106 | else 107 | curlCmd -s "$PROTO$REG/v2/_catalog?n=500" | jq -r '.repositories|.[]' 108 | fi 109 | 110 | ;; 111 | delete) 112 | repo=$1 113 | tag=$2 114 | response=$(curlCmd -v -s -H "Accept:application/vnd.docker.distribution.manifest.v2+json" "$PROTO$REG/v2/$repo/manifests/$tag" 2>&1) 115 | digest=$(echo "$response" | grep -i "< Docker-Content-Digest:"|awk '{print $3}') 116 | digest=${digest//[$'\t\r\n']} 117 | echo "DIGEST: $digest" 118 | result=$(curlCmd -s -o /dev/null -w "%{http_code}" -H "Accept:application/vnd.docker.distribution.manifest.v2+json" -X DELETE "$PROTO$REG/v2/$repo/manifests/$digest") 119 | if [ $result -eq 202 ]; then 120 | echo "Successfully deleted" 121 | exit 0 122 | else 123 | echo "Failed to delete" 124 | exit 3 125 | fi 126 | ;; 127 | esac 128 | 129 | -------------------------------------------------------------------------------- /replicator/setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.4.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | search = version='{current_version}' 8 | replace = version='{new_version}' 9 | 10 | [bumpversion:file:ngc_replicator/__init__.py] 11 | search = __version__ = '{current_version}' 12 | replace = __version__ = '{new_version}' 13 | 14 | [bumpversion:file:Makefile] 15 | 16 | [bdist_wheel] 17 | universal = 1 18 | 19 | [flake8] 20 | exclude = docs 21 | 22 | [aliases] 23 | test = pytest 24 | 25 | -------------------------------------------------------------------------------- /replicator/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """The setup script.""" 5 | 6 | from setuptools import setup, find_packages 7 | 8 | requirements = [ 9 | 'Click>=6.0', 10 | # TODO: put package requirements here 11 | ] 12 | 13 | setup_requirements = [ 14 | 'pytest-runner', 15 | # TODO(ryanolson): put setup requirements (distutils extensions, etc.) here 16 | ] 17 | 18 | test_requirements = [ 19 | 'pytest', 20 | # TODO: put package test requirements here 21 | ] 22 | 23 | setup( 24 | name='ngc_replicator', 25 | version='0.4.0', 26 | description="NGC Replication Service", 27 | author="Ryan Olson", 28 | author_email='rolson@nvidia.com', 29 | url='https://github.com/ryanolson/ngc_replicator', 30 | packages=find_packages(include=['ngc_replicator']), 31 | entry_points={ 32 | 'console_scripts': [ 33 | 'ngc_replicator=ngc_replicator.ngc_replicator:main' 34 | ] 35 | }, 36 | include_package_data=True, 37 | install_requires=requirements, 38 | zip_safe=False, 39 | keywords='ngc_replicator', 40 | classifiers=[ 41 | 'Development Status :: 2 - Pre-Alpha', 42 | 'Intended Audience :: Developers', 43 | 'Natural Language :: English', 44 | "Programming Language :: Python :: 2", 45 | 'Programming Language :: Python :: 2.6', 46 | 'Programming Language :: Python :: 2.7', 47 | 'Programming Language :: Python :: 3', 48 | 'Programming Language :: Python :: 3.3', 49 | 'Programming Language :: Python :: 3.4', 50 | 'Programming Language :: Python :: 3.5', 51 | ], 52 | test_suite='tests', 53 | tests_require=test_requirements, 54 | setup_requires=setup_requirements, 55 | ) 56 | -------------------------------------------------------------------------------- /replicator/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Unit test package for ngc_replicator.""" 4 | -------------------------------------------------------------------------------- /replicator/tests/test_ngc_replicator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import subprocess 5 | import sys 6 | import tempfile 7 | 8 | """Tests for `ngc_replicator` package.""" 9 | 10 | import pytest 11 | 12 | from ngc_replicator import ngc_replicator 13 | 14 | try: 15 | from .secrets import ngcpassword, dgxpassword 16 | HAS_SECRETS = True 17 | except Exception: 18 | HAS_SECRETS = False 19 | 20 | secrets = pytest.mark.skipif(not HAS_SECRETS, reason="No secrets.py file found") 21 | 22 | @secrets 23 | def nvsa_replicator(*, output_path): 24 | """ 25 | Instance of the test NGC Registry on compute.nvidia.com (project=nvsa) 26 | """ 27 | return ngc_replicator.Replicator( 28 | project="nvsa_clone", 29 | api_key=dgxpassword, 30 | exporter=True, 31 | output_path=output_path, 32 | min_version="16.04" 33 | ) 34 | 35 | 36 | @secrets 37 | def test_clone(): 38 | with tempfile.TemporaryDirectory() as tmpdir: 39 | state_file = os.path.join(tmpdir, "state.yml") 40 | assert not os.path.exists(state_file) 41 | replicator = nvsa_replicator(output_path=tmpdir) 42 | replicator.sync() 43 | assert os.path.exists(state_file) 44 | assert 'nvsa_clone/busybox' in replicator.state 45 | -------------------------------------------------------------------------------- /replicator/tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py26, py27, py33, py34, py35, flake8 3 | 4 | [travis] 5 | python = 6 | 3.5: py35 7 | 3.4: py34 8 | 3.3: py33 9 | 2.7: py27 10 | 2.6: py26 11 | 12 | [testenv:flake8] 13 | basepython=python3 14 | deps=flake8 15 | commands=flake8 ngc_replicator 16 | 17 | [testenv] 18 | setenv = 19 | PYTHONPATH = {toxinidir} 20 | deps = 21 | -r{toxinidir}/requirements_dev.txt 22 | commands = 23 | pip install -U pip 24 | py.test --basetemp={envtmpdir} 25 | 26 | 27 | ; If you want to make tox run the tests with the same versions, create a 28 | ; requirements.txt with the pinned versions and uncomment the following lines: 29 | ; deps = 30 | ; -r{toxinidir}/requirements.txt 31 | --------------------------------------------------------------------------------