├── README.md ├── LICENSE ├── Dockerfile ├── .gitignore └── start.sh /README.md: -------------------------------------------------------------------------------- 1 | # azure-agent-jupyter-minimal-notebook 2 | An agent for Azure Pipelines using jupyter/minimal-notebook as a base 3 | 4 | Build with 5 | ``` 6 | docker build --tag=basnijholt/azure-agent-jupyter-minimal-notebook:latest . 7 | ``` 8 | 9 | Run with 10 | ``` 11 | docker run -e AZP_URL=https://dev.azure.com/ -e AZP_TOKEN= -e AZP_AGENT_NAME=mydockeragent azure-agent-jupyter-minimal-notebook:latest 12 | ``` 13 | 14 | ## Notes 15 | * Used [this](https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/docker?view=azure-devops) as inspiration, but that image is Ubuntu 16, and we want 18.04. 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Bas Nijholt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jupyter/minimal-notebook 2 | 3 | # Switch from joyvan to root 4 | USER root 5 | 6 | # This prevents an error 7 | ENV XDG_RUNTIME_DIR="" 8 | 9 | # To make it easier for build and release pipelines to run apt-get, 10 | # configure apt to not require confirmation (assume the -y argument by default) 11 | ENV DEBIAN_FRONTEND=noninteractive 12 | RUN echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/90assumeyes 13 | 14 | # Make sure that we can download libicu55 15 | RUN apt-get update && \ 16 | apt-get install -y software-properties-common && \ 17 | add-apt-repository "deb http://security.ubuntu.com/ubuntu xenial-security main" 18 | 19 | # Install Azure Agent deps 20 | RUN apt-get update && \ 21 | apt-get install -y --no-install-recommends \ 22 | ca-certificates \ 23 | curl \ 24 | jq \ 25 | git \ 26 | iputils-ping \ 27 | libcurl4 \ 28 | libicu55 \ 29 | libunwind8 \ 30 | netcat 31 | 32 | # Install TeX deps such that we can convert `ipynb`s to `pdf` 33 | RUN apt-get update && \ 34 | apt-get install -y --no-install-recommends \ 35 | texlive-xetex texlive-fonts-recommended texlive-generic-recommended 36 | 37 | # Others 38 | RUN apt-get update && \ 39 | apt-get install -y --no-install-recommends \ 40 | ssh rsync 41 | 42 | # Because of https://github.com/git-lfs/git-lfs/issues/3571 the git-lfs from apt-get isn't good enough 43 | RUN conda install --yes git git-lfs 44 | 45 | # Set git-lfs settings globally (otherwise only refs are cloned) 46 | RUN git config --global --add filter.lfs.required true && \ 47 | git config --global --add filter.lfs.smudge "git-lfs smudge -- %f" && \ 48 | git config --global --add filter.lfs.process "git-lfs filter-process" && \ 49 | git config --global --add filter.lfs.clean "git-lfs clean -- %f" 50 | 51 | WORKDIR /azp 52 | 53 | COPY ./start.sh . 54 | RUN chmod +x start.sh 55 | 56 | CMD ["./start.sh"] 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | if [ -z "$AZP_URL" ]; then 5 | echo 1>&2 "error: missing AZP_URL environment variable" 6 | exit 1 7 | fi 8 | 9 | if [ -z "$AZP_TOKEN_FILE" ]; then 10 | if [ -z "$AZP_TOKEN" ]; then 11 | echo 1>&2 "error: missing AZP_TOKEN environment variable" 12 | exit 1 13 | fi 14 | 15 | AZP_TOKEN_FILE=/azp/.token 16 | echo -n $AZP_TOKEN > "$AZP_TOKEN_FILE" 17 | fi 18 | 19 | unset AZP_TOKEN 20 | 21 | if [ -n "$AZP_WORK" ]; then 22 | mkdir -p "$AZP_WORK" 23 | fi 24 | 25 | rm -rf /azp/agent 26 | mkdir /azp/agent 27 | cd /azp/agent 28 | 29 | export AGENT_ALLOW_RUNASROOT="1" 30 | 31 | cleanup() { 32 | if [ -e config.sh ]; then 33 | print_header "Cleanup. Removing Azure Pipelines agent..." 34 | 35 | ./config.sh remove --unattended \ 36 | --auth PAT \ 37 | --token $(cat "$AZP_TOKEN_FILE") 38 | fi 39 | } 40 | 41 | print_header() { 42 | lightcyan='\033[1;36m' 43 | nocolor='\033[0m' 44 | echo -e "${lightcyan}$1${nocolor}" 45 | } 46 | 47 | # Let the agent ignore the token env variables 48 | export VSO_AGENT_IGNORE=AZP_TOKEN,AZP_TOKEN_FILE 49 | 50 | print_header "1. Determining matching Azure Pipelines agent..." 51 | 52 | AZP_AGENT_RESPONSE=$(curl -LsS \ 53 | -u user:$(cat "$AZP_TOKEN_FILE") \ 54 | -H 'Accept:application/json;api-version=3.0-preview' \ 55 | "$AZP_URL/_apis/distributedtask/packages/agent?platform=linux-x64") 56 | 57 | if echo "$AZP_AGENT_RESPONSE" | jq . >/dev/null 2>&1; then 58 | AZP_AGENTPACKAGE_URL=$(echo "$AZP_AGENT_RESPONSE" \ 59 | | jq -r '.value | map([.version.major,.version.minor,.version.patch,.downloadUrl]) | sort | .[length-1] | .[3]') 60 | fi 61 | 62 | if [ -z "$AZP_AGENTPACKAGE_URL" -o "$AZP_AGENTPACKAGE_URL" == "null" ]; then 63 | echo 1>&2 "error: could not determine a matching Azure Pipelines agent - check that account '$AZP_URL' is correct and the token is valid for that account" 64 | exit 1 65 | fi 66 | 67 | print_header "2. Downloading and installing Azure Pipelines agent..." 68 | 69 | curl -LsS $AZP_AGENTPACKAGE_URL | tar -xz & wait $! 70 | 71 | source ./env.sh 72 | 73 | trap 'cleanup; exit 130' INT 74 | trap 'cleanup; exit 143' TERM 75 | 76 | print_header "3. Configuring Azure Pipelines agent..." 77 | 78 | ./config.sh --unattended \ 79 | --agent "${AZP_AGENT_NAME:-$(hostname)}" \ 80 | --url "$AZP_URL" \ 81 | --auth PAT \ 82 | --token $(cat "$AZP_TOKEN_FILE") \ 83 | --pool "${AZP_POOL:-Default}" \ 84 | --work "${AZP_WORK:-_work}" \ 85 | --replace \ 86 | --acceptTeeEula & wait $! 87 | 88 | # remove the administrative token before accepting work 89 | rm $AZP_TOKEN_FILE 90 | 91 | print_header "4. Running Azure Pipelines agent..." 92 | 93 | # `exec` the node runtime so it's aware of TERM and INT signals 94 | # AgentService.js understands how to handle agent self-update and restart 95 | exec ./externals/node/bin/node ./bin/AgentService.js interactive 96 | --------------------------------------------------------------------------------