├── .gitattributes ├── .github └── workflows │ ├── build-push.yml │ └── build.yml ├── README.md ├── build ├── Dockerfile ├── htoprc └── start-notebook.sh ├── run ├── README.md ├── config │ ├── jupyter_notebook_config.py │ ├── lab │ │ └── user-settings │ │ │ └── @jupyterlab │ │ │ ├── apputils-extension │ │ │ └── themes.jupyterlab-settings │ │ │ ├── console-extension │ │ │ └── tracker.jupyterlab-settings │ │ │ ├── extensionmanager-extension │ │ │ └── plugin.jupyterlab-settings │ │ │ ├── fileeditor-extension │ │ │ └── plugin.jupyterlab-settings │ │ │ ├── notebook-extension │ │ │ └── tracker.jupyterlab-settings │ │ │ └── statusbar-extension │ │ │ └── plugin.jupyterlab-settings │ └── nbconfig │ │ └── notebook.json ├── data │ └── .gitignore ├── notebooks │ └── .gitignore ├── run.sh └── secret │ └── .gitignore └── utils ├── create_env.sh ├── install_docker.sh └── install_nvidia_525.sh /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.github/workflows/build-push.yml: -------------------------------------------------------------------------------- 1 | name: Build and push 2 | 3 | on: 4 | pull_request: 5 | types: [closed] 6 | branches: master 7 | 8 | jobs: 9 | 10 | build-push: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | python: [3.8, 3.11] 18 | 19 | steps: 20 | - uses: actions/checkout@v2 21 | 22 | - name: Build 23 | run: | 24 | docker build build \ 25 | -t analysiscenter1/ds-py3:${{ matrix.python }} --build-arg PYTHON_VERSION=${{ matrix.python }} \ 26 | -t analysiscenter1/ds-py3:${{ matrix.python }}_$(echo ${{github.sha}} | cut -c1-7) --build-arg PYTHON_VERSION=${{ matrix.python }} 27 | 28 | - name: Push 29 | run: | 30 | echo ${{ secrets.DOCKER_HUB_PASS }} | docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin 31 | docker push analysiscenter1/ds-py3 --all-tags 32 | 33 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | pull_request 5 | 6 | jobs: 7 | 8 | build: 9 | 10 | runs-on: ubuntu-latest 11 | 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | python: [3.8, 3.11] 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | 20 | - run: | 21 | docker build build \ 22 | -t analysiscenter1/ds-py3:${{ matrix.python }} --build-arg PYTHON_VERSION=${{ matrix.python }} 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Docker containers with python 3 environment (with GPU support) for data science 2 | 3 | # Installation directory 4 | ``` 5 | git clone https://github.com/analysiscenter/ds-py3.git 6 | cd ds-py3 7 | ``` 8 | 9 | # NVIDIA driver 10 | Check whether the driver is installed and its version: 11 | ``` 12 | nvidia-smi 13 | ``` 14 | 15 | To install the latest NVIDIA driver execute `utils/install_nvidia_*.sh` and reboot. 16 | 17 | # Docker 18 | To install Docker and NVIDIA-docker2 execute `utils/install_docker.sh`. 19 | 20 | If the installation succeeds, the list of GPUs available within a docker container will be shown. 21 | 22 | 23 | # Container 24 | To prepare a docker environment run `utils/create_env.sh`. 25 | 26 | Map additional disks to subdirectories within `/notebooks`. 27 | 28 | Set a password in `run/config/jupyter_notebook_config.py`. 29 | 30 | 31 | # Run jupyter 32 | See [run/README.md](run/README.md) or just execute: 33 | ``` 34 | cd run 35 | ./run.sh 36 | ``` 37 | -------------------------------------------------------------------------------- /build/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:lunar 2 | LABEL maintainer="Roman Kh " 3 | 4 | # required to prevent interactions while installing packages (e.g. software-properties-common) 5 | ENV DEBIAN_FRONTEND=noninteractive 6 | 7 | ARG PYTHON_VERSION=local 8 | ENV PYTHON_VERSION=${PYTHON_VERSION} 9 | 10 | # install dependencies for pyenv 11 | RUN apt-get update && apt -y dist-upgrade && \ 12 | apt install -y --no-install-recommends make build-essential libssl-dev zlib1g-dev \ 13 | libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev \ 14 | xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ 15 | mecab-ipadic-utf8 ca-certificates git 16 | 17 | ENV PYENV_ROOT /root/.pyenv 18 | ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH 19 | 20 | RUN set -ex \ 21 | && curl https://pyenv.run | bash \ 22 | && pyenv update \ 23 | && pyenv install $PYTHON_VERSION \ 24 | && pyenv global $PYTHON_VERSION \ 25 | && pyenv rehash 26 | 27 | RUN apt install -y software-properties-common cmake gfortran \ 28 | libboost-dev libboost-system-dev libboost-filesystem-dev \ 29 | liblapack-dev libatlas-base-dev libopenblas-dev zip p7zip-full \ 30 | zlib1g-dev liblzma-dev liblz4-dev libzstd-dev libsnappy-dev \ 31 | libhdf5-dev libedit-dev libzmq3-dev libgl1-mesa-glx \ 32 | wget tmux nano sysstat htop mc tree unzip rsync \ 33 | libgl1-mesa-glx libglib2.0-0 pkg-config openssh-client 34 | 35 | RUN pip3 install -U numpy && \ 36 | pip3 install -U tbb && \ 37 | pip3 install numba==0.57.0rc1 && \ 38 | pip3 install torch torchvision 39 | 40 | RUN pip3 install -U pip && \ 41 | pip3 install -U poetry && \ 42 | pip3 install -U setuptools && \ 43 | apt install -y python3-venv && \ 44 | pip3 install -U virtualenv && \ 45 | pip3 install -U pipenv && \ 46 | pip3 install -U pipreqs && \ 47 | pip3 install -U cython && \ 48 | pip3 install -U toolz && \ 49 | pip3 install -U fsspec && \ 50 | pip3 install -U cloudpickle && \ 51 | pip3 install -U dill && \ 52 | pip3 install -U fastcache && \ 53 | pip3 install -U python-snappy && \ 54 | pip3 install -U blosc && \ 55 | pip3 install -U aiofiles && \ 56 | pip3 install -U packaging && \ 57 | pip3 install -U click && \ 58 | pip3 install -U itsdangerous && \ 59 | pip3 install -U requests && \ 60 | pip3 install -U multiprocess && \ 61 | pip3 install -U psutil && \ 62 | pip3 install -U eventlet && \ 63 | pip3 install -U greenlet && \ 64 | pip3 install -U aiohttp && \ 65 | pip3 install -U python-socketio && \ 66 | pip3 install -U flask && \ 67 | pip3 install -U flask-socketio && \ 68 | pip3 install -U pytest && \ 69 | pip3 install -U pytest-aiohttp && \ 70 | pip3 install -U pytest-flask && \ 71 | pip3 install -U pytest-cov && \ 72 | pip3 install -U tox && \ 73 | pip3 install -U autopep8 && \ 74 | pip3 install -U cookiecutter && \ 75 | pip3 install -U tqdm && \ 76 | pip3 install -U pylint && \ 77 | pip3 install -U connected-components-3d 78 | 79 | # install data manipulation packages 80 | RUN pip3 install -U zarr && \ 81 | pip3 install -U lmdb && \ 82 | pip3 install -U bottleneck && \ 83 | pip3 install -U numexpr && \ 84 | pip3 install -U tables && \ 85 | pip3 install -U h5py && \ 86 | pip3 install -U h5pickle && \ 87 | pip3 install -U hdf5plugin && \ 88 | pip3 install -U pandas && \ 89 | pip3 install -U feather-format && \ 90 | pip3 install -U fastparquet && \ 91 | pip3 install -U dask && \ 92 | pip3 install -U pint && \ 93 | pip3 install -U python-dateutil && \ 94 | pip3 install -U 'dvc[ssh]' && \ 95 | pip3 install -U cupy-cuda$(python3 -c 'import torch; print(torch.version.cuda)' | cut -c 1,2)x 96 | 97 | # install stats and ML libraries 98 | RUN pip3 install -U scipy && \ 99 | pip3 install -U scikit-learn && \ 100 | pip3 install -U sklearn-pandas && \ 101 | pip3 install -U statsmodels && \ 102 | pip3 install -U nltk && \ 103 | pip3 install -U xgboost && \ 104 | pip3 install -U catboost-dev[widget]==1.2rc0 && \ 105 | pip3 install -U lightgbm 106 | 107 | # install image libraries 108 | RUN apt install -y graphviz libfreetype6-dev libpng-dev libjpeg-dev && \ 109 | pip3 install -U graphviz && \ 110 | pip3 install -U matplotlib && \ 111 | pip3 install -U scikit-image && \ 112 | pip3 install -U imageio && \ 113 | pip3 install -U seaborn && \ 114 | pip3 install -U folium && \ 115 | pip3 install -U vispy && \ 116 | pip3 install -U plotly && \ 117 | pip3 install -U ipympl && \ 118 | pip3 install -U opencv-python 119 | 120 | # install special packages 121 | RUN pip3 install -U PyWavelets && \ 122 | pip3 install -U segyio && \ 123 | pip3 install -U lasio && \ 124 | pip3 install -U xlrd && \ 125 | pip3 install -U nvidia-ml-py3 && \ 126 | pip3 install -U fpdf && \ 127 | pip3 install -U sphinx && \ 128 | pip3 install -U gpustat && \ 129 | pip3 install -U py-nbtools && \ 130 | pip3 install -U batchflow && \ 131 | pip3 install -U segfast 132 | 133 | # install deep learning libraries 134 | RUN pip3 install -U einops 135 | 136 | # install jupyter and extensions 137 | RUN pip3 install -U nodejs && \ 138 | pip3 install -U jupyterlab && \ 139 | pip3 install -U jupyterlab_execute_time && \ 140 | pip3 install -U nbdime && \ 141 | nbdime extensions --enable 142 | 143 | # add short alias for gpustat 144 | RUN echo "alias gtop='gpustat -i 0.2'" >> ~/.bashrc 145 | 146 | COPY htoprc /root/.config/htop/htoprc 147 | COPY start-notebook.sh /usr/local/bin/start-notebook.sh 148 | 149 | WORKDIR /notebooks 150 | 151 | ENTRYPOINT ["/bin/sh", "-c"] 152 | CMD ["start-notebook.sh"] 153 | -------------------------------------------------------------------------------- /build/htoprc: -------------------------------------------------------------------------------- 1 | # Beware! This file is rewritten by htop when settings are changed in the interface. 2 | # The parser is also very primitive, and not human-friendly. 3 | fields=0 48 17 18 38 39 40 2 46 47 49 1 4 | sort_key=46 5 | sort_direction=1 6 | hide_threads=0 7 | hide_kernel_threads=1 8 | hide_userland_threads=0 9 | shadow_other_users=0 10 | show_thread_names=0 11 | show_program_path=1 12 | highlight_base_name=0 13 | highlight_megabytes=1 14 | highlight_threads=1 15 | tree_view=0 16 | header_margin=1 17 | detailed_cpu_time=0 18 | cpu_count_from_zero=0 19 | update_process_names=0 20 | account_guest_in_cpu_meter=0 21 | color_scheme=0 22 | delay=15 23 | left_meters=LeftCPUs2 Memory Swap 24 | left_meter_modes=1 1 1 25 | right_meters=RightCPUs2 Tasks LoadAverage Uptime 26 | right_meter_modes=1 2 2 2 27 | -------------------------------------------------------------------------------- /build/start-notebook.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export JUPYTER_CONFIG_DIR=/jupyter/config 4 | 5 | jupyter lab --no-browser --allow-root 6 | -------------------------------------------------------------------------------- /run/README.md: -------------------------------------------------------------------------------- 1 | # How to run a data science container 2 | 3 | In order to start a container you just need `./run.sh` 4 | 5 | Jupyter notebook config file is located in `./config` directory. You might want to change config, at least the password hash to access the notebook. 6 | 7 | 8 | **Prerequisites** 9 | 10 | You need [docker](https://docs.docker.com/engine/installation/linux/), [GPU drivers](http://www.nvidia.ru/Download/index.aspx) and [nvidia-docker](https://github.com/NVIDIA/nvidia-docker) installed. 11 | 12 | ## Settings 13 | 14 | Before running a container you might set some environment variables: 15 | 16 | ### DS_NAME 17 | default: `ds-py3` 18 | 19 | ### DS_NOTEBOOKS_DIR 20 | default: `./notebooks` 21 | 22 | Directory in the host system which is mapped to a container `/notebooks` directory where all the notebooks are stored 23 | 24 | ### DS_DATA_DIR 25 | default: `./data` 26 | 27 | Directory in the host system which is mapped to a container `/data` directory where all the data are stored 28 | 29 | ### DS_CONFIG_DIR 30 | default: `./config` 31 | 32 | Directory in the host system where `jupyter_notebook_config.py` is stored. It is mapped to `/jupyter/config` directory in the container. 33 | 34 | ### DS_SECRET_DIR 35 | default: `./secret` 36 | 37 | Directory in the host system where TLS certs are stored. It is mapped to `/jupyter/secret` directory in the container. 38 | 39 | ### DS_PORT 40 | default: `8888` 41 | 42 | Host port where jupyter notebook is listening. 43 | 44 | ### DS_IMAGE 45 | default: `analysiscenter1/ds-py3` 46 | 47 | Docker image to run in a container. 48 | 49 | ### DS_EXTRA_PORTS 50 | default: `2` 51 | 52 | Number of extra ports to open in a container, starting from the `DS_PORT` + 1. If equals to `0`, no extra ports are mapped.
53 | For example, if `DS_PORT=8892` and `DS_EXTRA_PORTS=2`, then the container is started with additional port mapping 8893:8893, 8894:8894. 54 | 55 | ## Examples 56 | `DS_PORT=8889 ./run.sh` - to run a container which can be accessed at `http://localhost:8889` 57 | `DS_NOTEBOOKS_DIR=/notebooks ./run.sh` - to store notebooks in the host directory `/notebooks` 58 | 59 | You can pass additional docker options, for instance:
60 | `DS_PORT=8889 ./run.sh -it` - to run a container interactively. 61 | 62 | To see docker options enabled by default, see the [runfile](../run/run.sh). 63 | -------------------------------------------------------------------------------- /run/config/jupyter_notebook_config.py: -------------------------------------------------------------------------------- 1 | c.NotebookApp.ip = '0.0.0.0' 2 | c.NotebookApp.port = 8888 3 | c.NotebookApp.open_browser = False 4 | c.NotebookApp.token = u'password' 5 | c.NotebookApp.allow_root = True 6 | c.NotebookApp.terminado_settings = { 'shell_command': ['bash'] } 7 | -------------------------------------------------------------------------------- /run/config/lab/user-settings/@jupyterlab/apputils-extension/themes.jupyterlab-settings: -------------------------------------------------------------------------------- 1 | { 2 | // Theme 3 | // @jupyterlab/apputils-extension:themes 4 | // Theme manager settings. 5 | // ************************************* 6 | 7 | // Selected Theme 8 | // Application-level visual styling theme 9 | "theme": "JupyterLab Dark", 10 | 11 | // Scrollbar Theming 12 | // Enable/disable styling of the application scrollbars 13 | "theme-scrollbars": true 14 | } 15 | -------------------------------------------------------------------------------- /run/config/lab/user-settings/@jupyterlab/console-extension/tracker.jupyterlab-settings: -------------------------------------------------------------------------------- 1 | { 2 | // Code Console 3 | // @jupyterlab/console-extension:tracker 4 | // Code Console settings. 5 | // ************************************* 6 | 7 | // Prompt Cell Configuration 8 | // The configuration for all prompt cells. 9 | "promptCellConfig": { 10 | "autoClosingBrackets": true, 11 | "cursorBlinkRate": 530, 12 | "fontFamily": null, 13 | "fontSize": null, 14 | "lineHeight": null, 15 | "lineNumbers": false, 16 | "lineWrap": "off", 17 | "matchBrackets": true, 18 | "readOnly": false, 19 | "insertSpaces": true, 20 | "tabSize": 4, 21 | "wordWrapColumn": 80, 22 | "rulers": [], 23 | "codeFolding": false, 24 | "lineWiseCopyCut": true 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /run/config/lab/user-settings/@jupyterlab/extensionmanager-extension/plugin.jupyterlab-settings: -------------------------------------------------------------------------------- 1 | { 2 | // Extension Manager 3 | // @jupyterlab/extensionmanager-extension:plugin 4 | // Extension manager settings. 5 | // ********************************************* 6 | 7 | // Disclaimed Status 8 | // Whether the user understand that extensions managed through this interface run arbitrary code that may be dangerous 9 | "disclaimed": true 10 | } 11 | -------------------------------------------------------------------------------- /run/config/lab/user-settings/@jupyterlab/fileeditor-extension/plugin.jupyterlab-settings: -------------------------------------------------------------------------------- 1 | { 2 | // Text Editor 3 | // @jupyterlab/fileeditor-extension:plugin 4 | // Text editor settings. 5 | // *************************************** 6 | 7 | // Editor Configuration 8 | // The configuration for all text editors. 9 | // If `fontFamily`, `fontSize` or `lineHeight` are `null`, 10 | // values from current theme are used. 11 | "editorConfig": { 12 | "cursorBlinkRate": 530, 13 | "fontFamily": null, 14 | "fontSize": null, 15 | "lineHeight": null, 16 | "lineNumbers": true, 17 | "lineWrap": "on", 18 | "wordWrapColumn": 80, 19 | "readOnly": false, 20 | "tabSize": 4, 21 | "insertSpaces": true, 22 | "matchBrackets": true, 23 | "autoClosingBrackets": true, 24 | "rulers": [120], 25 | "codeFolding": false, 26 | "showTrailingSpace": true 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /run/config/lab/user-settings/@jupyterlab/notebook-extension/tracker.jupyterlab-settings: -------------------------------------------------------------------------------- 1 | {"recordTiming": true, 2 | "codeCellConfig": {"matchBrackets" : true, 3 | "autoClosingBrackets" : true,},} 4 | -------------------------------------------------------------------------------- /run/config/lab/user-settings/@jupyterlab/statusbar-extension/plugin.jupyterlab-settings: -------------------------------------------------------------------------------- 1 | { 2 | // Status Bar 3 | // @jupyterlab/statusbar-extension:plugin 4 | // Status Bar settings. 5 | // ************************************** 6 | 7 | // Status Bar Visibility 8 | // Whether to show status bar or not 9 | "visible": true 10 | } 11 | -------------------------------------------------------------------------------- /run/config/nbconfig/notebook.json: -------------------------------------------------------------------------------- 1 | { 2 | "load_extensions": { 3 | "ipyvolume/extension": true, 4 | "jupyter-js-widgets/extension": true 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /run/data/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /run/notebooks/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /run/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | name=${DS_NAME:-ds-py3} 6 | notebooks_vol=${DS_NOTEBOOKS_DIR:-`pwd`/notebooks} 7 | data_vol=${DS_DATA_DIR:-`pwd`/data} 8 | config_vol=${DS_CONFIG_DIR:-`pwd`/config} 9 | secret_vol=${DS_SECRET_DIR:-`pwd`/secret} 10 | port=${DS_PORT:-8888} 11 | image=${DS_IMAGE:-analysiscenter1/ds-py3:3.8} 12 | extra_ports=${DS_EXTRA_PORTS:-2} 13 | 14 | ports="" 15 | if [ $extra_ports -gt 0 ]; then 16 | for i in $(seq 1 $extra_ports); do 17 | ports="$ports-p $((port+i)):$((port+i)) " 18 | done 19 | fi 20 | 21 | docker run -d --rm --name ${name} \ 22 | --gpus all --pid=host --shm-size=8G \ 23 | -p ${port}:8888 \ 24 | ${ports} \ 25 | -v ${notebooks_vol}:/notebooks \ 26 | -v ${data_vol}:/data \ 27 | -v ${config_vol}:/jupyter/config \ 28 | -v ${secret_vol}:/jupyter/secret \ 29 | $@ \ 30 | ${image} 31 | -------------------------------------------------------------------------------- /run/secret/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /utils/create_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # set up directory for notebooks 4 | sudo mkdir -p /notebooks 5 | rm -rf ../run/notebooks 6 | ln -s /notebooks ../run/notebooks 7 | 8 | # set up directory for data 9 | sudo mkdir -p /data 10 | rm -rf ../run/data 11 | ln -s /data ../run/data 12 | -------------------------------------------------------------------------------- /utils/install_docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # install Docker 4 | sudo apt install -y apt-transport-https ca-certificates curl software-properties-common 5 | 6 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 7 | 8 | sudo add-apt-repository \ 9 | "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ 10 | $(lsb_release -cs) \ 11 | stable" 12 | 13 | sudo apt update 14 | 15 | sudo apt install -y docker-ce docker-ce-cli containerd.io 16 | 17 | # install NVIDIA docker version 2 18 | curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - 19 | distribution=$(. /etc/os-release;echo $ID$VERSION_ID) 20 | curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | \ 21 | sudo tee /etc/apt/sources.list.d/nvidia-docker.list 22 | sudo apt update 23 | sudo apt install -y nvidia-container-toolkit 24 | 25 | # reload the Docker daemon configuration 26 | sudo systemctl restart docker 27 | -------------------------------------------------------------------------------- /utils/install_nvidia_525.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | sudo apt purge libnvidia* -y 4 | sudo apt purge nvidia* -y 5 | sudo apt dist-upgrade --fix-broken -y 6 | sudo add-apt-repository ppa:graphics-drivers/ppa -y 7 | sudo apt update 8 | sudo apt install -y --no-install-recommends nvidia-driver-525 nvidia-settings 9 | --------------------------------------------------------------------------------