├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── chainer ├── Dockerfile └── runner.sh ├── mxnet ├── Dockerfile └── runner.sh ├── pytorch ├── Dockerfile └── runner.sh ├── tensorflow ├── Dockerfile └── runner.sh └── xgboost ├── Dockerfile └── runner.sh /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:10.0-cudnn7-devel 2 | 3 | ENV DEBIAN_FRONTEND noninteractive 4 | ENV CUDA_HOME /usr/local/cuda 5 | 6 | # System dependencies 7 | RUN apt-get update && apt-get install --no-install-recommends -y \ 8 | build-essential \ 9 | curl \ 10 | wget \ 11 | git \ 12 | cmake \ 13 | vim \ 14 | pkg-config \ 15 | unzip \ 16 | libgtk2.0-dev \ 17 | imagemagick \ 18 | graphviz 19 | # libgtk2.0-dev is for OpenCV 20 | 21 | RUN ldconfig 22 | 23 | # Miniconda 3 24 | ENV PATH /opt/conda/bin:$PATH 25 | ENV LB_LIBRARY_PATH /opt/conda/lib:$LB_LIBRARY_PATH 26 | RUN curl -Ls https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/install-miniconda.sh && \ 27 | /bin/bash /tmp/install-miniconda.sh -b -p /opt/conda && \ 28 | conda update -n base conda && \ 29 | conda update --all -y 30 | 31 | # CNMeM - A simple memory manager for CUDA designed to help Deep Learning frameworks manage memory 32 | RUN git clone --depth 1 https://github.com/NVIDIA/cnmem.git /usr/src/cnmem && \ 33 | mkdir /usr/src/cnmem/build && cd /usr/src/cnmem/build && \ 34 | cmake .. && make -j install 35 | 36 | # NCCL - Optimized primitives for collective multi-GPU communication 37 | RUN git clone --depth 1 https://github.com/NVIDIA/nccl.git /usr/src/nccl && \ 38 | cd /usr/src/nccl && make -j install 39 | 40 | # Basic dependencies 41 | RUN conda install -y \ 42 | boost \ 43 | cython \ 44 | gensim \ 45 | hdf5 \ 46 | jupyterlab \ 47 | leveldb \ 48 | lmdb \ 49 | matplotlib \ 50 | mkl \ 51 | numpy \ 52 | openblas \ 53 | pandas \ 54 | pillow \ 55 | protobuf \ 56 | readline \ 57 | scipy 58 | 59 | RUN pip install \ 60 | h5py \ 61 | hyperdash \ 62 | nnpack \ 63 | pydot_ng \ 64 | scikit-image \ 65 | scikit-learn 66 | 67 | # OpenCV 68 | RUN conda install opencv3 -c menpo -y 69 | 70 | WORKDIR /workspace 71 | VOLUME /workspace 72 | 73 | RUN ln -s /usr/src /root/src && ln -s /project /root/project -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Yasuaki Uechi 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 14 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 15 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 16 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 19 | OR OTHER DEALINGS IN THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: build 2 | 3 | build: 4 | docker build -t uetchy/ml:base . 5 | docker build -t uetchy/ml:tensorflow ./tensorflow 6 | docker build -t uetchy/ml:pytorch ./pytorch 7 | docker build -t uetchy/ml:chainer ./chainer 8 | 9 | test: build 10 | docker run --runtime=nvidia --rm -it uetchy/ml:tensorflow test 11 | 12 | publish: build 13 | docker push uetchy/ml:base 14 | docker push uetchy/ml:tensorflow 15 | docker push uetchy/ml:pytorch 16 | docker push uetchy/ml:chainer 17 | 18 | bash: 19 | docker run --runtime=nvidia --rm -it uetchy/ml:base bash 20 | 21 | jupyter: 22 | docker run --runtime=nvidia --rm -p 8888:8888 -it uetchy/ml:base jupyter 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dockerfile for Machine Learning 2 | 3 | A Dockerfile for accelerated research process, consists of major machine learning libraries. 4 | 5 | - [DockerHub](https://registry.hub.docker.com/u/uetchy/ml/) 6 | 7 | ## Features 8 | 9 | - Ubuntu 18.04 10 | - Python 3.6 (Miniconda 3) 11 | - GPU accelerated (CUDA 10.0, cuDNN 7) 12 | - NCCL, CNMeM, Apex (PyTorch only) activated 13 | - Jupyter and OpenCV 3.0 included 14 | - Additional packages (Tensorboard, Hyperdash, etc) 15 | 16 | ## Available Deep Learning Frameworks 17 | 18 | - TensorFlow `uetchy/ml:tensorflow` 19 | - PyTorch `uetchy/ml:pytorch` 20 | - Chainer `uetchy/ml:chainer` 21 | - MXnet `uetchy/ml:mxnet` 22 | - XGBoost `uetchy/ml:xgboost` 23 | 24 | ## Install 25 | 26 | ### System Requirements 27 | 28 | - Docker 29 | - CUDA-enabled GPUs 30 | - CUDA Driver 31 | - CUDA Toolkit 32 | - nvidia-docker2 (https://github.com/NVIDIA/nvidia-docker) 33 | 34 | ### Pull the docker image from [DockerHub](https://registry.hub.docker.com/u/uetchy/ml/) 35 | 36 | ```bash 37 | docker pull uetchy/ml:tensorflow 38 | docker pull uetchy/ml:pytorch 39 | docker pull uetchy/ml:chainer 40 | ... 41 | ``` 42 | 43 | ### Launch Jupyter Lab on current directory 44 | 45 | ```bash 46 | docker run --runtime=nvidia -v $PWD:/workspace -p 8888:8888 -it uetchy/ml:pytorch jupyter 47 | open http://localhost:8888 48 | ``` 49 | 50 | ### Open Python REPL 51 | 52 | ```bash 53 | docker run --runtime=nvidia --rm -it uetchy/ml:base python 54 | ``` 55 | 56 | ### Run Bash Shell 57 | 58 | ```bash 59 | docker run --runtime=nvidia --rm -it uetchy/ml:tensorflow 60 | ``` 61 | 62 | ## List of Docker images for Data Science 63 | 64 | - [TensorFlow](https://hub.docker.com/r/tensorflow/tensorflow) 65 | - [Caffe2](https://hub.docker.com/r/caffe2ai/caffe2) 66 | - [MXNet](https://hub.docker.com/u/mxnet) 67 | - [Caffe](https://github.com/BVLC/caffe/tree/master/docker) 68 | 69 | # Contribution 70 | 71 | PRs are accepted. 72 | 73 | ## Contributors 74 | 75 | - Yasuaki Uechi 76 | - UpmostScarab 77 | - cyrusmvahid 78 | -------------------------------------------------------------------------------- /chainer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM uetchy/ml:base 2 | 3 | # Chainer 4 | RUN git clone --depth 1 https://github.com/pfnet/chainer.git /usr/src/chainer && \ 5 | pip install -U chainer cupy-cuda100 6 | 7 | COPY runner.sh /workspace/runner.sh 8 | ENTRYPOINT ["/workspace/runner.sh"] 9 | -------------------------------------------------------------------------------- /chainer/runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | case "$1" in 6 | "") 7 | bash 8 | ;; 9 | jupyter) 10 | jupyter lab --no-browser --allow-root --ip='*' 11 | ;; 12 | test) 13 | python -c "import chainer; chainer.print_runtime_info()" 14 | ;; 15 | *) 16 | $@ 17 | ;; 18 | esac 19 | 20 | exit 0 21 | -------------------------------------------------------------------------------- /mxnet/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM uetchy/ml:base 2 | 3 | # MXNet 4 | RUN git clone --depth 1 --recursive https://github.com/dmlc/mxnet /usr/src/mxnet && \ 5 | pip install mxnet-cu100 6 | 7 | COPY runner.sh /workspace/runner.sh 8 | ENTRYPOINT ["/workspace/runner.sh"] -------------------------------------------------------------------------------- /mxnet/runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | case "$1" in 6 | "") 7 | bash 8 | ;; 9 | jupyter) 10 | jupyter lab --no-browser --allow-root --ip='*' 11 | ;; 12 | test) 13 | python -c "import tensorflow as tf; print('TensorFlow', tf.__version__); tf.Session()" 14 | ;; 15 | *) 16 | $@ 17 | ;; 18 | esac 19 | 20 | exit 0 21 | -------------------------------------------------------------------------------- /pytorch/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM uetchy/ml:base 2 | 3 | # PyTorch 4 | RUN git clone --depth 1 https://github.com/pytorch/tutorials.git /usr/src/pytorch-tutorials && \ 5 | conda config --prepend channels pytorch && \ 6 | conda install pytorch torchvision cuda100 -y 7 | 8 | RUN pip install tensorboardX 9 | 10 | # Apex 11 | RUN git clone --depth 1 https://github.com/NVIDIA/apex.git /usr/src/apex && \ 12 | cd /usr/src/apex && python setup.py install --cuda_ext --cpp_ext 13 | 14 | COPY runner.sh /workspace/runner.sh 15 | ENTRYPOINT ["/workspace/runner.sh"] -------------------------------------------------------------------------------- /pytorch/runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | case "$1" in 6 | "") 7 | bash 8 | ;; 9 | jupyter) 10 | jupyter lab --no-browser --allow-root --ip='*' 11 | ;; 12 | test) 13 | python -c "import torch; print('PyTorch', torch.__version__, torch.cuda.current_device())" 14 | ;; 15 | *) 16 | $@ 17 | ;; 18 | esac 19 | 20 | exit 0 21 | -------------------------------------------------------------------------------- /tensorflow/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM uetchy/ml:base 2 | 3 | RUN pip install tensorflow-gpu tensorboard 4 | 5 | # RUN apt-get install openjdk-8-jdk -y && \ 6 | # echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" >> /etc/apt/sources.list.d/bazel.list && \ 7 | # curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ 8 | # apt-get update && \ 9 | # apt-get install bazel -y 10 | # see https://gist.github.com/PatWie/0c915d5be59a518f934392219ca65c3d 11 | # and https://github.com/tensorflow/tensorflow/blob/master/configure.py 12 | # and https://github.com/tensorflow/tensorflow/blob/master/tools/bazel.rc 13 | # for noninteractive tf build 14 | # RUN git clone --depth 1 -b r1.7 https://github.com/tensorflow/tensorflow.git /usr/src/tensorflow && \ 15 | # cd /usr/src/tensorflow && \ 16 | # PYTHON_BIN_PATH=$(which python) \ 17 | # PYTHON_LIB_PATH="$($PYTHON_BIN_PATH -c 'import site; print(site.getsitepackages()[0])')" \ 18 | # CUDA_TOOLKIT_PATH=/usr/local/cuda \ 19 | # CUDNN_INSTALL_PATH=/usr \ 20 | # TF_NEED_GCP=0 \ 21 | # TF_NEED_CUDA=1 \ 22 | # TF_CUDA_VERSION="$($CUDA_TOOLKIT_PATH/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p')" \ 23 | # TF_CUDA_COMPUTE_CAPABILITIES=6.1,5.2,3.5 \ 24 | # TF_NEED_HDFS=0 \ 25 | # TF_NEED_OPENCL=0 \ 26 | # TF_NEED_JEMALLOC=1 \ 27 | # TF_ENABLE_XLA=0 \ 28 | # TF_NEED_VERBS=0 \ 29 | # TF_CUDA_CLANG=0 \ 30 | # TF_CUDNN_VERSION="$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' $CUDNN_INSTALL_PATH/include/cudnn.h)" \ 31 | # TF_NEED_MKL=0 \ 32 | # TF_DOWNLOAD_MKL=0 \ 33 | # TF_NEED_MPI=0 \ 34 | # TF_NEED_OPENCL_SYCL=0 \ 35 | # TF_NEED_S3=0 \ 36 | # TF_NEED_KAFKA=0 \ 37 | # TF_NEED_TENSORRT=0 \ 38 | # TF_NEED_GDR=0 \ 39 | # TF_SET_ANDROID_WORKSPACE=0 \ 40 | # GCC_HOST_COMPILER_PATH=$(which gcc) \ 41 | # CC_OPT_FLAGS="-march=native" ./configure && \ 42 | # bazel build --config=opt --config=cuda //tensorflow/tools/pip_package:build_pip_package && \ 43 | # bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg && \ 44 | # pip install /tmp/tensorflow_pkg/tensorflow-1.7.0-cp36-cp36m-linux_x86_64.whl 45 | # cannot use GPUs within build process, you can also do GPU test manually: 46 | # python -c 'import tensorflow as tf;sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))' 47 | 48 | COPY runner.sh /workspace/runner.sh 49 | ENTRYPOINT ["/workspace/runner.sh"] 50 | -------------------------------------------------------------------------------- /tensorflow/runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | case "$1" in 6 | "") 7 | bash 8 | ;; 9 | jupyter) 10 | jupyter lab --no-browser --allow-root --ip='*' 11 | ;; 12 | test) 13 | python -c "import tensorflow as tf; print('TensorFlow', tf.__version__); tf.Session()" 14 | ;; 15 | *) 16 | $@ 17 | ;; 18 | esac 19 | 20 | exit 0 21 | -------------------------------------------------------------------------------- /xgboost/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM uetchy/ml:base 2 | 3 | # XGBoost 4 | RUN git clone --depth 1 --recursive https://github.com/dmlc/xgboost /usr/src/xgboost && \ 5 | mkdir /usr/src/xgboost/build && cd /usr/src/xgboost/build && \ 6 | cmake .. -DUSE_CUDA=ON && \ 7 | make -j4 && \ 8 | cd ../python-package && python setup.py install 9 | 10 | COPY runner.sh /workspace/runner.sh 11 | ENTRYPOINT ["/workspace/runner.sh"] -------------------------------------------------------------------------------- /xgboost/runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | case "$1" in 6 | "") 7 | bash 8 | ;; 9 | jupyter) 10 | jupyter lab --no-browser --allow-root --ip='*' 11 | ;; 12 | test) 13 | python -c "import tensorflow as tf; print('TensorFlow', tf.__version__); tf.Session()" 14 | python -c "import chainer; chainer.print_runtime_info()" 15 | ;; 16 | *) 17 | $@ 18 | ;; 19 | esac 20 | 21 | exit 0 22 | --------------------------------------------------------------------------------