├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── rm-rf_docker_stuff.bat ├── run_build.bat ├── run_notebook.bat └── theanorc /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: true 2 | dist: trusty 3 | 4 | language: python 5 | 6 | env: 7 | - KERAS_BACKEND=tensorflow 8 | - KERAS_BACKEND=theano 9 | python: 10 | - 2.7 11 | - 3.6 12 | services: 13 | - docker 14 | stage: Build Docker image 15 | 16 | install: docker build --build-arg python_version=$TRAVIS_PYTHON_VERSION -t geo-docker -f Dockerfile . 17 | 18 | stage: Test 19 | 20 | script: docker run -it -p 8888:8888 --env KERAS_BACKEND=$KERAS_BACKEND geo-docker 21 | 22 | jobs: 23 | include: 24 | - stage: Deploy to Docker.io 25 | provider: script 26 | before_script: echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin 27 | script: 28 | - docker images 29 | - docker tag geo-docker softwareunderground/geo-docker 30 | - docker push softwareunderground/geo-docker 31 | branch: master 32 | python: 3.6 33 | env: KERAS_BACKEND=tensorflow 34 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # set out our GPU lib versions 2 | # note even though we only specify the major version for cuDNN it will always pull 3 | 4 | ARG CUDA_V=9.0 5 | 6 | FROM nvidia/cuda:${CUDA_V}-devel 7 | 8 | ENV CUDA_VERSION ${CUDA_V} 9 | ENV CUDNN_VERSION 7.0.5.15 10 | 11 | # Install system packages 12 | RUN apt-get update && apt-get install -y --no-install-recommends \ 13 | bzip2 \ 14 | g++ \ 15 | git \ 16 | graphviz \ 17 | libgl1-mesa-glx \ 18 | libhdf5-dev \ 19 | openmpi-bin \ 20 | cuda-command-line-tools-9-0 \ 21 | wget && \ 22 | rm -rf /var/lib/apt/lists/* 23 | 24 | # Install correct CuDNN version for tensorflow 25 | LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}" 26 | RUN apt-get update && apt-get install -y --no-install-recommends \ 27 | libcudnn7=$CUDNN_VERSION-1+cuda9.0 \ 28 | libcudnn7-dev=$CUDNN_VERSION-1+cuda9.0 && \ 29 | rm -rf /var/lib/apt/lists/* 30 | 31 | # Install conda 32 | ENV CONDA_DIR /opt/conda 33 | ENV PATH $CONDA_DIR/bin:$PATH 34 | 35 | RUN wget --quiet --no-check-certificate https://repo.continuum.io/miniconda/Miniconda3-4.4.10-Linux-x86_64.sh && \ 36 | /bin/bash /Miniconda3-4.4.10-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ 37 | rm Miniconda3-4.4.10-Linux-x86_64.sh && \ 38 | echo export PATH=$CONDA_DIR/bin:'$PATH' > /etc/profile.d/conda.sh 39 | 40 | RUN conda update -n base conda 41 | RUN conda update openssl ca-certificates certifi 42 | RUN conda config --add channels conda-forge 43 | RUN apt-get install -y ca-certificates 44 | 45 | # Install Goodies 46 | ENV NB_USER geo 47 | ENV NB_UID 1000 48 | 49 | RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \ 50 | chown $NB_USER $CONDA_DIR -R && \ 51 | mkdir -p /src && \ 52 | chown $NB_USER /src 53 | 54 | USER $NB_USER 55 | 56 | ARG python_version=3.6 57 | 58 | RUN conda install -y python=${python_version} 59 | RUN conda config --set always_yes yes 60 | RUN pip install --upgrade pip 61 | RUN pip install https://cntk.ai/PythonWheel/GPU/cntk-2.1-cp36-cp36m-linux_x86_64.whl 62 | RUN pip install --no-cache-dir Cython 63 | 64 | ## Base Python Packages 65 | RUN conda install \ 66 | bcolz \ 67 | h5py \ 68 | matplotlib \ 69 | mkl \ 70 | nose \ 71 | notebook \ 72 | pygpu \ 73 | pyyaml \ 74 | six 75 | 76 | RUN pip install \ 77 | python-dotenv 78 | 79 | ## Data Science 80 | RUN conda install \ 81 | numpy \ 82 | scipy \ 83 | pandas \ 84 | tqdm \ 85 | colorcet \ 86 | seaborn \ 87 | networkx 88 | 89 | ## Image Processing 90 | RUN conda install \ 91 | Pillow \ 92 | scikit-image 93 | 94 | ## ML Packages 95 | RUN conda install \ 96 | scikit-learn \ 97 | six \ 98 | theano 99 | 100 | RUN pip install \ 101 | sklearn_pandas \ 102 | tensorflow-gpu \ 103 | tensorboardX \ 104 | jupyter-tensorboard \ 105 | livelossplot 106 | 107 | ## TPOT plus Dependencies 108 | RUN pip install \ 109 | deap \ 110 | update_checker \ 111 | tqdm \ 112 | stopit \ 113 | xgboost \ 114 | scikit-mdr \ 115 | skrebate \ 116 | tpot 117 | 118 | 119 | ### Torch (Because you're special) 120 | RUN conda install pytorch torchvision cuda90 -c pytorch \ 121 | && conda clean -ya 122 | 123 | RUN pip install git+https://github.com/pytorch/tnt.git@master 124 | 125 | # keras 126 | RUN git clone git://github.com/keras-team/keras.git /src && pip install -e /src[tests] && \ 127 | pip install git+git://github.com/keras-team/keras.git 128 | 129 | ## Geo Packages 130 | RUN conda install \ 131 | geopandas \ 132 | shapely \ 133 | dask 134 | 135 | RUN pip install \ 136 | obspy \ 137 | pynoddy \ 138 | gempy \ 139 | segyio \ 140 | bruges \ 141 | welly \ 142 | fiona \ 143 | rasterio \ 144 | simpeg \ 145 | lasio \ 146 | mplstereonet 147 | 148 | ## Package install over 149 | 150 | RUN conda clean -yt 151 | 152 | ADD theanorc /home/$NB_USER/.theanorc 153 | 154 | ENV PYTHONPATH='/src/:$PYTHONPATH' 155 | 156 | WORKDIR /src 157 | 158 | # Tensorboard 159 | EXPOSE 6006 160 | # Jupyter / iPython 161 | EXPOSE 8888 162 | 163 | CMD jupyter notebook --port=8888 --ip=0.0.0.0 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | All contributions tracked in this git repository are made by the 2 | awesome people at Software Underground, 2018+ 3 | 4 | 5 | ORIGINAL COPYRIGHT NOTICE from the keras repository 6 | 7 | All contributions by François Chollet: 8 | Copyright (c) 2015 - 2018, François Chollet. 9 | All rights reserved. 10 | 11 | All contributions by Google: 12 | Copyright (c) 2015 - 2018, Google, Inc. 13 | All rights reserved. 14 | 15 | All contributions by Microsoft: 16 | Copyright (c) 2017 - 2018, Microsoft, Inc. 17 | All rights reserved. 18 | 19 | All other contributions: 20 | Copyright (c) 2015 - 2018, the respective contributors. 21 | All rights reserved. 22 | 23 | Each contributor holds copyright over their respective contributions. 24 | The project versioning (Git) records all such contribution source information. 25 | 26 | LICENSE 27 | 28 | The MIT License (MIT) 29 | 30 | Permission is hereby granted, free of charge, to any person obtaining a copy 31 | of this software and associated documentation files (the "Software"), to deal 32 | in the Software without restriction, including without limitation the rights 33 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 34 | copies of the Software, and to permit persons to whom the Software is 35 | furnished to do so, subject to the following conditions: 36 | 37 | The above copyright notice and this permission notice shall be included in all 38 | copies or substantial portions of the Software. 39 | 40 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 41 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 42 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 43 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 44 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 45 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 46 | SOFTWARE. 47 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | help: 2 | @cat Makefile 3 | 4 | DATA?="${HOME}/Datasets" 5 | GPU?=0 6 | DOCKER_FILE=Dockerfile 7 | DOCKER=GPU=$(GPU) nvidia-docker 8 | BACKEND=tensorflow 9 | PYTHON_VERSION?=3.6 10 | CUDA_VERSION?=9.0 11 | CUDNN_VERSION?=7 12 | TEST=tests/ 13 | SRC?=$(shell dirname `pwd`) 14 | 15 | build: 16 | docker build -t geoml --build-arg python_version=$(PYTHON_VERSION) -f $(DOCKER_FILE) . 17 | 18 | bash: build 19 | $(DOCKER) run -it -v $(SRC):/src/workspace -v $(DATA):/src/workspace/data --env KERAS_BACKEND=$(BACKEND) geoml bash 20 | 21 | ipython: build 22 | $(DOCKER) run -it -v $(SRC):/src/workspace -v $(DATA):/src/workspace/data --env KERAS_BACKEND=$(BACKEND) geoml ipython 23 | 24 | notebook: build 25 | $(DOCKER) run -it -v $(SRC):/src/workspace -v $(DATA):/src/workspace/data --net=host --env KERAS_BACKEND=$(BACKEND) geoml 26 | 27 | test: build 28 | $(DOCKER) run -it -v $(SRC):/src/workspace -v $(DATA):/src/workspace/data --env KERAS_BACKEND=$(BACKEND) geoml py.test $(TEST) 29 | 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **DEPRECATED please use https://github.com/softwareunderground/subsurface-ml-docker** 2 | 3 | 4 | # geo-docker [![Build Status](https://travis-ci.org/JesperDramsch/geo-docker.svg?branch=master)](https://travis-ci.org/JesperDramsch/geo-docker) 5 | A docker image fully loaded with Geo* & ML related packages. 6 | 7 | 1. clone this repo into a folder *alongside* other repos or folders with code you want to execute in 8 | 1. build and launch a container with python notebook attached with `make notebook`. 9 | 1. In the notebook, browse to the `/workspace` folder, you should see the contents of your the parent directory mounted as a volume, which means read/write access from the container. 10 | 1. A folder `~/Datasets` is also mounted in the container at `/data` 11 | 12 | To change any of the mounted paths, or add more edit the Makefile 13 | 14 | At the moment, the configuration has first class setup for keras, as that is where it started out. 15 | 16 | ## What's in the Box 17 | A full Anaconda install is huge and we are adding to that with common ml and geo packages. To try and stop this getting too bloated we have stuck with a MiniConda base image, meaning we need to be exoplicit about what we add but we only get what we want. 18 | 19 | Some attept has been made to have sections in the `Dockerfile` in the hope that it's easier for people to customise to their needs. 20 | 21 | The container currently holds: 22 | - loads of packages 23 | - that probably should be listed 24 | - somewhere, maybe here 25 | - ..... 26 | 27 | ## Missing stuff 28 | Here is a list of packages that were not included initially, maybe these should be turned into issues! :) 29 | - GPRMax package 30 | - noddy executable for pynoddy 31 | - torch & pytorch 32 | - pygimli 33 | - fatiando 34 | - cupy - slow to install 35 | - devito 36 | 37 | There are some other things it would be nice to do too: 38 | 39 | - add a dead simple example notebook athe at least exercises the GPU via tensorflow, maybe even jsut an xor, or minst example or something. 40 | - setup tensorboard properly with jupyter-tensorflow and provide an example of how to use it 41 | 42 | ## Installing Docker 43 | 44 | General installation instructions are 45 | [on the Docker site](https://docs.docker.com/installation/), but we give some 46 | quick links here: 47 | 48 | * [OSX](https://docs.docker.com/installation/mac/): [docker toolbox](https://www.docker.com/toolbox) 49 | * [ubuntu](https://docs.docker.com/installation/ubuntulinux/) 50 | 51 | ## Running the container 52 | 53 | We are using `Makefile` to simplify docker commands within make commands. 54 | 55 | Build the container and start a Jupyter Notebook 56 | 57 | $ make notebook 58 | 59 | Build the container and start an iPython shell 60 | 61 | $ make ipython 62 | 63 | Build the container and start a bash 64 | 65 | $ make bash 66 | 67 | For GPU support install NVIDIA drivers (ideally latest) and 68 | [nvidia-docker](https://github.com/NVIDIA/nvidia-docker). Run using 69 | 70 | $ make notebook GPU=0 # or [ipython, bash] 71 | 72 | Switch keras between Theano and TensorFlow 73 | 74 | $ make notebook BACKEND=theano 75 | $ make notebook BACKEND=tensorflow 76 | 77 | Mount a volume for external data sets 78 | 79 | $ make DATA=~/mydata 80 | 81 | Prints all make tasks 82 | 83 | $ make help 84 | 85 | You can change Theano parameters by editing `/docker/theanorc`. 86 | 87 | Note: If you would have a problem running nvidia-docker you may try the old way 88 | we have used. But it is not recommended. If you find a bug in the nvidia-docker report 89 | it there please and try using the nvidia-docker as described above. 90 | 91 | $ export CUDA_SO=$(\ls /usr/lib/x86_64-linux-gnu/libcuda.* | xargs -I{} echo '-v {}:{}') 92 | $ export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') 93 | $ docker run -it -p 8888:8888 $CUDA_SO $DEVICES gcr.io/tensorflow/tensorflow:latest-gpu 94 | 95 | # License 96 | MIT 97 | 98 | # Credits 99 | This docker and Makefile layout was originally based on the [docker starter example in the keras repo](https://github.com/keras-team/keras/tree/master/docker). THe Docker file in particular has been customised to make it easier to see groups of related packages and add remove as necessary. But the makefile and instructions in this readme are pretty much as-is and lovely. The original repository available under [MIT here](https://github.com/keras-team/keras/blob/master/LICENSE) -------------------------------------------------------------------------------- /rm-rf_docker_stuff.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | FOR /f "tokens=*" %%i IN ('docker ps -aq') DO docker rm %%i 3 | FOR /f "tokens=*" %%i IN ('docker images --format "{{.ID}}"') DO docker rmi %%i -------------------------------------------------------------------------------- /run_build.bat: -------------------------------------------------------------------------------- 1 | docker build -t geoml -f Dockerfile . -------------------------------------------------------------------------------- /run_notebook.bat: -------------------------------------------------------------------------------- 1 | set src=E:\Development\euclidity 2 | set data=E:\Data 3 | set backend=tensorflow 4 | docker run -it -p 0.0.0.0:8888:8888 -v %src%:/src/workspace -v %data%:/src/workspace/data --env KERAS_BACKEND=%backend% geoml -------------------------------------------------------------------------------- /theanorc: -------------------------------------------------------------------------------- 1 | [global] 2 | floatX = float32 3 | optimizer=None 4 | device = cuda 5 | 6 | --------------------------------------------------------------------------------