├── .gitignore ├── Dockerfile ├── README.md ├── package.sh ├── run.sh ├── shared └── .gitkeep ├── update.sh └── utils.sh /.gitignore: -------------------------------------------------------------------------------- 1 | ml-examples 2 | jupyter.log 3 | shared 4 | *.tar -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | 3 | # [ tensorflow ] 4 | # https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/docker/Dockerfile 5 | 6 | RUN apt-get update; \ 7 | apt-get install -y --no-install-recommends \ 8 | build-essential \ 9 | curl \ 10 | libfreetype6-dev \ 11 | libpng12-dev \ 12 | libzmq3-dev \ 13 | pkg-config \ 14 | python \ 15 | python-dev \ 16 | rsync \ 17 | software-properties-common \ 18 | unzip; \ 19 | apt-get clean autoclean; \ 20 | apt-get autoremove -y; \ 21 | rm -rf /var/lib/apt/lists/* 22 | 23 | RUN curl -O https://bootstrap.pypa.io/get-pip.py; \ 24 | python get-pip.py; \ 25 | rm get-pip.py 26 | 27 | RUN pip --no-cache-dir install \ 28 | ipykernel \ 29 | jupyter \ 30 | matplotlib \ 31 | numpy \ 32 | scipy \ 33 | sklearn \ 34 | pandas \ 35 | Pillow \ 36 | tensorflow; \ 37 | python -m ipykernel.kernelspec 38 | 39 | # TensorBoard 40 | EXPOSE 6006 41 | 42 | # IPython 43 | EXPOSE 8888 44 | 45 | # [ pandas ] 46 | 47 | RUN pip --no-cache-dir install pandas 48 | 49 | # [ keras + theano ] 50 | # https://github.com/fchollet/keras/blob/master/docker/Dockerfile 51 | 52 | RUN apt-get update; \ 53 | apt-get install -y \ 54 | wget \ 55 | git \ 56 | libhdf5-dev \ 57 | g++ \ 58 | graphviz; \ 59 | apt-get clean autoclean; \ 60 | apt-get autoremove -y; \ 61 | rm -rf /var/lib/apt/lists/* 62 | 63 | RUN pip --no-cache-dir install h5py git+git://github.com/fchollet/keras.git 64 | 65 | # [ gensim ] 66 | 67 | RUN pip --no-cache-dir install gensim 68 | 69 | # [ torch ] 70 | # https://github.com/Kaixhin/dockerfiles/blob/master/torch/Dockerfile 71 | 72 | # Install git, apt-add-repository and dependencies for iTorch 73 | RUN apt-get update; \ 74 | apt-get install -y \ 75 | git \ 76 | software-properties-common \ 77 | ipython3 \ 78 | libssl-dev \ 79 | libzmq3-dev \ 80 | python-zmq \ 81 | python-pip; \ 82 | apt-get clean autoclean; \ 83 | apt-get autoremove -y; \ 84 | rm -rf /var/lib/apt/lists/* 85 | 86 | RUN git clone https://github.com/torch/distro.git /root/torch --recursive; \ 87 | cd /root/torch; bash install-deps; ./install.sh -b 88 | 89 | ENV LUA_PATH='/root/.luarocks/share/lua/5.1/?.lua;/root/.luarocks/share/lua/5.1/?/init.lua;/root/torch/install/share/lua/5.1/?.lua;/root/torch/install/share/lua/5.1/?/init.lua;./?.lua;/root/torch/install/share/luajit-2.1.0-beta1/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua' 90 | ENV LUA_CPATH='/root/.luarocks/lib/lua/5.1/?.so;/root/torch/install/lib/lua/5.1/?.so;./?.so;/usr/local/lib/lua/5.1/?.so;/usr/local/lib/lua/5.1/loadall.so' 91 | ENV PATH=/root/torch/install/bin:$PATH 92 | ENV LD_LIBRARY_PATH=/root/torch/install/lib:$LD_LIBRARY_PATH 93 | ENV DYLD_LIBRARY_PATH=/root/torch/install/lib:$DYLD_LIBRARY_PATH 94 | ENV LUA_CPATH='/root/torch/install/lib/?.so;'$LUA_CPATH 95 | 96 | RUN git clone https://github.com/facebook/iTorch.git ; \ 97 | cd iTorch ; luarocks make ; \ 98 | cd .. ; rm -rf iTorch 99 | 100 | # [ pyOSC ] 101 | 102 | # this version is more up to date than pip 103 | RUN pip --no-cache-dir install git+git://github.com/ptone/pyosc.git 104 | 105 | # [ dlib ] 106 | 107 | RUN apt-get update; \ 108 | apt-get install -y \ 109 | libopenblas-dev \ 110 | libboost-python-dev \ 111 | liblapack-dev; \ 112 | git clone https://github.com/davisking/dlib.git /root/dlib; \ 113 | cd /root/dlib; mkdir build; cd build; cmake ..; cmake --build .; \ 114 | cd /root/dlib; python setup.py install; \ 115 | cd /root ; rm -rf dlib; \ 116 | apt-get clean autoclean; \ 117 | apt-get autoremove -y; \ 118 | rm -rf /var/lib/apt/lists/* 119 | 120 | # [ Multicore-TSNE ] 121 | 122 | RUN git clone https://github.com/DmitryUlyanov/Multicore-TSNE.git ; \ 123 | cd Multicore-TSNE ; \ 124 | pip --no-cache-dir install -r requirements.txt ; \ 125 | python setup.py install ; \ 126 | cd .. ; rm -rf Multicore-TSNE 127 | 128 | RUN pip --no-cache-dir install \ 129 | scikit-image 130 | 131 | # [ lapjv ] 132 | 133 | RUN pip --no-cache-dir install \ 134 | cython \ 135 | git+git://github.com/gatagat/lapjv.git 136 | 137 | # [ torch-rnn ] 138 | 139 | RUN git clone https://github.com/deepmind/torch-hdf5 ; \ 140 | cd torch-hdf5 ; \ 141 | luarocks make hdf5-0-0.rockspec ; \ 142 | cd .. ; \ 143 | rm -rf torch-hdf5 144 | 145 | # [ neuraltalk2 ] 146 | 147 | RUN apt-get update; \ 148 | apt-get install -y \ 149 | libprotobuf-dev \ 150 | protobuf-compiler ;\ 151 | luarocks install loadcaffe -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Docker Pulls](https://img.shields.io/docker/pulls/kylemcdonald/ml-notebook.svg)](https://hub.docker.com/r/kylemcdonald/ml-notebook/) [![Docker Stars](https://img.shields.io/docker/stars/kylemcdonald/ml-notebook.svg)](https://hub.docker.com/r/kylemcdonald/ml-notebook/) 2 | 3 | ml-notebook 4 | =========== 5 | 6 | This project is aimed at providing an accessible and reproducible environment for a variety of machine learning toolkits, with a focus on deep learning toolkits. Instead of asking you to follow a set of complex setup instructions, ml-notebook asks you to wait while a tested, pre-built image is installed. 7 | 8 | The following tools are available inside the Ubuntu 14.04 image, with Jupyter as an interface: 9 | 10 | General: 11 | - [matplotlib](http://matplotlib.org/)* 12 | - [scipy](http://www.scipy.org/)* 13 | - [numpy](http://www.numpy.org/)* 14 | - [pyOSC](https://github.com/ptone/pyosc/blob/master/OSC.py) 15 | - [Jupyter](http://jupyter.org/) 16 | 17 | Machine Learning: 18 | - [gensim](https://radimrehurek.com/gensim/) 19 | - [scikit-learn](http://scikit-learn.org/stable/) 20 | - [scipy](http://www.scipy.org/)* 21 | - [pandas](http://pandas.pydata.org/)* 22 | 23 | Deep Learning: 24 | - [Caffe](http://caffe.berkeleyvision.org/) (with pycaffe) 25 | - [Chainer](http://chainer.org/) 26 | - [TensorFlow](http://tensorflow.org) 27 | - [Theano](http://deeplearning.net/software/theano/) (with [Keras](http://keras.io/) and [Lasagne](https://github.com/Lasagne/Lasagne)) 28 | - [Torch](http://torch.ch/) 29 | 30 | *These are requirements of other libraries, but also interesting in their own right. 31 | 32 | Usage 33 | ----- 34 | 35 | This is only tested this on OSX. Something similar should work on Linux, and possibly Windows with some changes. 36 | 37 | 1. Install [Docker](https://www.docker.com/). On Mac, use Docker for Mac. 38 | 2. Clone this repository. `git clone https://github.com/kylemcdonald/ml-notebook.git && cd ml-notebook` 39 | 3. (Optional) Run `./update.sh` __Note__: this downloads 2+GB of data and examples. If you just want to look around, browse [ml-examples](https://github.com/kylemcdonald/ml-examples). 40 | 4. Run `./run.sh` __Note__: this downloads another 2+GB of data, a pre-built image from Docker. 41 | 42 | Type `Ctrl-D` to exit the ml-notebook Docker. If you accidentally close the Terminal, the Jupyter notebook will keep running in the background. Whenever you want to run the environment again, just call `./run.sh`. Calling `./run.sh` when ml-notebook is running in the background will restart ml-notebook. 43 | 44 | Acknowledgements 45 | ---------------- 46 | 47 | Some of the deep learning toolkits are built based on Dockerfiles from [Kaixhin](https://github.com/Kaixhin/dockerfiles/). 48 | -------------------------------------------------------------------------------- /package.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source utils.sh 4 | 5 | checkdocker 6 | checkimage $IMAGE $IMAGE_FILE 7 | 8 | if [ ! -e $IMAGE_FILE ] ; then 9 | echo "Saving $IMAGE to $IMAGE_FILE" 10 | docker save $IMAGE > $IMAGE_FILE 11 | fi 12 | echo "$IMAGE_FILE is ready, zipping everything..." 13 | 14 | if [ ! -e $LOG_FILE ] ; then 15 | rm $LOG_FILE 16 | fi 17 | 18 | HASH=`git rev-parse HEAD | cut -c-8` 19 | ZIP_FILE="../ml-notebook-$HASH.zip" 20 | 21 | tar cf -q "$ZIP_FILE" ./ 22 | 23 | echo "Package is ready: $ZIP_FILE" -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source utils.sh 4 | 5 | # if the container wasn't automatically removed last time, remove it 6 | checkcontainer() { 7 | CONTAINER=$1 8 | if ( docker stats --no-stream $CONTAINER &>/dev/null ) ; then 9 | echo "The container is already running, stopping it..." 10 | docker stop $CONTAINER &>/dev/null 11 | if ( docker stats --no-stream $CONTAINER &>/dev/null ) ; then 12 | echo "The container still exists, removing it..." 13 | docker rm $CONTAINER &>/dev/null 14 | fi 15 | fi 16 | } 17 | 18 | # open the browser once the log file has the token 19 | openavailable() { 20 | HOST_IP=$1 21 | LOG_FILE=$2 22 | # remove the old log file if it exists 23 | [ -e $LOG_FILE ] && rm $LOG_FILE 24 | # wait up to 10 seconds for jupyter to start 25 | # this way this loop doesn't run indefinitely 26 | for i in `seq 1 10`; do 27 | if [ -e $LOG_FILE ]; then 28 | JUPYTER_URL=`grep -Eoh "http://$HOST_IP.+" $LOG_FILE` 29 | if [ -n "$JUPYTER_URL" ]; then 30 | open "$JUPYTER_URL" 31 | exit 32 | fi 33 | fi 34 | sleep 1 35 | done 36 | } 37 | 38 | # run docker 39 | runcontainer() { 40 | CONTAINER=$1 41 | HOST_IP=$2 42 | JUPYTER_PORT=$3 43 | DIR=`pwd` 44 | docker run -ti \ 45 | --rm \ 46 | --name "$CONTAINER" \ 47 | --publish "$JUPYTER_PORT:$JUPYTER_PORT" \ 48 | --env "HOST_IP=$HOST_IP" \ 49 | --workdir "/root/shared" \ 50 | --volume "$DIR/shared:/root/shared" \ 51 | $IMAGE \ 52 | /bin/bash -c " \ 53 | jupyter notebook --ip='*' >jupyter.log 2>&1 & sleep 1 ; \ 54 | echo 'Jupyter is writing to jupyter.log and running at:' ; \ 55 | sleep 1 ; grep 'http://$HOST_IP' jupyter.log ; \ 56 | echo 'To quit type \"exit\" or Control-D.' ; \ 57 | bash" 58 | } 59 | 60 | JUPYTER_PORT="8888" 61 | HOST_IP="localhost" 62 | 63 | checkdocker 64 | checkimage $IMAGE $IMAGE_FILE 65 | checkcontainer $CONTAINER 66 | openavailable $HOST_IP $LOG_FILE & 67 | runcontainer $CONTAINER $HOST_IP $JUPYTER_PORT -------------------------------------------------------------------------------- /shared/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylemcdonald/ml-notebook/8323b7d3a3020ec3d775fc548e40d219ad313495/shared/.gitkeep -------------------------------------------------------------------------------- /update.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source utils.sh 4 | 5 | checkdocker 6 | checkimage $IMAGE $IMAGE_FILE 7 | 8 | docker pull kylemcdonald/ml-notebook 9 | 10 | git pull origin master 11 | 12 | # into shared 13 | cd shared 14 | 15 | if [ ! -d ml-examples ] ; then 16 | git clone https://github.com/kylemcdonald/ml-examples.git 17 | fi 18 | cd ml-examples 19 | ./setup.sh 20 | cd - &>/dev/null 21 | 22 | if [ ! -d ml-examples-oF ] ; then 23 | git clone https://github.com/yusuketomoto/ml-examples-oF.git 24 | fi 25 | cd ml-examples-oF 26 | ./setup.sh 27 | cd - &>/dev/null 28 | 29 | cd .. 30 | # out of shared -------------------------------------------------------------------------------- /utils.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | CONTAINER="ml-notebook" 4 | IMAGE="kylemcdonald/$CONTAINER" 5 | IMAGE_FILE="$CONTAINER.tar" 6 | LOG_FILE="shared/jupyter.log" 7 | 8 | # check if docker is running 9 | checkdocker() { 10 | docker ps &> /dev/null 11 | DOCKER_RUNNING_CODE=$? 12 | if [ $DOCKER_RUNNING_CODE -ne 0 ]; then 13 | echo "Docker is not running. Start Docker first." 14 | exit 15 | fi 16 | } 17 | 18 | # download the image if necessary 19 | checkimage() { 20 | IMAGE=$1 21 | IMAGE_FILE=$2 22 | if ! ( docker images | grep "$IMAGE" &>/dev/null ) ; then 23 | if [ -e $IMAGE_FILE ]; then 24 | echo "The image will be loaded from $IMAGE_FILE (first time only, ~1 minute)." 25 | docker load < $IMAGE_FILE 26 | else 27 | echo "The image will be downloaded from docker (first time only, a few minutes)." 28 | docker pull $IMAGE 29 | fi 30 | fi 31 | } --------------------------------------------------------------------------------