├── .gitignore ├── settings.json ├── Makefile ├── Dockerfile ├── README.md └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/* 2 | !.vscode/settings.json 3 | !.vscode/tasks.json 4 | !.vscode/launch.json 5 | !.vscode/extensions.json 6 | *.code-workspace 7 | 8 | # Local History for Visual Studio Code 9 | .history/ 10 | -------------------------------------------------------------------------------- /settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "explorer.confirmDragAndDrop": false, 3 | "editor.formatOnType": true, 4 | "python.formatting.provider": "black", 5 | "editor.formatOnSave": true, 6 | "python.linting.pylintArgs": [ 7 | "--generated-members=numpy.*,torch.*,cv2.*", 8 | "--disable=C0111,C0115,C0116" 9 | ], 10 | "python.linting.flake8Args": [ 11 | "--max-line-length=120" 12 | ], 13 | "extensions.autoCheckUpdates": false, 14 | "extensions.autoUpdate": false 15 | } -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | build: 2 | docker build -f Dockerfile -t mldev . 3 | 4 | code: 5 | docker run \ 6 | --network=host \ 7 | -v $(WORKSPACE):/home/root/workspace \ 8 | -ti mldev \ 9 | /bin/zsh -c \ 10 | "source activate ml \ 11 | && code-server /home/root/workspace/ --verbose --port $(CPORT) --host 0.0.0.0 --auth none" 12 | 13 | jupyter: 14 | docker run \ 15 | --network=host \ 16 | -v $(WORKSPACE):/home/root/workspace \ 17 | -ti mldev \ 18 | /bin/zsh -c \ 19 | "source activate ml \ 20 | && cd /home/root/workspace/ \ 21 | && SHELL=/bin/zsh jupyter-lab --ip=0.0.0.0 --no-browser --port $(JPORT) --allow-root" 22 | 23 | code2: 24 | docker run \ 25 | --network=host \ 26 | -v $(WORKSPACE):/home/root/workspace \ 27 | -t mldev \ 28 | /bin/zsh -c \ 29 | "source activate ml \ 30 | && TERM=xterm code-server /home/root/workspace/ --log trace --verbose --port $(CPORT) --host 0.0.0.0 --auth none" 31 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:10.2-cudnn7-runtime-ubuntu18.04 2 | ENV PATH="/root/miniconda3/bin:${PATH}" 3 | ARG PATH="/root/miniconda3/bin:${PATH}" 4 | 5 | RUN apt-get update 6 | RUN apt-get install -y \ 7 | build-essential \ 8 | cmake \ 9 | curl \ 10 | gcc \ 11 | git \ 12 | libjpeg-dev \ 13 | libpng-dev \ 14 | libtiff-dev \ 15 | libgtk2.0-dev \ 16 | locales \ 17 | net-tools \ 18 | wget \ 19 | zsh \ 20 | && rm -rf /var/lib/apt/lists/* 21 | 22 | ENV ZSH_THEME agnoster 23 | RUN locale-gen en_US.UTF-8 24 | RUN /bin/sh -c chsh -s /bin/zsh 25 | 26 | ENV SHELL /bin/zsh 27 | ENV LANG=en_US.UTF-8 28 | 29 | RUN wget https://github.com/robbyrussell/oh-my-zsh/raw/master/tools/install.sh -O - | zsh || true 30 | 31 | RUN wget \ 32 | https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ 33 | && mkdir /root/.conda \ 34 | && sh Miniconda3-latest-Linux-x86_64.sh -b \ 35 | && rm -f Miniconda3-latest-Linux-x86_64.sh 36 | RUN conda create -y -n ml python=3.7 37 | 38 | COPY . / 39 | 40 | RUN /bin/zsh -c "source activate ml && pip install -r requirements.txt" 41 | 42 | RUN curl -fsSL https://code-server.dev/install.sh | sh 43 | RUN mkdir -p /root/.code-server/extensions 44 | 45 | RUN wget https://github.com/microsoft/vscode-python/releases/download/2020.5.86806/ms-python-release.vsix 46 | RUN code-server --install-extension ms-python-release.vsix 47 | 48 | RUN mkdir -p /root/.local/share/code-server/User/ 49 | COPY settings.json /root/.local/share/code-server/User/ 50 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ML Development Environment 2 | 3 | A fully fledged development environment for OSX, Windows, Linux 4 | 5 | ### Step - 1: Install docker 6 | 7 | You need docker! Check out https://docs.docker.com/get-docker/ on information on how to install docker for your system. 8 | 9 | 10 | ### Step - 2: NVIDIA docker runtime (not needed if you don't want to use GPUs) 11 | 12 | If you have NVIDIA drivers installed, you need the NVIDIA runtime to use GPUs in the development environment. 13 | Run the following commands if you are on Ubuntu to set up the NVIDIA runtimes. 14 | 15 | ``` 16 | # Add the package repositories 17 | distribution=$(. /etc/os-release;echo $ID$VERSION_ID) 18 | curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - 19 | curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list 20 | 21 | sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit 22 | sudo systemctl restart docker 23 | ``` 24 | 25 | For more information about the NVIDIA docker runtime, take a look here: https://github.com/NVIDIA/nvidia-docker 26 | 27 | ### Step - 3: Build the container 28 | 29 | ``` 30 | make build 31 | ``` 32 | 33 | ### Step - 4: Start the coding environment 34 | 35 | ``` 36 | WORKSPACE=[PATH_TO_YOUR_CODEBASE] CPORT=[PORT] make code 37 | ``` 38 | 39 | Where ```PATH_TO_YOUR_CODEBASE``` is the path to your code base where all the scripts/notebooks are located and ```PORT``` is the port you want to run the IDE on 40 | 41 | e.g. ```WORKSPACE=/home/abhishek/workspace/bert-sentiment CPORT=10012 make code``` 42 | 43 | ### Step - 5: Open the URL in broswer 44 | 45 | ```http://127.0.0.1:10012/``` 46 | 47 | And have fun coding! 48 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | albumentations==0.4.3 2 | alembic==1.4.2 3 | appdirs==1.4.3 4 | astor==0.8.1 5 | astroid==2.4.2 6 | astunparse==1.6.3 7 | attrs==19.3.0 8 | bcrypt==3.1.7 9 | black==19.10b0 10 | bleach==3.1.0 11 | boto3==1.10.48 12 | botocore==1.13.48 13 | cached-property==1.5.1 14 | cachetools==4.1.0 15 | catalyst==20.7 16 | catboost==0.20.2 17 | certifi==2020.6.20 18 | cffi==1.13.2 19 | chardet==3.0.4 20 | Click==7.0 21 | cliff==3.3.0 22 | cloudpickle==1.4.1 23 | cmaes==0.5.1 24 | cmd2==1.2.1 25 | colorama==0.4.3 26 | colorlog==4.1.0 27 | commonmark==0.9.1 28 | cryptography==2.9.2 29 | cycler==0.10.0 30 | Cython==0.29.21 31 | dask==2.17.2 32 | decorator==4.4.1 33 | defusedxml==0.6.0 34 | deprecation==2.1.0 35 | distributed==2.17.0 36 | distro==1.5.0 37 | docformatter==1.3 38 | docker==4.2.1 39 | docker-compose==1.26.0 40 | dockerpty==0.4.1 41 | docopt==0.6.2 42 | docutils==0.15.2 43 | easydict==1.9 44 | efficientnet-pytorch==0.6.3 45 | emoji==0.5.4 46 | entrypoints==0.3 47 | fastparquet==0.3.3 48 | filelock==3.0.12 49 | flake8==3.7.8 50 | flake8-bugbear==19.8.0 51 | Flask==1.1.1 52 | fsspec==0.7.4 53 | future==0.18.2 54 | gast==0.3.3 55 | gitdb==4.0.5 56 | gitdb2==2.0.5 57 | GitPython==3.1.7 58 | google-api-core==1.16.0 59 | google-api-python-client==1.8.0 60 | google-auth==1.13.1 61 | google-auth-httplib2==0.0.3 62 | google-auth-oauthlib==0.4.1 63 | google-pasta==0.1.8 64 | googleapis-common-protos==1.51.0 65 | graphviz==0.13.2 66 | grpcio==1.26.0 67 | gunicorn==20.0.4 68 | h5py==2.10.0 69 | HeapDict==1.0.1 70 | httplib2==0.17.2 71 | hyperopt==0.2.4 72 | idna==2.8 73 | imageio==2.6.1 74 | imagesize==1.2.0 75 | imgaug==0.2.6 76 | importlib-metadata==1.7.0 77 | ipykernel==5.1.3 78 | ipython==7.11.1 79 | ipython-genutils==0.2.0 80 | ipywidgets==7.5.1 81 | isort==4.3.21 82 | iterative-stratification==0.1.6 83 | itsdangerous==1.1.0 84 | jedi==0.15.2 85 | jeepney==0.4.3 86 | Jinja2==2.10.3 87 | jmespath==0.9.4 88 | joblib==0.14.1 89 | json5==0.8.5 90 | jsonpickle==1.4.1 91 | jsonschema==3.2.0 92 | jupyter==1.0.0 93 | jupyter-client==5.3.4 94 | jupyter-console==6.0.0 95 | jupyter-core==4.6.1 96 | jupyterlab==1.2.4 97 | jupyterlab-server==1.0.6 98 | kaggle==1.5.6 99 | Keras-Applications==1.0.8 100 | Keras-Preprocessing==1.1.0 101 | keyring==21.2.0 102 | kiwisolver==1.1.0 103 | lazy-object-proxy==1.4.3 104 | lightgbm==2.3.1 105 | llvmlite==0.31.0 106 | locket==0.2.0 107 | lycon==0.2.0 108 | Mako==1.1.3 109 | Markdown==3.1.1 110 | MarkupSafe==1.1.1 111 | matplotlib==3.1.2 112 | mccabe==0.6.1 113 | mistune==0.8.4 114 | more-itertools==8.0.2 115 | msgpack==1.0.0 116 | munch==2.5.0 117 | nbconvert==5.6.1 118 | nbformat==5.0.3 119 | networkx==2.4 120 | nltk==3.4.5 121 | notebook==6.0.2 122 | numba==0.48.0 123 | numpy==1.17.4 124 | oauth2client==4.1.3 125 | oauthlib==3.1.0 126 | olefile==0.46 127 | opencv-python==4.3.0.36 128 | opencv-python-headless==4.2.0.32 129 | opt-einsum==3.1.0 130 | optuna==1.5.0 131 | packaging==20.3 132 | pandas==1.0.4 133 | pandocfilters==1.4.2 134 | paramiko==2.7.1 135 | parso==0.5.2 136 | partd==1.1.0 137 | pathspec==0.8.0 138 | patsy==0.5.1 139 | pbr==5.4.5 140 | pexpect==4.7.0 141 | pickleshare==0.7.5 142 | Pillow-SIMD==7.0.0.post3 143 | pkginfo==1.5.0.1 144 | plotly==4.4.1 145 | pluggy==0.13.1 146 | pretrainedmodels==0.7.4 147 | prettytable==0.7.2 148 | prometheus-client==0.7.1 149 | prompt-toolkit==2.0.10 150 | protobuf==3.11.2 151 | psutil==5.7.0 152 | ptyprocess==0.6.0 153 | py==1.8.1 154 | py-cpuinfo==5.0.0 155 | py-gfm==0.1.4 156 | py-rouge==1.1 157 | pyaml==20.4.0 158 | pyarrow==0.16.0 159 | pyasn1==0.4.8 160 | pyasn1-modules==0.2.8 161 | pycodestyle==2.5.0 162 | pycparser==2.19 163 | pyflakes==2.1.1 164 | Pygments==2.5.2 165 | pylint==2.5.3 166 | PyNaCl==1.4.0 167 | pyparsing==2.4.6 168 | pyperclip==1.8.0 169 | pyrsistent==0.15.7 170 | pytest==5.3.2 171 | python-dateutil==2.8.1 172 | python-dotenv==0.13.0 173 | python-editor==1.0.4 174 | python-slugify==4.0.0 175 | pytorch-model-summary==0.1.1 176 | pytz==2019.3 177 | pyvips==2.1.12 178 | PyWavelets==1.1.1 179 | PyYAML==5.3 180 | pyzmq==18.1.1 181 | qtconsole==4.6.0 182 | readme-renderer==26.0 183 | recommonmark==0.6.0 184 | regex==2020.1.8 185 | requests==2.24.0 186 | requests-mock==1.7.0 187 | requests-oauthlib==1.3.0 188 | requests-toolbelt==0.9.1 189 | retrying==1.3.3 190 | rsa==4.0 191 | s3transfer==0.2.1 192 | sacred==0.8.1 193 | sacremoses==0.0.38 194 | sandesh==0.21 195 | scikit-image==0.16.2 196 | scikit-learn==0.23.1 197 | scikit-optimize==0.7.4 198 | scipy==1.4.1 199 | seaborn==0.10.0 200 | SecretStorage==3.1.2 201 | Send2Trash==1.5.0 202 | sentencepiece==0.1.85 203 | sh==1.12.14 204 | six==1.13.0 205 | smmap==3.0.2 206 | smmap2==3.0.1 207 | snowballstemmer==2.0.0 208 | sortedcontainers==2.1.0 209 | Sphinx==2.2.0 210 | SQLAlchemy==1.3.18 211 | statsmodels==0.11.1 212 | stevedore==3.2.0 213 | subword-nmt==0.3.7 214 | tblib==1.6.0 215 | tensorboard==2.2.2 216 | tensorboard-plugin-wit==1.7.0 217 | tensorboardX==2.1 218 | tensorflow==2.2.0 219 | tensorflow-estimator==2.2.0 220 | termcolor==1.1.0 221 | terminado==0.8.3 222 | testpath==0.4.4 223 | text-unidecode==1.3 224 | texttable==1.6.2 225 | threadpoolctl==2.1.0 226 | thrift==0.13.0 227 | tokenizers==0.7.0 228 | toml==0.10.0 229 | toolz==0.10.0 230 | torch==1.6.0 231 | torchvision==0.6.1 232 | tornado==6.0.3 233 | tqdm==4.46.1 234 | traitlets==4.3.3 235 | transformers==2.11.0 236 | tsfresh==0.16.0 237 | twine==3.1.1 238 | typed-ast==1.4.1 239 | typing-extensions==3.7.4.1 240 | Unidecode==1.1.1 241 | untokenize==0.1.1 242 | uritemplate==3.0.1 243 | urllib3==1.24.3 244 | wcwidth==0.1.8 245 | webencodings==0.5.1 246 | websocket-client==0.56.0 247 | websocket-server==0.4 248 | Werkzeug==0.16.0 249 | widgetsnbextension==3.5.1 250 | wtfml==0.0.3 251 | xgboost==0.90 252 | zict==2.0.0 253 | zipp==0.6.0 254 | --------------------------------------------------------------------------------