├── .gitignore ├── keys ├── config ├── ssh_sftp_key.pub └── ssh_sftp_key ├── notebooks ├── model_output │ └── weights-cnn-fashion-mnist.hdf5 └── conv-net-in-keras.ipynb ├── Dockerfile.mlflowserver ├── scripts ├── copy_know_hosts.sh └── create_experiments.sh ├── environment.yml ├── Dockerfile.jupyterlab ├── docker-compose.yml ├── LICENSE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | **/.ipynb* 2 | storage/** 3 | -------------------------------------------------------------------------------- /keys/config: -------------------------------------------------------------------------------- 1 | Host ekholabs-sftp 2 | HostName ekholabs-sftp 3 | PasswordAuthentication no 4 | IdentityFile ~/.ssh/keys/ssh_sftp_key 5 | Port 22 6 | -------------------------------------------------------------------------------- /notebooks/model_output/weights-cnn-fashion-mnist.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/artificially-ai/ai-engineering/HEAD/notebooks/model_output/weights-cnn-fashion-mnist.hdf5 -------------------------------------------------------------------------------- /Dockerfile.mlflowserver: -------------------------------------------------------------------------------- 1 | FROM python:3 2 | 3 | MAINTAINER Wilder Rodrigues (wilder.rodrigues@ekholabs.ai) 4 | 5 | RUN pip install mlflow pysftp 6 | 7 | ENV LC_ALL=C.UTF-8 8 | ENV LANG=C.UTF-8 9 | -------------------------------------------------------------------------------- /scripts/copy_know_hosts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "Copying known_hosts from 'ekholabs-mlflow' to 'ekholabs-jupyterlab'" 4 | docker cp ekholabs-mlflow:/root/.ssh/known_hosts . 5 | docker cp known_hosts ekholabs-jupyterlab:/home/jovyan/.ssh/known_hosts 6 | rm known_hosts 7 | echo "..." 8 | echo "Done!" 9 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: MLflow-Service 2 | dependencies: 3 | - python=3.6 4 | - pandas 5 | - numpy 6 | - keras 7 | - scipy 8 | - nltk 9 | - gensim 10 | - opencv 11 | - tensorflow 12 | - pip: 13 | - sklearn 14 | - matplotlib 15 | - jupyterlab 16 | - git+https://www.github.com/keras-team/keras-contrib.git 17 | - pydub 18 | - gym 19 | - mlflow 20 | - pysftp 21 | -------------------------------------------------------------------------------- /Dockerfile.jupyterlab: -------------------------------------------------------------------------------- 1 | FROM jupyter/datascience-notebook 2 | 3 | MAINTAINER Wilder Rodrigues (wilder.rodrigues@ekholabs.ai) 4 | 5 | USER $NB_USER 6 | 7 | RUN mkdir ~/.ssh 8 | 9 | # Install TensorFlow 10 | RUN conda install -c conda-forge tensorflow -y && \ 11 | conda install -c conda-forge numpy keras nltk gensim opencv -y 12 | 13 | # Install Keras Contrib 14 | RUN pip install npm jupyterlab mlflow pysftp git+https://www.github.com/keras-team/keras-contrib.git 15 | 16 | RUN jupyter serverextension enable --py jupyterlab 17 | -------------------------------------------------------------------------------- /scripts/create_experiments.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "Creating experiments on MLflow Tracking Server..." 4 | echo "..." 5 | echo "Creating experiment using Minio as storage..." 6 | docker exec -it ekholabs-mlflow /bin/sh -c "mlflow experiments create ekholabs-minio-storage --artifact-location s3://ai-models" 7 | echo "..." 8 | echo "Creating experiment using SFTP as storage..." 9 | docker exec -it ekholabs-mlflow /bin/sh -c "mlflow experiments create ekholabs-sftp-storage --artifact-location sftp://ekho@ekholabs-sftp/storage" 10 | echo "..." 11 | echo "Done!" 12 | -------------------------------------------------------------------------------- /keys/ssh_sftp_key.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDzYFcAqhpy94aSTY9a5dzh4pqBv3s1jTDgNyA74a+0CxnJKN9kOvoAq0j8pup9Uq0UU7U8s/DqLJJEuB6urwFz7jblDaFQwwR1KZF4BMBGbTSQDDV8lT3FwF42K+mjvgstSYTnKbmf25PMDSwudIwQ8+4hccPPQhkrGRDa37H7nTrvqBXyqXxBcU+OiXdZeN8oqz1SC12n6Zdb6s8HIyBE3UVWltfkGwn/N8cgU6Br3zKc2EewFVa2FcWPN9ezfFL6oUjUYmIf7NLxToKxNQ4z0jFignGFXVyZ30LjuAw/8z6T0HtHEJ7mhFuE8cD4XurRwE49SchLCYxmiOPngi+MHrn3q/0HMSJIPRiqp8T6IxfVjotV+eSXNSPx4NzT0c7OgWXyIauxyrEEpvRAmTEnLD+sv7Y/B09wsSyW/BzRIrMU+QoSSo+l+2D+tO3Ko0o48lr34Stl8V2dgV0qqocYdp1U1+I9+cXF63/+073x0Mxn6XWY4vUoB4/JxlyjG5LOk7GPlZcAuUC24WMoFo7A9LRgKYSEyJVNFD8TJj4cjdR2/JWd50n1AlirfFQFWJalu5DdFakbb8GUX4p0hjQjSNcFchOnm1gnowiQ1VyXIaHsZ/2+dmOsCN1kFIQwtSRyMd5RCAmJe6kAuR+98dXXanVk85W3wcpfoP+U4Troxw== ekho@3bbe745a4293 2 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | ekholabs-minio: 4 | image: minio/minio 5 | networks: 6 | - mlflow-net 7 | container_name: ekholabs-minio 8 | working_dir: "/ekholabs-minio/storage" 9 | volumes: 10 | - ./storage/minio:/ekholabs-minio/storage 11 | expose: 12 | - "9000" 13 | ports: 14 | - "9000:9000" 15 | environment: 16 | MINIO_ACCESS_KEY: ekholabs-minio 17 | MINIO_SECRET_KEY: ekholabs-minio-pass 18 | command: server /ekholabs-minio/storage 19 | ekholabs-sftp: 20 | image: atmoz/sftp 21 | networks: 22 | - mlflow-net 23 | container_name: ekholabs-sftp 24 | working_dir: "/home/ekho" 25 | volumes: 26 | - ./storage/sftp:/home/ekho/storage 27 | - ./keys/ssh_sftp_key.pub:/home/ekho/.ssh/keys/ssh_sftp_key.pub:ro 28 | expose: 29 | - "22" 30 | ports: 31 | - "22:22" 32 | command: ekho::1001 labs:pass:1002 33 | ekholabs-mlflow: 34 | image: "ekholabs/mlflow" 35 | container_name: "ekholabs-mlflow" 36 | networks: 37 | - mlflow-net 38 | build: 39 | context: . 40 | dockerfile: Dockerfile.mlflowserver 41 | depends_on: 42 | - ekholabs-sftp 43 | working_dir: "/ekholabs-mlflow" 44 | volumes: 45 | - ./storage:/ekholabs-mlflow 46 | - ./keys/ssh_sftp_key:/root/.ssh/keys/ssh_sftp_key:ro 47 | - ./keys/config:/root/.ssh/config:ro 48 | environment: 49 | MLFLOW_S3_ENDPOINT_URL: http://ekholabs-minio:9000 50 | AWS_ACCESS_KEY_ID: ekholabs-minio 51 | AWS_SECRET_ACCESS_KEY: ekholabs-minio-pass 52 | expose: 53 | - "5500" 54 | ports: 55 | - "5500:5500" 56 | command: > 57 | bash -c "sleep 3 58 | && ssh-keyscan -H ekholabs-sftp >> ~/.ssh/known_hosts 59 | && mlflow server --host 0.0.0.0 --port 5500 --file-store /ekholabs-mlflow/mlruns" 60 | ekholabs-jupyterlab: 61 | image: "ekholabs/jupyterlab" 62 | container_name: "ekholabs-jupyterlab" 63 | networks: 64 | - mlflow-net 65 | build: 66 | context: . 67 | dockerfile: Dockerfile.jupyterlab 68 | working_dir: "/ekholabs-notebooks" 69 | volumes: 70 | - ./notebooks:/ekholabs-notebooks 71 | - ./keys/ssh_sftp_key:/home/jovyan/.ssh/keys/ssh_sftp_key:ro 72 | - ./keys/config:/home/jovyan/.ssh/config:ro 73 | environment: 74 | MLFLOW_S3_ENDPOINT_URL: http://ekholabs-minio:9000 75 | AWS_ACCESS_KEY_ID: ekholabs-minio 76 | AWS_SECRET_ACCESS_KEY: ekholabs-minio-pass 77 | expose: 78 | - "8888" 79 | ports: 80 | - "8991:8888" 81 | 82 | networks: 83 | mlflow-net: 84 | -------------------------------------------------------------------------------- /keys/ssh_sftp_key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIJKgIBAAKCAgEA82BXAKoacveGkk2PWuXc4eKagb97NY0w4DcgO+GvtAsZySjf 3 | ZDr6AKtI/KbqfVKtFFO1PLPw6iySRLgerq8Bc+425Q2hUMMEdSmReATARm00kAw1 4 | fJU9xcBeNivpo74LLUmE5ym5n9uTzA0sLnSMEPPuIXHDz0IZKxkQ2t+x+50676gV 5 | 8ql8QXFPjol3WXjfKKs9Ugtdp+mXW+rPByMgRN1FVpbX5BsJ/zfHIFOga98ynNhH 6 | sBVWthXFjzfXs3xS+qFI1GJiH+zS8U6CsTUOM9IxYoJxhV1cmd9C47gMP/M+k9B7 7 | RxCe5oRbhPHA+F7q0cBOPUnISwmMZojj54IvjB6596v9BzEiSD0YqqfE+iMX1Y6L 8 | VfnklzUj8eDc09HOzoFl8iGrscqxBKb0QJkxJyw/rL+2PwdPcLEslvwc0SKzFPkK 9 | EkqPpftg/rTtyqNKOPJa9+ErZfFdnYFdKqqHGHadVNfiPfnFxet//tO98dDMZ+l1 10 | mOL1KAePycZcoxuSzpOxj5WXALlAtuFjKBaOwPS0YCmEhMiVTRQ/EyY+HI3UdvyV 11 | nedJ9QJYq3xUBViWpbuQ3RWpG2/BlF+KdIY0I0jXBXITp5tYJ6MIkNVclyGh7Gf9 12 | vnZjrAjdZBSEMLUkcjHeUQgJiXupALkfvfHV12p1ZPOVt8HKX6D/lOE66McCAwEA 13 | AQKCAgEAhHtOAPuX2Dtr0tpLjgSQn5m116k/TaBlGbiIYHBzL8PXLoyERlBgZNrY 14 | qUfHX58Qvn7uqnOdUiwfFy9O7HIxjUUANuGbjZYLKCgPmtCjRfMOaNdsijvHW41i 15 | +4Qd89dwyB88LIWnGSDzLGGDSmqBEH1ohGYkW/80KiTD6xFYFoq5WVobUHP4QruG 16 | viWQhXDpsX69q5qqScw7a3q/Zbg2SaZVp1QHg5FAeOCJEczUB8SBDaYiJ/UFtarW 17 | Vybj0T19AhenyI0hsxU8osTFGeeyqnziCyjM2PdAgDZreABpEqq7xsT3RnECvqMQ 18 | GqdrDwjENJfFP1GZqVcySWJ9BTn13PXTRfoj1eOalG6Gi2PxxHHUoLN/zG0USWw4 19 | MFOKXCdOqpDSlvpS0BleVx7drGaQFWQlc77lM7ggzWR0TFcxvp2nCBEhdwLbiFuE 20 | YEhorXTCgfYlWjmHGkIhJO/1PmE/dprwAS1vW98r57SYBI/EEbr8DQaR4thsyA62 21 | xky/pi2LaKfJR4GbngFulYzmD9GkRWgD2HFDDTuV0bG/DSKwsXeDrEo4PcCjpx6s 22 | zR3eJPdrzYf/8J8gexfhDkY97KnsfDqSGIl/9P0rPyKiTqyi6TTV1eIvfKPCuSUZ 23 | mCc5YP+aVf3oglOc8JKJ7Xl+AOOiQr8Li4lK42yKKpkGIjmhHeECggEBAP5+6Nr/ 24 | 8nS1agF6UJp2jHWXrCti4Zr9t6ep3JX0rfrUbybSX+NlJW+iAZauEUgEsK1C2pvs 25 | DpreBmRgxO85FXE7gwjUdmPw/cqKcqsn8EHBG2lyX9D5KP8PDul1RBv7u5szUvdO 26 | r+ZxX3ObufqXIP4TNz+cStXHjud6vndENgWMP1e9VMHYhzP8eRNi6P5Hn8Urrczc 27 | IJza5oRBEKLv7vc0pQ8cTBfLEbAbmcXkNGz2vzX8lb6+5aemLnEAC+fvsBEML9qY 28 | +2h5TP0jq11J/LXTChUR0mWj49lYzqkFrlsdkbMNrWVlWCpyXuBJ4kcd24Eh+qKT 29 | 4+zgFuHnC7MKkLcCggEBAPTQmtuumRlzP0k9E3Hh7uz2tfZRUmL1UmgHWAmbwzsj 30 | 81TihZmXH5M09F9/XEW5gzeeBVoGjGFcFbHBi81PgIocHYXLVT6kfUzTcs8sv8i0 31 | JN79A5DxmzjDmj8NenSj+iKsa13tn+WI5I8+eAOQVWs9zO+FLcz/5gpIdZg0KyyP 32 | lsR50kDueh2mTNWwaqtC0uFZzmA4fKLTeKCXhkkjiazVKsI/63gC7RerYrzvrz1o 33 | 8M5nau50ZQod86HM2613AM4aepNZeTS2ialoOjPPte2a+5ySgSvqGyeRooKRaoNz 34 | bfk5l5YLt8pAttkMFzq5M4IQCWK4q9g0ZVOhr9pfOHECggEAAb9ZVfef8JE2UyBF 35 | 4Wmi+UO7WOjq56eKVUpjS1xMvYYr5TZYljUhlqykGibD05zEocvtuFhcmU+g95t6 36 | frIDpa1k7vCWh4UmGPfuZaJ2kd0DlSvF4Kd1UpsF3TCC8TY/SYeGsFAIASsSHrcZ 37 | BdF31obHQFaQZPtjzc8X87WyMUOPQ8j3BgW4l/DJ3Ao301cmj7IHFXsFdk5CevH/ 38 | bT78gxIHeB0XoKoySuVFnwg8G+tt0K8VUCP7a7VGaFw5uLP6n4NX6MZYI5WBo9NU 39 | 3+UcKRtz3EUtDqmbbOihMqPpWJLRZ0tWv3V8r33tjNi1Bwv7b92N3QPq8rtvLMnr 40 | WKVf7QKCAQEAsIO8NEsmXkBHUDI2U/VYRQj5V/l4kptZaGJrKrphQMICD6EoPP7G 41 | qGgVDsoQqMFCiBZqu2dfTk+L4NexSj5Zarss2iwqcXk+Uuder1wPh9HDh/gb2yUp 42 | CvuVuoKoWXNlxhxVaasMj/5s9Y9QDxnBTQqJOeXniRi1L5LhHvX2bzG0vkMrwt9v 43 | svMWETkbIBKVZCBRefUggHJDVl8KVq/F85TbOKXHzIcnldpf3eRfkMKaZegRJI9N 44 | zsabxHylo+Pa88AJwzZO+x/mD/xzgqnv/bqyUkkb06L/Iz+RUd0xVEgk8f6BepFG 45 | WhDhBYv/L7sbFVskuUZ3uPLGYE+DMMtZMQKCAQEAqbnyByjpUb5AAVVxuX7l/He4 46 | +QTRK8iePJf3FNKif6xO+pootACPQf4F9nNcMcTvQQE9IcylRaFgIjpFd3sQSNob 47 | 6CONJ1exGECku4RAG4gOMX3N35p7SHbVAjhC9P9UokPF6ik4wKHdiCpnS6Eqh8SQ 48 | YiUnSrh+MGkf988ipIYN+VFCkYeyimG6a9tfEX1QmXO20MgNjjKbQM3po3VBPHyT 49 | Si1zlMxL2IcqkggD2AC6V4lg0fW+iKY2J904H1tKiUUDrGqSox/v6xcQ5pqxYg1c 50 | v+L7mbYacF0otJHD95+PP8uZ/P5+Bn0Ahe6gqPmdxSnezJtfbsS+NEHUplOHoA== 51 | -----END RSA PRIVATE KEY----- 52 | -------------------------------------------------------------------------------- /notebooks/conv-net-in-keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Convolutional Neural Network in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "Bulding a Convolutional Neural Network to classify Fashion-MNIST accessories." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Set seed for reproducibility" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import numpy as np\n", 31 | "np.random.seed(42)" 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "metadata": {}, 37 | "source": [ 38 | "#### Load dependencies" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "import os\n", 48 | "import keras\n", 49 | "from keras.datasets import fashion_mnist\n", 50 | "from keras.models import Sequential\n", 51 | "from keras.layers import Layer, Activation, Dense, Dropout, Conv2D, MaxPooling2D, Flatten, LeakyReLU, BatchNormalization\n", 52 | "from keras.callbacks import TensorBoard, ModelCheckpoint, EarlyStopping\n", 53 | "\n", 54 | "from keras import backend as K\n", 55 | "from keras.utils.generic_utils import get_custom_objects\n", 56 | "\n", 57 | "from keras_contrib.layers.advanced_activations import SineReLU" 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "metadata": {}, 63 | "source": [ 64 | "#### Load data" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "(X_train, y_train), (X_test, y_test) = fashion_mnist.load_data()" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "#### Preprocess data\n", 81 | "Flatten and normalise input data." 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "X_train = X_train.reshape(-1, 28, 28, 1)\n", 91 | "X_test = X_test.reshape(-1, 28, 28, 1)\n", 92 | "\n", 93 | "X_train = X_train.astype(\"float32\")/255.\n", 94 | "X_test = X_test.astype(\"float32\")/255." 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": null, 100 | "metadata": {}, 101 | "outputs": [], 102 | "source": [ 103 | "# One-hot encoded categories\n", 104 | "n_classes = 10\n", 105 | "y_train = keras.utils.to_categorical(y_train, n_classes)\n", 106 | "y_test = keras.utils.to_categorical(y_test, n_classes)" 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "metadata": {}, 112 | "source": [ 113 | "#### Test Predictions" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": null, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "model_output = 'model_output/weights-cnn-fashion-mnist.hdf5'\n", 123 | "saved_model = keras.models.load_model(model_output)" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": {}, 129 | "source": [ 130 | "#### Test Final Accuracy" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": null, 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [ 139 | "final_loss, final_acc = saved_model.evaluate(X_test, y_test, verbose = 1)\n", 140 | "print(\"Final loss: {0:.4f}, final accuracy: {1:.4f}\".format(final_loss, final_acc))" 141 | ] 142 | }, 143 | { 144 | "cell_type": "markdown", 145 | "metadata": {}, 146 | "source": [ 147 | "#### Log model and metrics in MLFlow" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": null, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [ 156 | "# If it doesn't show anything is because you forgot to copy the known_hosts. Please read the README again and make sure something sensible appears here.\n", 157 | "!cat ~/.ssh/known_hosts" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": null, 163 | "metadata": {}, 164 | "outputs": [], 165 | "source": [ 166 | "import mlflow\n", 167 | "import mlflow.keras\n", 168 | "\n", 169 | "mlflow.tracking.set_tracking_uri('http://ekholabs-mlflow:5500')\n", 170 | "\n", 171 | "with mlflow.start_run(experiment_id=2, source_version='0.0.1') as run:\n", 172 | " mlflow.log_param(key='backend', value='tensorflow')\n", 173 | " mlflow.log_param(key='backend_version', value='1.10.0')\n", 174 | " mlflow.log_metric('final_loss', final_loss)\n", 175 | " mlflow.log_metric('final_accuracy', final_acc)\n", 176 | " mlflow.log_artifact(model_output)\n" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": null, 182 | "metadata": {}, 183 | "outputs": [], 184 | "source": [] 185 | } 186 | ], 187 | "metadata": { 188 | "kernelspec": { 189 | "display_name": "Python 3", 190 | "language": "python", 191 | "name": "python3" 192 | }, 193 | "language_info": { 194 | "codemirror_mode": { 195 | "name": "ipython", 196 | "version": 3 197 | }, 198 | "file_extension": ".py", 199 | "mimetype": "text/x-python", 200 | "name": "python", 201 | "nbconvert_exporter": "python", 202 | "pygments_lexer": "ipython3", 203 | "version": "3.6.7" 204 | } 205 | }, 206 | "nbformat": 4, 207 | "nbformat_minor": 2 208 | } 209 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Artificial Intelligence Engineering 2 | 3 | This repository contains a Proof of Concept on how to integrate Jupyter Notebooks 4 | with MLflow, for AI models versioning and serving, and SFTP & Minio for artefacts 5 | storage. 6 | 7 | The PoC has been implemented in a way to mimic MLflow running as a remote tracking 8 | server, as a Docker container, and the SFTP and Minio also running as docker 9 | containers. To ease the way the Jupyter notebooks communicate with 10 | the MLflow tracking server, Jupyter Lab has also runs within a Docker container. 11 | 12 | All the three containers run with the assistance of Docker Compose, which also 13 | configures a custom network which is shared amongst the containers. 14 | 15 | ## Dependencies 16 | 17 | To get this PoC running on a MacBook Pro, one needs to install the following applications: 18 | 19 | * [Docker Engine](https://hub.docker.com/editions/community/docker-ce-desktop-mac) 20 | 21 | If you are not a fan of Docker and would like to have only JupyterLab and MLflow 22 | running locally, you will need the following: 23 | 24 | ## Step-by-step Setup 25 | 26 | Although you are encouraged to read all the sections below in order to understand 27 | better the whole setup, let's save some time and have a more concise step-by-step setup. 28 | 29 | 1. Run the Docker containers: 30 | - ```docker-compose up``` 31 | - The first run will take some time because it will have to pull some images and build other. 32 | 2. Once the containers are running, open another `terminal` tab and make the```copy_known_hosts.sh``` 33 | and the ```create_experiments.sh``` executables: 34 | - ```chmod +x copy_known_hosts.sh``` 35 | - ```chmod +x create_experiments.sh``` 36 | 3. Now copy the `known_hosts` to the JupytLab container: 37 | - ```./copy_known_hosts.sh``` 38 | 4. Create the experiments in the MLflow container: 39 | - ```./create_experiments.sh``` 40 | 5. Create a bucket on Minio: 41 | - Go the the Minio UI on `http://localhost:9000` 42 | - Click on the + sign on the bottom-right corner 43 | - Create a bucket called `ai-models` 44 | 6. Open JupyterLab: 45 | - Got to your browser and type `http://localhost:8991/lab` 46 | - Open the `conv-net-in-keras.ipynb` notebook and run all the cells 47 | - Go to MLflow UI on `http://localhost:5500` and check the experiments 48 | - Go to Minio UI and check the content of the bucket 49 | 50 | **If you want to understand why those steps were made, please keep reading.** 51 | 52 | ## Containers 53 | 54 | The three containers used are named as: 55 | 56 | 1. `ekholabs-minio`: 57 | - based on the `minio/minio` image. 58 | 2. `ekholabs-sftp`: 59 | - based on the `atmoz/sftp` image. 60 | 3. `ekholabs-mlflow`: 61 | - built from the `Dockerfile.mlflowserver`. 62 | 4. `ekholabs-jupyterlab`: 63 | - built from the `Dockerfile.jupyterlab`. 64 | 65 | ## Architecture 66 | 67 | A picture is worth a thousand words. When the picture is in `ASCII`, it's 68 | even better! ;) 69 | 70 | ```ascii 71 | -- MacBook Dev Environment ---------------------------------- 72 | | | 73 | | -- Docker Engine ------------------------ | 74 | | | | | 75 | | ------ | ------------ -------- | | 76 | | | User | -------> | JupyterLab | -------> | MLflow | | | 77 | | ------ | ------------ -------- | | 78 | | ^ | | / | | | 79 | | | | | / | | | 80 | | --------|------------|---------------- | | | 81 | | | | | | | 82 | | | | --------- | | | 83 | | | | | | | | 84 | | | V V | | | 85 | | | ------ ------- | | | 86 | | | | SFTP | | Minio | <---- | | 87 | | | ------ ------- | | | 88 | | | ^ | | | 89 | | | | | | | 90 | | | --------------------- | | 91 | | ----------------------------------------- | 92 | | | 93 | ------------------------------------------------------------- 94 | ``` 95 | 96 | 1. User runs models on notebooks served by JupyterLab; 97 | 2. JupyterLab notebooks store metrics, parameters and model on MLflow 98 | file storage; 99 | 3. JupyterLab notebooks store artefacts (aka model files) in the SFTP or Minio 100 | server, it depends on which experiment id is being used. The files are 101 | identified by the run id from MLflow; 102 | 4. The user can browse the experiments on MLflow via its UI. 103 | 5. The buckets kept on Minio are accessible via its UI. 104 | 105 | For the purpose of this PoC, the containers run on a local machine, MacBook Pro. 106 | However, for a robust and resilient experience, it is recommended that both MLflow 107 | and SFTP servers run on different machines. In addition to that, it is expected 108 | that the volumes used for storage are properly backed up. 109 | 110 | ## Running the Containers 111 | 112 | Running the containers with `docker-compose` is a no brainer. To accomplish that, 113 | please do the following: 114 | 115 | * Run -> ```docker-compose up``` 116 | - The command above should be enough to build the images based on their respective 117 | docker files and download the images used for the SFTP and Minio servers. 118 | 119 | Although you can also run on detached mode, I do recommend that the first run is 120 | nicer without it. Why? Because you can follow up on all the green letters printed out 121 | on your terminal. Besides that, if not enough, it's good to get acquainted with 122 | the logs of the services, in case you have errors happening during startup. 123 | 124 | * Run on detached mode -> ```docker-compose up -d``` 125 | 126 | ### Accessing the Frontends 127 | 128 | After starting the containers, you can access both JupyterLab, MLflow and Minio 129 | in the following way: 130 | 131 | 1. JupyterLab: http://localhost:8991/lab 132 | - Copy the token printed out on the terminal to be able to access JupyterLab 133 | 2. MLflow: http://localhost:5500 134 | 3. Minio: http://localhost:9000 135 | 136 | You will notice on the MLFow UI that only one experiment is available, the `Default` one. 137 | The intention behind this exercise is to be able to use SFTP and Minio as storage. 138 | Hence, the `Default` experiment is not a good choice to make. 139 | 140 | Let's explore other experiments. 141 | 142 | ### Creating Experiments on MLFow 143 | 144 | To start with, we have to create the experiments in the MLflow server. But how to do it? 145 | Easy: connect to the container and execute one command. Of course, once you are in 146 | the container. 147 | 148 | So, to start with, let's connect to the `ekholabs-mlflow`. To do that, run 149 | the command below: 150 | 151 | * ```docker exec -it ekholabs-mlflow /bin/bash``` 152 | 153 | If you type `mlflow --help` you will see a list of possible `commands` and `options`. 154 | 155 | To create our experiments, which will use SFTP and Minio as storage server, just 156 | execute the ```create_experiments.sh``` script. 157 | 158 | I also advice you to have a look at the script to understand how the experiments are created. 159 | If you face issues when running the script, please make sure the containers and running 160 | and that the script is executable (`chmod +x create_experiments.sh`). 161 | 162 | The script should be executed from the project root in the following way: 163 | 164 | * Run -> ```./scripts/create_experiments.sh``` 165 | 166 | Now, if you go back to the MLflow frontend, you will see that two experiments have been created. 167 | 168 | ## How does the Communication Works 169 | 170 | Before we get to the Jupyter notebooks, let's understand how the communication between 171 | the services work and what we need to do to work around some issues of having it all locally. 172 | 173 | If you take a look at the `docker-compose.yml` file, you will easily notice that the 174 | `ekholabs-sftp` container has the `ssh_sftp_key.pub` file mounted as a volume. That 175 | helps to ease communication with the SFTP server without using passwords. It also 176 | means that the private key has to be added to the other containers that will have to 177 | communicate with the SFTP server. 178 | 179 | Hence, looking further into the `docker-compose.yml` file, you will notice that both 180 | `ekholabs-mlflow` and `ekholabs-jupyterlab` have the `ssh_sftp_key` file mounted 181 | as a volume. Along with that file, which is the private key, we also have a SSH `config` 182 | file mounted as a volume. 183 | 184 | Almost there... hang on. 185 | 186 | Besides having both private key and SSH config files mounted on the volumes, we 187 | need one last thing: the `ekholabs-sftp` has to be added to the `known_hosts`, which 188 | goes under `~/.ssh/known_hosts` inside the containers. 189 | 190 | Adding that extra information in the MLflow container is pretty easy. It comes with 191 | OpenSHH installed, so just running the command below does the trick: 192 | 193 | * ```ssh-keyscan -H ekholabs-sftp >> ~/.ssh/known_hosts``` 194 | 195 | **Please, do not execute the line above**, this command is already part of the `docker-compose.yml`, 196 | which means that the host will be added to the `known_hosts` file of the MLflow container 197 | automatically at start-up. 198 | 199 | However, when it comes to the JupyterLab container, we do have an issue: it does not 200 | contain the OpenSHH packaged and we are not allowed to install it. Hence, `ssh-keyscan` 201 | won't work. 202 | 203 | What's the problem with that? Well, when the notebook tries to log the artefact on the 204 | tracking server (MLflow), `pysftp/Paramiko` will complain and throw and exception saying: 205 | *No hosts for key!* 206 | 207 | So, it can only mean one thing: we have to get the `known_hosts` from the `ekholabs-mlflow` 208 | container into the `ekholabs-jupyterlab` container. But how? Well, with **Docker**! I mean, 209 | with a shell script that will run some commands in the Docker containers. 210 | 211 | Take a peak inside the ```copy_known_hosts.sh``` shell script to understand what it's doing. 212 | Once you done, please execute that script from the root directory of the project. 213 | 214 | * Run --> ```./scripts/copy_known_hosts.sh``` 215 | 216 | Now you are - almost - good to go! 217 | 218 | **P.S.**: are you better than me at Docker sheiße? Please, contribute to this repo 219 | and get rid of so many steps. :) 220 | 221 | ## We need a Bucket 222 | 223 | *Well, completely unrelated, but I have a friend called Leonardo Bucket. Every time 224 | I do something with S3 and have to say or write the word `bucket`, it reminds me of him. :D* 225 | 226 | So, if we want to use Minio as storage, we do need a bucket. And, more over, the bucket 227 | has to be created before we try to store artefacts. 228 | 229 | If you have looked inside the ```create_experiments.sh```, you might have noticed that 230 | we expect a bucket called ```ai-models```, not *Leonardo*, there. 231 | 232 | To create a bucket, just go to Minio (http://localhost:9000) and the rest you should know. 233 | 234 | Ah!, do you need a *key id* and *secret* combination to login? Have a ```docker-compose.yml``` 235 | file, the Minio key id / secret pair is informed there. 236 | 237 | Done with the bucket? If so, now you are good to go. ;) 238 | 239 | ## Logging Metrics and Artefacts 240 | 241 | There is not much to say here. To actually see how it's done, go to your JupyterLab 242 | frontend and open the `conv-net-in-keras.ipynb` notebook. The last cell contains all 243 | the magic you need. 244 | 245 | ## SSH Keys 246 | 247 | This repository already contains the SSH keys that should be used to communicate with 248 | the SFTP service, from both MLflow and Jupyter notebooks perspective. 249 | 250 | If this setup is used to deploy a SFTP server for multiple users, with different 251 | SSH keys presumably, then the following information from the sub-session bellow 252 | has to be taken into account. 253 | 254 | ### Paramiko vs OpenSHH 255 | 256 | Keys generated on MacBooks work when connecting to the SFTP server using the `sftp` 257 | command from the shell. However, when the connection is established from within Jupyter 258 | the `pysftp` library is used, which uses `Paramiko` under the hood. And that brings issues! 259 | 260 | Do not waste your time googling a solution, trust me. The easiest / quickest thing 261 | to do is generate your keys on a Linux machine / Docker container. The key files 262 | under the `keys` directory have been created on a Docker container. 263 | 264 | ## WIP 265 | 266 | This is working in progress and more about MLflow features, like service models, 267 | will be added soon. 268 | --------------------------------------------------------------------------------