├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── assests ├── AutoDeploy_architecture.png ├── developer.md └── src.png ├── autodeploy ├── __init__.py ├── __version__.py ├── _backend │ ├── __init__.py │ ├── _database.py │ ├── _heartbeat.py │ ├── _rmq │ │ ├── _client.py │ │ └── _server.py │ └── redis │ │ └── redis_db.py ├── _schema │ ├── __init__.py │ ├── _schema.py │ └── _security.py ├── api.py ├── base │ ├── __init__.py │ ├── base_deploy.py │ ├── base_infere.py │ ├── base_loader.py │ ├── base_metric.py │ └── base_moniter_driver.py ├── config │ ├── __init__.py │ ├── config.py │ └── logging.conf ├── database │ ├── __init__.py │ ├── _database.py │ └── _models.py ├── dependencies │ ├── __init__.py │ └── _dependency.py ├── handlers │ ├── __init__.py │ └── _handlers.py ├── loader │ ├── __init__.py │ ├── _loaders.py │ └── _model_loader.py ├── logger │ ├── __init__.py │ └── logger.py ├── monitor.py ├── monitor │ ├── __init__.py │ ├── _drift_detection.py │ ├── _monitor.py │ └── _prometheus.py ├── predict.py ├── predict │ ├── __init__.py │ ├── _infere.py │ └── builder.py ├── register │ ├── __init__.py │ └── register.py ├── routers │ ├── __init__.py │ ├── _api.py │ ├── _model.py │ ├── _predict.py │ └── _security.py ├── security │ └── scheme.py ├── service │ └── _infere.py ├── testing │ └── load_test.py └── utils │ ├── registry.py │ └── utils.py ├── bin ├── autodeploy_start.sh ├── build.sh ├── load_test.sh ├── monitor_start.sh ├── predict_start.sh └── start.sh ├── configs ├── classification │ └── config.yaml ├── iris │ └── config.yaml └── prometheus.yml ├── dashboard └── dashboard.json ├── docker ├── Dockerfile ├── MonitorDockerfile ├── PredictDockerfile ├── PrometheusDockerfile └── docker-compose.yml ├── k8s ├── autodeploy-deployment.yaml ├── autodeploy-service.yaml ├── default-networkpolicy.yaml ├── grafana-deployment.yaml ├── grafana-service.yaml ├── horizontal-scale.yaml ├── metric-server.yaml ├── monitor-deployment.yaml ├── monitor-service.yaml ├── prediction-deployment.yaml ├── prediction-service.yaml ├── prometheus-deployment.yaml ├── prometheus-service.yaml ├── rabbitmq-deployment.yaml ├── rabbitmq-service.yaml ├── redis-deployment.yaml └── redis-service.yaml ├── notebooks ├── autodeploy_classification_inference.ipynb ├── classification_autodeploy.ipynb ├── convert to onnx.ipynb ├── horse_1.jpg ├── test_auto_deploy.ipynb └── zebra_1.jpg ├── pyproject.toml └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | *.pkl 132 | 133 | *.db 134 | .vim 135 | 136 | .DS_Store 137 | *.swp 138 | *.npy 139 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v2.2.1 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: end-of-file-fixer 9 | - id: check-yaml 10 | - id: check-added-large-files 11 | - id: check-symlinks 12 | 13 | - repo: https://github.com/pre-commit/mirrors-autopep8 14 | rev: v1.4.4 15 | hooks: 16 | - id: autopep8 17 | - repo: https://github.com/commitizen-tools/commitizen 18 | rev: master 19 | hooks: 20 | - id: commitizen 21 | stages: [commit-msg] 22 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 2.0.0 (2021-09-30) 2 | 3 | ### Feat 4 | 5 | - **kubernetes**: Added k8s configuration files 6 | 7 | ### Refactor 8 | 9 | - **load_test.sh**: removed load_test.sh from root 10 | 11 | ## 1.0.0 (2021-09-26) 12 | 13 | ### Refactor 14 | 15 | - **monitor-service**: refactored monitor service 16 | - **autodeploy**: refactored api monitor and prediction service with internal config addition 17 | - **backend-redis**: using internal config for redis configuraiton 18 | 19 | ### Feat 20 | 21 | - **_rmq-client-and-server**: use internal config for rmq configuration 22 | 23 | ## v0.1.1b (2021-09-25) 24 | 25 | ## 0.3beta (2021-09-18) 26 | 27 | ## 0.2beta (2021-09-12) 28 | 29 | ## 0.1beta (2021-09-09) 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 NeuralLink 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

AutoDeploy 2 | AutoDeploy - Automated machine learning model deployment. | Product Hunt

3 |
4 | 5 | Awesome Badge 6 | 7 | Star Badge 8 |
9 | Stars Badge 10 | Forks Badge 11 | Pull Requests Badge 12 | Issues Badge 13 | GitHub contributors 14 | License Badge 15 |
16 | 17 | ## What is AutoDeploy? 18 | 19 | Autodeploy src 20 | A one liner : 21 | For the DevOps nerds, AutoDeploy allows configuration based MLOps. 22 | 23 | For the rest : 24 | So you're a data scientist and have the greatest model on planet earth to classify dogs and cats! :). What next? It's a steeplearning cusrve from building your model to getting it to production. MLOps, Docker, Kubernetes, asynchronous, prometheus, logging, monitoring, versioning etc. Much more to do right before you The immediate next thoughts and tasks are 25 | 26 | - How do you get it out to your consumer to use as a service. 27 | - How do you monitor its use? 28 | - How do you test your model once deployed? And it can get trickier once you have multiple versions of your model. How do you perform 29 | A/B testing? 30 | - Can i configure custom metrics and monitor them? 31 | - What if my data distribution changes in production - how can i monitor data drift? 32 | - My models use different frameworks. Am i covered? 33 | ... and many more. 34 | 35 | # Architecture 36 | AutoDeploy architecture 37 | 38 | 39 | What if you could only configure a single file and get up and running with a single command. **That is what AutoDeploy is!** 40 | 41 | Read our [documentation](https://github.com/kartik4949/AutoDeploy/wiki) to know how to get setup and get to serving your models. 42 | 43 | # AutoDeploy monitoring dashboard 44 | AutoDeploy dashboard 45 | 46 | AutoDeploy dashboard 47 | 48 | and many more... 49 | 50 | # Feature Support. 51 | 52 | - [x] Single Configuration file support. 53 | - [x] Enterprise deployment architecture. 54 | - [x] Logging. 55 | - [x] Grafana Dashboards. 56 | - [x] Dynamic Database. 57 | - [x] Data Drift Monitoring. 58 | - [x] Async Model Monitoring. 59 | - [x] Network traffic monitoring. 60 | - [x] Realtime traffic simulation. 61 | - [x] Autoscaling of services. 62 | - [x] Kubernetes. 63 | - [x] Preprocess configuration. 64 | - [x] Posprocess configuration. 65 | - [x] Custom metrics configuration. 66 | 67 | ## Prerequisites 68 | - Install docker 69 | - For Ubuntu (and Linux distros) - [Install Docker on Ubuntu](https://docs.docker.com/engine/install/ubuntu/#installation-methods) 70 | - For Windows - [Install Docker on Windows](https://docs.docker.com/desktop/windows/install/) 71 | - For Mac - 72 | 73 | - Install docker-compose 74 | - For Ubuntu (and Linux distros) - [Install docker-compose on Linux](https://docs.docker.com/compose/install/) 75 | - For Windows and Mac 76 | 77 | ## Steps 78 | - Clone the repo : https://github.com/kartik4949/AutoDeploy 79 | - Download a sample model and dependencies 80 | - Run the command in a terminal from the AutoDeploy folder ``` wget https://github.com/kartik4949/AutoDeploy/files/7134516/model_dependencies.zip ``` 81 | - Extract the zip folder to get a **model_dependencies** folder 82 | - Have your model ready 83 | - Pickle file for scikitlearn 84 | - ONNX model for Tensorflow,Pytorch, MXNet etc. [How to convert to ONNX model](https://github.com/onnx/tutorials) 85 | - Create the model [dependencies](https://github.com/kartik4949/AutoDeploy/wiki/Setup-Model-Dependencies) 86 | - Copy the dependencies over to a **model_dependencies** folder 87 | - Setup [configuration](https://google.com) 88 | - Steps for Docker deployment 89 | - Build your docker image 90 | - ```bash build.sh -r path/to/model/requirements.txt -c path/to/model/config.yaml``` 91 | - Start your containers 92 | - ```bash start.sh -f path/to/config/file/in/autodeploy``` 93 | - Steps for Kubernetes 94 | - Build your docker image 95 | - ```bash build.sh -r path/to/model/requirements.txt -c path/to/model/config.yaml``` 96 | - Apply kubeconfig files 97 | - ``` kubectl -f k8s apply ``` 98 | - Print all pods 99 | - ``` kubectl get pod ``` 100 | - Port forwarding of api and grafana service 101 | - ``` kubectl port-forward autodeploy-pod-name 8000:8000 ``` 102 | - ``` kubectl port-forward grafana-pod-name 3000:3000 ``` 103 | 104 | ## Example (Docker deployment) - Iris Model Detection (Sci-Kit Learn). 105 | - Clone repo. 106 | - Dump your iris sklearn model via pickle, lets say `custom_model.pkl`. 107 | - Make a dir model_dependencies inside AutoDeploy. 108 | - Move `custom_model.pkl` to model_dependencies. 109 | - Create or import a reference `iris_reference.npy` file for data drift monitoring. 110 | - Note: `iris_reference.npy` is numpy reference array used to find drift in incomming data. 111 | - This reference data is usually in shape `(n, *shape_of_input)` e.g for iris data : np.zeros((100, 4)) 112 | - Shape (100, 4) means we are using 100 data points as reference for incomming input request. 113 | 114 | - Move `iris_reference.npy` to model_dependencies folder. 115 | - Refer below config file and make changes in configs/iris/config.yaml and save it. 116 | - Lastly make an empty reqs.txt file inside model_dependencies folder. 117 | ``` 118 | model: 119 | model_type: 'sklearn' 120 | model_path: 'custom_model.pkl' # Our model pickle file. 121 | model_file_type: 'pickle' 122 | version: '1.0.0' 123 | model_name: 'sklearn iris detection model.' 124 | endpoint: 'predict' 125 | protected: 0 126 | input_type: 'structured' 127 | server: 128 | name: 'autodeploy' 129 | port: 8000 130 | dependency: 131 | path: '/app/model_dependencies' 132 | input_schema: 133 | petal_length: 'float' 134 | petal_width: 'float' 135 | sepal_length: 'float' 136 | sepal_width: 'float' 137 | out_schema: 138 | out: 'int' 139 | probablity: 'float' 140 | status: 'int' 141 | monitor: 142 | server: 143 | name: 'rabbitmq' 144 | port: 5672 145 | data_drift: 146 | name: 'KSDrift' 147 | reference_data: 'iris_reference.npy' 148 | type: 'info' 149 | metrics: 150 | average_per_day: 151 | type: 'info' 152 | ``` 153 | - run ``` bash build.sh -r model_dependencies/reqs.txt -c configs/iris/config.yaml``` 154 | - run ``` bash start.sh -f configs/iris/config.yaml ``` 155 | 156 | Tada!! your model is deployed. 157 | 158 | ## Example (Docker deployment) - Classification Detection 159 | 160 | - Clone repo. 161 | - Convert the model to Onnx file `model.onnx`. 162 | - Make a dir model_dependencies inside AutoDeploy. 163 | - Move `model.onnx` to model_dependencies. 164 | - Create or import a reference `classification_reference.npy` file for data drift monitoring. 165 | - Move `classification_reference.npy` to model_dependencies folder. 166 | - Refer below config file and make changes in configs/iris/config.yaml and save it. 167 | 168 | ``` 169 | model: 170 | model_type: 'onnx' 171 | model_path: 'horse_zebra.onnx' 172 | model_file_type: 'onnx' 173 | version: '1.0.0' 174 | model_name: 'computer vision classification model.' 175 | endpoint: 'predict' 176 | protected: 0 177 | input_type: 'serialized' 178 | input_shape: [224, 224, 3] 179 | server: 180 | name: 'autodeploy' 181 | port: 8000 182 | preprocess: 'custom_preprocess_classification' 183 | input_schema: 184 | input: 'string' 185 | out_schema: 186 | out: 'int' 187 | probablity: 'float' 188 | status: 'int' 189 | dependency: 190 | path: '/app/model_dependencies' 191 | monitor: 192 | server: 193 | name: 'rabbitmq' 194 | port: 5672 195 | data_drift: 196 | name: 'KSDrift' 197 | reference_data: 'structured_ref.npy' 198 | type: 'info' 199 | custom_metrics: 'image_brightness' 200 | metrics: 201 | average_per_day: 202 | type: 'info' 203 | 204 | ``` 205 | - Make a reqs.txt file inside model_dependencies folder. 206 | - reqs.txt 207 | ``` 208 | pillow 209 | ``` 210 | 211 | - Make preprocess.py 212 | ``` 213 | import cv2 214 | import numpy as np 215 | 216 | from register import PREPROCESS 217 | 218 | @PREPROCESS.register_module(name='custom_preprocess') 219 | def iris_pre_processing(input): 220 | return input 221 | 222 | @PREPROCESS.register_module(name='custom_preprocess_classification') 223 | def custom_preprocess_fxn(input): 224 | _channels = 3 225 | _input_shape = (224, 224) 226 | _channels_first = 1 227 | input = cv2.resize( 228 | input[0], dsize=_input_shape, interpolation=cv2.INTER_CUBIC) 229 | if _channels_first: 230 | input = np.reshape(input, (_channels, *_input_shape)) 231 | else: 232 | input = np.reshape(input, (*_input_shape, _channels)) 233 | return np.asarray(input, np.float32) 234 | 235 | ``` 236 | - Make postproces.py 237 | 238 | ``` 239 | from register import POSTPROCESS 240 | 241 | @POSTPROCESS.register_module(name='custom_postprocess') 242 | def custom_postprocess_fxn(output): 243 | out_class, out_prob = output[0], output[1] 244 | output = {'out': output[0], 245 | 'probablity': output[1], 246 | 'status': 200} 247 | return output 248 | 249 | ``` 250 | - Make custom_metrics.py we will make a custom_metric to expose image_brightness 251 | ``` 252 | import numpy as np 253 | from PIL import Image 254 | from register import METRICS 255 | 256 | 257 | @METRICS.register_module(name='image_brightness') 258 | def calculate_brightness(image): 259 | image = Image.fromarray(np.asarray(image[0][0], dtype='uint8')) 260 | greyscale_image = image.convert('L') 261 | histogram = greyscale_image.histogram() 262 | pixels = sum(histogram) 263 | brightness = scale = len(histogram) 264 | 265 | for index in range(0, scale): 266 | ratio = histogram[index] / pixels 267 | brightness += ratio * (-scale + index) 268 | 269 | return 1.0 if brightness == 255 else brightness / scale 270 | 271 | ``` 272 | - run ``` bash build.sh -r model_dependencies/reqs.txt -c configs/classification/config.yaml ``` 273 | - run ``` bash start.sh -f configs/classification/config.yaml ``` 274 | - To monitor the custom metric `image_brightness`: goto grafana and add panel to the dashboard with image_brightness as metric. 275 | 276 | 277 | ## After deployment steps 278 | ### Model Endpoint 279 | - http://address:port/endpoint is your model endpoint e.g http://localhost:8000/predict 280 | 281 | ### Grafana 282 | - Open http://address:3000 283 | - Username and password both are `admin`. 284 | - Goto to add datasource. 285 | - Select first option prometheus. 286 | - Add http://prometheus:9090 in the source 287 | - Click save and test at bottom. 288 | - Goto dashboard and click import json file. 289 | - Upload dashboard/model.json avaiable in repository. 290 | - Now you have your dashboard ready!! feel free to add more panels with queries. 291 | 292 | ## Preprocess 293 | - Add preprocess.py in model_dependencies folder 294 | - from register module import PROCESS register, to register your preprocess functions. 295 | ``` 296 | from register import PREPROCESS 297 | ``` 298 | - decorate your preprocess function with `@PREPROCESS.register_module(name='custom_preprocess')` 299 | ``` 300 | @PREPROCESS.register_module(name='custom_preprocess') 301 | def function(input): 302 | # process input 303 | input = process(input) 304 | return input 305 | ``` 306 | - Remeber we will use `custom_preprocess` name in our config file, add this in your config file. 307 | ``` 308 | preprocess: custom_preprocess 309 | ``` 310 | 311 | ## Postprocess 312 | - Same as preprocess 313 | - Just remember schema of output from postprocess method should be same as definde in config file 314 | - i.e 315 | ``` 316 | out_schema: 317 | out: 'int' 318 | probablity: 'float' 319 | status: 'int' 320 | ``` 321 | 322 | ## Custom Metrics 323 | - from register import METRICS 324 | - register your function with METRIC decorator similar to preprocess 325 | - Example 1 : Simple single metric 326 | ``` 327 | import numpy as np 328 | from PIL import Image 329 | from register import METRICS 330 | 331 | 332 | @METRICS.register_module(name='image_brightness') 333 | def calculate_brightness(image): 334 | image = Image.fromarray(np.asarray(image[0][0], dtype='uint8')) 335 | greyscale_image = image.convert('L') 336 | histogram = greyscale_image.histogram() 337 | pixels = sum(histogram) 338 | brightness = scale = len(histogram) 339 | 340 | for index in range(0, scale): 341 | ratio = histogram[index] / pixels 342 | brightness += ratio * (-scale + index) 343 | 344 | return 1.0 if brightness == 255 else brightness / scale 345 | 346 | ``` 347 | - We will use `image_brightness` in config file to expose this metric function. 348 | ``` 349 | monitor: 350 | server: 351 | name: 'rabbitmq' 352 | port: 5672 353 | data_drift: 354 | name: 'KSDrift' 355 | reference_data: 'structured_ref.npy' 356 | type: 'info' 357 | custom_metrics: ['metric1', 'metric2'] 358 | metrics: 359 | average_per_day: 360 | type: 'info' 361 | ``` 362 | - Example 2: Advance metric with multiple metrcis functions 363 | 364 | ``` 365 | import numpy as np 366 | from PIL import Image 367 | from register import METRICS 368 | 369 | 370 | @METRICS.register_module(name='metric1') 371 | def calculate_brightness(image): 372 | return 1 373 | 374 | @METRICS.register_module(name='metric2') 375 | def metric2(image): 376 | return 2 377 | 378 | ``` 379 | - config looks like 380 | ``` 381 | monitor: 382 | server: 383 | name: 'rabbitmq' 384 | port: 5672 385 | data_drift: 386 | name: 'KSDrift' 387 | reference_data: 'structured_ref.npy' 388 | type: 'info' 389 | custom_metrics: ['metric1', 'metric2'] 390 | metrics: 391 | average_per_day: 392 | type: 'info' 393 | ``` 394 | 395 | 396 | ## License 397 | 398 | Distributed under the MIT License. See `LICENSE` for more information. 399 | 400 | 401 | ## Contact 402 | 403 | Kartik Sharma - kartik4949@gmail.com 404 | Nilav Ghosh - nilavghosh@gmail.com 405 | -------------------------------------------------------------------------------- /assests/AutoDeploy_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartik4949/AutoDeploy/af7b3b32954a574307849ababb05fa2f4a80f52e/assests/AutoDeploy_architecture.png -------------------------------------------------------------------------------- /assests/developer.md: -------------------------------------------------------------------------------- 1 | # formatter information: 2 | 3 | ## autopep8 command 4 | autopep8 --global-config ~/.config/pep8 --in-place --aggressive --recursive --indent-size=2 . 5 | 6 | ## pep8 file 7 | ``` 8 | [pep8] 9 | count = False 10 | ignore = E226,E302,E41 11 | max-line-length = 89 12 | statistics = True 13 | ``` 14 | -------------------------------------------------------------------------------- /assests/src.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartik4949/AutoDeploy/af7b3b32954a574307849ababb05fa2f4a80f52e/assests/src.png -------------------------------------------------------------------------------- /autodeploy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartik4949/AutoDeploy/af7b3b32954a574307849ababb05fa2f4a80f52e/autodeploy/__init__.py -------------------------------------------------------------------------------- /autodeploy/__version__.py: -------------------------------------------------------------------------------- 1 | version = '2.0.0' 2 | -------------------------------------------------------------------------------- /autodeploy/_backend/__init__.py: -------------------------------------------------------------------------------- 1 | from ._rmq._client import RabbitMQClient 2 | from ._rmq._server import RabbitMQConsume 3 | from ._database import Database 4 | from .redis.redis_db import RedisDB 5 | -------------------------------------------------------------------------------- /autodeploy/_backend/_database.py: -------------------------------------------------------------------------------- 1 | ''' A simple database class utility. ''' 2 | from database import _database as database, _models as models 3 | from logger import AppLogger 4 | 5 | applogger = AppLogger(__name__) 6 | logger = applogger.get_logger() 7 | 8 | 9 | class Database: 10 | ''' 11 | A database class that creates and stores incoming requests 12 | in user define database with given schema. 13 | Args: 14 | config (Config): A configuration object which contains configuration 15 | for the deployment. 16 | 17 | Attributes: 18 | config (Config): internal configuration object for configurations. 19 | 20 | Example: 21 | >> with Database(config) as db: 22 | >> .... db.store_request(item) 23 | 24 | ''' 25 | 26 | def __init__(self, config) -> None: 27 | self.config = config 28 | self.db = None 29 | 30 | @classmethod 31 | def bind(cls): 32 | models.Base.metadata.create_all(bind=database.engine) 33 | 34 | def setup(self): 35 | # create database engine and bind all. 36 | self.bind() 37 | self.db = database.SessionLocal() 38 | 39 | def close(self): 40 | self.db.close() 41 | 42 | def store_request(self, db_item) -> None: 43 | 44 | try: 45 | self.db.add(db_item) 46 | self.db.commit() 47 | self.db.refresh(db_item) 48 | except Exception as exc: 49 | logger.error( 50 | 'Some error occured while storing request in database.') 51 | raise Exception( 52 | 'Some error occured while storing request in database.') 53 | return db_item 54 | -------------------------------------------------------------------------------- /autodeploy/_backend/_heartbeat.py: -------------------------------------------------------------------------------- 1 | ''' a mixin class for heatbeat to rabbitmq server. ''' 2 | from time import sleep 3 | import threading 4 | 5 | # send heartbeat signal after 5 secs to rabbitmq. 6 | HEART_BEAT = 5 7 | 8 | class HeartBeatMixin: 9 | @staticmethod 10 | def _beat(connection): 11 | while True: 12 | # TODO: remove hardcode 13 | sleep(HEART_BEAT) 14 | connection.process_data_events() 15 | 16 | def beat(self): 17 | ''' process_data_events periodically. 18 | TODO: hackish way, think other way. 19 | ''' 20 | heartbeat = threading.Thread( 21 | target=self._beat, args=(self.connection,), daemon=True) 22 | heartbeat.start() 23 | -------------------------------------------------------------------------------- /autodeploy/_backend/_rmq/_client.py: -------------------------------------------------------------------------------- 1 | ''' a rabbitmq client class. ''' 2 | import json 3 | import asyncio 4 | from contextlib import suppress 5 | import uuid 6 | from typing import Dict 7 | 8 | import pika 9 | 10 | from _backend._heartbeat import HeartBeatMixin 11 | from logger import AppLogger 12 | 13 | logger = AppLogger(__name__).get_logger() 14 | 15 | 16 | class RabbitMQClient(HeartBeatMixin): 17 | def __init__(self, config): 18 | self.host = config.RABBITMQ_HOST 19 | self.port = config.RABBITMQ_PORT 20 | self.retries = config.RETRIES 21 | 22 | def connect(self): 23 | ''' a simple function to get connection to rmq ''' 24 | # connect to RabbitMQ Server. 25 | self.connection = pika.BlockingConnection( 26 | pika.ConnectionParameters(host=self.host, port=self.port)) 27 | 28 | self.channel = self.connection.channel() 29 | 30 | result = self.channel.queue_declare(queue='monitor', exclusive=False) 31 | self.callback_queue = result.method.queue 32 | self.channel.basic_qos(prefetch_count=1) 33 | 34 | def setupRabbitMq(self, ): 35 | ''' a simple setup for rabbitmq server connection 36 | and queue connection. 37 | ''' 38 | self.connect() 39 | self.beat() 40 | 41 | def _publish(self, body: Dict): 42 | self.channel.basic_publish( 43 | exchange='', 44 | routing_key='monitor', 45 | properties=pika.BasicProperties( 46 | reply_to=self.callback_queue, 47 | ), 48 | body=json.dumps(dict(body))) 49 | 50 | def publish_rbmq(self, body: Dict): 51 | try: 52 | self._publish(body) 53 | except: 54 | logger.error('Error while publish!!, restarting connection.') 55 | for i in range(self.retries): 56 | try: 57 | self._publish(body) 58 | except: 59 | continue 60 | else: 61 | logger.debug('published success after {i + 1} retries') 62 | return 63 | break 64 | 65 | logger.critical('cannot publish after retries!!') 66 | -------------------------------------------------------------------------------- /autodeploy/_backend/_rmq/_server.py: -------------------------------------------------------------------------------- 1 | ''' a simple rabbitmq server/consumer. ''' 2 | import pika 3 | 4 | from logger import AppLogger 5 | from _backend._heartbeat import HeartBeatMixin 6 | 7 | logger = AppLogger(__name__).get_logger() 8 | 9 | class RabbitMQConsume(HeartBeatMixin): 10 | def __init__(self, config) -> None: 11 | self.host = config.RABBITMQ_HOST 12 | self.port = config.RABBITMQ_PORT 13 | self.retries = config.RETRIES 14 | self.queue = config.RABBITMQ_QUEUE 15 | self.connection = None 16 | 17 | def setupRabbitMQ(self, callback): 18 | try: 19 | self.connection = pika.BlockingConnection( 20 | pika.ConnectionParameters(self.host, port=self.port)) 21 | except Exception as exc: 22 | logger.critical( 23 | 'Error occured while creating connnection in rabbitmq') 24 | raise Exception( 25 | 'Error occured while creating connnection in rabbitmq', exc) 26 | channel = self.connection.channel() 27 | 28 | channel.queue_declare(queue=self.queue) 29 | self.beat() 30 | 31 | channel.basic_consume( 32 | queue=self.queue, on_message_callback=callback, auto_ack=True) 33 | self.channel = channel 34 | -------------------------------------------------------------------------------- /autodeploy/_backend/redis/redis_db.py: -------------------------------------------------------------------------------- 1 | from uuid import uuid4 2 | import struct 3 | 4 | import redis 5 | import numpy as np 6 | 7 | 8 | class RedisDB: 9 | def __init__(self, config) -> None: 10 | self.config = config 11 | self.redis = redis.Redis(host = self.config.REDIS_SERVER, port = self.config.REDIS_PORT, db=0) 12 | 13 | def encode(self, input, shape): 14 | bytes = input.tobytes() 15 | _encoder = 'I'*len(shape) 16 | shape = struct.pack('>' + _encoder, *shape) 17 | return shape + bytes 18 | 19 | def pull(self, id, dtype, ndim): 20 | input_encoded = self.redis.get(id) 21 | shape = struct.unpack('>' + ('I'*ndim), input_encoded[:ndim*4]) 22 | a = np.frombuffer(input_encoded[ndim*4:], dtype=dtype).reshape(shape) 23 | return a 24 | 25 | def push(self, input, dtype, shape): 26 | input_hash = str(uuid4()) 27 | input = np.asarray(input, dtype=dtype) 28 | encoded_input = self.encode(input, shape=shape) 29 | self.redis.set(input_hash, encoded_input) 30 | return input_hash 31 | 32 | def pop(): 33 | NotImplementedError('Not implemented yet!') 34 | -------------------------------------------------------------------------------- /autodeploy/_schema/__init__.py: -------------------------------------------------------------------------------- 1 | from ._schema import UserIn, UserOut, create_model 2 | -------------------------------------------------------------------------------- /autodeploy/_schema/_schema.py: -------------------------------------------------------------------------------- 1 | ''' user input and output schema models. ''' 2 | from pydantic import create_model 3 | from utils import utils 4 | 5 | 6 | """ Simple user input schema. """ 7 | 8 | 9 | class UserIn: 10 | ''' 11 | `UserIn` pydantic model 12 | supports dynamic model attributes creation 13 | defined by user in configuration file. 14 | 15 | Args: 16 | UserInputSchema (pydantic.BaseModel): pydantic model. 17 | 18 | ''' 19 | 20 | def __init__(self, config, *args, **kwargs): 21 | self._model_attr = utils.annotator(dict(config.input_schema)) 22 | self.UserInputSchema = create_model( 23 | 'UserInputSchema', **self._model_attr) 24 | 25 | 26 | class UserOut: 27 | ''' 28 | 29 | `UserOut` pydantic model 30 | supports dynamic model attributes creation 31 | defined by user in configuration file. 32 | 33 | Args: 34 | UserOutputSchema (pydantic.BaseModel): pydantic model. 35 | ''' 36 | 37 | def __init__(self, config, *args, **kwargs): 38 | self._model_attr = utils.annotator(dict(config.out_schema)) 39 | self.UserOutputSchema = create_model( 40 | 'UserOutputSchema', **self._model_attr) 41 | -------------------------------------------------------------------------------- /autodeploy/_schema/_security.py: -------------------------------------------------------------------------------- 1 | """ Simple security user schema. """ 2 | from typing import Optional 3 | 4 | from pydantic import BaseModel 5 | 6 | 7 | class User(BaseModel): 8 | username: str 9 | email: Optional[str] = None 10 | full_name: Optional[str] = None 11 | disabled: Optional[bool] = None 12 | 13 | class UserInDB(User): 14 | hashed_password: str 15 | -------------------------------------------------------------------------------- /autodeploy/api.py: -------------------------------------------------------------------------------- 1 | """ A simple deploy service utility.""" 2 | import traceback 3 | import argparse 4 | import requests 5 | import os 6 | import json 7 | from typing import List, Dict, Tuple, Any 8 | from datetime import datetime 9 | 10 | import uvicorn 11 | import numpy as np 12 | from prometheus_fastapi_instrumentator import Instrumentator 13 | from fastapi import Depends, FastAPI, Request, HTTPException 14 | 15 | from config.config import Config, InternalConfig 16 | from utils import utils 17 | from logger import AppLogger 18 | from routers import AutoDeployRouter 19 | from routers import api_router 20 | from routers import ModelDetailRouter 21 | from routers import model_detail_router 22 | from routers import auth_router 23 | from base import BaseDriverService 24 | 25 | 26 | # ArgumentParser to get commandline args. 27 | parser = argparse.ArgumentParser() 28 | parser.add_argument("-o", "--mode", default='debug', type=str, 29 | help="model for running deployment ,mode can be PRODUCTION or DEBUG") 30 | args = parser.parse_args() 31 | 32 | # __main__ (root) logger instance construction. 33 | applogger = AppLogger(__name__) 34 | logger = applogger.get_logger() 35 | 36 | 37 | def set_middleware(app): 38 | @app.middleware('http') 39 | async def log_incoming_requests(request: Request, call_next): 40 | ''' 41 | Middleware to log incoming requests to server. 42 | 43 | Args: 44 | request (Request): incoming request payload. 45 | call_next: function to executing the request. 46 | Returns: 47 | response (Dict): reponse from the executing fxn. 48 | ''' 49 | logger.info(f'incoming payload to server. {request}') 50 | response = await call_next(request) 51 | return response 52 | 53 | 54 | class APIDriver(BaseDriverService): 55 | ''' 56 | APIDriver class for creating deploy driver which setups 57 | , registers routers with `app` and executes the server. 58 | 59 | This class is the main driver class responsible for creating 60 | and setupping the environment. 61 | Args: 62 | config (str): a config path. 63 | 64 | Note: 65 | `setup` method should get called before `register_routers`. 66 | 67 | ''' 68 | 69 | def __init__(self, config_path) -> None: 70 | # user config for configuring model deployment. 71 | self.user_config = Config(config_path).get_config() 72 | self.internal_config = InternalConfig() 73 | 74 | def setup(self, app) -> None: 75 | ''' 76 | Main setup function responsible for setting up 77 | the environment for model deployment. 78 | setups prediction and model routers. 79 | 80 | Setups Prometheus instrumentor 81 | ''' 82 | # print config 83 | logger.info(self.user_config) 84 | 85 | if isinstance(self.user_config.model.model_path, list): 86 | logger.info('Multi model deployment started...') 87 | 88 | # expose prometheus data to /metrics 89 | Instrumentator().instrument(app).expose(app) 90 | _schemas = self._setup_schema() 91 | 92 | apirouter = AutoDeployRouter(self.user_config, self.internal_config) 93 | apirouter.setup(_schemas) 94 | apirouter.register_router() 95 | 96 | modeldetailrouter = ModelDetailRouter(self.user_config) 97 | modeldetailrouter.register_router() 98 | set_middleware(app) 99 | 100 | # setup exception handlers 101 | self.handler_setup(app) 102 | 103 | def register_routers(self, app): 104 | ''' 105 | a helper function to register routers in the app. 106 | ''' 107 | self._app_include([api_router, model_detail_router, auth_router], app) 108 | 109 | def run(self, app): 110 | ''' 111 | The main executing function which runs the uvicorn server 112 | with the app instance and user configuration. 113 | ''' 114 | # run uvicorn server. 115 | uvicorn.run(app, port=self.internal_config.API_PORT, host="0.0.0.0") 116 | 117 | 118 | def main(): 119 | # create fastapi application 120 | app = FastAPI() 121 | deploydriver = APIDriver(os.environ['CONFIG']) 122 | deploydriver.setup(app) 123 | deploydriver.register_routers(app) 124 | deploydriver.run(app) 125 | 126 | 127 | if __name__ == "__main__": 128 | main() 129 | -------------------------------------------------------------------------------- /autodeploy/base/__init__.py: -------------------------------------------------------------------------------- 1 | from .base_infere import BaseInfere 2 | from .base_deploy import BaseDriverService 3 | from .base_loader import BaseLoader 4 | from .base_moniter_driver import BaseMonitorService 5 | from .base_metric import BaseMetric 6 | -------------------------------------------------------------------------------- /autodeploy/base/base_deploy.py: -------------------------------------------------------------------------------- 1 | ''' Base class for deploy driver service. ''' 2 | from abc import ABC, abstractmethod 3 | from typing import List, Dict, Tuple, Any 4 | 5 | from handlers import Handler, ModelException 6 | from _schema import UserIn, UserOut 7 | 8 | 9 | class BaseDriverService(ABC): 10 | def __init__(self, *args, **kwargs): 11 | ... 12 | 13 | def _setup_schema(self) -> Tuple[Any, Any]: 14 | ''' 15 | a function to setup input and output schema for 16 | server. 17 | 18 | ''' 19 | # create input and output schema for model endpoint api. 20 | output_model_schema = UserOut( 21 | self.user_config) 22 | input_model_schema = UserIn( 23 | self.user_config) 24 | return (input_model_schema, output_model_schema) 25 | 26 | @staticmethod 27 | def handler_setup(app): 28 | ''' 29 | a simple helper function to overide handlers 30 | ''' 31 | # create exception handlers for fastapi. 32 | handler = Handler() 33 | handler.overide_handlers(app) 34 | handler.create_handlers(app) 35 | 36 | def _app_include(self, routers, app): 37 | ''' 38 | a simple helper function to register routers 39 | with the fastapi `app`. 40 | 41 | ''' 42 | for route in routers: 43 | app.include_router(route) 44 | 45 | @abstractmethod 46 | def setup(self, *args, **kwargs): 47 | ''' 48 | abstractmethod for setup to be implemeneted in child class 49 | for setup of monitor driver. 50 | 51 | ''' 52 | raise NotImplementedError('setup function is not implemeneted.') 53 | -------------------------------------------------------------------------------- /autodeploy/base/base_infere.py: -------------------------------------------------------------------------------- 1 | ''' Base class for inference service. ''' 2 | from abc import ABC, abstractmethod 3 | 4 | 5 | class BaseInfere(ABC): 6 | def __init__(self, *args, **kwargs): 7 | ... 8 | 9 | @abstractmethod 10 | def infere(self, *args, **kwargs): 11 | ''' 12 | infere function which inferes the input 13 | data based on the model loaded. 14 | Needs to be overriden in child class with 15 | custom implementation. 16 | 17 | ''' 18 | raise NotImplementedError('Infere function is not implemeneted.') 19 | -------------------------------------------------------------------------------- /autodeploy/base/base_loader.py: -------------------------------------------------------------------------------- 1 | ''' Base class for loaders. ''' 2 | from abc import ABC, abstractmethod 3 | 4 | 5 | class BaseLoader(ABC): 6 | def __init__(self, *args, **kwargs): 7 | ... 8 | 9 | @abstractmethod 10 | def load(self, *args, **kwargs): 11 | ''' 12 | abstractmethod for loading model file. 13 | 14 | ''' 15 | raise NotImplementedError('load function is not implemeneted.') 16 | -------------------------------------------------------------------------------- /autodeploy/base/base_metric.py: -------------------------------------------------------------------------------- 1 | """ a simple model data drift detection monitering utilities. """ 2 | from abc import ABC, abstractmethod 3 | from typing import Dict 4 | 5 | import numpy as np 6 | 7 | from config import Config 8 | 9 | 10 | class BaseMetric(ABC): 11 | 12 | def __init__(self, config: Config, *args, **kwargs) -> None: 13 | self.config = config 14 | 15 | @abstractmethod 16 | def get_change(self, x: np.ndarray) -> Dict: 17 | ''' 18 | 19 | ''' 20 | raise NotImplementedError('setup function is not implemeneted.') 21 | -------------------------------------------------------------------------------- /autodeploy/base/base_moniter_driver.py: -------------------------------------------------------------------------------- 1 | ''' Base class for monitor driver service. ''' 2 | from abc import ABC, abstractmethod 3 | from typing import List, Dict, Tuple, Any 4 | 5 | from handlers import Handler, ModelException 6 | from _schema import UserIn, UserOut 7 | 8 | 9 | class BaseMonitorService(ABC): 10 | def __init__(self, *args, **kwargs): 11 | ... 12 | 13 | @abstractmethod 14 | def setup(self, *args, **kwargs): 15 | ''' 16 | abstractmethod for setup to be implemeneted in child class 17 | for setup of monitor driver. 18 | 19 | ''' 20 | raise NotImplementedError('setup function is not implemeneted.') 21 | 22 | def _setup_schema(self) -> Tuple[Any, Any]: 23 | ''' 24 | a function to setup input and output schema for 25 | server. 26 | 27 | ''' 28 | # create input and output schema for model endpoint api. 29 | input_model_schema = UserIn( 30 | self.user_config) 31 | output_model_schema = UserOut( 32 | self.user_config) 33 | return (input_model_schema, output_model_schema) 34 | 35 | @staticmethod 36 | def handler_setup(app): 37 | ''' 38 | a simple helper function to overide handlers 39 | ''' 40 | # create exception handlers for fastapi. 41 | handler = Handler() 42 | handler.overide_handlers(app) 43 | handler.create_handlers(app) 44 | 45 | def _app_include(self, routers, app): 46 | ''' 47 | a simple helper function to register routers 48 | with the fastapi `app`. 49 | 50 | ''' 51 | for route in routers: 52 | app.include_router(route) 53 | 54 | @abstractmethod 55 | def _load_monitor_algorithm(self, *args, **kwargs): 56 | ''' 57 | abstractmethod for loading monitor algorithm which needs 58 | to be overriden in child class. 59 | ''' 60 | 61 | raise NotImplementedError('load monitor function is not implemeneted.') 62 | -------------------------------------------------------------------------------- /autodeploy/config/__init__.py: -------------------------------------------------------------------------------- 1 | from .config import Config 2 | from .config import InternalConfig 3 | -------------------------------------------------------------------------------- /autodeploy/config/config.py: -------------------------------------------------------------------------------- 1 | """ A simple configuration class. """ 2 | from typing import Dict 3 | from os import path 4 | import copy 5 | import yaml 6 | 7 | 8 | class AttrDict(dict): 9 | """ Dictionary subclass whose entries can be accessed by attributes (as well 10 | as normally). 11 | """ 12 | 13 | def __init__(self, *args, **kwargs): 14 | super(AttrDict, self).__init__(*args, **kwargs) 15 | self.__dict__ = self 16 | 17 | @classmethod 18 | def from_nested_dicts(cls, data): 19 | """ Construct nested AttrDicts from nested dictionaries. 20 | Args: 21 | data (dict): a dictionary data. 22 | """ 23 | if not isinstance(data, dict): 24 | return data 25 | else: 26 | try: 27 | return cls({key: cls.from_nested_dicts(data[key]) for key in data}) 28 | except KeyError as ke: 29 | raise KeyError('key not found in data while loading config.') 30 | 31 | 32 | class Config(AttrDict): 33 | """ A Configuration Class. 34 | Args: 35 | config_file (str): a configuration file. 36 | """ 37 | 38 | def __init__(self, config_file): 39 | super().__init__() 40 | self.config_file = config_file 41 | self.config = self._parse_from_yaml() 42 | 43 | def as_dict(self): 44 | """Returns a dict representation.""" 45 | config_dict = {} 46 | for k, v in self.__dict__.items(): 47 | if isinstance(v, Config): 48 | config_dict[k] = v.as_dict() 49 | else: 50 | config_dict[k] = copy.deepcopy(v) 51 | return config_dict 52 | 53 | def __repr__(self): 54 | return repr(self.as_dict()) 55 | 56 | def __str__(self): 57 | print("Configurations:\n") 58 | try: 59 | return yaml.dump(self.as_dict(), indent=4) 60 | except TypeError: 61 | return str(self.as_dict()) 62 | 63 | def _parse_from_yaml(self) -> Dict: 64 | """Parses a yaml file and returns a dictionary.""" 65 | config_path = path.join(path.dirname(path.abspath(__file__)), self.config_file) 66 | try: 67 | with open(config_path, "r") as f: 68 | config_dict = yaml.load(f, Loader=yaml.FullLoader) 69 | return config_dict 70 | except FileNotFoundError as fnfe: 71 | raise FileNotFoundError('configuration file not found.') 72 | except Exception as exc: 73 | raise Exception('Error while loading config file.') 74 | 75 | def get_config(self): 76 | return AttrDict.from_nested_dicts(self.config) 77 | 78 | class InternalConfig(): 79 | """ An Internal Configuration Class. 80 | """ 81 | 82 | # model backend redis 83 | REDIS_SERVER = 'redis' 84 | REDIS_PORT = 6379 85 | 86 | # api endpoint 87 | API_PORT = 8000 88 | API_NAME = 'AutoDeploy' 89 | 90 | # Monitor endpoint 91 | MONITOR_PORT = 8001 92 | 93 | # Rabbitmq 94 | 95 | RETRIES = 3 96 | RABBITMQ_PORT = 5672 97 | RABBITMQ_HOST = 'rabbitmq' 98 | RABBITMQ_QUEUE = 'monitor' 99 | 100 | 101 | # model prediction service 102 | PREDICT_PORT = 8009 103 | PREDICT_ENDPOINT = 'model_predict' 104 | PREDICT_URL = 'prediction' 105 | PREDICT_INPUT_DTYPE = 'float32' 106 | -------------------------------------------------------------------------------- /autodeploy/config/logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root, monitor 3 | 4 | [handlers] 5 | keys=StreamHandler,FileHandler 6 | 7 | [formatters] 8 | keys=normalFormatter,detailedFormatter 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=FileHandler, StreamHandler 13 | 14 | [logger_monitor] 15 | level=INFO 16 | handlers=FileHandler, StreamHandler 17 | qualname=monitor 18 | propagate=0 19 | 20 | [handler_StreamHandler] 21 | class=StreamHandler 22 | level=DEBUG 23 | formatter=normalFormatter 24 | args=(sys.stdout,) 25 | 26 | [handler_FileHandler] 27 | class=FileHandler 28 | level=INFO 29 | formatter=detailedFormatter 30 | args=("app.log",) 31 | 32 | [formatter_normalFormatter] 33 | format=%(asctime)s loglevel=%(levelname)-6s logger=%(name)s %(funcName)s() L%(lineno)-4d %(message)s 34 | 35 | [formatter_detailedFormatter] 36 | format=%(asctime)s loglevel=%(levelname)-6s logger=%(name)s %(funcName)s() L%(lineno)-4d %(message)s call_trace=%(pathname)s L%(lineno)-4d 37 | -------------------------------------------------------------------------------- /autodeploy/database/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartik4949/AutoDeploy/af7b3b32954a574307849ababb05fa2f4a80f52e/autodeploy/database/__init__.py -------------------------------------------------------------------------------- /autodeploy/database/_database.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import create_engine 2 | from sqlalchemy.ext.declarative import declarative_base 3 | from sqlalchemy.orm import sessionmaker 4 | 5 | ''' SQL database url path. ''' 6 | SQLALCHEMY_DATABASE_URL = "sqlite:///./data_drifts.db" 7 | 8 | engine = create_engine( 9 | SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False} 10 | ) 11 | SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) 12 | 13 | Base = declarative_base() 14 | -------------------------------------------------------------------------------- /autodeploy/database/_models.py: -------------------------------------------------------------------------------- 1 | ''' a simple pydantic and sqlalchemy models utilities. ''' 2 | from os import path, environ 3 | 4 | from sqlalchemy import Boolean, Column, Integer, String, Float, BLOB 5 | 6 | from pydantic import BaseModel 7 | from pydantic.fields import ModelField 8 | from pydantic import create_model 9 | 10 | from typing import Any, Dict, Optional 11 | 12 | from database._database import Base 13 | from utils import utils 14 | from config import Config 15 | 16 | SQLTYPE_MAPPER = { 17 | 'float': Float, 18 | 'string': String, 19 | 'int': Integer, 20 | 'bool': Boolean} 21 | config_path = path.join(path.dirname(path.abspath(__file__)), 22 | environ['CONFIG']) 23 | 24 | config = Config(config_path).get_config() 25 | 26 | 27 | def set_dynamic_inputs(cls): 28 | ''' a decorator to set dynamic model attributes. ''' 29 | for k, v in dict(config.input_schema).items(): 30 | if config.model.input_type == 'serialized': 31 | if v == 'string': 32 | setattr(cls, k, Column(BLOB)) 33 | continue 34 | 35 | setattr(cls, k, Column(SQLTYPE_MAPPER[v])) 36 | return cls 37 | 38 | 39 | @set_dynamic_inputs 40 | class Requests(Base): 41 | ''' 42 | Requests class pydantic model with input_schema. 43 | ''' 44 | __tablename__ = "requests" 45 | 46 | id = Column(Integer, primary_key=True, index=True) 47 | time_stamp = Column(String) 48 | prediction = Column(Integer) 49 | is_drift = Column(Boolean, default=True) 50 | -------------------------------------------------------------------------------- /autodeploy/dependencies/__init__.py: -------------------------------------------------------------------------------- 1 | from ._dependency import LoadDependency 2 | -------------------------------------------------------------------------------- /autodeploy/dependencies/_dependency.py: -------------------------------------------------------------------------------- 1 | ''' a simple dependency class ''' 2 | import os 3 | import json 4 | import sys 5 | import glob 6 | import importlib 7 | 8 | from config import Config 9 | from register import PREPROCESS, POSTPROCESS, METRICS 10 | from logger import AppLogger 11 | 12 | logger = AppLogger(__name__).get_logger() 13 | 14 | 15 | class LoadDependency: 16 | ''' 17 | a dependency class which creates 18 | dependency on predict endpoints. 19 | 20 | Args: 21 | config (str): configuration file. 22 | 23 | ''' 24 | 25 | def __init__(self, config): 26 | self.config = config 27 | self.preprocess_fxn_name = config.get('preprocess', None) 28 | self.postprocess_fxn_name = config.get('postprocess', None) 29 | 30 | @staticmethod 31 | def convert_python_path(file): 32 | # TODO: check os. 33 | file = file.split('.')[-2] 34 | file = file.split('/')[-1] 35 | return file 36 | 37 | @property 38 | def postprocess_fxn(self): 39 | _fxns = list(POSTPROCESS.module_dict.values()) 40 | if self.postprocess_fxn_name: 41 | try: 42 | return POSTPROCESS.module_dict[self.postprocess_fxn_name] 43 | except KeyError as ke: 44 | raise KeyError( 45 | f'{self.postprocess_fxn_name} not found in {POSTPROCESS.keys()} keys') 46 | if _fxns: 47 | return _fxns[0] 48 | return None 49 | 50 | @property 51 | def preprocess_fxn(self): 52 | _fxns = list(PREPROCESS.module_dict.values()) 53 | if self.preprocess_fxn_name: 54 | try: 55 | return PREPROCESS.module_dict[self.preprocess_fxn_name] 56 | except KeyError as ke: 57 | raise KeyError( 58 | f'{self.preprocess_fxn_name} not found in {PREPROCESS.keys()} keys') 59 | if _fxns: 60 | return _fxns[0] 61 | return None 62 | 63 | def import_dependencies(self): 64 | # import to invoke the register 65 | try: 66 | path = self.config.dependency.path 67 | sys.path.append(path) 68 | _py_files = glob.glob(os.path.join(path, '*.py')) 69 | 70 | for file in _py_files: 71 | file = self.convert_python_path(file) 72 | importlib.import_module(file) 73 | except ImportError as ie: 74 | logger.error('could not import dependency from given path.') 75 | raise ImportError('could not import dependency from given path.') 76 | -------------------------------------------------------------------------------- /autodeploy/handlers/__init__.py: -------------------------------------------------------------------------------- 1 | from ._handlers import ModelException 2 | from ._handlers import Handler 3 | -------------------------------------------------------------------------------- /autodeploy/handlers/_handlers.py: -------------------------------------------------------------------------------- 1 | ''' A simple Exception handlers. ''' 2 | from fastapi import Request, status 3 | from fastapi.encoders import jsonable_encoder 4 | from fastapi.exceptions import RequestValidationError, StarletteHTTPException 5 | from fastapi.responses import JSONResponse, PlainTextResponse 6 | 7 | 8 | class ModelException(Exception): 9 | ''' Custom ModelException class 10 | Args: 11 | name (str): name of the exception. 12 | ''' 13 | 14 | def __init__(self, name: str): 15 | self.name = name 16 | 17 | 18 | async def model_exception_handler(request: Request, exception: ModelException): 19 | return JSONResponse(status_code=500, content={"message": "model failure."}) 20 | 21 | 22 | async def validation_exception_handler(request: Request, exc: RequestValidationError): 23 | return JSONResponse( 24 | status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, 25 | content=jsonable_encoder({"detail": exc.errors(), "body": exc.body}), 26 | ) 27 | 28 | 29 | async def http_exception_handler(request, exc): 30 | return PlainTextResponse(str(exc.detail), status_code=exc.status_code) 31 | 32 | 33 | class Handler: 34 | ''' Setup Handler class to setup 35 | and bind custom and overriden exceptions to fastapi 36 | application. 37 | 38 | ''' 39 | 40 | def overide_handlers(self, app): 41 | ''' A helper function to overide default fastapi handlers. 42 | Args: 43 | app: fastapi application. 44 | ''' 45 | _exc_app_request = app.exception_handler(RequestValidationError) 46 | _exc_app_starlette = app.exception_handler(StarletteHTTPException) 47 | 48 | _validation_exception_handler = _exc_app_request( 49 | validation_exception_handler) 50 | _starlette_exception_handler = _exc_app_starlette( 51 | http_exception_handler) 52 | 53 | def create_handlers(self, app): 54 | ''' A function to bind handlers to fastapi application. 55 | Args: 56 | app: fastapi application. 57 | ''' 58 | _exc_app_model = app.exception_handler(ModelException) 59 | _exce_model_exception_handler = _exc_app_model(model_exception_handler) 60 | -------------------------------------------------------------------------------- /autodeploy/loader/__init__.py: -------------------------------------------------------------------------------- 1 | from ._model_loader import ModelLoader 2 | -------------------------------------------------------------------------------- /autodeploy/loader/_loaders.py: -------------------------------------------------------------------------------- 1 | ''' A loader class utilities. ''' 2 | from os import path 3 | 4 | from logger import AppLogger 5 | from base import BaseLoader 6 | 7 | logger = AppLogger(__name__).get_logger() 8 | 9 | 10 | class PickleLoader(BaseLoader): 11 | ''' a simple PickleLoader class. 12 | class which loads pickle model file. 13 | Args: 14 | model_path (str): model file path. 15 | multi_model (bool): multi model flag. 16 | ''' 17 | 18 | def __init__(self, model_path, multi_model=False): 19 | self.model_path = model_path 20 | self.multi_model = multi_model 21 | 22 | def load(self): 23 | ''' a helper function to load model_path file. ''' 24 | import pickle 25 | # TODO: do handling 26 | try: 27 | if not self.multi_model: 28 | self.model_path = [self.model_path] 29 | models = [] 30 | for model in self.model_path: 31 | model_path = path.join(path.dirname(path.abspath(__file__)), model) 32 | with open(model_path, 'rb') as reader: 33 | models.append(pickle.load(reader)) 34 | return models 35 | except FileNotFoundError as fnfe: 36 | logger.error('model file not found...') 37 | raise FileNotFoundError('model file not found ...') 38 | 39 | class OnnxLoader(BaseLoader): 40 | ''' a simple OnnxLoader class. 41 | class which loads pickle model file. 42 | Args: 43 | model_path (str): model file path. 44 | multi_model (bool): multi model flag. 45 | ''' 46 | 47 | def __init__(self, model_path, multi_model=False): 48 | self.model_path = model_path 49 | self.multi_model = multi_model 50 | 51 | def model_assert(self, model_name): 52 | ''' a helper function to assert model file name. ''' 53 | if not model_name.endswith('.onnx'): 54 | logger.error( 55 | f'OnnxLoader save model extension is not .onnx but {model_name}') 56 | raise Exception( 57 | f'OnnxLoader save model extension is not .onnx but {model_name}') 58 | 59 | def load(self): 60 | ''' a function to load onnx model file. ''' 61 | import onnxruntime as ort 62 | 63 | try: 64 | if not self.multi_model: 65 | self.model_path = [self.model_path] 66 | models = [] 67 | for model in self.model_path: 68 | self.model_assert(model) 69 | model_path = path.join(path.dirname(path.abspath(__file__)), model) 70 | # onnx model load. 71 | sess = ort.InferenceSession(model_path) 72 | models.append(sess) 73 | return models 74 | except FileNotFoundError as fnfe: 75 | logger.error('model file not found...') 76 | raise FileNotFoundError('model file not found ...') 77 | -------------------------------------------------------------------------------- /autodeploy/loader/_model_loader.py: -------------------------------------------------------------------------------- 1 | ''' A simple Model Loader Class. ''' 2 | from loader._loaders import * 3 | from logger import AppLogger 4 | 5 | ALLOWED_MODEL_TYPES = ['pickle', 'hdf5', 'joblib', 'onnx'] 6 | 7 | logger = AppLogger(__name__).get_logger() 8 | 9 | 10 | class ModelLoader: 11 | ''' a driver class ModelLoader to setup and load 12 | model file based on file type. 13 | Args: 14 | model_path (str): model file path. 15 | model_type (str): model file type. 16 | ''' 17 | 18 | def __init__(self, model_path, model_type): 19 | self.model_path = model_path 20 | self.multi_model = False 21 | if isinstance(self.model_path, list): 22 | self.multi_model = True 23 | self.model_type = model_type 24 | 25 | def load(self): 26 | ''' a loading function which loads model file 27 | based on file type. ''' 28 | logger.info('model loading started') 29 | if self.model_type in ALLOWED_MODEL_TYPES: 30 | if self.model_type == 'pickle': 31 | loader = PickleLoader( 32 | self.model_path, multi_model=self.multi_model) 33 | return loader.load() 34 | 35 | elif self.model_type == 'onnx': 36 | loader = OnnxLoader( 37 | self.model_path, multi_model=self.multi_model) 38 | return loader.load() 39 | 40 | else: 41 | logger.error('model type is not allowed') 42 | raise ValueError('Model type is not supported yet!!!') 43 | logger.info('model loaded successfully!') 44 | -------------------------------------------------------------------------------- /autodeploy/logger/__init__.py: -------------------------------------------------------------------------------- 1 | from .logger import AppLogger 2 | -------------------------------------------------------------------------------- /autodeploy/logger/logger.py: -------------------------------------------------------------------------------- 1 | """ A simple app logger utility. """ 2 | import logging 3 | from os import path 4 | 5 | class AppLogger: 6 | ''' a simple logger class. 7 | logger class which takes configuration file and 8 | setups logging module. 9 | 10 | Args: 11 | file_name (str): name of file logger called from i.e `__name__` 12 | 13 | ''' 14 | 15 | def __init__(self, __file_name) -> None: 16 | # Create a custom logger 17 | config_path = path.join(path.dirname(path.abspath(__file__)), 18 | '../config/logging.conf') 19 | logging.config.fileConfig(config_path, disable_existing_loggers=False) 20 | self.__file_name = __file_name 21 | 22 | def get_logger(self): 23 | ''' 24 | a helper function to get logger with file name. 25 | 26 | ''' 27 | # get root logger 28 | logger = logging.getLogger(self.__file_name) 29 | return logger 30 | -------------------------------------------------------------------------------- /autodeploy/monitor.py: -------------------------------------------------------------------------------- 1 | ''' A monitoring utility micro service. ''' 2 | import json 3 | import os 4 | import sys 5 | from typing import Any, List, Dict, Optional, Union 6 | import functools 7 | 8 | import pika 9 | import uvicorn 10 | import numpy as np 11 | from fastapi import FastAPI 12 | from sqlalchemy.orm import Session 13 | from prometheus_client import start_http_server 14 | 15 | from base import BaseMonitorService 16 | from config import Config, InternalConfig 17 | from monitor import Monitor 18 | from database import _models as models 19 | from logger import AppLogger 20 | from monitor import PrometheusModelMetric 21 | from monitor import drift_detection_algorithms 22 | from _backend import RabbitMQConsume, Database 23 | 24 | applogger = AppLogger(__name__) 25 | logger = applogger.get_logger() 26 | 27 | ''' A simple Monitor Driver class. ''' 28 | 29 | 30 | class MonitorDriver(RabbitMQConsume, BaseMonitorService, Database): 31 | ''' 32 | A simple Monitor Driver class for creating monitoring model 33 | and listening to rabbitmq queue i.e Monitor. 34 | 35 | Ref: [https://www.rabbitmq.com/tutorials/tutorial-one-python.html] 36 | RabbitMQ is a message broker: it accepts and forwards messages. 37 | You can think about it as a post office: when you put the mail 38 | that you want posting in a post box, you can be sure that Mr. 39 | or Ms. Mailperson will eventually deliver the mail to your 40 | recipient. In this analogy, RabbitMQ is a post box, a post 41 | office and a postman 42 | 43 | 44 | Protocol: 45 | AMQP - The Advanced Message Queuing Protocol (AMQP) is an open 46 | standard for passing business messages between applications or 47 | organizations. It connects systems, feeds business processes 48 | with the information they need and reliably transmits onward the 49 | instructions that achieve their goals. 50 | 51 | Ref: [https://pika.readthedocs.io/en/stable/] 52 | Pika: 53 | Pika is a pure-Python implementation of the AMQP 0-9-1 protocol 54 | that tries to stay fairly independent of the underlying network 55 | support library. 56 | 57 | Attributes: 58 | config (Config): configuration file contains configuration. 59 | host (str): name of host to connect with rabbitmq server. 60 | queue (str): name of queue to connect to for consuming message. 61 | 62 | ''' 63 | 64 | def __init__(self, config) -> None: 65 | self.config = Config(config).get_config() 66 | self.internal_config = InternalConfig() 67 | super().__init__(self.internal_config) 68 | self.queue = self.internal_config.RABBITMQ_QUEUE 69 | self.drift_detection = None 70 | self.model_metric_port = self.internal_config.MONITOR_PORT 71 | self.database = Database(config) 72 | 73 | def _get_array(self, body: Dict) -> List: 74 | ''' 75 | 76 | A simple internal helper function `_get_array` function 77 | to convert request body to input for model monitoring. 78 | 79 | Args: 80 | body (Dict): a body request incoming to monitor service 81 | from rabbitmq. 82 | 83 | ''' 84 | input_schema = self.config.input_schema 85 | input = [] 86 | 87 | # TODO: do it better 88 | try: 89 | for k in input_schema.keys(): 90 | if self.config.model.input_type == 'serialized' and k == 'input': 91 | input.append(json.loads(body[k])) 92 | else: 93 | input.append(body[k]) 94 | except KeyError as ke: 95 | logger.error(f'{k} key not found') 96 | raise KeyError(f'{k} key not found') 97 | 98 | return [input] 99 | 100 | def _convert_str_to_blob(self, body): 101 | _body = {} 102 | for k, v in body.items(): 103 | if isinstance(v, str): 104 | _body[k] = bytes(v, 'utf-8') 105 | else: 106 | _body[k] = v 107 | return _body 108 | 109 | def _callback(self, ch: Any, method: Any, 110 | properties: Any, body: Dict) -> None: 111 | ''' 112 | a simple callback function attached for post processing on 113 | incoming message body. 114 | 115 | ''' 116 | 117 | try: 118 | body = json.loads(body) 119 | except JSONDecodeError as jde: 120 | logger.error('error while loading json object.') 121 | raise JSONDecodeError('error while loading json object.') 122 | 123 | input = self._get_array(body) 124 | input = np.asarray(input) 125 | output = body['prediction'] 126 | if self.drift_detection: 127 | drift_status = self.drift_detection.get_change(input) 128 | self.prometheus_metric.set_drift_status(drift_status) 129 | 130 | # modify data drift status 131 | body['is_drift'] = drift_status['data']['is_drift'] 132 | 133 | # store request data and model prediction in database. 134 | if self.config.model.input_type == 'serialized': 135 | body = self._convert_str_to_blob(body) 136 | request_store = models.Requests(**dict(body)) 137 | self.database.store_request(request_store) 138 | if self.drift_detection: 139 | logger.info( 140 | f'Data Drift Detection {self.config.monitor.data_drift.name} detected: {drift_status}') 141 | 142 | # expose prometheus_metric metrics 143 | self.prometheus_metric.expose(input, output) 144 | 145 | def _load_monitor_algorithm(self) -> Optional[Union[Monitor, None]]: 146 | reference_data = None 147 | monitor = None 148 | drift_name = self.config.monitor.data_drift.name 149 | reference_data = os.path.join( 150 | self.config.dependency.path, self.config.monitor.data_drift.reference_data) 151 | reference_data = np.load(reference_data) 152 | monitor = Monitor(self.config, drift_name, reference_data) 153 | return monitor 154 | 155 | def prometheus_server(self) -> None: 156 | # Start up the server to expose the metrics. 157 | start_http_server(self.model_metric_port) 158 | 159 | def setup(self, ) -> None: 160 | ''' 161 | a simple setup function to setup rabbitmq channel and queue connection 162 | to start consuming. 163 | 164 | This function setups the loading of model monitoring algorithm from config 165 | and define safe connection to rabbitmq. 166 | 167 | ''' 168 | self.prometheus_metric = PrometheusModelMetric(self.config) 169 | self.prometheus_metric.setup() 170 | if self.config.monitor.get('data_drift', None): 171 | self.drift_detection = self._load_monitor_algorithm() 172 | 173 | # setup rabbitmq channel and queue. 174 | self.setupRabbitMQ(self._callback) 175 | 176 | self.prometheus_server() 177 | 178 | # expose model deployment name. 179 | self.prometheus_metric.model_deployment_name.info( 180 | {'model_name': self.config.model.get('name', 'N/A')}) 181 | 182 | # expose model port name. 183 | self.prometheus_metric.monitor_port.info( 184 | {'model_monitoring_port': str(self.model_metric_port)}) 185 | 186 | # setup database i.e connect and bind. 187 | self.database.setup() 188 | 189 | def __call__(self) -> None: 190 | ''' 191 | __call__ for execution `start_consuming` method for consuming messages 192 | from queue. 193 | 194 | Consumers consume from queues. In order to consume messages there has 195 | to be a queue. When a new consumer is added, assuming there are already 196 | messages ready in the queue, deliveries will start immediately. 197 | 198 | ''' 199 | logger.info(' [*] Waiting for messages. To exit press CTRL+C') 200 | try: 201 | self.channel.start_consuming() 202 | except Exception as e: 203 | raise Exception('uncaught error while consuming message') 204 | finally: 205 | self.database.close() 206 | 207 | 208 | if __name__ == '__main__': 209 | monitordriver = MonitorDriver(os.environ['CONFIG']) 210 | monitordriver.setup() 211 | monitordriver() 212 | -------------------------------------------------------------------------------- /autodeploy/monitor/__init__.py: -------------------------------------------------------------------------------- 1 | from ._drift_detection import DriftDetectionAlgorithmsMixin 2 | from ._monitor import Monitor 3 | from ._prometheus import PrometheusModelMetric 4 | from ._drift_detection import drift_detection_algorithms 5 | -------------------------------------------------------------------------------- /autodeploy/monitor/_drift_detection.py: -------------------------------------------------------------------------------- 1 | """ Drift detection algorithms. """ 2 | import inspect 3 | import sys 4 | 5 | import alibi_detect.cd as alibi_drifts 6 | 7 | drift_detection_algorithms = ['KSDrift', 8 | 'AverageDriftDetection', 'AveragePerDay'] 9 | 10 | # TODO: implement all detection algorithms. 11 | class DriftDetectionAlgorithmsMixin: 12 | ''' 13 | A DriftDetection algorithms Mixin class 14 | contains helper function to setup and 15 | retrive detection algorithms modules. 16 | 17 | ''' 18 | 19 | def get_drift(self, name): 20 | ''' a method to retrive algorithm function 21 | from name. 22 | ''' 23 | _built_in_drift_detecition = {} 24 | _built_in_clsmembers = inspect.getmembers( 25 | sys.modules[__name__], inspect.isclass) 26 | for n, a in _built_in_clsmembers: 27 | _built_in_drift_detecition[n] = a 28 | 29 | if name not in _built_in_drift_detecition.keys(): 30 | return getattr(alibi_drifts, name) 31 | return _built_in_drift_detecition[name] 32 | -------------------------------------------------------------------------------- /autodeploy/monitor/_monitor.py: -------------------------------------------------------------------------------- 1 | """ a simple model data drift detection monitering utilities. """ 2 | from typing import Any, Dict, Text 3 | 4 | import numpy as np 5 | 6 | from config import Config 7 | from monitor import DriftDetectionAlgorithmsMixin 8 | 9 | 10 | class Monitor(DriftDetectionAlgorithmsMixin): 11 | ''' 12 | a simple `Monitor` class to drive and setup monitoring 13 | algorithms for monitor of inputs to the model 14 | 15 | Args: 16 | config (Config): a configuration instance. 17 | 18 | ''' 19 | 20 | def __init__(self, config: Config, drift_name: Text, 21 | reference_data: np.ndarray, *args, **kwargs) -> None: 22 | super(Monitor, self).__init__(*args, *kwargs) 23 | self.config = config 24 | _data_drift_instance = self.get_drift(drift_name) 25 | self._data_drift_instance = _data_drift_instance(reference_data) 26 | 27 | def get_change(self, x: np.ndarray) -> Dict: 28 | ''' 29 | a helper funciton to get data drift. 30 | 31 | ''' 32 | _status = self._data_drift_instance.predict(x) 33 | return _status 34 | -------------------------------------------------------------------------------- /autodeploy/monitor/_prometheus.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | import os 3 | import operator 4 | import glob 5 | import sys 6 | import importlib 7 | from typing import Dict 8 | 9 | from prometheus_client import Counter, Gauge, Summary, Info, Enum 10 | 11 | from monitor._drift_detection import drift_detection_algorithms 12 | from register import METRICS 13 | from logger import AppLogger 14 | 15 | logger = AppLogger(__name__).get_logger() 16 | 17 | 18 | _metric_types = {'gauge': Gauge, 'summary': Summary, 19 | 'counter': Counter, 'info': Info} 20 | 21 | 22 | class PrometheusModelMetric: 23 | ''' 24 | a `PrometheusModelMetric` class to setup and define 25 | prometheus_client metrics default and custom metrics 26 | from config file. 27 | 28 | Args: 29 | config (Config): a configuration instance. 30 | data_drift (Any): a data drift detection fxn. 31 | 32 | ''' 33 | 34 | def __init__(self, config) -> None: 35 | self.config = config 36 | self._metrics = defaultdict() 37 | self.custom_metric_fxn_name = config.monitor.get('custom_metrics', None) 38 | self.custom_metrics = None 39 | self.data_drift = None 40 | data_drift_meta = self.config.monitor.get('data_drift', None) 41 | if data_drift_meta: 42 | self.data_drift = Info(data_drift_meta.name, 'N/A') 43 | self.drift_status = None 44 | 45 | self.metric_list = [] 46 | 47 | metrics_meta = self.config.monitor.metrics 48 | if metrics_meta: 49 | for k, v in metrics_meta.items(): 50 | self._metrics.update( 51 | {k: _metric_types[v['type']](k, 'N/A')}) 52 | 53 | @staticmethod 54 | def convert_python_path(file): 55 | # TODO: check os. 56 | file = file.split('.')[-2] 57 | file = file.split('/')[-1] 58 | return file 59 | 60 | def import_custom_metric_files(self): 61 | try: 62 | path = self.config.dependency.path 63 | sys.path.append(path) 64 | _py = glob.glob(os.path.join(path, '*.py')) 65 | for file in _py: 66 | file = self.convert_python_path(file) 67 | if 'metric' in file: 68 | importlib.import_module(file) 69 | except ImportError as ie: 70 | logger.error('could not import dependency from given path.') 71 | raise ImportError('could not import dependency from given path.') 72 | 73 | def get_custom_metrics(self): 74 | ''' a fxn to get custom metrics dict or list from model dependencies module. ''' 75 | _fxns = METRICS.module_dict 76 | if self.custom_metric_fxn_name: 77 | if isinstance(self.custom_metric_fxn_name, list): 78 | _fxn_dict = {} 79 | _fxn_metrics = operator.itemgetter( 80 | *self.custom_metric_fxn_name)(METRICS.module_dict) 81 | for i, name in enumerate(self.custom_metric_fxn_name): 82 | _fxn_dict[name] = _fxn_metrics[i] 83 | return _fxn_dict 84 | elif isinstance(self.custom_metric_fxn_name, str): 85 | try: 86 | return METRICS.module_dict[self.custom_metric_fxn_name] 87 | except KeyError as ke: 88 | logger.error( 89 | f'{self.custom_metric_fxn_name} not found in {METRICS.keys()} keys') 90 | raise KeyError( 91 | f'{self.custom_metric_fxn_name} not found in {METRICS.keys()} keys') 92 | else: 93 | logger.error( 94 | f'wrong custom metric type {type(self.custom_metric_fxn_name)}, `list` or `str` expected.') 95 | raise Exception( 96 | f'wrong custom metric type {type(self.custom_metric_fxn_name)}, `list` or `str` expected.') 97 | if _fxns: 98 | return _fxns 99 | return None 100 | 101 | def default_model_metric(self): 102 | ''' 103 | a default_model_metric method which contains default 104 | model prometheus_client metrics. 105 | 106 | monitor_state: monitoring service state. 107 | monitor_port: stores monitor port number. 108 | model_deployment_name: stores model name. 109 | 110 | ''' 111 | self.monitor_state = Enum( 112 | 'monitor_service_state', 'Monitoring service state i.e up or down', states=['up', 'down']) 113 | self.monitor_port = Info('monitor_port', 'monitor service port number') 114 | self.model_deployment_name = Info( 115 | 'model_deployment_name', 'Name for model deployment.') 116 | self.model_output_score = Gauge( 117 | 'model_output_score', 'This is gauge to output model score.') 118 | self.data_drift_out = Gauge( 119 | 'data_drift_out', 'This is data drift output i.e binary.') 120 | 121 | def set_metrics_attributes(self): 122 | ''' 123 | a helper function to set custom metrics from 124 | config file in prometheus_client format. 125 | 126 | ''' 127 | for k, v in self._metrics.items(): 128 | setattr(self, k, v) 129 | 130 | def convert_str(self, status: Dict): 131 | ''' a helper function to convert dict to str ''' 132 | _dict = {} 133 | for k, v in status.items(): 134 | _dict.update({k: str(v)}) 135 | return _dict 136 | 137 | def expose(self, input, output): 138 | ''' Expose method to expose metrics to prometheus server. ''' 139 | if self.drift_status: 140 | status = self.convert_str(self.drift_status) 141 | self.data_drift.info(status) 142 | self.data_drift_out.set(self.drift_status['data']['is_drift']) 143 | 144 | self.monitor_state.state('up') 145 | self.model_output_score.set(output) 146 | for metric in self.metric_list: 147 | result = metric(input) 148 | self._metrics[metric.__name__].set(result) 149 | 150 | if self.custom_metrics: 151 | if isinstance(self.custom_metrics, dict): 152 | for name, metric in self.custom_metrics.items(): 153 | result = metric(input) 154 | self._metrics[name].set(result) 155 | elif callable(self.custom_metrics): 156 | result = self.custom_metrics(input) 157 | self._metrics[self.custom_metric_fxn_name].set(result) 158 | 159 | def setup_custom_metric(self): 160 | ''' a simple setup custom metric function to set 161 | custom functions. 162 | ''' 163 | custom_metrics = self.get_custom_metrics() 164 | self.custom_metrics = custom_metrics 165 | if isinstance(custom_metrics, dict): 166 | for name, module in custom_metrics.items(): 167 | self._metrics[name] = Gauge(name, 'N/A') 168 | elif callable(custom_metrics): 169 | self._metrics[self.custom_metric_fxn_name] = Gauge( 170 | self.custom_metric_fxn_name, 'N/A') 171 | 172 | def set_drift_status(self, status): 173 | self.drift_status = status 174 | 175 | def setup(self): 176 | ''' 177 | A setup function which binds custom and default prometheus_client 178 | metrics to PrometheusModelMetric class. 179 | 180 | ''' 181 | self.import_custom_metric_files() 182 | self.setup_custom_metric() 183 | self.set_metrics_attributes() 184 | self.default_model_metric() 185 | -------------------------------------------------------------------------------- /autodeploy/predict.py: -------------------------------------------------------------------------------- 1 | """ A simple prediction service utility.""" 2 | import traceback 3 | import argparse 4 | import requests 5 | import os 6 | import json 7 | from typing import List, Dict, Tuple, Any 8 | from datetime import datetime 9 | 10 | import uvicorn 11 | import numpy as np 12 | from prometheus_fastapi_instrumentator import Instrumentator 13 | from fastapi import Depends, FastAPI, Request, HTTPException 14 | 15 | from config.config import Config, InternalConfig 16 | from utils import utils 17 | from logger import AppLogger 18 | from routers import PredictRouter 19 | from routers import prediction_router 20 | from base import BaseDriverService 21 | 22 | 23 | # __main__ (root) logger instance construction. 24 | applogger = AppLogger(__name__) 25 | logger = applogger.get_logger() 26 | 27 | 28 | class PredictDriver(BaseDriverService): 29 | ''' 30 | PredictDriver class for creating prediction driver which 31 | takes input tensor and predicts with loaded model. 32 | 33 | This class is the main driver class responsible for creating 34 | and setupping the environment. 35 | Args: 36 | config (str): a config path. 37 | 38 | Note: 39 | `setup` method should get called before `register_routers`. 40 | 41 | ''' 42 | 43 | def __init__(self, config_path) -> None: 44 | # user config for configuring model deployment. 45 | self.user_config = Config(config_path).get_config() 46 | self.internal_config = InternalConfig() 47 | 48 | def setup(self, app) -> None: 49 | ''' 50 | Main setup function responsible for setting up 51 | the environment for model deployment. 52 | setups prediction and model routers. 53 | 54 | Setups Prometheus instrumentor 55 | ''' 56 | 57 | # expose prometheus data to /metrics 58 | Instrumentator().instrument(app).expose(app) 59 | 60 | apirouter = PredictRouter(self.user_config) 61 | apirouter.setup() 62 | apirouter.register_router() 63 | 64 | # setup exception handlers 65 | self.handler_setup(app) 66 | 67 | def register_routers(self, app): 68 | ''' 69 | a helper function to register routers in the app. 70 | ''' 71 | self._app_include([prediction_router], app) 72 | 73 | def run(self, app): 74 | ''' 75 | The main executing function which runs the uvicorn server 76 | with the app instance and user configuration. 77 | ''' 78 | # run uvicorn server. 79 | uvicorn.run(app, port=self.internal_config.PREDICT_PORT, host="0.0.0.0") 80 | 81 | 82 | def main(): 83 | # create fastapi application 84 | app = FastAPI() 85 | deploydriver = PredictDriver(os.environ['CONFIG']) 86 | deploydriver.setup(app) 87 | deploydriver.register_routers(app) 88 | deploydriver.run(app) 89 | 90 | 91 | if __name__ == "__main__": 92 | main() 93 | -------------------------------------------------------------------------------- /autodeploy/predict/__init__.py: -------------------------------------------------------------------------------- 1 | from ._infere import * 2 | -------------------------------------------------------------------------------- /autodeploy/predict/_infere.py: -------------------------------------------------------------------------------- 1 | """ Inference model classes """ 2 | import sklearn 3 | import numpy as np 4 | import cv2 5 | from fastapi.exceptions import RequestValidationError 6 | 7 | from register.register import INFER 8 | from base import BaseInfere 9 | 10 | 11 | @INFER.register_module(name='sklearn') 12 | class SkLearnInfere(BaseInfere): 13 | """ a SKLearn inference class. 14 | Args: 15 | config (Config): a configuring instance. 16 | model (Any): prediction model instance. 17 | """ 18 | 19 | def __init__(self, user_config, model): 20 | self.config = user_config 21 | self.model = model 22 | 23 | def infere(self, input): 24 | assert type(input) in [np.ndarray, list], 'Model input are not valid!' 25 | class_probablities = self.model.predict_proba([input]) 26 | out_class = np.argmax(class_probablities) 27 | return class_probablities[0][out_class], out_class 28 | 29 | @INFER.register_module(name='onnx') 30 | class OnnxInfere(BaseInfere): 31 | """ a Onnx inference class. 32 | Args: 33 | config (Config): a configuring instance. 34 | model (Any): prediction model instance. 35 | input_name (str): Onnx model input layer name. 36 | label_name (str): Onnx model output layer name. 37 | """ 38 | 39 | def __init__(self, user_config, model): 40 | self.config = user_config 41 | self.model = model 42 | self.input_name = self.model.get_inputs()[0].name 43 | self.label_name = self.model.get_outputs()[0].name 44 | 45 | def infere(self, input): 46 | ''' 47 | inference method to predict with onnx model 48 | on input data. 49 | 50 | Args: 51 | input (ndarray): numpy input array. 52 | 53 | ''' 54 | assert type(input) in [np.ndarray, list], 'Model input are not valid!' 55 | pred_onx = self.model.run( 56 | [self.label_name], {self.input_name: [input]})[0] 57 | return pred_onx 58 | -------------------------------------------------------------------------------- /autodeploy/predict/builder.py: -------------------------------------------------------------------------------- 1 | ''' Inference Model Builder classes. ''' 2 | import random 3 | 4 | import numpy as np 5 | 6 | from register.register import INFER 7 | from logger import AppLogger 8 | 9 | logger = AppLogger(__name__).get_logger() 10 | 11 | ''' A simple inference model builder. ''' 12 | 13 | 14 | class InfereBuilder: 15 | ''' 16 | A simple inference builder class which binds inference 17 | fxn with appropiate class from configuration file. 18 | 19 | Inference builder class builds request AB testing routing 20 | to route to correct model for inference according to the 21 | ab split provided in model configuration. 22 | 23 | Args: 24 | config (Config): configuration file. 25 | model (Model): list of models loaded for inference. 26 | 27 | ''' 28 | 29 | def __init__(self, config, model_list): 30 | self.config = config 31 | self.multi_model = False 32 | self.num_models = 1 33 | self._version = self.config.model.get('version', 'n.a') 34 | if len(model_list) > 1: 35 | self.num_models = len(model_list) 36 | self.multi_model = True 37 | logger.info('running multiple models..') 38 | 39 | _infere_class = INFER.get(self.config.model.model_type) 40 | self._infere_class_instances = [] 41 | 42 | for model in model_list: 43 | self._infere_class_instances.append(_infere_class(config, model)) 44 | 45 | def _get_split(self): 46 | ''' 47 | a helper function to get split weights for choice 48 | of routing model for inference. 49 | 50 | ''' 51 | if not self.config.model.get('ab_split', None): 52 | return [1 / self.num_models] * self.num_models 53 | return self.config.model.ab_split 54 | 55 | def _get_model_id_from_split(self): 56 | ''' 57 | a helper function to get model id to route request. 58 | 59 | ''' 60 | return random.choices(np.arange(self.num_models), 61 | weights=self._get_split())[0] 62 | 63 | def get_inference(self, input): 64 | ''' 65 | a function to get inference from model routed from split. 66 | ''' 67 | _idx = 0 68 | if self.multi_model: 69 | _idx = self._get_model_id_from_split() 70 | model_infere_detail = {'model': 'primary' if _idx == 71 | 0 else 'non_primary', 'id': _idx, 'version': self._version} 72 | logger.info(model_infere_detail) 73 | return self._infere_class_instances[_idx].infere( 74 | input), model_infere_detail 75 | -------------------------------------------------------------------------------- /autodeploy/register/__init__.py: -------------------------------------------------------------------------------- 1 | from .register import INFER, PREPROCESS, POSTPROCESS, METRICS 2 | -------------------------------------------------------------------------------- /autodeploy/register/register.py: -------------------------------------------------------------------------------- 1 | ''' a register for registering Inference, preprocess and postprocess modules. ''' 2 | from utils.registry import Registry 3 | 4 | INFER = Registry('Inference') 5 | PREPROCESS = Registry('Preprocess') 6 | POSTPROCESS = Registry('Postprocess') 7 | METRICS = Registry('Postprocess') 8 | -------------------------------------------------------------------------------- /autodeploy/routers/__init__.py: -------------------------------------------------------------------------------- 1 | from ._model import ModelDetailRouter, router as model_detail_router 2 | from ._predict import PredictRouter, router as prediction_router 3 | from ._api import AutoDeployRouter, router as api_router 4 | from ._security import login, router as auth_router 5 | -------------------------------------------------------------------------------- /autodeploy/routers/_api.py: -------------------------------------------------------------------------------- 1 | ''' a simple api router. ''' 2 | import traceback 3 | import argparse 4 | import os 5 | import requests 6 | from typing import List, Optional, Union, Text, Any 7 | import json 8 | from datetime import datetime 9 | 10 | import uvicorn 11 | import validators 12 | import pika 13 | import numpy as np 14 | from fastapi import APIRouter 15 | from fastapi import HTTPException 16 | from fastapi import Depends 17 | from sqlalchemy.orm import Session 18 | 19 | from config.config import Config 20 | from config.config import InternalConfig 21 | from utils import utils 22 | from handlers import Handler, ModelException 23 | from loader import ModelLoader 24 | from predict.builder import InfereBuilder 25 | from logger import AppLogger 26 | from database import _database as database, _models as models 27 | from dependencies import LoadDependency 28 | from logger import AppLogger 29 | from security.scheme import oauth2_scheme 30 | from _backend import Database, RabbitMQClient 31 | from _backend import RedisDB 32 | 33 | 34 | router = APIRouter() 35 | 36 | applogger = AppLogger(__name__) 37 | logger = applogger.get_logger() 38 | 39 | 40 | class AutoDeployRouter(RabbitMQClient, Database): 41 | ''' a simple api router class 42 | which routes api endpoint to fastapi 43 | application. 44 | 45 | Args: 46 | user_config (Config): a configuration instance. 47 | dependencies (LoadDependency): instance of LoadDependency. 48 | port (int): port number for monitor service 49 | host (str): hostname. 50 | Raises: 51 | BaseException: model prediction exception. 52 | 53 | 54 | ''' 55 | 56 | def __init__(self, config: Config, internal_config: InternalConfig) -> None: 57 | # user config for configuring model deployment. 58 | self.user_config = config 59 | self.internal_config = internal_config 60 | super(AutoDeployRouter, self).__init__(internal_config) 61 | self.dependencies = None 62 | 63 | # redis backend instance. 64 | 65 | self.backend_redis = RedisDB(self.internal_config) 66 | if config.get('dependency', None): 67 | self.dependencies = LoadDependency( 68 | config) 69 | 70 | self._dependency_fxn = None 71 | self._post_dependency_fxn = None 72 | self._protected = config.model.get('protected', False) 73 | 74 | def setup(self, schemas): 75 | ''' setup api endpoint method. 76 | 77 | Args: 78 | schemas (tuple): a user input and output schema tuple. 79 | ''' 80 | if self.user_config.model.model_type == 'onnx' and not self.user_config.get('postprocess', None): 81 | logger.error('Postprocess is required in model type `ONNX`.') 82 | raise Exception('Postprocess is required in model type `ONNX`.') 83 | 84 | self.input_model_schema = schemas[0] 85 | self.output_model_schema = schemas[1] 86 | _out_prop = self.output_model_schema.UserOutputSchema.schema()['properties'] 87 | 88 | if 'confidence' not in _out_prop.keys(): 89 | raise Exception('confidence is required in out schema.') 90 | 91 | if 'number' not in _out_prop['confidence']['type']: 92 | raise Exception('confidence should be a float type in out schema') 93 | 94 | # create database connection. 95 | self.bind() 96 | 97 | # create model loader instance. 98 | if isinstance(self.user_config.model.model_path, str): 99 | model_path = os.path.join(self.user_config.dependency.path, 100 | self.user_config.model.model_path) 101 | elif isinstance(self.user_config.model.model_path, list): 102 | model_path = [os.path.join(self.user_config.dependency.path, _model_path) 103 | for _model_path in self.user_config.model.model_path] 104 | else: 105 | logger.error('Invalid model path!!') 106 | raise Exception('Invalid model path!!') 107 | 108 | _model_loader = ModelLoader( 109 | model_path, self.user_config.model.model_file_type) 110 | __model = _model_loader.load() 111 | 112 | # inference model builder. 113 | self.__inference_executor = InfereBuilder(self.user_config, __model) 114 | 115 | # setupRabbitMq 116 | self.setupRabbitMq() 117 | 118 | self.dependencies.import_dependencies() 119 | 120 | # pick one function to register 121 | # TODO: picks first preprocess function. 122 | self._dependency_fxn = self.dependencies.preprocess_fxn 123 | 124 | self._post_dependency_fxn = self.dependencies.postprocess_fxn 125 | 126 | # input dtype. 127 | self.input_dtype = self.internal_config.PREDICT_INPUT_DTYPE 128 | 129 | def get_out_response(self, model_output): 130 | ''' a helper function to get ouput response. ''' 131 | # TODO: change status code. 132 | return {'out': model_output[1], 133 | 'confidence': model_output[0], 'status': 200} 134 | 135 | def _build_predict_url(self, input_hash_id, _input_array): 136 | _PREDICT_URL = f'http://{self.internal_config.PREDICT_URL}:{self.internal_config.PREDICT_PORT}/model_predict' 137 | return f'{_PREDICT_URL}?id={input_hash_id}&ndim={len(_input_array.shape)}' 138 | 139 | def register_router(self): 140 | ''' a main router registering funciton 141 | which registers the prediction service to 142 | user defined endpoint. 143 | ''' 144 | user_config = self.user_config 145 | input_model_schema = self.input_model_schema 146 | output_model_schema = self.output_model_schema 147 | preprocess_fxn = self._dependency_fxn 148 | _protected = self._protected 149 | postprocess_fxn = self._post_dependency_fxn 150 | 151 | @router.post(f'/{user_config.model.endpoint}', 152 | response_model=output_model_schema.UserOutputSchema) 153 | async def api_endpoint(payload: input_model_schema.UserInputSchema, token: Optional[Union[Text, Any]] = Depends(oauth2_scheme) if _protected else None): 154 | nonlocal self 155 | try: 156 | _input_array = [] 157 | 158 | # TODO: make a class for reading input payload. 159 | _input_type = self.user_config.model.get('input_type', 'na') 160 | if _input_type == 'structured': 161 | _input_array = np.asarray([v for k, v in payload]) 162 | 163 | elif _input_type == 'serialized': 164 | for k, v in payload: 165 | if isinstance(v, str): 166 | v = np.asarray(json.loads(v)) 167 | _input_array.append(v) 168 | elif _input_type == 'url': 169 | for k, v in payload: 170 | if validators.url(v): 171 | _input_array.append(utils.url_loader(v)) 172 | else: 173 | _input_array.append(v) 174 | if not _input_array: 175 | logger.critical('Could not read input payload.') 176 | raise ValueError('Could not read input payload.') 177 | 178 | cache = None 179 | if preprocess_fxn: 180 | _input_array = preprocess_fxn(_input_array) 181 | 182 | if isinstance(_input_array, tuple): 183 | _input_array = _input_array, cache 184 | 185 | input_hash_id = self.backend_redis.push( 186 | _input_array, dtype=self.input_dtype, shape=_input_array.shape) 187 | 188 | out_response = requests.post( 189 | self._build_predict_url(input_hash_id, _input_array)).json() 190 | 191 | except BaseException: 192 | logger.critical('Uncaught exception: %s', traceback.format_exc()) 193 | raise ModelException(name='structured_server') 194 | else: 195 | logger.debug('Model predict successfull.') 196 | 197 | _time_stamp = datetime.now() 198 | _request_store = {'time_stamp': str( 199 | _time_stamp), 'prediction': out_response['confidence'], 'is_drift': False} 200 | _request_store.update(dict(payload)) 201 | self.publish_rbmq(_request_store) 202 | 203 | return out_response 204 | -------------------------------------------------------------------------------- /autodeploy/routers/_model.py: -------------------------------------------------------------------------------- 1 | ''' a simple Model details router utilities. ''' 2 | import traceback 3 | 4 | from fastapi import APIRouter 5 | from fastapi import Request 6 | 7 | from logger import AppLogger 8 | 9 | router = APIRouter() 10 | 11 | applogger = AppLogger(__name__) 12 | logger = applogger.get_logger() 13 | 14 | 15 | @router.on_event('startup') 16 | async def startup(): 17 | logger.info('Connected to server...!!') 18 | 19 | 20 | class ModelDetailRouter: 21 | ''' 22 | a `ModelDetailRouter` class which routes `\model` 23 | to fastapi application. 24 | model detail endpoint gives model related details 25 | example: 26 | model: name of model. 27 | version: model version details. 28 | 29 | Args: 30 | user_config (Config): configuration instance. 31 | 32 | ''' 33 | 34 | def __init__(self, config) -> None: 35 | self.user_config = config 36 | 37 | def register_router(self): 38 | ''' 39 | a method to register model router to fastapi router. 40 | 41 | ''' 42 | @router.get('/model') 43 | async def model_details(): 44 | nonlocal self 45 | logger.debug('model detail request incomming.') 46 | try: 47 | out_response = {'model': self.user_config.model.model_name, 48 | 'version': self.user_config.model.version} 49 | except KeyError as e: 50 | logger.error('Please define model name and version in config.') 51 | except BaseException: 52 | logger.error("Uncaught exception: %s", traceback.format_exc()) 53 | return out_response 54 | -------------------------------------------------------------------------------- /autodeploy/routers/_predict.py: -------------------------------------------------------------------------------- 1 | ''' a simple prediction router. ''' 2 | import traceback 3 | import argparse 4 | import os 5 | import requests 6 | from typing import List, Optional, Union, Text, Any 7 | import json 8 | from datetime import datetime 9 | 10 | import uvicorn 11 | import validators 12 | import pika 13 | import numpy as np 14 | from fastapi import APIRouter 15 | from fastapi import HTTPException 16 | from fastapi import Depends 17 | from sqlalchemy.orm import Session 18 | 19 | from config.config import Config 20 | from utils import utils 21 | from handlers import Handler, ModelException 22 | from loader import ModelLoader 23 | from predict.builder import InfereBuilder 24 | from logger import AppLogger 25 | from database import _database as database, _models as models 26 | from dependencies import LoadDependency 27 | from logger import AppLogger 28 | from security.scheme import oauth2_scheme 29 | from _backend import Database, RabbitMQClient 30 | from _backend import RedisDB 31 | from dependencies import LoadDependency 32 | from config.config import InternalConfig 33 | 34 | 35 | router = APIRouter() 36 | 37 | applogger = AppLogger(__name__) 38 | logger = applogger.get_logger() 39 | 40 | 41 | class PredictRouter(RabbitMQClient, Database): 42 | ''' a simple prediction router class 43 | which routes model prediction endpoint to fastapi 44 | application. 45 | 46 | Args: 47 | user_config (Config): a configuration instance. 48 | dependencies (LoadDependency): instance of LoadDependency. 49 | port (int): port number for monitor service 50 | host (str): hostname. 51 | Raises: 52 | BaseException: model prediction exception. 53 | ''' 54 | 55 | def __init__(self, config: Config) -> None: 56 | # user config for configuring model deployment. 57 | self.user_config = config 58 | self.internal_config = InternalConfig() 59 | super(PredictRouter, self).__init__(self.internal_config) 60 | self.backend_redis = RedisDB(self.internal_config) 61 | if config.get('dependency', None): 62 | self.dependencies = LoadDependency( 63 | config) 64 | self._post_dependency_fxn = None 65 | 66 | def setup(self): 67 | ''' setup api endpoint method. 68 | ''' 69 | if self.user_config.model.model_type == 'onnx' and not self.user_config.get('postprocess', None): 70 | logger.error('Postprocess is required in model type `ONNX`.') 71 | raise Exception('Postprocess is required in model type `ONNX`.') 72 | 73 | # create model loader instance. 74 | if isinstance(self.user_config.model.model_path, str): 75 | model_path = os.path.join(self.user_config.dependency.path, 76 | self.user_config.model.model_path) 77 | elif isinstance(self.user_config.model.model_path, list): 78 | model_path = [os.path.join(self.user_config.dependency.path, _model_path) 79 | for _model_path in self.user_config.model.model_path] 80 | else: 81 | logger.error('Invalid model path!!') 82 | raise Exception('Invalid model path!!') 83 | _model_loader = ModelLoader( 84 | model_path, self.user_config.model.model_file_type) 85 | __model = _model_loader.load() 86 | self.dependencies.import_dependencies() 87 | 88 | self._post_dependency_fxn = self.dependencies.postprocess_fxn 89 | 90 | # inference model builder. 91 | self.__inference_executor = InfereBuilder(self.user_config, __model) 92 | 93 | def read_input_array(self, id, ndim): 94 | input = self.backend_redis.pull( 95 | id, dtype=self.internal_config.PREDICT_INPUT_DTYPE, ndim=int(ndim)) 96 | return input 97 | 98 | def register_router(self): 99 | ''' a main router registering funciton 100 | which registers the prediction service to 101 | user defined endpoint. 102 | ''' 103 | user_config = self.user_config 104 | postprocess_fxn = self._post_dependency_fxn 105 | 106 | @router.post('/model_predict') 107 | async def structured_server(id, ndim): 108 | nonlocal self 109 | try: 110 | _input_array = self.read_input_array(id, ndim) 111 | 112 | # model inference/prediction. 113 | model_output, _ = self.__inference_executor.get_inference( 114 | _input_array) 115 | 116 | if postprocess_fxn: 117 | out_response = postprocess_fxn(model_output) 118 | else: 119 | out_response = self.get_out_response(model_output) 120 | 121 | except BaseException: 122 | logger.error('uncaught exception: %s', traceback.format_exc()) 123 | raise ModelException(name='structured_server') 124 | else: 125 | logger.debug('model predict successfull.') 126 | return out_response 127 | -------------------------------------------------------------------------------- /autodeploy/routers/_security.py: -------------------------------------------------------------------------------- 1 | ''' a simple Security router utilities. ''' 2 | from typing import Optional 3 | 4 | from fastapi import APIRouter 5 | from fastapi import Depends, FastAPI, HTTPException, status 6 | from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm 7 | from pydantic import BaseModel 8 | 9 | from security.scheme import oauth2_scheme, fake_users_db, get_user, fake_decode_token, fake_hash_password 10 | from _schema._security import User, UserInDB 11 | 12 | router = APIRouter( 13 | prefix="/token", 14 | tags=["security"], 15 | responses={} 16 | ) 17 | @router.post("/") 18 | async def login(form_data: OAuth2PasswordRequestForm = Depends()): 19 | user_dict = fake_users_db.get(form_data.username) 20 | if not user_dict: 21 | raise HTTPException(status_code=400, detail="Incorrect username or password") 22 | user = UserInDB(**user_dict) 23 | hashed_password = fake_hash_password(form_data.password) 24 | if not hashed_password == user.hashed_password: 25 | raise HTTPException(status_code=400, detail="Incorrect username or password") 26 | 27 | return {"access_token": user.username, "token_type": "bearer"} 28 | -------------------------------------------------------------------------------- /autodeploy/security/scheme.py: -------------------------------------------------------------------------------- 1 | ''' security JWT token utilities. ''' 2 | from fastapi.security import OAuth2PasswordBearer 3 | from typing import Optional 4 | from fastapi import Depends, FastAPI, HTTPException, status 5 | from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm 6 | from pydantic import BaseModel 7 | 8 | from _schema._security import User, UserInDB 9 | 10 | oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") 11 | 12 | ''' 13 | Some fake users. 14 | 15 | TODO: These should be in a database 16 | ''' 17 | fake_users_db = { 18 | "johndoe": { 19 | "username": "johndoe", 20 | "full_name": "John Doe", 21 | "email": "johndoe@example.com", 22 | "hashed_password": "fakehashedsecret", 23 | "disabled": False, 24 | }, 25 | "alice": { 26 | "username": "alice", 27 | "full_name": "Alice Wonderson", 28 | "email": "alice@example.com", 29 | "hashed_password": "fakehashedsecret2", 30 | "disabled": True, 31 | }, 32 | } 33 | 34 | 35 | def get_user(db, username: str): 36 | if username in db: 37 | user_dict = db[username] 38 | return UserInDB(**user_dict) 39 | 40 | 41 | def fake_decode_token(token): 42 | ''' 43 | a helper function to decode token. 44 | TODO: 45 | This doesn't provide any security at all 46 | Check the next version 47 | ''' 48 | user = get_user(fake_users_db, token) 49 | return user 50 | 51 | 52 | async def get_current_user(token: str = Depends(oauth2_scheme)): 53 | user = fake_decode_token(token) 54 | if not user: 55 | raise HTTPException( 56 | status_code=status.HTTP_401_UNAUTHORIZED, 57 | detail="Invalid authentication credentials", 58 | headers={"WWW-Authenticate": "Bearer"}, 59 | ) 60 | return user 61 | 62 | 63 | async def get_current_active_user(current_user: User = Depends(get_current_user)): 64 | if current_user.disabled: 65 | raise HTTPException(status_code=400, detail="Inactive user") 66 | return current_user 67 | 68 | def fake_hash_password(password: str): 69 | return password 70 | 71 | 72 | def get_user(db, username: str): 73 | if username in db: 74 | user_dict = db[username] 75 | return UserInDB(**user_dict) 76 | 77 | 78 | def fake_decode_token(token): 79 | # This doesn't provide any security at all 80 | # Check the next version 81 | user = get_user(fake_users_db, token) 82 | return user 83 | 84 | 85 | async def get_current_user(token: str = Depends(oauth2_scheme)): 86 | user = fake_decode_token(token) 87 | if not user: 88 | raise HTTPException( 89 | status_code=status.HTTP_401_UNAUTHORIZED, 90 | detail="Invalid authentication credentials", 91 | headers={"WWW-Authenticate": "Bearer"}, 92 | ) 93 | return user 94 | 95 | 96 | async def get_current_active_user(current_user: User = Depends(get_current_user)): 97 | if current_user.disabled: 98 | raise HTTPException(status_code=400, detail="Inactive user") 99 | return current_user 100 | -------------------------------------------------------------------------------- /autodeploy/service/_infere.py: -------------------------------------------------------------------------------- 1 | """ Inference model classes """ 2 | import sklearn 3 | import numpy as np 4 | import cv2 5 | from fastapi.exceptions import RequestValidationError 6 | 7 | from register.register import INFER 8 | from base import BaseInfere 9 | 10 | 11 | @INFER.register_module(name='sklearn') 12 | class SkLearnInfere(BaseInfere): 13 | """ a SKLearn inference class. 14 | Args: 15 | config (Config): a configuring instance. 16 | model (Any): prediction model instance. 17 | """ 18 | 19 | def __init__(self, user_config, model): 20 | self.config = user_config 21 | self.model = model 22 | 23 | def infere(self, input): 24 | assert type(input) in [np.ndarray, list], 'Model input are not valid!' 25 | class_probablities = self.model.predict_proba([input]) 26 | out_class = np.argmax(class_probablities) 27 | return class_probablities[0][out_class], out_class 28 | 29 | @INFER.register_module(name='onnx') 30 | class OnnxInfere(BaseInfere): 31 | """ a Onnx inference class. 32 | Args: 33 | config (Config): a configuring instance. 34 | model (Any): prediction model instance. 35 | input_name (str): Onnx model input layer name. 36 | label_name (str): Onnx model output layer name. 37 | """ 38 | 39 | def __init__(self, user_config, model): 40 | self.config = user_config 41 | self.model = model 42 | self.input_name = self.model.get_inputs()[0].name 43 | self.label_name = self.model.get_outputs()[0].name 44 | 45 | def infere(self, input): 46 | ''' 47 | inference method to predict with onnx model 48 | on input data. 49 | 50 | Args: 51 | input (ndarray): numpy input array. 52 | 53 | ''' 54 | assert type(input) in [np.ndarray, list], 'Model input are not valid!' 55 | pred_onx = self.model.run( 56 | [self.label_name], {self.input_name: [input]})[0] 57 | return pred_onx 58 | -------------------------------------------------------------------------------- /autodeploy/testing/load_test.py: -------------------------------------------------------------------------------- 1 | """ a simple locust file for load testing. """ 2 | import os 3 | import json 4 | 5 | import numpy as np 6 | from locust import HttpUser, task 7 | 8 | from config.config import Config 9 | from utils import utils 10 | 11 | user_config = Config(os.environ['CONFIG']).get_config() 12 | 13 | 14 | class LoadTesting(HttpUser): 15 | """ LoadTesting class for load testing. """ 16 | 17 | @task 18 | def load_test_request(self): 19 | global user_config 20 | """ a simple request load test task. """ 21 | if user_config.model.input_type == 'serialized': 22 | input = utils.generate_random_from_schema( 23 | user_config.input_schema, shape=user_config.model.input_shape, serialized=True) 24 | elif user_config.mode.input_type == 'structured': 25 | input = utils.generate_random_from_schema(user_config.input_schema) 26 | self.client.post(f"/{user_config.model.endpoint}", json=input) 27 | -------------------------------------------------------------------------------- /autodeploy/utils/registry.py: -------------------------------------------------------------------------------- 1 | ''' autodeploy register utilities. ''' 2 | import inspect 3 | import warnings 4 | from functools import partial 5 | 6 | 7 | class Registry: 8 | """Registry. 9 | Registry Class which stores module references which can be used to 10 | apply pluging architecture and achieve flexiblity. 11 | """ 12 | 13 | def __init__(self, name): 14 | """__init__. 15 | 16 | Args: 17 | name: 18 | """ 19 | self._name = name 20 | self._module_dict = dict() 21 | 22 | def __len__(self): 23 | """__len__.""" 24 | return len(self._module_dict) 25 | 26 | def __contains__(self, key): 27 | """__contains__. 28 | 29 | Args: 30 | key: 31 | """ 32 | return self.get(key) is not None 33 | 34 | def __repr__(self): 35 | """__repr__.""" 36 | format_str = ( 37 | self.__class__.__name__ + f"(name={self._name}, " 38 | f"items={self._module_dict})" 39 | ) 40 | return format_str 41 | 42 | @property 43 | def name(self): 44 | """name.""" 45 | return self._name 46 | 47 | @property 48 | def module_dict(self): 49 | """module_dict.""" 50 | return self._module_dict 51 | 52 | def get(self, key): 53 | """get. 54 | 55 | Args: 56 | key: 57 | """ 58 | return self._module_dict.get(key, None) 59 | 60 | def _register_module(self, module_class, module_name=None, force=False): 61 | """_register_module. 62 | 63 | Args: 64 | module_class: Module class to register 65 | module_name: Module name to register 66 | force: forced injection in register 67 | """ 68 | 69 | if module_name is None: 70 | module_name = module_class.__name__ 71 | if not force and module_name in self._module_dict: 72 | raise KeyError( 73 | f"{module_name} is already registered " f"in {self.name}" 74 | ) 75 | self._module_dict[module_name] = module_class 76 | 77 | def register_module(self, name=None, force=False, module=None): 78 | """register_module. 79 | Registers module passed and stores in the modules dict. 80 | 81 | Args: 82 | name: module name. 83 | force: if forced inject register module if already present. default False. 84 | module: Module Reference. 85 | """ 86 | 87 | if module is not None: 88 | self._register_module( 89 | module_class=module, module_name=name, force=force 90 | ) 91 | return module 92 | 93 | if not (name is None or isinstance(name, str)): 94 | raise TypeError(f"name must be a str, but got {type(name)}") 95 | 96 | def _register(cls): 97 | self._register_module( 98 | module_class=cls, module_name=name, force=force 99 | ) 100 | return cls 101 | 102 | return _register 103 | -------------------------------------------------------------------------------- /autodeploy/utils/utils.py: -------------------------------------------------------------------------------- 1 | """ simple utilities functions. """ 2 | import random 3 | from typing import Text 4 | import json 5 | 6 | import requests 7 | import io 8 | from urllib.request import urlopen 9 | from PIL import Image 10 | import numpy as np 11 | 12 | from logger import AppLogger 13 | from database import _database as database 14 | 15 | # datatypes mapper dictionary. 16 | DATATYPES = {'string': str, 'int': int, 'float': float, 'bool': bool} 17 | 18 | logger = AppLogger(__name__).get_logger() 19 | 20 | 21 | def annotator(_dict): 22 | ''' 23 | a utility function to convert configuration input schema 24 | to pydantic model attributes dict. 25 | 26 | ''' 27 | __dict = {} 28 | for key, value in _dict.items(): 29 | if value not in DATATYPES: 30 | logger.error('input schema datatype is not valid.') 31 | # TODO: handle exception 32 | 33 | __dict[key] = (DATATYPES[value], ...) 34 | return __dict 35 | 36 | def generate_random_number(type=float): 37 | ''' a function to generate random number. ''' 38 | if isinstance(type, float): 39 | # TODO: remove hardcoded values 40 | return random.uniform(0.0, 10.0) 41 | return random.randint(0, 10) 42 | 43 | 44 | def generate_random_from_schema(schema, serialized=False, shape=None): 45 | ''' a function to generate random data in input schema format 46 | provided in coknfiguration file. 47 | ''' 48 | __dict = {} 49 | for k, v in dict(schema).items(): 50 | if v not in DATATYPES: 51 | logger.error('input schema datatype is not valid.') 52 | raise ValueError('input schema datatype is not valid.') 53 | if v == 'string' and serialized: 54 | input = np.random.uniform(size=shape) 55 | value = json.dumps(input.tolist()) 56 | else: 57 | v = DATATYPES[v] 58 | value = generate_random_number(v) 59 | __dict[k] = value 60 | return __dict 61 | 62 | 63 | def store_request(db, db_item): 64 | """ a helper function to store 65 | request in database. 66 | """ 67 | db.add(db_item) 68 | db.commit() 69 | db.refresh(db_item) 70 | return db_item 71 | 72 | 73 | def get_db(): 74 | db = database.SessionLocal() 75 | try: 76 | yield db 77 | finally: 78 | db.close() 79 | 80 | def url_loader(url: Text): 81 | ''' a simple function to read url images in numpy array. ''' 82 | response = requests.get(url) 83 | array_bytes = io.BytesIO(response.content) 84 | array = Image.open(array_bytes) 85 | return np.asarray(array) 86 | -------------------------------------------------------------------------------- /bin/autodeploy_start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while ! nc -z rabbitmq 5672; do sleep 3; done 4 | python3 /app/autodeploy/api.py 5 | -------------------------------------------------------------------------------- /bin/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # docker build commands to make docker images. 4 | 5 | helpFunction() 6 | { 7 | echo "" 8 | echo "Usage: $0 -r /app/model_dependencies/reqs.txt -c model configuration file." 9 | echo -e "\t-r Path to model dependencies reqs txt." 10 | echo -e "\t-c Path to model configuration." 11 | exit 1 12 | } 13 | 14 | while getopts "r:c:" opt 15 | do 16 | case "$opt" in 17 | r ) parameterR="$OPTARG" ;; 18 | c ) parameterC="$OPTARG" ;; 19 | ? ) helpFunction ;; 20 | esac 21 | done 22 | 23 | if [ -z "$parameterR" ] || [ -z "$parameterC" ] 24 | then 25 | echo "Some or all of the parameters are empty"; 26 | helpFunction 27 | fi 28 | 29 | # Begin script in case all parameters are correct 30 | echo "$parameterR" 31 | docker build -t autodeploy --build-arg MODEL_REQ=$parameterR --build-arg MODEL_CONFIG=$parameterC -f docker/Dockerfile . 32 | docker build -t prediction --build-arg MODEL_REQ=$parameterR --build-arg MODEL_CONFIG=$parameterC -f docker/PredictDockerfile . 33 | docker build -t monitor --build-arg MODEL_REQ=$parameterR --build-arg MODEL_CONFIG=$parameterC -f docker/MonitorDockerfile . 34 | docker build -t prometheus_server -f docker/PrometheusDockerfile . 35 | -------------------------------------------------------------------------------- /bin/load_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # load testing bash script 4 | helpFunction() 5 | { 6 | echo "" 7 | echo "Usage: $0 -f /app/configs/config.yaml" 8 | echo -e "\t-f A configuration file for deployment." 9 | exit 1 10 | } 11 | 12 | while getopts "f:" opt 13 | do 14 | case "$opt" in 15 | f ) parameterF="$OPTARG" ;; 16 | ? ) helpFunction ;; 17 | esac 18 | done 19 | 20 | if [ -z "$parameterF" ] 21 | then 22 | echo "Some or all of the parameters are empty"; 23 | helpFunction 24 | fi 25 | 26 | # Begin script in case all parameters are correct 27 | echo "$parameterF" 28 | cd ./autodeploy/ 29 | env CONFIG=$parameterF locust -f testing/load_test.py 30 | -------------------------------------------------------------------------------- /bin/monitor_start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while ! nc -z rabbitmq 5672; do sleep 3; done 4 | python3 /app/autodeploy/monitor.py 5 | -------------------------------------------------------------------------------- /bin/predict_start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while ! nc -z redis 6379; do sleep 3; done 4 | python3 /app/autodeploy/predict.py 5 | -------------------------------------------------------------------------------- /bin/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # docker compose commands to start the autodeploy service. 4 | helpFunction() 5 | { 6 | echo "" 7 | echo "Usage: $0 -f /app/configs/config.yaml" 8 | echo -e "\t-f A configuration file for deployment." 9 | exit 1 10 | } 11 | 12 | while getopts "f:" opt 13 | do 14 | case "$opt" in 15 | f ) parameterF="$OPTARG" ;; 16 | ? ) helpFunction ;; 17 | esac 18 | done 19 | 20 | if [ -z "$parameterF" ] 21 | then 22 | echo "Some or all of the parameters are empty"; 23 | helpFunction 24 | fi 25 | 26 | # Begin script in case all parameters are correct 27 | echo "$parameterF" 28 | CONFIG=$parameterF docker-compose -f docker/docker-compose.yml build 29 | CONFIG=$parameterF docker-compose -f docker/docker-compose.yml up 30 | -------------------------------------------------------------------------------- /configs/classification/config.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | model_type: 'onnx' 3 | model_path: ['horse_zebra.onnx', 'horse_zebra.onnx'] 4 | ab_split: [80,20] 5 | model_file_type: 'onnx' 6 | version: '1.0.0' 7 | model_name: 'computer vision classification model.' 8 | endpoint: 'predict' 9 | protected: 0 10 | input_type: 'serialized' 11 | input_shape: [224, 224, 3] 12 | preprocess: 'custom_preprocess_classification' 13 | postprocess: 'custom_postprocess' 14 | 15 | input_schema: 16 | input: 'string' 17 | out_schema: 18 | out: 'int' 19 | confidence: 'float' 20 | status: 'int' 21 | dependency: 22 | path: '/app/model_dependencies' 23 | monitor: 24 | data_drift: 25 | name: 'KSDrift' 26 | reference_data: 'structured_ref.npy' 27 | type: 'info' 28 | custom_metrics: 'image_brightness' 29 | metrics: 30 | average_per_day: 31 | type: 'info' 32 | -------------------------------------------------------------------------------- /configs/iris/config.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | model_type: 'sklearn' 3 | model_path: 'custom_model.pkl' 4 | model_file_type: 'pickle' 5 | version: '1.0.0' 6 | model_name: 'sklearn iris detection model.' 7 | endpoint: 'predict' 8 | protected: 0 9 | input_type: 'structured' 10 | dependency: 11 | path: '/app/model_dependencies' 12 | preprocess: 'custom_preprocess' 13 | postprocess: 'custom_postprocess' 14 | input_schema: 15 | petal_length: 'float' 16 | petal_width: 'float' 17 | sepal_length: 'float' 18 | sepal_width: 'float' 19 | out_schema: 20 | out: 'int' 21 | confidence: 'float' 22 | status: 'int' 23 | monitor: 24 | data_drift: 25 | name: 'KSDrift' 26 | reference_data: 'iris_reference.npy' 27 | type: 'info' 28 | custom_metrics: 'metric1' 29 | metrics: 30 | average_per_day: 31 | type: 'info' 32 | -------------------------------------------------------------------------------- /configs/prometheus.yml: -------------------------------------------------------------------------------- 1 | # prometheus.yml 2 | 3 | global: 4 | scrape_interval: 15s 5 | evaluation_interval: 30s 6 | # scrape_timeout is set to the global default (10s). 7 | 8 | scrape_configs: 9 | - job_name: prometheus 10 | 11 | honor_labels: true 12 | static_configs: 13 | - targets: 14 | - autodeploy:8000 # metrics from autodeploy service 15 | - monitor:8001 # metrics from monitor service 16 | -------------------------------------------------------------------------------- /dashboard/dashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "target": { 12 | "limit": 100, 13 | "matchAny": false, 14 | "tags": [], 15 | "type": "dashboard" 16 | }, 17 | "type": "dashboard" 18 | } 19 | ] 20 | }, 21 | "editable": true, 22 | "gnetId": null, 23 | "graphTooltip": 0, 24 | "id": 1, 25 | "links": [], 26 | "panels": [ 27 | { 28 | "collapsed": false, 29 | "datasource": null, 30 | "gridPos": { 31 | "h": 1, 32 | "w": 24, 33 | "x": 0, 34 | "y": 0 35 | }, 36 | "id": 16, 37 | "panels": [], 38 | "title": "Service Metrics", 39 | "type": "row" 40 | }, 41 | { 42 | "datasource": null, 43 | "gridPos": { 44 | "h": 8, 45 | "w": 4, 46 | "x": 0, 47 | "y": 1 48 | }, 49 | "id": 43, 50 | "options": { 51 | "content": "# Service Metrics\n\nService Metrics allow you to monitor \nhow well the service is performing and track metrics for no. of requests,\naverage data size/req etc. ", 52 | "mode": "markdown" 53 | }, 54 | "pluginVersion": "8.1.3", 55 | "timeFrom": null, 56 | "timeShift": null, 57 | "type": "text" 58 | }, 59 | { 60 | "datasource": null, 61 | "fieldConfig": { 62 | "defaults": { 63 | "mappings": [], 64 | "max": 1, 65 | "min": 0.8, 66 | "noValue": "No Data", 67 | "thresholds": { 68 | "mode": "absolute", 69 | "steps": [ 70 | { 71 | "color": "red", 72 | "value": null 73 | }, 74 | { 75 | "color": "yellow", 76 | "value": 0.95 77 | }, 78 | { 79 | "color": "green", 80 | "value": 0.99 81 | } 82 | ] 83 | }, 84 | "unit": "percentunit" 85 | }, 86 | "overrides": [] 87 | }, 88 | "gridPos": { 89 | "h": 8, 90 | "w": 5, 91 | "x": 4, 92 | "y": 1 93 | }, 94 | "id": 14, 95 | "options": { 96 | "orientation": "auto", 97 | "reduceOptions": { 98 | "calcs": [ 99 | "mean" 100 | ], 101 | "fields": "", 102 | "values": false 103 | }, 104 | "showThresholdLabels": false, 105 | "showThresholdMarkers": true, 106 | "text": {} 107 | }, 108 | "pluginVersion": "8.1.3", 109 | "targets": [ 110 | { 111 | "exemplar": true, 112 | "expr": "sum(rate(http_requests_total{status!=\"5xx\"}[30m])) / sum(rate(http_requests_total[30m]))", 113 | "interval": "", 114 | "legendFormat": "", 115 | "refId": "A" 116 | } 117 | ], 118 | "timeFrom": null, 119 | "timeShift": null, 120 | "title": "API Endpoint Success Rate", 121 | "type": "gauge" 122 | }, 123 | { 124 | "aliasColors": {}, 125 | "bars": false, 126 | "dashLength": 10, 127 | "dashes": false, 128 | "datasource": null, 129 | "fieldConfig": { 130 | "defaults": { 131 | "unit": "reqps" 132 | }, 133 | "overrides": [] 134 | }, 135 | "fill": 10, 136 | "fillGradient": 0, 137 | "gridPos": { 138 | "h": 9, 139 | "w": 15, 140 | "x": 9, 141 | "y": 1 142 | }, 143 | "hiddenSeries": false, 144 | "id": 2, 145 | "legend": { 146 | "alignAsTable": false, 147 | "avg": false, 148 | "current": false, 149 | "hideEmpty": false, 150 | "hideZero": false, 151 | "max": false, 152 | "min": false, 153 | "show": true, 154 | "total": false, 155 | "values": false 156 | }, 157 | "lines": true, 158 | "linewidth": 1, 159 | "nullPointMode": "null", 160 | "options": { 161 | "alertThreshold": true 162 | }, 163 | "percentage": false, 164 | "pluginVersion": "8.1.3", 165 | "pointradius": 2, 166 | "points": false, 167 | "renderer": "flot", 168 | "seriesOverrides": [], 169 | "spaceLength": 10, 170 | "stack": true, 171 | "steppedLine": false, 172 | "targets": [ 173 | { 174 | "exemplar": true, 175 | "expr": "sum(rate(http_requests_total[1m])) by (status) ", 176 | "interval": "", 177 | "legendFormat": "", 178 | "refId": "A" 179 | } 180 | ], 181 | "thresholds": [], 182 | "timeFrom": null, 183 | "timeRegions": [], 184 | "timeShift": null, 185 | "title": "Requests", 186 | "tooltip": { 187 | "shared": true, 188 | "sort": 0, 189 | "value_type": "individual" 190 | }, 191 | "type": "graph", 192 | "xaxis": { 193 | "buckets": null, 194 | "mode": "time", 195 | "name": null, 196 | "show": true, 197 | "values": [] 198 | }, 199 | "yaxes": [ 200 | { 201 | "format": "reqps", 202 | "label": null, 203 | "logBase": 1, 204 | "max": null, 205 | "min": "0", 206 | "show": true 207 | }, 208 | { 209 | "format": "short", 210 | "label": null, 211 | "logBase": 1, 212 | "max": null, 213 | "min": null, 214 | "show": true 215 | } 216 | ], 217 | "yaxis": { 218 | "align": false, 219 | "alignLevel": null 220 | } 221 | }, 222 | { 223 | "datasource": null, 224 | "fieldConfig": { 225 | "defaults": { 226 | "mappings": [], 227 | "noValue": "No Data", 228 | "thresholds": { 229 | "mode": "absolute", 230 | "steps": [ 231 | { 232 | "color": "green", 233 | "value": null 234 | }, 235 | { 236 | "color": "#EAB839", 237 | "value": 500 238 | } 239 | ] 240 | }, 241 | "unit": "decbytes" 242 | }, 243 | "overrides": [] 244 | }, 245 | "gridPos": { 246 | "h": 7, 247 | "w": 9, 248 | "x": 0, 249 | "y": 9 250 | }, 251 | "id": 31, 252 | "options": { 253 | "colorMode": "value", 254 | "graphMode": "none", 255 | "justifyMode": "auto", 256 | "orientation": "auto", 257 | "reduceOptions": { 258 | "calcs": [ 259 | "mean" 260 | ], 261 | "fields": "", 262 | "values": false 263 | }, 264 | "text": {}, 265 | "textMode": "auto" 266 | }, 267 | "pluginVersion": "8.1.3", 268 | "targets": [ 269 | { 270 | "exemplar": true, 271 | "expr": "sum(rate(http_request_size_bytes_sum{handler=\"/predict\"}[30m])) / sum(rate(http_request_size_bytes_count{handler=\"/predict\"}[30m]))", 272 | "interval": "", 273 | "legendFormat": "", 274 | "refId": "A" 275 | } 276 | ], 277 | "timeFrom": null, 278 | "timeShift": null, 279 | "title": "Average Request MBs", 280 | "type": "stat" 281 | }, 282 | { 283 | "aliasColors": {}, 284 | "bars": false, 285 | "dashLength": 10, 286 | "dashes": false, 287 | "datasource": null, 288 | "fieldConfig": { 289 | "defaults": { 290 | "unit": "decbytes" 291 | }, 292 | "overrides": [] 293 | }, 294 | "fill": 1, 295 | "fillGradient": 0, 296 | "gridPos": { 297 | "h": 7, 298 | "w": 15, 299 | "x": 9, 300 | "y": 10 301 | }, 302 | "hiddenSeries": false, 303 | "id": 30, 304 | "legend": { 305 | "avg": false, 306 | "current": false, 307 | "max": false, 308 | "min": false, 309 | "show": true, 310 | "total": false, 311 | "values": false 312 | }, 313 | "lines": true, 314 | "linewidth": 1, 315 | "nullPointMode": "null", 316 | "options": { 317 | "alertThreshold": true 318 | }, 319 | "percentage": false, 320 | "pluginVersion": "8.1.3", 321 | "pointradius": 2, 322 | "points": false, 323 | "renderer": "flot", 324 | "seriesOverrides": [], 325 | "spaceLength": 10, 326 | "stack": false, 327 | "steppedLine": false, 328 | "targets": [ 329 | { 330 | "exemplar": true, 331 | "expr": "sum(rate(http_request_size_bytes_sum{handler=\"/predict\"}[5m])) / sum(rate(http_request_size_bytes_count{handler=\"/predict\"}[5m]))", 332 | "interval": "", 333 | "legendFormat": "", 334 | "refId": "A" 335 | } 336 | ], 337 | "thresholds": [], 338 | "timeFrom": null, 339 | "timeRegions": [], 340 | "timeShift": null, 341 | "title": "Average Request Size(MB)", 342 | "tooltip": { 343 | "shared": true, 344 | "sort": 0, 345 | "value_type": "individual" 346 | }, 347 | "type": "graph", 348 | "xaxis": { 349 | "buckets": null, 350 | "mode": "time", 351 | "name": null, 352 | "show": true, 353 | "values": [] 354 | }, 355 | "yaxes": [ 356 | { 357 | "format": "decbytes", 358 | "label": null, 359 | "logBase": 1, 360 | "max": null, 361 | "min": "0", 362 | "show": true 363 | }, 364 | { 365 | "format": "short", 366 | "label": null, 367 | "logBase": 1, 368 | "max": null, 369 | "min": null, 370 | "show": true 371 | } 372 | ], 373 | "yaxis": { 374 | "align": false, 375 | "alignLevel": null 376 | } 377 | }, 378 | { 379 | "collapsed": false, 380 | "datasource": null, 381 | "gridPos": { 382 | "h": 1, 383 | "w": 24, 384 | "x": 0, 385 | "y": 17 386 | }, 387 | "id": 18, 388 | "panels": [], 389 | "title": "Model Metrics", 390 | "type": "row" 391 | }, 392 | { 393 | "datasource": null, 394 | "gridPos": { 395 | "h": 8, 396 | "w": 5, 397 | "x": 0, 398 | "y": 18 399 | }, 400 | "id": 10, 401 | "options": { 402 | "content": "# Model metrics\n\nModel metrics help to track \nmodel performance and visualize AI exaplainability.", 403 | "mode": "markdown" 404 | }, 405 | "pluginVersion": "8.1.3", 406 | "timeFrom": null, 407 | "timeShift": null, 408 | "type": "text" 409 | }, 410 | { 411 | "datasource": null, 412 | "fieldConfig": { 413 | "defaults": { 414 | "color": { 415 | "mode": "thresholds" 416 | }, 417 | "mappings": [], 418 | "thresholds": { 419 | "mode": "percentage", 420 | "steps": [ 421 | { 422 | "color": "green", 423 | "value": null 424 | }, 425 | { 426 | "color": "red", 427 | "value": 50 428 | } 429 | ] 430 | } 431 | }, 432 | "overrides": [] 433 | }, 434 | "gridPos": { 435 | "h": 8, 436 | "w": 4, 437 | "x": 5, 438 | "y": 18 439 | }, 440 | "id": 39, 441 | "options": { 442 | "reduceOptions": { 443 | "calcs": [ 444 | "mean" 445 | ], 446 | "fields": "", 447 | "values": false 448 | }, 449 | "showThresholdLabels": false, 450 | "showThresholdMarkers": true, 451 | "text": {} 452 | }, 453 | "pluginVersion": "8.1.3", 454 | "targets": [ 455 | { 456 | "exemplar": true, 457 | "expr": "model_output_score{} ", 458 | "interval": "", 459 | "legendFormat": "", 460 | "refId": "A" 461 | } 462 | ], 463 | "title": "Average Confidence", 464 | "transformations": [ 465 | { 466 | "id": "filterByValue", 467 | "options": { 468 | "filters": [ 469 | { 470 | "config": { 471 | "id": "greater", 472 | "options": { 473 | "value": 0 474 | } 475 | }, 476 | "fieldName": "model_output_score{instance=\"monitor:8001\", job=\"prometheus\"}" 477 | } 478 | ], 479 | "match": "any", 480 | "type": "include" 481 | } 482 | } 483 | ], 484 | "type": "gauge" 485 | }, 486 | { 487 | "datasource": null, 488 | "fieldConfig": { 489 | "defaults": { 490 | "color": { 491 | "mode": "palette-classic" 492 | }, 493 | "custom": { 494 | "axisLabel": "", 495 | "axisPlacement": "auto", 496 | "barAlignment": 0, 497 | "drawStyle": "line", 498 | "fillOpacity": 0, 499 | "gradientMode": "none", 500 | "hideFrom": { 501 | "legend": false, 502 | "tooltip": false, 503 | "viz": false 504 | }, 505 | "lineInterpolation": "linear", 506 | "lineWidth": 1, 507 | "pointSize": 5, 508 | "scaleDistribution": { 509 | "type": "linear" 510 | }, 511 | "showPoints": "auto", 512 | "spanNulls": false, 513 | "stacking": { 514 | "group": "A", 515 | "mode": "none" 516 | }, 517 | "thresholdsStyle": { 518 | "mode": "off" 519 | } 520 | }, 521 | "mappings": [], 522 | "thresholds": { 523 | "mode": "absolute", 524 | "steps": [ 525 | { 526 | "color": "green", 527 | "value": null 528 | }, 529 | { 530 | "color": "red", 531 | "value": 80 532 | } 533 | ] 534 | } 535 | }, 536 | "overrides": [] 537 | }, 538 | "gridPos": { 539 | "h": 8, 540 | "w": 15, 541 | "x": 9, 542 | "y": 18 543 | }, 544 | "id": 37, 545 | "options": { 546 | "legend": { 547 | "calcs": [], 548 | "displayMode": "list", 549 | "placement": "bottom" 550 | }, 551 | "tooltip": { 552 | "mode": "single" 553 | } 554 | }, 555 | "targets": [ 556 | { 557 | "exemplar": true, 558 | "expr": "model_output_score * 100", 559 | "interval": "", 560 | "legendFormat": "", 561 | "refId": "A" 562 | } 563 | ], 564 | "title": "Model Predictions", 565 | "transformations": [ 566 | { 567 | "id": "filterByValue", 568 | "options": { 569 | "filters": [ 570 | { 571 | "config": { 572 | "id": "greater", 573 | "options": { 574 | "value": 0 575 | } 576 | }, 577 | "fieldName": "{instance=\"monitor:8001\", job=\"prometheus\"}" 578 | } 579 | ], 580 | "match": "any", 581 | "type": "include" 582 | } 583 | } 584 | ], 585 | "type": "timeseries" 586 | }, 587 | { 588 | "datasource": null, 589 | "fieldConfig": { 590 | "defaults": { 591 | "color": { 592 | "mode": "thresholds" 593 | }, 594 | "mappings": [], 595 | "thresholds": { 596 | "mode": "absolute", 597 | "steps": [ 598 | { 599 | "color": "green", 600 | "value": null 601 | }, 602 | { 603 | "color": "red", 604 | "value": 80 605 | } 606 | ] 607 | } 608 | }, 609 | "overrides": [] 610 | }, 611 | "gridPos": { 612 | "h": 7, 613 | "w": 9, 614 | "x": 0, 615 | "y": 26 616 | }, 617 | "id": 41, 618 | "options": { 619 | "reduceOptions": { 620 | "calcs": [ 621 | "mean" 622 | ], 623 | "fields": "", 624 | "values": false 625 | }, 626 | "showThresholdLabels": false, 627 | "showThresholdMarkers": true, 628 | "text": {} 629 | }, 630 | "pluginVersion": "8.1.3", 631 | "targets": [ 632 | { 633 | "exemplar": true, 634 | "expr": "image_brightness{}", 635 | "interval": "", 636 | "legendFormat": "", 637 | "refId": "A" 638 | } 639 | ], 640 | "title": "Average image intensity (0-100)", 641 | "type": "gauge" 642 | }, 643 | { 644 | "aliasColors": {}, 645 | "bars": false, 646 | "dashLength": 10, 647 | "dashes": false, 648 | "datasource": null, 649 | "fieldConfig": { 650 | "defaults": { 651 | "unit": "s" 652 | }, 653 | "overrides": [] 654 | }, 655 | "fill": 1, 656 | "fillGradient": 0, 657 | "gridPos": { 658 | "h": 7, 659 | "w": 15, 660 | "x": 9, 661 | "y": 26 662 | }, 663 | "hiddenSeries": false, 664 | "id": 4, 665 | "legend": { 666 | "avg": false, 667 | "current": false, 668 | "max": false, 669 | "min": false, 670 | "show": true, 671 | "total": false, 672 | "values": false 673 | }, 674 | "lines": true, 675 | "linewidth": 1, 676 | "nullPointMode": "null", 677 | "options": { 678 | "alertThreshold": true 679 | }, 680 | "percentage": false, 681 | "pluginVersion": "8.1.3", 682 | "pointradius": 2, 683 | "points": false, 684 | "renderer": "flot", 685 | "seriesOverrides": [], 686 | "spaceLength": 10, 687 | "stack": false, 688 | "steppedLine": false, 689 | "targets": [ 690 | { 691 | "exemplar": true, 692 | "expr": "histogram_quantile(0.99, \n sum(\n rate(http_request_duration_seconds_bucket[1m])\n ) by (le)\n)", 693 | "interval": "", 694 | "legendFormat": "99th percentile", 695 | "refId": "A" 696 | }, 697 | { 698 | "expr": "histogram_quantile(0.95, \n sum(\n rate(fastapi_http_request_duration_seconds_bucket[1m])\n ) by (le)\n)", 699 | "interval": "", 700 | "legendFormat": "95th percentile", 701 | "refId": "B" 702 | }, 703 | { 704 | "expr": "histogram_quantile(0.50, \n sum(\n rate(fastapi_http_request_duration_seconds_bucket[1m])\n ) by (le)\n)", 705 | "interval": "", 706 | "legendFormat": "50th percentile", 707 | "refId": "C" 708 | } 709 | ], 710 | "thresholds": [], 711 | "timeFrom": null, 712 | "timeRegions": [], 713 | "timeShift": null, 714 | "title": "Latency (ms)", 715 | "tooltip": { 716 | "shared": true, 717 | "sort": 0, 718 | "value_type": "individual" 719 | }, 720 | "type": "graph", 721 | "xaxis": { 722 | "buckets": null, 723 | "mode": "time", 724 | "name": null, 725 | "show": true, 726 | "values": [] 727 | }, 728 | "yaxes": [ 729 | { 730 | "format": "s", 731 | "label": null, 732 | "logBase": 1, 733 | "max": null, 734 | "min": null, 735 | "show": true 736 | }, 737 | { 738 | "format": "short", 739 | "label": null, 740 | "logBase": 1, 741 | "max": null, 742 | "min": null, 743 | "show": true 744 | } 745 | ], 746 | "yaxis": { 747 | "align": false, 748 | "alignLevel": null 749 | } 750 | } 751 | ], 752 | "refresh": "5s", 753 | "schemaVersion": 30, 754 | "style": "dark", 755 | "tags": [], 756 | "templating": { 757 | "list": [] 758 | }, 759 | "time": { 760 | "from": "now-5m", 761 | "to": "now" 762 | }, 763 | "timepicker": {}, 764 | "timezone": "", 765 | "title": "Model Monitoring1", 766 | "uid": "JIdxtgSnk1", 767 | "version": 1 768 | } -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | ARG MODEL_REQ 3 | ARG MODEL_CONFIG 4 | RUN apt-get update \ 5 | && apt-get install python3 python3-pip -y \ 6 | && apt-get clean \ 7 | && apt-get autoremove 8 | 9 | # autodeploy requirements 10 | COPY ./requirements.txt /app/requirements.txt 11 | RUN python3 -m pip install -r /app/requirements.txt 12 | 13 | # user requirements 14 | COPY $MODEL_REQ /app/$MODEL_REQ 15 | RUN python3 -m pip install -r /app/$MODEL_REQ 16 | 17 | ENV TZ=Europe/Kiev 18 | RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 19 | 20 | RUN apt-get install -y libgl1-mesa-glx libglib2.0-0 libsm6 libxrender1 libxext6 -y 21 | RUN apt-get install iputils-ping netcat -y 22 | 23 | # autodeploy requirements 24 | COPY ./requirements.txt /app/requirements.txt 25 | RUN python3 -m pip install -r /app/requirements.txt 26 | 27 | # user requirements 28 | COPY $MODEL_REQ /app/$MODEL_REQ 29 | RUN python3 -m pip install -r /app/$MODEL_REQ 30 | 31 | ENV LC_ALL=C.UTF-8 32 | ENV LANG=C.UTF-8 33 | ENV LANGUAGE=C.UTF-8 34 | 35 | EXPOSE 8000 36 | COPY ./ app 37 | WORKDIR /app 38 | 39 | ENV CONFIG=/app/$MODEL_CONFIG 40 | 41 | RUN chmod +x /app/bin/autodeploy_start.sh 42 | 43 | CMD ["/app/bin/autodeploy_start.sh"] 44 | -------------------------------------------------------------------------------- /docker/MonitorDockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | ARG MODEL_REQ 3 | ARG MODEL_CONFIG 4 | RUN apt-get update \ 5 | && apt-get install python3 python3-pip -y \ 6 | && apt-get clean \ 7 | && apt-get autoremove 8 | 9 | ENV TZ=Europe/Kiev 10 | RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 11 | 12 | RUN apt-get install -y libgl1-mesa-glx libglib2.0-0 libsm6 libxrender1 libxext6 -y 13 | RUN apt-get install iputils-ping netcat -y 14 | 15 | 16 | # autodeploy requirements 17 | COPY ./requirements.txt /app/requirements.txt 18 | RUN python3 -m pip install -r /app/requirements.txt 19 | 20 | # user requirements 21 | COPY $MODEL_REQ /app/$MODEL_REQ 22 | RUN python3 -m pip install -r /app/$MODEL_REQ 23 | 24 | 25 | ENV LC_ALL=C.UTF-8 26 | ENV LANG=C.UTF-8 27 | ENV LANGUAGE=C.UTF-8 28 | 29 | EXPOSE 8001 30 | COPY ./ app 31 | WORKDIR /app 32 | 33 | ENV CONFIG=/app/$MODEL_CONFIG 34 | 35 | RUN chmod +x /app/bin/monitor_start.sh 36 | 37 | CMD ["/app/bin/monitor_start.sh"] 38 | -------------------------------------------------------------------------------- /docker/PredictDockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | ARG MODEL_REQ 3 | ARG MODEL_CONFIG 4 | RUN apt-get update \ 5 | && apt-get install python3 python3-pip -y \ 6 | && apt-get clean \ 7 | && apt-get autoremove 8 | 9 | # autodeploy requirements 10 | COPY ./requirements.txt /app/requirements.txt 11 | RUN python3 -m pip install -r /app/requirements.txt 12 | 13 | # user requirements 14 | COPY $MODEL_REQ /app/$MODEL_REQ 15 | RUN python3 -m pip install -r /app/$MODEL_REQ 16 | 17 | ENV TZ=Europe/Kiev 18 | RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 19 | 20 | RUN apt-get install -y libgl1-mesa-glx libglib2.0-0 libsm6 libxrender1 libxext6 -y 21 | RUN apt-get install iputils-ping netcat -y 22 | 23 | # autodeploy requirements 24 | COPY ./requirements.txt /app/requirements.txt 25 | RUN python3 -m pip install -r /app/requirements.txt 26 | 27 | # user requirements 28 | COPY $MODEL_REQ /app/$MODEL_REQ 29 | RUN python3 -m pip install -r /app/$MODEL_REQ 30 | 31 | ENV LC_ALL=C.UTF-8 32 | ENV LANG=C.UTF-8 33 | ENV LANGUAGE=C.UTF-8 34 | 35 | EXPOSE 8009 36 | COPY ./ app 37 | WORKDIR /app 38 | 39 | ENV CONFIG=/app/$MODEL_CONFIG 40 | 41 | RUN chmod +x /app/bin/predict_start.sh 42 | 43 | CMD ["/app/bin/predict_start.sh"] 44 | -------------------------------------------------------------------------------- /docker/PrometheusDockerfile: -------------------------------------------------------------------------------- 1 | FROM prom/prometheus:latest 2 | COPY ./configs/prometheus.yml /etc/prometheus/ 3 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | rabbitmq: 5 | image: rabbitmq:3-management 6 | restart: always 7 | ports: 8 | - "15672:15672" 9 | - "5672:5672" 10 | 11 | autodeploy: 12 | image: autodeploy:latest 13 | ports: 14 | - "8000:8000" 15 | links: 16 | - rabbitmq 17 | - prediction 18 | networks: 19 | - default 20 | environment: 21 | - CONFIG=/app/${CONFIG} 22 | 23 | monitor: 24 | image: monitor:latest 25 | ports: 26 | - "8001:8001" 27 | links: 28 | - rabbitmq 29 | networks: 30 | - default 31 | environment: 32 | - CONFIG=/app/${CONFIG} 33 | 34 | prometheus: 35 | image: prom/prometheus:latest 36 | ports: 37 | - "9090:9090" 38 | links: 39 | - rabbitmq 40 | - autodeploy 41 | volumes: 42 | - ../configs/prometheus.yml:/etc/prometheus/prometheus.yml 43 | 44 | grafana: 45 | image: grafana/grafana:latest 46 | ports: 47 | - "3000:3000" 48 | links: 49 | - prometheus 50 | 51 | redis: 52 | image: redis:alpine 53 | ports: 54 | - "6379:6379" 55 | restart: always 56 | 57 | prediction: 58 | image: prediction:latest 59 | ports: 60 | - "8009:8009" 61 | links: 62 | - redis 63 | networks: 64 | - default 65 | environment: 66 | - CONFIG=/app/${CONFIG} 67 | -------------------------------------------------------------------------------- /k8s/autodeploy-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: autodeploy 10 | name: autodeploy 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | io.kompose.service: autodeploy 16 | strategy: {} 17 | template: 18 | metadata: 19 | annotations: 20 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 21 | kompose.version: 1.22.0 (955b78124) 22 | creationTimestamp: null 23 | labels: 24 | io.kompose.network/default: "true" 25 | io.kompose.service: autodeploy 26 | spec: 27 | containers: 28 | - image: autodeploy:latest 29 | name: autodeploy 30 | imagePullPolicy: Never 31 | ports: 32 | - containerPort: 8000 33 | restartPolicy: Always 34 | status: {} 35 | -------------------------------------------------------------------------------- /k8s/autodeploy-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: autodeploy 10 | name: autodeploy 11 | spec: 12 | ports: 13 | - name: "8000" 14 | port: 8000 15 | targetPort: 8000 16 | selector: 17 | io.kompose.service: autodeploy 18 | type: LoadBalancer 19 | status: 20 | loadBalancer: {} 21 | -------------------------------------------------------------------------------- /k8s/default-networkpolicy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.k8s.io/v1 2 | kind: NetworkPolicy 3 | metadata: 4 | creationTimestamp: null 5 | name: default 6 | spec: 7 | ingress: 8 | - from: 9 | - podSelector: 10 | matchLabels: 11 | io.kompose.network/default: "true" 12 | podSelector: 13 | matchLabels: 14 | io.kompose.network/default: "true" 15 | -------------------------------------------------------------------------------- /k8s/grafana-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: grafana 10 | name: grafana 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | io.kompose.service: grafana 16 | strategy: {} 17 | template: 18 | metadata: 19 | annotations: 20 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 21 | kompose.version: 1.22.0 (955b78124) 22 | creationTimestamp: null 23 | labels: 24 | io.kompose.service: grafana 25 | spec: 26 | containers: 27 | - image: grafana/grafana:latest 28 | name: grafana 29 | ports: 30 | - containerPort: 3000 31 | resources: {} 32 | restartPolicy: Always 33 | status: {} 34 | -------------------------------------------------------------------------------- /k8s/grafana-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: grafana 10 | name: grafana 11 | spec: 12 | ports: 13 | - name: "3000" 14 | port: 3000 15 | targetPort: 3000 16 | selector: 17 | io.kompose.service: grafana 18 | type: NodePort 19 | -------------------------------------------------------------------------------- /k8s/horizontal-scale.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v1 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | name: prediction-hpa 5 | spec: 6 | scaleTargetRef: 7 | apiVersion: apps/v1 8 | kind: Deployment 9 | name: prediction 10 | minReplicas: 1 11 | maxReplicas: 10 12 | targetCPUUtilizationPercentage: 50 13 | -------------------------------------------------------------------------------- /k8s/metric-server.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | labels: 5 | k8s-app: metrics-server 6 | name: metrics-server 7 | namespace: kube-system 8 | --- 9 | apiVersion: rbac.authorization.k8s.io/v1 10 | kind: ClusterRole 11 | metadata: 12 | labels: 13 | k8s-app: metrics-server 14 | rbac.authorization.k8s.io/aggregate-to-admin: "true" 15 | rbac.authorization.k8s.io/aggregate-to-edit: "true" 16 | rbac.authorization.k8s.io/aggregate-to-view: "true" 17 | name: system:aggregated-metrics-reader 18 | rules: 19 | - apiGroups: 20 | - metrics.k8s.io 21 | resources: 22 | - pods 23 | - nodes 24 | verbs: 25 | - get 26 | - list 27 | - watch 28 | --- 29 | apiVersion: rbac.authorization.k8s.io/v1 30 | kind: ClusterRole 31 | metadata: 32 | labels: 33 | k8s-app: metrics-server 34 | name: system:metrics-server 35 | rules: 36 | - apiGroups: 37 | - "" 38 | resources: 39 | - pods 40 | - nodes 41 | - nodes/stats 42 | - namespaces 43 | - configmaps 44 | verbs: 45 | - get 46 | - list 47 | - watch 48 | --- 49 | apiVersion: rbac.authorization.k8s.io/v1 50 | kind: RoleBinding 51 | metadata: 52 | labels: 53 | k8s-app: metrics-server 54 | name: metrics-server-auth-reader 55 | namespace: kube-system 56 | roleRef: 57 | apiGroup: rbac.authorization.k8s.io 58 | kind: Role 59 | name: extension-apiserver-authentication-reader 60 | subjects: 61 | - kind: ServiceAccount 62 | name: metrics-server 63 | namespace: kube-system 64 | --- 65 | apiVersion: rbac.authorization.k8s.io/v1 66 | kind: ClusterRoleBinding 67 | metadata: 68 | labels: 69 | k8s-app: metrics-server 70 | name: metrics-server:system:auth-delegator 71 | roleRef: 72 | apiGroup: rbac.authorization.k8s.io 73 | kind: ClusterRole 74 | name: system:auth-delegator 75 | subjects: 76 | - kind: ServiceAccount 77 | name: metrics-server 78 | namespace: kube-system 79 | --- 80 | apiVersion: rbac.authorization.k8s.io/v1 81 | kind: ClusterRoleBinding 82 | metadata: 83 | labels: 84 | k8s-app: metrics-server 85 | name: system:metrics-server 86 | roleRef: 87 | apiGroup: rbac.authorization.k8s.io 88 | kind: ClusterRole 89 | name: system:metrics-server 90 | subjects: 91 | - kind: ServiceAccount 92 | name: metrics-server 93 | namespace: kube-system 94 | --- 95 | apiVersion: v1 96 | kind: Service 97 | metadata: 98 | labels: 99 | k8s-app: metrics-server 100 | name: metrics-server 101 | namespace: kube-system 102 | spec: 103 | ports: 104 | - name: https 105 | port: 443 106 | protocol: TCP 107 | targetPort: https 108 | selector: 109 | k8s-app: metrics-server 110 | --- 111 | apiVersion: apps/v1 112 | kind: Deployment 113 | metadata: 114 | labels: 115 | k8s-app: metrics-server 116 | name: metrics-server 117 | namespace: kube-system 118 | spec: 119 | selector: 120 | matchLabels: 121 | k8s-app: metrics-server 122 | strategy: 123 | rollingUpdate: 124 | maxUnavailable: 0 125 | template: 126 | metadata: 127 | labels: 128 | k8s-app: metrics-server 129 | spec: 130 | containers: 131 | - args: 132 | - --cert-dir=/tmp 133 | - --secure-port=443 134 | - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname 135 | - --kubelet-use-node-status-port 136 | - --metric-resolution=15s 137 | - --kubelet-insecure-tls 138 | - --kubelet-preferred-address-types=InternalIP 139 | image: k8s.gcr.io/metrics-server/metrics-server:v0.5.1 140 | imagePullPolicy: IfNotPresent 141 | livenessProbe: 142 | failureThreshold: 3 143 | httpGet: 144 | path: /livez 145 | port: https 146 | scheme: HTTPS 147 | periodSeconds: 10 148 | name: metrics-server 149 | ports: 150 | - containerPort: 443 151 | name: https 152 | protocol: TCP 153 | readinessProbe: 154 | failureThreshold: 3 155 | httpGet: 156 | path: /readyz 157 | port: https 158 | scheme: HTTPS 159 | initialDelaySeconds: 20 160 | periodSeconds: 10 161 | resources: 162 | requests: 163 | cpu: 100m 164 | memory: 200Mi 165 | securityContext: 166 | readOnlyRootFilesystem: true 167 | runAsNonRoot: true 168 | runAsUser: 1000 169 | volumeMounts: 170 | - mountPath: /tmp 171 | name: tmp-dir 172 | nodeSelector: 173 | kubernetes.io/os: linux 174 | priorityClassName: system-cluster-critical 175 | serviceAccountName: metrics-server 176 | volumes: 177 | - emptyDir: {} 178 | name: tmp-dir 179 | --- 180 | apiVersion: apiregistration.k8s.io/v1 181 | kind: APIService 182 | metadata: 183 | labels: 184 | k8s-app: metrics-server 185 | name: v1beta1.metrics.k8s.io 186 | spec: 187 | group: metrics.k8s.io 188 | groupPriorityMinimum: 100 189 | insecureSkipTLSVerify: true 190 | service: 191 | name: metrics-server 192 | namespace: kube-system 193 | version: v1beta1 194 | versionPriority: 100 195 | -------------------------------------------------------------------------------- /k8s/monitor-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: monitor 10 | name: monitor 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | io.kompose.service: monitor 16 | strategy: {} 17 | template: 18 | metadata: 19 | annotations: 20 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 21 | kompose.version: 1.22.0 (955b78124) 22 | creationTimestamp: null 23 | labels: 24 | io.kompose.network/default: "true" 25 | io.kompose.service: monitor 26 | spec: 27 | containers: 28 | - image: monitor:latest 29 | name: monitor 30 | imagePullPolicy: Never 31 | ports: 32 | - containerPort: 8001 33 | resources: {} 34 | restartPolicy: Always 35 | status: {} 36 | -------------------------------------------------------------------------------- /k8s/monitor-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: monitor 10 | name: monitor 11 | spec: 12 | ports: 13 | - name: "8001" 14 | port: 8001 15 | targetPort: 8001 16 | selector: 17 | io.kompose.service: monitor 18 | status: 19 | loadBalancer: {} 20 | -------------------------------------------------------------------------------- /k8s/prediction-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: prediction 10 | name: prediction 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | io.kompose.service: prediction 16 | strategy: {} 17 | template: 18 | metadata: 19 | annotations: 20 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 21 | kompose.version: 1.22.0 (955b78124) 22 | creationTimestamp: null 23 | labels: 24 | io.kompose.network/default: "true" 25 | io.kompose.service: prediction 26 | spec: 27 | containers: 28 | - image: prediction:latest 29 | name: prediction 30 | imagePullPolicy: Never 31 | ports: 32 | - containerPort: 8009 33 | resources: 34 | requests: 35 | cpu: "250m" 36 | restartPolicy: Always 37 | status: {} 38 | -------------------------------------------------------------------------------- /k8s/prediction-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: prediction 10 | name: prediction 11 | spec: 12 | ports: 13 | - name: "8009" 14 | port: 8009 15 | targetPort: 8009 16 | selector: 17 | io.kompose.service: prediction 18 | status: 19 | loadBalancer: {} 20 | -------------------------------------------------------------------------------- /k8s/prometheus-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: prometheus 10 | name: prometheus 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | io.kompose.service: prometheus 16 | strategy: 17 | type: Recreate 18 | template: 19 | metadata: 20 | annotations: 21 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 22 | kompose.version: 1.22.0 (955b78124) 23 | creationTimestamp: null 24 | labels: 25 | io.kompose.service: prometheus 26 | spec: 27 | containers: 28 | - image: prometheus_server:latest 29 | name: prometheus 30 | imagePullPolicy: Never 31 | ports: 32 | - containerPort: 9090 33 | resources: {} 34 | restartPolicy: Always 35 | status: {} 36 | -------------------------------------------------------------------------------- /k8s/prometheus-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: prometheus 10 | name: prometheus 11 | spec: 12 | ports: 13 | - name: "9090" 14 | port: 9090 15 | targetPort: 9090 16 | selector: 17 | io.kompose.service: prometheus 18 | -------------------------------------------------------------------------------- /k8s/rabbitmq-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: rabbitmq 10 | name: rabbitmq 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | io.kompose.service: rabbitmq 16 | strategy: {} 17 | template: 18 | metadata: 19 | annotations: 20 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 21 | kompose.version: 1.22.0 (955b78124) 22 | creationTimestamp: null 23 | labels: 24 | io.kompose.service: rabbitmq 25 | spec: 26 | containers: 27 | - image: rabbitmq:3-management 28 | name: rabbitmq 29 | ports: 30 | - containerPort: 15672 31 | - containerPort: 5672 32 | resources: {} 33 | restartPolicy: Always 34 | status: {} 35 | -------------------------------------------------------------------------------- /k8s/rabbitmq-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: rabbitmq 10 | name: rabbitmq 11 | spec: 12 | ports: 13 | - name: "15672" 14 | port: 15672 15 | targetPort: 15672 16 | - name: "5672" 17 | port: 5672 18 | targetPort: 5672 19 | selector: 20 | io.kompose.service: rabbitmq 21 | -------------------------------------------------------------------------------- /k8s/redis-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: redis 10 | name: redis 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | io.kompose.service: redis 16 | strategy: {} 17 | template: 18 | metadata: 19 | annotations: 20 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 21 | kompose.version: 1.22.0 (955b78124) 22 | creationTimestamp: null 23 | labels: 24 | io.kompose.service: redis 25 | spec: 26 | containers: 27 | - image: redis:alpine 28 | name: redis 29 | ports: 30 | - containerPort: 6379 31 | resources: {} 32 | restartPolicy: Always 33 | status: {} 34 | -------------------------------------------------------------------------------- /k8s/redis-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | annotations: 5 | kompose.cmd: ./kompose -f docker/docker-compose.yml convert 6 | kompose.version: 1.22.0 (955b78124) 7 | creationTimestamp: null 8 | labels: 9 | io.kompose.service: redis 10 | name: redis 11 | spec: 12 | ports: 13 | - name: "6379" 14 | port: 6379 15 | targetPort: 6379 16 | selector: 17 | io.kompose.service: redis 18 | status: 19 | loadBalancer: {} 20 | -------------------------------------------------------------------------------- /notebooks/horse_1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartik4949/AutoDeploy/af7b3b32954a574307849ababb05fa2f4a80f52e/notebooks/horse_1.jpg -------------------------------------------------------------------------------- /notebooks/test_auto_deploy.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartik4949/AutoDeploy/af7b3b32954a574307849ababb05fa2f4a80f52e/notebooks/test_auto_deploy.ipynb -------------------------------------------------------------------------------- /notebooks/zebra_1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartik4949/AutoDeploy/af7b3b32954a574307849ababb05fa2f4a80f52e/notebooks/zebra_1.jpg -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.commitizen] 2 | version = "2.0.0" 3 | version_files = [ 4 | "autodeploy/__version__.py", 5 | ] 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.26.0 2 | uvicorn==0.15.0 3 | redis==3.5.3 4 | fastapi==0.68.0 5 | scikit-learn==0.24.2 6 | pika==1.2.0 7 | sqlalchemy==1.4.23 8 | prometheus_fastapi_instrumentator 9 | pyyaml==5.4.1 10 | python-multipart==0.0.5 11 | alibi-detect 12 | onnxruntime 13 | validators 14 | --------------------------------------------------------------------------------