├── .gitignore ├── LICENCE ├── MAINTAINERS.md ├── README.md ├── base_docker_image ├── Dockerfile ├── download_source_code.py └── requirements.txt ├── cortex-serving-client-logo-2.drawio ├── cortex-serving-client-logo-2.jpg ├── cortex-serving-client-logo-2.svg ├── cortex-serving-client.png ├── cortex-serving-client.xcf ├── cortex_serving_client ├── __init__.py ├── command_line_wrapper.py ├── cortex_client.py ├── deployment_failed.py ├── hash_utils.py ├── printable_chars.py ├── resources │ ├── __init__.py │ └── main.py ├── retry_utils.py ├── s3.py ├── shell_utils.py └── zip_utils.py ├── example ├── README.md ├── cluster.yaml ├── dummy_dir │ ├── dummy_predictor.py │ └── requirements.txt └── example.py ├── model-manager-architecture ├── model-manager-architecture.jpg ├── publish_to_pypi.sh ├── requirements-dev.txt ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── data ├── batch_fail_deployment │ ├── fail_batch_predictor.py │ └── requirements.txt ├── batch_yes_deployment │ ├── batch_yes_predictor.py │ └── s3.py ├── fail_deployment │ └── fail_predictor.py ├── no_app_deployment │ ├── main.py │ └── yes_predictor.py └── yes_deployment │ ├── redeploy_yes_predictor.py │ └── yes_predictor.py ├── integration_tests.py ├── test_cortex_client.py └── test_printable_chars.py /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | __pycache__ 3 | /build 4 | /cortex_serving_client.egg-info 5 | /dist 6 | /tmp 7 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 Inspigroup s.r.o. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # How To Release A New Version 2 | 3 | Warning: Below is an experimental version. Automation will be added later. 4 | 5 | 1. Read [official how-to](https://packaging.python.org/guides/distributing-packages-using-setuptools/). 6 | 1. Re-test the project. 7 | 1. Make git-pull to make sure on latest sources. 8 | 1. Install reqs: 9 | 10 | ``` pip install -r requirements-dev.txt ``` 11 | 12 | 1. Select package version new_version where major and minor version should be equal to the supported cortex version and patch version is a custom version specific to this package. 13 | 14 | 1. Update and commit package version. 15 | 16 | ` vi setup.py ` 17 | 18 | 1. Update README.md to the newest version. 19 | 20 | 21 | 1. Execute and use your API token as a password: 22 | ``` 23 | bash ./publish_to_pypi.sh ${new_version}; 24 | ``` 25 | 26 | - TODO GPG signed Tag? https://github.com/scikit-build/ninja-python-distributions/blob/master/docs/make_a_release.rst 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Update: Cortex.dev bought by Databricks 2 | 3 | Consider migrating away from the Cortex.dev as it was bought by Databricks and is only maintained but not developed at the moment. Consider other options. You can do clusterless with directly EC2 wiht AWS cli scripsts, or Fargate. Or with cluster maintanance you can think about popular choice Terraform. For deploying consider using e.g. Kubernetes Python client. 4 | 5 | 6 | # Cortex Serving Client 7 | 8 | Cortex Serving Client makes Python serving automation simple. 9 | It is a Python wrapper around [Cortex's command-line client](https://cortex.dev) that provides garbage API collection. 10 | Cortex has [official Python client now](https://pypi.org/project/cortex/) ([source](https://github.com/cortexlabs/cortex/blob/e22985f8516fe8db930aaecd05269da99d5e7a93/pkg/cortex/client/cortex/client.py)), but this project offers advanced features (GC, temporary deployment, timeouts) not present in the vanilla. 11 | 12 | Main feature of this package is that you can use it on top of your codebase created for Cortex Version <= 0.34, meaning that: 13 | - deployment directory is automatically zipped and uploaded to S3 bucket 14 | - we prepared a base docker image that downloads this zipped code, unzips it and runs it in an Uvicorn worker 15 | - => you can simply deploy your `PythonPredictor` using Cortex 0.42 without having to wrap it inside your own docker image 16 | 17 | Additional features: 18 | - Automate your Cortex AWS cluster from Python. 19 | - Prevent accidental charges by auto-removing deployments that exceeded a timeout. 20 | - Execute operations: deploy, delete, get, get all. 21 | - Stream remote logs into the local log with thread name set to the API name. 22 | - Supported Cortex Version: 0.40.0 (See requirements.txt) 23 | 24 | Here is [a video about the package (version for Cortex 0.33 = before the big changes)](https://youtu.be/aU95dBAspr0?t=510). 25 | 26 | ## How Does It Work? 27 | 28 | After implementing your predictor module in a folder (see `example/dummy_dir`), 29 | you can deploy it to your Cortex cluster, 30 | and execute a prediction via a POST request. 31 | 32 | Here is [a video of the demo below](https://youtu.be/aU95dBAspr0?t=1261). 33 | 34 | ### Working Example 35 | Below is a snippet from [example.py](/example/example.py): 36 | 37 | The deployment dict has these additional fields compared to Cortex docs: 38 | - `"project_name":` in deployment root 39 | - name of the project, zipped source code is going to be uploaded to S3 path: `/.zip` 40 | - `predictor_path`: Module containing your predictor: `cls.__module__` = e.g. `predictors.my_predictor` 41 | - Optional `predictor_class_name`: `cls.__name__` of your predictor class, default is `PythonPredictor` 42 | - `"config":` in `container` specification 43 | - config dict that will be saved to `predictor_config.json` in the root of deployment dir 44 | - this file can then be loaded in `main.py` and passed to the PythonPredictor constructor = can be seen in `resources/main.py` 45 | 46 | 47 | ```python 48 | deployment = { 49 | "name": "dummy-a", 50 | "project_name": "test", 51 | "kind": "RealtimeAPI", 52 | "predictor_path": "dummy_predictor", 53 | "pod": { 54 | "containers": [ 55 | { 56 | "config": {"geo": "cz", "model_name": "CoolModel", "version": "000000-000000"}, 57 | "env": { 58 | "SECRET_ENV_VAR": "secret", 59 | }, 60 | "compute": {"cpu": '200m', "mem": f"{0.1}Gi"}, 61 | } 62 | ], 63 | }, 64 | } 65 | 66 | # Deploy 67 | with cortex.deploy_temporarily( 68 | deployment, 69 | deploy_dir="dummy_dir", 70 | api_timeout_sec=30 * 60, 71 | verbose=True, 72 | ) as get_result: 73 | # Predict 74 | response = post(get_result.endpoint, json={}).json() 75 | ``` 76 | 77 | ### Required changes for projects using Cortex version <= 0.34 78 | - optionally add `main.py` to the root of your cortex deployment folder 79 | - if there is no `main.py` in the root of the deployment folder, the default one from `resources/main.py` will be used 80 | - restructure your deployment dict to look like the one in `example.py` 81 | 82 | ### Garbage API Collection 83 | Garbage API collection auto-removes forgotten APIs to reduce costs. 84 | 85 | Each deployed API has a timeout period configured during deployment after which it definitely should not exist in the cluster anymore. 86 | This timeout is stored in a Postgres database table. 87 | Cortex client periodically checks currently deployed APIs and removes expired APIs from the cluster. 88 | 89 | ### Can You Rollback? 90 | How do you deal with new model failure in production? 91 | Do you have the ability to return to your model's previous working version? 92 | There is no generic solution for everybody. 93 | But you can implement the best suiting your needs using the Python API for Cortex. 94 | Having a plan B is a good idea. 95 | 96 | ## Our Use Case 97 | We use this project to automate deployment to auto-scalable AWS instances. 98 | The deployment management is part of application-specific Flask applications, 99 | which call to Python-Cortex-Serving-Client to command environment-dedicated Cortex cluster. 100 | 101 | In cases where multiple environments share a single cluster, a shared Cortex database Postgres instance is required. 102 | 103 | Read more about our use case in [Cortex Client release blog post](https://medium.com/@aiteamglami/serve-your-ml-models-in-aws-using-python-9908a4127a13). 104 | Or you can [watch a video about our use case](https://youtu.be/aU95dBAspr0?t=1164). 105 | 106 | ## Get Started 107 | This tutorial will help you to get [the basic example](/example/example.py) running under 15 minutes. 108 | 109 | ### Pre-requisites 110 | - Linux OS 111 | - Docker 112 | - Postgres 113 | 114 | 115 | ### Setup Database 116 | Follow instructions below to configure local database, 117 | or configure cluster database, 118 | and re-configure db in [the example script](/example/example.py). 119 | 120 | ```bash 121 | sudo su postgres; 122 | psql postgres postgres; 123 | create database cortex_test; 124 | create role cortex_test login password 'cortex_test'; 125 | grant all privileges on database cortex_test to cortex_test; 126 | ``` 127 | 128 | You may need to configure also 129 | ```bash 130 | 131 | vi /etc/postgresql/11/main/pg_hba.conf 132 | # change a matching line into following to allow localhost network access 133 | # host all all 127.0.0.1/32 trust 134 | 135 | sudo systemctl restart postgresql; 136 | ``` 137 | 138 | ### Install Cortex 139 | Supported [Cortex.dev](https://cortex.dev) version is a Python dependency version installed through `requirements.txt`. 140 | 141 | Cortex requires having [Docker](https://docs.docker.com/get-docker/) installed on your machine. 142 | 143 | ### Deploy Your First Model 144 | 145 | The deployment and prediction example resides in [the example script](/example/example.py). 146 | Make sure you have created a virtual environment, and installed requirements in `requirements.txt` and `requirements-dev.txt`, 147 | before you execute it. 148 | 149 | ## Contact Us 150 | [Submit an issue](https://github.com/glami/cortex-serving-client/issues) or a pull request if you have any problems or need an extra feature. 151 | -------------------------------------------------------------------------------- /base_docker_image/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | 3 | ENV PYTHONUNBUFFERED TRUE 4 | 5 | COPY requirements.txt /tmp/ 6 | RUN pip install --no-cache-dir -r /tmp/requirements.txt 7 | 8 | COPY download_source_code.py /tmp/ 9 | 10 | CMD python /tmp/download_source_code.py -------------------------------------------------------------------------------- /base_docker_image/download_source_code.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import subprocess 4 | import sys 5 | from pathlib import Path 6 | 7 | import boto3 8 | import zipfile 9 | 10 | BUCKET_NAME = os.environ["CSC_BUCKET_NAME"] 11 | S3_PATH = os.environ["CSC_S3_SOURCE_ZIP_PATH"] 12 | S3_SSE_KEY = os.environ["CSC_S3_SSE_KEY"] 13 | UVICORN_PORT = os.environ["CSC_UVICORN_PORT"] # CORTEX_* env vars are reserved for cortex 14 | DEPLOYMENT_DIR = "cortex_deployment" 15 | EXIT_CODE_ERROR = 4 16 | 17 | ADDITIONAL_BUCKET_KWARGS = { 18 | BUCKET_NAME: {"SSECustomerAlgorithm": "AES256", "SSECustomerKey": S3_SSE_KEY}, 19 | } 20 | 21 | logging.basicConfig( 22 | format="%(asctime)s : %(levelname)s : %(threadName)-10s : %(name)s : %(message)s", level=logging.INFO, 23 | ) 24 | 25 | logger = logging.getLogger(__name__) 26 | 27 | 28 | def download_file(s3_path, local_filepath, bucket_name, overwrite=True): 29 | local_filepath = str(local_filepath) 30 | s3_path = str(s3_path) 31 | 32 | if Path(local_filepath).exists() and not overwrite: 33 | logger.info(f"{local_filepath} exists and overwrite=False, skipping") 34 | else: 35 | Path(local_filepath).parent.mkdir(parents=True, exist_ok=True) 36 | logger.info(f"Downloading file from {bucket_name}:{s3_path} -> {local_filepath}") 37 | 38 | session = boto3.session.Session() 39 | session.client("s3").download_file( 40 | bucket_name, s3_path, local_filepath, ExtraArgs=ADDITIONAL_BUCKET_KWARGS[bucket_name] 41 | ) 42 | 43 | 44 | def unzip(filepath, targetdir): 45 | logger.info(f"Unzipping {filepath} -> {targetdir} ...") 46 | with zipfile.ZipFile(filepath, "r") as zip_ref: 47 | zip_ref.extractall(targetdir) 48 | 49 | 50 | def run_cmd(cmd): 51 | logger.info(f"Running {cmd} ...") 52 | os.system(cmd) 53 | 54 | 55 | def run_with_error_on_completion(cmd): 56 | logger.info(f"Running {cmd} ...") 57 | try: 58 | proc = subprocess.Popen(cmd, shell=True) 59 | proc.wait() 60 | finally: 61 | logger.info(f"Uvicorn process ended, exiting with code {EXIT_CODE_ERROR}!") 62 | sys.exit(EXIT_CODE_ERROR) 63 | 64 | 65 | def main(): 66 | # download code.zip 67 | local_path = "code.zip" 68 | download_file(S3_PATH, local_path, BUCKET_NAME, overwrite=True) 69 | # unzip it into cortex_deployment dir 70 | unzip(local_path, DEPLOYMENT_DIR) 71 | # set workdir to cortex_deployment 72 | os.chdir(str(Path.cwd() / DEPLOYMENT_DIR)) 73 | # set PYTHONPATH=/cortex_deployment 74 | os.environ["PYTHONPATH"] = str(Path.cwd()) 75 | # install requirements 76 | run_cmd(f"pip install -r requirements.txt") 77 | # run app 78 | run_with_error_on_completion(f"uvicorn --no-access-log --workers 1 --host 0.0.0.0 --port {UVICORN_PORT} " 79 | f"--proxy-headers --forwarded-allow-ips '*' main:app") 80 | 81 | 82 | if __name__ == "__main__": 83 | main() 84 | -------------------------------------------------------------------------------- /base_docker_image/requirements.txt: -------------------------------------------------------------------------------- 1 | annoy==1.16.3 2 | implicit==0.4.2 3 | psutil==5.6.7 4 | sortedcontainers==2.1.0 5 | sentry-sdk==0.20.2 6 | sqlalchemy==1.3.15 7 | numpy==1.22.0 8 | pandas==1.0.1 9 | boto3==1.12.11 10 | joblib==0.14.1 11 | psutil==5.6.7 12 | scipy==1.4.1 13 | tqdm==4.43.0 14 | uvicorn[standard]==0.15.0 15 | fastapi==0.68.0 -------------------------------------------------------------------------------- /cortex-serving-client-logo-2.drawio: -------------------------------------------------------------------------------- 1 | 5VpZk6JIF/019TgTrFb5qKIWtkCpuMDLBIuNbGIoivDrv3MBLVyqpyeiJ+aL6O6oKL2ZefPmuecuCfXC9+LzcG/tNkrirqMXjnHPL7z0wnEc0+LwiyR5JWFZplVJvL3v1rJPwcwv1rWQqaVH310fbiamSRKl/u5W6CTb7dpJb2TWfp9kt9O+J9HtrjvLWz8IZo4VPUqXvptuKumbyHzK39e+t0mv56tHYusyuRYcNpabZA0R33/he/skSatP8bm3jgi9Cy7VusEXo1fD9utt+jMLCjfm34rir28Ms14Mu/xfrTH/R+2ekxUd6wPXxqb5BQHYvaOPflxC1T2t96kPgMaWvY4+koOf+skW43aSpkmMCRENdC0n9PbJcev2kijZl6r47+W/ho5O5Hu0Nk12kFqHXeXC7/55Dau75Zadi5S5SPDZtVLrhe9UX7nB4eS9cN1zjPP2Pt5Vzsy7gr08H51iFxoF41vvU8aRktOY77JOnB1tfrQdc9NgzC0O5pKN7O20GBf9ozJ78+X3TWoPxUKL1eBjNkrc92mm+W8nl1v44+0ocrg2dKjROG7nZt4OjLyxZjsKzaC5n8u7ucgruXhyYuek6KGozd4yxX/LFZ/NzaGROnx0dIcDYbwUCzmXvfWQPdhbpeXw5vZmf+gab52i2hfrpU425ums1zVtOd4w7nunNc7bmO0c3UKpzlrIGeafSKfsX7ER7GF0tIq/s7ddzWvaxrVDa9U9mcOwuWdhc9OdM8TYTCxsfpEb3CKm/c2ZeDRXk9N0MO3DZqyLMpo71jtH830aNmza2XFaGNwgM/UHuz7HZpV9djxIzZWaGUs1cvJ2Y1z2XC4K3aHXloP+WekJjILf2Ccxl9HWep+QPNNmgqBKBsl3psT4hA/mcRrWN87YcuNF7nDRyYZvoatQfNmzluLe4dSNM5zfjzPqTCCdnLkaFdayffyYyedx0AFPPs+uxdOTwY02Djdvy347sTk2GnOjyI7np+lwUVj89OT0bu0wh+2gPDOwMTEHex7Hev9Y+7WB3e7krqYZfr6bqw1jgluVHSPC7diwHfxYMM77ghlviUviLYZxFLnM6LSW6FydTJYaWEoN/AIhlvnNRss73sdwGgFF4AOM4/JzrujOzRy513kjTpnLQT7lRxtgghic1GvOsGGxcUI3N5bTHWxnFL1fKFLfl4mHvBoYq26E+DzZw3Nk8LRmcHC5gejkMvLAyFcFQz/QLuVOTonkxcP4TKcoOgUQO5mxWYCZYEPpHWa9PEdAqXCH7Wy8pB3auearsRnPM1WXXytd7MkZDgJrNRW1oO+Pi5uTebJ0LtfD2vCiqzrBHCdox+ZWjVxClFjGT0V7CAZsVcbmO6nBtQ82L7dkzoyNQikUPa1taRPLd+572MK604dvBOth/7XaLzs9QfMOiY/hI67y8PPzYilCO3LIdgFERjqd0Jy3j4uVGgEpUSFu64YwRtwougy0vJx4rUrOUS0cXikcyMIzyXDao6bLQEyheSQTwBWgbvCqFPo1MgfwMDJWo3cnHoTWcnF0K8/wZT4oc9MuceJFjPXBkxNiHyUjFJuxWCI4XASEsLE888YqgqfffgqNr6wKWUWf/zL2kTbnlgm9HzChUAQVtUeTfoIJdLL3cn0VS5UuUdX74i3b+z9gu1LgjKKmT36G7W9j7hHJW3aeHxHtdbfXz4PFEboL2ME63CKf890N6saHHYGH8O1al8neQqFco8si+XstyZe8Dv6FZ62c14fM4ZFneUWSebVH80g2Z8fBnNH0Pq+VXHmeYzR9npe6S2YcWtb7KKJaLj85H+ae1X/OO8QpsnbsNBD8uKkI4sZeLsIy6+YVY4x4UKIAD2zN2kMUDfBk+R9e9VFhQ0RsYHPnk0MWI3/Dg5yKKmVQtzBc5HaJasSsdco7QnZXLcAoN7qZ03s+x+E2iL42Dxvyu/GWzbVTyLNxIDNqMRHUQuHv5gRUSca6zJVVMlDO95XU4NFRQK76QqYEDkNebI5TBXTLjm2SuteM1RXu93FXo5w8NV6eN+vlIqe4hkcip2S+u0PXkZSZiro+vZMrhZmtaa/hYmdyG4bGNGlyVIL5WdPdaqxZ8VF5CYfx6qoL55Vvc8TWhU3Tk5uzja7DyO/P9KmLupX5VzpE7MFovZuIfwNfGLBy4w77NbPZCN1kaKymG1RpioL7zPdOPepuNu0bT7KVwSmtBdfOrbxL3WnLWI7QSUCv/yS3UWyjv95Rj5caK6o41pLdmNy87DS/8eiKS84rLRM8tJcD5j5b0Z407r5H6DiqPZHx6+5x3uwu0c2MDnXXHRr0w7G7al1bd4fRwe5HKXH0MdtXeUoNtMhF/3O4R6TXLvedh9NhE9sx94llU47bRZNrB2AU2eD1BBcRZ7WI5IdOxUX8LTJ72Q6pc7OHA9Fcydcq+MnLKTKMWJRYYZ77PtrYWxXj0+hbj0FMKUdU0EyTvANiiEUc80qAOJuF/l3G3da6t4ruicS3WlfpKcgZZN+jw53Bm3miSjKq+gQxoAhjqS+oPSFXdeP8bfa0ng0HjNXrFkD+UJ7mmjWSs8ZdawjdN+jn1YzhGWlH967ki1yKvBtW825rzUWWu6hTyI4ne7uLnPiN6lJuc2kli+fX75dopLVX2bLcE57otIztgnGpjr6PThY3T6ueYneylkLLXkYMvPIKluYW6tBF9+X7eKUyqGHMg3zJIlPvjjYnRhrHbpw4jRwfGSqetOy4fTT1xn7lHtRhgLlSwmi8Ck+09+aMBcvUBPpzYxU+rDOHg8K69gtJ1lh3gxcqB/r80Q7ngj92nLVSd0YM7OhexncPpBuZp6zHGtem+0kMf/Ha1tzhLgFmUvZBtIeMp+kh+rr5QQ3Q30kGstvcm+SorcQ63cvVwGuO5bKEu2kw4TTJYceSI44p080wP6dM7/HqLEP9FThkVE/vlXLMC3NUizOtkfuZh0xM1eNHco6+q4F8rPah78rtd2nCjIOwINs1XUEfarDIzExTjv6V9LPlWmR5zKO+lNjPojKxSjHJ1bxDVYotx2YZ9RlH9LNnsl8NOvh80TXBWefo1GW2si08PluDM+NeKYjok2Fnv0CFYxSJMOpKsmSIn7YpHHDMVLKvlE9wD/CQ+TPo6ZBPPE2aIxNMLmcBNk55FuB0pnMhS9AeOeyHHZ2qHy/x+ZGM7EBPRX1WUy/GYSuv0e9ZZQO4kQFX7CfDVkegs0NPhnmFKikXXc116NsErvSFNLlb58EXNAc2BQbO5xFeB02SjxWmyvP5UihqPVoTHhX4FvvecPUZf9d+V5Ulj/jJ1ngTpwU1F1Ad+rjXhgfIrnzAXOBf2QO7sI+HHhR32YZckyqfkB66D5ON6D8hV7BvHxyQRcxHX4raqk9ybZYhYwts5V+yxaP+tIwL0qEAQ/gbnO0fyM/IxMjGHcSXB3/0S9/c6W7IP22h3hj+RX717m0HfkoB3YjlDL6qOHmPy9rvXCpUhgqUaLjnq0/vG11UEa++b5SfBZyNukdaq9JzAoojek4AjvDAGrFCVaviiEZjhYOqg3gpHPC6z8C22ufIM4VC3OLomYqqe1zNKwHdYVHGWIBY0OeC6neVK+f0ci8WWALnci+ltiOr7fjRGGKNeKtwVe5R+LI66n2KdaHkYWEQxugeiVvoIstY94i7iOsO7KT4JJsg1xEvODd8AV10F1FYYFnnIOSNYi5WHL+OwYaQoS4aPMyVch/iMnWplNPm1JXmFP+wu9AoH0syYSECI3BFphikfEN36YsdPxo7gxv4THf4SX2u52N034ZPwFX4hrAGZugaWK3MJ5RH56WNOBvs71zPRrGIGEXMToS7dYJWPU/KiKc368DXCveOWOU+dOMS7qZ6WMXal+uoDs3JH3XORKwUIeoVcj06KNyFwa/SjqLUUYQ1F2/XIW9TzADb+f06DtzhKi7erUEdAh+Ji4/6rmNlDYUvwOmAbKc7q0Lz2B/ESkvuM8C/D47IInLVoXoW46G29iuf6aRfwXfnZgxr4LMJcZbkiClDBCeLy7M84nkZX5VPuIqLN2tQUxSebhPlGpwdXODUnDCCXwPiP+1PvAy5Mhau8qqm4BzY02GV+zWoJfBvFSdBn3hGzwq9qvbAt2WcGLVv65i7rKkwuDnPzT6Bh7ibcA+2XeXAtPncJW48V9HT+jadBHUfm2mcib4qa9Et5FtP/fLZ1Tc9ja3l+XDznOYiW41Y7FX2d+sYt/Li2uM99Hz18+uWot/3q7g1SjvfXU2b/WNpjxW3d3awS9EP4kaR4k422qOnvei+fE9x5o2znTzI0Z+Gj/3t4njtbz/3q/ag5yjoabVg0ui3oZvbQP9ogx70fl3Z59qcGTvcAus6jXU3eNW9a3Qoe9vP/vTVHEaFMzyTbvT11NvDVt6ELebB5p1W3QMfzCdPSc3mk5dee3sZx+fCjR3cdl74bv1KbL1P1+cv37Wx1zd48Xm4TuJ1us8x5bKAr1/61a89X+uv2ecrRP7yonDTeH14FVr1a0vvqvrzzR4+1C/3/sGLPvHvX/RBi7870Cu+bOOn69nOcmgk21v0cm6TxthQYp+9pzuk+yS8vh9lRYi8veX6QOvy9m+bbEnzdz+K7kTV4sZbwrcB/f9FjmBuHcGKj54QnjhC+Lf80Po9/SAw4v+XH15/Tz9w4m088P+1H97+3g+R38DlBlTX3wP46s8PDsmRBpp+2a33PqxcE4625YQ2AP74lHV3ib9NywOJ3RdRggQ6aLfe9Y9JmOc++sKbX7tun6RWbSj/9mtcyQt3qa316Er2iSs59vXPV/Zfcmf793MnJ/yiUtV+8OefTPMf/9+797Lhb+TfP9rML0q9rdaf7G0V5Pk/31o/59XXf56A8fXzr8vKscYf6fH9/wE= -------------------------------------------------------------------------------- /cortex-serving-client-logo-2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/glami/cortex-serving-client/aa594b2061534d84fdaf320a3250a97eb0b66d92/cortex-serving-client-logo-2.jpg -------------------------------------------------------------------------------- /cortex-serving-client-logo-2.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /cortex-serving-client.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/glami/cortex-serving-client/aa594b2061534d84fdaf320a3250a97eb0b66d92/cortex-serving-client.png -------------------------------------------------------------------------------- /cortex-serving-client.xcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/glami/cortex-serving-client/aa594b2061534d84fdaf320a3250a97eb0b66d92/cortex-serving-client.xcf -------------------------------------------------------------------------------- /cortex_serving_client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/glami/cortex-serving-client/aa594b2061534d84fdaf320a3250a97eb0b66d92/cortex_serving_client/__init__.py -------------------------------------------------------------------------------- /cortex_serving_client/command_line_wrapper.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import subprocess 4 | import time 5 | from math import ceil 6 | from typing import List 7 | 8 | CORTEX_DEFAULT_COMMAND_SYSTEM_PROCESS_TIMEOUT = 3 * 60 9 | CORTEX_CMD_BASE_RETRY_SEC = 10 10 | 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | def _verbose_command_wrapper( 16 | cmd_arr: List[str], cwd: str = None, timeout: int = CORTEX_DEFAULT_COMMAND_SYSTEM_PROCESS_TIMEOUT, 17 | input: bytes = None, retry_count=3, allow_non_0_return_code_on_stdout_sub_strs=None, sleep_base_retry_sec=CORTEX_CMD_BASE_RETRY_SEC 18 | ): 19 | cmd_str = " ".join(cmd_arr) 20 | message = "" 21 | for retry in range(retry_count + 1): 22 | try: 23 | p = subprocess.run(cmd_arr, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, timeout=timeout, input=input) 24 | stdout = p.stdout.decode() 25 | stderr = p.stderr.decode() 26 | # check_returncode() does not print process output to the console automatically so custom below 27 | if p.returncode == 0: 28 | logger.debug(f"Successful command {cmd_arr} stdout is {json.dumps(stdout)}") 29 | return stdout 30 | 31 | elif allow_non_0_return_code_on_stdout_sub_strs is not None and any([s in stdout or s in stderr for s in allow_non_0_return_code_on_stdout_sub_strs]): 32 | logger.debug(f"Allowed unsuccessful command {cmd_arr} execution. Stdout {json.dumps(stdout)} or stderr {json.dumps(stderr)} matches one of {allow_non_0_return_code_on_stdout_sub_strs}") 33 | return stderr 34 | 35 | else: 36 | message = f"Non zero return code for command {cmd_str}! Output: {json.dumps(dict(stdout=stdout, stderr=stderr))}" 37 | 38 | except subprocess.TimeoutExpired as e: 39 | message = f'Timed out command {cmd_str}: {e} with stdout: "{e.output}" and stderr: "{e.stderr}"' 40 | 41 | if retry < retry_count: 42 | sleep_secs = ceil(sleep_base_retry_sec * 2 ** retry) 43 | logger.info(f"Retrying after {sleep_secs} sec: {message}") 44 | time.sleep(sleep_secs) 45 | 46 | else: 47 | raise ValueError(f"Retry count for command {cmd_str} exceeded: {message}") 48 | 49 | raise RuntimeError(f'Execution should never reach here: {message}') 50 | 51 | 52 | -------------------------------------------------------------------------------- /cortex_serving_client/cortex_client.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import json 3 | import logging 4 | import multiprocessing 5 | import os 6 | import subprocess 7 | import tempfile 8 | import time 9 | from contextlib import contextmanager 10 | from copy import deepcopy 11 | from datetime import datetime, timedelta 12 | from json import JSONDecodeError 13 | from math import ceil 14 | from pathlib import Path 15 | from threading import Thread, Lock 16 | from typing import NamedTuple, Optional, Dict, List, Callable 17 | 18 | import cortex 19 | import yaml 20 | from cortex.binary import get_cli_path 21 | from psycopg2._psycopg import DatabaseError 22 | from psycopg2.extras import NamedTupleCursor 23 | from psycopg2.pool import ThreadedConnectionPool 24 | 25 | from cortex_serving_client import s3 26 | from cortex_serving_client.command_line_wrapper import _verbose_command_wrapper 27 | from cortex_serving_client.deployment_failed import DeploymentFailed, COMPUTE_UNAVAILABLE_FAIL_TYPE, \ 28 | DEPLOYMENT_TIMEOUT_FAIL_TYPE, DEPLOYMENT_ERROR_FAIL_TYPE, DEPLOYMENT_JOB_NOT_DEPLOYED_FAIL_TYPE 29 | from cortex_serving_client.hash_utils import get_file_hash 30 | from cortex_serving_client.printable_chars import remove_non_printable 31 | from cortex_serving_client.retry_utils import create_always_retry_session 32 | from cortex_serving_client.s3 import BUCKET_NAME, BUCKET_SSE_KEY 33 | from cortex_serving_client.shell_utils import kill_process_with_children 34 | from cortex_serving_client.zip_utils import zip_dir, add_file_to_zip 35 | 36 | ENV_CSC_DEFAULT_DOCKER_IMAGE = "CSC_DEFAULT_BASE_DOCKER_IMAGE" 37 | 38 | KIND_REALTIME_API = 'RealtimeAPI' 39 | KIND_BATCH_API = 'BatchAPI' 40 | KIND_ASYNC_API = 'AsyncAPI' 41 | KIND_TASK_API = 'TaskAPI' 42 | KIND_DEFAULT = KIND_REALTIME_API 43 | 44 | NOT_DEPLOYED = "not deployed" 45 | 46 | """ 47 | Source: https://github.com/cortexlabs/cortex/blob/0.31/pkg/types/status/code.go 48 | On some places custom "not_deployed" status may be used. 49 | """ 50 | 51 | LIVE_STATUS = "status_live" 52 | UPDATING_STATUS = "status_updating" 53 | COMPUTE_UNAVAILABLE_STATUS = "status_stalled" 54 | CORTEX_STATUSES = [LIVE_STATUS, UPDATING_STATUS, "status_error", "status_oom", COMPUTE_UNAVAILABLE_STATUS] # 55 | NOT_DEPLOYED_STATUS = "status_not_deployed" 56 | 57 | # returned if cortex describe says Failed 58 | API_FAILED_STATUS = "api_failed_status" 59 | 60 | JOB_STATUS_SUCCEEDED = 'succeeded' 61 | JOB_STATUS_ENQUEUING = 'enqueuing' 62 | JOB_STATUS_RUNNING = 'running' 63 | JOB_STATUS_ENQUEUED_FAILED = 'failed_while_enqueuing' 64 | JOB_STATUS_COMPLETED_WITH_FAILURES = 'completed_with_failures' 65 | JOB_STATUS_WORKER_ERROR = 'worker_error' 66 | JOB_STATUS_OOM = 'out_of_memory' 67 | JOB_STATUS_TIMED_OUT = 'timed_out' 68 | JOB_STATUS_STOPPED = 'stopped' 69 | JOB_STATUS_UNEXPECTED_STATUS = 'unexpected_status' 70 | JOB_STATUS_PENDING = 'pending' 71 | 72 | JOB_FAIL_STATUSES = [ 73 | JOB_STATUS_ENQUEUED_FAILED, 74 | JOB_STATUS_WORKER_ERROR, 75 | JOB_STATUS_OOM, 76 | JOB_STATUS_TIMED_OUT, 77 | JOB_STATUS_STOPPED, 78 | API_FAILED_STATUS 79 | ] 80 | 81 | JOB_VALID_STATUSES = [ 82 | JOB_STATUS_SUCCEEDED, 83 | JOB_STATUS_PENDING, 84 | JOB_STATUS_ENQUEUING, 85 | JOB_STATUS_RUNNING, 86 | JOB_STATUS_COMPLETED_WITH_FAILURES, 87 | ] + JOB_FAIL_STATUSES 88 | 89 | 90 | CORTEX_DELETE_TIMEOUT_SEC = 10 * 60 91 | CORTEX_DEPLOY_REPORTED_TIMEOUT_SEC = 60 92 | CORTEX_DEFAULT_DEPLOYMENT_TIMEOUT = 20 * 60 93 | CORTEX_DEFAULT_API_TIMEOUT = CORTEX_DEFAULT_DEPLOYMENT_TIMEOUT 94 | CORTEX_MIN_API_TIMEOUT_SEC = CORTEX_DELETE_TIMEOUT_SEC 95 | CORTEX_DEPLOY_RETRY_BASE_SLEEP_SEC = 5 * 60 96 | CORTEX_STATUS_CHECK_SLEEP_SEC = 15 97 | INFINITE_TIMEOUT_SEC = 30 * 365 * 24 * 60 * 60 # 30 years 98 | WAIT_BEFORE_JOB_GET = int(os.environ.get('CORTEX_WAIT_BEFORE_JOB_GET', str(30))) 99 | N_RETRIES_BATCH_JOB = 5 100 | 101 | CORTEX_PATH = get_cli_path() 102 | 103 | logger = logging.getLogger('cortex_client') 104 | __cortex_client_instance = None 105 | 106 | # insert your base image id 107 | DEFAULT_DOCKER_IMAGE = os.environ.get(ENV_CSC_DEFAULT_DOCKER_IMAGE) 108 | DEFAULT_PORT = os.environ.get("CSC_DEFAULT_UVICORN_PORT", 8080) 109 | DEFAULT_PREDICTOR_CLASS_NAME = "PythonPredictor" 110 | 111 | 112 | class CortexClient: 113 | """ 114 | An object used to execute commands on Cortex, maintain API state in the db to collect garbage. 115 | 116 | Is thread-safe but not process-safe. 117 | """ 118 | 119 | def __init__(self, db_connection_pool: ThreadedConnectionPool, gc_interval_sec=30 * 60, cortex_env="aws"): 120 | self.db_connection_pool = db_connection_pool 121 | self._init_garbage_api_collector(gc_interval_sec) 122 | self.cortex_env = cortex_env 123 | self.cortex_vanilla = cortex.client(self.cortex_env) 124 | self.lock = Lock() 125 | logger.info(f'Constructing CortexClient for {CORTEX_PATH}.') 126 | 127 | def _prepare_deployment(self, deployment, deploy_dir): 128 | deployment = deepcopy(deployment) 129 | name = deployment["name"] 130 | 131 | if "kind" not in deployment: 132 | deployment["kind"] = KIND_DEFAULT 133 | 134 | assert len(deployment["pod"]['containers']) == 1, f"Number of containers must be 1 for simplicity! " \ 135 | f"Not {len(deployment['pod']['containers'])}" 136 | container = deployment["pod"]['containers'][0] 137 | 138 | docker_image = container.get("image", DEFAULT_DOCKER_IMAGE) 139 | if docker_image is None: 140 | raise ValueError(f'Image needs to be provided either in the container configuration or as environmental variable {ENV_CSC_DEFAULT_DOCKER_IMAGE}.') 141 | 142 | container['image'] = docker_image 143 | 144 | port = deployment["pod"].get('port', DEFAULT_PORT) 145 | deployment["pod"]["port"] = port 146 | 147 | if 'name' not in container: 148 | container['name'] = 'api' 149 | 150 | project_name = deployment.pop("project_name") 151 | predictor_path = deployment.pop("predictor_path") 152 | predictor_class_name = deployment.pop("predictor_class_name", DEFAULT_PREDICTOR_CLASS_NAME) 153 | bucket_name = deployment.pop("bucket_name", BUCKET_NAME) 154 | bucket_sse_key = deployment.pop("bucket_sse_key", BUCKET_SSE_KEY) 155 | config = container.pop("config", {}) 156 | with self.lock: 157 | with tempfile.TemporaryDirectory() as tmp_dir_name: 158 | fname = f"{name}.zip" 159 | local_zip_path = str(Path(tmp_dir_name) / fname) 160 | s3_path = f"{project_name}/{fname}" 161 | zip_dir(deploy_dir, local_zip_path) 162 | 163 | # dump config and add it to zipped deploy dir 164 | config_path = str(Path(tmp_dir_name) / 'predictor_config.json') 165 | with open(config_path, 'w') as f: 166 | json.dump(config, f) 167 | add_file_to_zip(local_zip_path, config_path) 168 | 169 | # add empty requirements.txt if it does not exist - it is mandatory 170 | requirements_path = Path(deploy_dir) / "requirements.txt" 171 | if not requirements_path.exists(): 172 | tmp_req_path = str(Path(tmp_dir_name) / 'requirements.txt') 173 | open(tmp_req_path, 'w').close() 174 | add_file_to_zip(local_zip_path, tmp_req_path) 175 | 176 | # add or check main.py 177 | main_path = Path(deploy_dir) / "main.py" 178 | if main_path.exists(): 179 | with open(main_path, 'rt') as f: 180 | code = ''.join(f.readlines()) 181 | assert '@app.post' in code, f"API {name} failed to deploy: @app.post not found in {main_path}! " \ 182 | f"main:app must be runnable by Uvicorn!" 183 | else: 184 | # add default main.py if it was not supplied 185 | add_file_to_zip(local_zip_path, Path(__file__).parent / 'resources' / 'main.py') 186 | 187 | predictor_filepath = Path(deploy_dir) / f"{predictor_path.replace('.', '/')}.py" 188 | try: 189 | with open(predictor_filepath, 'rt') as f: 190 | code = ''.join(f.readlines()) 191 | assert predictor_class_name in code, f"{predictor_class_name} not found in {predictor_filepath}!" 192 | except Exception as e: 193 | raise RuntimeError(f"API {name} failed to deploy:") from e 194 | 195 | s3.upload_file(local_zip_path, s3_path, bucket_name, bucket_sse_key, verbose=True) 196 | 197 | # add necessary env vars 198 | if 'env' not in container: 199 | container['env'] = {} 200 | container["env"]['CSC_BUCKET_NAME'] = bucket_name 201 | container["env"]['CSC_S3_SSE_KEY'] = bucket_sse_key 202 | container["env"]['CSC_S3_SOURCE_ZIP_PATH'] = s3_path 203 | container["env"]['CSC_UVICORN_PORT'] = port 204 | container["env"]['CSC_PREDICTOR_PATH'] = predictor_path 205 | container["env"]['CSC_PREDICTOR_CLASS_NAME'] = predictor_class_name 206 | # so redeploy actually deploys the new code 207 | container["env"]['CSC_HASH_OF_THE_CODE_DIR'] = get_file_hash(local_zip_path) 208 | 209 | # env must be string -> string map 210 | container['env'] = {str(k): str(v) for k, v in container['env'].items()} 211 | return deployment 212 | 213 | @staticmethod 214 | def _kill_log_worker(log_worker): 215 | if log_worker is not None and log_worker.is_alive(): 216 | logger.info(f"Killing process printing logs with pid: {log_worker.pid} ...") 217 | # log_worker.terminate() will not actually terminate the subprocess streaming logs :( 218 | kill_process_with_children(log_worker.pid) 219 | 220 | def deploy_single( 221 | self, 222 | deployment: Dict, 223 | deploy_dir, 224 | deployment_timeout_sec=CORTEX_DEFAULT_DEPLOYMENT_TIMEOUT, 225 | api_timeout_sec=CORTEX_DEFAULT_API_TIMEOUT, 226 | print_logs=False, 227 | n_retries=0 228 | ) -> 'CortexGetResult': 229 | """ 230 | Deploy an API until timeouts. Cortex docs https://docs.cortex.dev/deployments/deployment. 231 | 232 | Parameters 233 | ---------- 234 | 235 | deployment 236 | Cortex deployment config. See https://docs.cortex.dev/deployments/api-configuration 237 | deploy_dir 238 | Base directory of code to deploy to Cortex. 239 | deployment_timeout_sec 240 | Time to keep the API deploying. Including execution of predictor's `__init__`, 241 | which can be used to e.g. train a model. 242 | api_timeout_sec 243 | Time until API will be auto-deleted. Use `INFINITE_TIMEOUT_SEC` for infinite. 244 | print_logs 245 | Subscribe to Cortex logs of the API and print random pod's logs to stdout. 246 | n_retries 247 | Number of attempts to deploy until raising the failure. Retries only if the failure is not an application error. 248 | 249 | Returns 250 | ------- 251 | get_result 252 | Deployed API get result information. 253 | 254 | """ 255 | deployment = self._prepare_deployment(deployment, deploy_dir) 256 | name = deployment["name"] 257 | 258 | predictor_yaml_str = yaml.dump([deployment], default_style='"') 259 | 260 | if api_timeout_sec < CORTEX_MIN_API_TIMEOUT_SEC: 261 | logger.info(f'API timeout {api_timeout_sec} is be smaller than minimal API timeout {CORTEX_MIN_API_TIMEOUT_SEC}. Setting it to {CORTEX_MIN_API_TIMEOUT_SEC} to avoid excessive GC.') 262 | api_timeout_sec = CORTEX_MIN_API_TIMEOUT_SEC 263 | 264 | if deployment_timeout_sec < CORTEX_DEFAULT_DEPLOYMENT_TIMEOUT: 265 | logger.info(f'Deployment timeout {deployment_timeout_sec} is be smaller than default deployment timeout {CORTEX_DEFAULT_DEPLOYMENT_TIMEOUT}. Setting it to {CORTEX_DEFAULT_DEPLOYMENT_TIMEOUT}.') 266 | deployment_timeout_sec = CORTEX_DEFAULT_DEPLOYMENT_TIMEOUT 267 | 268 | # file has to be created the dir to which python predictors are described in yaml 269 | filename = f"{name}.yaml" 270 | filepath = f"{deploy_dir}/{filename}" 271 | 272 | log_worker = None 273 | for retry in range(n_retries + 1): 274 | try: 275 | self._collect_garbage() 276 | with str_to_public_temp_file(predictor_yaml_str, filepath, self.lock): 277 | gc_timeout_sec = deployment_timeout_sec + CORTEX_DELETE_TIMEOUT_SEC 278 | logger.info(f"Deployment {name} has deployment timeout {deployment_timeout_sec}sec and GC timeout {gc_timeout_sec}sec.") 279 | self._insert_or_update_gc_timeout(name, gc_timeout_sec) 280 | _verbose_command_wrapper([CORTEX_PATH, "deploy", filename, f"--env={self.cortex_env}", "--yes", "-o=json"], cwd=deploy_dir) 281 | 282 | if print_logs and deployment['kind'] == KIND_REALTIME_API: 283 | assert log_worker is None or not log_worker.is_alive(), f"Should be None or dead!" 284 | log_worker = self._cortex_logs_print_async(name) 285 | 286 | start_time = time.time() 287 | while True: 288 | if log_worker is not None and not log_worker.is_alive(): 289 | # necessary because cortex logs interrupts when pod is spun up and has to be run again 290 | log_worker = self._cortex_logs_print_async(name) 291 | 292 | get_result = self.get(name) 293 | time_since_start = ceil(time.time() - start_time) 294 | 295 | if get_result.status == LIVE_STATUS: 296 | self._kill_log_worker(log_worker) 297 | gc_timeout_sec = api_timeout_sec + CORTEX_DELETE_TIMEOUT_SEC 298 | logger.info(f"Deployment {name} successful. Setting API timeout {api_timeout_sec}sec and GC timeout {gc_timeout_sec}sec.") 299 | self._insert_or_update_gc_timeout(name, gc_timeout_sec) 300 | return get_result 301 | 302 | elif get_result.status == COMPUTE_UNAVAILABLE_STATUS: 303 | self.delete(name) 304 | raise DeploymentFailed( 305 | f'Deployment of {name} API could not start due to insufficient memory, CPU, GPU or Inf in the cluster after {time_since_start} secs.', 306 | COMPUTE_UNAVAILABLE_FAIL_TYPE, name, time_since_start) 307 | 308 | elif get_result.status != UPDATING_STATUS: 309 | self.delete(name) 310 | raise DeploymentFailed(f"Deployment of {name} failed with status {get_result.status} after {time_since_start} secs.", 311 | DEPLOYMENT_ERROR_FAIL_TYPE, name, time_since_start) 312 | 313 | if time_since_start > deployment_timeout_sec: 314 | self.delete(name) 315 | raise DeploymentFailed(f"Deployment of {name} timed out after {deployment_timeout_sec} secs.", 316 | DEPLOYMENT_TIMEOUT_FAIL_TYPE, name, time_since_start) 317 | 318 | logger.info(f"Sleeping during deployment of {name} until next status check. Current status: {get_result.status}.") 319 | time.sleep(CORTEX_STATUS_CHECK_SLEEP_SEC) 320 | 321 | except DeploymentFailed as e: 322 | if retry == n_retries or e.failure_type == DEPLOYMENT_ERROR_FAIL_TYPE: 323 | self._kill_log_worker(log_worker) 324 | raise e 325 | 326 | else: 327 | sleep_secs = ceil(CORTEX_DEPLOY_RETRY_BASE_SLEEP_SEC * 2 ** retry) 328 | logger.warning(f'Retrying {retry + 1} time after sleep of {sleep_secs} secs due to deployment failure: {e}') 329 | time.sleep(sleep_secs) 330 | 331 | raise RuntimeError('Execution should never reach here.') 332 | 333 | 334 | @contextmanager 335 | def deploy_temporarily( 336 | self, 337 | deployment: Dict, 338 | deploy_dir: str, 339 | deployment_timeout_sec=CORTEX_DEFAULT_DEPLOYMENT_TIMEOUT, 340 | api_timeout_sec=CORTEX_DEFAULT_API_TIMEOUT, 341 | print_logs=False, 342 | n_retries=0 343 | ) -> 'CortexGetResult': 344 | """ 345 | Deploy an API until timeouts. Cortex docs https://docs.cortex.dev/deployments/deployment. 346 | 347 | Parameters 348 | ---------- 349 | 350 | deployment 351 | Cortex deployment config. See https://docs.cortex.dev/deployments/api-configuration 352 | deploy_dir 353 | Base directory of code to deploy to Cortex. 354 | deployment_timeout_sec 355 | Time to keep the API deploying. Including execution of predictor's `__init__`, 356 | which can be used to e.g. train a model. 357 | api_timeout_sec 358 | Time until API will be auto-deleted. Use `INFINITE_TIMEOUT_SEC` for infinite. 359 | print_logs 360 | Subscribe to Cortex logs of the API and print random pod's logs to stdout. 361 | n_retries 362 | Number of attempts to deploy until raising the failure. Retries only if the failure is not an application error. 363 | 364 | Returns 365 | ------- 366 | get_result 367 | Deployed API get result information. 368 | 369 | """ 370 | 371 | try: 372 | yield self.deploy_single(deployment, deploy_dir, deployment_timeout_sec, api_timeout_sec, print_logs, n_retries) 373 | 374 | finally: 375 | self.delete(deployment["name"]) 376 | 377 | def deploy_batch_api_and_run_job(self, 378 | deployment: dict, 379 | job_spec: dict, 380 | deploy_dir: str, 381 | deployment_timeout_sec=CORTEX_DEFAULT_DEPLOYMENT_TIMEOUT, 382 | api_timeout_sec=CORTEX_DEFAULT_API_TIMEOUT, 383 | print_logs=False, 384 | verbose=False, 385 | n_retries=0) -> "CortexGetResult": 386 | 387 | deployment['kind'] = deployment.get('kind', KIND_BATCH_API) 388 | assert deployment['kind'] == KIND_BATCH_API, f"Deployment['kind'] must be {KIND_BATCH_API}!" 389 | 390 | # has to be in BatchAPI, no idea why ... 391 | deployment['pod']['containers'][0]['command'] = ["python", "/tmp/download_source_code.py"] 392 | 393 | with self.deploy_temporarily( 394 | deployment, 395 | deploy_dir=deploy_dir, 396 | deployment_timeout_sec=deployment_timeout_sec, 397 | api_timeout_sec=api_timeout_sec, 398 | print_logs=print_logs, 399 | n_retries=n_retries 400 | ) as get_result: 401 | 402 | logger.info(f'BatchAPI {deployment["name"]} deployed. Starting a job.') 403 | with create_always_retry_session() as session: 404 | job_id = None 405 | for retry_job_get in range(N_RETRIES_BATCH_JOB): 406 | for retry_job_submit in range(3): 407 | try: 408 | job_json = session.post(get_result.endpoint, json=job_spec, timeout=10 * 60).json() 409 | 410 | except JSONDecodeError as e: 411 | logger.warning(f'Job submission response could not be decoded: {e}.') 412 | job_json = None 413 | 414 | if job_json is not None and 'job_id' in job_json: 415 | job_id = job_json['job_id'] 416 | break 417 | 418 | else: 419 | sleep_time = 3 * 2 ** retry_job_submit 420 | logger.info(f'Retrying job creation request after {sleep_time}.') 421 | time.sleep(sleep_time) 422 | continue 423 | 424 | if job_id is None: 425 | raise ValueError(f'job_id not in job_json {json.dumps(job_json)}') 426 | 427 | log_worker = None 428 | if print_logs: 429 | log_worker = self._cortex_logs_print_async(deployment['name'], job_id) 430 | 431 | # Don't call job too early: https://gitter.im/cortexlabs/cortex?at=5f7fe4c01cbba72b63cb745f 432 | time.sleep(WAIT_BEFORE_JOB_GET * 2 ** retry_job_get) 433 | 434 | job_status = JOB_STATUS_ENQUEUING 435 | while job_status in (JOB_STATUS_ENQUEUING, JOB_STATUS_RUNNING): 436 | if log_worker is not None and not log_worker.is_alive(): 437 | # necessary because cortex logs interrupts when pod is spun up and has to be run again 438 | log_worker = self._cortex_logs_print_async(deployment['name'], job_id) 439 | 440 | job_result = self.get(deployment["name"], job_id) 441 | job_status = job_result.status 442 | if verbose: 443 | logger.info(f"Job {job_id} has status: {job_status}") 444 | time.sleep(30) 445 | 446 | if log_worker is not None: 447 | logger.info(f"Terminating process printing logs from {deployment['name']} job_id={job_id} ...") 448 | log_worker.terminate() 449 | 450 | if job_status in [NOT_DEPLOYED_STATUS, JOB_STATUS_UNEXPECTED_STATUS] + JOB_FAIL_STATUSES: 451 | # TODO request fixes improvements https://gitter.im/cortexlabs/cortex?at=5f7fe4c01cbba72b63cb745f 452 | # Sleep in after job submission. 453 | logger.warning(f'Job unexpectedly undeployed or failed with status {job_status}. Retrying with sleep.') 454 | logger.info(f"Finished retry {retry_job_get+1}/{N_RETRIES_BATCH_JOB}") 455 | continue 456 | 457 | else: 458 | logger.info(f'BatchAPI {deployment["name"]} job {job_id} ended with status {job_status}. Deleting the BatchAPI.') 459 | return job_result 460 | 461 | raise DeploymentFailed(f'Job unexpectedly undeployed or failed with status {job_status}.', DEPLOYMENT_JOB_NOT_DEPLOYED_FAIL_TYPE, 462 | deployment['name'], -1) 463 | 464 | def postpone_api_timeout(self, name: str, timeout_timestamp: datetime): 465 | ultimate_timeout = timeout_timestamp + timedelta(seconds=CORTEX_DELETE_TIMEOUT_SEC) 466 | with self._open_db_cursor() as cursor: 467 | cursor.execute( 468 | "update cortex_api_timeout set ultimate_timeout = %s, modified = transaction_timestamp() where api_name = %s", 469 | [ultimate_timeout, name], 470 | ) 471 | 472 | def raise_on_cluster_down(self): 473 | try: 474 | self.get_all() 475 | 476 | except (ValueError, JSONDecodeError) as e: 477 | raise ValueError(f"Cluster is likely down: {e}.") from e 478 | 479 | def get(self, name, job_id=None) -> "CortexGetResult": 480 | job_id_or_empty = [job_id] if job_id is not None else [] 481 | cmd = [CORTEX_PATH, "get", name] + job_id_or_empty + [f"--env={self.cortex_env}", "-o=json"] 482 | out = _verbose_command_wrapper(cmd, allow_non_0_return_code_on_stdout_sub_strs=[NOT_DEPLOYED]) 483 | try: 484 | json_dict = json.loads(out.strip()) 485 | 486 | except JSONDecodeError as e: 487 | logger.debug(f'Encountered {e} but ignoring.') 488 | json_dict = None 489 | 490 | lines = out.splitlines() 491 | first_line = lines[0] if len(lines) > 0 else '' 492 | if json_dict is not None: 493 | if job_id is not None: 494 | job_status = json_dict['job_status']["status"] 495 | if job_status not in JOB_VALID_STATUSES: 496 | logger.warning(f"Unexpected status: {job_status} of job: {job_id}!") 497 | job_status = JOB_STATUS_UNEXPECTED_STATUS 498 | return CortexGetResult(job_status, None, json_dict) 499 | 500 | first_json_dict = json_dict[0] 501 | if first_json_dict['spec']['kind'] in [KIND_BATCH_API, KIND_TASK_API]: 502 | # BatchAPI has no status 503 | return CortexGetResult(LIVE_STATUS, first_json_dict['endpoint'], first_json_dict) 504 | 505 | else: 506 | # check RealtimeAPI or AsyncAPI 507 | cmd_describe = [CORTEX_PATH, "describe", name, f"--env={self.cortex_env}"] 508 | describe_out = _verbose_command_wrapper(cmd_describe, 509 | allow_non_0_return_code_on_stdout_sub_strs=[NOT_DEPLOYED]) 510 | if 'Failed' in describe_out: 511 | # Failed appears in the output only if there is >0 of Failed replicas 512 | return CortexGetResult(API_FAILED_STATUS, None, describe_out) 513 | else: 514 | return CortexGetResult(status_from_dict(first_json_dict['status']), first_json_dict['endpoint'], first_json_dict) 515 | 516 | elif NOT_DEPLOYED in first_line: 517 | return CortexGetResult(NOT_DEPLOYED_STATUS, None, dict()) 518 | 519 | else: 520 | raise ValueError(f"For api {name} with job_id {job_id} got unsupported Cortex output:\n{out}") 521 | 522 | def get_all(self) -> List["CortexGetAllStatus"]: 523 | input_in_case_endpoint_prompt = b"n\n" 524 | out = _verbose_command_wrapper([CORTEX_PATH, "get", f"--env={self.cortex_env}", "-o=json"], input=input_in_case_endpoint_prompt) 525 | json_dict = json.loads(out.strip()) 526 | return [CortexGetAllStatus(e['metadata']['name'], status_from_dict(e['status']) if e['metadata']['kind'] == KIND_REALTIME_API else LIVE_STATUS, e) for e in json_dict] 527 | 528 | def delete(self, name, force=False, timeout_sec=CORTEX_DELETE_TIMEOUT_SEC, cursor=None) -> "CortexGetResult": 529 | """ 530 | Executes delete and checks that the API was deleted. If not tries force delete. If that fails, raises exception. 531 | """ 532 | 533 | delete_cmd = [CORTEX_PATH, "delete", name, f"--env={self.cortex_env}", "-o=json"] 534 | if force: 535 | delete_cmd.append("-f") 536 | 537 | accept_deletion_if_asked_input = b"y\n" 538 | try: 539 | self._open_cursor_if_none(cursor, self._modify_ultimate_timeout_to_delete_timeout, name) 540 | start_time = time.time() 541 | while True: 542 | get_result = self.get(name) 543 | if get_result.status == NOT_DEPLOYED_STATUS: 544 | return get_result 545 | 546 | else: 547 | _verbose_command_wrapper(delete_cmd, input=accept_deletion_if_asked_input, 548 | allow_non_0_return_code_on_stdout_sub_strs=[NOT_DEPLOYED]) 549 | 550 | if start_time + timeout_sec < time.time(): 551 | if force: 552 | raise ValueError(f'Timeout of force delete {delete_cmd} after {timeout_sec}.') 553 | 554 | else: 555 | logger.error(f'Timeout of delete cmd {delete_cmd} after {timeout_sec} seconds. Will attempt to force delete now.') 556 | return self.delete(name, force=True, timeout_sec=timeout_sec, cursor=cursor) 557 | 558 | logger.info(f"During delete of {name} sleeping until next status check. Current result: {get_result.status}.") 559 | time.sleep(CORTEX_STATUS_CHECK_SLEEP_SEC) 560 | 561 | finally: 562 | self._open_cursor_if_none(cursor, self._del_db_api_row, name) 563 | 564 | def _cortex_logs_print_async(self, name, job_id=None): 565 | def listen_on_logs(cmd_arr): 566 | logger.debug(f"Started log watch process pid: {os.getpid()} ...") 567 | with subprocess.Popen(cmd_arr, stdout=subprocess.PIPE) as logs_sp: 568 | while True: 569 | # returns None while subprocess is running 570 | retcode = logs_sp.poll() 571 | line = logs_sp.stdout.readline().decode(encoding='utf-8') 572 | print_line = remove_non_printable(line.rstrip('\n')) 573 | if len(print_line) > 0: 574 | logger.info(print_line) 575 | if retcode is not None: 576 | break 577 | 578 | if job_id is None: 579 | cmd_arr = [CORTEX_PATH, "logs", "--yes", "--random-pod", name, f"--env={self.cortex_env}"] 580 | 581 | else: 582 | cmd_arr = [CORTEX_PATH, "logs", "--yes", "--random-pod", name, job_id, f"--env={self.cortex_env}"] 583 | 584 | worker = multiprocessing.Process(target=listen_on_logs, args=(cmd_arr,), daemon=False, name=f'api_{name}') 585 | worker.start() 586 | return worker 587 | 588 | @staticmethod 589 | def _modify_ultimate_timeout_to_delete_timeout(cur, name): 590 | """ Will update modified time to prevent GC raise conditions and shorten timeout to the amount needed to delete the api.""" 591 | try: 592 | cur.execute(""" 593 | update cortex_api_timeout 594 | set ultimate_timeout = transaction_timestamp() + %s * interval '1 second', 595 | modified = transaction_timestamp() 596 | where api_name = %s""", 597 | [CORTEX_DELETE_TIMEOUT_SEC, name]) 598 | 599 | except DatabaseError: 600 | logger.warning(f"Ignoring exception during cortex_api_timeout record for {name} modifying ultimate_timeout to delete the api", exc_info=True) 601 | 602 | def _insert_or_update_gc_timeout(self, name: str, gc_timeout_seconds_from_now: float): 603 | with self._open_db_cursor() as cursor: 604 | cursor.execute( 605 | """ 606 | insert into cortex_api_timeout(api_name, ultimate_timeout) values (%s, transaction_timestamp() + %s * interval '1 second') 607 | on conflict (api_name) do update 608 | set ultimate_timeout = transaction_timestamp() + %s * interval '1 second', modified = transaction_timestamp() 609 | """, 610 | [name, gc_timeout_seconds_from_now, gc_timeout_seconds_from_now], 611 | ) 612 | 613 | @staticmethod 614 | def _del_db_api_row(cur, name): 615 | try: 616 | cur.execute("delete from cortex_api_timeout where api_name = %s", [name]) 617 | 618 | except DatabaseError: 619 | logger.warning(f"Ignoring exception during cortex_api_timeout record for {name} deletion", exc_info=True) 620 | 621 | def _open_db_cursor(self): 622 | return open_pg_cursor(self.db_connection_pool) 623 | 624 | def _open_cursor_if_none(self, cursor, fun: Callable, *args): 625 | if cursor is not None: 626 | fun(cursor, *args) 627 | 628 | else: 629 | with self._open_db_cursor() as cursor: 630 | fun(cursor, *args) 631 | 632 | def _collect_garbage(self): 633 | logger.info(f"Starting garbage collection.") 634 | try: 635 | with self._open_db_cursor() as cur: 636 | # remove timed out - deployed and recorded 637 | cur.execute( 638 | "select api_name from cortex_api_timeout where ultimate_timeout < transaction_timestamp() for update" 639 | ) 640 | for api_row in cur.fetchall(): 641 | api_name = api_row.api_name 642 | if self.get(api_name).status != NOT_DEPLOYED_STATUS: 643 | logger.warning(f"Collecting Cortex garbage - timed-out deployed API: {api_name}") 644 | self.delete(api_name, cursor=cur) 645 | 646 | else: 647 | logger.warning(f"Collecting Cortex garbage - timed-out db row: {api_name}") 648 | self._del_db_api_row(cur, api_name) 649 | 650 | cur.execute("commit") 651 | 652 | deployed_api_names = [row.api for row in self.get_all()] 653 | 654 | # Remove recorded but not deployed - fast to avoid conflicts 655 | cur.execute( 656 | "select api_name from cortex_api_timeout where modified + %s * interval '1 second' < transaction_timestamp() and not (api_name = ANY(%s))", 657 | [CORTEX_DELETE_TIMEOUT_SEC, deployed_api_names], 658 | ) 659 | if cur.rowcount > 0: 660 | apis = [r.api_name for r in cur.fetchall()] 661 | logger.warning(f"Collecting Cortex garbage - recorded but not deployed: {apis}") 662 | cur.execute( 663 | "delete from cortex_api_timeout where modified + %s * interval '1 second' < transaction_timestamp() and not (api_name = ANY(%s))", 664 | [CORTEX_DELETE_TIMEOUT_SEC, deployed_api_names], 665 | ) 666 | cur.execute("commit") 667 | 668 | # Remove deployed but not recorded 669 | cur.execute("select api_name from cortex_api_timeout") 670 | recorded_apis_set = set([r.api_name for r in cur.fetchall()]) 671 | for deployed_not_recorded_name in set(deployed_api_names).difference(recorded_apis_set): 672 | logger.warning(f"Collecting Cortex garbage - deployed not recorded: {deployed_not_recorded_name}") 673 | self.delete(deployed_not_recorded_name, cursor=cur) 674 | 675 | except Exception as e: 676 | logger.info(f"Ignoring unexpected exception that occurred during Garbage Collection: {e}", exc_info=e) 677 | 678 | def _init_garbage_api_collector(self, interval_sec): 679 | with self._open_db_cursor() as cur: 680 | cur.execute( 681 | """ 682 | SELECT EXISTS ( 683 | SELECT FROM pg_catalog.pg_class c 684 | JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace 685 | WHERE n.nspname = current_schema() 686 | AND c.relname = 'cortex_api_timeout' 687 | ); 688 | """ 689 | ) 690 | if not cur.fetchone().exists: 691 | cur.execute( 692 | """ 693 | create table cortex_api_timeout ( 694 | api_name varchar(255) primary key, 695 | ultimate_timeout timestamp not null, 696 | modified timestamp default transaction_timestamp() not null, 697 | created timestamp default transaction_timestamp() not null 698 | ) 699 | """ 700 | ) 701 | 702 | self.looping_thread = Thread(target=lambda: self._start_gc_loop(interval_sec), daemon=True) 703 | self.looping_thread.start() 704 | 705 | def _start_gc_loop(self, interval_sec): 706 | while True: 707 | time.sleep(interval_sec) 708 | self._collect_garbage() 709 | 710 | 711 | @contextmanager 712 | def open_pg_cursor(db_connection_pool, key=None): 713 | try: 714 | with db_connection_pool.getconn(key) as conn: 715 | conn.autocommit = True 716 | with conn.cursor() as cur: 717 | yield cur 718 | 719 | finally: 720 | db_connection_pool.putconn(conn, key) 721 | 722 | 723 | def status_from_dict(status_dict): 724 | ready = status_dict['ready'] 725 | requested = status_dict['requested'] 726 | up_to_date = status_dict['up_to_date'] 727 | 728 | if ready == requested and ready == up_to_date and ready > 0: 729 | return LIVE_STATUS 730 | else: 731 | return UPDATING_STATUS 732 | 733 | 734 | class CortexGetAllStatus(NamedTuple): 735 | api: str 736 | status: str 737 | response: dict 738 | 739 | 740 | class CortexGetResult(NamedTuple): 741 | status: str 742 | endpoint: Optional[str] 743 | response: dict 744 | 745 | 746 | def file_to_str(path: str) -> str: 747 | with open(path, "r") as f: 748 | return f.read() 749 | 750 | 751 | @contextlib.contextmanager 752 | def str_to_public_temp_file(string: str, filepath: str, lock: Lock) -> str: 753 | with lock: 754 | with open(filepath, "w+") as f: 755 | f.write(string) 756 | 757 | yield filepath 758 | 759 | os.remove(filepath) 760 | 761 | 762 | def get_cortex_client_instance_with_pool(db_connection_pool: ThreadedConnectionPool, gc_interval_sec=15 * 60, cortex_env="aws"): 763 | global __cortex_client_instance 764 | if __cortex_client_instance is not None: 765 | return __cortex_client_instance 766 | 767 | else: 768 | __cortex_client_instance = CortexClient(db_connection_pool, gc_interval_sec, cortex_env) 769 | return __cortex_client_instance 770 | 771 | 772 | def get_cortex_client_instance(pg_user, pg_password, pg_db, pg_host='127.0.0.1', pg_port='5432', min_conn=0, max_conn=3, gc_interval_sec=15 * 60, cortex_env="aws"): 773 | global __cortex_client_instance 774 | if __cortex_client_instance is not None: 775 | return __cortex_client_instance 776 | 777 | else: 778 | pg_user = os.environ.get("CORTEX_CLIENT_USERNAME", pg_user) 779 | pg_password = os.environ.get("CORTEX_CLIENT_PASSWORD", pg_password) 780 | pg_host = os.environ.get("CORTEX_CLIENT_HOSTNAME", pg_host) 781 | pg_port = os.environ.get("CORTEX_CLIENT_PORT", pg_port) 782 | pg_db = os.environ.get("CORTEX_CLIENT_DATABASE", pg_db) 783 | db_connection_pool = ThreadedConnectionPool(minconn=min_conn, maxconn=max_conn, user=pg_user, password=pg_password, 784 | host=pg_host, port=pg_port, 785 | database=pg_db, cursor_factory=NamedTupleCursor) 786 | 787 | __cortex_client_instance = CortexClient(db_connection_pool, gc_interval_sec, cortex_env) 788 | return __cortex_client_instance 789 | -------------------------------------------------------------------------------- /cortex_serving_client/deployment_failed.py: -------------------------------------------------------------------------------- 1 | COMPUTE_UNAVAILABLE_FAIL_TYPE = 'compute_unavailable' 2 | DEPLOYMENT_TIMEOUT_FAIL_TYPE = 'deployment_timeout' 3 | DEPLOYMENT_ERROR_FAIL_TYPE = 'deployment_error' 4 | DEPLOYMENT_JOB_NOT_DEPLOYED_FAIL_TYPE = 'deployment_job_not_deployed' 5 | 6 | FAILURE_TYPES = (COMPUTE_UNAVAILABLE_FAIL_TYPE, DEPLOYMENT_TIMEOUT_FAIL_TYPE, DEPLOYMENT_ERROR_FAIL_TYPE, DEPLOYMENT_JOB_NOT_DEPLOYED_FAIL_TYPE) 7 | 8 | 9 | class DeploymentFailed(RuntimeError): 10 | 11 | def __init__(self, message: str, failure_type: str, deployment_name: dict, time_to_fail_sec: float): 12 | super().__init__(message) 13 | self.failure_type = failure_type 14 | self.deployment_name = deployment_name 15 | self.time_to_fail_sec = time_to_fail_sec 16 | -------------------------------------------------------------------------------- /cortex_serving_client/hash_utils.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | 3 | 4 | def get_file_hash(filepath) -> str: 5 | """ 6 | Returns SHA256 hash of the file. 7 | """ 8 | with open(filepath, 'rb') as f: 9 | return hashlib.sha256(f.read()).hexdigest() 10 | -------------------------------------------------------------------------------- /cortex_serving_client/printable_chars.py: -------------------------------------------------------------------------------- 1 | import string 2 | import re 3 | 4 | 5 | PRINTABLE_CHARS_REGEX = re.compile(f'[^{re.escape(string.printable)}]') 6 | 7 | 8 | def remove_non_printable(s: str) -> str: 9 | return PRINTABLE_CHARS_REGEX.sub('', s) 10 | -------------------------------------------------------------------------------- /cortex_serving_client/resources/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/glami/cortex-serving-client/aa594b2061534d84fdaf320a3250a97eb0b66d92/cortex_serving_client/resources/__init__.py -------------------------------------------------------------------------------- /cortex_serving_client/resources/main.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import json 3 | import logging 4 | import os 5 | import re 6 | 7 | from fastapi import FastAPI 8 | from starlette import status 9 | from starlette.requests import Request 10 | from starlette.responses import PlainTextResponse, JSONResponse 11 | 12 | 13 | JOB_COMPLETE_PAYLOAD = '"job_complete"' 14 | ON_JOB_COMPLETE_PATH = "/on-job-complete" 15 | 16 | 17 | def get_class(module_path: str, class_name: str): 18 | """ 19 | @param module_path: cls.__module__ = 'predictors.my_predictor' 20 | @param class_name: cls.__name__ = 'MyPythonPredictor' 21 | @return: The imported class ready to be instantiated. 22 | """ 23 | module = importlib.import_module(module_path) 24 | return getattr(module, class_name) 25 | 26 | 27 | app = FastAPI() 28 | app.ready = False 29 | app.predictor = None 30 | logger = logging.getLogger(__name__) 31 | 32 | 33 | @app.on_event("startup") 34 | def startup(): 35 | with open("predictor_config.json", "r") as f: 36 | config = json.load(f) 37 | 38 | predictor_file_path = os.environ['CSC_PREDICTOR_PATH'] 39 | predictor_cls_name = os.environ['CSC_PREDICTOR_CLASS_NAME'] 40 | predictor_cls = get_class(predictor_file_path, predictor_cls_name) 41 | app.predictor = predictor_cls(config) 42 | app.ready = True 43 | 44 | 45 | @app.get("/healthz") 46 | def healthz(): 47 | if app.ready: 48 | return PlainTextResponse("ok") 49 | return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) 50 | 51 | 52 | @app.post("/") 53 | def handle_post_or_batch(request: Request): 54 | response = app.predictor.predict(request.state.payload) 55 | if response is not None: 56 | return response 57 | 58 | 59 | @app.post(ON_JOB_COMPLETE_PATH) 60 | def on_job_complete(): 61 | app.predictor.on_job_complete() 62 | 63 | 64 | @app.middleware("http") 65 | async def parse_payload(request: Request, call_next): 66 | content_type = request.headers.get("content-type", "").lower() 67 | 68 | if request.url.path.endswith(ON_JOB_COMPLETE_PATH) or content_type.startswith("text/plain"): 69 | try: 70 | body = await get_text_body(content_type, request) 71 | request.state.payload = body 72 | 73 | except Exception as e: 74 | log_exception(content_type, request) 75 | return PlainTextResponse(content=str(e), status_code=400) 76 | 77 | elif content_type.startswith("multipart/form") or content_type.startswith( 78 | "application/x-www-form-urlencoded" 79 | ): 80 | try: 81 | request.state.payload = await request.form() 82 | 83 | except Exception as e: 84 | log_exception(content_type, request) 85 | return PlainTextResponse(content=str(e), status_code=400) 86 | 87 | elif content_type.startswith("application/json"): 88 | try: 89 | request.state.payload = await request.json() 90 | 91 | except json.JSONDecodeError as e: 92 | log_exception(content_type, request) 93 | return JSONResponse(content={"error": str(e)}, status_code=400) 94 | 95 | except Exception as e: 96 | log_exception(content_type, request) 97 | return JSONResponse(content={"error": str(e)}, status_code=400) 98 | 99 | else: 100 | request.state.payload = await request.body() 101 | 102 | response = await call_next(request) 103 | return response 104 | 105 | 106 | async def get_text_body(content_type, request: Request) -> str: 107 | charset = "utf-8" 108 | matches = re.findall(r"charset=(\S+)", content_type) 109 | if len(matches) > 0: 110 | charset = matches[-1].rstrip(";") 111 | 112 | body = (await request.body()).decode(charset) 113 | return body 114 | 115 | 116 | def log_exception(content_type, request): 117 | logger.exception(f"Couldn't parse {request.method} request of content type {content_type} to {request.url}") 118 | -------------------------------------------------------------------------------- /cortex_serving_client/retry_utils.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | from typing import Callable, Tuple 3 | 4 | import logging 5 | 6 | import inspect 7 | 8 | import requests 9 | from requests.adapters import HTTPAdapter, Retry 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | def retry_on_exception( 15 | fun: Callable, catched_exceptions: Tuple = (Exception,), max_retries: int = 3, starting_sleep_secs: float = 3.0 16 | ): 17 | ex = None 18 | fun_logger = None 19 | for retry in range(max_retries): 20 | try: 21 | return fun() 22 | 23 | except Exception as e: 24 | if isinstance(e, catched_exceptions): 25 | if not fun_logger: 26 | fun_logger = logging.getLogger(inspect.getmodule(fun).__name__) 27 | 28 | ex = e 29 | sleep_secs = starting_sleep_secs * 2 ** retry 30 | fun_logger.info(f"Retrying {retry+1} time with sleep {sleep_secs} secs.") 31 | sleep(sleep_secs) 32 | 33 | raise RuntimeError(f"Too many retries ({max_retries}). Last exception: {ex}.") from ex 34 | 35 | 36 | def create_always_retry_session(backoff_sec=3): 37 | s = requests.Session() 38 | http_adapter = HTTPAdapter(max_retries=Retry(total=5, backoff_factor=backoff_sec, method_whitelist=False)) 39 | s.mount('http://', http_adapter) 40 | s.mount('https://', http_adapter) 41 | return s 42 | -------------------------------------------------------------------------------- /cortex_serving_client/s3.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from pathlib import Path 4 | 5 | import boto3 6 | 7 | # insert Cortex Serving Client bucket name 8 | BUCKET_NAME = os.environ["CSC_BUCKET_NAME"] 9 | BUCKET_SSE_KEY = os.environ['CSC_S3_SSE_KEY'] 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | def _get_extra_args(sse_key): 15 | return {"SSECustomerAlgorithm": "AES256", "SSECustomerKey": sse_key} 16 | 17 | 18 | def upload_file(local_filepath, s3_path, bucket_name=BUCKET_NAME, sse_key=BUCKET_SSE_KEY, verbose=True): 19 | local_filepath = str(local_filepath) 20 | s3_path = str(s3_path) 21 | 22 | if verbose: 23 | logger.info(f"Uploading local file {local_filepath} -> {bucket_name}:{s3_path}") 24 | 25 | session = boto3.session.Session() 26 | session.client("s3").upload_file( 27 | local_filepath, bucket_name, s3_path, ExtraArgs=_get_extra_args(sse_key) 28 | ) 29 | 30 | 31 | def upload_fileobj(fp, s3_path, bucket_name=BUCKET_NAME, sse_key=BUCKET_SSE_KEY, verbose=True): 32 | s3_path = str(s3_path) 33 | 34 | if verbose: 35 | logger.info(f"Uploading local file object -> {bucket_name}:{s3_path}") 36 | 37 | session = boto3.session.Session() 38 | session.client("s3").upload_fileobj(fp, bucket_name, s3_path, ExtraArgs=_get_extra_args(sse_key)) 39 | 40 | 41 | def download_fileobj(s3_path, fp, bucket_name=BUCKET_NAME, sse_key=BUCKET_SSE_KEY, verbose=True): 42 | s3_path = str(s3_path) 43 | 44 | if verbose: 45 | logger.info(f"Downloading file object from {bucket_name}:{s3_path}") 46 | 47 | session = boto3.session.Session() 48 | session.client("s3").download_fileobj(bucket_name, s3_path, fp, ExtraArgs=_get_extra_args(sse_key)) 49 | 50 | 51 | def download_file(s3_path, local_filepath, bucket_name=BUCKET_NAME, sse_key=BUCKET_SSE_KEY, overwrite=True): 52 | local_filepath = str(local_filepath) 53 | s3_path = str(s3_path) 54 | 55 | if Path(local_filepath).exists() and not overwrite: 56 | logger.info(f"{local_filepath} exists and overwrite=False, skipping") 57 | else: 58 | Path(local_filepath).parent.mkdir(parents=True, exist_ok=True) 59 | logger.info(f"Downloading file from {bucket_name}:{s3_path} -> {local_filepath}") 60 | 61 | session = boto3.session.Session() 62 | session.client("s3").download_file( 63 | bucket_name, s3_path, local_filepath, ExtraArgs=_get_extra_args(sse_key) 64 | ) 65 | -------------------------------------------------------------------------------- /cortex_serving_client/shell_utils.py: -------------------------------------------------------------------------------- 1 | import psutil 2 | 3 | 4 | def kill_process_with_children(parent_pid): 5 | parent = psutil.Process(parent_pid) 6 | for child in parent.children(recursive=True): 7 | child.kill() 8 | parent.kill() -------------------------------------------------------------------------------- /cortex_serving_client/zip_utils.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import zipfile 3 | from pathlib import Path 4 | 5 | 6 | def zip_dir(dir_path, archive_path: str): 7 | """ 8 | Zips contents of a directory without the root directory. 9 | Example usage: 10 | zip_dir('dummy_dir', 'code.zip') 11 | will archive contents of directory 'dummy_dir' into a file 'code.zip' in CWD 12 | :param dir_path: Path to directory to compress. 13 | :param archive_path: Path to the zip file. 14 | """ 15 | archive_path = str(archive_path) 16 | if archive_path.endswith('.zip'): 17 | archive_path = archive_path[:-4] # remove .zip 18 | else: 19 | raise RuntimeError(f"archive_path {archive_path} must end with '.zip'!") 20 | 21 | shutil.make_archive(archive_path, 'zip', root_dir=dir_path, base_dir='.') 22 | 23 | 24 | def add_file_to_zip(archive_path, file_path): 25 | with zipfile.ZipFile(archive_path, "a") as zf: 26 | zf.write(file_path, Path(file_path).name) 27 | -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | ### Example README 2 | 3 | ### Simple test 4 | 1. create instance based on Ubuntu 18 in AWS in e.g. `eu-west-1` 5 | 2. `pip install -r requirements.txt` 6 | 3. `pip install -r requirements-dev.txt` 7 | 4. install Docker, if you don't have it already: 8 | ```shell 9 | sudo apt -y install apt-transport-https ca-certificates curl gnupg-agent software-properties-common 10 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 11 | sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" 12 | sudo apt update 13 | sudo apt -y install docker-ce docker-ce-cli containerd.io 14 | sudo groupadd docker; sudo gpasswd -a $USER docker 15 | ``` 16 | 4. `cortex cluster up example/cluster.yaml` 17 | 5. install local postgresql DB: 18 | ```bash 19 | sudo su postgres; 20 | psql postgres postgres; 21 | create database cortex_test; 22 | create role cortex_test login password 'cortex_test'; 23 | grant all privileges on database cortex_test to cortex_test; 24 | ``` 25 | 26 | You may need to configure also 27 | ```bash 28 | 29 | vi /etc/postgresql/11/main/pg_hba.conf 30 | # change a matching line into following to allow localhost network access 31 | # host all all 127.0.0.1/32 trust 32 | 33 | sudo systemctl restart postgresql; 34 | ``` 35 | 5. Optionally create a CSC S3 bucket or use an existing one 36 | 6. Optionally build and push `base_docker_image` to AWS ECR to get its ID or use you own 37 | 7. set env vars according to CSC bucket info: 38 | ```shell 39 | CSC_BUCKET_NAME= 40 | CSC_S3_SSE_KEY= 41 | DEFAULT_BASE_DOCKER_IMAGE= 42 | ``` 43 | 7. run `integration_tests.py` 44 | 45 | 46 | ### Local base image test 47 | Edit the `download_source_code.py` script to load source code from disk and not S3 48 | because the locally run docker image will not have AWS credentials. 49 | 50 | Build docker image: 51 | ``` 52 | docker build . -t predictor 53 | ``` 54 | 55 | Run docker image: 56 | ``` 57 | docker run -p 8080:8080 predictor 58 | ``` 59 | 60 | Make request to the running container: 61 | ``` 62 | curl -X POST -H "Content-Type: application/json" -d '{"msg": "hello world"}' localhost:8080 63 | ``` 64 | 65 | Kill all docker containers (or just kill this one): 66 | ``` 67 | docker stop `docker ps -q` 68 | ``` 69 | -------------------------------------------------------------------------------- /example/cluster.yaml: -------------------------------------------------------------------------------- 1 | # This is example cluster config 2 | 3 | # EKS cluster name for cortex (default: cortex) 4 | cluster_name: cortex-serving-client-test 5 | 6 | # AWS region 7 | region: eu-west-1 8 | 9 | # instance type 10 | node_groups: 11 | - name: ng-spot-1 12 | instance_type: t2.small 13 | 14 | # instance volume size (GB) (default: 50) 15 | instance_volume_size: 20 16 | 17 | min_instances: 0 18 | max_instances: 5 19 | 20 | spot: true 21 | 22 | # second node group instead of on_demand_backup 23 | - name: ng-demand-1 24 | instance_type: t2.small 25 | 26 | # instance volume size (GB) (default: 50) 27 | instance_volume_size: 20 28 | 29 | min_instances: 0 30 | max_instances: 5 31 | 32 | spot: false 33 | 34 | 35 | subnet_visibility: public 36 | nat_gateway: none 37 | 38 | # whether the API load balancer should be internet-facing or internal (default: "internet-facing") 39 | # note: if using "internal", you must configure VPC Peering or an API Gateway VPC Link to connect to your APIs (see www.cortex.dev/guides/vpc-peering or www.cortex.dev/guides/api-gateway) 40 | api_load_balancer_scheme: internet-facing # must be "internet-facing" or "internal" 41 | 42 | # whether the operator load balancer should be internet-facing or internal (default: "internet-facing") 43 | # note: if using "internal", you must configure VPC Peering to connect your CLI to your cluster operator (see www.cortex.dev/guides/vpc-peering) 44 | operator_load_balancer_scheme: internet-facing # must be "internet-facing" or "internal" 45 | 46 | tags: 47 | Application: cortex-serving-client-test 48 | Environment: test 49 | Department: ai 50 | -------------------------------------------------------------------------------- /example/dummy_dir/dummy_predictor.py: -------------------------------------------------------------------------------- 1 | class PythonPredictor: 2 | def __init__(self, config): 3 | self.config = config 4 | print(f'Initialized the model with config: {config}') 5 | 6 | def predict(self, payload): 7 | print('Predicting!') 8 | return f"Predictor config: {self.config} Received payload type: {type(payload)}, payload: {payload}" 9 | 10 | -------------------------------------------------------------------------------- /example/dummy_dir/requirements.txt: -------------------------------------------------------------------------------- 1 | uvicorn[standard]==0.15.0 2 | fastapi==0.68.0 -------------------------------------------------------------------------------- /example/example.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | logging.basicConfig( 4 | format="%(asctime)s : %(levelname)s : %(threadName)-10s : %(name)s : %(message)s", level=logging.INFO, 5 | ) 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | from requests import post 10 | from cortex_serving_client.cortex_client import get_cortex_client_instance 11 | 12 | 13 | # Instantiate Cortex Client 14 | cortex = get_cortex_client_instance( 15 | cortex_env='cortex-serving-client-test', 16 | pg_user='cortex_test', 17 | pg_password='cortex_test', 18 | pg_db='cortex_test', 19 | ) 20 | 21 | # Deployment config 22 | deployment = { 23 | "name": "dummy-a", 24 | "project_name": "test", 25 | "predictor_path": "dummy_predictor", 26 | "kind": "RealtimeAPI", 27 | "pod": { 28 | "containers": [ 29 | { 30 | "config": {"geo": "cz", "model_name": "CoolModel", "version": "000000-000000"}, 31 | "env": { 32 | "SECRET_ENV_VAR": "secret", 33 | }, 34 | "compute": {"cpu": '200m', "mem": f"{0.1}Gi"}, 35 | } 36 | ], 37 | }, 38 | "autoscaling": { 39 | }, 40 | } 41 | 42 | # Deploy 43 | with cortex.deploy_temporarily( 44 | deployment, 45 | deploy_dir="dummy_dir", 46 | api_timeout_sec=30 * 60, 47 | verbose=True, 48 | ) as get_result: 49 | # Predict 50 | response = post(get_result.endpoint, json={"question": "Do you love ML?"}).json() 51 | 52 | # Print the response 53 | logger.info(f"API Response: {response}") 54 | # assert result['yes'] 55 | -------------------------------------------------------------------------------- /model-manager-architecture: -------------------------------------------------------------------------------- 1 | 7LzHluPIsiX6NbVWv8HJBS2GECRAEARAaGLSC1oSWhFf3+7MSJ1Vde49dV9POiIjgwBdwE1s22bujD9w4blLY9gXty5Jmz8wJNn/wMU/MAwlMOwP+A9JXp/v0AT1+UY+lslHo283rPJIP24iH3eXMkmnHxrOXdfMZf/jzbhr2zSef7gXjmO3/dgs65ofZ+3DPP3lhhWHza93vTKZi4+7KMV+e0NOy7z4mJrB6M9vRGFc52O3tB/z/YHh2fvr89vP8MtYHwudijDptu9u4ac/cGHsuvnzq+cupA2U7Rexfe53/pN3vz73mLbzv9PhQ09r2CzplyemGtCVj8CLHL4wxi5ZgIwxRAjnNO9GoK3xa6vxS7Mvd8Bc0bd77zXOry9yzcqmEbqmG9+X+JmC36DhNI9dnX55p+3aFN78IhsEXORjmJRgUT+1ybp2/jAflADXYVPmLbho0gw+wNSHcdnm8F3ko/V309Mo/Ab32RP8Bi/WdJxLYAbcxzBz138bxYYX4r+IL9N+LAv51kJ9zyoS8FYxPxs4L3gJrHQOyzYdP1r/qqQPvcHp0/27Wx9Kk9Lumc7jCzT5eBenPwzow8FQ/ON6+2auBPJxr/jOUnHm42b44SL517G/mQl48WEpv7ca9her+QIBt7AFrgGMAuHGuChn4JzLmP6dHRAih57Jv7UD9D+1gw/9kL+xhQ8v/UFv/4CeUORHPVG/qomif6Mm4h/QEkpSv+hpHoEh/oEJEJvTvulen1+PXdNA6PrVrf+XbNvG//eLAtME4OXHZTfORZd3bdicvt3l3yCYJh8m/51UmzBKG/4rTv6s7Dkc53PZfGn9vv7QLFRs2iYcBHhwGTVdXNsFWM/79nedwNV3Xap0nl8f1+Eyd+DWt0dWu7dXo78YHzRLjCHO+Nd3vkQC7Cd7w361JuT99Z0dxsCEoP/zSTgVX6WS7uXsf3lm8PoB738iP67E/btm4uvLRQvMwP/+4rte8PJbt/fVl35/asxTt4zxl7iHMR/RNhzz9KOdFl4elwo7sbu4Cw/uypOW9y+U+oh40BL+HU9APiE4TfzgDGCQz9dj2oRzuf4YfH9n+h+jG13Zzt+GxtgfnQxHiE8s++Mon1f50fH7gPjTWF/x8etY5I8DfZbMXwz0pWGXZVP6Q5u3w36VzL/lw4zeIvexjrgXR+k9Idz+9xoCsf0ato0xTUoYpX/rwKCtoVv2f+7Hv3Wc3zjYd/7+xci/Gfbju3f+WSN/XxnpWAIR/324/e9aPvlfs/x/IZ8w+gvT+jArgvlHLJ8gf4wuxI8D/CMm+FUyf0USnekbJ/xLBvjNnKAFhNHUNcucArLwBU3h3a9X2I+m9CfB/wei+APA/g1Mf+V9P9GFv6SBX8jkb1jhT+z2fGZO/0M84msm8h2RQMnfEAkUZf/cqP5dJvEnfvA7u/hJ3b/E1OT99auwvvKvH9jej4EX/QGO/l37efMNo5vKuex+CMVftKz+1OBZJskb+n4J3j/bRdTNc/f8hV/+hl6y76/fWOQGSbIFusM7G8imv43Gf4z+sxf8A+ZE/GROGPIrL/0dLaX/AVr6J8b0a07xizH94NofqX8YfU3CvhPRT8ZFk/D7L/KLHo7x3HNYzfiUxz32aZq78V0k4IGg5z+V/N94xr+tEBjaflbA79z5S17wH8aNf2E/Bg70J5LzjwSOP5EN8St36aY5H9MJ3BV/jRjfqfXPPParH/7isX+SbHwPOH8C9r9Q/l+gDEEY5Iz8Zfz52bTKbqI/lcB0p09JOIf/jDNTxE+xgUY/sdQv5sQCBs7g3339Jlgg7CeKRAG/pFmEYukvdvw/4O7kL1bAedYvuu+WuSlbIPQvZb5fJPy3JvF79f5iKN9Z2c9KC7cJ/7R1Y/1u+BO0iCxHE3/BTf4RsP4xbcLJX7ECRX8b++lPX2D0f0CF9N/Twl/KQv+tquH/44z/vRohQ376oqSvtJH+HTSgJPHpt3iAYZ8Q4j83IJRCf1Hq/6sg/RMVpD81k+9z27/2338j4f0wgb9NeP9DVkIhzCcWpb59/chRWfwTwEL66z/qx/H/3QrPX8+CY/gnCv3u68dJ/qT684+xIxr/lR2NXZxO09t7wvnXWjrUyjv6/egkf5u8fE1zAPMqj+94dA9X914vyf9Bir+3MurP4OjrptzHiH98v7H1e+qLMiT9z9Ba9hP7gzb/BQb/aZT/QWpL/1pv/19CB+Xz/wro/xfh78X2VYUx2jn71zN80LvuePW3Guo/X+9DPuEYgf+IKug/Yt8kgX36nsH/mMRhJAsYPgt+ff6H/zfxEQVBn6YRFCVYAiOxHz0Kp9lPBA489su/fwsf/4eq49JURFIoKi3Hnkl+M5j/bVL/+gZNf1WX+uD4y7Ph4rn7q3LQ32QS3znmb7OVf4DIfd3c/YtqDf4b/k/9A9WaP3GeX5P4X0QM0an/LwaJ3wjqz+kt+6Nlfh36e2KL/f8rFuzvxfIjrP+u/Pfndawsy7A4/h3SJlREkdRXc/uNwL8X7F8p9Vd5/1+U53/Bk8vn53Ld37rxf7skFE79Z7fOyh0qkH9PyX25i3y5A15/Lu5wny+xc9Wn+Xuj/cxW4D+C4+5WHShmzvHcnQNX4DcHnuCMbbzIcTeByy/g5/1b5LabyHOPE2gKfnKJzx1JuOdXIa8vwr2+ihxzE/JJF7ZaEc2TJuYMaHi6C/lmnfbL/bTB66+j329gxDsYMYej3UEv9WPo/+AHjr5xnClw5wvn5SJnXTiuAO/tJ5jaM3dOfq/xdH73+OErBo8BVsTVJw6aC3Y+cdIdSOUEnvMEGt9M4d5dxLsD2p44IJUTb54k/nXa7/fccryTHT9Oztl8+JdKEZzG9etAL5BEtYkirxxMedW9tW9mXd+ORHRbz+2ji6JUrpNcrghhOa1Wu1Xg62I32AJjnBDzFJ66L8/Cb1BHUBs/PTcn8PlJBrI8C8a75elugicUeO4k81t9FrjbZ+1+e+qCLy773T0XiHlz44dkdtFJPvOP8tYodr1y/r2xecK61ZqkvLpZLW+Muse0CnWXczcgDyhV8XSHkuIMVzuZf/HtOHT0B8bXBlzLY76D1wG1gtfFFaoLPI+PzGORcrTMGU2yMtKhJ6CRYOcEv1l6yKu78rxcnIuv+KJcbWQoPSYH45z+9drv5vp81pEsS8iSsvsm5bFlpZxjUjTlPvfsPrYIsvSZ8ZyDWmpm0XTkCynZl0oCs89Ju/jiE8EjclaxET0diYlTloQtqW9ir1op+6eoE+cOXy+0XzcmklrLqa07hNOfOEHzu2s+mcE/NdMASGkqUBjqAxw6j8fVN8MLcjaRTSInFhlk+wQCxTkH6Mb3FcMMtxFF9nl18RcmupoWXXDGc9y1kDc9RY+CFDrH0ByST2mhYwpuiwTZrwjcL/ZhN8yKQhzWk4sgwn06YSd2p/BnG1bDg2NMMe4fJiMK5xH1x9abY3pEsUnFHd04vXDBJY1t8U4KQaP3NtYounqcrjKt3vVO7Zdq98EjIkQ7Av3wz5TC9V5hEF8amoNOg2NFm6w6e65rrPkQPOCiJrlZKYCjvGmcnWORWHs6C6RnF6g9hE89zBztKHfaGmtoCd2oY3RkzPHMcTOG4A6jeoyUphP9zJ7NLr9UewKJGH+dDnwsM6FYtxiRB//yUk9Bie6y11dTa5EQ1j39fgqJxN/F6uw79W1LoPATB7gCz2/OkwqhjeXP6RowkeaHp1nbKn5jWd5QrHXHiIy4n1iMkBdfIYmh0MnkqlrBVaOdSwLFm1BnN8BE+nZnU6NK0mqp9GcxWahH6e2ioaOmozaLO0tilAZ0wHBvDk5n97BfO7WK6NEycQEk8IbnTBuFxKxxxSyNoO9kf7kycZlQ2HgHY/jYIRHT4pWshSbcndHrfmOWnlwo6wJNJmnnvZ0ufnGk9knbVlWfkmKXI7g+YcR6/wrMDXt6uIQ85AlJBTZEfMPbqj6jbKyz72GIRme+p51oRx8bZbMUWi0vjV69BeJGDCV3aRJh6I/beG5kppWix4BZyCE8sbjnta28lBRFIn/l7H/yfZZai8OAPvb17pqZx1oe7RaaGnayeMl0k2I3dtjSwYgWWUtDjVpdFJWTcAArx7H1OcIXc31DTcfWqYOU29ICHITTNv/u2wOqOb4djiYVMwpoyI5KdGFux/Z8KimLNxFxSc0YJzduBpj8UhXPOXn4aOwKHuZzTN7tkRyiG4SjYdpkezRIBRvZUC/GsXFuvZqYB+rQ0Nz2+97QeHoR07nBPFqn99lB12g+Y8FW1aPcIAhdNLMhIEF2SXGOEJ8Wf/jAYCnMDxXM89ymRpKgA+DEE2GXFWBJCzsryh6ndMy/OImMZ2909KcXRHqaeJWYLnrjhIozrutzktZs8Zs+SW5hM6m8kmmGr+NTtoTU0CuXPDv5XOCpCqFipzG8lourqndcqothCJPVaYpnJWJGhOlhssuVL2QWRenJgMWSGPKdVSc2ps8tw/vOXfecmJ6aEeA2/6iSvOLZkODEmOBXu18cR0/KUjxFz0WvJOJKJ9YrzZT7IRXL8azs4lGVSzAxbugoxPAEwovH08AUROPVnTVEjMpfX1x7Ys7mjXCFmDedxr/OdA0gRCfeb5bzsbhWQs3K1tXDXRhb/nUj6QaXEX2pR5fqmPPUo3DoQo27y+SlyzNjmeGpSdD/SWPspTwI6ZLgcvLySiWAbZins75qN0SX3L2yT2wUXYYLNbRNmNCKpIvPlrivxnCqX1n+hEHCHvQN6r6BUene+q/bw5RC7YEV6f5S/bnFRsqAmFmqyqgXXHROqKveU4BrnWu8TscAGtYiPiFARsIc6qGnq354n+s5lmfxSQnAs8r+Uv7XvtXjSQWB3EMfgfGUO4XlO6K2j844NgdPbgd4z4Cp85nSzIzW93uBqdH5ZtANJES9nM8Ax0Ej7XNPk3FRHuGIQcOQzHGmPAz7ptnPdbRtx85mciOecWe94o91Nsak9hAJp159J7unU+bBPY6eu7E3MJbsh0LKg8yCBmJq9Zp9hA8UDS6aGZqrq8V05oiq0lnJzg5kv5FVcQDjqp5nO+7J8PIKybhCfQ5w8MHdXsHzEZF+DRgIPyGtGCfhcS2r3tkdZ9lkF9J6RqOqa6jQAhErvCGjt+Ttr8y1CY7nFPprLM3hra4JTiBaCqrE4qe0Hyw18LsnubusdtYVenc3mqgu+07Vov/YaF72Y3Ix2yE/1TrHvGoLsYInFuQbzysY9xgRar4BSuJkbWsJnNbrAqm8NGCzrmWv7mKlIZZac8BdC6JftoKgEWDNgxUIs84KCdUlmye4Na+THtM4Nen2imO5F14PZ0U6v28Q12tdXF5DsJ2S3V2Y4uHdz9SdBRrkSRvFtiuOMHvqIedtvIbGmKK3ttDw5Amo1vkKzcM38DsMhOW5NygAbzSAzSvaOu/YcSsxlhgDV4OU7AKllgjAuLpx6f7LBllRQQEzCxiESfBfh0L2OtFngTd4O0MOu4rYzGA4c4BskpOSMmfafB4R6GGLAxUMEkrUd8AwcH17RlUAPdLrIT3ZMJyRyHkCCxhCEiuhDkfcccEyeHVkjxzJdbnyJAA3hYKrsx7K5VHE6gAgvlgQ7IqrjodF4TwsVRLNd/o+0p5T7YCx0cNOP/BKuukhGITprshUX3vj7LU6aQZiIJy7B0BmKztlZw2vaxo5dvd0FMI0CgDMAvwCdC1Mp/EiBDjh9jrpKpsb2oWpOPnzPsZ047Qp+DUN17o1Xlek5QCyQQV6NSlsU64cQrpfxzYrHLYEaGexcpofUEa8gKzs1W1ZQu2UGIY3vgX/X6waerA1gf9XPuFVb7ZhjndZWdN7XSEvECDXzoFmlnKhfq+zbsX6KEuukS4DQnZh2BHhcyRWN2lrJXobIXRA91I4dSHEtLw6J//07PW6AHGPhxHzMFLMK1vFcIvukh8wJVB1n2TW2ps0bIhsyQJPcZYQ0fG9o+UXZ48YPilOfncqiQ0ZkTbp57x5YJRbn5n+nrkmGj8ipVrqBxez9DITDdJOB/YKTextszkd3EKL3Ge1HzNHlwsvj33N4OmUifaDM8/re3HXqtUeyJoSWogEirJxOhpq6bykO20jI6GOswKpuqxKsVFWo2cbYbrNlXIx4vtiyUJp2qGEtN0Z3ffLpJ/UXeYSZZop0+Q1ZAarqlD+8iC1rhLMI4iuO+T250mBGbZBj3hHfzZi6AOBlrDWsh+JnvEJ04PZGeKV7R5466Zfn9pZbHEfNzGWgXD8CmmqMZZ1qWaayNgaxdFrjNvt5mfXVIi/64mub4bLn6QcmylmZGPXA8rG1vFBPHYbDDjljwuBTWwc8K3hHpssy+kMITpmR42m4Ib32bWN0b3iOIhiZxP94Rk2vRq/PgNryTd0ELY7pqUgr6RaSj0u9Ww+xuOUkr2iHdQbU4ayYg90wtcOXVn3Bsmba+CGdaRQavDRUeegUGtnMRFJjOwEUaJcIM91wZRp1dNEy9YumLIlaV+D5Yr6vU5XBMgfwREg2Z66Fi6EocFCtC8LcX695aoy/jBMhGVsuLLr55UtVft1mjR1jwdoTRqswxzdE12h89FXwCfPuKH/9OCfp4jAFC6cgjt/WY2MAA89f6xnheux6cnoSNj9M+KZUMbOKLczIGHgrv3u72U3SpkjBkbq4fOc0nvO5uucMZFc5OBs0Wct3kOtV0k18YzuMiDH6y5cyJQkAG3mDXFIVi2BT5+Uiazie0ZfRshA3H1EYPGpYvsF1WUnLeiVJD1yHnrqcaQB+cjWyn+dj3S8ouMiFutepYDq3Xanq/fncU+emN7MU7qpRzirCCarJDJ4JLp3SSbO1yJ6gmSncI8VxqzjOl9XzSPxS3qd8VoVXtwAl3cZZEl7jQdTzuRVudZDW45xJQx2PeA6+VDesS3hq99BXHt06rvEc0+gjKwGhoBUeFYO1md3UrZXEAdDQ2y2nbCHx14tRjg7pOkKfh8RDK4txyOyy+iELVHICYmCqUEj5ytIE85CUZIXOZ+wEnFoqJRNHIw3vgEGiCAgZg3ghznYwhnpOwMJ7W6wNWx5TulDMTKR3neP7eYTq8FC1Ls3dwMdssAEua99I27j1Urbg748UiTjoXmYMSKqh4I9sufteRdX8VnNMcKbmdmT+fLIUpE4RykgZ3xYLJoYd6vonF1ZP56eflR7VRrX+YY9o0C70Lcw25LjqQHCIhWItvawlJunYivn+5omJMkN6L2bdH0zL3ZO9shJIgxv1WNk0kPCu+L7eS+GsddpGxr1lZ3H8aLKUAl8X/UXkDPK2YU5kTTG+rtqy5Gr9RWZCW3ID+oYHLl/fQAWwXP6tpjmG99rBEdqbXi2pJeGKvaiEeJCIVp2kkmZfMCgbzRMSakgl4JRyMRvc7wxVISYuRMfm3fIkIFvIMOGfqBcxIWo3lEIJJWmiuTsaLNt2mLe8LKLmV0OX+AxiDcBYS6JCgHZhqVPha1y5nHYFnoaqbJrMlW9EGvK0iV5tCdSUYxTzaBLimWus4xs79JL1AuwCiV1iYlspCBdWHMxpLufajvIYEBQXd+k2YR153NPIc6zSwJsuRL+0j+U9cZam1F2W0JEqLio+/KyWx4Xpdsz6KMkIin/EI27wzQ3E824tQy9B60PJoiLT2od5gRZtHrRjGS0PQuZA+slWCDpwORbrAOc4kHaVWsrtkkvJGkAwwnlTkIi6iaFS/cwSwW5s4ki4aDhrprLXudclQHkVGGu/8DnW8/2WP2g24tk4np96elCQB5iEEftLM++mOobKSvhOu+U7VVzNYrh41myYVWH1gvfrFzL8ai1V5y9ZyWNyxOHj81qSYjspvmDHmgr2QITPaPdYUkBkvnMKZ0ExZuPapxlTwRvUzt+NnuHUp9j+PSmUSU1w7gxL8t6TWq7xna9xMTysuwDhJ3nnTwfAD74NHuI7oNgH1K1+g16VQwtelq2WlN0JQ4Kwr5gOhTaM4E9aYfszSEChhTydOl5WKMUvSfWemf4BO03O43KhfKad5YTjQRiOsTt0DXu3KFnhovvBtYklyTuHX4JhOQRPdt8SM5d41TLQVRRmxKx22PJq7PmF6uy7TKAgEJbPVCukXzubyBFj+86ifNxBiugTlawdHpFcMAuSY9KG+rA3YSx2d5h1b1dpvcIO3VHwIuYzvu0YiGPk/GZGK0y1VlHodEFzDqxaMWYoJ9+9JB1k3gCLx/s5Zox3Ilu43PtCN1nXGqShXU8Om1eB+4bu4zVrNq0ywKSHdqqYLSASFYmxyIcB+8YMexUn4ZnPCbHRFQsigPyC2Rj4V9e25CGwvh7oqvt7kgfM9WBe6OXK14hM8wDkgIHszU2Mj++XckfXWESist5H2/wCZaEIEazTFnnWDyo7KaX6QkGSVgBQ9o+2MFvWlxLY74aQLCwJBOnQnNAMsCTeFXOJ3IodDqdaVbVDTxIPgQk3d6IbqgrktEWjNGWvLI7oYCuDPd54huNRi/vM2VrGjAz9BUTVomvbf+47dVBQcYOuQhI4LNAxMEVwZzVp56yxyqfQwWsHXD/iKa8U7YgNCt/fm7//dz2e+4KwH/6edkTsx/4e+77l1yAnl/4m+dcKQMPv0zHSUB+44N5QKHgbzPCmqp9mZCeLwYfkDVwGCB8P87ukF+UIkj0HJjoOc/IhrGUpURmsVnbfZGTgRFHdJ75HSRICMDkuRyCoT0sarzxRiGI1cCauCjLV8x3cKseNHPqMR1xZUieOLWqfW45WhlKp5q8floVXfOoRLOY8kYerxvygAx45ChavGshdr7Lgpnwt0zUJH11I6fnrl1XsMYAq8GRaLyTv/udTgO4WHDvBrVlHSyfwRiKJSTUQvFmliy0Gmu9jPFX7YPkwO2NtKtYo8w+RoSU4e4aNg7j9HM7WNVAzhWtM72xvS0hhQanWOe4NjJgvjC6UCuKG52EPiloBF0ix4pNXJtQ3JMqhbsEYprisR6S6NIAdKYXzUzG2bNQN3AtyQpBeuIbcRrAR6indfQ9zR2vhOnW6O0a6oK6xfElQzcYIq2YkBh6zQWUjIbX5ZWWgeFs9x4KoEljcltj8QFWwoWWpqDYNLQuSU7sLTiMQAIZDpIhSVI+nyoeGWYGq6PPyD8aQtMi0nwCO5SHQJ7ZFoVpCpdg/MoreKLsHoVvzvqUhLwnqIcIO/LBCbntWOKIwjxdNbGaD8LOiUdJ7BIPyF75YtBDvmmtsIPQNBJBH6ZuY9qvtWyqWJ7SfQ/JMY/zvMrvaUiRGJLIIzSVLjJdKkeCkBizlAqKJ3MgB/e6gmDRrA4gH8C1z7x0a+NUN18S5AyhYTLYbGBlmGXhiOuyD4/GKPhjJ7tCJ6e1InWakTGP82/PvQLButZ6t16iQHgJIZgQvZCS5LAmlp0sOdU68+Q9UbH3j5fxPAjzePkkO7fpdHT6tWnRo8eERbRtyw/rQEMX2bkdd9qoX+uCDWEEbGsW1rmuK1Uy0Yp83KpKvKSrlB2++MpH9wlyF3aqI/yWbsXha7og6Dxvnimn6xBMowNVnaVp1ky0iUR+uAo3baDMXbSvXpWpkhFoximPsDZOMCFQzTOD7mUf0Jl7d7cwDK0zrSzzRhXh54fS0CRNLn5khpoRZuMshdlySuyUzndUc1I+QQtz1yWZfQC6k9dcbLbKngekCOjS43BcLAnoOHxebrex5etHsvKGQXLu0KNoes+AscOkMmEoNrQ6BIWuwrsF563RaXz1r+k1VSfyerVTlOFm1gxFSqdhiZHvs0PO9EH2ExJCeh76zI4jDbaazxRPBmrtlQtMhnfo8WXFUMx6lxnRyp4Q3eshrTA2y0Mb9qpZqacfOmpYlomzZX0+yzgsQa8+IdXv52MVh5VZ6P4c0pLV2qtyhxvRggbshZQz7HqvnTR9o977YWxiBsPqiPD9JSuRdIL1lAkh/zw5RddR7A3WjbNvPeClCGl6b5R4ZnmG+YLrOSjT+tyr7DqShTsHX3sBpN5o+i2Udx8bidsED9v+rMDF75ZeVhmyq7AJB5EcNgJIdV5XQoLPrIL1PVg5wBNsNHrxo5dVVg5tjoxBCl8aGba04sS3SxNLwLOl9aldaQBtVy0DDbA45b+bRf6+B5il/zwLr7y1Y3llZSAvlf3p2ZT1jb1ct35eUdZqYEUG855ouN5W+wqTmTj9aAE4UfMuf28c/vkOkBxsAGX3Bt7P06VlxdG2dv/YcbcRf4/WHkTbZwh7H+/EBz6p/lAOoKyD/eiYl63REbTIqpHLNucZPYahXxwt8W1U1UfzuFUlVcdjt+f6gyBZWIVtvcycEB7WQ69NdkdnKg5aQ1wF9xQQXHwzq9l9jA7Ld7f19AxW7JGCfHcS9yuLWY+AmSJIDNhSZc/oIZXtBSTiF0i7DkgnqmcG4o3lIBDBWw16hR6wvJPRNiXOgI/B0ubAtkgKraHTTbtMyqWGgY3hYtwgyBYBAmhmJfs2oP3ReaVGo6N1ekHvaYkZ/tIQ7NVn0bWvrlRTV4HKUfd8uVy2uOZEdDNeqn9OAynJUckLtBe57AQvwy2KFE0NKV8boigMpOKTPLysp7SlK3MtsrJK90ncdDk/cg5mTrGr+XJ63qGpoyhu+jqGsOXiWq/r2j/Lqzq2XPvgqZY3zptEVO1Ox4kVXpdgWdrQc+kSECFJuLdzBKHE5pkB7Wvx/KQMw9ZtyGbE+LQ+7PFMn6pQzFPcTzxhngvbEWTjnqQ8HumHiQGdHjtKnq0NRN8zfy+6vqNndyqoVRyEYIuq18N/pJgdKUHSas0Q6tX4CFjT3oFELe0SP0pzaWp58qqSS+hqkWhFPqn42qSUZdLtmlogEZTfeeAYvkZr6XFraqtMbLfKz6edjnTyPMHi0WrHy6h1bjnVcZC+rClQjTy+PA6UhFARgnSQd4qTmciJ0gaGMbyQnUKm85XyKG+4TMPlgZGJ42BWXD39m6LsB288uMVsKHcQk1E2n7M+KiSs2XRLKKG+5TytsE2Fa9jF6WNfq9jeLzxPc3Fz8hVRXssUszT6aTzu5PhseV7m8lF7USBuG7D4CgQR4Jj1vFBr7uPSY4LRftnWBVWmEStCu8nklFVwQasEk0ivl9fWrgwn3F44GY+ehDA0mXFTXoMFyY5o+RlXqgzi7WlDjsFcjKxjU8FIhaiHPTPXPbuOKZ3zKry1A5vJvnHhl4T05v11e1aReiL0B8MfFgiUzoRiUettTkwuyRPfvT5k+3An8mGjRwd5gMDHU+hOAsMqSJ0NU3t15mMz3ZtKpUmeyRdkjhFm615r0zcTmlLPrEEC1oVVA+zIWlnzR2Xo5nLLNPmhRx3IEYzRG4xGtts7sbZR7AGiXm3XHaZxNkCyLr1O1HgN8Ni9t6srbh+F/bha3RP+ME2nuTeO8ircc94EnOkohNKWo/1youEunFiqszdXPNxbTjiucmPAKHkBtxIaAi0C7t6WOcFrGKvfQQcrwBvA+utj3YZTJb1HvNckfoapwO7GEO6EGcTcVwHJUuyG9kIMq+YxTd5YnGvvxHDoBI06deCxkTg/UXf2AdbZ5nSyWwdh7wrFEoSJ1AFgQdmzGiom2ASCqM1JKVL6fnNfdyLqxm1f6hEpKU164pSDiZQ630bhZaJT5K1hekkcPdHvMHOhqamYzSfk/zzDwcei4d8RADhWQCAPZ6ooyr57UGlbRfIZsMvH3m3SzG0cjz2cB21S+2xmO7djpn2fHym5v2CmeLkDdbzaowvuHZb05RIa9ERvobsYyuIfQV+Z3cN4TbN3nRX8YhcOjLuBO12U9trZeDfrWBVigJ/iaTCYm/YasPpeMRKeB9Azj5Ux/bSf9ed11W416c74oi0VkCi3XwmyNMTmYZa7mJ/7BYNVdQ9bxjA8HCQKrs9CSTRdpI4RDiXS78MuwUKj0rCkLFPhlmXEJF36ekyQikLUTTZHgMfh3lLMqHgKFKoggBzIllxCGAoo1SJ0e74FVx4fts3y5TlOAO3HUD7EnmQBJXqT1OiM5cnmH+MNec7aAwEwUnSa5z+fPcyElqIYSe9mVKH88Ck8uy9DdGG3gTZhEeyWvSYxE24XOq4KMlZhABMaWJzkOLNKUFhWcDcYbbiFvR5170ilphohE5aCSKhFee8mrqnvp0QXDFiNdulekcF6SbXV47iuGcsa3Tl9sk2KqG49Xmil1wnrlt4N8bVL1xPcTixkImIWF0mmlrqhPQKCbsaG9h33KPNk3RGas22HLg49Yao85S+c/LDzcVFLuzsihWQV7Gl7kJn2vulb1M43hnjBxSF9vsSa2UiS0/U7R1snh/y2s3WVl2ZNVh2EpyE5/Ec9YAh6OVaKNSVZZI5RzVUTHlliJMZLx4xqSHihG7DUgDKrksysrmebt3MTZCyiX9YBU1eQCVmrEeuqDCDsbq68fBkoxLsXx6xmQhni0o5J2+Ta18MwlWs/lxOqSRLia1ufe9qKtuXqClSbFNNouKZBy9nNZbCGptQdciTxGRuaw3G1PBevBFpBOmHDwE7p89CqnIX8ZekG5Jbww23NrB1G1mW6yj0RgMSMOdHCy9JAStYTeZE97dVjQ3SSn8MJT+tKeZYyTnZs/da6ZKCpEkqW4E5la9PXgju5S+tpcPN1FlDUUXjWqJ4n2hx05Pq0CgB2nJ/4lgsgrleuAYBAgIxyPpyYMwC2IRBKq+ffmLfaEPPqbrDA3AHOCynwWuK9DcAO/Uuu7NnWiauGKoBhAX+OAaTCndTHeztcgKUPYkDaZLF1FoBhcw4A2LmLXQjx2Xm3uvC3FsxRjyAutMN16QILqU81gxTUeCEAfWzliZOYIZphEeCOYQh9l++oGwV8PwBKF4cDHVSpVwcv7JRlQiZJVbb7vPS8OAuulwvnG3xDTButFZpCnQEvmZvX6u79aIdXKZ0BUTAnzbARJsgU9UIQojdLTN3WZ8pNznZHm/SJVZcF6y6YATK3Dp4AiV7Y3QgsM6COMCgkoo5KdnsJxINXvby9RSe9kicWu0wMo0Yw/6AhbzOLQE2GOWcuaE/M5eNWKr1EEIpCLhf86qPOdYTbtDoIsTL/XGEZpJxntTeePRlqtHfAqgTM9tN1fMDzTC56p0+xKrD4cT3BN6ZmpF7p3Br4KyFTenmlAoaHImBSc3N+AkymJAX32hjPSYLQeBpiybqrVrk27asJzGlAGMxscL/YRvyCnHeEfeYRPYYoV4vq3ZZKEW0TWHzx87Kzl3vNZ4WLNtodmebAPbSyKEL/ZcTJ4VXPOZxDks7RbrE7t1ikkvQMVBv0AtnDXu19nafW4dFtmWutF1yT6XQLfB0lFjbJokpL1OLqHzZYBEBcEPg57fVaZtZH9TGagpd/6wjBcxCT1Ngp26uDyUnVd9krKQSKqFAOmT5kej9JBtOxSMbOPc6pPhsMQds9A5EtlipbiQtn3blbzXD9hoXhhRXnU9THXUESqJ3yUHUCBOGdCeBRCG4OMZe6nN3L4bBYbuQPPOgBjKF0Qd/YjUw+V0PrPlSIAQKT06pi40vlS11dGGsv5mgsVWysEaOZKy3Z6PW4K8vtli0VZNJyWFxf3jOsDm244ha58c4+G6WyZegonYCTKO+NR1hUPIvDazJeRVjXj70hh7MkNWOebvmAms+4F+MjFJa8Ns/sU3XC7w8MkAsFUOQZSUCvna5opnPYxHpj7xnI7njS72SEfmRCdL6mNiNmofvmBXA3KYDIOfgtYt3GR2qb7+LdNPqY0zIJymd2FFWOt0zVGMJDDsMjIUhdvp2tJqJgsXhJZNlzjmmqZfEJKF/KXC9l8joWYEES4ra5AgunqxXnxkHzuIK/92+bcnXWAzu/plu64Kt05BPyPhITmKlZyqic7BjjKgp32SBn5McZVpKTs08OvffolYMCICLH8KgpPB+zOFVebOKzPsmpACuV5+yOsC8AhXofoB3JVytPnhLCvffJPbFh2SJjEeIULRd3GRojch5SwBKP/F0gcIw2Hy2XTbC5McMrZRWDSfXFC90IPUx5uP1kLN7LM6VdKmgxPod4JiZi7Bnn8gQYzwYhlkGfFHcBpj4jWEjLWpeigB+eu9FmZleWYhutW3x55njA+pcEy3azbienZ8808K3IV95V3NGcTSu0+l4fpZeGOw1YqL0+UHx6FJXYVmnmzVu8RJTgWbsX2e7Y08pDDWxYKc17T9Cltie2vZKqCWrb36/j7IxjYO75ZrYGSvP7ddJuXdTPXblXASBWKecrMMEH9sHTWNPoryyUx+f4HIS777CAO0p7RxuCNcd+agya02J05yXyUlSPF2ARns8fGWn1S2MJ9Hqp0i279FvdVMUjAOA4PfCS3DeedEd2Q4/wemsaDaVmgu2XCB4TPT8G2Z3GJ0KuV5s5Xg/Mn2yEe7QD767NVeQyapLENrmseBPe8O7Jys14RyLtesuT2XdgLUoIzj4NX3RsyvZm1rsm+5hYYz6PpU0+Zwb+IRFeJ48VffiHjuX4oWc80jEOxGOQhKmkd3U2LJJLneJo5nUGam5aXbopOVVsHiS4pEEZ/ho6PgVPOp09hX0lSLqmSLciAzTlybM/F67gZkOflQdDdUnnsvJ09QlYCNcXlfavPuOpmPyuerNnORMDTMyn6D1Ylh5fB3ONxU6rg8YSaN+Na6Dly+8LH6hIZbysheUttkI/X3jGHPfZflkfGdO0iJMx3NdxIyQ1vhvXBklOx6RsA6VkAykV1AOSOra3snNLojayv88yc0+n3d/S+XFs++vYGfvyEYhi6evr6D6CZGZIGahAj3daHS2CPGAV61ZDH+TpcsXZm+/I89X+LBLAHjUwItYo73WIpH91kpv1WUKc4BR1e5IuxUQvAm7Yz1KTqHtxkhrt6pXHxO1l0XUEBKrbzClM4EWOb2AlVSRLGAZtCQtX8UiEo/fswylxUaSuZFhoh8AuXqnTTrqUjpwPaZfbjqNfZDLINXoDWResvMyvxlsBgZ3rAIblK7/cqlHH210sswtcEWqMMJgT5tKNurzdZfgoDt2u8sMIK9IdbggzMWARTXZh8fOBn+AuvjMy6Ehf5VftCuN2qK8xNYvWfuAxhGO3eCQcRat4IOWb4m7ZSY/phT4pr0HOnzA+nUPgjGeYCHBDPZyTIV3E9Npj7RhcRRAKIAAfWJtoF90nL8bWZi8B88N2rI6+b8MUKPQcObfPkc7SYO1XujHrqzRyuRKKIuqZJ2bJEzqk0BaYDWgI1mwJdtmG3oVhIQwsIaNgBdOXJ6ybe0TsMNtNDgq9o0eU2Sd9kEFqs+y5EWa1XHsXhH0MANLQYnYFJEnvZMwwF0wsCRZS54faWTsWlggbEpF/v+2yoNGlRuMxLq9XWPHxMXfrTjPZX+GGYHrJIYlpIhq3hwijprS4Uy7JnoAnG06Et8HuY7tp3s1bXlThS9e6xz6aucZJHDeQRJda3frcFrWKx8f9PHHDaRjlpq58sb67sBbTWxOjuHe4Z4jVIU5Y07b5hmj2y/TcpCslmScfG2bj+7h8hc7/XE1jYPPm1YYaKvtITK+n4H6w0bbqmc6mnWwlc9vO0xltLdYfU7HjHZh505JZL0DWm/piX4Cor4tF06/qScvyJnRPh1ZBpIi8ar/27XWIGdS8FZ7mohHaVY5REHtAvWyalIxVrGilYytif4G558W53Q8LYUalC2BWssXopjOnqzPr/gtim8M9JenYieh2TAlWLdducTT1vYfQOpd0SHSCEYmBcYSeWQ4A4pknVal+YSA26cPIYSyiP+68e+eW0pgXtGcyCj3TG1nBz5ugR7LzkiD0o1dRNEKjVkSaEy4njsYRybtSC73daOxXNrSKziBly1HajiVZhQxaBlY7M1QhnsKEwFi/QFTLsmYCU6PbACsd9Hye0+rqrFVPwmxcHnssD0A8fNzg4HIlwozuRg7Io35caTjZlmHFg5JB1McK9HZhEN8bzkvvy+XlksvGhW6N94Mt6GPotafd+KYJbLwH+YOLNa+8VZrpQoF0hGc93wHJZTMiaDBcN8+qqv4q4Nutp1Lp9gj3ysxhqp8dOptV2chamP5kFKVgjnc1XyiPNU4RNvcxFA34iLK8aZAfVRmhFc7mzFCWJC51yxwpHgo4qKETAWDaF3Hxj6M5mjhu+/jex3i1q9jKIiEuZ5mZWRGtqhGq9NsT9xNazIYXdHbKHwi8dF5uTaoxGUl4vK8lZDhcK5yqFNAghrJvqNRe+uoQX+SO1E/SRYoN27Bi5B+8eLj9nUHvtZr7V+4JsslXAT+wcz4N9/uA67RwotHQXrzzhX8fvi27VLjehWG07EIEmWPy2kDKCLJHryZcN2pMJ4cDEEoDD6n13LXXQaY8Vi5r9DqyPF8oGxmAM5zSaI1kxoEJ80UPXyi5nsXHmL2TDOm+aa8U3UGSEVrJi8RRFWSiqJEEEUhSdnR1djOnOyx+QqXiq+5mt+JWt4FpZ+1pM/BUijOyouFmCx3NpMCEE3qzTsataFE81CDxBBHPdV3Td28wT9Kb5zP215Sg8RNjDQ5rGO6xI6h2Agp0twdQe73EKQe34DULVu5n5K5mHEBALN9dq9JUicg2WzD8dr0QSc6E+NjIfh8pZfsMUj6770tJomInrQ+NzYfoYt2sbt/FCKlpbi4QpRcAkIxi4E2Jj3LiUN4jJqk4QMCucRNd7P1eB0JnH/5rda7FzbWCTDvsiaSPuGWWPOS0u3dcTnw9i6erpHJAA3WbFrprJueYEFQ6kiUKRhnILvnuQKlBWbcFWsupjpo2Ee5pmHgY+7yxnonAgwGoB1nY5HdIypOo8zhxWZoiyZiftwNuRVsNvmJcuzh4V6q07HXff4rpXM0m5rXAZikqTAYm5ZbmbJgWKWWOdVNFpmpIwZ0Vx6Pm4SqMlapQocvcRps+09bdZC7+9Fhvdm+HGst7uOzMiiyjYUge7y0XAzvPJgV34vVNuL9gIoG6IdPAQB6dyrs9cgY8yUgNY2NyRxfrN8280BnEJt682YWj4J3dOC6yDCFXECpNK2lAK7Hkxp5/Y6PrYYwY8z5+vMNTNjoJTHosyg0zAb4yPu+rAoix68kspnvf9+FTmHuCGp6pvFszSGeScq0fL7fqSKk45HyOxe1inK6GdgorhqZykJfBkgJLFfgg6zoFl3DIbNY45ejNZg0/88jrjwAJ7wl+ji49Q726O3+S2ru5Ok7YT6emHF+Y1muq1iBqxYw2XsC/yXU200lNjIjXJOQRcmGZM/hihGFB9zNMh2uOKYlgdfueRiPyKnWSPq9wjc8dv4yIghnoWXSiK6sIuL2/NoR9dpTD3knR3+/zA6QRIdnzjEXHtCvqL2Cd7/M0Ofy4Gy/xxDhY7u3U6EY5sXUlJlWpCwtWuEdHec+ZhgAq6m80P+G5URgv1RTc6bn25Xsz8NYmModw6Q2a3+zNJ+bMVEFiVz22HryyhTES5YjHRO3oAXQSTroTRH6FDXBHoO7banUF13Gph3D2yFcB4qtQJZZwdpR7W47WfQIIdgJRyHXtK3gZo6Ef1QpONyFolAPkq0lXiC/wAwZnCyLgvRWuxWUrnPyk4CODv25VlagXdpRuvNrqOsfpiuLusGpfdk9sDAnmEmhXDACJ2++epRnwDFwOyQdO6GiK1uy9r67XNFK2G9/Hie1RxyZN43AgpFhflHKlivkKEixl9GzXfTEHMpYY7lTlBQa/U7LdoIMC5r43YVkeSwVa66M6YVWlVD0xGUOQhTeQsre58T691z1u1/vr0S+h4eRbUWanNrRSpOVDKqYMJ4CSb+iKTe5eigXhBdZIw8kfyF0+eGtl7/uG2y8R1lAsPYs22tX0ZPcJJJEtjKqLLtSVyTmJHExXYw5SicVlXqRDWibTB8ShMpLIV0lOtJaNhtx58p4O2xh5KeFWomBtMZleDT+tc0WGWUXyk3tjHIW4Im351hdzhifZX2HJGNn1ucS4GobiUC0xQa+12HS7s9IqbYWiTY6BVFsuPMogQMYbijHO9sywGjXV7y08TC8kEfpI8HZ4H4CK76v0yp88V8uiR3NZi/seASCNAgn6Dx+59BXDprB4Xc7FBP9aAl9Uaq869DGPTJYr8mnXGRBLn+/8O+WTSD2FCzKHWhQ+lrQeaSEIBQqhqTDodIEzl1h8XXRBxHJ1sU5IcH1EkjTaSIjCv0zOuydaXfLW76jEYnIhvha6MHfYkzboY1rROdHO/t7fUP5myjep1lq4ncXr7pLaT8KI46qevQNrIiM4RadQnrXNHTVGQ7T9QXgX9wYCPXujm+gwTwZvuV3rZG0P9zYTFJYwIPsdkm6zEqbS8b3KXemZQN7hlWNY4mjsBtYrsTwF98IMkxe/da/3yNhE0ZjFe/LmQwMrPY+jF25ydU8uOKA2s/NKZD+kEtE/kBuODbF6TgKOI3fWtlB+4xVc0wmdKUiwJs9iHJae6SvG5cCF+L4O3h8oeMIjbJmSgeTHvESd/dqJ6T6Xd6J6jHe2WIQFn2PYoZQAA7+8uV/x4Nq8IzGbNpXZRCpNx5c+UeLr9TxeFL9VnkV4404Mrkk8novTtbYhm3Zvk+JpV/cVRFIU6Sm54cW+iGwiYLLtz6U1CgjF8WhxBmmG4LgPxe9ATgc/n4m0xQqZE6zSmycAHvAcIyzCn+FO5HLdps7OgRLg/Xs3PCDUmOd7m+ymxd1b3u0Jpc3W67YMs4WkzWgq8dAVkGl6mxngr1dcEEtgPJbQbTqElSP1RcckXT248OWWyiniH4/qePAPOzTHZHlSHHLmr4ahDxFVyUx55ICukOKKMw0ftXpKAAa0vLfaXoBLg8jYRCncVwtJXHi87GOhjuolP3iME2NubUL0wPlKao5W5jjJstbBVgxlEOvjUgqqdAGBfO4xMdZ8LwnR3YGnV/hun96WQWnvnDzwEWzB3JkciSiKCDCATg+zfEl648Fp5JUSkbVfKmQNL3Ri5XpGUakP6zIwuYZJr1rzmCpEenLWEPReriNRkKRrZy6sD9de/BRXGm6wJi+PqEe7bfjXStJP46ZoOM0X++WzoRNwnXk4L20QYojUpMFEplfDT3CxmumTK8DzyIPqldMDxAIDIAjh9sb5AUMCETVhYfBnuHns456F2v5CIeyUH8+AENsUjfY4u9yGLAzbsr+m1yYH8EN4WvXQN+4hoGJw9DcsrRiWIp/X1xBQTQ/XeL3RTHqjywxEneTMQp+0M2Vmzz357Ju5m8J9HkWTvlB2aSO5fFJ9z3HNE3kTzqIxTPQxEiCRnavr+vOH1K6SXu9IFhpR0pd6iklCgxqhtyTR+28OkNMhron9krFeZ2RMnd/bmQc5AaPzIzvBB/qcRZl2x4WiSUTHYY7YgEmkcTCCRvhT2M3Ywco+pGBz3x1KGjoYjO7mLD7U1bpxbchvISMnl2NJVvlSeELtdq87KWnl4b5PCae8O9yUjZn5LRthIXpYVV+n8wH4yQ1Dju4ZPZL2NE2HyquDgE4K1SShJ0VgJWCqS5ZJ708Xm3hzRzVRsw89Hy7tbt10+ZbCQw5swRNXagvK802DBzTNxdXKF+P5yXP3ajkdhTjvMeVlP43Tch8rL0DmQbA2lKiBFTbNJbptSats9M5QzefzXC5vGulCaz32Ug93mpB+viKsNZ8mLZ+JBVElGp/OhsTrOhPLCSa+3qVzZ1r28WnmNztPew9XFp3dMj6O+VKf78UdeGN0xQTCuR1dTyVEDWRtCpSHBMOrRovgMjacNLURVfYFfxC63ZLiLiekewhziahwL6pHkhRzj9vaB5OJOne2PoxTVomlcdXh+Zpzs9fTretVI6/YVdyjVZpXf8Dk2Bppa8iDqwqVOcn2go89yC7e9QJX8Rw0s3Z0lnXME4I8k9j+JWsGNzAA/vwDnp45zbF6gmCqTWYMUhn3dO0g3gFHigsDiNlK2YfnKICBFd691wx/IRw6HMoZmRC7jI2QzwlAIUO/eTAwe7s12JlEsFCwo04wylsh0C8OHgEW9l0XRitWb2khOutiOBl3vlN3VIUx4IKaDOfHZD1kRKgw1SIQuwYj2i6xnScrWyrMCq5CWAG98UWDvSNKZGG283/au45l15Hk+jVaqgPeLAHCe4LwO3jCEt59vVB886TX0zOKaWlGK13eG3GJYIEAKivznKyqPBhoDIKjbfBj/5f2jfJd3C6jy2MXMh9atkEZiblNBJbZt5xGGUbrN98ficlbkMw/mQOifN+F/MLXSQftq4I+AaGBJX/YvJQ4fCKgMSh983W3nsrArleT+nWLUPkR8wueO5dMQRV3tSYYk3cnMWZbA/yNmJvPvhfPmpab9hf4xloj52wWU/iDAFzHOdnW+3F5BlTRLwj/YChXFfsXR1JWOAfzggToFMAg67mUy0zB0ycoPQ4xUKs4+NBK6jzfc137sF21OLaeCFI4IHrMwDdfYRgGY9iI+mTxyJvsFTiyg5q5FaTwVDHAvjKWgfpgFsWNfUYmA/xzr58Ywz0f9oDQQTWbNx+7n7+Rl5dnFlQFSzuHw3cHGGA7MtY6PzogVvH6P3tg/fRKgtU5pD3uUIpNwArKzbNWxf2moXpvgIJxQ/mxKKVrkvTvnhTOtppsd9/miX77ngdIcwrlhSXWTuY0Q7VOdAarROfJ8MnIyPBTCtR1ILwuI/2oAcztpuYeGjmKujrXjHDqRrygRYqJnYSPI56UJ5tRbmZv6I35oimwUlGPAzNNGSZS1KZ/ItdNkLsoN55PdurysRz65OROkBPZudB6+xDC2W0NkdWrGS7NDfcILzyL6uXcjuAkPR2qS9SMT0w4fFPP8F3SjRQ7NFZHH5H2XiP+Mjux49TqcgZzpsXrIG4U6yOw93erigQ7+jgQn2yHdbnHKDTTCf4ZkFcWHFkogWwBLjR8zWssY7K+QLNmbeEfJKAZwCCq/qNAwEsYIhhWHz6AvK5xV7dY1PpMjPKPLeYPUg+iW35Ed9Ziby5K+yEydmWJ5gTnknesx75viB1qFfnyGoPLmIAzUSlarwif1OG91IEIwcn2gIADtiNGqRRyJlcoxMvRbBFjyWeDG/1lFfoJOI/8Nn0AbSkOdpMdClOTKsg1xuVtaafFCvwPqGdDzDM3R7iVnKl5m9c4POdBztwO7/pDl2eBfCcxZMqsZH8sN4tpNWmCF2VKojSvDFFHgHnMJQzjqMdASxZR7vIkacIC8E85ATadKafeAUsRh0+gPJUYVK+R0bJuQEBAwhgTRy3AywlPUbOK2dtxdtEOyLN0WsWG3xbVqqeLC8fkdigMfMDJxPWYwYSUkJLvkQs7mqllhMMh1u/dJ28ypY63+x3uQeYM3nCzKi/uvOc3bWcpLwDN2G8Ue9FgHH0nMgu18gpbSMnaOYxqIN/uMSf7DTB3hirVxX+JrmspXZGZtihg2gvZjUVhN1S7iF42R555dh0CchuCM1fldBGAxxXUCPcHufXe1l+5lfngm62MIhCbniAwDSNk1I5pu90s4FnJ5uGsH4Z4rcjTN+sGKeJoemitpx/pTS63AQu+S8hY3EpvwCf2VJKVWZ68knYwYYHy0LH36OcD6WqdrEsKbW5khjul1ufzpI6Y58JZ4PENbbuW3sxzXeEX+mMLI37T2Ogsi1bSxIRndiSYH05hIsgOcfaNgcJNGPVKLXhes656mg2QJBQSBdwRJOXZZx3iwmn7qNzowHWaKHL2sp7umzUZM5IlI3y0Q1wtY0i8mnemGToG1vahq4KxGRuz0I8+sYWJ4l6e22SR+57G1VfwBD5NLq0R+Uxzl4KPyI38V/eilOmzX/0KGewNDxWEyXCCdIZFq2AF57iQexVnYK+tofmF3ZaDYfpqxxvKSyc+9Eyjc79axs1k2yh7igmaPwtq8jYCFL64nTFOoxMaTMU3Q3iN341bYCnC/fcOCt183P3nghlo/g4OAKfH24yNCjE5ThR3+ct/Xi9mvXmGpXXe8kDa10AvbvNxCXiN5fpJICcx6TCjL2LGKlvXnXKlQcL+Zwu5qCqdg7kB8QXSwF8KyGgf100cmq89YxSy5xGtGb/f7r5WrwrQSumSCgxWke9SH3cBqRYbTF1mDxgvtNYd3Rs7DkOVtnVEhtMFumZOSrLSH0t5UusZN+Lz05WLVK4v1EjhI8Z4XmJZrJzI5IbAxB68HiRADM+5KzN4xXd8sVJ7dcskWIAFh/EbGZOArbr+x+IV9jt2GvxRBBj5nZEEeGXUPZp3u+HsXNjk9z0IKd6M00TyQaT15TmLB/3Sjuhh+hJTPwvmnT3jHGafRLvwBvpxiMoN070vRCr1++2+824u18Lo/RklboRXEu1hrNNDdBb6k7ullF2kjQsQg9hzLZ72FuP1AXdrnN7+6nkDDjzE9UPvBmpHsL3HZpP0uS1BqwMeaXf8GD1mpkUs9jU8Yn0lxOP49uxgZG4MqPjNOOAj1ILCHiDDJqEI7fo9GRi3YwWjH7PXmNtysoEyPECc1+FSDo4P4lzXoAgNWCRW3jG1/FYMmnHDD6wRwtMThQ4tzt5oFt4jPhvwrKTGZbWRFxWocc5xbzYKN5NE7c1KtTZPvLhC+mdSI/VHwmnGl2GuiME0Z7cuKQmn7x0uTcIoMUNKrV49RjUyAKk9xA2GdcNDA8GIxvi5vDUkC2F9yk0RbL3iKZ0EqeqHQJtCHRcq3izcKj4uv+bsD7OVZoPdg3ta74eqHXcMrxxs7EAIta3ecQyYTbFKPT8RjYdYgoXf/R4qoK5Mrk/FmZ0tAONndYkWp9V+5SAH8ZZ2YNTxosHGhyDiOFWiSiE8cjb60EDyY8clXAA9B0mqQUW0P6Ym6ZADV+7x+10xxjIFh7J6BWGbEKXzk7YeKBaY6OCzyvktMDCl/QMmJoYZSiSU7mj7jR7Nq5h9ZHKNFuxyAKXvBIpBIgtmk3ybAoUTy/M7b8cASviAI++mbqAIgevavFcH/Z8f7EsRiUNc3lgZJAbQ55IpmJYsg6cTJvWETcu9ngxAJmJ8c6OHMLIWo+H2uZVr6SFJTKKohM82KiTSUIMudd5NwqKhRsVrN4xE136IVZhn/Xqcdv1uzDiXAzNfDcUTmwRkWOuspTWje7WNNd+8X9okAFafFAy9+2oWJskRtPG8Xj5SjHMrjUTu1oUkmyxlg0p5n4jH3fVM+8l2vtiyxBKjfm/iFht0e3hx/wCFRtmJDKFUuPwhRmwCodRjbbVeN+edhxxUqu5B1mD1eppP1/dozvPPpVvohA9v1oiH9IDzxRzgO5hAvtTqYV+KjLKkZJgPeDefT8ApwArm6kRux6LPVC0J07sOicpA0PgxeoqTvvrj4MWJqbPpQYwKNu5tDH8sn/cnzVSUwEgCCOPUTk/A2RRMCgsYfyDPtlCXZxPSeheczXmDX/GQ4NFsJF+YO1aM9P0Q3h0G9grIi4S0BEpzyx1yECTMc8NF7wsE1mIhaG1pmHdCXesYcS++3FFbM4SJT7ASkp29Ku7C2y71WE5Z2veXxMvSPMYlIvqWaNlZvDbV1R5QK7VCbpWsDU1bseKbpunvgD+Rmh3HCI2KIvIld3cQusFEbV9pvjMJTLtu3r5yRzD5jTM6XUBNU7imtJiYMG6n+P72xe50GlFMRiqZ5Y1EMTdmBTF8SLuFPD7qHlMnzT5P7tZR6QCMSs1Ka0GiFVGSOX4H32HesKEmCT+hz3G+zux3zFwY/ywh8BBQffDxxHH0tm8OYnRxyU8LZSeNkzDqJjLuC1f29CBv8v3eZYuZzrmdRxnwzih77PGxc/Z0JTmUNyhxxxwPGXsncPFGN6FVtJyZaNOtXbN7dCxbv8AvVE/Jx06jYSFnYOXgUBhp/B27EE/vYxDWAyKQIGsjaSclxMPn6TAFfiT6nILPV/CckJN2yX7R12BaJ97eENFepH5A7oR9V92H2GLVADxpJl4Mhkr4HjVtN20KpKOZKJgKrSvF8PWdVEmOfLdVRn5BeRfq6T3RTakH8MZ51B3hE9d7LdzTlCqUo98+phHnaW0Xpwb0sZWLB4MIbwKYAgu8LUQCfv5Zn7PnE8knZBzYzzKPQwd7SZtlbK10Yq79MtLjqL85Jix96Lw0u9Tk0CGhUvSLwJFh8uFcP22K1dB6BpF+iSfqk9vza+J685nkKuSnUBww8cPC3zNc2xXDhYTkiedBobV4OR9zpwQxktERDi7buc+PKRhqn7opGen9jdtjVo53zKbQosQg348kdSVIHVzYQX/YvWss1eVDte7osUHNT5wmwcTFSD1ddRrnvlpz4+MJsVaM4zxM+ndjGcsNPpMPdWvSl0/vaC43dQNKErAvUjUNQ89sDFMutjfMPThioPvFTnAdju9Wko13c5us1h1gI1Fy+CbAKqOIx9GqJW2iiNXT0CQjf/hWnDvvC5dOTQYLlCDLN/Y8eAVnzkslwvFPCV85SvFXefnYo+3gnYXDYrlk/BLj8SPJQi+NDBtLTCek3usuRJ3k51Mo0Vn3uem0lcBcX2MS/w6R9QbTuJcuReCsXd9lcL9lhxJg8ppz8jnP1CHryhTw/iqWInJi4tsuxOpguy19Gbjn64VMz2nywMlKXL5FsTCGdYYv/A+vF71SNzu9MYW3sK2FGRuvS9+E9ME+4RCUDmGPoGrXS5bx2tmII6Qi9hs6ZMzEqPqd8T6BQTDykbXa3ghKeO7W7RC508pvp+3gvlpbJ7U9WjlcmIJ03pMM3U8fACfB9SKqxXOWr8xqNdAF4kJjw5N1KV75DXPBXA8kLjPP3a79Td10WKefEXCi1no7yVWFDb+ZQNyCcDsqHffm/RJhNsmDK+1VXe4Q8oxO6UyjZR/CYgkhCmCS+Wwm1DrW2Nelk45ie5x8+oABowRoblTjOibld5zSIVV9dzkJu7/yjeAGm1dr4+KNTBSHFaObjSFMtLkiBL2cRDrYiulU6cAlGQJSmkbjQa7b2ur5EN6FImN6DKUvZ398jhoPlroOcRKVGNApViqoa5k9ROi9YJv+NEIsqiowY7vLoDIRC65uOqibW8INakSOU7YYM+85RLpQ6ozi9rlxo1q7oWeUFKAU9zGIMkakcLKBwjXp5d98FQOIy2tStMEzalDPWN6mh9d+0p6JbjIbTYxYf5hZGFeNz8Q8rGnFuGBQ+CZ7/g9KwPK+tpm95hThiazDG3NE8ZxRK8HzolIhE9Trmmy8XdJyCt+kBM8l1X9RCWA2r7Ia4ew0BqMAVhi+8RlwvOImNBMsTWwwvDKvi7SHSJCgKi6RvXAYtjXvjqPezZZhRQP5oPZdyPA8PVCtNHaUHvY1jIZIwYPrsjRckYS7C6qX07TZnsmfTTw21/kc7Z44+VMcKkBxnm1qau/TOe0SVytdTh1Lf9Z8dTBiGTsF36A7IZgDbNMvLcWXGT2Q6aa0iTrSgdrfox6dHE/bjzVn8KZfmbOQBwJCP/kex2Uoq3ZDOzWcOfcjyZ3rIN0vUPDiN5RkFthz7nbQI3+vPInVUqIMn9clOrJ0fMtZ3wGmhSZjbsu02AQqh4p6Z1GiZ8EyfI3oIht+k1HeAZoWMiGSwkNKFmnNPOD3sq1B/kkvB3+Kit7DJBbyiBR6Arwv2bI29kDWz93TI9Gc5N08sPU6VYH+dJUY+2NInGiWwdv48heubg4LowNL5yOZskm9nwTkfBX39dVWPZzfuliCamWrf62v4+74OotH/AmGOAMSSgI7c48Hfg3+clP6VZ1MmxbP8SxGXbdl7c2kiSWmUAUWKdChacoQFhJLTIvjlek9niExFWHQ3XtQbMR47s6kCOI9G5dTYqhUf+mUYzNUt17wm5IBP5lUpdUKe4CIC6Rvl+n9dGC+0d1sKvRAZxTtrfEgay+AUiQsm5jn8bkI0sbm4SYKSukT89OeTAIj4CC2jaUZltnDZ5SXPu67+VazXhRS0FLx7B3o9mqGC89oGiyx1SQ97rWj80RaVWOredZOM10jizxkv8OD04VXvUCuTgdbi1ZWyqeCPz8Nku+SNC5Y+2wdtw38M38MauxnOF2L+2qdWkza8w6zQTu3Baw+ePKCL7jHSFD/7dGe4kPsQ3nMhcZhb5T1iVmc2SDvGPqmbZRsL6wjsLNdjM0N5JoeVBWdY70JH45B89T58gWvzKxYn7tef0Gz0o5pqFMrfTsU7tQsuUIK5cUvTii3uF2Ge1zejxglPP8D5Yq+6S8Va0jI5oMk4P8koPnda8mQN5Fn4/akKYjm9RNeBPnkJUplW0DdKs7DSbBmiDXIDy4KE9IsKbzvT/tgH7TangLUZs233hD9fBHwDKs43K/QULg3U4I1dzUpSWKna2PJl41E8meHpOmxQtJ4osSwkDkwrqq5FN4pX4fJxvSbSl6p1Ny/K7mIICZOJzRvYje1knvw3N7kTyWU+HxukiMuGktuEhjHNSJ+4IEIe1p8wjDt2DBnv+fFUgkDo5yuEfgAuHPD4JzD6kfFt+1j0fofJS65CoST+xw3bVMR4uoGWiL88XCR9d1Slgtrlu/xGMs9dwoxWBBPyleGYEwnJ9NIegDZvQWrMBw0xo4bPqnr0nSxJePH41DlikgICenw8FtSPV66Wo5uwloOKJ/mE7E6ZwKWEpcqTkP1YFIEArOvOJ1vjxF3Y1H20hh2+dnkufM0cKvBGjYjCQ3qk4wjwo+hIZ+o/ITrJ3ow0mszafJC1/RVqJVm/NlCtP/dS8PJdzKKpKvT85opgucur/24nufCdTeWk+X7VrUuiJ0b7NfOdaJOtJnIMPT5SJFx5RptfVuRlPTcmj5zauRI+IN6Xa8ETxR1svMTGJaYdzk8UM4cyuTu3qHP/JBFgD7cJyRwrqe/TeYUlQuVTkZ9LUImaTfwQGLztS71JDzXPfCEHJqLmvI1GLdqRsrvKycGJE20TCP1KSTq2TOJ19gzFOzNCKeZNqhGdSSIIWIZjk7pn4jMIOkBZuWeQH8E5f418i0/ZZ3+SgkS/u0vIls/fn5Kvf4i9oLhv2F/QxQSu1v+EzTF/85N/FH/7g+ioj8PAMmwr5LQTwEXYlw/Pz6A0jB4/XroR9sfcmx3q1c+bUCYE4EeLZAH/UWI9Md5/4Q+6X3G5a8kZP+WzvSvYj5/OfSPi/T9LaGg30sJ/etM56e27E/T+aP0/E8F4l/t5KdU0L/ASIg/dMH/y+n9M+T08qNagp/XfP8fguO/4X95xx2/fIw7f77p7z4Nfn3zoxWJwD8P/FfD77vftbTyqbrNAlzLL1b8V1J7/3qJv78zAP63en3470cO8teO8x/W5PvDiYjfKPqXn39Mhu/f/rzE3t95sn/UfmZAAQfIuTvzsy7z3xyg/ycqpf9Drbn/ztP8n0iXwr/r4H+Hf4P+CcKlgGF9QAD8r48DSfMfstwo/x8= -------------------------------------------------------------------------------- /model-manager-architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/glami/cortex-serving-client/aa594b2061534d84fdaf320a3250a97eb0b66d92/model-manager-architecture.jpg -------------------------------------------------------------------------------- /publish_to_pypi.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -uxe; 3 | cd "$(dirname "$0")"; 4 | 5 | 6 | echo "INFO: Check no uncommitted changes"; 7 | if git diff HEAD --name-only|grep -v 'cortex_client\|test\|.ipynb'; then 8 | echo "There are uncommitted changes! Please stash them and try again."; 9 | exit 1; 10 | fi; 11 | 12 | tag="$1" 13 | echo "Check tag in setup.py" 14 | if ! grep -q "${tag}" setup.py; then 15 | echo "Tag is not configured in the setup.py"; 16 | exit 1; 17 | fi; 18 | 19 | git tag "${tag}" || true; 20 | git push --tags; 21 | 22 | rm -r dist/ build/ cortex_serving_client.egg-info/ || true; 23 | python3 setup.py sdist bdist_wheel; 24 | python3 -m twine upload dist/* --username __token__; -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | requests==2.25.1 2 | 3 | # PyPI distribution requirements: 4 | setuptools==54.2.0 5 | wheel==0.36.2 6 | twine==3.4.1 7 | uvicorn[standard]==0.15.0 8 | fastapi==0.68.0 9 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cortex==0.42.0 2 | PyYAML>=3.12 3 | psycopg2-binary>=2.8.6 4 | boto3>=1.12.11 5 | psutil>=5.6.7 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("requirements.txt", "r") as fs: 4 | reqs = [r for r in fs.read().splitlines() if (len(r) > 0 and not r.startswith("#"))] 5 | 6 | with open("README.md", "r") as fh: 7 | long_description = fh.read() 8 | 9 | setuptools.setup( 10 | name="cortex-serving-client", 11 | version="0.42.0", 12 | author="Vaclav Kosar, Antonin Hoskovec, Radek Bartyzal", 13 | author_email="vaclav.kosar@glami.cz, antonin.hoskovec@glami.cz, radek.bartyzal@glami.cz", 14 | description="Cortex.dev ML Serving Client for Python with garbage API collection.", 15 | long_description=long_description, 16 | long_description_content_type="text/markdown", 17 | url="https://github.com/glami/cortex-serving-client", 18 | packages=setuptools.find_packages(exclude=("test*",)), install_requires=reqs, 19 | classifiers=[ 20 | "Programming Language :: Python :: 3", 21 | "License :: OSI Approved :: MIT License", 22 | "Operating System :: OS Independent", 23 | "Topic :: Utilities", 24 | ], 25 | python_requires='>=3.6', 26 | ) 27 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/glami/cortex-serving-client/aa594b2061534d84fdaf320a3250a97eb0b66d92/tests/__init__.py -------------------------------------------------------------------------------- /tests/data/batch_fail_deployment/fail_batch_predictor.py: -------------------------------------------------------------------------------- 1 | class PythonPredictor: 2 | def __init__(self, config): 3 | raise RuntimeError(f"Intentional testing error!") 4 | 5 | def predict(self, payload): 6 | return dict(yes=True, payload=payload) 7 | 8 | def on_job_complete(self): 9 | print("Processed all the uploaded batches.") 10 | -------------------------------------------------------------------------------- /tests/data/batch_fail_deployment/requirements.txt: -------------------------------------------------------------------------------- 1 | annoy==1.16.3 2 | implicit==0.4.2 3 | psutil==5.6.7 4 | sortedcontainers==2.1.0 5 | sentry-sdk==0.20.2 6 | sqlalchemy==1.3.15 7 | numpy==1.22.0 8 | pandas==1.0.1 9 | boto3==1.12.11 10 | joblib==0.14.1 11 | psutil==5.6.7 12 | scipy==1.4.1 13 | tqdm==4.43.0 14 | uvicorn[standard]==0.15.0 15 | gunicorn==20.0.4 16 | fastapi==0.68.0 -------------------------------------------------------------------------------- /tests/data/batch_yes_deployment/batch_yes_predictor.py: -------------------------------------------------------------------------------- 1 | import io 2 | import pickle 3 | 4 | import s3 5 | 6 | 7 | class PythonPredictor: 8 | def __init__(self, config): 9 | print(f"Init with config {config}") 10 | self.test_id = config['test_id'] 11 | 12 | def predict(self, payload): 13 | processed_batch = dict(yes=True, payload=payload) 14 | 15 | s3_path = f'test/batch-yes/{sum(payload)}.json' 16 | with io.BytesIO() as fp: 17 | pickle.dump(processed_batch, fp) 18 | fp.seek(0) 19 | s3.upload_fileobj(fp, s3_path) 20 | print(f"Uploaded processed batch to S3: {s3_path}") 21 | 22 | def on_job_complete(self): 23 | print("Processing all the uploaded batches.") 24 | with io.BytesIO(self.test_id.encode()) as fp: 25 | fp.seek(0) 26 | s3.upload_fileobj(fp, s3_path=f'test/batch-yes/{self.test_id}.json', verbose=True) 27 | 28 | print("Processed all the uploaded batches.") 29 | -------------------------------------------------------------------------------- /tests/data/batch_yes_deployment/s3.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from pathlib import Path 4 | 5 | import boto3 6 | 7 | # insert Cortex Serving Client bucket name 8 | BUCKET_NAME = os.environ["CSC_BUCKET_NAME"] 9 | BUCKET_SSE_KEY = os.environ['CSC_S3_SSE_KEY'] 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | def _get_extra_args(sse_key): 15 | return {"SSECustomerAlgorithm": "AES256", "SSECustomerKey": sse_key} 16 | 17 | 18 | def upload_file(local_filepath, s3_path, bucket_name=BUCKET_NAME, sse_key=BUCKET_SSE_KEY, verbose=True): 19 | local_filepath = str(local_filepath) 20 | s3_path = str(s3_path) 21 | 22 | if verbose: 23 | logger.info(f"Uploading local file {local_filepath} -> {bucket_name}:{s3_path}") 24 | 25 | session = boto3.session.Session() 26 | session.client("s3").upload_file( 27 | local_filepath, bucket_name, s3_path, ExtraArgs=_get_extra_args(sse_key) 28 | ) 29 | 30 | 31 | def upload_fileobj(fp, s3_path, bucket_name=BUCKET_NAME, sse_key=BUCKET_SSE_KEY, verbose=True): 32 | s3_path = str(s3_path) 33 | 34 | if verbose: 35 | logger.info(f"Uploading local file object -> {bucket_name}:{s3_path}") 36 | 37 | session = boto3.session.Session() 38 | session.client("s3").upload_fileobj(fp, bucket_name, s3_path, ExtraArgs=_get_extra_args(sse_key)) 39 | 40 | 41 | def download_fileobj(s3_path, fp, bucket_name=BUCKET_NAME, sse_key=BUCKET_SSE_KEY, verbose=True): 42 | s3_path = str(s3_path) 43 | 44 | if verbose: 45 | logger.info(f"Downloading file object from {bucket_name}:{s3_path}") 46 | 47 | session = boto3.session.Session() 48 | session.client("s3").download_fileobj(bucket_name, s3_path, fp, ExtraArgs=_get_extra_args(sse_key)) 49 | 50 | 51 | def download_file(s3_path, local_filepath, bucket_name=BUCKET_NAME, sse_key=BUCKET_SSE_KEY, overwrite=True): 52 | local_filepath = str(local_filepath) 53 | s3_path = str(s3_path) 54 | 55 | if Path(local_filepath).exists() and not overwrite: 56 | logger.info(f"{local_filepath} exists and overwrite=False, skipping") 57 | else: 58 | Path(local_filepath).parent.mkdir(parents=True, exist_ok=True) 59 | logger.info(f"Downloading file from {bucket_name}:{s3_path} -> {local_filepath}") 60 | 61 | session = boto3.session.Session() 62 | session.client("s3").download_file( 63 | bucket_name, s3_path, local_filepath, ExtraArgs=_get_extra_args(sse_key) 64 | ) 65 | -------------------------------------------------------------------------------- /tests/data/fail_deployment/fail_predictor.py: -------------------------------------------------------------------------------- 1 | class PythonPredictor: 2 | def __init__(self, config): 3 | raise RuntimeError('Intentional testing exception') 4 | 5 | def predict(self, payload): 6 | return dict(yes=True) 7 | -------------------------------------------------------------------------------- /tests/data/no_app_deployment/main.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from fastapi import FastAPI, status, Body 4 | from fastapi.responses import PlainTextResponse 5 | from yes_predictor import PythonPredictor 6 | 7 | my_app = FastAPI() 8 | my_app.ready = False 9 | my_app.predictor = None 10 | 11 | 12 | @my_app.on_event("startup") 13 | def startup(): 14 | with open("predictor_config.json", "r") as f: 15 | config = json.load(f) 16 | my_app.predictor = PythonPredictor(config) 17 | my_app.ready = True 18 | 19 | 20 | @my_app.get("/healthz") 21 | def healthz(): 22 | if my_app.ready: 23 | return PlainTextResponse("ok") 24 | return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) 25 | 26 | 27 | @my_app.post("/") 28 | def post_handler(payload: dict = Body(...)): 29 | return my_app.predictor.predict(payload) 30 | -------------------------------------------------------------------------------- /tests/data/no_app_deployment/yes_predictor.py: -------------------------------------------------------------------------------- 1 | class PythonPredictor: 2 | def __init__(self, config): 3 | pass 4 | 5 | def predict(self, payload): 6 | return dict(yes=True) 7 | -------------------------------------------------------------------------------- /tests/data/yes_deployment/redeploy_yes_predictor.py: -------------------------------------------------------------------------------- 1 | class PythonPredictor: 2 | def __init__(self, config): 3 | print(f"Initted predictor with config: {config}") 4 | pass 5 | 6 | def predict(self, payload): 7 | return dict(yes=False) -------------------------------------------------------------------------------- /tests/data/yes_deployment/yes_predictor.py: -------------------------------------------------------------------------------- 1 | class PythonPredictor: 2 | def __init__(self, config): 3 | print(f"Initted predictor with config: {config}") 4 | pass 5 | 6 | def predict(self, payload): 7 | return dict(yes=True) 8 | -------------------------------------------------------------------------------- /tests/integration_tests.py: -------------------------------------------------------------------------------- 1 | import io 2 | import logging 3 | import os 4 | import pickle 5 | from concurrent.futures import ThreadPoolExecutor 6 | from time import sleep 7 | from unittest import skip 8 | from uuid import uuid4 9 | 10 | from cortex_serving_client import s3 11 | 12 | logging.basicConfig( 13 | format="%(asctime)s : %(levelname)s : %(threadName)-10s : %(name)s : %(message)s", level=logging.INFO 14 | ) 15 | 16 | 17 | from unittest.mock import patch 18 | from requests import post 19 | import unittest 20 | 21 | from cortex_serving_client.cortex_client import get_cortex_client_instance, NOT_DEPLOYED_STATUS, JOB_STATUS_SUCCEEDED, \ 22 | KIND_BATCH_API 23 | from cortex_serving_client.deployment_failed import DeploymentFailed, DEPLOYMENT_TIMEOUT_FAIL_TYPE, \ 24 | DEPLOYMENT_JOB_NOT_DEPLOYED_FAIL_TYPE, DEPLOYMENT_ERROR_FAIL_TYPE 25 | 26 | logger = logging.getLogger(__name__) 27 | 28 | 29 | class IntegrationTests(unittest.TestCase): 30 | """ 31 | These tests require Docker and Cortex. 32 | """ 33 | 34 | cortex = get_cortex_client_instance( 35 | cortex_env='cortex-serving-client-test', # if you create cluster by using example/cluster.yaml 36 | pg_user='cortex_test', 37 | pg_password='cortex_test', 38 | pg_db='cortex_test', 39 | ) 40 | 41 | @staticmethod 42 | def _get_deployment_dict(name, env=None): 43 | if env is None: 44 | env = {} 45 | return { 46 | "name": name, 47 | "project_name": "test", 48 | "predictor_path": "yes_predictor", 49 | "pod": { 50 | "containers": [ 51 | { 52 | "compute": {"cpu": '200m', "mem": f"{0.1}Gi"}, 53 | "env": env, 54 | } 55 | ], 56 | }, 57 | "node_groups": ['ng-spot-1'] 58 | } 59 | 60 | @skip 61 | def test_thread_safety(self): 62 | def get_deploy_dict(a): 63 | deployment = IntegrationTests._get_deployment_dict("yes-api") 64 | cortex_ = get_cortex_client_instance( 65 | cortex_env='cortex-serving-client-test', # if you create cluster by using example/cluster.yaml 66 | pg_user='cortex_test', 67 | pg_password='cortex_test', 68 | pg_db='cortex_test', 69 | ) 70 | x = cortex_._prepare_deployment(deployment, "./data/yes_deployment") 71 | return len(x) 72 | 73 | with ThreadPoolExecutor(max_workers=40) as e: 74 | futures = [] 75 | for _ in range(100): 76 | futures.append(e.submit(get_deploy_dict, 1)) 77 | 78 | for f in futures: 79 | logger.info(f.result()) # necessary to get the exceptions to the main thread 80 | 81 | def test_deploy_yes(self): 82 | deployment = self._get_deployment_dict("yes-api") 83 | 84 | with self.cortex.deploy_temporarily( 85 | deployment, 86 | deploy_dir="./data/yes_deployment", 87 | print_logs=True, 88 | api_timeout_sec=10 * 60, 89 | ) as get_result: 90 | for _ in range(10): 91 | logger.info(f"Sending request to test logging terminated correctly ...") 92 | invalid_result = post(get_result.endpoint, data='invalid_json', headers={'content-type': 'application/json'}) 93 | self.assertEqual(invalid_result.status_code, 400) 94 | result = post(get_result.endpoint, json={}).json() 95 | sleep(1) 96 | 97 | # extra delete can occur, should not cause failure. Non-force deletes are tested in other cases. 98 | self.cortex.delete(deployment['name'], force=True) 99 | 100 | self.assertTrue(result['yes']) 101 | self.assertEqual(self.cortex.get(deployment['name']).status, NOT_DEPLOYED_STATUS) 102 | 103 | def test_redeploy(self): 104 | deployment = self._get_deployment_dict("yes-api") 105 | 106 | with self.cortex.deploy_temporarily( 107 | deployment, 108 | deploy_dir="./data/yes_deployment", 109 | print_logs=True, 110 | api_timeout_sec=10 * 60, 111 | ) as get_result: 112 | result = post(get_result.endpoint, json={}).json() 113 | self.assertTrue(result['yes']) 114 | sleep(1) 115 | 116 | # redeploy test 117 | logger.info(f"\n --- Testing redeploy --- ") 118 | with open("./data/yes_deployment/yes_predictor.py", 'rt') as f: 119 | orig_code = f.readlines() 120 | try: 121 | with open("./data/yes_deployment/redeploy_yes_predictor.py", 'rt') as f: 122 | redeploy_code = f.readlines() 123 | with open("./data/yes_deployment/yes_predictor.py", 'wt') as f: 124 | f.writelines(redeploy_code) 125 | logger.info(f"Changed code in yes_predictor.py, calling deploy again ...") 126 | 127 | deployment = self._get_deployment_dict("yes-api") 128 | self.cortex.deploy_single( 129 | deployment, 130 | deploy_dir="./data/yes_deployment", 131 | print_logs=True, 132 | api_timeout_sec=10 * 60, 133 | ) 134 | 135 | for _ in range(10): 136 | logger.info(f"Sending request to test logging terminated and code has been updated ...") 137 | result = post(get_result.endpoint, json={}).json() 138 | self.assertEqual(result['yes'], False) 139 | sleep(1) 140 | finally: 141 | with open("./data/yes_deployment/yes_predictor.py", 'wt') as f: 142 | f.writelines(orig_code) 143 | logger.info(f"Reset code in yes_predictor.py to original.") 144 | 145 | # extra delete can occur, should not cause failure. Non-force deletes are tested in other cases. 146 | self.cortex.delete(deployment['name'], force=True) 147 | 148 | self.assertEqual(self.cortex.get(deployment['name']).status, NOT_DEPLOYED_STATUS) 149 | 150 | def test_deploy_no_predictor(self): 151 | deployment = self._get_deployment_dict("no-predictor-api") 152 | 153 | with self.assertRaises(FileNotFoundError): 154 | with self.cortex.deploy_temporarily( 155 | deployment, 156 | deploy_dir="data/no_predictor_deployment", 157 | api_timeout_sec=10 * 60, 158 | ) as get_result: 159 | self.fail(f'Deployment should fail but {get_result.status}.') 160 | 161 | def test_deploy_no_app(self): 162 | # also tests that if main.py is present it will be used instead of the default one 163 | deployment = self._get_deployment_dict("no-app-api") 164 | 165 | with self.assertRaises(AssertionError): 166 | with self.cortex.deploy_temporarily( 167 | deployment, 168 | deploy_dir="./data/no_app_deployment", 169 | api_timeout_sec=10 * 60, 170 | ) as get_result: 171 | self.fail(f'Deployment should fail but {get_result.status}.') 172 | 173 | def test_deploy_fail(self): 174 | deployment = self._get_deployment_dict("fail-api") 175 | deployment["predictor_path"] = "fail_predictor" 176 | 177 | try: 178 | with patch(target="cortex_serving_client.cortex_client.CORTEX_DEFAULT_DEPLOYMENT_TIMEOUT", new=5*60): 179 | with self.cortex.deploy_temporarily( 180 | deployment, 181 | deploy_dir="./data/fail_deployment", 182 | api_timeout_sec=5 * 60, 183 | print_logs=True, 184 | deployment_timeout_sec=5 * 60, 185 | ) as get_result: 186 | self.fail(f'Deployment should fail but {get_result.status}.') 187 | 188 | except DeploymentFailed as e: 189 | self.assertEqual(e.failure_type, DEPLOYMENT_ERROR_FAIL_TYPE) 190 | 191 | self.assertEqual(self.cortex.get(deployment['name']).status, NOT_DEPLOYED_STATUS) 192 | 193 | def test_deploy_timeout_fail(self): 194 | deployment = self._get_deployment_dict("timeout-api") 195 | 196 | try: 197 | with patch(target="cortex_serving_client.cortex_client.CORTEX_DEFAULT_DEPLOYMENT_TIMEOUT", new=0): 198 | with self.cortex.deploy_temporarily( 199 | deployment, 200 | deploy_dir="./data/yes_deployment", 201 | api_timeout_sec=10 * 60, 202 | deployment_timeout_sec=0 203 | ) as get_result: 204 | self.fail(f'Deployment should fail but {get_result.status}.') 205 | 206 | except DeploymentFailed as e: 207 | self.assertEqual(e.failure_type, DEPLOYMENT_TIMEOUT_FAIL_TYPE) 208 | 209 | self.assertEqual(self.cortex.get(deployment['name']).status, NOT_DEPLOYED_STATUS) 210 | 211 | def test_deploy_job_yes(self): 212 | env = { 213 | 'CSC_BUCKET_NAME': os.environ["CSC_BUCKET_NAME"], 214 | 'CSC_S3_SSE_KEY': os.environ['CSC_S3_SSE_KEY'] 215 | } 216 | test_id = str(uuid4()) 217 | deployment = self._get_deployment_dict("job-yes", env) 218 | deployment['kind'] = KIND_BATCH_API 219 | deployment["predictor_path"] = "batch_yes_predictor" 220 | deployment['pod']['containers'][0]['config'] = dict(test_id=test_id) 221 | 222 | job_spec = { 223 | "workers": 2, 224 | "item_list": {"items": [1, 2, 3, 4], "batch_size": 2}, 225 | } 226 | job_result = self.cortex.deploy_batch_api_and_run_job( 227 | deployment, 228 | job_spec, 229 | deploy_dir="./data/batch_yes_deployment", 230 | api_timeout_sec=10 * 60, 231 | print_logs=True, 232 | verbose=True, 233 | ) 234 | assert job_result.status == JOB_STATUS_SUCCEEDED 235 | self.assertEqual(self.cortex.get(deployment['name']).status, NOT_DEPLOYED_STATUS) 236 | 237 | with io.BytesIO() as fp: 238 | s3.download_fileobj(f'test/batch-yes/{test_id}.json', fp) 239 | logger.info(f'read the test {test_id} {fp.read().decode()}') 240 | 241 | for batch in [[1, 2], [3, 4]]: 242 | s3_path = f'test/batch-yes/{sum(batch)}.json' 243 | with io.BytesIO() as fp: 244 | s3.download_fileobj(s3_path, fp) 245 | fp.seek(0) 246 | result = pickle.load(fp) 247 | 248 | logger.info(f"{s3_path}: {result}") 249 | assert result['yes'] 250 | 251 | with io.BytesIO() as fp: 252 | pickle.dump({'yes': False, 'info': 'this should be rewritten!'}, fp) 253 | fp.seek(0) 254 | s3.upload_fileobj(fp, s3_path) 255 | logger.info(f"Replaced {s3_path} with dummy file.") 256 | 257 | def test_deploy_job_worker_error(self): 258 | deployment = self._get_deployment_dict("job-worker-err") 259 | deployment['kind'] = KIND_BATCH_API 260 | deployment["predictor_path"] = "fail_batch_predictor" 261 | 262 | job_spec = { 263 | "workers": 1, 264 | "item_list": {"items": [1, 2, 3, 4], "batch_size": 2}, 265 | } 266 | try: 267 | with patch(target="cortex_serving_client.cortex_client.N_RETRIES_BATCH_JOB", new=2): 268 | job_result = self.cortex.deploy_batch_api_and_run_job( 269 | deployment, 270 | job_spec, 271 | deploy_dir="./data/batch_fail_deployment", 272 | api_timeout_sec=10 * 60, 273 | print_logs=True, 274 | verbose=True, 275 | ) 276 | except DeploymentFailed as e: 277 | self.assertEqual(e.failure_type, DEPLOYMENT_JOB_NOT_DEPLOYED_FAIL_TYPE) 278 | 279 | self.assertEqual(self.cortex.get(deployment['name']).status, NOT_DEPLOYED_STATUS) 280 | 281 | 282 | 283 | 284 | -------------------------------------------------------------------------------- /tests/test_cortex_client.py: -------------------------------------------------------------------------------- 1 | # Basic logging config 2 | import logging 3 | 4 | import requests 5 | 6 | logging.basicConfig( 7 | format="%(asctime)s : %(levelname)s : %(threadName)-10s : %(name)s : %(message)s", level=logging.DEBUG, 8 | ) 9 | 10 | import subprocess 11 | import unittest 12 | from cortex_serving_client.cortex_client import _verbose_command_wrapper, \ 13 | NOT_DEPLOYED_STATUS 14 | from cortex_serving_client.retry_utils import create_always_retry_session 15 | 16 | 17 | class CortexClientTest(unittest.TestCase): 18 | 19 | def test_subprocess_timeout(self): 20 | try: 21 | p = subprocess.run(['grep', 'test'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=1) 22 | print(p) 23 | 24 | except subprocess.TimeoutExpired as e: 25 | print(f'{e} with stdout: "{e.output}" and stderr: "{e.stderr}"') 26 | 27 | def test_not_deployed(self): 28 | _verbose_command_wrapper(['cat', NOT_DEPLOYED_STATUS], allow_non_0_return_code_on_stdout_sub_strs=[NOT_DEPLOYED_STATUS], timeout=1, sleep_base_retry_sec=0.1) 29 | try: 30 | _verbose_command_wrapper(['cat', NOT_DEPLOYED_STATUS], timeout=1, sleep_base_retry_sec=0.1) 31 | self.fail() 32 | 33 | except ValueError as e: 34 | pass 35 | 36 | def test_request_retry(self): 37 | with self.assertRaises(requests.exceptions.ConnectionError): 38 | with create_always_retry_session(backoff_sec=0) as session: 39 | # this does not simulate problematic scenario of whitelist methods and read-error urllib3.util.retry.Retry.increment 40 | session.post("http://127.0.0.1:33333") 41 | -------------------------------------------------------------------------------- /tests/test_printable_chars.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from cortex_serving_client.printable_chars import remove_non_printable 4 | 5 | 6 | class PrintableCharsTest(unittest.TestCase): 7 | 8 | def test_remove_non_printable(self): 9 | self.assertEqual('test test text text', remove_non_printable('test test \b text \btext')) 10 | --------------------------------------------------------------------------------