├── .gitignore ├── CLA.md ├── LICENSE ├── README.md └── components └── inference-service ├── README.md ├── build.sh ├── charts ├── Chart.yaml ├── templates │ ├── _helpers.tpl │ ├── monaiinferenceservice-clusterrole.yaml │ ├── monaiinferenceservice-clusterrolebinding.yaml │ ├── monaiinferenceservice-deployment.yaml │ ├── monaiinferenceservice-payload-volume-claim.yaml │ ├── monaiinferenceservice-payload-volume.yaml │ ├── monaiinferenceservice-service.yaml │ └── monaiinferenceservice-serviceaccount.yaml └── values.yaml ├── dockerfile ├── docs ├── design-diagram.jpg └── spec.md ├── examples └── example.py ├── monaiinference ├── __init__.py ├── handler │ ├── __init__.py │ ├── config.py │ ├── kubernetes.py │ └── payload.py └── main.py ├── requirements-dev.txt ├── requirements.txt ├── setup.py └── tests └── test.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Editors 2 | .vscode/ 3 | .idea/ 4 | 5 | # Vagrant 6 | .vagrant/ 7 | 8 | # Mac/OSX 9 | .DS_Store 10 | 11 | # Windows 12 | Thumbs.db 13 | 14 | # Source for the following rules: https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore 15 | # Byte-compiled / optimized / DLL files 16 | __pycache__/ 17 | *.py[cod] 18 | *$py.class 19 | 20 | # C extensions 21 | *.so 22 | 23 | # Distribution / packaging 24 | .Python 25 | build/ 26 | develop-eggs/ 27 | dist/ 28 | downloads/ 29 | eggs/ 30 | .eggs/ 31 | lib/ 32 | lib64/ 33 | parts/ 34 | sdist/ 35 | var/ 36 | wheels/ 37 | *.egg-info/ 38 | .installed.cfg 39 | *.egg 40 | MANIFEST 41 | 42 | # PyInstaller 43 | # Usually these files are written by a python script from a template 44 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 45 | *.manifest 46 | *.spec 47 | 48 | # Installer logs 49 | pip-log.txt 50 | pip-delete-this-directory.txt 51 | 52 | # Unit test / coverage reports 53 | htmlcov/ 54 | .tox/ 55 | .nox/ 56 | .coverage 57 | .coverage.* 58 | .cache 59 | nosetests.xml 60 | coverage.xml 61 | *.cover 62 | .hypothesis/ 63 | .pytest_cache/ 64 | 65 | # Translations 66 | *.mo 67 | *.pot 68 | 69 | # Django stuff: 70 | *.log 71 | local_settings.py 72 | db.sqlite3 73 | 74 | # Flask stuff: 75 | instance/ 76 | .webassets-cache 77 | 78 | # Scrapy stuff: 79 | .scrapy 80 | 81 | # Sphinx documentation 82 | docs/_build/ 83 | 84 | # PyBuilder 85 | target/ 86 | 87 | # Jupyter Notebook 88 | .ipynb_checkpoints 89 | 90 | # IPython 91 | profile_default/ 92 | ipython_config.py 93 | 94 | # pyenv 95 | .python-version 96 | 97 | # celery beat schedule file 98 | celerybeat-schedule 99 | 100 | # SageMath parsed files 101 | *.sage.py 102 | 103 | # Environments 104 | .env 105 | .venv/ 106 | env/ 107 | venv/ 108 | ENV/ 109 | env.bak/ 110 | venv.bak/ 111 | 112 | # Spyder project settings 113 | .spyderproject 114 | .spyproject 115 | 116 | # Rope project settings 117 | .ropeproject 118 | 119 | # mkdocs documentation 120 | /site 121 | 122 | # mypy 123 | .mypy_cache/ 124 | .dmypy.json 125 | dmypy.json -------------------------------------------------------------------------------- /CLA.md: -------------------------------------------------------------------------------- 1 | ## Individual Contributor License Agreement (CLA) 2 | 3 | **Thank you for submitting your contributions to this project.** 4 | 5 | By signing this CLA, you agree that the following terms apply to all of your past, present and future contributions 6 | to the project. 7 | 8 | ### License. 9 | 10 | You hereby represent that all present, past and future contributions are governed by the 11 | [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0) 12 | copyright statement. 13 | 14 | This entails that to the extent possible under law, you transfer all copyright and related or neighboring rights 15 | of the code or documents you contribute to the project itself or its maintainers. 16 | Furthermore you also represent that you have the authority to perform the above waiver 17 | with respect to the entirety of you contributions. 18 | 19 | ### Moral Rights. 20 | 21 | To the fullest extent permitted under applicable law, you hereby waive, and agree not to 22 | assert, all of your “moral rights” in or relating to your contributions for the benefit of the project. 23 | 24 | ### Third Party Content. 25 | 26 | If your Contribution includes or is based on any source code, object code, bug fixes, configuration changes, tools, 27 | specifications, documentation, data, materials, feedback, information or other works of authorship that were not 28 | authored by you (“Third Party Content”) or if you are aware of any third party intellectual property or proprietary 29 | rights associated with your Contribution (“Third Party Rights”), 30 | then you agree to include with the submission of your Contribution full details respecting such Third Party 31 | Content and Third Party Rights, including, without limitation, identification of which aspects of your 32 | Contribution contain Third Party Content or are associated with Third Party Rights, the owner/author of the 33 | Third Party Content and Third Party Rights, where you obtained the Third Party Content, and any applicable 34 | third party license terms or restrictions respecting the Third Party Content and Third Party Rights. For greater 35 | certainty, the foregoing obligations respecting the identification of Third Party Content and Third Party Rights 36 | do not apply to any portion of a Project that is incorporated into your Contribution to that same Project. 37 | 38 | ### Representations. 39 | 40 | You represent that, other than the Third Party Content and Third Party Rights identified by 41 | you in accordance with this Agreement, you are the sole author of your Contributions and are legally entitled 42 | to grant the foregoing licenses and waivers in respect of your Contributions. If your Contributions were 43 | created in the course of your employment with your past or present employer(s), you represent that such 44 | employer(s) has authorized you to make your Contributions on behalf of such employer(s) or such employer 45 | (s) has waived all of their right, title or interest in or to your Contributions. 46 | 47 | ### Disclaimer. 48 | 49 | To the fullest extent permitted under applicable law, your Contributions are provided on an "as is" 50 | basis, without any warranties or conditions, express or implied, including, without limitation, any implied 51 | warranties or conditions of non-infringement, merchantability or fitness for a particular purpose. You are not 52 | required to provide support for your Contributions, except to the extent you desire to provide support. 53 | 54 | ### No Obligation. 55 | 56 | You acknowledge that the maintainers of this project are under no obligation to use or incorporate your contributions 57 | into the project. The decision to use or incorporate your contributions into the project will be made at the 58 | sole discretion of the maintainers or their authorized delegates. 59 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | project-monai 3 |

4 | 5 | 💡 If you want to know more about MONAI Deploy WG vision, overall structure, and guidelines, please read [MONAI Deploy](https://github.com/Project-MONAI/monai-deploy) main repo first. 6 | 7 | # MONAI Deploy Application Server (DEPRECATED - NOT TO BE USED NOR REFERENCED) 8 | [![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](LICENSE) 9 | 10 | Application server that will run [MAPs](https://github.com/Project-MONAI/monai-deploy/blob/main/guidelines/monai-application-package.md) (MONAI Application Package). 11 | 12 | First version's (v0.1) scope will include a basic component called the MONAI Inference Service ([MIS](./components/inference-service/README.md)). 13 | 14 | MIS is a RESTful Service which supports [MONAI workloads](https://github.com/Project-MONAI/monai-deploy/blob/main/guidelines/monai-workloads.md#synchronous-computational-workload) that can be completed within the timeframe of a single HTTP request/response. 15 | 16 | ## Contributing 17 | 18 | For guidance on making a contribution to MONAI Deploy App Server, see the [contributing guidelines](https://github.com/Project-MONAI/monai-deploy/blob/main/CONTRIBUTING.md). 19 | 20 | ## Community 21 | 22 | To participate, please join the MONAI Deploy App Server weekly meetings on the [calendar](https://calendar.google.com/calendar/u/0/embed?src=c_954820qfk2pdbge9ofnj5pnt0g@group.calendar.google.com&ctz=America/New_York) and review the [meeting notes](https://docs.google.com/document/d/1wY-WyJNDox5Wk1yKOZulHsV48Y-XnSltDoiDfTI6s_4/edit?usp=sharing). 23 | 24 | Join the conversation on Twitter [@ProjectMONAI](https://twitter.com/ProjectMONAI) or join our [Slack channel](https://forms.gle/QTxJq3hFictp31UM9). 25 | 26 | Ask and answer questions over on [MONAI Deploy App Server's GitHub Discussions tab](https://github.com/Project-MONAI/monai-deploy-app-server/discussions). 27 | 28 | ## Links 29 | 30 | - Website: 31 | - Code: 32 | - Project tracker: 33 | - Issue tracker: 34 | -------------------------------------------------------------------------------- /components/inference-service/README.md: -------------------------------------------------------------------------------- 1 | # MONAI Inference Service 2 | 3 | [![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](https://opensource.org/licenses/Apache-2.0) 4 | 5 | MONAI Inference Service(MIS) is a server that runs MONAI Application Packages [MAP](https://github.com/Project-MONAI/monai-deploy/blob/main/guidelines/monai-application-package.md) in a [Kubernetes](https://kubernetes.io/) cluster. It shares the same 6 | principles with [MONAI](https://github.com/Project-MONAI). 7 | 8 | ## Glossary 9 | 10 | 1. [Features](#features) 11 | 2. [Installation](#installation) 12 | - [Building the MIS Container](#building-the-mis-container) 13 | - [Helm Chart Configuration](#helm-chart-configuration) 14 | - [Helm Chart Deployment](#helm-chart-deployment) 15 | 3. [Submitting Inference Requests](#submitting-inference-requests) 16 | 17 | ## Features 18 | 19 | > _The codebase is currently under active development._ 20 | 21 | - Register a MAP in the Helm Charts of MIS. 22 | - Upload inputs via a REST API request and make them available to the MAP container. 23 | - Provision resources for the MAP container. 24 | - Provide outputs of the MAP container to the client which made the request. 25 | 26 | ## Installation 27 | 28 | MIS supports following OS with **GPU/CUDA** enabled. 29 | 30 | - Ubuntu 31 | 32 | MIS is intended to be deployed as a microservice in a [Kubernetes](https://kubernetes.io/) cluster. 33 | 34 | ### Building the MIS Container 35 | 36 | To build the MIS container, you can simply run: 37 | ```bash 38 | ./build.sh 39 | ``` 40 | 41 | To build the MIS container manually, you can run: 42 | ```bash 43 | docker build -f dockerfile -t monai/inference-service:0.1 . 44 | ``` 45 | 46 | ### Helm Chart Configuration 47 | Helm charts are located in the charts folder. 48 | 49 | All helm chart configuration values are listed in the `values.yaml` file in the charts folder. 50 | 51 | #### MIS Image and Tag 52 | MIS container image can be set in the `monaiInferenceService` field of the images section. 53 | 54 | The container tag for MIS can be set in the `monaiInferenceServiceTag` of the images section. 55 | 56 | #### MIS Kubernetes Service Type 57 | MIS supports two Kubernetes [service types](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) types: NodePort and ClusterIp. 58 | 59 | This can be set in the `serviceType` field of the server section. 60 | 61 | The default value of `serviceType` is `NodePort`. 62 | 63 | #### MIS Node Port 64 | The node port can be set in the `nodePort` field of the server section. If the `serviceType` is set to `NodePort`, the IP address of the machine on which MIS is deployed along with the node port can be used to reach the MIS. 65 | 66 | #### MIS Target Port 67 | The target port can be set in the `targetPort` field of the server section. Regardless of service type, if a client is on a machine belonging to the Kubernetes cluster on which MIS is deployed, cluster IP of the MIS kubernetes service along with the target port can be used to reach the MIS. 68 | 69 | You can obtain the cluster IP of the MIS Kubernetes service by doing a `kubectl get svc`. 70 | 71 | For example, 72 | ```bash 73 | user@hostname:~$ kubectl get svc 74 | NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE 75 | kubernetes ClusterIP 10.96.0.1 443/TCP 8d 76 | monai-inference-service NodePort 10.97.138.32 8000:32000/TCP 4s 77 | ``` 78 | 79 | Under the entry `monai-inference-service`, note the IP registered under the `CLUSTER-IP` section. This is the Cluster IP of the MIS. 80 | 81 | #### MIS Volume Host Path 82 | To register the host path on which the payload volume for the MAP resides, record the host path in the `hostVolumePath` field of the `payloadService` sub-section of the `server` section. Please make sure that this directory has read, write, and execute permissions for the user, group, and all other users `rwxrwxrwx` (Running `chmod 777 ` will achomplish this). 83 | 84 | #### MAP Configuration 85 | The `map` sub-section in the `server` section has all the configuration values for the MAP. 86 | - urn: This represents the container "\:\" to be deployed by MIS. For example, `urn: ubuntu:latest`. 87 | - entrypoint: String value which defines entry point command for MAP Container. For example, `entrypoint: "/bin/echo Hello"`. 88 | - cpu: Integer value which defines the CPU limit assigned to the MAP container. This value can not be less than 1. For example, `cpu: 1`. 89 | - memory: Integer value in Megabytes which defines the Memory limit assigned to the MAP container. This value can not be less than 256. For example, `memory: 8192`. 90 | - gpu: Integer value which defines the number of GPUs assigned to the MAP container. This value can not be less than 0. For example, `gpu: 0`. 91 | - inputPath: Input directory path of MAP Container. For example, `inputPath: "/var/monai/input"`. An environment variable `MONAI_INPUTPATH` is mounted in the MAP container with it's value equal to the one provided for this field. 92 | - outputPath: Output directory path of MAP Container. For example, `outputPath: "/var/monai/output"`. An environment variable `MONAI_OUTPUTPATH` is mounted in the MAP container with it's value equal to the one provided for this field. 93 | - modelPath: Model directory path of MAP Container. For example, `modelPath: "/opt/monai/models"`. This is an optional field. An environment variable `MONAI_MODELPATH` is mounted in the MAP container with it's value equal to the one provided for this field. 94 | 95 | ### Helm Chart Deployment 96 | 97 | In order to install the helm chart, please run: 98 | ```bash 99 | helm install monai ./charts 100 | ``` 101 | 102 | ## Submitting Inference Requests 103 | #### Making a request with `curl` 104 | 105 | With MIS running, a user can make an inference request to the service using the `/upload` POST endpoint with the **cluster IP** and **port** from running `kubectl get svc` and a compressed .zip file containing all the input payload files (eg. input.zip) 106 | 107 | #### Usage: 108 | 109 | 110 | curl -X 'POST' 'http://`:8000 OR :32000`/upload/' \\ \ 111 |     -H 'accept: application/json' \\ \ 112 |     -H 'Content-Type: multipart/form-data' \\ \ 113 |     -F 'file=@``;type=application/x-zip-compressed' \\ \ 114 |     -o output.zip 115 | 116 | For example: 117 | ```bash 118 | curl -X 'POST' 'http://10.97.138.32:8000/upload/' \ 119 | -H 'accept: application/json' \ 120 | -H 'Content-Type: multipart/form-data' \ 121 | -F 'file=@input.zip;type=application/x-zip-compressed' \ 122 | -o output.zip 123 | ``` 124 | 125 | To view the FastAPI generated UI for an instance of MIS, have the service running and then on any browser, navigate to `http://HOST_IP:32000/docs` (ex. http://10.110.21.31:32000/docs) 126 | -------------------------------------------------------------------------------- /components/inference-service/build.sh: -------------------------------------------------------------------------------- 1 | docker build -f dockerfile -t monai/inference-service:0.1 . -------------------------------------------------------------------------------- /components/inference-service/charts/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | apiVersion: v1 13 | description: A Helm chart for Kubernetes 14 | name: monai-inference-service 15 | version: 0.1 -------------------------------------------------------------------------------- /components/inference-service/charts/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | {{/* vim: set filetype=mustache: */}} 13 | {{/* 14 | Expand the name of the chart. 15 | */}} 16 | {{- define "monai.name" -}} 17 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 18 | {{- end -}} 19 | 20 | {{/* 21 | Create a default fully qualified app name. 22 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 23 | If release name contains chart name it will be used as a full name. 24 | */}} 25 | {{- define "monai.fullname" -}} 26 | {{- if .Values.fullnameOverride -}} 27 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 28 | {{- else -}} 29 | {{- $name := default .Chart.Name .Values.nameOverride -}} 30 | {{- if contains $name .Release.Name -}} 31 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 32 | {{- else -}} 33 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 34 | {{- end -}} 35 | {{- end -}} 36 | {{- end -}} 37 | 38 | {{/* 39 | Create chart name and version as used by the chart label. 40 | */}} 41 | {{- define "monai.chart" -}} 42 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 43 | {{- end -}} 44 | -------------------------------------------------------------------------------- /components/inference-service/charts/templates/monaiinferenceservice-clusterrole.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | apiVersion: rbac.authorization.k8s.io/v1 13 | kind: ClusterRole 14 | metadata: 15 | name: {{ .Values.server.names.clusterRole }} 16 | rules: 17 | - apiGroups: 18 | - "" 19 | resources: 20 | - persistentvolumes 21 | - persistentvolumeclaims 22 | - pods 23 | - pods/exec 24 | - pods/log 25 | - pods/status 26 | verbs: 27 | - create 28 | - get 29 | - list 30 | - watch 31 | - update 32 | - patch 33 | - delete 34 | -------------------------------------------------------------------------------- /components/inference-service/charts/templates/monaiinferenceservice-clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | apiVersion: rbac.authorization.k8s.io/v1 13 | kind: ClusterRoleBinding 14 | metadata: 15 | name: {{ .Values.server.names.clusterRoleBinding }} 16 | roleRef: 17 | apiGroup: rbac.authorization.k8s.io 18 | kind: ClusterRole 19 | name: {{ .Values.server.names.clusterRole }} 20 | subjects: 21 | - kind: ServiceAccount 22 | name: {{ .Values.server.names.serviceAccount }} 23 | namespace: {{ .Release.Namespace }} -------------------------------------------------------------------------------- /components/inference-service/charts/templates/monaiinferenceservice-deployment.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | apiVersion: apps/v1 13 | kind: Deployment 14 | metadata: 15 | name: {{ .Values.server.names.deployment }} 16 | labels: 17 | app: {{ .Release.Name }}-inferenceservice 18 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 19 | release: {{ .Release.Name }} 20 | heritage: {{ .Release.Service }} 21 | spec: 22 | selector: 23 | matchLabels: 24 | app: {{ .Release.Name }}-inferenceservice 25 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 26 | release: {{ .Release.Name }} 27 | heritage: {{ .Release.Service }} 28 | template: 29 | metadata: 30 | labels: 31 | app: {{ .Release.Name }}-inferenceservice 32 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 33 | release: {{ .Release.Name }} 34 | heritage: {{ .Release.Service }} 35 | spec: 36 | {{- with .Values.images.pullSecrets }} 37 | imagePullSecrets: 38 | {{- toYaml . | nindent 8 }} 39 | {{- end }} 40 | serviceAccountName: {{ .Values.server.names.serviceAccount }} 41 | volumes: 42 | - name: {{ .Release.Name }}-volume 43 | persistentVolumeClaim: 44 | claimName: {{ .Values.server.names.volumeClaim }} 45 | containers: 46 | - name: inference-service 47 | image: "{{ .Values.images.monaiInferenceService }}:{{ .Values.images.monaiInferenceServiceTag }}" 48 | imagePullPolicy: IfNotPresent 49 | # Note that the container's payload storage path currently must be the same as the 50 | # host path, since any persistent volumes created for the MAP containers must also 51 | # point to the original host path. 52 | args: [ 53 | "--map-urn", "{{ .Values.server.map.urn }}", 54 | "--map-entrypoint", "{{ .Values.server.map.entrypoint }}", 55 | "--map-cpu", "{{ .Values.server.map.cpu }}", 56 | "--map-memory", "{{ .Values.server.map.memory }}", 57 | "--map-gpu", "{{ .Values.server.map.gpu }}", 58 | "--map-input-path", "{{ .Values.server.map.inputPath }}", 59 | "--map-output-path", "{{ .Values.server.map.outputPath }}", 60 | "--map-model-path", "{{ .Values.server.map.modelPath }}", 61 | "--payload-host-path", "{{ .Values.server.payloadService.hostVolumePath }}", 62 | "--port", "{{ .Values.server.targetPort }}"] 63 | ports: 64 | - name: apiservice-port 65 | containerPort: {{ .Values.server.targetPort }} 66 | protocol: TCP 67 | resources: 68 | requests: 69 | cpu: {{ .Values.server.map.cpu }} 70 | memory: "{{ .Values.server.map.memory }}Mi" 71 | ephemeral-storage: "1Gi" 72 | limits: 73 | ephemeral-storage: "10Gi" 74 | volumeMounts: 75 | - mountPath: {{ .Values.server.payloadService.hostVolumePath }} 76 | name: {{ .Release.Name }}-volume 77 | -------------------------------------------------------------------------------- /components/inference-service/charts/templates/monaiinferenceservice-payload-volume-claim.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | apiVersion: v1 13 | kind: PersistentVolumeClaim 14 | metadata: 15 | name: {{ .Values.server.names.volumeClaim }} 16 | spec: 17 | storageClassName: {{ .Values.server.names.storageClass }} 18 | accessModes: 19 | - ReadWriteOnce 20 | resources: 21 | requests: 22 | storage: "10Gi" 23 | -------------------------------------------------------------------------------- /components/inference-service/charts/templates/monaiinferenceservice-payload-volume.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | apiVersion: v1 13 | kind: PersistentVolume 14 | metadata: 15 | name: {{ .Values.server.names.volume }} 16 | labels: 17 | type: local 18 | spec: 19 | storageClassName: {{ .Values.server.names.storageClass }} 20 | capacity: 21 | storage: "10Gi" 22 | volumeMode: Filesystem 23 | accessModes: 24 | - ReadWriteOnce 25 | hostPath: 26 | path: {{ .Values.server.payloadService.hostVolumePath }} 27 | type: "DirectoryOrCreate" 28 | -------------------------------------------------------------------------------- /components/inference-service/charts/templates/monaiinferenceservice-service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | apiVersion: v1 13 | kind: Service 14 | metadata: 15 | name: {{ .Values.server.names.service }} 16 | labels: 17 | app: {{ .Release.Name }}-inferenceservice 18 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 19 | release: {{ .Release.Name }} 20 | heritage: {{ .Release.Service }} 21 | spec: 22 | type: {{ .Values.server.serviceType }} 23 | ports: 24 | - name: apiservice-port 25 | port: {{ .Values.server.targetPort }} 26 | nodePort: {{ .Values.server.nodePort }} 27 | targetPort: apiservice-port 28 | selector: 29 | app: {{ .Release.Name }}-inferenceservice 30 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 31 | release: {{ .Release.Name }} 32 | heritage: {{ .Release.Service }} 33 | -------------------------------------------------------------------------------- /components/inference-service/charts/templates/monaiinferenceservice-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | apiVersion: v1 13 | kind: ServiceAccount 14 | metadata: 15 | name: {{ .Values.server.names.serviceAccount }} -------------------------------------------------------------------------------- /components/inference-service/charts/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | # Default values for MONAI Inference Service. 13 | # This is a YAML-formatted file. 14 | # Declare variables to be passed into your templates. 15 | images: 16 | monaiInferenceService: monai/inference-service 17 | monaiInferenceServiceTag: 0.1 18 | 19 | ######################################################## 20 | # Configuration Values for MONAI Inference Service # 21 | ######################################################## 22 | 23 | server: 24 | names: 25 | clusterRole: monai-inference-service-cluster-role 26 | clusterRoleBinding: monai-inference-service-binding 27 | deployment: monai-inference-service 28 | service: monai-inference-service 29 | serviceAccount: monai-inference-service-service-account 30 | storageClass: monai-inference-service-storage-class 31 | volume: monai-inference-service-payload-volume 32 | volumeClaim: monai-inference-service-payload-volume-claim 33 | 34 | serviceType: NodePort # Alternatively: ClusterIp if only in cluster clients will exist 35 | nodePort: 32000 36 | pullSecrets: [] 37 | targetPort: 8000 38 | 39 | # Configuration for the payload service in the MONAI Inference Service. 40 | payloadService: 41 | # The path on the node running MONAI Inference Service where a payload will be stored. 42 | # The input directory and output directory that are created by MONAI Inference Service 43 | # will exist as a directory inside this path. 44 | # (e.g. "/monai/payload/input"). 45 | # Please make sure that this directory has read, write, and execute permissions for the user, 46 | # group, and all other users `rwxrwxrwx`. Running `chmod 777 ` will achomplish this. 47 | hostVolumePath: "/monai/payload" 48 | 49 | # MAP configuration. 50 | map: 51 | # MAP Container : to de deployed by MONAI Inference Service. 52 | # For example, urn: "ubuntu:latest" 53 | urn: ":" 54 | 55 | # String value which defines entry point command for MAP Container. 56 | # For example, entrypoint: "/bin/echo Hello" 57 | entrypoint: "" 58 | 59 | # Integer value which defines the CPU limit assigned to the MAP container. 60 | # This value can not be less than 1. 61 | cpu: 1 62 | 63 | # Integer value in Megabytes which defines the Memory limit assigned to the MAP container. 64 | # This value can not be less than 256. 65 | memory: 8192 66 | 67 | # Integer value which defines the number of GPUs assigned to the MAP container. 68 | # This value can not be less than 0. 69 | gpu: 0 70 | 71 | # Input directory path of MAP Container. 72 | # An environment variable `MONAI_INPUTPATH` is mounted in the MAP container 73 | # with it's value equal to the one provided for this field. 74 | inputPath: "/var/monai/input" 75 | 76 | # Output directory path of MAP Container. 77 | # An environment variable `MONAI_OUTPUTPATH` is mounted in the MAP container 78 | # with it's value equal to the one provided for this field. 79 | outputPath: "/var/monai/output" 80 | 81 | # Model directory path of MAP Container. 82 | # For example, modelPath: /opt/monai/models. 83 | # An environment variable `MONAI_MODELPATH` is mounted in the MAP container 84 | # with it's value equal to the one provided for this field. 85 | modelPath: "" 86 | -------------------------------------------------------------------------------- /components/inference-service/dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | FROM python:3.9-slim-buster 13 | LABEL "base"="python:3.9-slim-buster" 14 | 15 | ARG APTVER_CURL=7.64.0-4+deb10u2 16 | ARG APTVER_TRANSPORT_HTTPS=1.8.2.2 17 | ARG APTVER_GNUPG2=2.2.12-1+deb10u1 18 | 19 | RUN apt-get update \ 20 | && apt-get install -y --no-install-recommends \ 21 | apt-transport-https=${APTVER_TRANSPORT_HTTPS} \ 22 | gnupg2=${APTVER_GNUPG2} \ 23 | curl=${APTVER_CURL} \ 24 | && apt-get update \ 25 | && apt-get install --no-install-recommends -y libgssapi-krb5-2 \ 26 | build-essential \ 27 | unixodbc-dev 28 | 29 | RUN python -m pip install --upgrade pip 30 | 31 | ADD ./requirements.txt /monai_inference/requirements.txt 32 | RUN python -m pip install -r /monai_inference/requirements.txt 33 | 34 | ADD ./monaiinference /monai_inference/monaiinference 35 | 36 | ENV PYTHONPATH "${PYTHONPATH}:/monai_inference/" 37 | 38 | ENTRYPOINT ["/usr/local/bin/python", "/monai_inference/monaiinference/main.py"] 39 | -------------------------------------------------------------------------------- /components/inference-service/docs/design-diagram.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Project-MONAI/monai-deploy-app-server/aa84cc7f64177c524f5e825e5608011d16b0a8a6/components/inference-service/docs/design-diagram.jpg -------------------------------------------------------------------------------- /components/inference-service/docs/spec.md: -------------------------------------------------------------------------------- 1 | ## Overview 2 | As data scientists & application developers build AI models they need a way to deploy these applications in production. MONAI Inference Service will be used to deploy a MONAI application. This proposal documents the requirements and the design for the MONAI Inference Service (MIS). 3 | 4 | ## Goal 5 | The goal for this proposal is to enlist, prioritize and provide clarity on the requirements for MIS. Developers working on different software modules in MIS SHALL use this specification as a guideline when designing and implementing software for the Service. 6 | 7 | ## Success Criteria 8 | MIS SHALL provide a REST API for client to communicate with. 9 | 10 | MIS SHALL support configuration of the [MONAI Application Package(MAP)](https://github.com/Project-MONAI/monai-deploy/blob/main/guidelines/monai-application-package.md]) used to service inference requests. 11 | 12 | MIS SHALL provide an REST API to upload inputs to perform inferencing on. 13 | 14 | MIS SHALL return inference results as part of the response to the originating inference request. 15 | 16 | ## Requirements 17 | 18 | ### Support for Specific MONAI Workloads 19 | MIS SHALL support [MONAI workloads](https://github.com/Project-MONAI/monai-deploy/blob/main/guidelines/monai-workloads.md#synchronous-computational-workload) which can be completed within the timeframe of a single HTTP request/response. 20 | 21 | ### Deployable on MONAI Operating Environments 22 | MIS SHALL run on Development/Integration Server environments as defined in [MONAI Operating Environments](https://github.com/Project-MONAI/monai-deploy/blob/main/guidelines/MONAI-Operating-Environments.md#developmentintegration-server-environment). 23 | 24 | ### API Interface 25 | MIS SHALL provide a REST API which utilizes the functionality of the HTTP transport. 26 | 27 | ### Consistent and Robust Logging 28 | MIS SHALL provide consistent and robust logging about its operations. 29 | 30 | ### Register single MAP configuration before MIS startup 31 | MIS SHALL allow clients to provide MAP configuration as part of MIS' deployment configuration. 32 | 33 | ### Fulfill an inference request with uploaded file inputs 34 | MIS SHALL fulfill an inference request with uploaded file inputs. 35 | 36 | ### Provision resources for an inference request 37 | MIS SHALL provision CPU, memory, and GPU resources for an inference request as defined in the configuration. 38 | 39 | ### Provide results of inference request 40 | MIS SHALL provide results of inference request as a part of the response to the request. 41 | 42 | ### SHALL NOT persist request inputs or inference results 43 | MIS SHALL NOT persist inference request inputs or inference results beyond the lifetime of the originating inferencing request. 44 | 45 | ## Limitations 46 | MIS SHALL service inference requests one at a time. 47 | 48 | ## Design 49 | 50 | ### Mechanism for deployment 51 | MIS SHALL be deployed via a Helm chart in a Kubernetes cluster. 52 | 53 | ### Mechanism for MAP registration 54 | MIS SHALL use the deployment configuration specified in the Helm charts for registering the MAP to run. 55 | 56 | ### Mechanism for inference request handling 57 | - Clients submit an inference request using a POST REST API call to the MIS. 58 | - MIS SHALL decompress the inputs provided in the inference request into a location that can be accessed by the MAP container. 59 | - MIS SHALL deploy a [Kubernetes job](https://kubernetes.io/docs/concepts/workloads/controllers/job/) for the MAP container using the configuration specified in the Helm charts of the MIS. 60 | - MIS SHALL then monitor the progress of the Kubernetes job. 61 | - MIS SHALL return a success code [200 OK] (https://en.wikipedia.org/wiki/List_of_HTTP_status_codes) when the job completes along with a zipped version of the outputs. 62 | 63 | ![Block Diagram describing inference request workflow in MIS](./design-diagram.jpg) 64 | 65 | ### Detail data input and output formats 66 | - MIS SHALL only accept inputs in a `.zip` format. 67 | - The zipped file inputs SHALL be decompressed and loaded directly in the input folder of a MAP. 68 | - The output of a MAP SHALL be compressed by the MIS and sent back as a part of the response of the inference request. 69 | 70 | ### Mechanism for error handling 71 | - If clients submit an inference request when a request is currently being fulfilled, MIS SHALL return a response with the HTTP error code 500 along with the message denoting that another inference request is currently being fulfilled. 72 | - If the Kubernetes job does not complete within the timeout(50 seconds), MIS SHALL terminate the job and return the HTTP error code [500 Internal Server Error](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes) along with the message denoting that the inference request timed out. 73 | - If the Kubernetes job fails, MIS SHALL return a response with the HTTP error code 500 along with the message denoting that the MAP deployed for the inference request failed. 74 | 75 | ### Mechanism for logging 76 | MIS logs can be obtained from Kubernetes. 77 | -------------------------------------------------------------------------------- /components/inference-service/examples/example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Project-MONAI/monai-deploy-app-server/aa84cc7f64177c524f5e825e5608011d16b0a8a6/components/inference-service/examples/example.py -------------------------------------------------------------------------------- /components/inference-service/monaiinference/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | -------------------------------------------------------------------------------- /components/inference-service/monaiinference/handler/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | -------------------------------------------------------------------------------- /components/inference-service/monaiinference/handler/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | 13 | class ServerConfig: 14 | """Class that defines object to store MONAI Inference configuration specifications""" 15 | 16 | def __init__(self, map_urn: str, map_entrypoint: str, map_cpu: int, map_memory: int, 17 | map_gpu: int, map_input_path: str, map_output_path: str, map_model_path: str, 18 | payload_host_path: str): 19 | """Constructor for Payload Provider class 20 | 21 | Args: 22 | map_urn (str): MAP Container : to de deployed for inference 23 | map_entrypoint (str): Entry point command for MAP Container 24 | map_cpu (int): Maximum CPU cores needed by MAP Container 25 | map_memory (int): Maximum memory in Megabytes needed by MAP Container 26 | map_gpu (int): Maximum GPUs needed by MAP Container 27 | map_input_path (str): Input directory path of MAP Container 28 | map_output_path (str): Output directory path of MAP Container 29 | map_model_path (str): Model directory path of MAP Container 30 | payload_host_path (str): Host path of payload directory 31 | """ 32 | self.map_urn = map_urn 33 | self.map_entrypoint = map_entrypoint 34 | self.map_cpu = map_cpu 35 | self.map_memory = map_memory 36 | self.map_gpu = map_gpu 37 | self.map_input_path = map_input_path 38 | self.map_output_path = map_output_path 39 | self.map_model_path = map_model_path 40 | self.payload_host_path = payload_host_path 41 | -------------------------------------------------------------------------------- /components/inference-service/monaiinference/handler/kubernetes.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | import enum 13 | import logging 14 | import os 15 | import time 16 | from pathlib import Path 17 | 18 | from monaiinference.handler.config import ServerConfig 19 | 20 | from kubernetes import client 21 | from kubernetes.client import models 22 | 23 | API_VERSION_FOR_PODS = "v1" 24 | API_VERSION_FOR_PERSISTENT_VOLUME = "v1" 25 | API_VERSION_FOR_PERSISTENT_VOLUME_CLAIM = "v1" 26 | DEFAULT_NAMESPACE = "default" 27 | DEFAULT_STORAGE_SPACE = "10Gi" 28 | DIRECTORY_OR_CREATE = "DirectoryOrCreate" 29 | ENV_MONAI_INPUTPATH="MONAI_INPUTPATH" 30 | ENV_MONAI_OUTPUTPATH="MONAI_OUTPUTPATH" 31 | ENV_MONAI_MODELPATH="MONAI_MODELPATH" 32 | IF_NOT_PRESENT = "IfNotPresent" 33 | MAP = "map" 34 | MONAI = "monai" 35 | POD = "Pod" 36 | POD_NAME = "monai-pod" 37 | PERSISTENT_VOLUME = "PersistentVolume" 38 | PERSISTENT_VOLUME_CLAIM = "PersistentVolumeClaim" 39 | PERSISTENT_VOLUME_CLAIM_NAME = "monai-volume-claim" 40 | PERSISTENT_VOLUME_NAME = "monai-volume" 41 | READ_WRITE_ONCE = "ReadWriteOnce" 42 | RESTART_POLICY_NEVER = "Never" 43 | STORAGE = "storage" 44 | STORAGE_CLASS_NAME = "monai-storage-class" 45 | WAIT_TIME_FOR_POD_COMPLETION = 50 46 | 47 | logger = logging.getLogger('MIS_Kubernetes') 48 | 49 | 50 | class KubernetesHandler: 51 | """Class to handle interactions with kubernetes for fulflling an inference request.""" 52 | 53 | def __init__(self, config: ServerConfig): 54 | """Constructor of the base KubernetesHandler class 55 | 56 | Args: 57 | config (ServerConfig): Instance of ServerConfig class with MONAI Inference 58 | configuration specifications 59 | """ 60 | # Initialize kubernetes client and handler configuration. 61 | self.kubernetes_core_client = client.CoreV1Api() 62 | self.config = config 63 | 64 | def __build_resources_requests(self) -> models.V1ResourceRequirements: 65 | # Derive CPU, memory(in Megabytes) and GPU limits for container from handler configuration. 66 | limits = { 67 | "cpu": str(self.config.map_cpu), 68 | "memory": str(self.config.map_memory) + "Mi", 69 | "nvidia.com/gpu": str(self.config.map_gpu) 70 | } 71 | 72 | resources = models.V1ResourceRequirements(limits=limits) 73 | return resources 74 | 75 | def __build_container_template(self) -> models.V1Container: 76 | # Derive container POSIX input path for defining input mount. 77 | input_path = Path(os.path.join("/", self.config.map_input_path)).as_posix() 78 | 79 | # Define input volume mount. 80 | input_mount = models.V1VolumeMount( 81 | name=PERSISTENT_VOLUME_CLAIM_NAME, 82 | mount_path=input_path, 83 | sub_path=input_path[1:], 84 | read_only=True 85 | ) 86 | 87 | # Derive container POSIX output path for defining output mount. 88 | output_path = Path(os.path.join("/", self.config.map_output_path)).as_posix() 89 | 90 | # Define output volume mount. 91 | output_mount = models.V1VolumeMount( 92 | name=PERSISTENT_VOLUME_CLAIM_NAME, 93 | mount_path=output_path, 94 | sub_path=output_path[1:], 95 | ) 96 | 97 | # Build Shared Memory volume mount. 98 | shared_memory_volume_mount = models.V1VolumeMount( 99 | mount_path="/dev/shm", 100 | name="shared-memory", 101 | read_only=False 102 | ) 103 | 104 | input_env = models.V1EnvVar(name=ENV_MONAI_INPUTPATH, value=self.config.map_input_path) 105 | output_env = models.V1EnvVar(name=ENV_MONAI_OUTPUTPATH, value=self.config.map_output_path) 106 | model_env = models.V1EnvVar(name=ENV_MONAI_MODELPATH, value=self.config.map_model_path) 107 | 108 | # Build container object. 109 | container = models.V1Container( 110 | name=MAP, 111 | image=self.config.map_urn, 112 | command=self.config.map_entrypoint, 113 | image_pull_policy=IF_NOT_PRESENT, 114 | env=[input_env, output_env, model_env], 115 | resources=self.__build_resources_requests(), 116 | volume_mounts=[input_mount, output_mount, shared_memory_volume_mount] 117 | ) 118 | 119 | return container 120 | 121 | def __build_kubernetes_pod(self) -> models.V1Pod: 122 | container = self.__build_container_template() 123 | 124 | # Build pod object. 125 | pod = models.V1Pod( 126 | api_version=API_VERSION_FOR_PODS, 127 | kind=POD, 128 | metadata=models.V1ObjectMeta( 129 | name=POD_NAME, 130 | labels={ 131 | "pod-name": POD_NAME, 132 | "pod-type": MONAI 133 | } 134 | ), 135 | spec=models.V1PodSpec( 136 | containers=[container], 137 | restart_policy=RESTART_POLICY_NEVER, 138 | volumes=[ 139 | models.V1Volume( 140 | name=PERSISTENT_VOLUME_CLAIM_NAME, 141 | persistent_volume_claim=models.V1PersistentVolumeClaimVolumeSource( 142 | claim_name=PERSISTENT_VOLUME_CLAIM_NAME, 143 | ), 144 | ), 145 | models.V1Volume( 146 | name="shared-memory", 147 | empty_dir=models.V1EmptyDirVolumeSource 148 | ( 149 | medium="Memory", 150 | ) 151 | ) 152 | ] 153 | ) 154 | ) 155 | 156 | return pod 157 | 158 | def __build_kubernetes_persistent_volume(self) -> models.V1PersistentVolume: 159 | persistent_volume = models.V1PersistentVolume( 160 | api_version=API_VERSION_FOR_PERSISTENT_VOLUME, 161 | kind=PERSISTENT_VOLUME, 162 | metadata=models.V1ObjectMeta( 163 | name=PERSISTENT_VOLUME_NAME, 164 | labels={ 165 | "volume-type": MONAI 166 | } 167 | ), 168 | spec=models.V1PersistentVolumeSpec( 169 | access_modes=[READ_WRITE_ONCE], 170 | capacity={ 171 | STORAGE: DEFAULT_STORAGE_SPACE, 172 | }, 173 | host_path=models.V1HostPathVolumeSource( 174 | path=self.config.payload_host_path, 175 | type=DIRECTORY_OR_CREATE, 176 | ), 177 | storage_class_name=STORAGE_CLASS_NAME, 178 | ) 179 | ) 180 | 181 | return persistent_volume 182 | 183 | def __build_kubernetes_persistent_volume_claim(self) -> models.V1PersistentVolumeClaim: 184 | persistent_volume_claim = models.V1PersistentVolumeClaim( 185 | api_version=API_VERSION_FOR_PERSISTENT_VOLUME_CLAIM, 186 | kind=PERSISTENT_VOLUME_CLAIM, 187 | metadata=models.V1ObjectMeta( 188 | name=PERSISTENT_VOLUME_CLAIM_NAME, 189 | labels={ 190 | "volume-claim-type": MONAI 191 | } 192 | ), 193 | spec=models.V1PersistentVolumeClaimSpec( 194 | access_modes=[READ_WRITE_ONCE], 195 | resources=models.V1ResourceRequirements( 196 | requests={ 197 | STORAGE: DEFAULT_STORAGE_SPACE, 198 | } 199 | ), 200 | storage_class_name=STORAGE_CLASS_NAME, 201 | ) 202 | ) 203 | 204 | return persistent_volume_claim 205 | 206 | def create_kubernetes_pod(self): 207 | """Create a kubernetes pod and the Persistent Volume and Persistent Volume Claim needed by the pod. 208 | """ 209 | 210 | try: 211 | # Create a Kubernetes Persistent Volume. 212 | pv = self.__build_kubernetes_persistent_volume() 213 | self.kubernetes_core_client.create_persistent_volume(pv) 214 | logger.info(f'Created Persistent Volume {pv.metadata.name}') 215 | except Exception as e: 216 | logger.error(e, exc_info=True) 217 | raise e 218 | 219 | try: 220 | # Create a Kubernetes Persistent Volume Claim. 221 | pvc = self.__build_kubernetes_persistent_volume_claim() 222 | self.kubernetes_core_client.create_namespaced_persistent_volume_claim(namespace=DEFAULT_NAMESPACE, body=pvc) 223 | logger.info(f'Created Persistent Volume Claim {pvc.metadata.name}') 224 | except Exception as e: 225 | logger.error(e, exc_info=True) 226 | self.kubernetes_core_client.delete_persistent_volume(name=PERSISTENT_VOLUME_NAME) 227 | raise e 228 | 229 | try: 230 | # Create a Kubernetes Pod. 231 | pod = self.__build_kubernetes_pod() 232 | self.kubernetes_core_client.create_namespaced_pod( 233 | namespace=DEFAULT_NAMESPACE, 234 | body=pod 235 | ) 236 | 237 | logger.info(f'Created pod {pod.metadata.name}') 238 | except Exception as e: 239 | self.kubernetes_core_client.delete_namespaced_persistent_volume_claim( 240 | namespace=DEFAULT_NAMESPACE, name=PERSISTENT_VOLUME_CLAIM_NAME) 241 | self.kubernetes_core_client.delete_persistent_volume(name=PERSISTENT_VOLUME_NAME) 242 | logger.error(e, exc_info=True) 243 | raise e 244 | 245 | def delete_kubernetes_pod(self): 246 | """Delete a kubernetes pod and the Persistent Volume and Persistent Volume Claim created for the pod. 247 | """ 248 | 249 | # Delete the Kubernetes Pod, Persistent Volume Claim and Persistent Volume. 250 | try: 251 | self.kubernetes_core_client.delete_namespaced_pod(name=POD_NAME, namespace=DEFAULT_NAMESPACE) 252 | logger.info(f'Deleted pod {POD_NAME}') 253 | except Exception as e: 254 | logger.error(e, exc_info=True) 255 | 256 | try: 257 | self.kubernetes_core_client.delete_namespaced_persistent_volume_claim( 258 | namespace=DEFAULT_NAMESPACE, name=PERSISTENT_VOLUME_CLAIM_NAME) 259 | logger.info(f'Deleted Persistent Volume Claim {PERSISTENT_VOLUME_CLAIM_NAME}') 260 | except Exception as e: 261 | logger.error(e, exc_info=True) 262 | 263 | try: 264 | self.kubernetes_core_client.delete_persistent_volume(name=PERSISTENT_VOLUME_NAME) 265 | logger.info(f'Deleted Persistent Volume {PERSISTENT_VOLUME_NAME}') 266 | except Exception as e: 267 | logger.error(e, exc_info=True) 268 | 269 | def watch_kubernetes_pod(self): 270 | """Watch the status of kubernetes pod until it completes or it times out. 271 | 272 | Returns: 273 | PodStatus: Enum which denotes a pod status. 274 | """ 275 | polling_time = 1 276 | current_sleep_time = 0 277 | status = PodStatus.Pending 278 | 279 | # Check every `polling_time` seconds if pod has completed(successfully/failed). 280 | # If Pod does not complete within timeout, return last reported status(Pending/Running) of pod. 281 | # If pod is in a pending state with ImagePullBackOff error, then quit checking for pod status 282 | # and return error along with Pending status. 283 | 284 | while (current_sleep_time < WAIT_TIME_FOR_POD_COMPLETION): 285 | pod = self.kubernetes_core_client.read_namespaced_pod(name=POD_NAME, namespace=DEFAULT_NAMESPACE) 286 | if (pod.status is None): 287 | continue 288 | 289 | pod_status = pod.status.phase 290 | 291 | if (pod_status == "Pending"): 292 | status = PodStatus.Pending 293 | 294 | container_statuses = pod.status.container_statuses 295 | if (container_statuses is None): 296 | continue 297 | 298 | container_status = container_statuses[0] 299 | if (container_status.state.waiting is not None and 300 | container_status.state.waiting.reason == "ImagePullBackOff"): 301 | logger.warning(f'Pod {POD_NAME} in Pending State: Image Pull Back Off') 302 | break 303 | elif (pod_status == "Running"): 304 | status = PodStatus.Running 305 | elif (pod_status == "Succeeded"): 306 | status = PodStatus.Succeeded 307 | break 308 | elif (pod_status == "Failed"): 309 | status = PodStatus.Failed 310 | break 311 | else: 312 | logger.warning(f'Unknown pod status {pod.status.phase}') 313 | 314 | time.sleep(polling_time) 315 | current_sleep_time += polling_time 316 | 317 | logger.info(f'Pod status is {status} after {current_sleep_time} seconds') 318 | 319 | return status 320 | 321 | 322 | class PodStatus(enum.Enum): 323 | Pending = 1, 324 | Running = 2, 325 | Succeeded = 3, 326 | Failed = 4 327 | -------------------------------------------------------------------------------- /components/inference-service/monaiinference/handler/payload.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | import logging 13 | import os 14 | import shutil 15 | import zipfile 16 | from pathlib import Path 17 | 18 | from fastapi import File, UploadFile 19 | from fastapi.responses import FileResponse 20 | 21 | logger = logging.getLogger('MIS_Payload') 22 | 23 | 24 | class PayloadProvider: 25 | """Class to handle interactions with payload I/O and Monai Inference Service 26 | shared volumes""" 27 | 28 | def __init__(self, host_path: str, input_path: str, output_path: str): 29 | """Constructor for Payload Provider class 30 | 31 | Args: 32 | host_path (str): Absolute path of shared volume for payloads 33 | input_path (str): Relative path of input sub-directory within shared volume for payloads 34 | output_path (str): Relative path of input sub-directory within shared volume for payloads 35 | """ 36 | self._host_path = host_path 37 | self._input_path = input_path.strip('/') 38 | self._output_path = output_path.strip('/') 39 | 40 | PayloadProvider.clean_directory(self._host_path) 41 | 42 | abs_input_path = Path(os.path.join(self._host_path, self._input_path)) 43 | abs_input_path.mkdir(parents=True, exist_ok=True) 44 | os.chmod(abs_input_path, 0o777) 45 | 46 | abs_output_path = Path(os.path.join(self._host_path, self._output_path)) 47 | abs_output_path.mkdir(parents=True, exist_ok=True) 48 | os.chmod(abs_output_path, 0o777) 49 | 50 | 51 | def upload_input_payload(self, file: UploadFile=File(...)): 52 | """Uploads and extracts input payload .zip provided by user to input folder within MIS container 53 | 54 | Args: 55 | file (UploadFile, optional): .zip file provided by user to be moved 56 | and extracted in shared volume directory for input payloads. Defaults to File(...). 57 | """ 58 | 59 | abs_input_path = os.path.join(self._host_path, self._input_path) 60 | # Clean input payload directory of any lingering content 61 | PayloadProvider.clean_directory(abs_input_path) 62 | 63 | abs_output_path = os.path.join(self._host_path, self._output_path) 64 | # Clean output payload directory of any lingering content 65 | PayloadProvider.clean_directory(abs_output_path) 66 | 67 | # Read contents of .zip file arguement and write it to input payload folder 68 | target_path = f'{abs_input_path}/{file.filename}' 69 | f = open(f'{target_path}', 'wb') 70 | content = file.file.read() 71 | f.write(content) 72 | f.close() 73 | 74 | # Extract contents of .zip into input payload folder 75 | with zipfile.ZipFile(target_path, 'r') as zip_ref: 76 | zip_ref.extractall(abs_input_path) 77 | 78 | # Remove compressed input payload .zip file 79 | os.remove(target_path) 80 | 81 | logger.info(f'Extracted {target_path} into {abs_input_path}') 82 | 83 | def stream_output_payload(self) -> FileResponse: 84 | """Compresses output payload directory and returns .zip as FileResponse object 85 | 86 | Returns: 87 | FileResponse: Asynchronous object for FastAPI to stream compressed .zip folder with 88 | the output payload from running the MONAI Application Package 89 | """ 90 | abs_output_path = os.path.join(self._host_path, self._output_path) 91 | abs_zip_path = os.path.join(self._host_path, 'output.zip') 92 | 93 | # Compress output payload directory into .zip file in root payload directory 94 | with zipfile.ZipFile(abs_zip_path, 'w', zipfile.ZIP_DEFLATED) as zip_file: 95 | for root_dir, dirs, files in os.walk(abs_output_path): 96 | for file in files: 97 | zip_file.write(os.path.join(root_dir, file), 98 | os.path.relpath(os.path.join(root_dir, file), 99 | os.path.join(abs_output_path, '..'))) 100 | 101 | logger.info(f'Compressed {abs_output_path} into {abs_zip_path}') 102 | 103 | # Move compressed .zip into output payload directory 104 | target_zip_path = os.path.join(abs_output_path, 'output.zip') 105 | shutil.move(abs_zip_path, target_zip_path) 106 | 107 | # Return stream of resulting .zip file using the FastAPI FileResponse object 108 | logger.info(f'Returning stream of {target_zip_path}') 109 | return FileResponse(target_zip_path) 110 | 111 | @staticmethod 112 | def clean_directory(dir_path: str): 113 | """Cleans contents of a directory, but does not delete directory itself 114 | 115 | Args: 116 | dir_path (str): Path to of directory to be cleaned 117 | """ 118 | 119 | deletion_files = [f for f in os.listdir(dir_path)] 120 | 121 | for f in deletion_files: 122 | deletion_path = os.path.join(dir_path, f) 123 | if os.path.isdir(deletion_path): 124 | shutil.rmtree(deletion_path) 125 | else: 126 | os.remove(deletion_path) 127 | -------------------------------------------------------------------------------- /components/inference-service/monaiinference/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | import argparse 13 | import logging 14 | from threading import Lock 15 | 16 | import uvicorn 17 | from fastapi import FastAPI, File, HTTPException, UploadFile 18 | from fastapi.middleware.cors import CORSMiddleware 19 | from fastapi.responses import FileResponse 20 | from kubernetes import config 21 | from starlette.middleware import Middleware 22 | from starlette.routing import Host 23 | 24 | from monaiinference.handler.config import ServerConfig 25 | from monaiinference.handler.kubernetes import KubernetesHandler, PodStatus 26 | from monaiinference.handler.payload import PayloadProvider 27 | 28 | MIS_HOST = "0.0.0.0" 29 | 30 | logging_config = { 31 | 'version': 1, 'disable_existing_loggers': True, 32 | 'formatters': {'default': {'()': 'uvicorn.logging.DefaultFormatter', 33 | 'fmt': '%(levelprefix)s %(message)s', 'use_colors': None}, 34 | 'access': {'()': 'uvicorn.logging.AccessFormatter', 35 | 'fmt': '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s'}}, 36 | 'handlers': {'default': {'formatter': 'default', 'class': 'logging.StreamHandler', 'stream': 'ext://sys.stderr'}, 37 | 'access': {'formatter': 'access', 'class': 'logging.StreamHandler', 'stream': 'ext://sys.stdout'}}, 38 | 'loggers': {'uvicorn': {'handlers': ['default'], 'level': 'INFO'}, 39 | 'uvicorn.error': {'level': 'INFO', 'handlers': ['default'], 'propagate': True}, 40 | 'uvicorn.access': {'handlers': ['access'], 'level': 'INFO', 'propagate': False}, 41 | 'MIS_Main': {'handlers': ['default'], 'level': 'INFO'}, 42 | 'MIS_Payload': {'handlers': ['default'], 'level': 'INFO'}, 43 | 'MIS_Kubernetes': {'handlers': ['default'], 'level': 'INFO'} 44 | }, 45 | } 46 | 47 | logger = logging.getLogger('MIS_Main') 48 | app = FastAPI( 49 | middleware=[ 50 | Middleware( 51 | CORSMiddleware, 52 | allow_origins=["*"], 53 | allow_credentials=True, 54 | allow_methods=["*"], 55 | allow_headers=["*"], 56 | ) 57 | ], 58 | ) 59 | request_mutex = Lock() 60 | 61 | 62 | def main(): 63 | """Driver method that parses arguements and intializes providers 64 | """ 65 | parser = argparse.ArgumentParser() 66 | parser.add_argument('--map-urn', type=str, required=True, 67 | help="MAP Container : to de deployed for inference") 68 | parser.add_argument('--map-entrypoint', type=str, required=True, 69 | help="Entry point command for MAP Container") 70 | parser.add_argument('--map-cpu', type=int, required=True, help="Maximum CPU cores needed by MAP Container") 71 | parser.add_argument('--map-memory', type=int, required=True, 72 | help="Maximum memory in Megabytes needed by MAP Container") 73 | parser.add_argument('--map-gpu', type=int, required=True, help="Maximum GPUs needed by MAP Container") 74 | parser.add_argument('--map-input-path', type=str, required=True, 75 | help="Input directory path of MAP Container") 76 | parser.add_argument('--map-output-path', type=str, required=True, 77 | help="Output directory path of MAP Container") 78 | parser.add_argument('--map-model-path', type=str, required=False, 79 | help="Model directory path of MAP Container") 80 | parser.add_argument('--payload-host-path', type=str, required=True, 81 | help="Host path of payload directory") 82 | parser.add_argument('--port', type=int, required=False, default=8000, 83 | help="Host port of MONAI Inference Service") 84 | 85 | args = parser.parse_args() 86 | 87 | if (args.map_cpu < 1): 88 | raise Exception(f'MAP cpu value can not be less than 1, provided value is \"{args.map_cpu}\"') 89 | if (args.map_gpu < 0): 90 | raise Exception(f'MAP gpu value can not be less than 0, provided value is \"{args.map_gpu}\"') 91 | if (args.map_memory < 256): 92 | raise Exception(f'MAP memory value can not be less than 256, provided value is \"{args.map_memory}\"') 93 | 94 | config.load_incluster_config() 95 | 96 | service_config = ServerConfig(args.map_urn, args.map_entrypoint.split(' '), args.map_cpu, 97 | args.map_memory, args.map_gpu, args.map_input_path, 98 | args.map_output_path, args.map_model_path, args.payload_host_path) 99 | kubernetes_handler = KubernetesHandler(service_config) 100 | payload_provider = PayloadProvider(args.payload_host_path, 101 | args.map_input_path, 102 | args.map_output_path) 103 | 104 | @app.post("/upload/") 105 | def upload_file(file: UploadFile = File(...)) -> FileResponse: 106 | """Defines REST POST Endpoint for Uploading input payloads. 107 | Will trigger inference job sequentially after uploading payload 108 | 109 | Args: 110 | file (UploadFile, optional): .zip file provided by user to be moved 111 | and extracted in shared volume directory for input payloads. Defaults to File(...). 112 | 113 | Returns: 114 | FileResponse: Asynchronous object for FastAPI to stream compressed .zip folder with 115 | the output payload from running the MONAI Application Package 116 | """ 117 | logger.info("/upload/ Request Received") 118 | if not request_mutex.acquire(False): 119 | logger.info("Request rejected as MIS is currently servicing another request") 120 | raise HTTPException( 121 | status_code=500, 122 | detail="Request timed out since MAP container's pod was in pending state after timeout") 123 | else: 124 | logger.info("Acquired resource lock") 125 | 126 | try: 127 | payload_provider.upload_input_payload(file) 128 | kubernetes_handler.create_kubernetes_pod() 129 | 130 | try: 131 | pod_status = kubernetes_handler.watch_kubernetes_pod() 132 | finally: 133 | kubernetes_handler.delete_kubernetes_pod() 134 | 135 | if (pod_status is PodStatus.Pending): 136 | logger.error("Request timed out since MAP container's pod was in pending state after timeout") 137 | raise HTTPException( 138 | status_code=500, 139 | detail="Request timed out since MAP container's pod was in pending state after timeout") 140 | elif (pod_status is PodStatus.Running): 141 | logger.error("Request timed out since MAP container's pod was in running state after timeout") 142 | raise HTTPException( 143 | status_code=500, 144 | detail="Request timed out since MAP container's pod was in running state after timeout") 145 | elif (pod_status is PodStatus.Failed): 146 | logger.info("Request failed since MAP container's pod failed") 147 | raise HTTPException(status_code=500, detail="Request failed since MAP container's pod failed") 148 | elif (pod_status is PodStatus.Succeeded): 149 | logger.info("MAP container's pod completed") 150 | return payload_provider.stream_output_payload() 151 | except Exception as e: 152 | logging.error(e, exc_info=True) 153 | finally: 154 | logger.info("Releasing resource lock") 155 | request_mutex.release() 156 | 157 | print(f'MAP URN: \"{args.map_urn}\"') 158 | print(f'MAP entrypoint: \"{args.map_entrypoint}\"') 159 | print(f'MAP cpu: \"{args.map_cpu}\"') 160 | print(f'MAP memory: \"{args.map_memory}\"') 161 | print(f'MAP gpu: \"{args.map_gpu}\"') 162 | print(f'MAP input path: \"{args.map_input_path}\"') 163 | print(f'MAP output path: \"{args.map_output_path}\"') 164 | print(f'MAP model path: \"{args.map_model_path}\"') 165 | print(f'payload host path: \"{args.payload_host_path}\"') 166 | print(f'MIS host: \"{MIS_HOST}\"') 167 | print(f'MIS port: \"{args.port}\"') 168 | 169 | uvicorn.run(app, host=MIS_HOST, port=args.port, log_config=logging_config) 170 | 171 | 172 | if __name__ == "__main__": 173 | main() 174 | -------------------------------------------------------------------------------- /components/inference-service/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | flake8 3 | autopep8 -------------------------------------------------------------------------------- /components/inference-service/requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi 2 | uvicorn 3 | python-multipart 4 | kubernetes==19.15.0 5 | -------------------------------------------------------------------------------- /components/inference-service/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 - 2021 MONAI Consortium 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | import setuptools 13 | import os 14 | 15 | release_version = "0.0.0" 16 | long_description = "" 17 | # with open("README.md", "r") as fh: 18 | # long_description = fh.read() 19 | 20 | # # Install required packages from requirements.txt file 21 | requirements_relative_path = "./requirements.txt" 22 | package_folder = os.path.dirname(os.path.realpath(__file__)) 23 | requirements_path = package_folder + requirements_relative_path 24 | install_requires = [] 25 | if os.path.isfile(requirements_path): 26 | with open(requirements_path) as f: 27 | install_requires = f.read().splitlines() 28 | 29 | setuptools.setup( 30 | name="monai-inference-service", 31 | author="MONAI Deploy", 32 | version=release_version, 33 | description="MONAI Infrence Service", 34 | long_description=long_description, 35 | long_description_content_type="text/markdown", 36 | url="https://docs.nvidia.com/clara/deploy/", 37 | install_requires=install_requires, 38 | packages=setuptools.find_packages('.'), 39 | entry_points={ 40 | 'console_scripts': [ 41 | 'mis = monaiinference.main:main' 42 | ] 43 | }, 44 | classifiers=[ 45 | "Programming Language :: Python :: 3", 46 | "Operating System :: OS Independent", 47 | ], 48 | python_requires='>=3.6', 49 | ) 50 | -------------------------------------------------------------------------------- /components/inference-service/tests/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Project-MONAI/monai-deploy-app-server/aa84cc7f64177c524f5e825e5608011d16b0a8a6/components/inference-service/tests/test.py --------------------------------------------------------------------------------