├── .coveragerc_py36 ├── .coveragerc_py37 ├── .coveragerc_py38 ├── .flake8 ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ ├── documentation-request.md │ └── feature_request.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── NOTICE ├── README.rst ├── VERSION ├── artifacts ├── config.properties └── mms-entrypoint.py ├── buildspec-container-pr.yml ├── buildspec-deploy.yml ├── buildspec-release.yml ├── buildspec.yml ├── docker ├── 1.4.0 │ └── final │ │ ├── Dockerfile.cpu │ │ ├── Dockerfile.eia │ │ └── Dockerfile.gpu ├── 1.4.1 │ ├── py2 │ │ ├── Dockerfile.cpu │ │ ├── Dockerfile.eia │ │ ├── Dockerfile.gpu │ │ ├── config.properties │ │ ├── dockerd-entrypoint.sh │ │ └── mms-entrypoint.py │ └── py3 │ │ ├── Dockerfile.cpu │ │ ├── Dockerfile.eia │ │ ├── Dockerfile.gpu │ │ ├── config.properties │ │ └── mms-entrypoint.py ├── 1.5.1 │ ├── py2 │ │ └── Dockerfile.eia │ └── py3 │ │ └── Dockerfile.eia ├── 1.6.0 │ ├── py2 │ │ ├── Dockerfile.cpu │ │ └── Dockerfile.gpu │ └── py3 │ │ ├── Dockerfile.cpu │ │ └── Dockerfile.gpu └── artifacts │ ├── config.properties │ └── mms-entrypoint.py ├── setup.cfg ├── setup.py ├── src └── sagemaker_mxnet_serving_container │ ├── __init__.py │ ├── default_inference_handler.py │ ├── handler_service.py │ ├── mxnet_module_transformer.py │ ├── serving.py │ └── utils.py ├── test ├── conftest.py ├── container │ ├── 1.4.1 │ │ └── Dockerfile.dlc.eia │ └── 1.7.0 │ │ ├── Dockerfile.dlc.cpu │ │ ├── Dockerfile.dlc.gpu │ │ └── Dockerfile.mxnet.cpu ├── integration │ ├── __init__.py │ ├── local │ │ ├── test_default_model_fn.py │ │ └── test_hosting.py │ └── sagemaker │ │ ├── __init__.py │ │ ├── test_batch_transform.py │ │ ├── test_elastic_inference.py │ │ ├── test_hosting.py │ │ └── timeout.py ├── resources │ ├── default_handlers │ │ ├── model.tar.gz │ │ └── model │ │ │ ├── code │ │ │ └── empty_module.py │ │ │ ├── model-0000.params │ │ │ ├── model-shapes.json │ │ │ └── model-symbol.json │ ├── dummy_hosting │ │ └── code │ │ │ └── dummy_hosting_module.py │ └── mnist │ │ ├── images │ │ ├── 04.json │ │ └── 07.csv │ │ └── model │ │ └── model.tar.gz ├── unit │ ├── test_default_inference_handler.py │ ├── test_handler_service.py │ ├── test_mxnet_module_transformer.py │ ├── test_serving.py │ └── test_utils.py └── utils │ ├── __init__.py │ ├── image_utils.py │ └── local_mode_utils.py └── tox.ini /.coveragerc_py36: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | timid = True 4 | 5 | [report] 6 | exclude_lines = 7 | pragma: no cover 8 | pragma: py3 no cover 9 | if six.PY2 10 | elif six.PY2 11 | 12 | partial_branches = 13 | pragma: no cover 14 | pragma: py3 no cover 15 | if six.PY3 16 | elif six.PY3 17 | 18 | show_missing = True 19 | 20 | fail_under = 90 21 | -------------------------------------------------------------------------------- /.coveragerc_py37: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | timid = True 4 | 5 | [report] 6 | exclude_lines = 7 | pragma: no cover 8 | pragma: py3 no cover 9 | if six.PY2 10 | elif six.PY2 11 | 12 | partial_branches = 13 | pragma: no cover 14 | pragma: py3 no cover 15 | if six.PY3 16 | elif six.PY3 17 | 18 | show_missing = True 19 | 20 | fail_under = 90 21 | -------------------------------------------------------------------------------- /.coveragerc_py38: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | timid = True 4 | 5 | [report] 6 | exclude_lines = 7 | pragma: no cover 8 | pragma: py3 no cover 9 | if six.PY2 10 | elif six.PY2 11 | 12 | partial_branches = 13 | pragma: no cover 14 | pragma: py3 no cover 15 | if six.PY3 16 | elif six.PY3 17 | 18 | show_missing = True 19 | 20 | fail_under = 90 21 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | application_import_names = image_utils, local_mode_utils, integration, sagemaker_mxnet_serving_container, test, timeout, utils 3 | import-order-style = google 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: File a report to help us reproduce and fix the problem 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To reproduce** 14 | A clear, step-by-step set of instructions to reproduce the bug. 15 | 16 | **Expected behavior** 17 | A clear and concise description of what you expected to happen. 18 | 19 | **Screenshots or logs** 20 | If applicable, add screenshots or logs to help explain your problem. 21 | 22 | **System information** 23 | A description of your system. Please provide: 24 | - **Toolkit version**: 25 | - **Framework version**: 26 | - **Python version**: 27 | - **CPU or GPU**: 28 | - **Custom Docker image (Y/N)**: 29 | 30 | **Additional context** 31 | Add any other context about the problem here. 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Ask a question related to SageMaker 4 | url: https://stackoverflow.com/questions/tagged/amazon-sagemaker 5 | about: Use Stack Overflow to ask and answer questions about SageMaker 6 | - name: Ask a question related to MXNet 7 | url: https://discuss.mxnet.io 8 | about: Use the MXNet Forum to ask and answer questions about MXNet 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Documentation request 3 | about: Request improved documentation 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **What did you find confusing? Please describe.** 11 | A clear and concise description of what you found confusing. Ex. I tried to [...] but I didn't understand how to [...] 12 | 13 | **Describe how documentation can be improved** 14 | A clear and concise description of where documentation was lacking and how it can be improved. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the documentation request here. 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest new functionality for this toolkit 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the feature you'd like** 11 | A clear and concise description of the functionality you want. 12 | 13 | **How would this feature be used? Please describe.** 14 | A clear and concise description of the use case for this feature. Please provide an example, if possible. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | 6 | By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | **/*.egg-info 3 | **/__pycache__ 4 | .pytest_cache 5 | **/*.pyc 6 | .tox/* 7 | .coverage 8 | test/resources/local_mode_lock 9 | test-toolkit/resources/local_mode_lock 10 | .idea/* 11 | *~ 12 | .DS_Store 13 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v1.5.5 (2022-05-23) 4 | 5 | ### Bug Fixes and Other Changes 6 | 7 | * Fix input reshape in default handler 8 | 9 | ## v1.5.4 (2022-01-08) 10 | 11 | ### Bug Fixes and Other Changes 12 | 13 | * stop pinning sagemaker-inference toolkit version (remove upper bound) 14 | 15 | ## v1.5.3 (2021-12-17) 16 | 17 | ### Bug Fixes and Other Changes 18 | 19 | * Upgrade MMS_VERSION to 1.1.6 in Dockerfiles. 20 | 21 | ## v1.5.2 (2021-12-16) 22 | 23 | ### Bug Fixes and Other Changes 24 | 25 | * Bump SM inference toolkit version to 1.5.7 26 | 27 | ## v1.5.1 (2021-04-12) 28 | 29 | ### Bug Fixes and Other Changes 30 | 31 | * Add exception for import_module 32 | 33 | ## v1.5.0 (2020-09-14) 34 | 35 | ### Features 36 | 37 | * support mxnet 1.7.0 38 | 39 | ### Bug Fixes and Other Changes 40 | 41 | * drop py27, add py38 support 42 | 43 | ## v1.4.6 (2020-08-17) 44 | 45 | ### Bug Fixes and Other Changes 46 | 47 | * Add support for Accept headers with multiple MIME types 48 | 49 | ## v1.4.5 (2020-08-06) 50 | 51 | ### Bug Fixes and Other Changes 52 | 53 | * upgrade sagemaker-inference version up to 1.5.2 54 | 55 | ## v1.4.4.post1 (2020-06-25) 56 | 57 | ### Testing and Release Infrastructure 58 | 59 | * add issue templates 60 | 61 | ## v1.4.4.post0 (2020-06-16) 62 | 63 | ### Testing and Release Infrastructure 64 | 65 | * fix release build. 66 | * Make docker folder read only, remove unused tests, update repo name, remove incorrect documentation. 67 | 68 | ## v1.4.4 (2020-05-20) 69 | 70 | ### Bug Fixes and Other Changes 71 | 72 | * Remove unnecessary EI packages 73 | 74 | ## v1.4.3 (2020-05-19) 75 | 76 | ### Bug Fixes and Other Changes 77 | 78 | * Install openssl from source in py3 1.5.1 EI image 79 | 80 | ## v1.4.2 (2020-05-18) 81 | 82 | ### Bug Fixes and Other Changes 83 | 84 | * Add Dockerfile MXNet 1.5.1 for Elastic Inference 85 | 86 | ## v1.4.1 (2020-05-14) 87 | 88 | ### Bug Fixes and Other Changes 89 | 90 | * Skip GluonCV and GluonNLP for EIA containers 91 | * Revert "infra: remove unused scripts." 92 | 93 | ## v1.4.0 (2020-05-12) 94 | 95 | ### Features 96 | 97 | * Python 3.7 support 98 | 99 | ### Testing and Release Infrastructure 100 | 101 | * remove unused scripts. 102 | 103 | ## v1.3.3.post0 (2020-04-30) 104 | 105 | ### Testing and Release Infrastructure 106 | 107 | * use tox in buildspecs 108 | 109 | ## v1.3.3 (2020-04-28) 110 | 111 | ### Bug Fixes and Other Changes 112 | 113 | * Improve error message in handler_service.py 114 | 115 | ## v1.3.2 (2020-04-01) 116 | 117 | ### Bug Fixes and Other Changes 118 | 119 | * upgrade inference-toolkit version 120 | 121 | ## v1.3.1 (2020-04-01) 122 | 123 | ### Bug Fixes and Other Changes 124 | 125 | * add model_dir to python path at service initialization 126 | * add gluoncv 127 | 128 | ## v1.3.0 (2020-03-30) 129 | 130 | ### Features 131 | 132 | * install mxnet-inference toolkit from PyPI. 133 | 134 | ## v1.2.6.post0 (2020-03-24) 135 | 136 | ### Testing and Release Infrastructure 137 | 138 | * refactor toolkit tests. 139 | 140 | ## v1.2.6 (2020-03-11) 141 | 142 | ### Bug Fixes and Other Changes 143 | 144 | * Test MME hosting with MultiDataModel 145 | 146 | ## v1.2.5 (2020-03-09) 147 | 148 | ### Bug Fixes and Other Changes 149 | 150 | * Upgrade the version of sagemaker-inference 151 | 152 | ## v1.2.4 (2020-03-04) 153 | 154 | ### Bug Fixes and Other Changes 155 | 156 | * Modify HandlerService to use context model_dir 157 | 158 | ## v1.2.3 (2020-02-20) 159 | 160 | ### Bug Fixes and Other Changes 161 | 162 | * copy all tests to test-toolkit folder. 163 | 164 | ## v1.2.2 (2020-02-19) 165 | 166 | ### Bug Fixes and Other Changes 167 | 168 | * change remove multi-model label from dockerfiles 169 | 170 | ## v1.2.1 (2020-02-17) 171 | 172 | ### Bug Fixes and Other Changes 173 | 174 | * update: Update license URL 175 | 176 | ## v1.2.0 (2020-02-12) 177 | 178 | ### Features 179 | 180 | * Add release to PyPI. Change package name to sagemaker-mxnet-inference. 181 | 182 | ### Bug Fixes and Other Changes 183 | 184 | * Add GluonNLP 185 | * Update AWS-MXNet version to 1.6.0 - official release of 1.6 186 | * Update build artifacts 187 | * Revert "change: install python-dateutil explicitly for botocore (#81)" 188 | * pin in setuptools in py2 containers 189 | * make build context the directory with the given dockerfile 190 | * update copyright year in license header 191 | * add comments about sagemaker-inference version 192 | * release 1.6.0 Dockerfiles 193 | * install python-dateutil explicitly for botocore 194 | * use regional endpoint for STS in builds 195 | * Update toolkit version 196 | 197 | ### Testing and Release Infrastructure 198 | 199 | * properly fail build if has-matching-changes fails 200 | * properly fail build if has-matching-changes fails 201 | 202 | ## v1.1.5 (2019-10-22) 203 | 204 | ### Bug fixes and other changes 205 | 206 | * update instance type region availability 207 | 208 | ## v1.1.4 (2019-09-26) 209 | 210 | ### Bug fixes and other changes 211 | 212 | * build context in release build 213 | * make consistent build context for py versions 214 | * Revert "Update build context on 1.4.1 EI dockerfiles (#64)" 215 | * a typo in build_all script 216 | * Update build context on 1.4.1 EI dockerfiles 217 | * correct typo in buildspec 218 | * enable PR build 219 | * separate pip upgrade from other installs 220 | * New py2 dlc dockerfiles 221 | * New py3 dlc dockerfiles 222 | 223 | ## v1.1.3.post0 (2019-08-29) 224 | 225 | ### Documentation changes 226 | 227 | * Update the build instructions to match reality. 228 | 229 | ## v1.1.3 (2019-08-28) 230 | 231 | ### Bug fixes and other changes 232 | 233 | * retry mms until it's ready 234 | 235 | ## v1.1.2 (2019-08-17) 236 | 237 | ### Bug fixes and other changes 238 | 239 | * split cpu and gpu tests for deployments in buildspec-release.yml 240 | * add missing placeholder in test for non-eia test command. 241 | 242 | ## v1.1.1 (2019-08-15) 243 | 244 | ### Bug fixes and other changes 245 | 246 | * fix flake8 247 | * update no-p2 regions and no-p3 regions. 248 | * Skipping EIA test if accelerator type is None. 249 | 250 | ## v1.1.0 (2019-07-18) 251 | 252 | ### Features 253 | 254 | * add MXNet 1.4.1 Dockerfiles 255 | 256 | ### Bug fixes and other changes 257 | 258 | * use Python 2 build logic for EI images during release 259 | * add missing files needed for building MXNet 1.4.1 Python 2 images 260 | * configure flake8 to ignore the docker/ directory 261 | 262 | ## v1.0.6 (2019-07-03) 263 | 264 | ### Bug fixes and other changes 265 | 266 | * add retries to all integ tests commands in buildspec-release.yml 267 | 268 | ## v1.0.5 (2019-07-03) 269 | 270 | ### Bug fixes and other changes 271 | 272 | * fix account number for EI deployment test in buildspec-release.yml 273 | 274 | ## v1.0.4 (2019-07-01) 275 | 276 | ### Bug fixes and other changes 277 | 278 | * remove unnecessary pytest marks 279 | * add retries to remote integ tests in buildspec-release.yml 280 | * update tests to except pytest's ExceptionInfo object 281 | * parametrize Python version and processor type in integ tests 282 | 283 | ## v1.0.3 (2019-06-28) 284 | 285 | ### Bug fixes and other changes 286 | 287 | * fix account number in deployment test command 288 | 289 | ## v1.0.2 (2019-06-28) 290 | 291 | ### Bug fixes and other changes 292 | 293 | * remove nonexistent EI GPU images from buildspec-release.yml 294 | 295 | ## v1.0.1 (2019-06-27) 296 | 297 | ### Bug fixes and other changes 298 | 299 | * add release buildspec 300 | * Fix SageMaker integration tests 301 | * skip GPU test in regions with limited p2s or p3s 302 | * Add link to SageMaker integ test setup requirements to README 303 | * Add SageMaker Elastic Inference test 304 | 305 | ## v1.0.0 306 | 307 | Initial commit 308 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check [existing open](https://github.com/aws/sagemaker-mxnet-inference-toolkit/issues), or [recently closed](https://github.com/aws/sagemaker-mxnet-inference-toolkit/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/aws/sagemaker-mxnet-inference-toolkit/labels/help%20wanted) issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](https://github.com/aws/sagemaker-mxnet-inference-toolkit/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include src/sagemaker_mxnet_serving_container * 2 | 3 | include VERSION 4 | include LICENSE 5 | include README.rst 6 | 7 | prune test 8 | 9 | recursive-exclude * __pycache__ 10 | recursive-exclude * *.py[co] 11 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | sagemaker-mxnet-inference-toolkit 2 | Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ================================= 2 | SageMaker MXNet Inference Toolkit 3 | ================================= 4 | 5 | SageMaker MXNet Inference Toolkit is an open-source library for serving MXNet models on Amazon SageMaker. 6 | This library provides default pre-processing, predict and postprocessing for certain MXNet model types and 7 | utilizes the `SageMaker Inference Toolkit `__ 8 | for starting up the model server, which is responsible for handling inference requests. 9 | 10 | For training, see `SageMaker MXNet Training Toolkit `__. 11 | 12 | For the Dockerfiles used for building SageMaker MXNet Containers, see `AWS Deep Learning Containers `__. 13 | 14 | For information on using MXNet on Amazon SageMaker, please refer to the `SageMaker Python SDK documentation `__. 15 | 16 | Contributing 17 | ------------ 18 | 19 | Please read `CONTRIBUTING.md `__ 20 | for details on our code of conduct, and the process for submitting pull requests to us. 21 | 22 | License 23 | ------- 24 | 25 | SageMaker MXNet Containers is licensed under the Apache 2.0 License. 26 | It is copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 27 | The license is available at: http://aws.amazon.com/apache2.0/ 28 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 1.5.6.dev0 2 | -------------------------------------------------------------------------------- /artifacts/config.properties: -------------------------------------------------------------------------------- 1 | vmargs=-Xmx128m -XX:-UseLargePages -XX:+UseG1GC -XX:MaxMetaspaceSize=32M -XX:MaxDirectMemorySize=10m -XX:+ExitOnOutOfMemoryError 2 | model_store=/opt/ml/model 3 | load_models=ALL 4 | inference_address=http://0.0.0.0:8080 5 | management_address=http://0.0.0.0:8081 6 | -------------------------------------------------------------------------------- /artifacts/mms-entrypoint.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | import shlex 15 | import subprocess 16 | import sys 17 | 18 | 19 | if sys.argv[1] == 'serve': 20 | from sagemaker_mxnet_serving_container import serving 21 | serving.main() 22 | else: 23 | subprocess.check_call(shlex.split(' '.join(sys.argv[1:]))) 24 | 25 | # prevent docker exit 26 | subprocess.call(['tail', '-f', '/dev/null']) 27 | -------------------------------------------------------------------------------- /buildspec-container-pr.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | phases: 4 | pre_build: 5 | commands: 6 | - PR_NUM=$(echo $CODEBUILD_SOURCE_VERSION | grep -o '[0-9]\+') 7 | - echo 'Pull request number:' $PR_NUM '. No value means this build is not from pull request.' 8 | 9 | build: 10 | commands: 11 | 12 | - error_cmd="echo 'In order to make changes to the docker files, please, use https://github.com/aws/deep-learning-containers repository.' && exit 1" 13 | - execute-command-if-has-matching-changes "$error_cmd" "docker/" 14 | -------------------------------------------------------------------------------- /buildspec-deploy.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | phases: 4 | build: 5 | commands: 6 | - PACKAGE_FILE="$CODEBUILD_SRC_DIR_ARTIFACT_1/sagemaker_mxnet_inference-*.tar.gz" 7 | 8 | # publish to pypi 9 | - publish-pypi-package $PACKAGE_FILE 10 | -------------------------------------------------------------------------------- /buildspec-release.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | phases: 4 | build: 5 | commands: 6 | # prepare the release (update versions, changelog etc.) 7 | - git-release --prepare 8 | 9 | # run linters 10 | - tox -e flake8,twine 11 | 12 | # run unit tests 13 | - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN= 14 | AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION= 15 | tox -e py36,py37,py38 -- test/unit 16 | 17 | # run local integ tests 18 | #- $(aws ecr get-login --no-include-email --region us-west-2) 19 | #- IGNORE_COVERAGE=- tox -e py27,py36 -- test/integ/local 20 | 21 | # run sagemaker integ tests 22 | #- IGNORE_COVERAGE=- tox -e py27,py36 -- test/integ/sagemaker 23 | 24 | # generate the distribution package 25 | - python3 setup.py sdist 26 | 27 | # publish the release to github 28 | - git-release --publish 29 | 30 | artifacts: 31 | files: 32 | - dist/sagemaker_mxnet_inference-*.tar.gz 33 | name: ARTIFACT_1 34 | discard-paths: yes 35 | -------------------------------------------------------------------------------- /buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | env: 4 | variables: 5 | FRAMEWORK_VERSION: '1.7.0' 6 | EIA_FRAMEWORK_VERSION: '1.4.1' 7 | CPU_INSTANCE_TYPE: 'ml.c4.xlarge' 8 | GPU_INSTANCE_TYPE: 'ml.p2.xlarge' 9 | EI_ACCELERATOR_TYPE: 'ml.eia1.medium' 10 | ECR_REPO: 'sagemaker-test' 11 | GITHUB_REPO: 'sagemaker-mxnet-serving-container' 12 | DLC_ACCOUNT: '763104351884' 13 | SETUP_FILE: 'setup_cmds.sh' 14 | SETUP_CMDS: '#!/bin/bash\npython3.6 -m pip install --upgrade pip\npython3.6 -m pip install -U -e .\npython3.6 -m pip install -U -e .[test]' 15 | 16 | phases: 17 | pre_build: 18 | commands: 19 | - start-dockerd 20 | - ACCOUNT=$(aws --region $AWS_DEFAULT_REGION sts --endpoint-url https://sts.$AWS_DEFAULT_REGION.amazonaws.com get-caller-identity --query 'Account' --output text) 21 | - PREPROD_IMAGE="$ACCOUNT.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$ECR_REPO" 22 | - PR_NUM=$(echo $CODEBUILD_SOURCE_VERSION | grep -o '[0-9]\+') 23 | - BUILD_ID="$(echo $CODEBUILD_BUILD_ID | sed -e 's/:/-/g')" 24 | - echo 'Pull request number:' $PR_NUM '. No value means this build is not from pull request.' 25 | 26 | build: 27 | commands: 28 | - TOX_PARALLEL_NO_SPINNER=1 29 | - PY_COLORS=0 30 | 31 | # run linters 32 | - tox -e flake8,twine 33 | 34 | # run unit tests 35 | - tox -e py36,py37,py38 test/unit 36 | 37 | # define tags 38 | - GENERIC_CPU_TAG="$FRAMEWORK_VERSION-mxnet-cpu-$BUILD_ID" 39 | - DLC_CPU_TAG="$FRAMEWORK_VERSION-dlc-cpu-$BUILD_ID" 40 | - DLC_GPU_TAG="$FRAMEWORK_VERSION-dlc-gpu-$BUILD_ID" 41 | - DLC_EIA_TAG="$EIA_FRAMEWORK_VERSION-dlc-eia-$BUILD_ID" 42 | 43 | # run local CPU integration tests (build and push the image to ECR repo) 44 | - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --build-image --push-image --dockerfile-type mxnet.cpu --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $GENERIC_CPU_TAG" 45 | - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" "artifacts/*" 46 | - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --build-image --push-image --dockerfile-type dlc.cpu --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $DLC_CPU_TAG" 47 | - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" "artifacts/*" 48 | 49 | # launch remote GPU instance 50 | - prefix='ml.' 51 | - instance_type=${GPU_INSTANCE_TYPE#"$prefix"} 52 | - create-key-pair 53 | - launch-ec2-instance --instance-type $instance_type --ami-name dlami-ubuntu 54 | 55 | # build GPU images because they are too big and take too long to build as part of the test 56 | - python3.6 setup.py sdist 57 | - build_dir="test/container/$FRAMEWORK_VERSION" 58 | - $(aws ecr get-login --registry-ids $DLC_ACCOUNT --no-include-email --region $AWS_DEFAULT_REGION) 59 | - docker build -f "$build_dir/Dockerfile.dlc.gpu" -t $PREPROD_IMAGE:$DLC_GPU_TAG --build-arg region=$AWS_DEFAULT_REGION . 60 | # push images to ECR 61 | - $(aws ecr get-login --registry-ids $ACCOUNT --no-include-email --region $AWS_DEFAULT_REGION) 62 | - docker push $PREPROD_IMAGE:$DLC_GPU_TAG 63 | 64 | # run GPU local integration tests 65 | - printf "$SETUP_CMDS" > $SETUP_FILE 66 | - dlc_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $DLC_GPU_TAG" 67 | - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$dlc_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\"" 68 | - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" "artifacts/*" 69 | 70 | # run CPU sagemaker integration tests 71 | - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $GENERIC_CPU_TAG" 72 | - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" "artifacts/*" 73 | - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $DLC_CPU_TAG" 74 | - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" "artifacts/*" 75 | 76 | # run GPU sagemaker integration tests 77 | - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $DLC_GPU_TAG" 78 | - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" "artifacts/*" 79 | 80 | # run EIA tests 81 | - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker/test_elastic_inference.py --build-image --push-image --dockerfile-type dlc.eia --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $EIA_FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $DLC_EIA_TAG --accelerator-type $EI_ACCELERATOR_TYPE" 82 | - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" "artifacts/*" 83 | 84 | finally: 85 | # shut down remote GPU instance 86 | - cleanup-gpu-instances 87 | - cleanup-key-pairs 88 | 89 | # remove ECR image 90 | - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GENERIC_CPU_TAG 91 | - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$DLC_CPU_TAG 92 | - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$DLC_GPU_TAG 93 | -------------------------------------------------------------------------------- /docker/1.4.0/final/Dockerfile.cpu: -------------------------------------------------------------------------------- 1 | ARG py_version 2 | 3 | FROM awsdeeplearningteam/mxnet-model-server:base-cpu-py$py_version 4 | 5 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 6 | 7 | RUN apt-get update && \ 8 | apt-get -y install --no-install-recommends \ 9 | libopencv-dev \ 10 | build-essential \ 11 | && rm -rf /var/lib/apt/lists/* 12 | 13 | WORKDIR / 14 | 15 | COPY dist/sagemaker_mxnet_serving_container.tar.gz /sagemaker_mxnet_serving_container.tar.gz 16 | 17 | RUN pip install --no-cache mxnet-mkl==1.4.0 \ 18 | keras-mxnet==2.2.4.1 \ 19 | onnx==1.4.1 \ 20 | /sagemaker_mxnet_serving_container.tar.gz && \ 21 | rm /sagemaker_mxnet_serving_container.tar.gz 22 | 23 | # This is here to make our installed version of OpenCV work. 24 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 25 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 26 | RUN ln -s /dev/null /dev/raw1394 27 | 28 | ENV PYTHONDONTWRITEBYTECODE=1 \ 29 | PYTHONUNBUFFERED=1 \ 30 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 31 | PYTHONIOENCODING=UTF-8 \ 32 | LANG=C.UTF-8 \ 33 | LC_ALL=C.UTF-8 34 | 35 | ENTRYPOINT [] 36 | -------------------------------------------------------------------------------- /docker/1.4.0/final/Dockerfile.eia: -------------------------------------------------------------------------------- 1 | ARG py_version 2 | 3 | FROM awsdeeplearningteam/mxnet-model-server:base-cpu-py$py_version 4 | 5 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 6 | 7 | RUN apt-get update && \ 8 | apt-get -y install --no-install-recommends \ 9 | libopencv-dev \ 10 | build-essential \ 11 | && rm -rf /var/lib/apt/lists/* 12 | 13 | WORKDIR / 14 | 15 | COPY dist/sagemaker_mxnet_serving_container.tar.gz /sagemaker_mxnet_serving_container.tar.gz 16 | 17 | RUN pip install --no-cache https://s3.amazonaws.com/amazonei-apachemxnet/amazonei_mxnet-1.4.0-py2.py3-none-manylinux1_x86_64.whl \ 18 | keras-mxnet==2.2.4.1 \ 19 | onnx==1.4.1 \ 20 | /sagemaker_mxnet_serving_container.tar.gz && \ 21 | rm /sagemaker_mxnet_serving_container.tar.gz 22 | 23 | # This is here to make our installed version of OpenCV work. 24 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 25 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 26 | RUN ln -s /dev/null /dev/raw1394 27 | 28 | ENV PYTHONDONTWRITEBYTECODE=1 \ 29 | PYTHONUNBUFFERED=1 \ 30 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 31 | PYTHONIOENCODING=UTF-8 \ 32 | LANG=C.UTF-8 \ 33 | LC_ALL=C.UTF-8 34 | 35 | ENTRYPOINT [] 36 | -------------------------------------------------------------------------------- /docker/1.4.0/final/Dockerfile.gpu: -------------------------------------------------------------------------------- 1 | ARG py_version 2 | 3 | FROM awsdeeplearningteam/mxnet-model-server:base-gpu-py$py_version 4 | 5 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 6 | 7 | RUN apt-get update && \ 8 | apt-get -y install --no-install-recommends \ 9 | libopencv-dev \ 10 | build-essential \ 11 | && rm -rf /var/lib/apt/lists/* 12 | 13 | WORKDIR / 14 | 15 | COPY dist/sagemaker_mxnet_serving_container.tar.gz /sagemaker_mxnet_serving_container.tar.gz 16 | 17 | RUN pip install --no-cache mxnet-cu92mkl==1.4.0 \ 18 | keras-mxnet==2.2.4.1 \ 19 | onnx==1.4.1 \ 20 | /sagemaker_mxnet_serving_container.tar.gz && \ 21 | rm /sagemaker_mxnet_serving_container.tar.gz 22 | 23 | # This is here to make our installed version of OpenCV work. 24 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 25 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 26 | RUN ln -s /dev/null /dev/raw1394 27 | 28 | ENV PYTHONDONTWRITEBYTECODE=1 \ 29 | PYTHONUNBUFFERED=1 \ 30 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 31 | PYTHONIOENCODING=UTF-8 \ 32 | LANG=C.UTF-8 \ 33 | LC_ALL=C.UTF-8 34 | 35 | ENTRYPOINT [] 36 | -------------------------------------------------------------------------------- /docker/1.4.1/py2/Dockerfile.cpu: -------------------------------------------------------------------------------- 1 | FROM awsdeeplearningteam/mxnet-model-server:base-cpu-py2.7 2 | 3 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 4 | 5 | RUN apt-get update && \ 6 | apt-get -y install --no-install-recommends \ 7 | libopencv-dev \ 8 | build-essential \ 9 | && rm -rf /var/lib/apt/lists/* 10 | 11 | WORKDIR / 12 | 13 | COPY sagemaker_mxnet_serving_container.tar.gz /sagemaker_mxnet_serving_container.tar.gz 14 | 15 | RUN pip install --no-cache mxnet-mkl==1.4.1 \ 16 | keras-mxnet==2.2.4.1 \ 17 | numpy==1.14.5 \ 18 | onnx==1.4.1 \ 19 | /sagemaker_mxnet_serving_container.tar.gz && \ 20 | rm /sagemaker_mxnet_serving_container.tar.gz 21 | 22 | # This is here to make our installed version of OpenCV work. 23 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 24 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 25 | RUN ln -s /dev/null /dev/raw1394 26 | 27 | ENV PYTHONDONTWRITEBYTECODE=1 \ 28 | PYTHONUNBUFFERED=1 \ 29 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 30 | PYTHONIOENCODING=UTF-8 \ 31 | LANG=C.UTF-8 \ 32 | LC_ALL=C.UTF-8 33 | 34 | ENTRYPOINT [] 35 | -------------------------------------------------------------------------------- /docker/1.4.1/py2/Dockerfile.eia: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | LABEL maintainer="Amazon AI" 4 | 5 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 6 | 7 | ARG MMS_VERSION=1.0.5 8 | ARG PYTHON=python 9 | ARG HEALTH_CHECK_VERSION=1.3.3 10 | 11 | RUN apt-get update && \ 12 | apt-get -y install --no-install-recommends \ 13 | build-essential \ 14 | ca-certificates \ 15 | curl \ 16 | git \ 17 | libopencv-dev \ 18 | openjdk-8-jdk-headless \ 19 | python-dev \ 20 | python-pip \ 21 | vim \ 22 | wget \ 23 | zlib1g-dev \ 24 | && rm -rf /var/lib/apt/lists/* 25 | 26 | # See http://bugs.python.org/issue19846 27 | ENV LANG=C.UTF-8 \ 28 | PYTHONDONTWRITEBYTECODE=1 \ 29 | PYTHONUNBUFFERED=1 \ 30 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 31 | PYTHONIOENCODING=UTF-8 \ 32 | LC_ALL=C.UTF-8 33 | 34 | WORKDIR / 35 | 36 | RUN wget https://amazonei-healthcheck.s3.amazonaws.com/v${HEALTH_CHECK_VERSION}/ei_health_check_${HEALTH_CHECK_VERSION}.tar.gz -O /opt/ei_health_check_${HEALTH_CHECK_VERSION}.tar.gz \ 37 | && tar -xvf /opt/ei_health_check_${HEALTH_CHECK_VERSION}.tar.gz -C /opt/ \ 38 | && rm -rf /opt/ei_health_check_${HEALTH_CHECK_VERSION}.tar.gz \ 39 | && chmod a+x /opt/ei_health_check/bin/health_check 40 | 41 | RUN pip install --no-cache-dir --upgrade \ 42 | pip \ 43 | setuptools 44 | 45 | RUN pip install --no-cache-dir \ 46 | https://s3.amazonaws.com/amazonei-apachemxnet/amazonei_mxnet-1.4.1-py2.py3-none-manylinux1_x86_64.whl \ 47 | mxnet-model-server==$MMS_VERSION \ 48 | keras-mxnet==2.2.4.1 \ 49 | # setuptools<45.0.0 because support for py2 stops with 45.0.0 50 | # https://github.com/pypa/setuptools/blob/master/CHANGES.rst#v4500 51 | "setuptools<45.0.0" \ 52 | onnx==1.4.1 \ 53 | # use sagemaker-inference version comaptible with MMS_VERSION=1.0.5 54 | sagemaker-inference==1.1.0 \ 55 | "sagemaker-mxnet-inference<2" 56 | 57 | # This is here to make our installed version of OpenCV work. 58 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 59 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 60 | RUN ln -s /dev/null /dev/raw1394 61 | 62 | RUN useradd -m model-server \ 63 | && mkdir -p /home/model-server/tmp \ 64 | && chown -R model-server /home/model-server 65 | 66 | COPY mms-entrypoint.py /usr/local/bin/dockerd-entrypoint.py 67 | COPY config.properties /home/model-server 68 | 69 | RUN chmod +x /usr/local/bin/dockerd-entrypoint.py 70 | 71 | EXPOSE 8080 8081 72 | ENV TEMP=/home/model-server/tmp 73 | ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] 74 | CMD ["mxnet-model-server", "--start", "--mms-config", "/home/model-server/config.properties"] 75 | -------------------------------------------------------------------------------- /docker/1.4.1/py2/Dockerfile.gpu: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:10.0-cudnn7-runtime-ubuntu16.04 2 | 3 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 4 | 5 | RUN apt-get update && \ 6 | DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ 7 | fakeroot \ 8 | ca-certificates \ 9 | dpkg-dev \ 10 | g++ \ 11 | python-dev \ 12 | openjdk-8-jdk-headless \ 13 | curl \ 14 | vim \ 15 | && rm -rf /var/lib/apt/lists/* \ 16 | && cd /tmp \ 17 | && curl -O https://bootstrap.pypa.io/get-pip.py \ 18 | && python get-pip.py 19 | 20 | RUN update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 21 | 22 | RUN apt-get update && \ 23 | apt-get -y install --no-install-recommends \ 24 | libopencv-dev \ 25 | build-essential \ 26 | && rm -rf /var/lib/apt/lists/* 27 | 28 | WORKDIR / 29 | 30 | COPY sagemaker_mxnet_serving_container.tar.gz /sagemaker_mxnet_serving_container.tar.gz 31 | 32 | RUN pip install --no-cache mxnet-cu100mkl==1.4.1 \ 33 | mxnet-model-server==1.0.5 \ 34 | keras-mxnet==2.2.4.1 \ 35 | numpy==1.14.5 \ 36 | onnx==1.4.1 \ 37 | /sagemaker_mxnet_serving_container.tar.gz && \ 38 | rm /sagemaker_mxnet_serving_container.tar.gz 39 | 40 | # This is here to make our installed version of OpenCV work. 41 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 42 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 43 | RUN ln -s /dev/null /dev/raw1394 44 | 45 | ENV PYTHONDONTWRITEBYTECODE=1 \ 46 | PYTHONUNBUFFERED=1 \ 47 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 48 | PYTHONIOENCODING=UTF-8 \ 49 | LANG=C.UTF-8 \ 50 | LC_ALL=C.UTF-8 51 | 52 | RUN mkdir -p /home/model-server/tmp 53 | 54 | COPY dockerd-entrypoint.sh /usr/local/bin/dockerd-entrypoint.sh 55 | COPY config.properties /home/model-server 56 | 57 | RUN chmod +x /usr/local/bin/dockerd-entrypoint.sh 58 | 59 | EXPOSE 8080 8081 60 | 61 | WORKDIR /home/model-server 62 | ENV TEMP=/home/model-server/tmp 63 | CMD ["serve"] 64 | ENTRYPOINT [] 65 | -------------------------------------------------------------------------------- /docker/1.4.1/py2/config.properties: -------------------------------------------------------------------------------- 1 | # vmargs=-Xmx128m -XX:-UseLargePages -XX:+UseG1GC -XX:MaxMetaspaceSize=32M -XX:MaxDirectMemorySize=10m -XX:+ExitOnOutOfMemoryError 2 | model_store=/opt/ml/model 3 | load_models=ALL 4 | inference_address=http://0.0.0.0:8080 5 | management_address=http://0.0.0.0:8081 6 | # management_address=unix:/tmp/management.sock 7 | # number_of_netty_threads=0 8 | # netty_client_threads=0 9 | # default_response_timeout=120 10 | # default_workers_per_model=0 11 | # job_queue_size=100 12 | # async_logging=false 13 | # number_of_gpu=1 14 | # cors_allowed_origin 15 | # cors_allowed_methods 16 | # cors_allowed_headers 17 | # keystore=src/test/resources/keystore.p12 18 | # keystore_pass=changeit 19 | # keystore_type=PKCS12 20 | # private_key_file=src/test/resources/key.pem 21 | # certificate_file=src/test/resources/certs.pem 22 | # max_response_size=6553500 23 | # max_request_size=6553500 24 | # blacklist_env_vars= 25 | decode_input_request=false 26 | # enable_envvars_config=false -------------------------------------------------------------------------------- /docker/1.4.1/py2/dockerd-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"). You 6 | # may not use this file except in compliance with the License. A copy of 7 | # the License is located at 8 | # 9 | # http://aws.amazon.com/apache2.0/ 10 | # 11 | # or in the "license" file accompanying this file. This file is 12 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 13 | # ANY KIND, either express or implied. See the License for the specific 14 | # language governing permissions and limitations under the License. 15 | 16 | set -e 17 | 18 | if [[ "$1" = "serve" ]]; then 19 | shift 1 20 | mxnet-model-server --start --mms-config config.properties 21 | else 22 | eval "$@" 23 | fi 24 | 25 | # prevent docker exit 26 | tail -f /dev/null 27 | -------------------------------------------------------------------------------- /docker/1.4.1/py2/mms-entrypoint.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | import shlex 14 | import subprocess 15 | import sys 16 | 17 | from sagemaker_mxnet_serving_container import serving 18 | 19 | if sys.argv[1] == 'serve': 20 | serving.main() 21 | else: 22 | subprocess.check_call(shlex.split(' '.join(sys.argv[1:]))) 23 | 24 | # prevent docker exit 25 | subprocess.call(['tail', '-f', '/dev/null']) 26 | -------------------------------------------------------------------------------- /docker/1.4.1/py3/Dockerfile.cpu: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | LABEL maintainer="Amazon AI" 4 | 5 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 6 | 7 | ARG MMS_VERSION=1.0.5 8 | ARG MX_VERSION=1.4.1 9 | ARG PYTHON=python3 10 | ARG PYTHON_PIP=python3-pip 11 | ARG PIP=pip3 12 | ARG PYTHON_VERSION=3.6.8 13 | 14 | RUN apt-get update && \ 15 | apt-get -y install --no-install-recommends \ 16 | build-essential \ 17 | ca-certificates \ 18 | curl \ 19 | git \ 20 | libopencv-dev \ 21 | openjdk-8-jdk-headless \ 22 | vim \ 23 | wget \ 24 | zlib1g-dev \ 25 | && rm -rf /var/lib/apt/lists/* 26 | 27 | RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ 28 | tar -xvf Python-$PYTHON_VERSION.tgz && cd Python-$PYTHON_VERSION && \ 29 | ./configure && make && make install && \ 30 | apt-get update && apt-get install -y --no-install-recommends libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev && \ 31 | make && make install && rm -rf ../Python-$PYTHON_VERSION* && \ 32 | ln -s /usr/local/bin/pip3 /usr/bin/pip 33 | 34 | RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 35 | 36 | RUN ${PIP} --no-cache-dir install --upgrade pip setuptools 37 | 38 | WORKDIR / 39 | 40 | COPY sagemaker_mxnet_serving_container.tar.gz /sagemaker_mxnet_serving_container.tar.gz 41 | 42 | RUN ${PIP} install --no-cache-dir mxnet-mkl==$MX_VERSION \ 43 | mxnet-model-server==$MMS_VERSION \ 44 | keras-mxnet==2.2.4.1 \ 45 | numpy==1.14.5 \ 46 | onnx==1.4.1 \ 47 | /sagemaker_mxnet_serving_container.tar.gz && \ 48 | rm /sagemaker_mxnet_serving_container.tar.gz 49 | 50 | # This is here to make our installed version of OpenCV work. 51 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 52 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 53 | RUN ln -s /dev/null /dev/raw1394 54 | 55 | ENV PYTHONDONTWRITEBYTECODE=1 \ 56 | PYTHONUNBUFFERED=1 \ 57 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 58 | PYTHONIOENCODING=UTF-8 \ 59 | LANG=C.UTF-8 \ 60 | LC_ALL=C.UTF-8 61 | 62 | RUN useradd -m model-server \ 63 | && mkdir -p /home/model-server/tmp \ 64 | && chown -R model-server /home/model-server 65 | 66 | COPY mms-entrypoint.py /usr/local/bin/dockerd-entrypoint.py 67 | COPY config.properties /home/model-server 68 | 69 | RUN chmod +x /usr/local/bin/dockerd-entrypoint.py 70 | 71 | EXPOSE 8080 8081 72 | ENV TEMP=/home/model-server/tmp 73 | ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] 74 | CMD ["mxnet-model-server", "--start", "--mms-config", "/home/model-server/config.properties"] 75 | -------------------------------------------------------------------------------- /docker/1.4.1/py3/Dockerfile.eia: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | LABEL maintainer="Amazon AI" 4 | 5 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 6 | 7 | ARG MMS_VERSION=1.0.5 8 | ARG PYTHON=python3 9 | ARG PYTHON_VERSION=3.6.8 10 | ARG HEALTH_CHECK_VERSION=1.3.3 11 | 12 | RUN apt-get update \ 13 | && apt-get -y install --no-install-recommends \ 14 | build-essential \ 15 | ca-certificates \ 16 | curl \ 17 | git \ 18 | libopencv-dev \ 19 | openjdk-8-jdk-headless \ 20 | vim \ 21 | wget \ 22 | zlib1g-dev \ 23 | && rm -rf /var/lib/apt/lists/* 24 | 25 | # See http://bugs.python.org/issue19846 26 | ENV LANG=C.UTF-8 \ 27 | PYTHONDONTWRITEBYTECODE=1 \ 28 | PYTHONUNBUFFERED=1 \ 29 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 30 | PYTHONIOENCODING=UTF-8 \ 31 | LC_ALL=C.UTF-8 32 | 33 | RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz \ 34 | && tar -xvf Python-$PYTHON_VERSION.tgz \ 35 | && cd Python-$PYTHON_VERSION \ 36 | && ./configure \ 37 | && make \ 38 | && make install \ 39 | && apt-get update \ 40 | && apt-get install -y --no-install-recommends \ 41 | libreadline-gplv2-dev \ 42 | libncursesw5-dev \ 43 | libssl-dev \ 44 | libsqlite3-dev \ 45 | tk-dev \ 46 | libgdbm-dev \ 47 | libc6-dev \ 48 | libbz2-dev \ 49 | && make \ 50 | && make install \ 51 | && rm -rf ../Python-$PYTHON_VERSION* \ 52 | && ln -s /usr/local/bin/pip3 /usr/bin/pip \ 53 | && ln -s $(which ${PYTHON}) /usr/local/bin/python 54 | 55 | WORKDIR / 56 | 57 | RUN wget https://amazonei-healthcheck.s3.amazonaws.com/v${HEALTH_CHECK_VERSION}/ei_health_check_${HEALTH_CHECK_VERSION}.tar.gz -O /opt/ei_health_check_${HEALTH_CHECK_VERSION}.tar.gz \ 58 | && tar -xvf /opt/ei_health_check_${HEALTH_CHECK_VERSION}.tar.gz -C /opt/ \ 59 | && rm -rf /opt/ei_health_check_${HEALTH_CHECK_VERSION}.tar.gz \ 60 | && chmod a+x /opt/ei_health_check/bin/health_check 61 | 62 | RUN pip install --no-cache-dir --upgrade \ 63 | pip \ 64 | setuptools 65 | 66 | RUN pip install --no-cache-dir \ 67 | https://s3.amazonaws.com/amazonei-apachemxnet/amazonei_mxnet-1.4.1-py2.py3-none-manylinux1_x86_64.whl \ 68 | mxnet-model-server==$MMS_VERSION \ 69 | keras-mxnet==2.2.4.1 \ 70 | onnx==1.4.1 \ 71 | # use sagemaker-inference version comaptible with MMS_VERSION=1.0.5 72 | sagemaker-inference==1.1.0 \ 73 | "sagemaker-mxnet-inference<2" 74 | 75 | # This is here to make our installed version of OpenCV work. 76 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 77 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 78 | RUN ln -s /dev/null /dev/raw1394 79 | 80 | RUN useradd -m model-server \ 81 | && mkdir -p /home/model-server/tmp \ 82 | && chown -R model-server /home/model-server 83 | 84 | COPY mms-entrypoint.py /usr/local/bin/dockerd-entrypoint.py 85 | COPY config.properties /home/model-server 86 | 87 | RUN chmod +x /usr/local/bin/dockerd-entrypoint.py 88 | 89 | EXPOSE 8080 8081 90 | ENV TEMP=/home/model-server/tmp 91 | ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] 92 | CMD ["mxnet-model-server", "--start", "--mms-config", "/home/model-server/config.properties"] 93 | -------------------------------------------------------------------------------- /docker/1.4.1/py3/Dockerfile.gpu: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:10.0-cudnn7-runtime-ubuntu16.04 2 | 3 | LABEL maintainer="Amazon AI" 4 | 5 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 6 | 7 | ARG MMS_VERSION=1.0.5 8 | ARG MX_VERSION=1.4.1 9 | ARG PYTHON=python3 10 | ARG PYTHON_PIP=python3-pip 11 | ARG PIP=pip3 12 | ARG PYTHON_VERSION=3.6.8 13 | 14 | RUN apt-get update && \ 15 | apt-get -y install --no-install-recommends \ 16 | build-essential \ 17 | ca-certificates \ 18 | curl \ 19 | git \ 20 | libopencv-dev \ 21 | openjdk-8-jdk-headless \ 22 | vim \ 23 | wget \ 24 | zlib1g-dev \ 25 | && rm -rf /var/lib/apt/lists/* 26 | 27 | RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ 28 | tar -xvf Python-$PYTHON_VERSION.tgz && cd Python-$PYTHON_VERSION && \ 29 | ./configure && make && make install && \ 30 | apt-get update && apt-get install -y --no-install-recommends libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev && \ 31 | make && make install && rm -rf ../Python-$PYTHON_VERSION* && \ 32 | ln -s /usr/local/bin/pip3 /usr/bin/pip 33 | 34 | RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 35 | 36 | RUN ${PIP} --no-cache-dir install --upgrade pip setuptools 37 | 38 | WORKDIR / 39 | 40 | COPY sagemaker_mxnet_serving_container.tar.gz /sagemaker_mxnet_serving_container.tar.gz 41 | 42 | RUN ${PIP} install --no-cache-dir mxnet-cu100mkl==$MX_VERSION \ 43 | mxnet-model-server==$MMS_VERSION \ 44 | keras-mxnet==2.2.4.1 \ 45 | numpy==1.14.5 \ 46 | onnx==1.4.1 \ 47 | /sagemaker_mxnet_serving_container.tar.gz && \ 48 | rm /sagemaker_mxnet_serving_container.tar.gz 49 | 50 | # This is here to make our installed version of OpenCV work. 51 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 52 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 53 | RUN ln -s /dev/null /dev/raw1394 54 | 55 | ENV PYTHONDONTWRITEBYTECODE=1 \ 56 | PYTHONUNBUFFERED=1 \ 57 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 58 | PYTHONIOENCODING=UTF-8 \ 59 | LANG=C.UTF-8 \ 60 | LC_ALL=C.UTF-8 61 | 62 | RUN useradd -m model-server \ 63 | && mkdir -p /home/model-server/tmp \ 64 | && chown -R model-server /home/model-server 65 | 66 | COPY mms-entrypoint.py /usr/local/bin/dockerd-entrypoint.py 67 | COPY config.properties /home/model-server 68 | 69 | RUN chmod +x /usr/local/bin/dockerd-entrypoint.py 70 | 71 | EXPOSE 8080 8081 72 | ENV TEMP=/home/model-server/tmp 73 | ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] 74 | CMD ["mxnet-model-server", "--start", "--mms-config", "/home/model-server/config.properties"] 75 | -------------------------------------------------------------------------------- /docker/1.4.1/py3/config.properties: -------------------------------------------------------------------------------- 1 | vmargs=-Xmx128m -XX:-UseLargePages -XX:+UseG1GC -XX:MaxMetaspaceSize=32M -XX:MaxDirectMemorySize=10m -XX:+ExitOnOutOfMemoryError 2 | model_store=/opt/ml/model 3 | load_models=ALL 4 | inference_address=http://0.0.0.0:8080 5 | management_address=http://0.0.0.0:8081 6 | -------------------------------------------------------------------------------- /docker/1.4.1/py3/mms-entrypoint.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | import shlex 15 | import subprocess 16 | import sys 17 | 18 | from sagemaker_mxnet_serving_container import serving 19 | 20 | if sys.argv[1] == 'serve': 21 | serving.main() 22 | else: 23 | subprocess.check_call(shlex.split(' '.join(sys.argv[1:]))) 24 | 25 | # prevent docker exit 26 | subprocess.call(['tail', '-f', '/dev/null']) 27 | -------------------------------------------------------------------------------- /docker/1.5.1/py2/Dockerfile.eia: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | LABEL maintainer="Amazon AI" 4 | 5 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 6 | 7 | ARG MMS_VERSION=1.1.6 8 | ARG PYTHON=python 9 | ARG HEALTH_CHECK_VERSION=1.6.3 10 | 11 | RUN apt-get update && \ 12 | apt-get -y install --no-install-recommends \ 13 | build-essential \ 14 | ca-certificates \ 15 | curl \ 16 | git \ 17 | libopencv-dev \ 18 | openjdk-8-jdk-headless \ 19 | python-dev \ 20 | python-pip \ 21 | vim \ 22 | wget \ 23 | zlib1g-dev \ 24 | && apt-get clean \ 25 | && rm -rf /var/lib/apt/lists/* 26 | 27 | # See http://bugs.python.org/issue19846 28 | ENV LANG=C.UTF-8 \ 29 | PYTHONDONTWRITEBYTECODE=1 \ 30 | PYTHONUNBUFFERED=1 \ 31 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 32 | PYTHONIOENCODING=UTF-8 \ 33 | LC_ALL=C.UTF-8 34 | 35 | WORKDIR / 36 | 37 | RUN wget https://amazonei-tools.s3.amazonaws.com/v${HEALTH_CHECK_VERSION}/ei_tools_${HEALTH_CHECK_VERSION}.tar.gz -O /opt/ei_tools_${HEALTH_CHECK_VERSION}.tar.gz \ 38 | && tar -xvf /opt/ei_tools_${HEALTH_CHECK_VERSION}.tar.gz -C /opt/ \ 39 | && rm -rf /opt/ei_tools_${HEALTH_CHECK_VERSION}.tar.gz \ 40 | && chmod a+x /opt/ei_tools/bin/health_check 41 | 42 | RUN pip install --no-cache-dir --upgrade \ 43 | pip \ 44 | setuptools 45 | 46 | RUN pip install --no-cache-dir \ 47 | https://s3.amazonaws.com/amazonei-apachemxnet/amazonei_mxnet-1.5.1-py2.py3-none-manylinux1_x86_64.whl \ 48 | multi-model-server==$MMS_VERSION \ 49 | keras-mxnet==2.2.4.1 \ 50 | # setuptools<45.0.0 because support for py2 stops with 45.0.0 51 | # https://github.com/pypa/setuptools/blob/master/CHANGES.rst#v4500 52 | "setuptools<45.0.0" \ 53 | numpy==1.16.5 \ 54 | onnx==1.4.1 \ 55 | "sagemaker-mxnet-inference<2" 56 | 57 | # This is here to make our installed version of OpenCV work. 58 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 59 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 60 | RUN ln -s /dev/null /dev/raw1394 61 | 62 | RUN useradd -m model-server \ 63 | && mkdir -p /home/model-server/tmp \ 64 | && chown -R model-server /home/model-server 65 | 66 | COPY mms-entrypoint.py /usr/local/bin/dockerd-entrypoint.py 67 | COPY config.properties /home/model-server 68 | 69 | RUN chmod +x /usr/local/bin/dockerd-entrypoint.py 70 | 71 | EXPOSE 8080 8081 72 | ENV TEMP=/home/model-server/tmp 73 | ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] 74 | CMD ["multi-model-server", "--start", "--mms-config", "/home/model-server/config.properties"] 75 | -------------------------------------------------------------------------------- /docker/1.5.1/py3/Dockerfile.eia: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | LABEL maintainer="Amazon AI" 4 | 5 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 6 | 7 | ARG MMS_VERSION=1.1.6 8 | ARG PYTHON=python3 9 | ARG PYTHON_VERSION=3.6.8 10 | ARG HEALTH_CHECK_VERSION=1.6.3 11 | 12 | RUN apt-get update \ 13 | && apt-get -y install --no-install-recommends \ 14 | build-essential \ 15 | ca-certificates \ 16 | curl \ 17 | git \ 18 | libopencv-dev \ 19 | openjdk-8-jdk-headless \ 20 | vim \ 21 | wget \ 22 | zlib1g-dev \ 23 | && apt-get clean \ 24 | && rm -rf /var/lib/apt/lists/* 25 | 26 | # See http://bugs.python.org/issue19846 27 | ENV LANG=C.UTF-8 \ 28 | PYTHONDONTWRITEBYTECODE=1 \ 29 | PYTHONUNBUFFERED=1 \ 30 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 31 | PYTHONIOENCODING=UTF-8 \ 32 | LC_ALL=C.UTF-8 33 | 34 | RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz \ 35 | && tar -xvf Python-$PYTHON_VERSION.tgz \ 36 | && cd Python-$PYTHON_VERSION \ 37 | && ./configure \ 38 | && make \ 39 | && make install \ 40 | && apt-get update \ 41 | && apt-get install -y --no-install-recommends \ 42 | libreadline-gplv2-dev \ 43 | libncursesw5-dev \ 44 | libssl-dev \ 45 | libsqlite3-dev \ 46 | tk-dev \ 47 | libgdbm-dev \ 48 | libc6-dev \ 49 | libbz2-dev \ 50 | && make \ 51 | && make install \ 52 | && rm -rf ../Python-$PYTHON_VERSION* \ 53 | && ln -s /usr/local/bin/pip3 /usr/bin/pip \ 54 | && ln -s $(which ${PYTHON}) /usr/local/bin/python 55 | 56 | WORKDIR / 57 | 58 | RUN wget https://amazonei-tools.s3.amazonaws.com/v${HEALTH_CHECK_VERSION}/ei_tools_${HEALTH_CHECK_VERSION}.tar.gz -O /opt/ei_tools_${HEALTH_CHECK_VERSION}.tar.gz \ 59 | && tar -xvf /opt/ei_tools_${HEALTH_CHECK_VERSION}.tar.gz -C /opt/ \ 60 | && rm -rf /opt/ei_tools_${HEALTH_CHECK_VERSION}.tar.gz \ 61 | && chmod a+x /opt/ei_tools/bin/health_check 62 | 63 | RUN pip install --no-cache-dir --upgrade \ 64 | pip \ 65 | setuptools 66 | 67 | RUN pip install --no-cache-dir \ 68 | https://s3.amazonaws.com/amazonei-apachemxnet/amazonei_mxnet-1.5.1-py2.py3-none-manylinux1_x86_64.whl \ 69 | multi-model-server==$MMS_VERSION \ 70 | keras-mxnet==2.2.4.1 \ 71 | numpy==1.17.4 \ 72 | onnx==1.4.1 \ 73 | "sagemaker-mxnet-inference<2" 74 | 75 | # Install openssl-1.1.1g to override default openssl-1.0.2g 76 | RUN wget https://www.openssl.org/source/openssl-1.1.1g.tar.gz \ 77 | && apt-get remove --purge -y openssl \ 78 | && rm -rf /usr/include/openssl \ 79 | && apt-get update \ 80 | && apt-get install -y \ 81 | ca-certificates \ 82 | openjdk-8-jdk-headless \ 83 | && tar -xzvf openssl-1.1.1g.tar.gz \ 84 | && cd openssl-1.1.1g \ 85 | && ./config \ 86 | && make \ 87 | && make test \ 88 | && make install \ 89 | && cd ../ \ 90 | && rm -rf openssl-* \ 91 | && apt-get clean \ 92 | && rm -rf /var/lib/apt/lists/* 93 | 94 | # Purging default openssl in above command also prevents the new installation of openssl 95 | # from finding certificates at the pre-existing path. This env variable fixes the issue. 96 | ENV SSL_CERT_DIR=/etc/ssl/certs 97 | 98 | # This is here to make our installed version of OpenCV work. 99 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 100 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 101 | RUN ln -s /dev/null /dev/raw1394 102 | 103 | RUN useradd -m model-server \ 104 | && mkdir -p /home/model-server/tmp \ 105 | && chown -R model-server /home/model-server 106 | 107 | COPY mms-entrypoint.py /usr/local/bin/dockerd-entrypoint.py 108 | COPY config.properties /home/model-server 109 | 110 | RUN chmod +x /usr/local/bin/dockerd-entrypoint.py 111 | 112 | EXPOSE 8080 8081 113 | ENV TEMP=/home/model-server/tmp 114 | ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] 115 | CMD ["multi-model-server", "--start", "--mms-config", "/home/model-server/config.properties"] 116 | -------------------------------------------------------------------------------- /docker/1.6.0/py2/Dockerfile.cpu: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | LABEL maintainer="Amazon AI" 4 | 5 | # Specify accept-bind-to-port LABEL for inference pipelines to use SAGEMAKER_BIND_TO_PORT 6 | # https://docs.aws.amazon.com/sagemaker/latest/dg/inference-pipeline-real-time.html 7 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 8 | # Specify multi-models LABEL to indicate container is capable of loading and serving multiple models concurrently 9 | # https://docs.aws.amazon.com/sagemaker/latest/dg/build-multi-model-build-container.html 10 | LABEL com.amazonaws.sagemaker.capabilities.multi-models=true 11 | 12 | ARG MMS_VERSION=1.1.6 13 | ARG MX_URL=https://aws-mxnet-pypi.s3-us-west-2.amazonaws.com/1.6.0/aws_mxnet_mkl-1.6.0-py2.py3-none-manylinux1_x86_64.whl 14 | ARG PYTHON=python 15 | ARG PYTHON_PIP=python-pip 16 | ARG PIP=pip 17 | 18 | ENV PYTHONDONTWRITEBYTECODE=1 \ 19 | PYTHONUNBUFFERED=1 \ 20 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 21 | PYTHONIOENCODING=UTF-8 \ 22 | LANG=C.UTF-8 \ 23 | LC_ALL=C.UTF-8 \ 24 | TEMP=/home/model-server/tmp 25 | 26 | RUN apt-get update \ 27 | && apt-get -y install --no-install-recommends \ 28 | build-essential \ 29 | ca-certificates \ 30 | curl \ 31 | git \ 32 | libopencv-dev \ 33 | openjdk-8-jdk-headless \ 34 | vim \ 35 | wget \ 36 | zlib1g-dev \ 37 | && apt-get clean \ 38 | && rm -rf /var/lib/apt/lists/* 39 | 40 | RUN apt-get update \ 41 | && apt-get install -y \ 42 | ${PYTHON} \ 43 | ${PYTHON_PIP} 44 | 45 | RUN ${PIP} --no-cache-dir install --upgrade \ 46 | pip \ 47 | setuptools 48 | 49 | WORKDIR / 50 | 51 | RUN ${PIP} install --no-cache-dir \ 52 | ${MX_URL} \ 53 | # setuptools<45.0.0 because support for py2 stops with 45.0.0 54 | # https://github.com/pypa/setuptools/blob/master/CHANGES.rst#v4500 55 | "setuptools<45.0.0" \ 56 | multi-model-server==$MMS_VERSION \ 57 | keras-mxnet==2.2.4.1 \ 58 | numpy==1.16.5 \ 59 | onnx==1.4.1 \ 60 | "sagemaker-mxnet-inference<2" 61 | 62 | # This is here to make our installed version of OpenCV work. 63 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 64 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 65 | RUN ln -s /dev/null /dev/raw1394 66 | 67 | RUN useradd -m model-server \ 68 | && mkdir -p /home/model-server/tmp \ 69 | && chown -R model-server /home/model-server 70 | 71 | COPY mms-entrypoint.py /usr/local/bin/dockerd-entrypoint.py 72 | COPY config.properties /home/model-server 73 | 74 | RUN chmod +x /usr/local/bin/dockerd-entrypoint.py 75 | 76 | RUN curl https://aws-dlc-licenses.s3.amazonaws.com/aws-mxnet-1.6.0/license.txt -o /license.txt 77 | 78 | EXPOSE 8080 8081 79 | ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] 80 | CMD ["multi-model-server", "--start", "--mms-config", "/home/model-server/config.properties"] 81 | -------------------------------------------------------------------------------- /docker/1.6.0/py2/Dockerfile.gpu: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu16.04 2 | 3 | LABEL maintainer="Amazon AI" 4 | 5 | # Specify accept-bind-to-port LABEL for inference pipelines to use SAGEMAKER_BIND_TO_PORT 6 | # https://docs.aws.amazon.com/sagemaker/latest/dg/inference-pipeline-real-time.html 7 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 8 | 9 | ARG MMS_VERSION=1.1.6 10 | ARG MX_URL=https://aws-mxnet-pypi.s3-us-west-2.amazonaws.com/1.6.0/aws_mxnet_cu101mkl-1.6.0-py2.py3-none-manylinux1_x86_64.whl 11 | ARG PYTHON=python 12 | ARG PYTHON_PIP=python-pip 13 | ARG PIP=pip 14 | 15 | ENV PYTHONDONTWRITEBYTECODE=1 \ 16 | PYTHONUNBUFFERED=1 \ 17 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 18 | PYTHONIOENCODING=UTF-8 \ 19 | LANG=C.UTF-8 \ 20 | LC_ALL=C.UTF-8 \ 21 | TEMP=/home/model-server/tmp 22 | 23 | RUN apt-get update \ 24 | && apt-get -y install --no-install-recommends \ 25 | build-essential \ 26 | ca-certificates \ 27 | curl \ 28 | git \ 29 | libopencv-dev \ 30 | openjdk-8-jdk-headless \ 31 | vim \ 32 | wget \ 33 | zlib1g-dev \ 34 | && apt-get clean \ 35 | && rm -rf /var/lib/apt/lists/* 36 | 37 | RUN apt-get update \ 38 | && apt-get install -y \ 39 | ${PYTHON} \ 40 | ${PYTHON_PIP} 41 | 42 | RUN ${PIP} --no-cache-dir install --upgrade \ 43 | pip \ 44 | setuptools 45 | 46 | WORKDIR / 47 | 48 | RUN ${PIP} install --no-cache-dir \ 49 | ${MX_URL} \ 50 | # setuptools<45.0.0 because support for py2 stops with 45.0.0 51 | # https://github.com/pypa/setuptools/blob/master/CHANGES.rst#v4500 52 | "setuptools<45.0.0" \ 53 | multi-model-server==$MMS_VERSION \ 54 | keras-mxnet==2.2.4.1 \ 55 | numpy==1.16.5 \ 56 | onnx==1.4.1 \ 57 | "sagemaker-mxnet-inference<2" 58 | 59 | # This is here to make our installed version of OpenCV work. 60 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 61 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 62 | RUN ln -s /dev/null /dev/raw1394 63 | 64 | RUN useradd -m model-server \ 65 | && mkdir -p /home/model-server/tmp \ 66 | && chown -R model-server /home/model-server 67 | 68 | COPY mms-entrypoint.py /usr/local/bin/dockerd-entrypoint.py 69 | COPY config.properties /home/model-server 70 | 71 | RUN chmod +x /usr/local/bin/dockerd-entrypoint.py 72 | 73 | RUN curl https://aws-dlc-licenses.s3.amazonaws.com/aws-mxnet-1.6.0/license.txt -o /license.txt 74 | 75 | EXPOSE 8080 8081 76 | ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] 77 | CMD ["multi-model-server", "--start", "--mms-config", "/home/model-server/config.properties"] 78 | -------------------------------------------------------------------------------- /docker/1.6.0/py3/Dockerfile.cpu: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | LABEL maintainer="Amazon AI" 4 | 5 | # Specify accept-bind-to-port LABEL for inference pipelines to use SAGEMAKER_BIND_TO_PORT 6 | # https://docs.aws.amazon.com/sagemaker/latest/dg/inference-pipeline-real-time.html 7 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 8 | # Specify multi-models LABEL to indicate container is capable of loading and serving multiple models concurrently 9 | # https://docs.aws.amazon.com/sagemaker/latest/dg/build-multi-model-build-container.html 10 | LABEL com.amazonaws.sagemaker.capabilities.multi-models=true 11 | 12 | ARG MMS_VERSION=1.1.6 13 | ARG MX_URL=https://aws-mxnet-pypi.s3-us-west-2.amazonaws.com/1.6.0/aws_mxnet_mkl-1.6.0-py2.py3-none-manylinux1_x86_64.whl 14 | ARG PYTHON=python3 15 | ARG PYTHON_PIP=python3-pip 16 | ARG PIP=pip3 17 | ARG PYTHON_VERSION=3.6.8 18 | 19 | ENV PYTHONDONTWRITEBYTECODE=1 \ 20 | PYTHONUNBUFFERED=1 \ 21 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 22 | PYTHONIOENCODING=UTF-8 \ 23 | LANG=C.UTF-8 \ 24 | LC_ALL=C.UTF-8 \ 25 | TEMP=/home/model-server/tmp 26 | 27 | RUN apt-get update \ 28 | && apt-get -y install --no-install-recommends \ 29 | build-essential \ 30 | ca-certificates \ 31 | curl \ 32 | git \ 33 | libopencv-dev \ 34 | openjdk-8-jdk-headless \ 35 | vim \ 36 | wget \ 37 | zlib1g-dev \ 38 | && apt-get clean \ 39 | && rm -rf /var/lib/apt/lists/* 40 | 41 | RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz \ 42 | && tar -xvf Python-$PYTHON_VERSION.tgz \ 43 | && cd Python-$PYTHON_VERSION \ 44 | && ./configure \ 45 | && make \ 46 | && make install \ 47 | && apt-get update \ 48 | && apt-get install -y --no-install-recommends \ 49 | libreadline-gplv2-dev \ 50 | libncursesw5-dev \ 51 | libssl-dev \ 52 | libsqlite3-dev \ 53 | tk-dev \ 54 | libgdbm-dev \ 55 | libc6-dev \ 56 | libbz2-dev \ 57 | && make \ 58 | && make install \ 59 | && rm -rf ../Python-$PYTHON_VERSION* \ 60 | && ln -s /usr/local/bin/pip3 /usr/bin/pip 61 | 62 | RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 63 | 64 | RUN ${PIP} --no-cache-dir install --upgrade \ 65 | pip \ 66 | setuptools 67 | 68 | WORKDIR / 69 | 70 | RUN ${PIP} install --no-cache-dir \ 71 | ${MX_URL} \ 72 | git+git://github.com/dmlc/gluon-nlp.git@v0.9.0 \ 73 | gluoncv==0.6.0 \ 74 | multi-model-server==$MMS_VERSION \ 75 | keras-mxnet==2.2.4.1 \ 76 | numpy==1.17.4 \ 77 | onnx==1.4.1 \ 78 | "sagemaker-mxnet-inference<2" 79 | 80 | # This is here to make our installed version of OpenCV work. 81 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 82 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 83 | RUN ln -s /dev/null /dev/raw1394 84 | 85 | RUN useradd -m model-server \ 86 | && mkdir -p /home/model-server/tmp \ 87 | && chown -R model-server /home/model-server 88 | 89 | COPY mms-entrypoint.py /usr/local/bin/dockerd-entrypoint.py 90 | COPY config.properties /home/model-server 91 | 92 | RUN chmod +x /usr/local/bin/dockerd-entrypoint.py 93 | 94 | RUN curl https://aws-dlc-licenses.s3.amazonaws.com/aws-mxnet-1.6.0/license.txt -o /license.txt 95 | 96 | EXPOSE 8080 8081 97 | ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] 98 | CMD ["multi-model-server", "--start", "--mms-config", "/home/model-server/config.properties"] 99 | -------------------------------------------------------------------------------- /docker/1.6.0/py3/Dockerfile.gpu: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu16.04 2 | 3 | LABEL maintainer="Amazon AI" 4 | 5 | # Specify accept-bind-to-port LABEL for inference pipelines to use SAGEMAKER_BIND_TO_PORT 6 | # https://docs.aws.amazon.com/sagemaker/latest/dg/inference-pipeline-real-time.html 7 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 8 | 9 | ARG MMS_VERSION=1.1.6 10 | ARG MX_URL=https://aws-mxnet-pypi.s3-us-west-2.amazonaws.com/1.6.0/aws_mxnet_cu101mkl-1.6.0-py2.py3-none-manylinux1_x86_64.whl 11 | ARG PYTHON=python3 12 | ARG PYTHON_PIP=python3-pip 13 | ARG PIP=pip3 14 | ARG PYTHON_VERSION=3.6.8 15 | 16 | ENV PYTHONDONTWRITEBYTECODE=1 \ 17 | PYTHONUNBUFFERED=1 \ 18 | LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ 19 | PYTHONIOENCODING=UTF-8 \ 20 | LANG=C.UTF-8 \ 21 | LC_ALL=C.UTF-8 \ 22 | TEMP=/home/model-server/tmp 23 | 24 | RUN apt-get update \ 25 | && apt-get -y install --no-install-recommends \ 26 | build-essential \ 27 | ca-certificates \ 28 | curl \ 29 | git \ 30 | libopencv-dev \ 31 | openjdk-8-jdk-headless \ 32 | vim \ 33 | wget \ 34 | zlib1g-dev \ 35 | && apt-get clean \ 36 | && rm -rf /var/lib/apt/lists/* 37 | 38 | RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz \ 39 | && tar -xvf Python-$PYTHON_VERSION.tgz \ 40 | && cd Python-$PYTHON_VERSION \ 41 | && ./configure \ 42 | && make \ 43 | && make install \ 44 | && apt-get update \ 45 | && apt-get install -y --no-install-recommends \ 46 | libreadline-gplv2-dev \ 47 | libncursesw5-dev \ 48 | libssl-dev \ 49 | libsqlite3-dev \ 50 | tk-dev \ 51 | libgdbm-dev \ 52 | libc6-dev \ 53 | libbz2-dev \ 54 | && make \ 55 | && make install \ 56 | && rm -rf ../Python-$PYTHON_VERSION* \ 57 | && ln -s /usr/local/bin/pip3 /usr/bin/pip 58 | 59 | RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 60 | 61 | RUN ${PIP} --no-cache-dir install --upgrade \ 62 | pip \ 63 | setuptools 64 | 65 | WORKDIR / 66 | 67 | RUN ${PIP} install --no-cache-dir \ 68 | ${MX_URL} \ 69 | git+git://github.com/dmlc/gluon-nlp.git@v0.9.0 \ 70 | gluoncv==0.6.0 \ 71 | multi-model-server==$MMS_VERSION \ 72 | keras-mxnet==2.2.4.1 \ 73 | numpy==1.17.4 \ 74 | onnx==1.4.1 \ 75 | "sagemaker-mxnet-inference<2" 76 | 77 | # This is here to make our installed version of OpenCV work. 78 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 79 | # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? 80 | RUN ln -s /dev/null /dev/raw1394 81 | 82 | RUN useradd -m model-server \ 83 | && mkdir -p /home/model-server/tmp \ 84 | && chown -R model-server /home/model-server 85 | 86 | COPY mms-entrypoint.py /usr/local/bin/dockerd-entrypoint.py 87 | COPY config.properties /home/model-server 88 | 89 | RUN chmod +x /usr/local/bin/dockerd-entrypoint.py 90 | 91 | RUN curl https://aws-dlc-licenses.s3.amazonaws.com/aws-mxnet-1.6.0/license.txt -o /license.txt 92 | 93 | EXPOSE 8080 8081 94 | ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] 95 | CMD ["multi-model-server", "--start", "--mms-config", "/home/model-server/config.properties"] 96 | -------------------------------------------------------------------------------- /docker/artifacts/config.properties: -------------------------------------------------------------------------------- 1 | vmargs=-Xmx128m -XX:-UseLargePages -XX:+UseG1GC -XX:MaxMetaspaceSize=32M -XX:MaxDirectMemorySize=10m -XX:+ExitOnOutOfMemoryError 2 | model_store=/opt/ml/model 3 | load_models=ALL 4 | inference_address=http://0.0.0.0:8080 5 | management_address=http://0.0.0.0:8081 6 | -------------------------------------------------------------------------------- /docker/artifacts/mms-entrypoint.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | import shlex 15 | import subprocess 16 | import sys 17 | 18 | 19 | if sys.argv[1] == 'serve': 20 | from sagemaker_mxnet_serving_container import serving 21 | serving.main() 22 | else: 23 | subprocess.check_call(shlex.split(' '.join(sys.argv[1:]))) 24 | 25 | # prevent docker exit 26 | subprocess.call(['tail', '-f', '/dev/null']) 27 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | addopts = 3 | --verbose 4 | 5 | 6 | [flake8] 7 | max-line-length = 100 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | from glob import glob 16 | import os 17 | 18 | from setuptools import find_packages, setup 19 | 20 | 21 | def read(fname): 22 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 23 | 24 | 25 | setup( 26 | name='sagemaker_mxnet_inference', 27 | version=read('VERSION').strip(), 28 | description='Open source library for creating MXNet containers for serving on SageMaker.', 29 | 30 | packages=find_packages(where='src', exclude=('test',)), 31 | package_dir={'': 'src'}, 32 | py_modules=[os.path.splitext(os.path.basename(path))[0] for path in glob('src/*.py')], 33 | 34 | long_description=read('README.rst'), 35 | author='Amazon Web Services', 36 | url='https://github.com/aws/sagemaker-mxnet-inference-toolkit', 37 | license='Apache License 2.0', 38 | 39 | classifiers=[ 40 | "Development Status :: 5 - Production/Stable", 41 | "Intended Audience :: Developers", 42 | "Natural Language :: English", 43 | "License :: OSI Approved :: Apache Software License", 44 | "Programming Language :: Python", 45 | 'Programming Language :: Python :: 2.7', 46 | 'Programming Language :: Python :: 3.6', 47 | 'Programming Language :: Python :: 3.7', 48 | ], 49 | 50 | # support sagemaker-inference==1.1.0 for mxnet 1.4 eia image and 51 | install_requires=['sagemaker-inference>=1.5.0', 'retrying==1.3.3'], 52 | extras_require={ 53 | 'test': ['tox', 'flake8', 'pytest', 'pytest-cov', 'pytest-xdist', 'pytest-rerunfailures', 54 | 'mock', 'sagemaker==1.62.0', 'docker-compose', 'mxnet==1.7.0.post1', 'awslogs', 55 | 'requests_mock'] 56 | }, 57 | 58 | entry_points={ 59 | 'console_scripts': 'serve=sagemaker_mxnet_serving_container.serving:main' 60 | } 61 | ) 62 | -------------------------------------------------------------------------------- /src/sagemaker_mxnet_serving_container/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/sagemaker-mxnet-inference-toolkit/addf27925e3f7d51c05be05a31d16a64d88839f9/src/sagemaker_mxnet_serving_container/__init__.py -------------------------------------------------------------------------------- /src/sagemaker_mxnet_serving_container/default_inference_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import os 16 | 17 | import mxnet as mx 18 | from sagemaker_inference import ( 19 | content_types, 20 | decoder, 21 | default_inference_handler, 22 | encoder, 23 | errors, 24 | ) 25 | 26 | from sagemaker_mxnet_serving_container.utils import ( 27 | get_default_context, 28 | parse_accept, 29 | read_data_shapes, 30 | ) 31 | 32 | PREFERRED_BATCH_SIZE_PARAM = 'SAGEMAKER_DEFAULT_MODEL_FIRST_DIMENSION_SIZE' 33 | INFERENCE_ACCELERATOR_PRESENT_ENV = 'SAGEMAKER_INFERENCE_ACCELERATOR_PRESENT' 34 | 35 | DEFAULT_MODEL_NAME = 'model' 36 | DEFAULT_MODEL_FILENAMES = { 37 | 'symbol': 'model-symbol.json', 38 | 'params': 'model-0000.params', 39 | 'shapes': 'model-shapes.json', 40 | } 41 | 42 | 43 | class DefaultMXNetInferenceHandler(default_inference_handler.DefaultInferenceHandler): 44 | VALID_CONTENT_TYPES = (content_types.JSON, content_types.NPY) 45 | 46 | def default_model_fn(self, model_dir, preferred_batch_size=1): 47 | """Function responsible for loading the model. This implementation is designed to work with 48 | the default save function provided for MXNet training. 49 | 50 | Args: 51 | model_dir (str): The directory where model files are stored 52 | preferred_batch_size (int): preferred batch size of the model's data shape. 53 | Defaults to 1. 54 | 55 | Returns: 56 | mxnet.mod.Module: the loaded model. 57 | 58 | """ 59 | for f in DEFAULT_MODEL_FILENAMES.values(): 60 | path = os.path.join(model_dir, f) 61 | if not os.path.exists(path): 62 | raise ValueError('Failed to load model with default model_fn: missing file {}.' 63 | 'Expected files: {}'.format(f, [file_name for _, file_name 64 | in DEFAULT_MODEL_FILENAMES.items()])) 65 | 66 | shapes_file = os.path.join(model_dir, DEFAULT_MODEL_FILENAMES['shapes']) 67 | preferred_batch_size = preferred_batch_size or os.environ.get(PREFERRED_BATCH_SIZE_PARAM) 68 | data_names, data_shapes = read_data_shapes(shapes_file, preferred_batch_size) 69 | 70 | sym, args, aux = mx.model.load_checkpoint(os.path.join(model_dir, DEFAULT_MODEL_NAME), 0) 71 | 72 | ctx = mx.eia() if os.environ.get(INFERENCE_ACCELERATOR_PRESENT_ENV) == 'true' else get_default_context() 73 | 74 | mod = mx.mod.Module(symbol=sym, context=ctx, data_names=data_names, label_names=None) 75 | mod.bind(for_training=False, data_shapes=data_shapes) 76 | mod.set_params(args, aux, allow_missing=True) 77 | 78 | return mod 79 | 80 | def default_input_fn(self, input_data, content_type): 81 | """Take request data and deserialize it into an MXNet NDArray for prediction. 82 | When an InvokeEndpoint operation is made against an Endpoint running SageMaker model server, 83 | the model server receives two pieces of information: 84 | 85 | - The request's content type, for example "application/json" 86 | - The request data 87 | 88 | The ``input_fn`` is responsible for preprocessing request data before prediction. 89 | 90 | Args: 91 | input_data (obj): the request data 92 | content_type (str): the request's content type 93 | 94 | Returns: 95 | mxnet.nd.array: an MXNet NDArray 96 | 97 | Raises: 98 | sagemaker_inference.errors.UnsupportedFormatError: if an unsupported content type is used. 99 | 100 | """ 101 | if content_type in self.VALID_CONTENT_TYPES: 102 | np_array = decoder.decode(input_data, content_type) 103 | return mx.nd.array(np_array).as_in_context(get_default_context()) 104 | else: 105 | raise errors.UnsupportedFormatError(content_type) 106 | 107 | def default_output_fn(self, prediction, accept): 108 | """Serialize the prediction into a response. 109 | 110 | Args: 111 | prediction (mxnet.nd.array): an MXNet NDArray that is the result of a prediction 112 | accept (str): the accept content type expected by the client 113 | 114 | Returns: 115 | obj: prediction data. 116 | 117 | Raises: 118 | sagemaker_inference.errors.UnsupportedFormatError: if an unsupported content type is used. 119 | 120 | """ 121 | for content_type in parse_accept(accept): 122 | if content_type in self.VALID_CONTENT_TYPES: 123 | return encoder.encode(prediction.asnumpy().tolist(), content_type) 124 | raise errors.UnsupportedFormatError(accept) 125 | 126 | 127 | class DefaultModuleInferenceHandler(DefaultMXNetInferenceHandler): 128 | VALID_CONTENT_TYPES = (content_types.JSON, content_types.CSV, content_types.NPY) 129 | 130 | def default_input_fn(self, input_data, content_type, model=None): 131 | """Take request data and deserialize it into an object for prediction. 132 | When an InvokeEndpoint operation is made against an Endpoint running SageMaker model server, 133 | the model server receives two pieces of information: 134 | 135 | - The request's content type, for example "application/json" 136 | - The request data 137 | 138 | The ``input_fn`` is responsible for preprocessing request data before prediction. 139 | 140 | Args: 141 | input_data (obj): the request data 142 | content_type (str): the request's content type 143 | model (obj): an MXNet model 144 | 145 | Returns: 146 | mxnet.io.NDArrayIter: data ready for prediction. 147 | 148 | Raises: 149 | sagemaker_inference.errors.UnsupportedFormatError: if an unsupported content type is used. 150 | 151 | """ 152 | if content_type not in self.VALID_CONTENT_TYPES: 153 | raise errors.UnsupportedFormatError(content_type) 154 | 155 | np_array = decoder.decode(input_data, content_type) 156 | ndarray = mx.nd.array(np_array).as_in_context(get_default_context()) 157 | 158 | # We require model to only have one input 159 | [data_shape] = model.data_shapes 160 | 161 | # Reshape flattened CSV as specified by the model 162 | if content_type == content_types.CSV: 163 | _, data = data_shape 164 | # infer batch dimension from input ndarray 165 | if isinstance(data, tuple): 166 | target_shape = (-1,) + data[1:] 167 | elif isinstance(data, list): 168 | target_shape = [-1] + data[1:] 169 | else: 170 | raise TypeError("Input shape has to be list or tuple.") 171 | 172 | ndarray = ndarray.reshape(target_shape) 173 | 174 | # Batch size is the first dimension of model input 175 | model_batch_size = data_shape[1][0] 176 | 177 | # no padding when batch size is 1 178 | pad_rows = 0 if model_batch_size == 1 else model_batch_size - ndarray.shape[0] % model_batch_size 179 | 180 | model_input = mx.io.NDArrayIter(ndarray, batch_size=model_batch_size, last_batch_handle='pad') 181 | 182 | if pad_rows: 183 | # Update the getpad method on the model_input data iterator to return the amount of 184 | # padding. MXNet will ignore the last getpad() rows during Module predict. 185 | def _getpad(): 186 | return pad_rows 187 | 188 | model_input.getpad = _getpad 189 | 190 | return model_input 191 | 192 | def default_predict_fn(self, data, model): 193 | """Use the model to create a prediction for the data. 194 | 195 | Args: 196 | data (mxnet.io.NDArrayIter): input data for prediction 197 | model (mxnet.module.BaseModule): an MXNet Module 198 | 199 | Returns: 200 | list: the prediction result. This will be either a list of ``mxnet.nd.array`` or 201 | a list of lists of ``mxnet.nd.array`` 202 | 203 | """ 204 | return model.predict(data) 205 | 206 | 207 | class DefaultGluonBlockInferenceHandler(DefaultMXNetInferenceHandler): 208 | def default_predict_fn(self, data, block): 209 | """Use the model to create a prediction for the data. 210 | 211 | Args: 212 | data (mxnet.nd.array): input data for prediction (deserialized by ``input_fn``) 213 | block (mxnet.gluon.block.Block): a Gluon neural network 214 | 215 | Returns: 216 | mxnet.nd.array: the prediction result 217 | 218 | """ 219 | return block(data) 220 | -------------------------------------------------------------------------------- /src/sagemaker_mxnet_serving_container/handler_service.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import importlib 16 | import logging 17 | import os 18 | 19 | import mxnet as mx 20 | from sagemaker_inference import environment 21 | from sagemaker_inference.default_handler_service import DefaultHandlerService 22 | from sagemaker_inference.transformer import Transformer 23 | 24 | from sagemaker_mxnet_serving_container.default_inference_handler import DefaultGluonBlockInferenceHandler, \ 25 | DefaultMXNetInferenceHandler 26 | from sagemaker_mxnet_serving_container.mxnet_module_transformer import MXNetModuleTransformer 27 | 28 | PYTHON_PATH_ENV = "PYTHONPATH" 29 | logging.basicConfig(level=logging.ERROR) 30 | 31 | 32 | class HandlerService(DefaultHandlerService): 33 | """Handler service that is executed by the model server. 34 | 35 | Determines specific default inference handlers to use based on the type MXNet model being used. 36 | 37 | This class extends ``DefaultHandlerService``, which define the following: 38 | - The ``handle`` method is invoked for all incoming inference requests to the model server. 39 | - The ``initialize`` method is invoked at model server start up. 40 | 41 | Based on: https://github.com/awslabs/multi-model-server/blob/master/docs/custom_service.md 42 | 43 | """ 44 | def __init__(self): 45 | self._service = None 46 | 47 | @staticmethod 48 | def _user_module_transformer(model_dir=environment.model_dir): 49 | try: 50 | user_module = importlib.import_module(environment.Environment().module_name) 51 | except ModuleNotFoundError as e: 52 | logging.error("import_module exception: {}".format(e)) 53 | raise ValueError('import_module exception: {}'.format(e)) 54 | 55 | if hasattr(user_module, 'transform_fn'): 56 | return Transformer(default_inference_handler=DefaultMXNetInferenceHandler()) 57 | 58 | model_fn = getattr(user_module, 'model_fn', DefaultMXNetInferenceHandler().default_model_fn) 59 | 60 | model = model_fn(model_dir) 61 | if isinstance(model, mx.module.BaseModule): 62 | return MXNetModuleTransformer() 63 | elif isinstance(model, mx.gluon.block.Block): 64 | return Transformer(default_inference_handler=DefaultGluonBlockInferenceHandler()) 65 | else: 66 | raise ValueError('Unsupported model type: {}. Did you forget to implement ' 67 | '`transform_fn` or `model_fn` in your entry-point?' 68 | .format(model.__class__.__name__)) 69 | 70 | def initialize(self, context): 71 | """Calls the Transformer method that validates the user module against 72 | the SageMaker inference contract. 73 | """ 74 | properties = context.system_properties 75 | model_dir = properties.get("model_dir") 76 | 77 | # add model_dir/code to python path 78 | code_dir_path = "{}:".format(model_dir + '/code') 79 | if PYTHON_PATH_ENV in os.environ: 80 | os.environ[PYTHON_PATH_ENV] = code_dir_path + os.environ[PYTHON_PATH_ENV] 81 | else: 82 | os.environ[PYTHON_PATH_ENV] = code_dir_path 83 | 84 | self._service = self._user_module_transformer(model_dir) 85 | super(HandlerService, self).initialize(context) 86 | -------------------------------------------------------------------------------- /src/sagemaker_mxnet_serving_container/mxnet_module_transformer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import inspect 16 | 17 | from sagemaker_inference.transformer import Transformer 18 | 19 | from sagemaker_mxnet_serving_container.default_inference_handler import DefaultModuleInferenceHandler 20 | 21 | 22 | class MXNetModuleTransformer(Transformer): 23 | """Custom ``Transformer``, which passes the model object to the input_fn, 24 | as required in the default_input_fn for Module based MXNet models. 25 | """ 26 | def __init__(self): 27 | super(MXNetModuleTransformer, self).__init__(DefaultModuleInferenceHandler()) 28 | 29 | def _default_transform_fn(self, model, input_data, content_type, accept): 30 | data = self._call_input_fn(input_data, content_type, model) 31 | prediction = self._predict_fn(data, model) 32 | result = self._output_fn(prediction, accept) 33 | return result 34 | 35 | # The default_input_fn for Modules requires access to the model object. 36 | # Originally, the input_fn allowed for input_data, content_type and model. 37 | # https://github.com/aws/sagemaker-python-sdk/tree/a6b26d63e1420bf2283d7981f46a9f58b9163165#model-serving 38 | # However, the current documentation indicates only two parameters: 39 | # input_data and content_type 40 | # Thus the following has to be added to not break input_fn that are 41 | # abiding with the current documentation. 42 | def _call_input_fn(self, input_data, content_type, model): 43 | try: # PY3 44 | argspec = inspect.getfullargspec(self._input_fn) 45 | 46 | except AttributeError: # PY2 47 | argspec = inspect.getargspec(self._input_fn) 48 | 49 | if 'model' in argspec.args: 50 | return self._input_fn(input_data, content_type, model) 51 | 52 | return self._input_fn(input_data, content_type) 53 | -------------------------------------------------------------------------------- /src/sagemaker_mxnet_serving_container/serving.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import os 16 | from subprocess import CalledProcessError 17 | 18 | from retrying import retry 19 | from sagemaker_inference import model_server 20 | 21 | from sagemaker_mxnet_serving_container import handler_service 22 | 23 | HANDLER_SERVICE = handler_service.__name__ 24 | 25 | DEFAULT_ENV_VARS = { 26 | 'MXNET_CPU_WORKER_NTHREADS': '1', 27 | 'MXNET_CPU_PRIORITY_NTHREADS': '1', 28 | 'MXNET_KVSTORE_REDUCTION_NTHREADS': '1', 29 | 'OMP_NUM_THREADS': '1', 30 | } 31 | 32 | 33 | def _update_mxnet_env_vars(): 34 | for k, v in DEFAULT_ENV_VARS.items(): 35 | if k not in os.environ: 36 | os.environ[k] = v 37 | 38 | 39 | def _retry_if_error(exception): 40 | return isinstance(exception, CalledProcessError or OSError) 41 | 42 | 43 | @retry(stop_max_delay=1000 * 50, 44 | retry_on_exception=_retry_if_error) 45 | def _start_model_server(): 46 | # there's a race condition that causes the model server command to 47 | # sometimes fail with 'bad address'. more investigation needed 48 | # retry starting mms until it's ready 49 | model_server.start_model_server(handler_service=HANDLER_SERVICE) 50 | 51 | 52 | def main(): 53 | _update_mxnet_env_vars() 54 | _start_model_server() 55 | -------------------------------------------------------------------------------- /src/sagemaker_mxnet_serving_container/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import json 16 | 17 | import mxnet as mx 18 | 19 | 20 | def get_default_context(): 21 | """Get the default context. 22 | 23 | Returns: 24 | context : The corresponding CPU context. 25 | 26 | """ 27 | # TODO mxnet ctx - better default, allow user control 28 | return mx.cpu() 29 | 30 | 31 | def read_data_shapes(path, preferred_batch_size=1): 32 | """Read the data name and data shape required by the MXNet module. 33 | 34 | Args: 35 | path (str): an MXNet NDArray that is the result of a prediction 36 | preferred_batch_size (int): the accept content type expected by the client 37 | 38 | Returns: 39 | tuple: A list of names for data required by the module along with 40 | a list of (name, shape) pairs specifying the data inputs to this module. 41 | 42 | """ 43 | with open(path, 'r') as f: 44 | signatures = json.load(f) 45 | 46 | data_names = [] 47 | data_shapes = [] 48 | 49 | for s in signatures: 50 | name = s['name'] 51 | data_names.append(name) 52 | 53 | shape = s['shape'] 54 | 55 | if preferred_batch_size: 56 | shape[0] = preferred_batch_size 57 | 58 | data_shapes.append((name, shape)) 59 | 60 | return data_names, data_shapes 61 | 62 | 63 | # TODO (@bvveeram): This function is also implemented in the 64 | # sagemaker-inference package. Once the MXNet 1.4 EIA image is deprecated, pin 65 | # the sagemaker-inference dependency to >= 1.5.0, remove this function, and use 66 | # the equivalent function in sagemaker-inference. 67 | def parse_accept(accept): 68 | """Parses the Accept header sent with a request. 69 | 70 | Args: 71 | accept (str): the value of an Accept header. 72 | Returns: 73 | (list): A list containing the MIME types that the client is able to 74 | understand. 75 | """ 76 | return accept.replace(" ", "").split(",") 77 | -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import logging 16 | import os 17 | 18 | import boto3 19 | import pytest 20 | from sagemaker import LocalSession, Session 21 | from sagemaker.mxnet import MXNet 22 | 23 | from utils import image_utils 24 | 25 | logger = logging.getLogger(__name__) 26 | logging.getLogger('boto').setLevel(logging.INFO) 27 | logging.getLogger('botocore').setLevel(logging.INFO) 28 | logging.getLogger('factory.py').setLevel(logging.INFO) 29 | logging.getLogger('auth.py').setLevel(logging.INFO) 30 | logging.getLogger('connectionpool.py').setLevel(logging.INFO) 31 | 32 | DIR_PATH = os.path.dirname(os.path.realpath(__file__)) 33 | 34 | # These regions have some p2 and p3 instances, but not enough for automated testing 35 | NO_P2_REGIONS = ['ca-central-1', 'eu-central-1', 'eu-west-2', 'us-west-1', 'eu-west-3', 36 | 'eu-north-1', 'sa-east-1', 'ap-east-1', 'me-south-1'] 37 | NO_P3_REGIONS = ['ap-southeast-1', 'ap-southeast-2', 'ap-south-1', 'ca-central-1', 38 | 'eu-central-1', 'eu-west-2', 'us-west-1', 'eu-west-3', 'eu-north-1', 39 | 'sa-east-1', 'ap-east-1', 'me-south-1'] 40 | 41 | 42 | def pytest_addoption(parser): 43 | parser.addoption('--build-image', '-B', action='store_true') 44 | parser.addoption('--push-image', '-P', action='store_true') 45 | parser.addoption('--dockerfile-type', '-T', 46 | choices=['dlc.cpu', 'dlc.gpu', 'mxnet.cpu', 'dlc.eia'], 47 | default='mxnet.cpu') 48 | parser.addoption('--dockerfile', '-D', default=None) 49 | parser.addoption('--docker-base-name', default='sagemaker-mxnet-inference') 50 | parser.addoption('--region', default='us-west-2') 51 | parser.addoption('--framework-version', default=MXNet.LATEST_VERSION) 52 | parser.addoption('--py-version', default='3', choices=['2', '3', '2,3']) 53 | parser.addoption('--processor', default='cpu', choices=['gpu', 'cpu', 'cpu,gpu']) 54 | parser.addoption('--aws-id', default=None) 55 | parser.addoption('--instance-type', default=None) 56 | parser.addoption('--accelerator-type', default=None) 57 | # If not specified, will default to {framework-version}-{processor}-py{py-version} 58 | parser.addoption('--tag', default=None) 59 | 60 | 61 | @pytest.fixture(scope='session', name='dockerfile_type') 62 | def fixture_dockerfile_type(request): 63 | return request.config.getoption('--dockerfile-type') 64 | 65 | 66 | @pytest.fixture(scope='session', name='dockerfile') 67 | def fixture_dockerfile(request, dockerfile_type): 68 | dockerfile = request.config.getoption('--dockerfile') 69 | return dockerfile if dockerfile else 'Dockerfile.{}'.format(dockerfile_type) 70 | 71 | 72 | @pytest.fixture(scope='session', name='build_image', autouse=True) 73 | def fixture_build_image(request, framework_version, dockerfile, image_uri, region): 74 | build_image = request.config.getoption('--build-image') 75 | if build_image: 76 | return image_utils.build_image(framework_version=framework_version, 77 | dockerfile=dockerfile, 78 | image_uri=image_uri, 79 | region=region, 80 | cwd=os.path.join(DIR_PATH, '..')) 81 | 82 | return image_uri 83 | 84 | 85 | @pytest.fixture(scope='session', name='push_image', autouse=True) 86 | def fixture_push_image(request, image_uri, region, aws_id): 87 | push_image = request.config.getoption('--push-image') 88 | if push_image: 89 | return image_utils.push_image(image_uri, region, aws_id) 90 | return None 91 | 92 | 93 | def pytest_generate_tests(metafunc): 94 | if 'py_version' in metafunc.fixturenames: 95 | py_version_params = ['py' + v for v in metafunc.config.getoption('--py-version').split(',')] 96 | metafunc.parametrize('py_version', py_version_params, scope='session') 97 | 98 | if 'processor' in metafunc.fixturenames: 99 | processor_params = metafunc.config.getoption('--processor').split(',') 100 | metafunc.parametrize('processor', processor_params, scope='session') 101 | 102 | 103 | @pytest.fixture(scope='session') 104 | def docker_base_name(request): 105 | return request.config.getoption('--docker-base-name') 106 | 107 | 108 | @pytest.fixture(scope='session') 109 | def region(request): 110 | return request.config.getoption('--region') 111 | 112 | 113 | @pytest.fixture(scope='session') 114 | def framework_version(request): 115 | return request.config.getoption('--framework-version') 116 | 117 | 118 | @pytest.fixture(scope='session') 119 | def aws_id(request): 120 | return request.config.getoption('--aws-id') 121 | 122 | 123 | @pytest.fixture(scope='session') 124 | def tag(request, framework_version, processor, py_version): 125 | provided_tag = request.config.getoption('--tag') 126 | default_tag = '{}-{}-{}'.format(framework_version, processor, py_version) 127 | return provided_tag if provided_tag is not None else default_tag 128 | 129 | 130 | @pytest.fixture(scope='session') 131 | def instance_type(request, processor): 132 | provided_instance_type = request.config.getoption('--instance-type') 133 | default_instance_type = 'ml.c4.xlarge' if processor == 'cpu' else 'ml.p2.xlarge' 134 | return provided_instance_type if provided_instance_type is not None else default_instance_type 135 | 136 | 137 | @pytest.fixture(scope='session') 138 | def accelerator_type(request): 139 | return request.config.getoption('--accelerator-type') 140 | 141 | 142 | @pytest.fixture(name='docker_registry', scope='session') 143 | def fixture_docker_registry(aws_id, region): 144 | return '{}.dkr.ecr.{}.amazonaws.com'.format(aws_id, region) if aws_id else None 145 | 146 | 147 | @pytest.fixture(name='image_uri', scope='session') 148 | def fixture_image_uri(docker_registry, docker_base_name, tag): 149 | if docker_registry: 150 | return '{}/{}:{}'.format(docker_registry, docker_base_name, tag) 151 | return '{}:{}'.format(docker_base_name, tag) 152 | 153 | 154 | @pytest.fixture(scope='session') 155 | def sagemaker_session(region): 156 | return Session(boto_session=boto3.Session(region_name=region)) 157 | 158 | 159 | @pytest.fixture(scope='session') 160 | def sagemaker_local_session(region): 161 | return LocalSession(boto_session=boto3.Session(region_name=region)) 162 | 163 | 164 | @pytest.fixture(scope='session') 165 | def local_instance_type(processor): 166 | return 'local' if processor == 'cpu' else 'local_gpu' 167 | 168 | 169 | @pytest.fixture(autouse=True) 170 | def skip_gpu_instance_restricted_regions(region, instance_type): 171 | no_p2 = region in NO_P2_REGIONS and instance_type.startswith('ml.p2') 172 | no_p3 = region in NO_P3_REGIONS and instance_type.startswith('ml.p3') 173 | if no_p2 or no_p3: 174 | pytest.skip('Skipping GPU test in region {} to avoid insufficient capacity'.format(region)) 175 | -------------------------------------------------------------------------------- /test/container/1.4.1/Dockerfile.dlc.eia: -------------------------------------------------------------------------------- 1 | ARG region 2 | FROM 763104351884.dkr.ecr.$region.amazonaws.com/mxnet-inference-eia:1.4.1-cpu-py3 3 | 4 | COPY dist/sagemaker_mxnet_inference-*.tar.gz /sagemaker_mxnet_inference.tar.gz 5 | RUN pip install --upgrade --no-cache-dir /sagemaker_mxnet_inference.tar.gz && \ 6 | rm /sagemaker_mxnet_inference.tar.gz 7 | 8 | # use sagemaker-inference version comaptible with MMS_VERSION=1.0.5 9 | RUN pip install --upgrade --no-cache-dir sagemaker-inference==1.1.0 10 | -------------------------------------------------------------------------------- /test/container/1.7.0/Dockerfile.dlc.cpu: -------------------------------------------------------------------------------- 1 | ARG region 2 | FROM 763104351884.dkr.ecr.$region.amazonaws.com/mxnet-inference:1.6.0-cpu-py2 3 | 4 | COPY dist/sagemaker_mxnet_inference-*.tar.gz /sagemaker_mxnet_inference.tar.gz 5 | RUN pip install --upgrade --no-cache-dir /sagemaker_mxnet_inference.tar.gz && \ 6 | rm /sagemaker_mxnet_inference.tar.gz 7 | -------------------------------------------------------------------------------- /test/container/1.7.0/Dockerfile.dlc.gpu: -------------------------------------------------------------------------------- 1 | ARG region 2 | FROM 763104351884.dkr.ecr.$region.amazonaws.com/mxnet-inference:1.6.0-gpu-py3 3 | 4 | COPY dist/sagemaker_mxnet_inference-*.tar.gz /sagemaker_mxnet_inference.tar.gz 5 | RUN pip install --upgrade --no-cache-dir /sagemaker_mxnet_inference.tar.gz && \ 6 | rm /sagemaker_mxnet_inference.tar.gz 7 | -------------------------------------------------------------------------------- /test/container/1.7.0/Dockerfile.mxnet.cpu: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 2 | 3 | LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true 4 | LABEL com.amazonaws.sagemaker.capabilities.multi-models=true 5 | 6 | ARG MMS_VERSION=1.1.6 7 | 8 | ENV SAGEMAKER_SERVING_MODULE sagemaker_mxnet_serving_container.serving:main 9 | ENV TEMP=/home/model-server/tmp 10 | 11 | # No interactive mode to install OpenCV 12 | ENV DEBIAN_FRONTEND=noninteractive 13 | 14 | RUN apt-get update \ 15 | && apt-get install -y --no-install-recommends \ 16 | software-properties-common \ 17 | build-essential \ 18 | ca-certificates \ 19 | curl \ 20 | git \ 21 | openssh-client \ 22 | openssh-server \ 23 | vim \ 24 | wget \ 25 | python3-dev \ 26 | && ln -s -f /usr/bin/python3 /usr/bin/python \ 27 | && ln -s /usr/local/bin/pip3 /usr/bin/pip 28 | 29 | RUN apt-get install -y --no-install-recommends \ 30 | libgl1-mesa-glx \ 31 | libglib2.0-0 \ 32 | libopencv-dev \ 33 | libsm6 \ 34 | libxext6 \ 35 | libxrender-dev \ 36 | openjdk-8-jdk-headless \ 37 | && apt-get clean \ 38 | && rm -rf /var/lib/apt/lists/* 39 | 40 | # MXNet requires pip 19.3.1 due to being backwards compatible 41 | # with Python2 42 | RUN cd /tmp && \ 43 | curl -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ 44 | python get-pip.py 'pip==19.1' && rm get-pip.py 45 | 46 | RUN pip install --upgrade pip 47 | RUN pip install --no-cache mxnet==1.7.0.post1 \ 48 | multi-model-server==$MMS_VERSION 49 | 50 | COPY dist/sagemaker_mxnet_inference-*.tar.gz /sagemaker_mxnet_inference.tar.gz 51 | RUN pip install --no-cache-dir /sagemaker_mxnet_inference.tar.gz && \ 52 | rm /sagemaker_mxnet_inference.tar.gz 53 | 54 | # This is here to make our installed version of OpenCV work. 55 | # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 56 | RUN ln -s /dev/null /dev/raw1394 57 | 58 | RUN useradd -m model-server \ 59 | && mkdir -p /home/model-server/tmp \ 60 | && chown -R model-server /home/model-server 61 | 62 | COPY artifacts/mms-entrypoint.py /usr/local/bin/dockerd-entrypoint.py 63 | COPY artifacts/config.properties /home/model-server 64 | 65 | RUN chmod +x /usr/local/bin/dockerd-entrypoint.py 66 | 67 | EXPOSE 8080 8081 68 | ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] 69 | CMD ["multi-model-server", "--start", "--mms-config", "/home/model-server/config.properties"] 70 | -------------------------------------------------------------------------------- /test/integration/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import os 16 | 17 | RESOURCE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'resources')) 18 | 19 | # EI is currently only supported in the following regions 20 | # regions were derived from https://aws.amazon.com/machine-learning/elastic-inference/pricing/ 21 | EI_SUPPORTED_REGIONS = ['us-east-1', 'us-east-2', 'us-west-2', 'eu-west-1', 'ap-northeast-1', 'ap-northeast-2'] 22 | -------------------------------------------------------------------------------- /test/integration/local/test_default_model_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import os 16 | 17 | import pytest 18 | import requests 19 | from sagemaker.mxnet.model import MXNetModel 20 | 21 | from integration import RESOURCE_PATH 22 | from utils import local_mode_utils 23 | 24 | DEFAULT_HANDLER_PATH = os.path.join(RESOURCE_PATH, 'default_handlers') 25 | MODEL_PATH = os.path.join(DEFAULT_HANDLER_PATH, 'model') 26 | SCRIPT_PATH = os.path.join(MODEL_PATH, 'code', 'empty_module.py') 27 | 28 | 29 | @pytest.fixture(scope='module') 30 | def predictor(image_uri, sagemaker_local_session, local_instance_type): 31 | model = MXNetModel('file://{}'.format(MODEL_PATH), 32 | 'SageMakerRole', 33 | SCRIPT_PATH, 34 | image=image_uri, 35 | sagemaker_session=sagemaker_local_session) 36 | 37 | with local_mode_utils.lock(): 38 | try: 39 | predictor = model.deploy(1, local_instance_type) 40 | yield predictor 41 | finally: 42 | predictor.delete_endpoint() 43 | 44 | 45 | def test_default_model_fn(predictor): 46 | input = [[1, 2]] 47 | output = predictor.predict(input) 48 | assert [[4.9999918937683105]] == output 49 | 50 | 51 | def test_default_model_fn_content_type(predictor): 52 | r = requests.post('http://localhost:8080/invocations', json=[[1, 2]]) 53 | assert 'application/json' == r.headers['Content-Type'] 54 | assert [[4.9999918937683105]] == r.json() 55 | -------------------------------------------------------------------------------- /test/integration/local/test_hosting.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import os 16 | 17 | from sagemaker.mxnet.model import MXNetModel 18 | from sagemaker.predictor import StringDeserializer 19 | 20 | from integration import RESOURCE_PATH 21 | from utils import local_mode_utils 22 | 23 | HOSTING_RESOURCE_PATH = os.path.join(RESOURCE_PATH, 'dummy_hosting') 24 | MODEL_PATH = os.path.join(HOSTING_RESOURCE_PATH, 'code') 25 | SCRIPT_PATH = os.path.join(HOSTING_RESOURCE_PATH, 'code', 'dummy_hosting_module.py') 26 | 27 | 28 | # The image should use the model_fn and transform_fn defined 29 | # in the user-provided script when serving. 30 | def test_hosting(image_uri, sagemaker_local_session, local_instance_type): 31 | model = MXNetModel('file://{}'.format(MODEL_PATH), 32 | 'SageMakerRole', 33 | SCRIPT_PATH, 34 | image=image_uri, 35 | sagemaker_session=sagemaker_local_session) 36 | 37 | with local_mode_utils.lock(): 38 | try: 39 | predictor = model.deploy(1, local_instance_type) 40 | predictor.serializer = None 41 | predictor.deserializer = StringDeserializer() 42 | predictor.accept = None 43 | predictor.content_type = None 44 | 45 | input = 'some data' 46 | output = predictor.predict(input) 47 | assert input == output 48 | finally: 49 | predictor.delete_endpoint() 50 | -------------------------------------------------------------------------------- /test/integration/sagemaker/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/sagemaker-mxnet-inference-toolkit/addf27925e3f7d51c05be05a31d16a64d88839f9/test/integration/sagemaker/__init__.py -------------------------------------------------------------------------------- /test/integration/sagemaker/test_batch_transform.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import json 16 | import os 17 | from urllib.parse import urlparse 18 | 19 | from sagemaker import utils 20 | from sagemaker.mxnet.model import MXNetModel 21 | 22 | from integration import RESOURCE_PATH 23 | from integration.sagemaker import timeout 24 | 25 | SCRIPT_PATH = os.path.join(RESOURCE_PATH, 'default_handlers', 'model', 'code', 'empty_module.py') 26 | MNIST_PATH = os.path.join(RESOURCE_PATH, 'mnist') 27 | MODEL_PATH = os.path.join(MNIST_PATH, 'model', 'model.tar.gz') 28 | 29 | DATA_FILE = '07.csv' 30 | DATA_PATH = os.path.join(MNIST_PATH, 'images', DATA_FILE) 31 | 32 | 33 | def test_batch_transform(sagemaker_session, image_uri, instance_type, framework_version): 34 | s3_prefix = 'mxnet-serving/mnist' 35 | model_data = sagemaker_session.upload_data(path=MODEL_PATH, key_prefix=s3_prefix) 36 | model = MXNetModel(model_data, 37 | 'SageMakerRole', 38 | SCRIPT_PATH, 39 | image=image_uri, 40 | framework_version=framework_version, 41 | sagemaker_session=sagemaker_session) 42 | 43 | transformer = model.transformer(1, instance_type) 44 | with timeout.timeout_and_delete_model_with_transformer(transformer, sagemaker_session, minutes=20): 45 | input_data = sagemaker_session.upload_data(path=DATA_PATH, key_prefix=s3_prefix) 46 | 47 | job_name = utils.unique_name_from_base('test-mxnet-serving-batch') 48 | transformer.transform(input_data, content_type='text/csv', job_name=job_name) 49 | transformer.wait() 50 | 51 | prediction = _transform_result(sagemaker_session.boto_session, transformer.output_path) 52 | assert prediction == 7 53 | 54 | 55 | def _transform_result(boto_session, output_path): 56 | s3 = boto_session.resource('s3', region_name=boto_session.region_name) 57 | 58 | parsed_url = urlparse(output_path) 59 | bucket_name = parsed_url.netloc 60 | prefix = parsed_url.path[1:] 61 | 62 | output_obj = s3.Object(bucket_name, '{}/{}.out'.format(prefix, DATA_FILE)) 63 | output = output_obj.get()['Body'].read().decode('utf-8') 64 | 65 | probabilities = json.loads(output)[0] 66 | return probabilities.index(max(probabilities)) 67 | -------------------------------------------------------------------------------- /test/integration/sagemaker/test_elastic_inference.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import os 16 | 17 | import pytest 18 | from sagemaker import utils 19 | from sagemaker.mxnet import MXNetModel 20 | 21 | from integration import EI_SUPPORTED_REGIONS, RESOURCE_PATH 22 | from integration.sagemaker.timeout import timeout_and_delete_endpoint_by_name 23 | 24 | DEFAULT_HANDLER_PATH = os.path.join(RESOURCE_PATH, 'default_handlers') 25 | MODEL_PATH = os.path.join(DEFAULT_HANDLER_PATH, 'model.tar.gz') 26 | SCRIPT_PATH = os.path.join(DEFAULT_HANDLER_PATH, 'model', 'code', 'empty_module.py') 27 | 28 | 29 | @pytest.fixture(autouse=True) 30 | def skip_if_no_accelerator(accelerator_type): 31 | if accelerator_type is None: 32 | pytest.skip('Skipping because accelerator type was not provided') 33 | 34 | 35 | @pytest.fixture(autouse=True) 36 | def skip_if_non_supported_ei_region(region): 37 | if region not in EI_SUPPORTED_REGIONS: 38 | pytest.skip('EI is not supported in {}'.format(region)) 39 | 40 | 41 | @pytest.mark.skip_if_non_supported_ei_region() 42 | @pytest.mark.skip_if_no_accelerator() 43 | def test_elastic_inference(image_uri, sagemaker_session, instance_type, accelerator_type, framework_version): 44 | endpoint_name = utils.unique_name_from_base('test-mxnet-ei') 45 | 46 | with timeout_and_delete_endpoint_by_name(endpoint_name=endpoint_name, 47 | sagemaker_session=sagemaker_session, 48 | minutes=20): 49 | prefix = 'mxnet-serving/default-handlers' 50 | model_data = sagemaker_session.upload_data(path=MODEL_PATH, key_prefix=prefix) 51 | model = MXNetModel(model_data=model_data, 52 | entry_point=SCRIPT_PATH, 53 | role='SageMakerRole', 54 | image=image_uri, 55 | framework_version=framework_version, 56 | sagemaker_session=sagemaker_session) 57 | 58 | predictor = model.deploy(initial_instance_count=1, 59 | instance_type=instance_type, 60 | accelerator_type=accelerator_type, 61 | endpoint_name=endpoint_name) 62 | 63 | output = predictor.predict([[1, 2]]) 64 | assert [[4.9999918937683105]] == output 65 | -------------------------------------------------------------------------------- /test/integration/sagemaker/test_hosting.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import os 16 | 17 | from sagemaker import utils 18 | from sagemaker.mxnet.model import MXNetModel 19 | 20 | from integration import RESOURCE_PATH 21 | from integration.sagemaker import timeout 22 | 23 | DEFAULT_HANDLER_PATH = os.path.join(RESOURCE_PATH, 'default_handlers') 24 | MODEL_PATH = os.path.join(DEFAULT_HANDLER_PATH, 'model.tar.gz') 25 | SCRIPT_PATH = os.path.join(DEFAULT_HANDLER_PATH, 'model', 'code', 'empty_module.py') 26 | 27 | 28 | def test_hosting(sagemaker_session, image_uri, instance_type, framework_version): 29 | prefix = 'mxnet-serving/default-handlers' 30 | model_data = sagemaker_session.upload_data(path=MODEL_PATH, key_prefix=prefix) 31 | model = MXNetModel(model_data, 32 | 'SageMakerRole', 33 | SCRIPT_PATH, 34 | image=image_uri, 35 | framework_version=framework_version, 36 | sagemaker_session=sagemaker_session) 37 | 38 | endpoint_name = utils.unique_name_from_base('test-mxnet-serving') 39 | with timeout.timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): 40 | predictor = model.deploy(1, instance_type, endpoint_name=endpoint_name) 41 | 42 | output = predictor.predict([[1, 2]]) 43 | assert [[4.9999918937683105]] == output 44 | -------------------------------------------------------------------------------- /test/integration/sagemaker/timeout.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | from contextlib import contextmanager 16 | import logging 17 | import signal 18 | from time import sleep 19 | 20 | from awslogs.core import AWSLogs 21 | from botocore.exceptions import ClientError 22 | 23 | LOGGER = logging.getLogger('timeout') 24 | 25 | 26 | class TimeoutError(Exception): 27 | pass 28 | 29 | 30 | @contextmanager 31 | def timeout(seconds=0, minutes=0, hours=0): 32 | """ 33 | Add a signal-based timeout to any block of code. 34 | If multiple time units are specified, they will be added together to determine time limit. 35 | Usage: 36 | with timeout(seconds=5): 37 | my_slow_function(...) 38 | Args: 39 | - seconds: The time limit, in seconds. 40 | - minutes: The time limit, in minutes. 41 | - hours: The time limit, in hours. 42 | """ 43 | 44 | limit = seconds + 60 * minutes + 3600 * hours 45 | 46 | def handler(signum, frame): 47 | raise TimeoutError('timed out after {} seconds'.format(limit)) 48 | 49 | try: 50 | signal.signal(signal.SIGALRM, handler) 51 | signal.alarm(limit) 52 | 53 | yield 54 | finally: 55 | signal.alarm(0) 56 | 57 | 58 | @contextmanager 59 | def timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, seconds=0, minutes=45, hours=0): 60 | with timeout(seconds=seconds, minutes=minutes, hours=hours) as t: 61 | no_errors = False 62 | try: 63 | yield [t] 64 | no_errors = True 65 | finally: 66 | attempts = 3 67 | 68 | while attempts > 0: 69 | attempts -= 1 70 | try: 71 | sagemaker_session.delete_endpoint(endpoint_name) 72 | LOGGER.info('deleted endpoint {}'.format(endpoint_name)) 73 | 74 | _show_logs(endpoint_name, 'Endpoints', sagemaker_session) 75 | if no_errors: 76 | _cleanup_logs(endpoint_name, 'Endpoints', sagemaker_session) 77 | break 78 | except ClientError as ce: 79 | if ce.response['Error']['Code'] == 'ValidationException': 80 | # avoids the inner exception to be overwritten 81 | pass 82 | # trying to delete the resource again in 10 seconds 83 | sleep(10) 84 | 85 | 86 | @contextmanager 87 | def timeout_and_delete_model_with_transformer(transformer, sagemaker_session, seconds=0, minutes=0, hours=0): 88 | with timeout(seconds=seconds, minutes=minutes, hours=hours) as t: 89 | no_errors = False 90 | try: 91 | yield [t] 92 | no_errors = True 93 | finally: 94 | attempts = 3 95 | 96 | while attempts > 0: 97 | attempts -= 1 98 | try: 99 | transformer.delete_model() 100 | LOGGER.info('deleted SageMaker model {}'.format(transformer.model_name)) 101 | 102 | _show_logs(transformer.model_name, 'Models', sagemaker_session) 103 | if no_errors: 104 | _cleanup_logs(transformer.model_name, 'Models', sagemaker_session) 105 | break 106 | except ClientError as ce: 107 | if ce.response['Error']['Code'] == 'ValidationException': 108 | pass 109 | sleep(10) 110 | 111 | 112 | def _show_logs(resource_name, resource_type, sagemaker_session): 113 | log_group = '/aws/sagemaker/{}/{}'.format(resource_type, resource_name) 114 | try: 115 | # print out logs before deletion for debuggability 116 | LOGGER.info('cloudwatch logs for log group {}:'.format(log_group)) 117 | logs = AWSLogs(log_group_name=log_group, log_stream_name='ALL', start='1d', 118 | aws_region=sagemaker_session.boto_session.region_name) 119 | logs.list_logs() 120 | except Exception: 121 | LOGGER.exception('Failure occurred while listing cloudwatch log group %s. Swallowing exception but printing ' 122 | 'stacktrace for debugging.', log_group) 123 | 124 | 125 | def _cleanup_logs(resource_name, resource_type, sagemaker_session): 126 | log_group = '/aws/sagemaker/{}/{}'.format(resource_type, resource_name) 127 | try: 128 | # print out logs before deletion for debuggability 129 | LOGGER.info('deleting cloudwatch log group {}:'.format(log_group)) 130 | cwl_client = sagemaker_session.boto_session.client('logs') 131 | cwl_client.delete_log_group(logGroupName=log_group) 132 | LOGGER.info('deleted cloudwatch log group: {}'.format(log_group)) 133 | except Exception: 134 | LOGGER.exception('Failure occurred while cleaning up cloudwatch log group %s. ' 135 | 'Swallowing exception but printing stacktrace for debugging.', log_group) 136 | -------------------------------------------------------------------------------- /test/resources/default_handlers/model.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/sagemaker-mxnet-inference-toolkit/addf27925e3f7d51c05be05a31d16a64d88839f9/test/resources/default_handlers/model.tar.gz -------------------------------------------------------------------------------- /test/resources/default_handlers/model/code/empty_module.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | # nothing here... we are testing default model loading and handlers 15 | -------------------------------------------------------------------------------- /test/resources/default_handlers/model/model-0000.params: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/sagemaker-mxnet-inference-toolkit/addf27925e3f7d51c05be05a31d16a64d88839f9/test/resources/default_handlers/model/model-0000.params -------------------------------------------------------------------------------- /test/resources/default_handlers/model/model-shapes.json: -------------------------------------------------------------------------------- 1 | [{"shape": [1, 2], "name": "data"}] -------------------------------------------------------------------------------- /test/resources/default_handlers/model/model-symbol.json: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": [ 3 | { 4 | "op": "null", 5 | "name": "data", 6 | "inputs": [] 7 | }, 8 | { 9 | "op": "null", 10 | "name": "fc1_weight", 11 | "attr": {"num_hidden": "1"}, 12 | "inputs": [] 13 | }, 14 | { 15 | "op": "null", 16 | "name": "fc1_bias", 17 | "attr": {"num_hidden": "1"}, 18 | "inputs": [] 19 | }, 20 | { 21 | "op": "FullyConnected", 22 | "name": "fc1", 23 | "attr": {"num_hidden": "1"}, 24 | "inputs": [[0, 0, 0], [1, 0, 0], [2, 0, 0]] 25 | }, 26 | { 27 | "op": "null", 28 | "name": "lin_reg_label", 29 | "inputs": [] 30 | }, 31 | { 32 | "op": "LinearRegressionOutput", 33 | "name": "lro", 34 | "inputs": [[3, 0, 0], [4, 0, 0]] 35 | } 36 | ], 37 | "arg_nodes": [0, 1, 2, 4], 38 | "node_row_ptr": [0, 1, 2, 3, 4, 5, 6], 39 | "heads": [[5, 0, 0]], 40 | "attrs": {"mxnet_version": ["int", 1100]} 41 | } -------------------------------------------------------------------------------- /test/resources/dummy_hosting/code/dummy_hosting_module.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | 15 | class DummyModel(object): 16 | def predict(self, data): 17 | return data 18 | 19 | 20 | def model_fn(model_dir): 21 | return DummyModel() 22 | 23 | 24 | def transform_fn(model, data, input_content_type, output_content_type): 25 | return data, "application/json" 26 | -------------------------------------------------------------------------------- /test/resources/mnist/images/04.json: -------------------------------------------------------------------------------- 1 | [[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19607843458652496, 0.8784313797950745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.27450981736183167, 0.11372549086809158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4745098054409027, 0.9058823585510254, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5803921818733215, 0.658823549747467, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01568627543747425, 0.7647058963775635, 0.9058823585510254, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3764705955982208, 0.8235294222831726, 0.04313725605607033, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.2705882489681244, 0.9882352948188782, 0.5254902243614197, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4470588266849518, 0.9882352948188782, 0.08235294371843338, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1764705926179886, 0.9254902005195618, 0.8509804010391235, 0.0470588244497776, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.7529411911964417, 0.9882352948188782, 0.08235294371843338, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.658823549747467, 0.9686274528503418, 0.2078431397676468, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07058823853731155, 1.0, 0.9921568632125854, 0.08235294371843338, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3294117748737335, 0.9490196108818054, 0.8274509906768799, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5529412031173706, 0.9921568632125854, 0.7411764860153198, 0.019607843831181526, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.6627451181411743, 0.9882352948188782, 0.4156862795352936, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.125490203499794, 0.9098039269447327, 0.9803921580314636, 0.25882354378700256, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05882352963089943, 0.8823529481887817, 0.9882352948188782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5254902243614197, 0.9882352948188782, 0.8274509906768799, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.08627451211214066, 0.9882352948188782, 0.6431372761726379, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.6627451181411743, 0.9882352948188782, 0.6549019813537598, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03529411926865578, 0.800000011920929, 0.8196078538894653, 0.07058823853731155, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.08627451211214066, 0.9921568632125854, 0.9921568632125854, 0.41960784792900085, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.6627451181411743, 0.9882352948188782, 0.7803921699523926, 0.3333333432674408, 0.3333333432674408, 0.3333333432674408, 0.3333333432674408, 0.5058823823928833, 0.6431372761726379, 0.7647058963775635, 0.9882352948188782, 0.9882352948188782, 0.4156862795352936, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.16078431904315948, 0.6666666865348816, 0.9607843160629272, 0.9882352948188782, 0.9882352948188782, 0.9882352948188782, 0.9882352948188782, 0.9098039269447327, 0.9058823585510254, 0.9843137264251709, 0.9882352948188782, 0.9882352948188782, 0.03529411926865578, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1921568661928177, 0.3294117748737335, 0.3294117748737335, 0.3294117748737335, 0.3294117748737335, 0.0, 0.0, 0.6313725709915161, 0.9882352948188782, 0.9882352948188782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.49803921580314636, 0.9882352948188782, 0.9882352948188782, 0.1764705926179886, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.501960813999176, 0.9921568632125854, 0.9921568632125854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.49803921580314636, 0.9882352948188782, 0.9882352948188782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.529411792755127, 0.9882352948188782, 0.95686274766922, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.9098039269447327, 0.9254902005195618, 0.43529412150382996, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.7019608020782471, 0.25882354378700256, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]] -------------------------------------------------------------------------------- /test/resources/mnist/model/model.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/sagemaker-mxnet-inference-toolkit/addf27925e3f7d51c05be05a31d16a64d88839f9/test/resources/mnist/model/model.tar.gz -------------------------------------------------------------------------------- /test/unit/test_default_inference_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import json 16 | import os 17 | 18 | from mock import call, Mock, mock_open, patch 19 | import mxnet as mx 20 | import pytest 21 | from sagemaker_inference import content_types, errors 22 | 23 | from sagemaker_mxnet_serving_container.default_inference_handler import DefaultGluonBlockInferenceHandler, \ 24 | DefaultModuleInferenceHandler, DefaultMXNetInferenceHandler 25 | 26 | MODEL_DIR = 'foo/model' 27 | 28 | ################################################################################# 29 | # DefaultMXNetInferenceHandler Tests 30 | 31 | 32 | def test_default_mxnet_valid_content_types(): 33 | assert DefaultMXNetInferenceHandler().VALID_CONTENT_TYPES == (content_types.JSON, content_types.NPY) 34 | 35 | 36 | @patch('mxnet.cpu') 37 | @patch('mxnet.mod.Module') 38 | @patch('mxnet.model.load_checkpoint') 39 | @patch('os.path.exists', return_value=True) 40 | def test_default_model_fn(path_exists, mx_load_checkpoint, mx_module, mx_cpu): 41 | sym = Mock() 42 | args = Mock() 43 | aux = Mock() 44 | mx_load_checkpoint.return_value = [sym, args, aux] 45 | 46 | mx_context = Mock() 47 | mx_cpu.return_value = mx_context 48 | 49 | data_name = 'foo' 50 | data_shape = [1] 51 | signature = json.dumps([{'name': data_name, 'shape': data_shape}]) 52 | 53 | with patch('six.moves.builtins.open', mock_open(read_data=signature)): 54 | DefaultMXNetInferenceHandler().default_model_fn(MODEL_DIR) 55 | 56 | mx_load_checkpoint.assert_called_with(os.path.join(MODEL_DIR, 'model'), 0) 57 | 58 | init_call = call(symbol=sym, context=mx_context, data_names=[data_name], label_names=None) 59 | assert init_call in mx_module.mock_calls 60 | 61 | model = mx_module.return_value 62 | model.bind.assert_called_with(for_training=False, data_shapes=[(data_name, data_shape)]) 63 | model.set_params.assert_called_with(args, aux, allow_missing=True) 64 | 65 | 66 | @patch('mxnet.eia', create=True) 67 | @patch('mxnet.mod.Module') 68 | @patch('mxnet.model.load_checkpoint') 69 | @patch('os.path.exists', return_value=True) 70 | @patch.dict(os.environ, {'SAGEMAKER_INFERENCE_ACCELERATOR_PRESENT': 'true'}, clear=True) 71 | def test_default_model_fn_with_accelerator(path_exists, mx_load_checkpoint, mx_module, mx_eia): 72 | sym = Mock() 73 | args = Mock() 74 | aux = Mock() 75 | mx_load_checkpoint.return_value = [sym, args, aux] 76 | 77 | eia_context = Mock() 78 | mx_eia.return_value = eia_context 79 | 80 | data_name = 'foo' 81 | data_shape = [1] 82 | signature = json.dumps([{'name': data_name, 'shape': data_shape}]) 83 | 84 | with patch('six.moves.builtins.open', mock_open(read_data=signature)): 85 | DefaultMXNetInferenceHandler().default_model_fn(MODEL_DIR) 86 | 87 | mx_load_checkpoint.assert_called_with(os.path.join(MODEL_DIR, 'model'), 0) 88 | 89 | init_call = call(symbol=sym, context=eia_context, data_names=[data_name], label_names=None) 90 | assert init_call in mx_module.mock_calls 91 | 92 | model = mx_module.return_value 93 | model.bind.assert_called_with(for_training=False, data_shapes=[(data_name, data_shape)]) 94 | model.set_params.assert_called_with(args, aux, allow_missing=True) 95 | 96 | 97 | @patch('sagemaker_inference.decoder.decode', return_value=[0]) 98 | def test_mxnet_default_input_fn_with_json(decode): 99 | input_data = Mock() 100 | content_type = 'application/json' 101 | 102 | deserialized_data = DefaultMXNetInferenceHandler().default_input_fn(input_data, content_type) 103 | 104 | decode.assert_called_with(input_data, content_type) 105 | assert deserialized_data == mx.nd.array([0]) 106 | 107 | 108 | @patch('sagemaker_inference.decoder.decode', return_value=[0]) 109 | def test_mxnet_default_input_fn_with_npy(decode): 110 | input_data = Mock() 111 | content_type = 'application/x-npy' 112 | 113 | deserialized_data = DefaultMXNetInferenceHandler().default_input_fn(input_data, content_type) 114 | 115 | decode.assert_called_with(input_data, content_type) 116 | assert deserialized_data == mx.nd.array([0]) 117 | 118 | 119 | @patch('mxnet.eia', create=True) 120 | @patch('mxnet.nd.array') 121 | @patch('sagemaker_inference.decoder.decode', return_value=[0]) 122 | @patch.dict(os.environ, {'SAGEMAKER_INFERENCE_ACCELERATOR_PRESENT': 'true'}, clear=True) 123 | def test_mxnet_default_input_fn_with_accelerator(decode, mx_ndarray, mx_eia): 124 | ndarray = Mock() 125 | mx_ndarray.return_value = ndarray 126 | 127 | DefaultMXNetInferenceHandler().default_input_fn(Mock(), 'application/json') 128 | 129 | ndarray.as_in_context.assert_called_with(mx.cpu()) 130 | 131 | 132 | def test_mxnet_default_input_fn_invalid_content_type(): 133 | with pytest.raises(errors.UnsupportedFormatError) as e: 134 | DefaultMXNetInferenceHandler().default_input_fn(None, 'bad/content-type') 135 | e.match('Content type bad/content-type is not supported by this framework') 136 | 137 | 138 | @patch('sagemaker_inference.encoder.encode', return_value=str()) 139 | def test_mxnet_default_output_fn(encode): 140 | prediction = mx.ndarray.zeros(1) 141 | accept = 'application/json' 142 | 143 | response = DefaultMXNetInferenceHandler().default_output_fn(prediction, accept) 144 | 145 | flattened_prediction = prediction.asnumpy().tolist() 146 | encode.assert_called_with(flattened_prediction, accept) 147 | 148 | assert isinstance(response, str) 149 | 150 | 151 | @patch('sagemaker_inference.encoder.encode', return_value=str()) 152 | def test_mxnet_default_output_fn_multiple_content_types(encode): 153 | prediction = mx.ndarray.zeros(1) 154 | accept = 'application/unsupported, application/json, application/x-npy' 155 | 156 | response = DefaultMXNetInferenceHandler().default_output_fn(prediction, accept) 157 | 158 | flattened_prediction = prediction.asnumpy().tolist() 159 | encode.assert_called_with(flattened_prediction, "application/json") 160 | 161 | assert isinstance(response, str) 162 | 163 | 164 | def test_mxnet_default_output_fn_invalid_content_type(): 165 | with pytest.raises(errors.UnsupportedFormatError) as e: 166 | DefaultMXNetInferenceHandler().default_output_fn(None, 'bad/content-type') 167 | e.match('Content type bad/content-type is not supported by this framework') 168 | 169 | 170 | ################################################################################# 171 | # DefaultModuleInferenceHandler Tests 172 | 173 | 174 | def test_default_module_valid_content_types(): 175 | assert DefaultModuleInferenceHandler().VALID_CONTENT_TYPES == \ 176 | (content_types.JSON, content_types.CSV, content_types.NPY) 177 | 178 | 179 | @patch('mxnet.io.NDArrayIter') 180 | @patch('sagemaker_inference.decoder.decode', return_value=[0]) 181 | def test_module_default_input_fn_with_json(decode, mx_ndarray_iter): 182 | model = Mock(data_shapes=[(1, (1,))]) 183 | 184 | input_data = Mock() 185 | content_type = 'application/json' 186 | DefaultModuleInferenceHandler().default_input_fn(input_data, content_type, model) 187 | 188 | decode.assert_called_with(input_data, content_type) 189 | init_call = call(mx.nd.array([0]), batch_size=1, last_batch_handle='pad') 190 | assert init_call in mx_ndarray_iter.mock_calls 191 | 192 | 193 | @patch('mxnet.nd.array') 194 | @patch('mxnet.io.NDArrayIter') 195 | @patch('sagemaker_inference.decoder.decode', return_value=[0]) 196 | def test_module_default_input_fn_with_csv(decode, mx_ndarray_iter, mx_ndarray): 197 | ndarray = Mock(shape=(1, (1,))) 198 | ndarray.reshape.return_value = ndarray 199 | ndarray.as_in_context.return_value = ndarray 200 | mx_ndarray.return_value = ndarray 201 | 202 | model = Mock(data_shapes=[(1, (1,))]) 203 | 204 | input_data = Mock() 205 | content_type = 'text/csv' 206 | DefaultModuleInferenceHandler().default_input_fn(input_data, content_type, model) 207 | 208 | decode.assert_called_with(input_data, content_type) 209 | ndarray.reshape.assert_called_with((-1,)) 210 | init_call = call(mx.nd.array([0]), batch_size=1, last_batch_handle='pad') 211 | assert init_call in mx_ndarray_iter.mock_calls 212 | 213 | 214 | @patch('mxnet.io.NDArrayIter') 215 | @patch('sagemaker_inference.decoder.decode', return_value=[0]) 216 | def test_module_default_input_fn_with_npy(decode, mx_ndarray_iter): 217 | model = Mock(data_shapes=[(1, (1,))]) 218 | 219 | input_data = Mock() 220 | content_type = 'application/x-npy' 221 | DefaultModuleInferenceHandler().default_input_fn(input_data, content_type, model) 222 | 223 | decode.assert_called_with(input_data, content_type) 224 | init_call = call(mx.nd.array([0]), batch_size=1, last_batch_handle='pad') 225 | assert init_call in mx_ndarray_iter.mock_calls 226 | 227 | 228 | @patch('mxnet.eia', create=True) 229 | @patch('mxnet.nd.array') 230 | @patch('mxnet.io.NDArrayIter') 231 | @patch('sagemaker_inference.decoder.decode', return_value=[0]) 232 | @patch.dict(os.environ, {'SAGEMAKER_INFERENCE_ACCELERATOR_PRESENT': 'true'}, clear=True) 233 | def test_module_default_input_fn_with_accelerator(decode, mx_ndarray_iter, mx_ndarray, mx_eia): 234 | ndarray = Mock(shape=(1, (1,))) 235 | ndarray.as_in_context.return_value = ndarray 236 | mx_ndarray.return_value = ndarray 237 | 238 | model = Mock(data_shapes=[(1, (1,))]) 239 | DefaultModuleInferenceHandler().default_input_fn(Mock(), 'application/json', model) 240 | 241 | ndarray.as_in_context.assert_called_with(mx.cpu()) 242 | 243 | 244 | def test_module_default_input_fn_invalid_content_type(): 245 | with pytest.raises(errors.UnsupportedFormatError) as e: 246 | DefaultModuleInferenceHandler().default_input_fn(None, 'bad/content-type') 247 | e.match('Content type bad/content-type is not supported by this framework') 248 | 249 | 250 | def test_module_default_predict_fn(): 251 | module = Mock() 252 | data = Mock() 253 | 254 | DefaultModuleInferenceHandler().default_predict_fn(data, module) 255 | module.predict.assert_called_with(data) 256 | 257 | ################################################################################# 258 | # DefaultGluonBlockInferenceHandler Tests 259 | 260 | 261 | def test_gluon_default_predict_fn(): 262 | data = [0] 263 | block = Mock() 264 | 265 | DefaultGluonBlockInferenceHandler().default_predict_fn(data, block) 266 | 267 | block.assert_called_with(data) 268 | -------------------------------------------------------------------------------- /test/unit/test_handler_service.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | from mock import MagicMock, Mock, patch 16 | import mxnet as mx 17 | import pytest 18 | from sagemaker_inference import environment 19 | from sagemaker_inference.default_inference_handler import DefaultInferenceHandler 20 | from sagemaker_inference.transformer import Transformer 21 | 22 | from sagemaker_mxnet_serving_container.default_inference_handler import DefaultGluonBlockInferenceHandler 23 | from sagemaker_mxnet_serving_container.handler_service import HandlerService 24 | from sagemaker_mxnet_serving_container.mxnet_module_transformer import MXNetModuleTransformer 25 | 26 | MODULE_NAME = 'module_name' 27 | 28 | 29 | @patch('sagemaker_mxnet_serving_container.handler_service.HandlerService._user_module_transformer') 30 | @patch('sagemaker_inference.default_handler_service.DefaultHandlerService.initialize') 31 | def test_handler_service(user_module_transformer, initialize): 32 | service = HandlerService() 33 | 34 | properties = { 35 | 'model_dir': '/opt/ml/models/model-name' 36 | } 37 | 38 | def getitem(key): 39 | return properties[key] 40 | 41 | context = MagicMock() 42 | context.system_properties.__getitem__.side_effect = getitem 43 | service.initialize(context) 44 | 45 | assert isinstance(service._service, Mock) 46 | 47 | 48 | class UserModuleTransformFn: 49 | def __init__(self): 50 | self.transform_fn = Mock() 51 | 52 | 53 | @patch('sagemaker_inference.environment.Environment') 54 | @patch('importlib.import_module', return_value=UserModuleTransformFn()) 55 | def test_user_module_transform_fn(import_module, env): 56 | env.return_value.module_name = MODULE_NAME 57 | transformer = HandlerService._user_module_transformer() 58 | 59 | import_module.assert_called_once_with(MODULE_NAME) 60 | assert isinstance(transformer._default_inference_handler, DefaultInferenceHandler) 61 | assert isinstance(transformer, Transformer) 62 | 63 | 64 | class UserModuleModelFn: 65 | def __init__(self): 66 | self.model_fn = Mock() 67 | 68 | 69 | @patch('sagemaker_inference.environment.Environment') 70 | @patch('importlib.import_module', return_value=UserModuleModelFn()) 71 | def test_user_module_mxnet_module_transformer(import_module, env): 72 | env.return_value.module_name = MODULE_NAME 73 | import_module.return_value.model_fn.return_value = mx.module.BaseModule() 74 | 75 | transformer = HandlerService._user_module_transformer() 76 | 77 | import_module.assert_called_once_with(MODULE_NAME) 78 | assert isinstance(transformer, MXNetModuleTransformer) 79 | 80 | 81 | @patch('sagemaker_inference.environment.Environment') 82 | @patch('sagemaker_mxnet_serving_container.default_inference_handler.DefaultMXNetInferenceHandler.default_model_fn') 83 | @patch('importlib.import_module', return_value=object()) 84 | def test_default_inference_handler_mxnet_gluon_transformer(import_module, model_fn, env): 85 | env.return_value.module_name = MODULE_NAME 86 | model_fn.return_value = mx.gluon.block.Block() 87 | 88 | transformer = HandlerService._user_module_transformer() 89 | 90 | import_module.assert_called_once_with(MODULE_NAME) 91 | model_fn.assert_called_once_with(environment.model_dir) 92 | assert isinstance(transformer, Transformer) 93 | assert isinstance(transformer._default_inference_handler, DefaultGluonBlockInferenceHandler) 94 | 95 | 96 | @patch('sagemaker_inference.environment.Environment') 97 | @patch('importlib.import_module', return_value=UserModuleModelFn()) 98 | def test_user_module_unsupported(import_module, env): 99 | env.return_value.module_name = MODULE_NAME 100 | 101 | with pytest.raises(ValueError) as e: 102 | HandlerService._user_module_transformer() 103 | 104 | import_module.assert_called_once_with(MODULE_NAME) 105 | e.match('Unsupported model type') 106 | 107 | 108 | @patch('sagemaker_inference.environment.Environment') 109 | def test_user_module_notfound(env): 110 | env.return_value.module_name = MODULE_NAME 111 | 112 | with pytest.raises(ValueError) as e: 113 | HandlerService._user_module_transformer() 114 | 115 | e.match('import_module exception') 116 | -------------------------------------------------------------------------------- /test/unit/test_mxnet_module_transformer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | from mock import Mock, patch 16 | 17 | from sagemaker_mxnet_serving_container.mxnet_module_transformer import MXNetModuleTransformer 18 | 19 | CONTENT_TYPE = 'content' 20 | ACCEPT = 'accept' 21 | DATA = 'data' 22 | MODEL = 'foo' 23 | 24 | PREPROCESSED_DATA = 'preprocessed_data' 25 | PREDICT_RESULT = 'prediction_result' 26 | PROCESSED_RESULT = 'processed_result' 27 | 28 | 29 | def _input_fn_with_model(input_data, content_type, model): 30 | return PREPROCESSED_DATA 31 | 32 | 33 | @patch('importlib.import_module', return_value=object()) 34 | def test_default_transform_fn(import_module): 35 | predict_fn = Mock(return_value=PREDICT_RESULT) 36 | output_fn = Mock(return_value=PROCESSED_RESULT) 37 | 38 | module_transformer = MXNetModuleTransformer() 39 | module_transformer._input_fn = _input_fn_with_model 40 | module_transformer._predict_fn = predict_fn 41 | module_transformer._output_fn = output_fn 42 | 43 | result = module_transformer._default_transform_fn(MODEL, DATA, CONTENT_TYPE, ACCEPT) 44 | 45 | predict_fn.assert_called_once_with(PREPROCESSED_DATA, MODEL) 46 | output_fn.assert_called_once_with(PREDICT_RESULT, ACCEPT) 47 | assert PROCESSED_RESULT == result 48 | 49 | 50 | @patch('importlib.import_module', return_value=object()) 51 | def test_call_input_fn(import_module): 52 | module_transformer = MXNetModuleTransformer() 53 | module_transformer._input_fn = _input_fn_with_model 54 | 55 | result = module_transformer._call_input_fn(DATA, CONTENT_TYPE, MODEL) 56 | 57 | assert PREPROCESSED_DATA == result 58 | 59 | 60 | def _input_fn_without_model(input_data, content_type): 61 | return PREPROCESSED_DATA 62 | 63 | 64 | @patch('importlib.import_module', return_value=object()) 65 | def test_call_input_fn_without_model_arg(import_module): 66 | module_transformer = MXNetModuleTransformer() 67 | module_transformer._input_fn = _input_fn_without_model 68 | 69 | result = module_transformer._call_input_fn(MODEL, DATA, CONTENT_TYPE) 70 | 71 | assert PREPROCESSED_DATA == result 72 | -------------------------------------------------------------------------------- /test/unit/test_serving.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import os 16 | 17 | from mock import patch 18 | 19 | from sagemaker_mxnet_serving_container.serving import _update_mxnet_env_vars, DEFAULT_ENV_VARS, HANDLER_SERVICE, main 20 | 21 | 22 | @patch('sagemaker_inference.model_server.start_model_server') 23 | @patch('sagemaker_mxnet_serving_container.serving._update_mxnet_env_vars') 24 | def test_main(update_mxnet_env_vars, model_server): 25 | main() 26 | 27 | update_mxnet_env_vars.assert_called_once() 28 | model_server.assert_called_once_with(handler_service=HANDLER_SERVICE) 29 | 30 | 31 | @patch.dict(os.environ, dict(), clear=True) 32 | def test_update_env_vars(): 33 | assert bool(os.environ) is False 34 | 35 | _update_mxnet_env_vars() 36 | 37 | assert DEFAULT_ENV_VARS == os.environ 38 | -------------------------------------------------------------------------------- /test/unit/test_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import json 16 | 17 | from mock import Mock, mock_open, patch 18 | 19 | from sagemaker_mxnet_serving_container.utils import get_default_context, read_data_shapes 20 | 21 | MODEL_DIR = 'foo/model' 22 | 23 | 24 | @patch('mxnet.cpu') 25 | def test_context(mx_cpu): 26 | mx_context = Mock() 27 | mx_cpu.return_value = mx_context 28 | 29 | default_context = get_default_context() 30 | 31 | assert default_context == mx_context 32 | 33 | 34 | def test_read_data_shapes(): 35 | data_name = 'foo' 36 | data_shape = [1] 37 | signature = json.dumps([{'name': data_name, 'shape': data_shape}]) 38 | 39 | with patch('six.moves.builtins.open', mock_open(read_data=signature)): 40 | data_names, data_shapes = read_data_shapes(MODEL_DIR) 41 | 42 | assert len(data_names) == 1 43 | assert len(data_shapes) == 1 44 | assert data_names[0] == data_name 45 | assert data_shapes[0][1] == data_shape 46 | -------------------------------------------------------------------------------- /test/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | -------------------------------------------------------------------------------- /test/utils/image_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | import os 16 | import subprocess 17 | import sys 18 | 19 | CYAN_COLOR = '\033[36m' 20 | END_COLOR = '\033[0m' 21 | DLC_AWS_ID = '763104351884' 22 | 23 | 24 | def build_image(framework_version, dockerfile, image_uri, region, cwd='.'): 25 | _check_call('python setup.py sdist') 26 | 27 | if 'dlc' in dockerfile: 28 | ecr_login(region, DLC_AWS_ID) 29 | 30 | dockerfile_location = os.path.join('test', 'container', framework_version, dockerfile) 31 | 32 | subprocess.check_call( 33 | ['docker', 'build', '-t', image_uri, '-f', dockerfile_location, '--build-arg', 34 | 'region={}'.format(region), cwd], cwd=cwd) 35 | print('created image {}'.format(image_uri)) 36 | return image_uri 37 | 38 | 39 | def push_image(ecr_image, region, aws_id): 40 | ecr_login(region, aws_id) 41 | _check_call('docker push {}'.format(ecr_image)) 42 | 43 | 44 | def ecr_login(region, aws_id): 45 | login = _check_call('aws ecr get-login --registry-ids {} '.format(aws_id) 46 | + '--no-include-email --region {}'.format(region)) 47 | _check_call(login.decode('utf-8').rstrip('\n')) 48 | 49 | 50 | def _check_call(cmd, *popenargs, **kwargs): 51 | if isinstance(cmd, str): 52 | cmd = cmd.split(" ") 53 | _print_cmd(cmd) 54 | return subprocess.check_output(cmd, *popenargs, **kwargs) 55 | 56 | 57 | def _print_cmd(cmd): 58 | print('executing docker command: {}{}{}'.format(CYAN_COLOR, ' '.join(cmd), END_COLOR)) 59 | sys.stdout.flush() 60 | -------------------------------------------------------------------------------- /test/utils/local_mode_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from __future__ import absolute_import 14 | 15 | from contextlib import contextmanager 16 | import fcntl 17 | import os 18 | import tarfile 19 | import time 20 | 21 | from integration import RESOURCE_PATH 22 | 23 | LOCK_PATH = os.path.join(RESOURCE_PATH, 'local_mode_lock') 24 | 25 | 26 | @contextmanager 27 | def lock(): 28 | # Since Local Mode uses the same port for serving, we need a lock in order 29 | # to allow concurrent test execution. 30 | local_mode_lock_fd = open(LOCK_PATH, 'w') 31 | local_mode_lock = local_mode_lock_fd.fileno() 32 | 33 | fcntl.lockf(local_mode_lock, fcntl.LOCK_EX) 34 | 35 | try: 36 | yield 37 | finally: 38 | time.sleep(5) 39 | fcntl.lockf(local_mode_lock, fcntl.LOCK_UN) 40 | 41 | 42 | def assert_output_files_exist(output_path, directory, files): 43 | with tarfile.open(os.path.join(output_path, '{}.tar.gz'.format(directory))) as tar: 44 | for f in files: 45 | tar.getmember(f) 46 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | [tox] 6 | envlist = flake8,py27,py36,py37 7 | skip_missing_interpreters = False 8 | 9 | [flake8] 10 | max-line-length = 120 11 | exclude = 12 | .git 13 | __pycache__ 14 | .tox 15 | test/resources 16 | venv/ 17 | docker/ 18 | artifacts/ 19 | max-complexity = 10 20 | ignore = 21 | C901, 22 | E203, # whitespace before ':': Black disagrees with and explicitly violates this. 23 | E722 24 | FI10, 25 | FI12, 26 | FI13, 27 | FI14, 28 | FI15, 29 | FI16, 30 | FI17, 31 | FI18, # __future__ import "annotations" missing -> check only Python 3.7 compatible 32 | FI50, 33 | FI51, 34 | FI52, 35 | FI53, 36 | FI54, 37 | FI55, 38 | FI56, 39 | FI57, 40 | W503 41 | 42 | require-code = True 43 | 44 | [testenv] 45 | passenv = 46 | AWS_ACCESS_KEY_ID 47 | AWS_SECRET_ACCESS_KEY 48 | AWS_SESSION_TOKEN 49 | AWS_CONTAINER_CREDENTIALS_RELATIVE_URI 50 | AWS_DEFAULT_REGION 51 | # {posargs} can be passed in by additional arguments specified when invoking tox. 52 | # Can be used to specify which tests to run, e.g.: tox -- -s 53 | commands = 54 | coverage run --rcfile .coveragerc_{envname} --source sagemaker_mxnet_serving_container -m pytest {posargs} 55 | {env:IGNORE_COVERAGE:} coverage report --fail-under=90 --include *sagemaker_mxnet_serving_container* 56 | deps = .[test] 57 | 58 | [testenv:flake8] 59 | basepython = python 60 | deps = 61 | flake8 62 | flake8-future-import 63 | flake8-import-order 64 | commands = flake8 65 | 66 | [testenv:twine] 67 | basepython = python3 68 | # https://github.com/pypa/twine/blob/master/docs/changelog.rst 69 | deps = 70 | twine>=1.12.0 71 | # https://packaging.python.org/guides/making-a-pypi-friendly-readme/#validating-restructuredtext-markup 72 | commands = 73 | python setup.py sdist 74 | twine check dist/*.tar.gz 75 | --------------------------------------------------------------------------------