├── .github ├── generate-toc └── workflows │ ├── e2e-test-deploy.yaml │ ├── help-action.yml │ └── slash-command-dispatcher.yaml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── bentoctl_sagemaker ├── __init__.py ├── create_deployable.py ├── generate.py ├── parameters.py ├── registry_utils.py ├── sagemaker │ ├── README.md │ ├── serve │ ├── service.py │ └── template.j2 └── templates │ ├── terraform_default.tf │ └── terraform_with_data_capture.tf ├── operator_config.py ├── requirements.txt └── tests ├── classifier.py └── test_api.py /.github/generate-toc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script was taken from https://github.com/ekalinin/github-markdown-toc 4 | # checkout the repo for how to use it 5 | # 6 | # Internal Working 7 | # Steps: 8 | # 9 | # 1. Download corresponding html file for some README.md: 10 | # curl -s $1 11 | # 12 | # 2. Discard rows where no substring 'user-content-' (github's markup): 13 | # awk '/user-content-/ { ... 14 | # 15 | # 3.1 Get last number in each row like ' ... sitemap.js.*<\/h/)+2, RLENGTH-5) 24 | # 25 | # 5. Find anchor and insert it inside "(...)": 26 | # substr($0, match($0, "href=\"[^\"]+?\" ")+6, RLENGTH-8) 27 | # 28 | 29 | gh_toc_version="0.8.0" 30 | 31 | gh_user_agent="gh-md-toc v$gh_toc_version" 32 | 33 | # 34 | # Download rendered into html README.md by its url. 35 | # 36 | # 37 | gh_toc_load() { 38 | local gh_url=$1 39 | 40 | if type curl &>/dev/null; then 41 | curl --user-agent "$gh_user_agent" -s "$gh_url" 42 | elif type wget &>/dev/null; then 43 | wget --user-agent="$gh_user_agent" -qO- "$gh_url" 44 | else 45 | echo "Please, install 'curl' or 'wget' and try again." 46 | exit 1 47 | fi 48 | } 49 | 50 | # 51 | # Converts local md file into html by GitHub 52 | # 53 | # -> curl -X POST --data '{"text": "Hello world github/linguist#1 **cool**, and #1!"}' https://api.github.com/markdown 54 | #

Hello world github/linguist#1 cool, and #1!

'" 55 | gh_toc_md2html() { 56 | local gh_file_md=$1 57 | URL=https://api.github.com/markdown/raw 58 | 59 | if [ ! -z "$GH_TOC_TOKEN" ]; then 60 | TOKEN=$GH_TOC_TOKEN 61 | else 62 | TOKEN_FILE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/token.txt" 63 | if [ -f "$TOKEN_FILE" ]; then 64 | TOKEN="$(cat $TOKEN_FILE)" 65 | fi 66 | fi 67 | if [ ! -z "${TOKEN}" ]; then 68 | AUTHORIZATION="Authorization: token ${TOKEN}" 69 | fi 70 | 71 | # echo $URL 1>&2 72 | OUTPUT=$(curl -s \ 73 | --user-agent "$gh_user_agent" \ 74 | --data-binary @"$gh_file_md" \ 75 | -H "Content-Type:text/plain" \ 76 | -H "$AUTHORIZATION" \ 77 | "$URL") 78 | 79 | if [ "$?" != "0" ]; then 80 | echo "XXNetworkErrorXX" 81 | fi 82 | if [ "$(echo "${OUTPUT}" | awk '/API rate limit exceeded/')" != "" ]; then 83 | echo "XXRateLimitXX" 84 | else 85 | echo "${OUTPUT}" 86 | fi 87 | } 88 | 89 | 90 | # 91 | # Is passed string url 92 | # 93 | gh_is_url() { 94 | case $1 in 95 | https* | http*) 96 | echo "yes";; 97 | *) 98 | echo "no";; 99 | esac 100 | } 101 | 102 | # 103 | # TOC generator 104 | # 105 | gh_toc(){ 106 | local gh_src=$1 107 | local gh_src_copy=$1 108 | local gh_ttl_docs=$2 109 | local need_replace=$3 110 | local no_backup=$4 111 | local no_footer=$5 112 | 113 | if [ "$gh_src" = "" ]; then 114 | echo "Please, enter URL or local path for a README.md" 115 | exit 1 116 | fi 117 | 118 | 119 | # Show "TOC" string only if working with one document 120 | if [ "$gh_ttl_docs" = "1" ]; then 121 | 122 | echo "Table of Contents" 123 | echo "=================" 124 | echo "" 125 | gh_src_copy="" 126 | 127 | fi 128 | 129 | if [ "$(gh_is_url "$gh_src")" == "yes" ]; then 130 | gh_toc_load "$gh_src" | gh_toc_grab "$gh_src_copy" 131 | if [ "${PIPESTATUS[0]}" != "0" ]; then 132 | echo "Could not load remote document." 133 | echo "Please check your url or network connectivity" 134 | exit 1 135 | fi 136 | if [ "$need_replace" = "yes" ]; then 137 | echo 138 | echo "!! '$gh_src' is not a local file" 139 | echo "!! Can't insert the TOC into it." 140 | echo 141 | fi 142 | else 143 | local rawhtml=$(gh_toc_md2html "$gh_src") 144 | if [ "$rawhtml" == "XXNetworkErrorXX" ]; then 145 | echo "Parsing local markdown file requires access to github API" 146 | echo "Please make sure curl is installed and check your network connectivity" 147 | exit 1 148 | fi 149 | if [ "$rawhtml" == "XXRateLimitXX" ]; then 150 | echo "Parsing local markdown file requires access to github API" 151 | echo "Error: You exceeded the hourly limit. See: https://developer.github.com/v3/#rate-limiting" 152 | TOKEN_FILE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/token.txt" 153 | echo "or place GitHub auth token here: ${TOKEN_FILE}" 154 | exit 1 155 | fi 156 | local toc=`echo "$rawhtml" | gh_toc_grab "$gh_src_copy"` 157 | echo "$toc" 158 | if [ "$need_replace" = "yes" ]; then 159 | if grep -Fxq "" $gh_src && grep -Fxq "" $gh_src; then 160 | echo "Found markers" 161 | else 162 | echo "You don't have or in your file...exiting" 163 | exit 1 164 | fi 165 | local ts="<\!--ts-->" 166 | local te="<\!--te-->" 167 | local dt=`date +'%F_%H%M%S'` 168 | local ext=".orig.${dt}" 169 | local toc_path="${gh_src}.toc.${dt}" 170 | local toc_header="## Table of Contents" 171 | local toc_footer="" 172 | # http://fahdshariff.blogspot.ru/2012/12/sed-mutli-line-replacement-between-two.html 173 | # clear old TOC 174 | sed -i${ext} "/${ts}/,/${te}/{//!d;}" "$gh_src" 175 | 176 | # add header 177 | echo -e "\n${toc_header}\n" > "${toc_path}" 178 | 179 | # create toc file 180 | echo "${toc}" >> "$toc_path" 181 | 182 | # add footer 183 | if [ "${no_footer}" != "yes" ]; then 184 | echo -e "\n${toc_footer}\n" >> "$toc_path" 185 | fi 186 | 187 | # insert toc file 188 | if [[ "`uname`" == "Darwin" ]]; then 189 | sed -i "" "/${ts}/r ${toc_path}" "$gh_src" 190 | else 191 | sed -i "/${ts}/r ${toc_path}" "$gh_src" 192 | fi 193 | echo 194 | if [ "${no_backup}" = "yes" ]; then 195 | rm ${toc_path} ${gh_src}${ext} 196 | fi 197 | echo "!! TOC was added into: '$gh_src'" 198 | if [ -z "${no_backup}" ]; then 199 | echo "!! Origin version of the file: '${gh_src}${ext}'" 200 | echo "!! TOC added into a separate file: '${toc_path}'" 201 | fi 202 | echo 203 | fi 204 | fi 205 | } 206 | 207 | # 208 | # Grabber of the TOC from rendered html 209 | # 210 | # $1 - a source url of document. 211 | # It's need if TOC is generated for multiple documents. 212 | # 213 | gh_toc_grab() { 214 | common_awk_script=' 215 | modified_href = "" 216 | split(href, chars, "") 217 | for (i=1;i <= length(href); i++) { 218 | c = chars[i] 219 | res = "" 220 | if (c == "+") { 221 | res = " " 222 | } else { 223 | if (c == "%") { 224 | res = "\\x" 225 | } else { 226 | res = c "" 227 | } 228 | } 229 | modified_href = modified_href res 230 | } 231 | print sprintf("%*s", (level-1)*3, "") "* [" text "](" gh_url modified_href ")" 232 | ' 233 | if [ `uname -s` == "OS/390" ]; then 234 | grepcmd="pcregrep -o" 235 | echoargs="" 236 | awkscript='{ 237 | level = substr($0, length($0), 1) 238 | text = substr($0, match($0, /a>.*<\/h/)+2, RLENGTH-5) 239 | href = substr($0, match($0, "href=\"([^\"]+)?\"")+6, RLENGTH-7) 240 | '"$common_awk_script"' 241 | }' 242 | else 243 | grepcmd="grep -Eo" 244 | echoargs="-e" 245 | awkscript='{ 246 | level = substr($0, length($0), 1) 247 | text = substr($0, match($0, /a>.*<\/h/)+2, RLENGTH-5) 248 | href = substr($0, match($0, "href=\"[^\"]+?\"")+6, RLENGTH-7) 249 | '"$common_awk_script"' 250 | }' 251 | fi 252 | href_regex='href=\"[^\"]+?\"' 253 | 254 | # if closed is on the new line, then move it on the prev line 255 | # for example: 256 | # was: The command foo1 257 | # 258 | # became: The command foo1 259 | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n<\/h/<\/h/g' | 260 | 261 | # find strings that corresponds to template 262 | $grepcmd '//g' | sed 's/<\/code>//g' | 266 | 267 | # remove g-emoji 268 | sed 's/]*[^<]*<\/g-emoji> //g' | 269 | 270 | # now all rows are like: 271 | # ... / placeholders" 300 | echo " $app_name - Create TOC for markdown from STDIN" 301 | echo " $app_name --help Show help" 302 | echo " $app_name --version Show version" 303 | return 304 | fi 305 | 306 | if [ "$1" = '--version' ]; then 307 | echo "$gh_toc_version" 308 | echo 309 | echo "os: `lsb_release -d | cut -f 2`" 310 | echo "kernel: `cat /proc/version`" 311 | echo "shell: `$SHELL --version`" 312 | echo 313 | for tool in curl wget grep awk sed; do 314 | printf "%-5s: " $tool 315 | echo `$tool --version | head -n 1` 316 | done 317 | return 318 | fi 319 | 320 | if [ "$1" = "-" ]; then 321 | if [ -z "$TMPDIR" ]; then 322 | TMPDIR="/tmp" 323 | elif [ -n "$TMPDIR" -a ! -d "$TMPDIR" ]; then 324 | mkdir -p "$TMPDIR" 325 | fi 326 | local gh_tmp_md 327 | if [ `uname -s` == "OS/390" ]; then 328 | local timestamp=$(date +%m%d%Y%H%M%S) 329 | gh_tmp_md="$TMPDIR/tmp.$timestamp" 330 | else 331 | gh_tmp_md=$(mktemp $TMPDIR/tmp.XXXXXX) 332 | fi 333 | while read input; do 334 | echo "$input" >> "$gh_tmp_md" 335 | done 336 | gh_toc_md2html "$gh_tmp_md" | gh_toc_grab "" 337 | return 338 | fi 339 | 340 | if [ "$1" = '--insert' ]; then 341 | need_replace="yes" 342 | shift 343 | fi 344 | 345 | if [ "$1" = '--no-backup' ]; then 346 | need_replace="yes" 347 | no_backup="yes" 348 | shift 349 | fi 350 | 351 | if [ "$1" = '--hide-footer' ]; then 352 | need_replace="yes" 353 | no_footer="yes" 354 | shift 355 | fi 356 | 357 | for md in "$@" 358 | do 359 | echo "" 360 | gh_toc "$md" "$#" "$need_replace" "$no_backup" "$no_footer" 361 | done 362 | 363 | echo "" 364 | echo "Created by [gh-md-toc](https://github.com/ekalinin/github-markdown-toc)" 365 | } 366 | 367 | # 368 | # Entry point 369 | # 370 | gh_toc_app "$@" 371 | 372 | -------------------------------------------------------------------------------- /.github/workflows/e2e-test-deploy.yaml: -------------------------------------------------------------------------------- 1 | # Run secret-dependent integration tests only after /ok-to-test approval 2 | on: 3 | pull_request: 4 | repository_dispatch: 5 | types: [test-e2e-deploy-command] 6 | 7 | name: e2e tests 8 | 9 | jobs: 10 | # Repo owner has commented / on a (fork-based) pull request 11 | test-e2e-deploy: 12 | runs-on: ubuntu-latest 13 | if: 14 | github.event_name == 'repository_dispatch' && 15 | github.event.client_payload.slash_command.command == 'test-e2e-deploy' && 16 | github.event.client_payload.slash_command.args.named.sha != '' && 17 | contains(github.event.client_payload.pull_request.head.sha, github.event.client_payload.slash_command.args.named.sha) 18 | steps: 19 | 20 | # Check out merge commit 21 | - name: Fork based /test-e2e-deploy checkout 22 | uses: actions/checkout@v2 23 | with: 24 | ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' 25 | 26 | # 27 | 28 | - run: aws --version 29 | - name: Configure AWS Credentials 30 | uses: aws-actions/configure-aws-credentials@v1 31 | with: 32 | aws-access-key-id: ${{ secrets.AWS_KEY_ID }} 33 | aws-secret-access-key: ${{ secrets.AWS_SEC }} 34 | aws-region: 'us-west-1' 35 | - name: Setup python 3.8 36 | uses: actions/setup-python@v2 37 | with: 38 | python-version: '3.8' 39 | - name: Install dependencies 40 | run: python -m pip install -r requirements.txt 41 | - name: Run Integration tests for deploy 42 | run: python tests/test_api.py 43 | 44 | # Update check run called "integration-fork" 45 | - uses: actions/github-script@v1 46 | id: update-check-run 47 | if: ${{ always() }} 48 | env: 49 | number: ${{ github.event.client_payload.pull_request.number }} 50 | job: ${{ github.job }} 51 | # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run 52 | conclusion: ${{ job.status }} 53 | with: 54 | github-token: ${{ secrets.GITHUB_TOKEN }} 55 | script: | 56 | const { data: pull } = await github.pulls.get({ 57 | ...context.repo, 58 | pull_number: process.env.number 59 | }); 60 | const ref = pull.head.sha; 61 | 62 | const { data: checks } = await github.checks.listForRef({ 63 | ...context.repo, 64 | ref 65 | }); 66 | 67 | const check = checks.check_runs.filter(c => c.name === process.env.job); 68 | 69 | const { data: result } = await github.checks.update({ 70 | ...context.repo, 71 | check_run_id: check[0].id, 72 | status: 'completed', 73 | conclusion: process.env.conclusion 74 | }); 75 | 76 | return result; 77 | -------------------------------------------------------------------------------- /.github/workflows/help-action.yml: -------------------------------------------------------------------------------- 1 | name: help action 2 | on: 3 | repository_dispatch: 4 | types: [help-command] 5 | 6 | jobs: 7 | display-help-info: 8 | runs-on: ubuntu-latest 9 | if: 10 | github.event_name == 'repository_dispatch' && 11 | github.event.client_payload.slash_command.command == 'help' 12 | steps: 13 | - name: Create comment 14 | uses: peter-evans/create-or-update-comment@v1 15 | with: 16 | issue-number: ${{ github.event.client_payload.github.payload.issue.number }} 17 | body: | 18 | Hello @${{ github.event.client_payload.github.actor }}! 19 | These are the list of commands available in this repo. 20 | 21 | | Command | Description | 22 | | ------- | ----------- | 23 | | /test-e2e-deploy sha=_"first 7 char of commit sha"_ | Runs the e2e test for this deployment tool. | 24 | -------------------------------------------------------------------------------- /.github/workflows/slash-command-dispatcher.yaml: -------------------------------------------------------------------------------- 1 | # If someone with write access comments "/ok-to-test" on a pull request, emit a repository_dispatch event 2 | name: slash command dispatcher 3 | 4 | on: 5 | issue_comment: 6 | types: [created] 7 | 8 | jobs: 9 | test-e2e-deploy: 10 | runs-on: ubuntu-latest 11 | # Only run for PRs, not issue comments 12 | if: ${{ github.event.issue.pull_request }} 13 | steps: 14 | # Generate a GitHub App installation access token from an App ID and private key 15 | # To create a new GitHub App: 16 | # https://developer.github.com/apps/building-github-apps/creating-a-github-app/ 17 | # See app.yml for an example app manifest 18 | - name: Generate token 19 | id: generate_token 20 | uses: tibdex/github-app-token@v1 21 | with: 22 | app_id: ${{ secrets.DEPLOY_BOT_APP_ID }} 23 | private_key: ${{ secrets.DEPLOY_BOT_PRIVATE_KEY }} 24 | 25 | - name: Slash Command Dispatch 26 | uses: peter-evans/slash-command-dispatch@v2 27 | env: 28 | TOKEN: ${{ steps.generate_token.outputs.token }} 29 | with: 30 | token: ${{ env.TOKEN }} # GitHub App installation access token 31 | #token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} # PAT or OAuth token will also work 32 | reaction-token: ${{ secrets.GITHUB_TOKEN }} 33 | issue-type: pull-request 34 | commands: test-e2e-deploy 35 | permission: write 36 | 37 | help-command: 38 | runs-on: ubuntu-latest 39 | steps: 40 | # Checkout is necessary here due to referencing a local action. 41 | # It's also necessary when using the 'config-from-file' option. 42 | # Otherwise, avoid using checkout to keep this workflow fast. 43 | - uses: actions/checkout@v2 44 | 45 | - name: Generate token 46 | id: generate_token 47 | uses: tibdex/github-app-token@v1 48 | with: 49 | app_id: ${{ secrets.DEPLOY_BOT_APP_ID }} 50 | private_key: ${{ secrets.DEPLOY_BOT_PRIVATE_KEY }} 51 | 52 | - name: slash command dispatch 53 | uses: peter-evans/slash-command-dispatch@v2 54 | env: 55 | TOKEN: ${{ steps.generate_token.outputs.token }} 56 | with: 57 | token: ${{ env.TOKEN }} 58 | commands: help 59 | permission: none 60 | issue-type: both 61 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # Editors 132 | .idea 133 | .vscode 134 | *.egg-info 135 | 136 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | fmt: 2 | # Format the file in repo 3 | terraform fmt ./bentoctl_sagemaker/templates/*.tf && \ 4 | echo "Formated terraform templates" 5 | isort bentoctl_sagemaker tests 6 | black bentoctl_sagemaker tests 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## ⚠️ BentoCTL project has been deprecated 2 | 3 | Plese see the latest BentoML documentation on OCI-container based deployment workflow: https://docs.bentoml.com/ 4 | 5 | ## AWS Sagemaker Operator 6 | 7 | Sagemaker is a fully managed service for building ML models. BentoML provides great support 8 | for deploying BentoService to AWS Sagemaker without the additional process and work from users. With [BentoML serving framework](https://github.com/bentoml/BentoML) and [bentoctl](https://github.com/bentoml/bentoctl) users can enjoy the performance and scalability of Sagemaker with any popular ML frameworks. 9 | 10 | > **Note:** This operator is compatible with BentoML version 1.0.0 and above. For older versions, please switch to the branch `pre-v1.0` and follow the instructions in the README.md. 11 | 12 | 13 | ## Table of Contents 14 | 15 | * [Quickstart with bentoctl](#quickstart-with-bentoctl) 16 | * [Configuration options](#configuration-options) 17 | * [Troubleshooting](#troubleshooting) 18 | 19 | 20 | ## Quickstart with bentoctl 21 | 22 | This quickstart will walk you through deploying a bento as an AWS Sagemaker Endpoint. Make sure to go through the [prerequisites](#prerequisites) section and follow the instructions to set everything up. 23 | 24 | ### Prerequisites 25 | 26 | 1. BentoML version 1.0 or above. Please follow the [Installation guide](https://docs.bentoml.org/en/latest/installation.html). 27 | 2. Terraform - [Terraform](https://www.terraform.io/) is a tool for building, configuring, and managing infrastructure. Installation instruction: www.terraform.io/downloads 28 | 3. AWS CLI - installed and configured with an AWS account with permission to Sagemaker, Lambda and ECR. Please follow the [Installation guide](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html). 29 | 4. Docker - Install instruction: [docs.docker.com/install](https://docs.docker.com/install) 30 | 5. A built Bento project. For this guide, we will use the Iris classifier bento from the [BentoML quickstart guide](https://docs.bentoml.org/en/latest/quickstart.html#quickstart). You can also use your own Bentos that are available locally. 31 | 32 | ### Steps 33 | 34 | 1. Install bentoctl via pip 35 | ``` 36 | $ pip install bentoctl 37 | ``` 38 | 39 | 2. Install AWS Sagemaker operator 40 | 41 | Bentoctl will install the official AWS Sagemaker operator and its dependencies. 42 | 43 | ``` 44 | $ bentoctl operator install aws-sagemaker 45 | ``` 46 | 47 | 3. Initialize deployment with bentoctl 48 | 49 | Follow the interactive guide to initialize the deployment project. 50 | 51 | ```bash 52 | $ bentoctl init 53 | 54 | Bentoctl Interactive Deployment Config Builder 55 | 56 | Welcome! You are now in interactive mode. 57 | 58 | This mode will help you setup the deployment_config.yaml file required for 59 | deployment. Fill out the appropriate values for the fields. 60 | 61 | (deployment config will be saved to: ./deployment_config.yaml) 62 | 63 | api_version: v1 64 | name: quickstart 65 | operator: aws-sagemaker 66 | template: terraform 67 | spec: 68 | region: ap-south-1 69 | instance_type: ml.t2.medium 70 | initial_instance_count: 1 71 | timeout: 60 72 | enable_data_capture: False 73 | destination_s3_uri: 74 | initial_sampling_percentage: 1 75 | filename for deployment_config [deployment_config.yaml]: 76 | deployment config generated to: deployment_config.yaml 77 | ✨ generated template files. 78 | - ./main.tf 79 | - ./bentoctl.tfvars 80 | ``` 81 | This will also run the `bentoctl generate` command for you and will generate the `main.tf` terraform file, which specifies the resources to be created and the `bentoctl.tfvars` file which contains the values for the variables used in the `main.tf` file. 82 | 83 | 4. Build and push AWS sagemaker compatible docker image to the registry 84 | 85 | Bentoctl will build and push the sagemaker compatible docker image to the AWS ECR repository. 86 | 87 | ```bash 88 | bentoctl build -b iris_classifier:latest -f deployment_config.yaml 89 | 90 | Step 1/22 : FROM bentoml/bento-server:1.0.0a6-python3.8-debian-runtime 91 | ---> 046bc2e28220 92 | Step 2/22 : ARG UID=1034 93 | ---> Using cache 94 | ---> f44cfa910c52 95 | Step 3/22 : ARG GID=1034 96 | ---> Using cache 97 | ---> e4d5aed007af 98 | Step 4/22 : RUN groupadd -g $GID -o bentoml && useradd -m -u $UID -g $GID -o -r bentoml 99 | ---> Using cache 100 | ---> fa8ddcfa15cf 101 | ... 102 | Step 22/22 : CMD ["bentoml", "serve", ".", "--production"] 103 | ---> Running in 28eccee2f650 104 | ---> 98bc66e49cd9 105 | Successfully built 98bc66e49cd9 106 | Successfully tagged quickstart:kiouq7wmi2gmockr 107 | 🔨 Image build! 108 | Created the repository quickstart 109 | The push refers to repository 110 | [213386773652.dkr.ecr.ap-south-1.amazonaws.com/quickstart] 111 | kiouq7wmi2gmockr: digest: 112 | sha256:e1a468e6b9ceeed65b52d0ee2eac9e3cd1a57074eb94db9c263be60e4db98881 size: 3250 113 | 63984d77b4da: Pushed 114 | 2bc5eef20c91: Pushed 115 | ... 116 | da0af9cdde98: Layer already exists 117 | e5baccb54724: Layer already exists 118 | 🚀 Image pushed! 119 | ✨ generated template files. 120 | - ./bentoctl.tfvars 121 | - ./startup_script.sh 122 | ``` 123 | The iris-classifier service is now built and pushed into the container registry and the required terraform files have been created. Now we can use terraform to perform the deployment. 124 | 125 | 5. Apply Deployment with Terraform 126 | 127 | 1. Initialize terraform project. This installs the AWS provider and sets up the terraform folders. 128 | ```bash 129 | $ terraform init 130 | ``` 131 | 132 | 2. Apply terraform project to create Sagemaker deployment 133 | 134 | ```bash 135 | $ terraform apply -var-file=bentoctl.tfvars -auto-approve 136 | 137 | aws_iam_role.iam_role_lambda: Creating... 138 | aws_iam_role.iam_role_sagemaker: Creating... 139 | aws_apigatewayv2_api.lambda: Creating... 140 | aws_apigatewayv2_api.lambda: Creation complete after 1s [id=rwfej5qsf6] 141 | aws_cloudwatch_log_group.api_gw: Creating... 142 | aws_cloudwatch_log_group.api_gw: Creation complete after 1s [id=/aws/api_gw/quickstart-gw] 143 | aws_apigatewayv2_stage.lambda: Creating... 144 | aws_apigatewayv2_stage.lambda: Creation complete after 3s [id=$default] 145 | aws_iam_role.iam_role_sagemaker: Creation complete after 7s [id=quickstart-sagemaker-iam-role] 146 | aws_sagemaker_model.sagemaker_model: Creating... 147 | aws_iam_role.iam_role_lambda: Creation complete after 8s [id=quickstart-lambda-iam-role] 148 | aws_lambda_function.fn: Creating... 149 | ... 150 | 151 | 152 | Apply complete! Resources: 1 added, 0 changed, 0 destroyed. 153 | 154 | Outputs: 155 | 156 | endpoint = "https://rwfej5qsf6.execute-api.ap-south-1.amazonaws.com/" 157 | ecr_image_tag = "213386773652.dkr.ecr.ap-south-1.amazonaws.com/quickstart:sfx3dagmpogmockr" 158 | ``` 159 | 160 | 6. Test deployed endpoint 161 | 162 | The `iris_classifier` uses the `/classify` endpoint for receiving requests so the full URL for the classifier will be in the form `{EndpointUrl}/classify`. 163 | 164 | ```bash 165 | URL=$(terraform output -json | jq -r .endpoint.value)classify 166 | curl -i \ 167 | --header "Content-Type: application/json" \ 168 | --request POST \ 169 | --data '[5.1, 3.5, 1.4, 0.2]' \ 170 | $URL 171 | 172 | HTTP/2 200 173 | date: Thu, 14 Apr 2022 23:02:45 GMT 174 | content-type: application/json 175 | content-length: 1 176 | apigw-requestid: Ql8zbicdSK4EM5g= 177 | 178 | 0% 179 | ``` 180 | 181 | > Note: You can also [invoke the Sagemaker endpoint directly](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_runtime_InvokeEndpoint.html). If there is only one service, SageMaker deployment will choose that one. If there is more than one, you can specify which service to use by passing the `X-Amzn-SageMaker-Custom-Attributes` header with the name of the service as value. 182 | 183 | 7. Delete deployment 184 | Use the `bentoctl destroy` command to remove the registry and the deployment 185 | 186 | ```bash 187 | bentoctl destroy -f deployment_config.yaml 188 | 189 | ## Configuration Options 190 | 191 | A sample configuration file has been given has been provided [here](sagemaker_config.json). Feel free to copy it over and change it for you specific deployment values 192 | 193 | * `region`: AWS region where Sagemaker endpoint is deploying to 194 | * `instance_type`: The ML compute instance type for Sagemaker endpoint. See https://docs.aws.amazon.com/cli/latest/reference/sagemaker/create-endpoint-config.html for available instance types 195 | * `initial_instance_count`: Number of instances to launch initially. 196 | * `timeout`: timeout for API request in seconds 197 | * `enable_data_capture`: Enable Sagemaker capture data from requests and responses and store the captured data to AWS S3 198 | * `destination_s3_uri`: S3 bucket path for store captured data 199 | * `initial_sampling_percentage`: Percentage of the data will be captured to S3 bucket. 200 | 201 | ## Troubleshooting 202 | By default sagemaker is configured with cloudwatch for metrics and logs. To see the cloudwatch logs for the deployment 203 | 204 | 1. Open the Amazon Cloudwatch console at https://console.aws.amazon.com/cloudwatch/. 205 | 2. In the navigation pane, choose Logs -> Log groups. 206 | 3. Head over to /aws/sagemaker/Endpoints/-endpoint 207 | 4. Choose the latest logs streams 208 | -------------------------------------------------------------------------------- /bentoctl_sagemaker/__init__.py: -------------------------------------------------------------------------------- 1 | from bentoctl_sagemaker.create_deployable import create_deployable 2 | from bentoctl_sagemaker.generate import generate 3 | from bentoctl_sagemaker.registry_utils import create_repository, delete_repository 4 | 5 | __all__ = ["generate", "create_deployable", "create_repository", "delete_repository"] 6 | -------------------------------------------------------------------------------- /bentoctl_sagemaker/create_deployable.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | import shutil 5 | from pathlib import Path 6 | from sys import version_info 7 | from typing import Any 8 | 9 | from attr import asdict 10 | from bentoml._internal.bento.bento import BentoInfo 11 | from bentoml._internal.bento.build_config import DockerOptions 12 | from bentoml._internal.bento.gen import generate_dockerfile 13 | 14 | if version_info >= (3, 8): 15 | from shutil import copytree 16 | else: 17 | from backports.shutil_copytree import copytree 18 | 19 | root_dir = Path(os.path.abspath(os.path.dirname(__file__)), "sagemaker") 20 | SERVICE_PATH = os.path.join(root_dir, "service.py") 21 | TEMPLATE_PATH = os.path.join(root_dir, "template.j2") 22 | SERVE_PATH = os.path.join(root_dir, "serve") 23 | 24 | 25 | def create_deployable( 26 | bento_path: str, 27 | destination_dir: str, 28 | bento_metadata: dict[str, Any], 29 | overwrite_deployable: bool, 30 | ) -> str: 31 | """ 32 | The deployable is the bento along with all the modifications (if any) 33 | requried to deploy to the cloud service. 34 | 35 | Parameters 36 | ---------- 37 | bento_path: str 38 | Path to the bento from the bento store. 39 | destination_dir: str 40 | directory to create the deployable into. 41 | bento_metadata: dict 42 | metadata about the bento. 43 | 44 | Returns 45 | ------- 46 | docker_context_path : str 47 | path to the docker context. 48 | """ 49 | deployable_path = Path(destination_dir) 50 | 51 | # copy over the bento bundle 52 | copytree(bento_path, deployable_path, dirs_exist_ok=True) 53 | 54 | bento_metafile = Path(bento_path, "bento.yaml") 55 | with bento_metafile.open("r", encoding="utf-8") as metafile: 56 | info = BentoInfo.from_yaml_file(metafile) 57 | 58 | options = asdict(info.docker) 59 | options["dockerfile_template"] = TEMPLATE_PATH 60 | 61 | dockerfile_path = deployable_path.joinpath("env", "docker", "Dockerfile") 62 | with dockerfile_path.open("w", encoding="utf-8") as dockerfile: 63 | dockerfile.write( 64 | generate_dockerfile( 65 | DockerOptions(**options).with_defaults(), 66 | str(deployable_path), 67 | use_conda=not info.conda.is_empty(), 68 | ) 69 | ) 70 | 71 | # copy sagemaker service.py 72 | shutil.copy( 73 | SERVICE_PATH, 74 | os.path.join(deployable_path, "sagemaker_service.py"), 75 | ) 76 | 77 | # then copy the serve script 78 | serve_fspath = os.path.join(deployable_path, "serve") 79 | shutil.copy(SERVE_PATH, serve_fspath) 80 | # permission 755 is required for entry script 'serve' 81 | os.chmod(serve_fspath, 0o755) 82 | 83 | return str(deployable_path) 84 | -------------------------------------------------------------------------------- /bentoctl_sagemaker/generate.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | from bentoctl.exceptions import TemplateExists, TemplateTypeNotDefined 5 | 6 | from bentoctl_sagemaker.parameters import DeploymentParams 7 | 8 | data_capture_schema = { 9 | "enable_data_capture": { 10 | "required": False, 11 | "default": False, 12 | "type": "boolean", 13 | "coerce": bool, 14 | "help_message": "Enable data capture for the sagemaker deployment", 15 | }, 16 | "destination_s3_uri": { 17 | "required": False, 18 | "default": "", 19 | "type": "string", 20 | "help_message": "S3 URI for the data capture for the sagemaker deployment", 21 | }, 22 | "initial_sampling_percentage": { 23 | "required": False, 24 | "type": "integer", 25 | "default": "1", 26 | "coerce": int, 27 | "help_message": "Percentage of the data capture for the sagemaker deployment. Value between 0 and 100", 28 | "max": 100, 29 | "min": 1, 30 | }, 31 | } 32 | 33 | 34 | def copy_template(template_name: str, destination_dir: str): 35 | TERRAFORM_TEMPLATE_FILE_NAME = "main.tf" 36 | template_name = template_name + ".tf" 37 | 38 | template_file = os.path.join(destination_dir, TERRAFORM_TEMPLATE_FILE_NAME) 39 | if os.path.exists(template_file): 40 | raise TemplateExists(template_file) 41 | 42 | shutil.copyfile( 43 | os.path.join(os.path.dirname(__file__), f"templates/{template_name}"), 44 | template_file, 45 | ) 46 | 47 | return template_file 48 | 49 | 50 | def generate_terraform_template(spec: dict, destination_dir: str): 51 | if spec.get("enable_data_capture") == True: 52 | return copy_template("terraform_with_data_capture", destination_dir) 53 | else: 54 | return copy_template("terraform_default", destination_dir) 55 | 56 | 57 | def generate_terraform_values(name: str, spec: dict, destination_dir: str): 58 | TERRAFORM_VALUES_FILE_NAME = "bentoctl.tfvars" 59 | 60 | if spec.get("enable_data_capture") == False: 61 | for key in data_capture_schema: 62 | try: 63 | del spec[key] 64 | except KeyError: 65 | continue 66 | elif spec.get("enable_data_capture") == True: 67 | del spec["enable_data_capture"] 68 | 69 | params = DeploymentParams(name, spec, "terraform") 70 | values_file = os.path.join(destination_dir, TERRAFORM_VALUES_FILE_NAME) 71 | params.to_params_file(values_file) 72 | 73 | return values_file 74 | 75 | 76 | def generate( 77 | name: str, 78 | spec: dict, 79 | template_type: str, 80 | destination_dir: str, 81 | values_only: bool = True, 82 | ): 83 | """ 84 | generates the template corresponding to the template_type. 85 | 86 | Parameters 87 | ---------- 88 | name : str 89 | deployment name to be used by the template. This name will be used 90 | to create the resource names. 91 | spec : dict 92 | The properties of the deployment (specifications) passed from the 93 | deployment_config's `spec` section. 94 | template_type: str 95 | The type of template that is to be generated by the operator. The 96 | available ones are [terraform, cloudformation] 97 | destination_dir: str 98 | The directory into which the files are generated. 99 | values_only: bool 100 | Generate only the values files. 101 | 102 | Returns 103 | ------- 104 | generated_path : str 105 | The path for the generated template. 106 | """ 107 | generated_files = [] 108 | 109 | if template_type == "terraform": 110 | if not values_only: 111 | template_file_path = generate_terraform_template(spec, destination_dir) 112 | generated_files.append(template_file_path) 113 | values_file_path = generate_terraform_values(name, spec, destination_dir) 114 | generated_files.append(values_file_path) 115 | else: 116 | raise TemplateTypeNotDefined(template_type) 117 | 118 | return generated_files 119 | -------------------------------------------------------------------------------- /bentoctl_sagemaker/parameters.py: -------------------------------------------------------------------------------- 1 | from collections import UserDict 2 | 3 | DEPLOYMENT_PARAMS_WARNING = """# This file is maintained automatically by 4 | # "bentoctl generate" and "bentoctl build" commands. 5 | # Manual edits may be lost the next time these commands are run. 6 | 7 | """ 8 | 9 | 10 | def parse_image_tag(image_tag: str): 11 | registry_url, tag = image_tag.split("/") 12 | repository, version = tag.split(":") 13 | 14 | return registry_url, repository, version 15 | 16 | 17 | class DeploymentParams(UserDict): 18 | def __init__(self, name, spec, template_type): 19 | if "image_tag" in spec: 20 | _, image_repository, image_version = parse_image_tag(spec["image_tag"]) 21 | spec["image_repository"] = image_repository 22 | spec["image_version"] = image_version 23 | 24 | super().__init__({"deployment_name": name, **spec}) 25 | self.template_type = template_type 26 | 27 | def to_params_file(self, file_path): 28 | if self.template_type == "terraform": 29 | self.generate_terraform_tfvars_file(file_path) 30 | 31 | @classmethod 32 | def from_params_file(cls, file_path): 33 | pass 34 | 35 | def generate_terraform_tfvars_file(self, file_path): 36 | params = [] 37 | for param_name, param_value in self.items(): 38 | params.append(f'{param_name} = "{param_value}"') 39 | 40 | with open(file_path, "w") as params_file: 41 | params_file.write(DEPLOYMENT_PARAMS_WARNING) 42 | params_file.write("\n".join(params)) 43 | params_file.write("\n") 44 | -------------------------------------------------------------------------------- /bentoctl_sagemaker/registry_utils.py: -------------------------------------------------------------------------------- 1 | import base64 2 | 3 | import boto3 4 | 5 | 6 | def get_ecr_login_info(region, repository_id): 7 | ecr_client = boto3.client("ecr", region) 8 | token = ecr_client.get_authorization_token(registryIds=[repository_id]) 9 | username, password = ( 10 | base64.b64decode(token["authorizationData"][0]["authorizationToken"]) 11 | .decode("utf-8") 12 | .split(":") 13 | ) 14 | registry_url = token["authorizationData"][0]["proxyEndpoint"] 15 | 16 | return registry_url, username, password 17 | 18 | 19 | def get_repository(ecr_client, repository_name): 20 | result = ecr_client.describe_repositories(repositoryNames=[repository_name]) 21 | repository_id = result["repositories"][0]["registryId"] 22 | repository_uri = result["repositories"][0]["repositoryUri"] 23 | 24 | return repository_id, repository_uri 25 | 26 | 27 | def create_ecr_repository_if_not_exists(region, repository_name): 28 | ecr_client = boto3.client("ecr", region) 29 | try: 30 | result = ecr_client.describe_repositories(repositoryNames=[repository_name]) 31 | repository_id = result["repositories"][0]["registryId"] 32 | repository_uri = result["repositories"][0]["repositoryUri"] 33 | except ecr_client.exceptions.RepositoryNotFoundException: 34 | result = ecr_client.create_repository(repositoryName=repository_name) 35 | repository_id = result["repository"]["registryId"] 36 | repository_uri = result["repository"]["repositoryUri"] 37 | return repository_id, repository_uri 38 | 39 | 40 | def create_repository(repository_name, operator_spec): 41 | """ 42 | Create ECR repository and return the information. 43 | """ 44 | repo_id, _ = create_ecr_repository_if_not_exists( 45 | operator_spec["region"], repository_name 46 | ) 47 | registry_url, username, password = get_ecr_login_info( 48 | operator_spec["region"], repo_id 49 | ) 50 | repository_url = f"{registry_url}/{repository_name}" 51 | 52 | return repository_url, username, password 53 | 54 | 55 | def delete_repository(repository_name, operator_spec): 56 | """ 57 | Delete the ECR repository created 58 | """ 59 | ecr_client = boto3.client("ecr", operator_spec.get("region")) 60 | try: 61 | get_repository(ecr_client, repository_name) 62 | ecr_client.delete_repository(repositoryName=repository_name, force=True) 63 | except ecr_client.exceptions.RepositoryNotFoundException: 64 | print(f"Repository {repository_name} not found. Skipping registry cleanup") 65 | -------------------------------------------------------------------------------- /bentoctl_sagemaker/sagemaker/README.md: -------------------------------------------------------------------------------- 1 | ## Sagemaker Requirements 2 | 3 | how sagemaker runs its docker images. 4 | 5 | 1. sagemaker overides the default CMD command with service and 6 | calls docker image as `docker run serve`. In our docker container the 7 | `serve` script is what call `bentoml serve` 8 | 9 | 2. expose port 8080 10 | 11 | 3. defines 2 endpoints 12 | /invocation - this is the endpoint into which the request comes in 13 | / ping - sagemaker checks this to ensure the container is running 14 | 15 | more details - [Use Your Own Inference Code with Hosting Services - Amazon SageMaker](https://docs.aws.amazon.com/sagemaker/latest/dg/your-algorithms-inference-code.html) 16 | 17 | ## Sagemaker_service 18 | 19 | custom bentoml service that loads the bentoml service present locally. It loads a middleware which redirects the traffic comming from `/ping` to `/livez` in the bentoml service and `/invocation` to the curresponding bentoml path based on the header `AWS_CUSTOM_ENDPOINT_HEADER = "X-Amzn-SageMaker-Custom-Attributes"`. 20 | 21 | 22 | ## Running locally 23 | 24 | 1. build with bentoctl - `bentoctl build --debug --dry-run` 25 | 2. docker run with the docker tag from the output of the previous step. `docker run -p 8080:8080 serve`. 26 | -------------------------------------------------------------------------------- /bentoctl_sagemaker/sagemaker/serve: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This implement the sagemaker serving service shell. It starts nginx and gunicorn. 4 | # Parameter Env Var Default Value 5 | # number of workers BENTO_SERVER_TIMEOUT 60s 6 | # timeout GUNICORN_WORKER_COUNT number of cpu cores / 2 + 1 7 | # api name API_NAME None 8 | 9 | import os 10 | import signal 11 | import subprocess 12 | import sys 13 | 14 | 15 | def sigterm_handler(bentoserver_pid): 16 | try: 17 | os.kill(bentoserver_pid, signal.SIGTERM) 18 | except OSError: 19 | pass 20 | 21 | sys.exit(0) 22 | 23 | 24 | def start_bentoml_production_server(): 25 | bento_server = subprocess.Popen( 26 | [ 27 | "bentoml", 28 | "serve", 29 | "--production", 30 | "--port", 31 | "8080", 32 | "sagemaker_service:svc", 33 | ] 34 | ) 35 | signal.signal(signal.SIGTERM, lambda: sigterm_handler(bento_server.pid)) 36 | 37 | pids = {bento_server.pid} 38 | while True: 39 | pid, _ = os.wait() 40 | if pid in pids: 41 | break 42 | print("Inference server exiting") 43 | sigterm_handler(bento_server.pid) 44 | 45 | 46 | if __name__ == "__main__": 47 | start_bentoml_production_server() 48 | -------------------------------------------------------------------------------- /bentoctl_sagemaker/sagemaker/service.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import bentoml 4 | from bentoml.exceptions import BentoMLException 5 | from starlette.requests import Request 6 | from starlette.types import ASGIApp, Receive, Scope, Send 7 | 8 | AWS_SAGEMAKER_SERVE_PORT = 8080 9 | AWS_CUSTOM_ENDPOINT_HEADER = "X-Amzn-SageMaker-Custom-Attributes" 10 | BENTOML_HEALTH_CHECK_PATH = "/livez" 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | # use standalone_load so that the path is not changed back 15 | # after loading. 16 | svc = bentoml.load(".", standalone_load=True) 17 | 18 | 19 | class SagemakerMiddleware: 20 | def __init__( 21 | self, 22 | app: ASGIApp, 23 | ) -> None: 24 | self.app = app 25 | 26 | async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: 27 | if scope["type"] == "http": 28 | req = Request(scope, receive) 29 | if req.url.path == "/ping": 30 | scope["path"] = BENTOML_HEALTH_CHECK_PATH 31 | 32 | if req.url.path == "/invocations": 33 | if AWS_CUSTOM_ENDPOINT_HEADER not in req.headers: 34 | if len(svc.apis) == 1: 35 | # only one api, use it 36 | api_path, *_ = svc.apis 37 | logger.info( 38 | f"'{AWS_CUSTOM_ENDPOINT_HEADER}' not found in request header. Using defualt {api_path} service." 39 | ) 40 | else: 41 | logger.error( 42 | f"'{AWS_CUSTOM_ENDPOINT_HEADER}' not found inside request header. If you are directly invoking the Sagemaker Endpoint pass in the '{AWS_CUSTOM_ENDPOINT_HEADER}' with the bentoml service name that you want to invoke." 43 | ) 44 | raise BentoMLException( 45 | f"'{AWS_CUSTOM_ENDPOINT_HEADER}' not found inside request header." 46 | ) 47 | else: 48 | api_path = req.headers[AWS_CUSTOM_ENDPOINT_HEADER] 49 | if api_path not in svc.apis: 50 | message = f"API Service passed via the '{AWS_CUSTOM_ENDPOINT_HEADER}' header '{api_path}' not found in the bentoml service." 51 | logger.error(message) 52 | raise BentoMLException(message) 53 | scope["path"] = "/" + api_path 54 | 55 | await self.app(scope, receive, send) 56 | 57 | 58 | svc.add_asgi_middleware(SagemakerMiddleware) 59 | -------------------------------------------------------------------------------- /bentoctl_sagemaker/sagemaker/template.j2: -------------------------------------------------------------------------------- 1 | {% extends bento_base_template %} 2 | {% block SETUP_BENTO_ENTRYPOINT %} 3 | EXPOSE 8080 4 | USER bentoml 5 | RUN chmod +x {{ bento__path }}/serve 6 | 7 | ENV PATH="${PATH}:{{ bento__path }}" 8 | {% endblock %} 9 | -------------------------------------------------------------------------------- /bentoctl_sagemaker/templates/terraform_default.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | aws = { 4 | source = "hashicorp/aws" 5 | version = "~> 4.0.0" 6 | } 7 | archive = { 8 | source = "hashicorp/archive" 9 | version = "~> 2.2.0" 10 | } 11 | } 12 | } 13 | 14 | provider "aws" { 15 | region = var.region 16 | } 17 | 18 | ################################################################################ 19 | # Input variable definitions 20 | ################################################################################ 21 | 22 | variable "deployment_name" { 23 | type = string 24 | } 25 | 26 | variable "image_tag" { 27 | type = string 28 | } 29 | 30 | variable "image_repository" { 31 | type = string 32 | } 33 | 34 | variable "image_version" { 35 | type = string 36 | } 37 | 38 | variable "region" { 39 | type = string 40 | } 41 | 42 | variable "timeout" { 43 | type = number 44 | } 45 | 46 | variable "instance_type" { 47 | type = string 48 | } 49 | variable "initial_instance_count" { 50 | type = number 51 | } 52 | 53 | ################################################################################ 54 | # Resource definitions 55 | ################################################################################ 56 | 57 | data "aws_ecr_repository" "service" { 58 | name = var.image_repository 59 | } 60 | 61 | data "aws_ecr_image" "service_image" { 62 | repository_name = data.aws_ecr_repository.service.name 63 | image_tag = var.image_version 64 | } 65 | 66 | resource "aws_iam_role" "iam_role_sagemaker" { 67 | name = "${var.deployment_name}-sagemaker-iam-role" 68 | managed_policy_arns = ["arn:aws:iam::aws:policy/AmazonSageMakerFullAccess"] 69 | assume_role_policy = jsonencode({ 70 | Version = "2012-10-17" 71 | Statement = [ 72 | { 73 | Action = "sts:AssumeRole" 74 | Effect = "Allow" 75 | Sid = "" 76 | Principal = { 77 | Service = "sagemaker.amazonaws.com" 78 | } 79 | }, 80 | ] 81 | }) 82 | } 83 | 84 | resource "aws_sagemaker_model" "sagemaker_model" { 85 | lifecycle { 86 | create_before_destroy = true 87 | } 88 | name = "${var.deployment_name}-model-${var.image_version}" 89 | execution_role_arn = resource.aws_iam_role.iam_role_sagemaker.arn 90 | primary_container { 91 | image = "${data.aws_ecr_repository.service.repository_url}@${data.aws_ecr_image.service_image.id}" 92 | mode = "SingleModel" 93 | } 94 | } 95 | 96 | resource "aws_sagemaker_endpoint_configuration" "endpoint_config" { 97 | lifecycle { 98 | create_before_destroy = true 99 | } 100 | name = "${var.deployment_name}-endpoint-config-${var.image_version}" 101 | 102 | production_variants { 103 | initial_instance_count = var.initial_instance_count 104 | initial_variant_weight = 1.0 105 | instance_type = var.instance_type 106 | model_name = aws_sagemaker_model.sagemaker_model.name 107 | variant_name = "default" 108 | } 109 | } 110 | 111 | resource "aws_sagemaker_endpoint" "sagemaker_endpoint" { 112 | name = "${var.deployment_name}-endpoint" 113 | endpoint_config_name = aws_sagemaker_endpoint_configuration.endpoint_config.name 114 | } 115 | 116 | # Lambda function as a proxy to invoke Sagemaker Endpoint 117 | resource "aws_iam_role" "iam_role_lambda" { 118 | name = "${var.deployment_name}-lambda-iam-role" 119 | managed_policy_arns = [ 120 | "arn:aws:iam::aws:policy/AmazonSageMakerFullAccess", 121 | "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" 122 | ] 123 | assume_role_policy = jsonencode({ 124 | Version = "2012-10-17" 125 | Statement = [ 126 | { 127 | Action = "sts:AssumeRole" 128 | Effect = "Allow" 129 | Sid = "" 130 | Principal = { 131 | Service = "lambda.amazonaws.com" 132 | } 133 | }, 134 | ] 135 | }) 136 | } 137 | 138 | data "archive_file" "lambda_inline_zip" { 139 | type = "zip" 140 | output_path = "/tmp/lambda_zip_inline.zip" 141 | source { 142 | content = <=1.0.0rc1 2 | boto3 3 | docker 4 | pandas 5 | rich 6 | numpy 7 | pillow 8 | backports.shutil_copytree -------------------------------------------------------------------------------- /tests/classifier.py: -------------------------------------------------------------------------------- 1 | # iris_classifier.py 2 | from bentoml import BentoService, api, env 3 | from bentoml.adapters import DataframeInput, FileInput, ImageInput, JsonInput 4 | from bentoml.types import FileLike, JsonSerializable 5 | 6 | 7 | @env(infer_pip_packages=True) 8 | class TestService(BentoService): 9 | @api(input=DataframeInput(), batch=True) 10 | def dfapi(self, df): 11 | print(df) 12 | return df 13 | 14 | @api(input=JsonInput(), batch=False) 15 | def jsonapi(self, json: JsonSerializable): 16 | print(json) 17 | return json 18 | 19 | @api(input=FileInput(), batch=False) 20 | def fileapi(self, file_stream: FileLike): 21 | print(file_stream) 22 | if file_stream.bytes_ is not None: 23 | return file_stream.bytes_ 24 | else: 25 | return file_stream._stream.read() 26 | 27 | @api(input=ImageInput(), batch=False) 28 | def imageapi(self, img): 29 | print(img.shape) 30 | return img.shape 31 | -------------------------------------------------------------------------------- /tests/test_api.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import sys 4 | import tempfile 5 | import time 6 | 7 | import requests 8 | from classifier import TestService 9 | from pandas import DataFrame 10 | 11 | sys.path.append("./") 12 | from delete import delete 13 | from deploy import deploy 14 | from describe import describe 15 | 16 | 17 | class Setup: 18 | def __init__(self): 19 | """ 20 | Setup the deployment on the deployment choosen 21 | """ 22 | self.deployment_name = "sagemaker_bento_deploy_test" 23 | self.dirpath = tempfile.mkdtemp() 24 | print("temp dir {} created!".format(self.dirpath)) 25 | self.saved_dir = os.path.join(self.dirpath, "saved_dir") 26 | 27 | # make config file 28 | config = """ 29 | { 30 | "region": "us-west-1", 31 | "instance_type": "ml.t2.medium", 32 | "initial_instance_count": 1, 33 | "workers": 3, 34 | "timeout": 60, 35 | "enable_data_capture": false, 36 | "data_capture_s3_prefix": "s3://bucket-name/optional/predix", 37 | "data_capture_sample_percent": 100 38 | } 39 | """ 40 | self.config_file = os.path.join(self.dirpath, "config.json") 41 | with open(self.config_file, "w") as f: 42 | f.write(config) 43 | 44 | # make bento service 45 | os.mkdir(self.saved_dir) 46 | test_service = TestService() 47 | # test_service.pack() 48 | test_service.save_to_dir(self.saved_dir) 49 | 50 | @staticmethod 51 | def check_if_up(url, num_attempts=5, wait_time=20): 52 | attempt = 0 53 | while attempt < num_attempts: 54 | try: 55 | if requests.post(url).status_code == 400: 56 | print("Ok!") 57 | return True 58 | else: 59 | print("not Ok", end=" ") 60 | time.sleep(wait_time) 61 | except Exception as e: 62 | print(e) 63 | time.sleep(wait_time) 64 | finally: 65 | attempt += 1 66 | return False 67 | 68 | def make_deployment(self): 69 | deploy(self.saved_dir, self.deployment_name, self.config_file) 70 | info_json = describe(self.deployment_name, self.config_file) 71 | url = info_json["EndpointURL"] + "/{}" 72 | 73 | self.check_if_up(url.format("dfapi"), num_attempts=2) 74 | 75 | return url 76 | 77 | def teardown(self): 78 | delete(self.deployment_name, self.config_file) 79 | shutil.rmtree(self.dirpath) 80 | print("Removed {}!".format(self.dirpath)) 81 | 82 | 83 | def test_json(url): 84 | """ 85 | GIVEN the api is deployed 86 | WHEN a valid json is given 87 | THEN accepts the binary_data and returns it 88 | """ 89 | headers = {"content-type": "application/json"} 90 | input_json = "[[1, 2, 3, 4]]" 91 | resp = requests.post(url, data=input_json, headers=headers) 92 | assert resp.ok 93 | assert resp.content == bytearray(input_json, "ascii") 94 | 95 | 96 | def test_df(url): 97 | """ 98 | GIVEN the api is deployed 99 | WHEN a dataframe is passed, as json or csv 100 | THEN accepts the binary_data and returns it 101 | """ 102 | input_array = [[1, 2, 3, 4]] 103 | 104 | # request as json 105 | resp = requests.post(url, json=input_array) 106 | assert resp.ok 107 | assert DataFrame(resp.json()).to_json() == DataFrame(input_array).to_json() 108 | 109 | # request as csv 110 | headers = {"content-type": "text/csv"} 111 | csv = DataFrame(input_array).to_csv(index=False) 112 | resp = requests.post(url, data=csv, headers=headers) 113 | assert resp.ok 114 | assert DataFrame(resp.json()).to_json() == DataFrame(input_array).to_json() 115 | 116 | 117 | def test_files(url): 118 | """ 119 | GIVEN the api is deployed 120 | WHEN a file is passed either as raw bytes with any content-type or as mulitpart/form 121 | THEN it accepts the binary_data and returns it 122 | """ 123 | binary_data = b"test" 124 | 125 | # request with raw data 126 | headers = {"content-type": "image/jpeg"} 127 | resp = requests.post(url, data=binary_data, headers=headers) 128 | assert resp.ok 129 | assert resp.content == b'"test"' 130 | 131 | # request mulitpart/form-data 132 | file = {"audio": ("test", binary_data)} 133 | resp = requests.post(url, files=file) 134 | assert resp.ok 135 | assert resp.content == b'"test"' 136 | 137 | 138 | def test_image(url): 139 | """ 140 | GIVEN the api is deployed 141 | WHEN an image is passed as bytes or mulitpart/form-data 142 | THEN it accepts it and returns the size 143 | """ 144 | from io import BytesIO 145 | 146 | import numpy as np 147 | from PIL import Image 148 | 149 | img = Image.fromarray(np.uint8(np.random.rand(10, 10, 3) * 256)) 150 | byte_io = BytesIO() 151 | img.save(byte_io, "png") 152 | img_bytes = byte_io.getvalue() 153 | byte_io.close() 154 | 155 | # request with raw data 156 | headers = { 157 | "content-type": "image/jpeg", 158 | } 159 | resp = requests.post(url.format("imageapi"), headers=headers, data=img_bytes) 160 | assert resp.ok 161 | assert resp.content == b"[10, 10, 3]" 162 | 163 | # request mulitpart/form-data 164 | resp = requests.post( 165 | url.format("imageapi"), 166 | files={ 167 | "image": ( 168 | "test.png", 169 | img_bytes, 170 | ) 171 | }, 172 | ) 173 | assert resp.ok 174 | assert resp.content == b"[10, 10, 3]" 175 | 176 | 177 | if __name__ == "__main__": 178 | 179 | setup = Setup() 180 | failed = False 181 | try: 182 | url = setup.make_deployment() 183 | print(url) 184 | except Exception as e: 185 | print("Setup failed") 186 | raise e 187 | else: 188 | # setup successful! 189 | print("Setup successful") 190 | 191 | # list of tests to perform 192 | TESTS = [ 193 | (test_df, "dfapi"), 194 | (test_files, "fileapi"), 195 | (test_json, "jsonapi"), 196 | (test_image, "imageapi"), 197 | ] 198 | 199 | for test_func, endpoint in TESTS: 200 | try: 201 | print("Testing endpoint /{}...".format(endpoint), end="") 202 | test_func(url.format(endpoint)) 203 | print("\033[92m passed! \033[0m") 204 | except Exception as e: 205 | print("\033[91m failed! \033[0m") 206 | print("\nTest at endpoint /{} failded: ".format(endpoint), e) 207 | failed = True 208 | finally: 209 | setup.teardown() 210 | 211 | if failed: 212 | sys.exit(1) 213 | else: 214 | sys.exit(0) 215 | --------------------------------------------------------------------------------