├── .github └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── amazon_forecast_tutorial ├── common │ ├── data │ │ └── item-demand-time.csv │ ├── images │ │ ├── amazon_forecast.png │ │ └── forecast_workflow.png │ └── util │ │ ├── __init__.py │ │ ├── fcst_utils.py │ │ └── notebook_utils.py └── forecast_electricityusagedata.ipynb ├── autogluon └── tabular-prediction │ └── AutoGluon-Tabular-with-SageMaker │ ├── AutoGluon_Tabular_SageMaker.ipynb │ ├── README.md │ ├── container-inference │ ├── Dockerfile.inference │ └── build_push_inference.sh │ ├── container-training │ ├── Dockerfile.training │ ├── build_push_training.sh │ ├── inference.py │ └── train.py │ ├── setup.sh │ └── utils │ └── ag_utils.py ├── autopilot └── autopilot_customer_churn.ipynb ├── distributed_training ├── Dockerfile ├── README.md ├── build_and_push.sh ├── dist-training-profile.ipynb ├── e2e_mask_rcnn_R_50_FPN_1x_16GPU_4bs.yaml ├── e2e_mask_rcnn_R_50_FPN_1x_32GPU_4bs.yaml ├── e2e_mask_rcnn_R_50_FPN_1x_64GPU_4bs.yaml ├── e2e_mask_rcnn_R_50_FPN_1x_8GPU_4bs.yaml ├── images │ ├── .ipynb_checkpoints │ │ └── sm_debugger_insights-checkpoint.png │ ├── sm_debugger_insights.png │ ├── sm_debugger_nav.png │ ├── sm_debugger_sys_util.png │ ├── smdataparallel_horovod.png │ └── smdataparallel_pytorchddp.png ├── train_pytorch_single_maskrcnn.py ├── train_pytorch_smdataparallel_maskrcnn.py └── upload_coco2017_to_s3.sh ├── docs ├── 404.html ├── _print │ ├── about │ │ ├── featured-background.jpg │ │ ├── featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_1920x1080_fill_q75_catmullrom_bottom.jpg │ │ └── featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_960x540_fill_q75_catmullrom_bottom.jpg │ └── docs │ │ ├── ai-services │ │ ├── images │ │ │ ├── rekognition │ │ │ │ ├── celeb.png │ │ │ │ ├── compare-face.png │ │ │ │ ├── moderation.png │ │ │ │ ├── ppe.png │ │ │ │ ├── rek-obj.png │ │ │ │ └── text.png │ │ │ └── textract │ │ │ │ └── textract.png │ │ └── picture.png │ │ └── お客様事例集 │ │ └── picture.png ├── about │ ├── _print │ │ └── index.html │ ├── featured-background.jpg │ ├── featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_1920x1080_fill_q75_catmullrom_bottom.jpg │ ├── featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_960x540_fill_q75_catmullrom_bottom.jpg │ └── index.html ├── blog │ ├── 2018 │ │ ├── 10 │ │ │ └── 06 │ │ │ │ ├── easy-documentation-with-docsy │ │ │ │ ├── featured-sunset-get.png │ │ │ │ ├── featured-sunset-get_hu69849a7cdb847c2393a7b3a7f6061c86_387442_250x125_fill_catmullrom_center_2.png │ │ │ │ ├── featured-sunset-get_hu69849a7cdb847c2393a7b3a7f6061c86_387442_600x300_fill_catmullrom_smart1_2.png │ │ │ │ ├── featured-sunset-get_hu69849a7cdb847c2393a7b3a7f6061c86_387442_640x0_resize_catmullrom_2.png │ │ │ │ └── index.html │ │ │ │ └── the-second-blog-post │ │ │ │ └── index.html │ │ └── 01 │ │ │ └── 04 │ │ │ └── another-great-release │ │ │ └── index.html │ ├── _print │ │ └── index.html │ ├── index.html │ ├── news │ │ ├── _print │ │ │ └── index.html │ │ ├── index.html │ │ └── page │ │ │ └── 1 │ │ │ └── index.html │ ├── page │ │ └── 1 │ │ │ └── index.html │ └── releases │ │ ├── _print │ │ └── index.html │ │ ├── index.html │ │ └── page │ │ └── 1 │ │ └── index.html ├── community │ ├── _print │ │ └── index.html │ └── index.html ├── css │ ├── prism.css │ └── swagger-ui.css ├── docs │ ├── _print │ │ └── index.html │ ├── ai-services │ │ ├── _print │ │ │ └── index.html │ │ ├── images │ │ │ ├── _print │ │ │ │ └── index.html │ │ │ ├── index.html │ │ │ ├── rekognition │ │ │ │ ├── _print │ │ │ │ │ └── index.html │ │ │ │ ├── celeb.png │ │ │ │ ├── compare-face.png │ │ │ │ ├── index.html │ │ │ │ ├── moderation.png │ │ │ │ ├── ppe.png │ │ │ │ ├── rek-obj.png │ │ │ │ └── text.png │ │ │ └── textract │ │ │ │ ├── _print │ │ │ │ └── index.html │ │ │ │ ├── index.html │ │ │ │ └── textract.png │ │ ├── index.html │ │ ├── picture.png │ │ ├── pubref1 │ │ │ └── index.html │ │ ├── table-data │ │ │ ├── _print │ │ │ │ └── index.html │ │ │ ├── forecast │ │ │ │ ├── _print │ │ │ │ │ └── index.html │ │ │ │ └── index.html │ │ │ ├── fraud │ │ │ │ ├── _print │ │ │ │ │ └── index.html │ │ │ │ └── index.html │ │ │ ├── index.html │ │ │ └── personalize │ │ │ │ ├── _print │ │ │ │ └── index.html │ │ │ │ └── index.html │ │ ├── text │ │ │ ├── _print │ │ │ │ └── index.html │ │ │ ├── comprehend │ │ │ │ ├── _print │ │ │ │ │ └── index.html │ │ │ │ └── index.html │ │ │ ├── index.html │ │ │ ├── polly │ │ │ │ ├── _print │ │ │ │ │ └── index.html │ │ │ │ └── index.html │ │ │ └── translate │ │ │ │ ├── _print │ │ │ │ └── index.html │ │ │ │ └── index.html │ │ └── voice │ │ │ ├── _print │ │ │ └── index.html │ │ │ ├── index.html │ │ │ └── transcribe │ │ │ ├── _print │ │ │ └── index.html │ │ │ └── index.html │ ├── examples │ │ ├── _print │ │ │ └── index.html │ │ └── index.html │ ├── index.html │ ├── ml-services │ │ ├── _print │ │ │ └── index.html │ │ ├── index.html │ │ ├── sagemaker-dive-deep │ │ │ ├── _print │ │ │ │ └── index.html │ │ │ └── index.html │ │ └── sagemaker-tutorial │ │ │ ├── _print │ │ │ └── index.html │ │ │ └── index.html │ ├── usecase │ │ ├── _print │ │ │ └── index.html │ │ ├── image │ │ │ └── index.html │ │ ├── index.html │ │ ├── nlp │ │ │ └── index.html │ │ └── table-data │ │ │ └── index.html │ ├── usecases │ │ ├── _print │ │ │ └── index.html │ │ ├── image │ │ │ └── index.html │ │ ├── index.html │ │ ├── nlp │ │ │ └── index.html │ │ └── table-data │ │ │ └── index.html │ ├── お客様事例集 │ │ ├── _print │ │ │ └── index.html │ │ ├── index.html │ │ ├── picture.png │ │ └── pubref1 │ │ │ └── index.html │ └── 参考になりそうな外部記事 │ │ ├── _print │ │ └── index.html │ │ ├── index.html │ │ └── parameter-reference │ │ └── index.html ├── examples │ ├── _print │ │ └── index.html │ └── index.html ├── favicons │ ├── android-144x144.png │ ├── android-192x192.png │ ├── android-36x36.png │ ├── android-48x48.png │ ├── android-72x72.png │ ├── android-96x96.png │ ├── apple-touch-icon-180x180.png │ ├── favicon-1024.png │ ├── favicon-16x16.png │ ├── favicon-256.png │ ├── favicon-32x32.png │ ├── favicon.ico │ ├── pwa-192x192.png │ ├── pwa-512x512.png │ ├── tile150x150.png │ ├── tile310x150.png │ ├── tile310x310.png │ └── tile70x70.png ├── featured-background.jpg ├── featured-background.png ├── featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_1920x1080_fill_q75_catmullrom_top.jpg ├── featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_960x540_fill_q75_catmullrom_top.jpg ├── featured-background_hu8a39ca4bb86027cf9971f48ddb45157c_1460964_1920x1080_fill_catmullrom_top_2.png ├── featured-background_hu8a39ca4bb86027cf9971f48ddb45157c_1460964_960x540_fill_catmullrom_top_2.png ├── index.html ├── index.xml ├── js │ ├── main.min.2698ba6d60966a95b30f3160f89208b4219c5535421f52ee6147801ad45cc471.js │ ├── prism.js │ ├── swagger-ui-bundle.js │ └── swagger-ui-standalone-preset.js ├── robots.txt ├── scss │ ├── main.min.7a701bfef5f2c2e872654520afd482ce033f034b72dfa66f90c8b5638e0febec.css │ ├── main.min.b31ffd97dd6798655a172c11b526e2c16608629dbeabd6fe8bf1f609994e0818.css │ └── main.min.c63ec56da86340c64de55b0b9c654a2beea1e27d957474a842892958b79a4a75.css ├── search │ └── index.html ├── sitemap.xml └── webfonts │ ├── fa-brands-400.eot │ ├── fa-brands-400.svg │ ├── fa-brands-400.ttf │ ├── fa-brands-400.woff │ ├── fa-brands-400.woff2 │ ├── fa-regular-400.eot │ ├── fa-regular-400.svg │ ├── fa-regular-400.ttf │ ├── fa-regular-400.woff │ ├── fa-regular-400.woff2 │ ├── fa-solid-900.eot │ ├── fa-solid-900.svg │ ├── fa-solid-900.ttf │ ├── fa-solid-900.woff │ └── fa-solid-900.woff2 ├── edge_inference └── greengrass-ml-inference │ ├── .gitignore │ ├── README.md │ ├── build_mnist_classifier.ipynb │ ├── greengrass_ml_inference.ipynb │ ├── image │ ├── AWS-Architecture-Icons-Deck_For-Dark-BG_20200911.pptx │ ├── image01.png │ ├── image02.png │ ├── image03.png │ ├── image04.png │ ├── image05.png │ ├── image06.png │ ├── image07.png │ ├── image08.png │ ├── image09.png │ ├── image10.png │ ├── image11.png │ ├── image12.png │ ├── image13.png │ ├── image14.png │ └── image15.png │ └── src │ ├── classifier_train.py │ └── ggv2 │ └── components │ ├── artifacts │ ├── com.example.IoTPublisher │ │ ├── 1.0.0 │ │ │ ├── Dockerfile │ │ │ └── IoTPublisher.py │ │ └── 1.0.1 │ │ │ ├── Dockerfile │ │ │ ├── IoTPublisher.py │ │ │ └── classifier │ │ │ ├── compiled.meta │ │ │ ├── compiled.params │ │ │ ├── compiled.so │ │ │ ├── compiled_model.json │ │ │ ├── dlr.h │ │ │ ├── libdlr.so │ │ │ └── manifest │ ├── com.example.Publisher │ │ └── 1.0.0 │ │ │ └── publisher.py │ └── com.example.Subscriber │ │ └── 1.0.0 │ │ ├── classifier.h5 │ │ └── subscriber.py │ └── recipes │ ├── com.example.Publisher-1.0.0.yaml │ └── com.example.Subscriber-1.0.0.yaml ├── hpo_pytorch_mnist ├── input.html ├── mnist.py └── pytorch_mnist.ipynb ├── life-science ├── BreastCancerPrediction_R.ipynb └── covid19-built-in-image-classification.ipynb ├── mlops └── sagemaker-pipelines │ └── sagemaker-pipelines-sample │ ├── .gitignore │ ├── README.md │ ├── media │ ├── 1.png │ ├── 2.png │ ├── 3.png │ ├── 4.png │ ├── 5.png │ ├── 6.png │ └── 7.png │ ├── postprocess │ └── postprocess.py │ ├── preprocess │ └── preprocess.py │ └── sagemaker-pipelines-sample.ipynb ├── model-deployment ├── TF2-model-deploy.ipynb ├── paddleocr.ipynb └── pytorch-deeplab.ipynb ├── nlp_amazon_review ├── BlazingText │ └── blazingtext.ipynb └── GluonNLP_BERT │ ├── gluonnlp_bert.ipynb │ └── src │ ├── bert │ ├── data │ │ ├── __init__.py │ │ └── transform.py │ └── model │ │ ├── __init__.py │ │ └── classification.py │ └── train_and_deploy.py ├── package-lock.json ├── package.json ├── sagemaker-experiments └── pytorch_mnist │ ├── figures │ └── experiments-overview.jpg │ ├── pytorch_mnist.ipynb │ └── src │ ├── mnist_deploy.py │ └── mnist_train.py ├── sagemaker_processing ├── Feature_engineering_with_RAPIDS_on_SageMaker_Processing │ ├── Feature_engineering_with_RAPIDS_on_SageMaker_Processing.ipynb │ └── container │ │ └── Dockerfile └── processing_with_bring_your_own_container_for_beginner │ ├── 0_data_preparation.ipynb │ ├── 1_preprocess.ipynb │ ├── container │ ├── Dockerfile │ └── preprocess_code │ │ └── preprocess.py │ ├── media │ ├── 0_data_preparation.png │ ├── 1_preprocess.png │ └── README.md │ └── preprocess_script │ └── preprocess.py ├── step-functions-data-science-sdk ├── model-train-and-evaluate │ └── step_functions_mlworkflow_scikit_learn_data_processing_and_model_evaluation.ipynb └── model-train-evaluate-compare │ ├── step_functions_mlworkflow_scikit_learn_data_processing_and_model_evaluation_with_experiments.ipynb │ └── studio_step_functions_mlworkflow_scikit_learn_data_processing_and_model_evaluation_with_experiments.ipynb ├── tensorflow2_training_and_serving ├── mnist.py └── tensorflow2_training_and_serving.ipynb ├── workshop └── lab_bring-your-own-model │ ├── img │ └── sagemaker-data-model.png │ ├── pytorch │ └── pytorch.ipynb │ └── tensorflow │ ├── cnn_mnist_after.py │ ├── cnn_mnist_before.py │ └── tensorflow.ipynb └── xgboost_customer_churn ├── xgboost_customer_churn.ipynb └── xgboost_customer_churn_neo.ipynb /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | 6 | By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.exe 7 | *.o 8 | *.so 9 | 10 | # Packages # 11 | ############ 12 | # it's better to unpack these files and commit the raw source 13 | # git has its own built in compression methods 14 | *.7z 15 | *.dmg 16 | *.gz 17 | *.iso 18 | *.jar 19 | *.rar 20 | *.tar 21 | *.zip 22 | 23 | # Logs and databases # 24 | ###################### 25 | *.log 26 | *.sql 27 | *.sqlite 28 | 29 | # OS generated files # 30 | ###################### 31 | .DS_Store 32 | .DS_Store? 33 | ._* 34 | .Spotlight-V100 35 | .Trashes 36 | ehthumbs.db 37 | Thumbs.db 38 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check [existing open](https://github.com/aws-samples/amazon-sagemaker-examples-jp/issues), or [recently closed](https://github.com/aws-samples/amazon-sagemaker-examples-jp/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/aws-samples/amazon-sagemaker-examples-jp/labels/help%20wanted) issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](https://github.com/aws-samples/amazon-sagemaker-examples-jp/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | amazon-sagemaker-examples-jp 2 | Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Archived 2 | --- 3 | # Amazon SageMaker Examples JP 4 | **サンプルコードの対象範囲を広げて、[こちらのリポジトリ](https://github.com/aws-samples/aws-ml-jp) に移行しました。** 5 | 6 | [Amazon SageMaker Examples](https://github.com/awslabs/amazon-sagemaker-examples) の日本語訳や、オリジナルのサンプルコードのためのレポジトリです。 7 | AWS 目黒オフィスで SageMaker 体験ハンズオンを定期的に開催しています [[connpass](https://awsj-ml.connpass.com/)]。 8 | 9 | Japanese translation of [Amazon SageMaker Examples](https://github.com/awslabs/amazon-sagemaker-examples) and other original sample codes, which will be used in Amazon SageMaker hands-on workshops in Japan. 10 | 11 | ## Notice 12 | 13 | 現在、SageMaker Python SDK v2 に合わせてサンプルコードの改修を行っています。改修済みのものを master branch で公開しています。改修していないものを legacy branch で公開しています。改修のリクエストなどは [issue](https://github.com/aws-samples/amazon-sagemaker-examples-jp/issues) からご連絡ください。 14 | 15 | ## License 16 | 17 | This library is licensed under the Apache 2.0 License. 18 | -------------------------------------------------------------------------------- /amazon_forecast_tutorial/common/images/amazon_forecast.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/amazon_forecast_tutorial/common/images/amazon_forecast.png -------------------------------------------------------------------------------- /amazon_forecast_tutorial/common/images/forecast_workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/amazon_forecast_tutorial/common/images/forecast_workflow.png -------------------------------------------------------------------------------- /amazon_forecast_tutorial/common/util/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from .fcst_utils import * 3 | from .notebook_utils import * 4 | 5 | -------------------------------------------------------------------------------- /amazon_forecast_tutorial/common/util/fcst_utils.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | import gzip 4 | 5 | import boto3 6 | import botocore.exceptions 7 | 8 | import pandas as pd 9 | import matplotlib.pyplot as plt 10 | 11 | import util.notebook_utils 12 | 13 | 14 | def wait_till_delete(callback, check_time = 5, timeout = None): 15 | 16 | elapsed_time = 0 17 | while timeout is None or elapsed_time < timeout: 18 | try: 19 | out = callback() 20 | except botocore.exceptions.ClientError as e: 21 | # When given the resource not found exception, deletion has occured 22 | if e.response['Error']['Code'] == 'ResourceNotFoundException': 23 | print('Successful delete') 24 | return 25 | else: 26 | raise 27 | time.sleep(check_time) # units of seconds 28 | elapsed_time += check_time 29 | 30 | raise TimeoutError( "Forecast resource deletion timed-out." ) 31 | 32 | 33 | def wait(callback, time_interval = 10): 34 | 35 | status_indicator = util.notebook_utils.StatusIndicator() 36 | 37 | while True: 38 | status = callback()['Status'] 39 | status_indicator.update(status) 40 | if status in ('ACTIVE', 'CREATE_FAILED'): break 41 | time.sleep(time_interval) 42 | 43 | status_indicator.end() 44 | 45 | return (status=="ACTIVE") 46 | 47 | 48 | def load_exact_sol(fname, item_id, is_schema_perm=False): 49 | exact = pd.read_csv(fname, header = None) 50 | exact.columns = ['item_id', 'timestamp', 'target'] 51 | if is_schema_perm: 52 | exact.columns = ['timestamp', 'target', 'item_id'] 53 | return exact.loc[exact['item_id'] == item_id] 54 | 55 | 56 | def get_or_create_iam_role( role_name ): 57 | 58 | iam = boto3.client("iam") 59 | 60 | assume_role_policy_document = { 61 | "Version": "2012-10-17", 62 | "Statement": [ 63 | { 64 | "Effect": "Allow", 65 | "Principal": { 66 | "Service": "forecast.amazonaws.com" 67 | }, 68 | "Action": "sts:AssumeRole" 69 | } 70 | ] 71 | } 72 | 73 | try: 74 | create_role_response = iam.create_role( 75 | RoleName = role_name, 76 | AssumeRolePolicyDocument = json.dumps(assume_role_policy_document) 77 | ) 78 | role_arn = create_role_response["Role"]["Arn"] 79 | print("Created", role_arn) 80 | except iam.exceptions.EntityAlreadyExistsException: 81 | print("The role " + role_name + " exists, ignore to create it") 82 | role_arn = boto3.resource('iam').Role(role_name).arn 83 | 84 | print("Attaching policies") 85 | 86 | iam.attach_role_policy( 87 | RoleName = role_name, 88 | PolicyArn = "arn:aws:iam::aws:policy/AmazonForecastFullAccess" 89 | ) 90 | 91 | iam.attach_role_policy( 92 | RoleName=role_name, 93 | PolicyArn='arn:aws:iam::aws:policy/AmazonS3FullAccess', 94 | ) 95 | 96 | print("Waiting for a minute to allow IAM role policy attachment to propagate") 97 | time.sleep(60) 98 | 99 | print("Done.") 100 | return role_arn 101 | 102 | 103 | def delete_iam_role( role_name ): 104 | iam = boto3.client("iam") 105 | iam.detach_role_policy( PolicyArn = "arn:aws:iam::aws:policy/AmazonS3FullAccess", RoleName = role_name ) 106 | iam.detach_role_policy( PolicyArn = "arn:aws:iam::aws:policy/AmazonForecastFullAccess", RoleName = role_name ) 107 | iam.delete_role(RoleName=role_name) 108 | 109 | 110 | def plot_forecasts(fcsts, exact, freq = '1H', forecastHorizon=24, time_back = 80): 111 | p10 = pd.DataFrame(fcsts['Forecast']['Predictions']['p10']) 112 | p50 = pd.DataFrame(fcsts['Forecast']['Predictions']['p50']) 113 | p90 = pd.DataFrame(fcsts['Forecast']['Predictions']['p90']) 114 | pred_int = p50['Timestamp'].apply(lambda x: pd.Timestamp(x)) 115 | fcst_start_date = pred_int.iloc[0] 116 | fcst_end_date = pred_int.iloc[-1] 117 | time_int = exact['timestamp'].apply(lambda x: pd.Timestamp(x)) 118 | plt.plot(time_int[-time_back:],exact['target'].values[-time_back:], color = 'r') 119 | plt.plot(pred_int, p50['Value'].values, color = 'k') 120 | plt.fill_between(p50['Timestamp'].values, 121 | p10['Value'].values, 122 | p90['Value'].values, 123 | color='b', alpha=0.3); 124 | plt.axvline(x=pd.Timestamp(fcst_start_date), linewidth=3, color='g', ls='dashed') 125 | plt.axvline(x=pd.Timestamp(fcst_end_date), linewidth=3, color='g', ls='dashed') 126 | plt.xticks(rotation=30) 127 | plt.legend(['Target', 'Forecast'], loc = 'lower left') 128 | 129 | 130 | def extract_gz( src, dst ): 131 | 132 | print( f"Extracting {src} to {dst}" ) 133 | 134 | with open(dst, 'wb') as fd_dst: 135 | with gzip.GzipFile( src, 'rb') as fd_src: 136 | data = fd_src.read() 137 | fd_dst.write(data) 138 | 139 | print("Done.") 140 | 141 | -------------------------------------------------------------------------------- /amazon_forecast_tutorial/common/util/notebook_utils.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import io 3 | import ipywidgets 4 | 5 | 6 | widget_table = {} 7 | 8 | def create_text_widget( name, placeholder, default_value="" ): 9 | 10 | if name in widget_table: 11 | widget = widget_table[name] 12 | if name not in widget_table: 13 | widget = ipywidgets.Text( description = name, placeholder = placeholder, value=default_value ) 14 | widget_table[name] = widget 15 | display(widget) 16 | 17 | return widget 18 | 19 | 20 | class StatusIndicator: 21 | 22 | def __init__(self): 23 | self.previous_status = None 24 | self.need_newline = False 25 | 26 | def update( self, status ): 27 | if self.previous_status != status: 28 | if self.need_newline: 29 | sys.stdout.write("\n") 30 | sys.stdout.write( status + " ") 31 | self.need_newline = True 32 | self.previous_status = status 33 | else: 34 | sys.stdout.write(".") 35 | self.need_newline = True 36 | sys.stdout.flush() 37 | 38 | def end(self): 39 | if self.need_newline: 40 | sys.stdout.write("\n") 41 | 42 | -------------------------------------------------------------------------------- /autogluon/tabular-prediction/AutoGluon-Tabular-with-SageMaker/README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | 5 | # AutoGluon Tabular with Amazon SageMaker 6 | 7 | [AutoGluon](https://github.com/awslabs/autogluon) automates machine learning tasks enabling you to easily achieve strong predictive performance in your applications. With just a few lines of code, you can train and deploy high-accuracy deep learning models on tabular, image, and text data. 8 | This example shows how to use AutoGluon-Tabular with Amazon SageMaker by creating custom containers. -------------------------------------------------------------------------------- /autogluon/tabular-prediction/AutoGluon-Tabular-with-SageMaker/container-inference/Dockerfile.inference: -------------------------------------------------------------------------------- 1 | ARG REGISTRY_URI 2 | FROM ${REGISTRY_URI} 3 | 4 | RUN pip install autogluon 5 | RUN pip install PrettyTable 6 | 7 | # Defines inference.py as script entrypoint 8 | ENV SAGEMAKER_PROGRAM inference.py 9 | -------------------------------------------------------------------------------- /autogluon/tabular-prediction/AutoGluon-Tabular-with-SageMaker/container-inference/build_push_inference.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Get account 4 | account=${1} 5 | 6 | # Get the region defined in the current configuration 7 | region=${2:-us-east-1} 8 | 9 | # The name of our algorithm 10 | algorithm_name=${3:-sagemaker-autogluon-inference} 11 | 12 | uri_prefix=${4} 13 | fullname="${uri_prefix}/${algorithm_name}:latest" 14 | 15 | # Get the registry id 16 | registry_id=${5} 17 | registry_uri=${6} 18 | 19 | # If the repository doesn't exist in ECR, create it. 20 | aws ecr describe-repositories --region ${region} --repository-names "${algorithm_name}" > /dev/null 2>&1 21 | if [ $? -ne 0 ] 22 | then 23 | aws ecr create-repository --region ${region} --repository-name "${algorithm_name}" > /dev/null 24 | fi 25 | 26 | # Get the login command from ECR and execute it directly 27 | $(aws ecr get-login --region ${region} --no-include-email) 28 | $(aws ecr get-login --registry-ids ${registry_id} --region ${region} --no-include-email) 29 | 30 | # Build the docker image, tag with full name and then push it to ECR 31 | docker build -t ${algorithm_name} -f container-inference/Dockerfile.inference . --build-arg REGISTRY_URI=${registry_uri} 32 | docker tag ${algorithm_name} ${fullname} 33 | docker push ${fullname} 34 | -------------------------------------------------------------------------------- /autogluon/tabular-prediction/AutoGluon-Tabular-with-SageMaker/container-training/Dockerfile.training: -------------------------------------------------------------------------------- 1 | ARG REGISTRY_URI 2 | FROM ${REGISTRY_URI} 3 | 4 | RUN pip install autogluon 5 | RUN pip install PrettyTable 6 | RUN pip install bokeh 7 | 8 | RUN apt-get update \ 9 | && apt-get install -y --no-install-recommends graphviz libgraphviz-dev pkg-config \ 10 | && rm -rf /var/lib/apt/lists/* \ 11 | && pip install pygraphviz 12 | 13 | ENV PATH="/opt/ml/code:${PATH}" 14 | 15 | # Copies the training code inside the container 16 | COPY container-training/train.py /opt/ml/code/train.py 17 | COPY container-training/inference.py /opt/ml/code/inference.py 18 | 19 | # Install seaborn for plot 20 | RUN pip install seaborn 21 | 22 | # this environment variable is used by the SageMaker PyTorch container to determine our user code directory. 23 | ENV SAGEMAKER_SUBMIT_DIRECTORY /opt/ml/code 24 | 25 | # Defines train.py as script entrypoint 26 | ENV SAGEMAKER_PROGRAM train.py 27 | -------------------------------------------------------------------------------- /autogluon/tabular-prediction/AutoGluon-Tabular-with-SageMaker/container-training/build_push_training.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Get account 4 | account=${1} 5 | 6 | # Get the region defined in the current configuration 7 | region=${2:-us-east-1} 8 | 9 | # The name of our algorithm 10 | algorithm_name=${3:-sagemaker-autogluon-training} 11 | 12 | uri_prefix=${4} 13 | fullname="${uri_prefix}/${algorithm_name}:latest" 14 | 15 | # Get the registry id 16 | registry_id=${5} 17 | registry_uri=${6} 18 | 19 | # If the repository doesn't exist in ECR, create it. 20 | aws ecr describe-repositories --region ${region} --repository-names "${algorithm_name}" > /dev/null 2>&1 21 | if [ $? -ne 0 ] 22 | then 23 | aws ecr create-repository --region ${region} --repository-name "${algorithm_name}" > /dev/null 24 | fi 25 | 26 | # Get the login command from ECR and execute it directly 27 | $(aws ecr get-login --region ${region} --no-include-email) 28 | $(aws ecr get-login --registry-ids ${registry_id} --region ${region} --no-include-email) 29 | 30 | # Build the docker image, tag with full name and then push it to ECR 31 | docker build -t ${algorithm_name} -f container-training/Dockerfile.training . --build-arg REGISTRY_URI=${registry_uri} 32 | docker tag ${algorithm_name} ${fullname} 33 | docker push ${fullname} 34 | -------------------------------------------------------------------------------- /autogluon/tabular-prediction/AutoGluon-Tabular-with-SageMaker/container-training/inference.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import argparse 4 | import logging 5 | import warnings 6 | import time 7 | import json 8 | import subprocess 9 | import copy 10 | 11 | warnings.filterwarnings('ignore', category=FutureWarning) 12 | 13 | import numpy as np 14 | import pandas as pd 15 | import pickle 16 | from io import StringIO 17 | from timeit import default_timer as timer 18 | from itertools import islice 19 | from collections import Counter 20 | 21 | with warnings.catch_warnings(): 22 | warnings.filterwarnings('ignore', category=DeprecationWarning) 23 | from prettytable import PrettyTable 24 | from autogluon import TabularPrediction as task 25 | 26 | def make_str_table(df): 27 | table = PrettyTable(['index']+list(df.columns)) 28 | for row in df.itertuples(): 29 | table.add_row(row) 30 | return str(table) 31 | 32 | def take(n, iterable): 33 | "Return first n items of the iterable as a list" 34 | return list(islice(iterable, n)) 35 | 36 | def preprocess(df, columns, target): 37 | features = copy.deepcopy(columns) 38 | features.remove(target) 39 | first_row_list = df.iloc[0].tolist() 40 | 41 | if set(first_row_list) >= set(features): 42 | df.drop(0, inplace=True) 43 | if len(first_row_list) == len(columns): 44 | df.columns = columns 45 | if len(first_row_list) == len(features): 46 | df.columns = features 47 | 48 | return df 49 | 50 | # ------------------------------------------------------------ # 51 | # Hosting methods # 52 | # ------------------------------------------------------------ # 53 | 54 | def model_fn(model_dir): 55 | """ 56 | Load the gluon model. Called once when hosting service starts. 57 | :param: model_dir The directory where model files are stored. 58 | :return: a model (in this case a Gluon network) and the column info. 59 | """ 60 | print(f'Loading model from {model_dir} with contents {os.listdir(model_dir)}') 61 | net = task.load(model_dir, verbosity=True) 62 | with open(f'{model_dir}/code/columns.pkl', 'rb') as f: 63 | column_dict = pickle.load(f) 64 | return net, column_dict 65 | 66 | 67 | def transform_fn(models, data, input_content_type, output_content_type): 68 | """ 69 | Transform a request using the Gluon model. Called once per request. 70 | :param models: The Gluon model and the column info. 71 | :param data: The request payload. 72 | :param input_content_type: The request content type. ('text/csv') 73 | :param output_content_type: The (desired) response content type. ('text/csv') 74 | :return: response payload and content type. 75 | """ 76 | start = timer() 77 | net = models[0] 78 | column_dict = models[1] 79 | 80 | # text/csv 81 | if input_content_type == 'text/csv': 82 | 83 | # Load dataset 84 | columns = column_dict['columns'] 85 | df = pd.read_csv(StringIO(data), header=None) 86 | df_preprosessed = preprocess(df, columns, net.label_column) 87 | ds = task.Dataset(df=df_preprosessed) 88 | 89 | try: 90 | predictions = net.predict(ds) 91 | except: 92 | try: 93 | predictions = net.predict(ds.fillna(0.0)) 94 | warnings.warn('Filled NaN\'s with 0.0 in order to predict.') 95 | except Exception as e: 96 | response_body = e 97 | return response_body, output_content_type 98 | 99 | # Print prediction counts, limit in case of regression problem 100 | pred_counts = Counter(predictions.tolist()) 101 | n_display_items = 30 102 | if len(pred_counts) > n_display_items: 103 | print(f'Top {n_display_items} prediction counts: ' 104 | f'{dict(take(n_display_items, pred_counts.items()))}') 105 | else: 106 | print(f'Prediction counts: {pred_counts}') 107 | 108 | # Form response 109 | output = StringIO() 110 | pd.DataFrame(predictions).to_csv(output, header=False, index=False) 111 | response_body = output.getvalue() 112 | 113 | # If target column passed, evaluate predictions performance 114 | target = net.label_column 115 | if target in ds: 116 | print(f'Label column ({target}) found in input data. ' 117 | 'Therefore, evaluating prediction performance...') 118 | try: 119 | performance = net.evaluate_predictions(y_true=ds[target], 120 | y_pred=predictions, 121 | auxiliary_metrics=True) 122 | print(json.dumps(performance, indent=4, default=pd.DataFrame.to_json)) 123 | time.sleep(0.1) 124 | except Exception as e: 125 | # Print exceptions on evaluate, continue to return predictions 126 | print(f'Exception: {e}') 127 | else: 128 | raise NotImplementedError("content_type must be 'text/csv'") 129 | 130 | elapsed_time = round(timer()-start,3) 131 | print(f'Elapsed time: {round(timer()-start,3)} seconds') 132 | 133 | return response_body, output_content_type 134 | -------------------------------------------------------------------------------- /autogluon/tabular-prediction/AutoGluon-Tabular-with-SageMaker/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo -n true 4 | if [ $? -eq 0 ]; then 5 | echo "The user has root access." 6 | else 7 | echo "The user does not have root access. Everything required to run the notebook is already installed and setup. We are good to go!" 8 | exit 0 9 | fi 10 | 11 | # Do we have GPU support? 12 | nvidia-smi > /dev/null 2>&1 13 | if [ $? -eq 0 ]; then 14 | # check if we have nvidia-docker 15 | NVIDIA_DOCKER=`rpm -qa | grep -c nvidia-docker2` 16 | if [ $NVIDIA_DOCKER -eq 0 ]; then 17 | # Install nvidia-docker2 18 | #sudo pkill -SIGHUP dockerd 19 | sudo yum -y remove docker 20 | sudo yum -y install docker-17.09.1ce-1.111.amzn1 21 | 22 | sudo /etc/init.d/docker start 23 | 24 | curl -s -L https://nvidia.github.io/nvidia-docker/amzn1/nvidia-docker.repo | sudo tee /etc/yum.repos.d/nvidia-docker.repo 25 | sudo yum install -y nvidia-docker2-2.0.3-1.docker17.09.1.ce.amzn1 26 | sudo cp daemon.json /etc/docker/daemon.json 27 | sudo pkill -SIGHUP dockerd 28 | echo "installed nvidia-docker2" 29 | else 30 | echo "nvidia-docker2 already installed. We are good to go!" 31 | fi 32 | fi 33 | 34 | # This is common for both GPU and CPU instances 35 | 36 | # check if we have docker-compose 37 | docker-compose version >/dev/null 2>&1 38 | if [ $? -ne 0 ]; then 39 | # install docker compose 40 | pip install docker-compose 41 | fi 42 | 43 | # check if we need to configure our docker interface 44 | SAGEMAKER_NETWORK=`docker network ls | grep -c sagemaker-local` 45 | if [ $SAGEMAKER_NETWORK -eq 0 ]; then 46 | docker network create --driver bridge sagemaker-local 47 | fi 48 | 49 | # Notebook instance Docker networking fixes 50 | RUNNING_ON_NOTEBOOK_INSTANCE=`sudo iptables -S OUTPUT -t nat | grep -c 169.254.0.2` 51 | 52 | # Get the Docker Network CIDR and IP for the sagemaker-local docker interface. 53 | SAGEMAKER_INTERFACE=br-`docker network ls | grep sagemaker-local | cut -d' ' -f1` 54 | DOCKER_NET=`ip route | grep $SAGEMAKER_INTERFACE | cut -d" " -f1` 55 | DOCKER_IP=`ip route | grep $SAGEMAKER_INTERFACE | cut -d" " -f12` 56 | 57 | # check if both IPTables and the Route Table are OK. 58 | IPTABLES_PATCHED=`sudo iptables -S PREROUTING -t nat | grep -c 169.254.0.2` 59 | ROUTE_TABLE_PATCHED=`sudo ip route show table agent | grep -c $SAGEMAKER_INTERFACE` 60 | 61 | if [ $RUNNING_ON_NOTEBOOK_INSTANCE -gt 0 ]; then 62 | 63 | if [ $ROUTE_TABLE_PATCHED -eq 0 ]; then 64 | # fix routing 65 | sudo ip route add $DOCKER_NET via $DOCKER_IP dev $SAGEMAKER_INTERFACE table agent 66 | else 67 | echo "SageMaker instance route table setup is ok. We are good to go." 68 | fi 69 | 70 | if [ $IPTABLES_PATCHED -eq 0 ]; then 71 | sudo iptables -t nat -A PREROUTING -i $SAGEMAKER_INTERFACE -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 169.254.0.2:9081 72 | echo "iptables for Docker setup done" 73 | else 74 | echo "SageMaker instance routing for Docker is ok. We are good to go!" 75 | fi 76 | fi 77 | -------------------------------------------------------------------------------- /distributed_training/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG region 2 | 3 | FROM 763104351884.dkr.ecr.${region}.amazonaws.com/pytorch-training:1.8.1-gpu-py36-cu111-ubuntu18.04 4 | 5 | RUN pip install ninja yacs cython matplotlib tqdm opencv-python pybind11==2.5.0 'git+https://github.com/NVIDIA/dllogger' 6 | 7 | RUN pip --no-cache-dir --no-cache install 'git+https://github.com/NVIDIA/cocoapi#egg=pycocotools&subdirectory=PythonAPI' 8 | 9 | RUN cd /root && \ 10 | rm -rf apex && \ 11 | git clone https://github.com/NVIDIA/apex && \ 12 | cd apex && \ 13 | python setup.py install --cuda_ext --cpp_ext 14 | 15 | RUN cd /root && \ 16 | rm -rf DeepLearningExamples && \ 17 | git clone --recursive https://github.com/HerringForks/DeepLearningExamples.git && \ 18 | cd DeepLearningExamples/PyTorch/Segmentation/MaskRCNN/pytorch/ && \ 19 | python setup.py build develop 20 | -------------------------------------------------------------------------------- /distributed_training/README.md: -------------------------------------------------------------------------------- 1 | # Amazon SageMakerで大型学習ジョブのコスト最適化 2 | 3 | 4 | データ量が増えてきて、深層学習モデルの学習に時間がかかりすぎていませんか？ 5 | 6 | このノートブックでは、深層学習モデルの学習にかかる時間とコストを最適化する手順を、SageMaker Data Parallel LibraryとSageMaker Debuggerを使いながらご紹介します。 7 | 8 | SageMaker Data Parallelism Libraryは、AWS上での分散学習実行に最適化されているため、HorovodやPyTorch DDPなどの他の分散学習フレームワークに比べて並列化効率がより高いことが実証されています。詳細は、[論文](https://www.amazon.science/publications/herring-rethinking-the-parameter-server-at-scale-for-the-cloud)を参照してください。 9 | 10 | 今回は、例としてMask RCNNモデルに[COCO2017データセット](https://cocodataset.org/#home)を分散学習していく過程を紹介します。 11 | 12 | なお、このノートブックはSageMaker Studio上のPython 3 (PyTorch 1.6 Python 3.6 GPU Optimized)環境で動作確認をしております。 13 | 14 | 他のフレームワークやモデルでの分散学習のスクリプトは[こちら](https://github.com/HerringForks/DeepLearningExamples)を参照してください。 15 | 16 | 17 | ## 学習時間と課金額のトレードオフでトータルコスト最適化を考える 18 | 19 | SageMaker上で分散学習を実施することで、学習時間と課金額のトレードオフがコントロールできるようになります。具体的には、約11%の金額増加で約86%も学習時間を短縮できる、といった例が挙げられます。こちらのトレードオフを詳しくみてみましょう。 20 | 21 | インスタンスの使用料金をPドル/時、学習にかかる時間をT時間、並列化効率(インスタンス x個で学習時間がx倍にどれくらい近くなるか)をS%として、１個のインスタンス（またはGPU）からN個のインスタンス（またはGPU）に分散学習する場合を考えてみましょう。 22 | 23 | 学習にかかる時間はT時間から(T/N)/S時間に減り、インスタンス使用料金はT * Pドルから(T/N)/S * P * N = T/S * Pドルへ（S = 1、すなわち並列化効率100%とならない限りは）増加します。並列化効率が十分高ければ、大幅な学習時間短縮を少ない金額の増加で買うことができる、というわけです。コストを時間と課金額の組み合わせと捉えると、分散学習が１つのコスト最適化のツールとなり、かつ並列化効率がそのトレードオフを決定する重要な要因であることがご理解いただけると思います。 24 | また、学習時間に制限があり、データ量が増える中で時間内に学習を終える必要があるかもしれません。その際にも、並列化効率が鍵となり、どのくらいの課金額の増加でどのくらいの学習時間短縮が狙えるか調査することが大切になってきます。 25 | 26 | 例えば、ml.p3.2xlargeインスタンスで24時間学習に時間を取られている場合、8つのインスタンスに90%の並列化効率で分散学習が実現できれば、時間と課金額は以下のように変化します。 27 | 28 | 分散学習なしでは24 * 3.825 = 91.8ドル（us-west-2リージョンで[SageMaker Savings Plan](https://aws.amazon.com/about-aws/whats-new/2021/04/amazon-sagemaker-announces-a-price-reduction-in-instances-and-sagemaker-savings-plan/)を使用しない場合）の課金と24時間というコストが発生します。 29 | 30 | 上記の条件で分散学習をすると、24/0.9 * 3.825 = 102ドルの課金と24/8/0.9 = 3.33時間というコストになります。 31 | 32 | この場合、最初に記載したように約11%の金額増加で約86%の学習時間短縮が期待されます。どちらのトータルコストが良いかどうかは、ビジネス上で深層学習のモデル学習時間短縮がどれくらい重要かによります。しかし、モデルを何度も再学習する必要に迫られる中で、これからデータ量が増えていくことが予測されるならば、分散学習を１つの選択肢として持っておくのは悪くないかもしれません。 -------------------------------------------------------------------------------- /distributed_training/build_and_push.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # This script shows how to build the Docker image and push it to ECR to be ready for use 3 | # by SageMaker. 4 | # The argument to this script is the image name. This will be used as the image on the local 5 | # machine and combined with the account and region to form the repository name for ECR. 6 | # set region 7 | 8 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 9 | 10 | if [ "$#" -eq 3 ]; then 11 | region=$1 12 | image=$2 13 | tag=$3 14 | else 15 | echo "usage: $0 $1 $2 " 16 | exit 1 17 | fi 18 | 19 | # Get the account number associated with the current IAM credentials 20 | account=$(aws sts get-caller-identity --query Account --output text) 21 | 22 | if [ $? -ne 0 ] 23 | then 24 | exit 255 25 | fi 26 | 27 | 28 | fullname="${account}.dkr.ecr.${region}.amazonaws.com/${image}:${tag}" 29 | 30 | # If the repository doesn't exist in ECR, create it. 31 | aws ecr describe-repositories --region ${region} --repository-names "${image}" > /dev/null 2>&1 32 | if [ $? -ne 0 ]; then 33 | echo "creating ECR repository : ${fullname} " 34 | aws ecr create-repository --region ${region} --repository-name "${image}" > /dev/null 35 | fi 36 | 37 | $(aws ecr get-login --no-include-email --region ${region} --registry-ids 763104351884) 38 | docker build ${DIR}/ -t ${image} -f ${DIR}/Dockerfile --build-arg region=${region} 39 | docker tag ${image} ${fullname} 40 | 41 | # Get the login command from ECR and execute it directly 42 | $(aws ecr get-login --region ${region} --no-include-email) 43 | docker push ${fullname} 44 | if [ $? -eq 0 ]; then 45 | echo "Amazon ECR URI: ${fullname}" 46 | else 47 | echo "Error: Image build and push failed" 48 | exit 1 49 | fi -------------------------------------------------------------------------------- /distributed_training/e2e_mask_rcnn_R_50_FPN_1x_16GPU_4bs.yaml: -------------------------------------------------------------------------------- 1 | MODEL: 2 | META_ARCHITECTURE: "GeneralizedRCNN" 3 | WEIGHT: "catalog://ImageNetPretrained/MSRA/R-50" 4 | BACKBONE: 5 | CONV_BODY: "R-50-FPN" 6 | OUT_CHANNELS: 256 7 | RPN: 8 | USE_FPN: True 9 | ANCHOR_STRIDE: (4, 8, 16, 32, 64) 10 | PRE_NMS_TOP_N_TRAIN: 2000 11 | PRE_NMS_TOP_N_TEST: 1000 12 | POST_NMS_TOP_N_TEST: 1000 13 | FPN_POST_NMS_TOP_N_TEST: 1000 14 | ROI_HEADS: 15 | USE_FPN: True 16 | ROI_BOX_HEAD: 17 | POOLER_RESOLUTION: 7 18 | POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) 19 | POOLER_SAMPLING_RATIO: 2 20 | FEATURE_EXTRACTOR: "FPN2MLPFeatureExtractor" 21 | PREDICTOR: "FPNPredictor" 22 | ROI_MASK_HEAD: 23 | POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) 24 | FEATURE_EXTRACTOR: "MaskRCNNFPNFeatureExtractor" 25 | PREDICTOR: "MaskRCNNC4Predictor" 26 | POOLER_RESOLUTION: 14 27 | POOLER_SAMPLING_RATIO: 2 28 | RESOLUTION: 28 29 | SHARE_BOX_FEATURE_EXTRACTOR: False 30 | MASK_ON: True 31 | DATASETS: 32 | TRAIN: ("coco_2017_train",) 33 | TEST: ("coco_2017_val",) 34 | DATALOADER: 35 | SIZE_DIVISIBILITY: 32 36 | SOLVER: 37 | BASE_LR: 0.01 38 | WEIGHT_DECAY: 0.0001 39 | STEPS: (4320, 5760) 40 | MAX_ITER: 1000 41 | IMS_PER_BATCH: 64 42 | WARMUP_FACTOR: 0.001 43 | WARMUP_ITERS: 1000 44 | TEST: 45 | IMS_PER_BATCH: 64 46 | 47 | -------------------------------------------------------------------------------- /distributed_training/e2e_mask_rcnn_R_50_FPN_1x_32GPU_4bs.yaml: -------------------------------------------------------------------------------- 1 | MODEL: 2 | META_ARCHITECTURE: "GeneralizedRCNN" 3 | WEIGHT: "catalog://ImageNetPretrained/MSRA/R-50" 4 | BACKBONE: 5 | CONV_BODY: "R-50-FPN" 6 | OUT_CHANNELS: 256 7 | RPN: 8 | USE_FPN: True 9 | ANCHOR_STRIDE: (4, 8, 16, 32, 64) 10 | PRE_NMS_TOP_N_TRAIN: 2000 11 | PRE_NMS_TOP_N_TEST: 1000 12 | POST_NMS_TOP_N_TEST: 1000 13 | FPN_POST_NMS_TOP_N_TEST: 1000 14 | ROI_HEADS: 15 | USE_FPN: True 16 | ROI_BOX_HEAD: 17 | POOLER_RESOLUTION: 7 18 | POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) 19 | POOLER_SAMPLING_RATIO: 2 20 | FEATURE_EXTRACTOR: "FPN2MLPFeatureExtractor" 21 | PREDICTOR: "FPNPredictor" 22 | ROI_MASK_HEAD: 23 | POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) 24 | FEATURE_EXTRACTOR: "MaskRCNNFPNFeatureExtractor" 25 | PREDICTOR: "MaskRCNNC4Predictor" 26 | POOLER_RESOLUTION: 14 27 | POOLER_SAMPLING_RATIO: 2 28 | RESOLUTION: 28 29 | SHARE_BOX_FEATURE_EXTRACTOR: False 30 | MASK_ON: True 31 | DATASETS: 32 | TRAIN: ("coco_2017_train",) 33 | TEST: ("coco_2017_val",) 34 | DATALOADER: 35 | SIZE_DIVISIBILITY: 32 36 | SOLVER: 37 | BASE_LR: 0.02 38 | WEIGHT_DECAY: 0.0001 39 | STEPS: (4320, 5760) 40 | MAX_ITER: 1000 41 | IMS_PER_BATCH: 128 42 | WARMUP_FACTOR: 0.001 43 | WARMUP_ITERS: 1000 44 | TEST: 45 | IMS_PER_BATCH: 64 46 | 47 | -------------------------------------------------------------------------------- /distributed_training/e2e_mask_rcnn_R_50_FPN_1x_64GPU_4bs.yaml: -------------------------------------------------------------------------------- 1 | MODEL: 2 | META_ARCHITECTURE: "GeneralizedRCNN" 3 | WEIGHT: "catalog://ImageNetPretrained/MSRA/R-50" 4 | BACKBONE: 5 | CONV_BODY: "R-50-FPN" 6 | OUT_CHANNELS: 256 7 | RPN: 8 | USE_FPN: True 9 | ANCHOR_STRIDE: (4, 8, 16, 32, 64) 10 | PRE_NMS_TOP_N_TRAIN: 2000 11 | PRE_NMS_TOP_N_TEST: 1000 12 | POST_NMS_TOP_N_TEST: 1000 13 | FPN_POST_NMS_TOP_N_TEST: 1000 14 | ROI_HEADS: 15 | USE_FPN: True 16 | ROI_BOX_HEAD: 17 | POOLER_RESOLUTION: 7 18 | POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) 19 | POOLER_SAMPLING_RATIO: 2 20 | FEATURE_EXTRACTOR: "FPN2MLPFeatureExtractor" 21 | PREDICTOR: "FPNPredictor" 22 | ROI_MASK_HEAD: 23 | POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) 24 | FEATURE_EXTRACTOR: "MaskRCNNFPNFeatureExtractor" 25 | PREDICTOR: "MaskRCNNC4Predictor" 26 | POOLER_RESOLUTION: 14 27 | POOLER_SAMPLING_RATIO: 2 28 | RESOLUTION: 28 29 | SHARE_BOX_FEATURE_EXTRACTOR: False 30 | MASK_ON: True 31 | DATASETS: 32 | TRAIN: ("coco_2017_train",) 33 | TEST: ("coco_2017_val",) 34 | DATALOADER: 35 | SIZE_DIVISIBILITY: 32 36 | SOLVER: 37 | BASE_LR: 0.04 38 | WEIGHT_DECAY: 0.0001 39 | STEPS: (4320, 5760) 40 | MAX_ITER: 1500 41 | IMS_PER_BATCH: 256 42 | WARMUP_FACTOR: 0.001 43 | WARMUP_ITERS: 1000 44 | TEST: 45 | IMS_PER_BATCH: 64 46 | 47 | -------------------------------------------------------------------------------- /distributed_training/e2e_mask_rcnn_R_50_FPN_1x_8GPU_4bs.yaml: -------------------------------------------------------------------------------- 1 | MODEL: 2 | META_ARCHITECTURE: "GeneralizedRCNN" 3 | WEIGHT: "catalog://ImageNetPretrained/MSRA/R-50" 4 | BACKBONE: 5 | CONV_BODY: "R-50-FPN" 6 | OUT_CHANNELS: 256 7 | RPN: 8 | USE_FPN: True 9 | ANCHOR_STRIDE: (4, 8, 16, 32, 64) 10 | PRE_NMS_TOP_N_TRAIN: 2000 11 | PRE_NMS_TOP_N_TEST: 1000 12 | POST_NMS_TOP_N_TEST: 1000 13 | FPN_POST_NMS_TOP_N_TEST: 1000 14 | ROI_HEADS: 15 | USE_FPN: True 16 | ROI_BOX_HEAD: 17 | POOLER_RESOLUTION: 7 18 | POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) 19 | POOLER_SAMPLING_RATIO: 2 20 | FEATURE_EXTRACTOR: "FPN2MLPFeatureExtractor" 21 | PREDICTOR: "FPNPredictor" 22 | ROI_MASK_HEAD: 23 | POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) 24 | FEATURE_EXTRACTOR: "MaskRCNNFPNFeatureExtractor" 25 | PREDICTOR: "MaskRCNNC4Predictor" 26 | POOLER_RESOLUTION: 14 27 | POOLER_SAMPLING_RATIO: 2 28 | RESOLUTION: 28 29 | SHARE_BOX_FEATURE_EXTRACTOR: False 30 | MASK_ON: True 31 | DATASETS: 32 | TRAIN: ("coco_2017_train",) 33 | TEST: ("coco_2017_val",) 34 | DATALOADER: 35 | SIZE_DIVISIBILITY: 32 36 | SOLVER: 37 | BASE_LR: 0.005 38 | WEIGHT_DECAY: 0.0001 39 | STEPS: (4320, 5760) 40 | MAX_ITER: 1000 41 | IMS_PER_BATCH: 32 42 | WARMUP_FACTOR: 0.001 43 | WARMUP_ITERS: 1000 44 | TEST: 45 | IMS_PER_BATCH: 64 46 | 47 | -------------------------------------------------------------------------------- /distributed_training/images/.ipynb_checkpoints/sm_debugger_insights-checkpoint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/distributed_training/images/.ipynb_checkpoints/sm_debugger_insights-checkpoint.png -------------------------------------------------------------------------------- /distributed_training/images/sm_debugger_insights.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/distributed_training/images/sm_debugger_insights.png -------------------------------------------------------------------------------- /distributed_training/images/sm_debugger_nav.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/distributed_training/images/sm_debugger_nav.png -------------------------------------------------------------------------------- /distributed_training/images/sm_debugger_sys_util.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/distributed_training/images/sm_debugger_sys_util.png -------------------------------------------------------------------------------- /distributed_training/images/smdataparallel_horovod.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/distributed_training/images/smdataparallel_horovod.png -------------------------------------------------------------------------------- /distributed_training/images/smdataparallel_pytorchddp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/distributed_training/images/smdataparallel_pytorchddp.png -------------------------------------------------------------------------------- /distributed_training/upload_coco2017_to_s3.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # credit of https://github.com/awslabs/amazon-sagemaker-examples/blob/master/advanced_functionality/distributed_tensorflow_mask_rcnn/prepare-s3-bucket.sh 3 | 4 | if [ "$#" -ne 2 ]; then 5 | echo "usage: $0 " # TODO: fix usage message to reflect that bucket prefix is also required 6 | exit 1 7 | fi 8 | 9 | S3_BUCKET=$1 10 | S3_PREFIX=$2 11 | 12 | # Stage directory must be on EBS volume with 100 GB available space 13 | STAGE_DIR=$HOME/SageMaker/coco-2017-$(date +"%Y-%m-%d-%H-%M-%S") 14 | 15 | echo "Create stage directory: $STAGE_DIR" 16 | mkdir -p $STAGE_DIR 17 | 18 | # wget -O $STAGE_DIR/train2017.zip http://images.cocodataset.org/zips/train2017.zip 19 | aws s3 cp s3://fast-ai-coco/train2017.zip $STAGE_DIR/train2017.zip 20 | echo "Extracting $STAGE_DIR/train2017.zip" 21 | unzip -o $STAGE_DIR/train2017.zip -d $STAGE_DIR | awk 'BEGIN {ORS="="} {if(NR%1000==0)print "="}' 22 | echo "Done." 23 | rm $STAGE_DIR/train2017.zip 24 | 25 | # wget -O $STAGE_DIR/val2017.zip http://images.cocodataset.org/zips/val2017.zip 26 | aws s3 cp s3://fast-ai-coco/val2017.zip $STAGE_DIR/val2017.zip 27 | echo "Extracting $STAGE_DIR/val2017.zip" 28 | unzip -o $STAGE_DIR/val2017.zip -d $STAGE_DIR | awk 'BEGIN {ORS="="} {if(NR%1000==0)print "="}' 29 | echo "Done." 30 | rm $STAGE_DIR/val2017.zip 31 | 32 | # wget -O $STAGE_DIR/annotations_trainval2017.zip http://images.cocodataset.org/annotations/annotations_trainval2017.zip 33 | aws s3 cp s3://fast-ai-coco/annotations_trainval2017.zip $STAGE_DIR/annotations_trainval2017.zip 34 | unzip -o $STAGE_DIR/annotations_trainval2017.zip -d $STAGE_DIR 35 | rm $STAGE_DIR/annotations_trainval2017.zip 36 | 37 | echo "`date`: Uploading extracted files to s3://$S3_BUCKET/$S3_PREFIX [ eta 12 minutes ]" 38 | aws s3 cp --recursive $STAGE_DIR s3://$S3_BUCKET/$S3_PREFIX | awk 'BEGIN {ORS="="} {if(NR%100==0)print "="}' 39 | echo "Done." 40 | 41 | echo "Delete stage directory: $STAGE_DIR" 42 | rm -rf $STAGE_DIR 43 | echo "Success." -------------------------------------------------------------------------------- /docs/_print/about/featured-background.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/_print/about/featured-background.jpg -------------------------------------------------------------------------------- /docs/_print/about/featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_1920x1080_fill_q75_catmullrom_bottom.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/_print/about/featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_1920x1080_fill_q75_catmullrom_bottom.jpg -------------------------------------------------------------------------------- /docs/_print/about/featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_960x540_fill_q75_catmullrom_bottom.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/_print/about/featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_960x540_fill_q75_catmullrom_bottom.jpg -------------------------------------------------------------------------------- /docs/_print/docs/ai-services/images/rekognition/celeb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/_print/docs/ai-services/images/rekognition/celeb.png -------------------------------------------------------------------------------- /docs/_print/docs/ai-services/images/rekognition/compare-face.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/_print/docs/ai-services/images/rekognition/compare-face.png -------------------------------------------------------------------------------- /docs/_print/docs/ai-services/images/rekognition/moderation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/_print/docs/ai-services/images/rekognition/moderation.png -------------------------------------------------------------------------------- /docs/_print/docs/ai-services/images/rekognition/ppe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/_print/docs/ai-services/images/rekognition/ppe.png -------------------------------------------------------------------------------- /docs/_print/docs/ai-services/images/rekognition/rek-obj.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/_print/docs/ai-services/images/rekognition/rek-obj.png -------------------------------------------------------------------------------- /docs/_print/docs/ai-services/images/rekognition/text.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/_print/docs/ai-services/images/rekognition/text.png -------------------------------------------------------------------------------- /docs/_print/docs/ai-services/images/textract/textract.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/_print/docs/ai-services/images/textract/textract.png -------------------------------------------------------------------------------- /docs/_print/docs/ai-services/picture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/_print/docs/ai-services/picture.png -------------------------------------------------------------------------------- /docs/_print/docs/お客様事例集/picture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/_print/docs/お客様事例集/picture.png -------------------------------------------------------------------------------- /docs/about/featured-background.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/about/featured-background.jpg -------------------------------------------------------------------------------- /docs/about/featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_1920x1080_fill_q75_catmullrom_bottom.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/about/featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_1920x1080_fill_q75_catmullrom_bottom.jpg -------------------------------------------------------------------------------- /docs/about/featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_960x540_fill_q75_catmullrom_bottom.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/about/featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_960x540_fill_q75_catmullrom_bottom.jpg -------------------------------------------------------------------------------- /docs/blog/2018/10/06/easy-documentation-with-docsy/featured-sunset-get.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/blog/2018/10/06/easy-documentation-with-docsy/featured-sunset-get.png -------------------------------------------------------------------------------- /docs/blog/2018/10/06/easy-documentation-with-docsy/featured-sunset-get_hu69849a7cdb847c2393a7b3a7f6061c86_387442_250x125_fill_catmullrom_center_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/blog/2018/10/06/easy-documentation-with-docsy/featured-sunset-get_hu69849a7cdb847c2393a7b3a7f6061c86_387442_250x125_fill_catmullrom_center_2.png -------------------------------------------------------------------------------- /docs/blog/2018/10/06/easy-documentation-with-docsy/featured-sunset-get_hu69849a7cdb847c2393a7b3a7f6061c86_387442_600x300_fill_catmullrom_smart1_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/blog/2018/10/06/easy-documentation-with-docsy/featured-sunset-get_hu69849a7cdb847c2393a7b3a7f6061c86_387442_600x300_fill_catmullrom_smart1_2.png -------------------------------------------------------------------------------- /docs/blog/2018/10/06/easy-documentation-with-docsy/featured-sunset-get_hu69849a7cdb847c2393a7b3a7f6061c86_387442_640x0_resize_catmullrom_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/blog/2018/10/06/easy-documentation-with-docsy/featured-sunset-get_hu69849a7cdb847c2393a7b3a7f6061c86_387442_640x0_resize_catmullrom_2.png -------------------------------------------------------------------------------- /docs/blog/news/page/1/index.html: -------------------------------------------------------------------------------- 1 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/blog/news/ -------------------------------------------------------------------------------- /docs/blog/page/1/index.html: -------------------------------------------------------------------------------- 1 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/blog/ -------------------------------------------------------------------------------- /docs/blog/releases/page/1/index.html: -------------------------------------------------------------------------------- 1 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/blog/releases/ -------------------------------------------------------------------------------- /docs/css/prism.css: -------------------------------------------------------------------------------- 1 | /* PrismJS 1.21.0 2 | https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript+bash+c+csharp+cpp+go+java+markdown+python+scss+sql+toml+yaml&plugins=toolbar+copy-to-clipboard */ 3 | /** 4 | * prism.js default theme for JavaScript, CSS and HTML 5 | * Based on dabblet (http://dabblet.com) 6 | * @author Lea Verou 7 | */ 8 | 9 | code[class*="language-"], 10 | pre[class*="language-"] { 11 | color: black; 12 | background: none; 13 | text-shadow: 0 1px white; 14 | font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; 15 | font-size: 1em; 16 | text-align: left; 17 | white-space: pre; 18 | word-spacing: normal; 19 | word-break: normal; 20 | word-wrap: normal; 21 | line-height: 1.5; 22 | 23 | -moz-tab-size: 4; 24 | -o-tab-size: 4; 25 | tab-size: 4; 26 | 27 | -webkit-hyphens: none; 28 | -moz-hyphens: none; 29 | -ms-hyphens: none; 30 | hyphens: none; 31 | } 32 | 33 | pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection, 34 | code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection { 35 | text-shadow: none; 36 | background: #b3d4fc; 37 | } 38 | 39 | pre[class*="language-"]::selection, pre[class*="language-"] ::selection, 40 | code[class*="language-"]::selection, code[class*="language-"] ::selection { 41 | text-shadow: none; 42 | background: #b3d4fc; 43 | } 44 | 45 | @media print { 46 | code[class*="language-"], 47 | pre[class*="language-"] { 48 | text-shadow: none; 49 | } 50 | } 51 | 52 | /* Code blocks */ 53 | pre[class*="language-"] { 54 | padding: 1em; 55 | margin: .5em 0; 56 | overflow: auto; 57 | } 58 | 59 | :not(pre) > code[class*="language-"], 60 | pre[class*="language-"] { 61 | background: #f5f2f0; 62 | } 63 | 64 | /* Inline code */ 65 | :not(pre) > code[class*="language-"] { 66 | padding: .1em; 67 | border-radius: .3em; 68 | white-space: normal; 69 | } 70 | 71 | .token.comment, 72 | .token.prolog, 73 | .token.doctype, 74 | .token.cdata { 75 | color: slategray; 76 | } 77 | 78 | .token.punctuation { 79 | color: #999; 80 | } 81 | 82 | .token.namespace { 83 | opacity: .7; 84 | } 85 | 86 | .token.property, 87 | .token.tag, 88 | .token.boolean, 89 | .token.number, 90 | .token.constant, 91 | .token.symbol, 92 | .token.deleted { 93 | color: #905; 94 | } 95 | 96 | .token.selector, 97 | .token.attr-name, 98 | .token.string, 99 | .token.char, 100 | .token.builtin, 101 | .token.inserted { 102 | color: #690; 103 | } 104 | 105 | .token.operator, 106 | .token.entity, 107 | .token.url, 108 | .language-css .token.string, 109 | .style .token.string { 110 | color: #9a6e3a; 111 | /* This background color was intended by the author of this theme. */ 112 | background: hsla(0, 0%, 100%, .5); 113 | } 114 | 115 | .token.atrule, 116 | .token.attr-value, 117 | .token.keyword { 118 | color: #07a; 119 | } 120 | 121 | .token.function, 122 | .token.class-name { 123 | color: #DD4A68; 124 | } 125 | 126 | .token.regex, 127 | .token.important, 128 | .token.variable { 129 | color: #e90; 130 | } 131 | 132 | .token.important, 133 | .token.bold { 134 | font-weight: bold; 135 | } 136 | .token.italic { 137 | font-style: italic; 138 | } 139 | 140 | .token.entity { 141 | cursor: help; 142 | } 143 | 144 | div.code-toolbar { 145 | position: relative; 146 | } 147 | 148 | div.code-toolbar > .toolbar { 149 | position: absolute; 150 | top: .3em; 151 | right: .2em; 152 | transition: opacity 0.3s ease-in-out; 153 | opacity: 0; 154 | } 155 | 156 | div.code-toolbar:hover > .toolbar { 157 | opacity: 1; 158 | } 159 | 160 | /* Separate line b/c rules are thrown out if selector is invalid. 161 | IE11 and old Edge versions don't support :focus-within. */ 162 | div.code-toolbar:focus-within > .toolbar { 163 | opacity: 1; 164 | } 165 | 166 | div.code-toolbar > .toolbar .toolbar-item { 167 | display: inline-block; 168 | } 169 | 170 | div.code-toolbar > .toolbar a { 171 | cursor: pointer; 172 | } 173 | 174 | div.code-toolbar > .toolbar button { 175 | background: none; 176 | border: 0; 177 | color: inherit; 178 | font: inherit; 179 | line-height: normal; 180 | overflow: visible; 181 | padding: 0; 182 | -webkit-user-select: none; /* for button */ 183 | -moz-user-select: none; 184 | -ms-user-select: none; 185 | } 186 | 187 | div.code-toolbar > .toolbar a, 188 | div.code-toolbar > .toolbar button, 189 | div.code-toolbar > .toolbar span { 190 | color: #bbb; 191 | font-size: .8em; 192 | padding: 0 .5em; 193 | background: #f5f2f0; 194 | background: rgba(224, 224, 224, 0.2); 195 | box-shadow: 0 2px 0 0 rgba(0,0,0,0.2); 196 | border-radius: .5em; 197 | } 198 | 199 | div.code-toolbar > .toolbar a:hover, 200 | div.code-toolbar > .toolbar a:focus, 201 | div.code-toolbar > .toolbar button:hover, 202 | div.code-toolbar > .toolbar button:focus, 203 | div.code-toolbar > .toolbar span:hover, 204 | div.code-toolbar > .toolbar span:focus { 205 | color: inherit; 206 | text-decoration: none; 207 | } 208 | 209 | -------------------------------------------------------------------------------- /docs/docs/ai-services/images/rekognition/celeb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/docs/ai-services/images/rekognition/celeb.png -------------------------------------------------------------------------------- /docs/docs/ai-services/images/rekognition/compare-face.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/docs/ai-services/images/rekognition/compare-face.png -------------------------------------------------------------------------------- /docs/docs/ai-services/images/rekognition/moderation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/docs/ai-services/images/rekognition/moderation.png -------------------------------------------------------------------------------- /docs/docs/ai-services/images/rekognition/ppe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/docs/ai-services/images/rekognition/ppe.png -------------------------------------------------------------------------------- /docs/docs/ai-services/images/rekognition/rek-obj.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/docs/ai-services/images/rekognition/rek-obj.png -------------------------------------------------------------------------------- /docs/docs/ai-services/images/rekognition/text.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/docs/ai-services/images/rekognition/text.png -------------------------------------------------------------------------------- /docs/docs/ai-services/images/textract/textract.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/docs/ai-services/images/textract/textract.png -------------------------------------------------------------------------------- /docs/docs/ai-services/picture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/docs/ai-services/picture.png -------------------------------------------------------------------------------- /docs/docs/お客様事例集/picture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/docs/お客様事例集/picture.png -------------------------------------------------------------------------------- /docs/favicons/android-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/android-144x144.png -------------------------------------------------------------------------------- /docs/favicons/android-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/android-192x192.png -------------------------------------------------------------------------------- /docs/favicons/android-36x36.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/android-36x36.png -------------------------------------------------------------------------------- /docs/favicons/android-48x48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/android-48x48.png -------------------------------------------------------------------------------- /docs/favicons/android-72x72.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/android-72x72.png -------------------------------------------------------------------------------- /docs/favicons/android-96x96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/android-96x96.png -------------------------------------------------------------------------------- /docs/favicons/apple-touch-icon-180x180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/apple-touch-icon-180x180.png -------------------------------------------------------------------------------- /docs/favicons/favicon-1024.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/favicon-1024.png -------------------------------------------------------------------------------- /docs/favicons/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/favicon-16x16.png -------------------------------------------------------------------------------- /docs/favicons/favicon-256.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/favicon-256.png -------------------------------------------------------------------------------- /docs/favicons/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/favicon-32x32.png -------------------------------------------------------------------------------- /docs/favicons/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/favicon.ico -------------------------------------------------------------------------------- /docs/favicons/pwa-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/pwa-192x192.png -------------------------------------------------------------------------------- /docs/favicons/pwa-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/pwa-512x512.png -------------------------------------------------------------------------------- /docs/favicons/tile150x150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/tile150x150.png -------------------------------------------------------------------------------- /docs/favicons/tile310x150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/tile310x150.png -------------------------------------------------------------------------------- /docs/favicons/tile310x310.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/tile310x310.png -------------------------------------------------------------------------------- /docs/favicons/tile70x70.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/favicons/tile70x70.png -------------------------------------------------------------------------------- /docs/featured-background.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/featured-background.jpg -------------------------------------------------------------------------------- /docs/featured-background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/featured-background.png -------------------------------------------------------------------------------- /docs/featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_1920x1080_fill_q75_catmullrom_top.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_1920x1080_fill_q75_catmullrom_top.jpg -------------------------------------------------------------------------------- /docs/featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_960x540_fill_q75_catmullrom_top.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/featured-background_hu376e1fbab6ce6c455a2b3aa5c258c0d9_496231_960x540_fill_q75_catmullrom_top.jpg -------------------------------------------------------------------------------- /docs/featured-background_hu8a39ca4bb86027cf9971f48ddb45157c_1460964_1920x1080_fill_catmullrom_top_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/featured-background_hu8a39ca4bb86027cf9971f48ddb45157c_1460964_1920x1080_fill_catmullrom_top_2.png -------------------------------------------------------------------------------- /docs/featured-background_hu8a39ca4bb86027cf9971f48ddb45157c_1460964_960x540_fill_catmullrom_top_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/featured-background_hu8a39ca4bb86027cf9971f48ddb45157c_1460964_960x540_fill_catmullrom_top_2.png -------------------------------------------------------------------------------- /docs/js/main.min.2698ba6d60966a95b30f3160f89208b4219c5535421f52ee6147801ad45cc471.js: -------------------------------------------------------------------------------- 1 | (function($){'use strict';$(function(){$('[data-toggle="tooltip"]').tooltip();$('[data-toggle="popover"]').popover();$('.popover-dismiss').popover({trigger:'focus'})});function bottomPos(element){return element.offset().top+element.outerHeight();} 2 | $(function(){var promo=$(".js-td-cover");if(!promo.length){return} 3 | var promoOffset=bottomPos(promo);var navbarOffset=$('.js-navbar-scroll').offset().top;var threshold=Math.ceil($('.js-navbar-scroll').outerHeight());if((promoOffset-navbarOffset)';a.href='#'+heading.id;heading.insertAdjacentElement('beforeend',a);heading.addEventListener('mouseenter',function(){a.style.visibility='initial';});heading.addEventListener('mouseleave',function(){a.style.visibility='hidden';});}});});}(jQuery));;(function($){'use strict';var Search={init:function(){$(document).ready(function(){$(document).on('keypress','.td-search-input',function(e){if(e.keyCode!==13){return} 6 | var query=$(this).val();var searchPage="https://aws-samples.github.io/amazon-sagemaker-examples-jp/search/?q="+query;document.location=searchPage;return false;});});},};Search.init();}(jQuery));; -------------------------------------------------------------------------------- /docs/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * -------------------------------------------------------------------------------- /docs/sitemap.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/text/comprehend/ 7 | 2021-03-09T14:31:16+09:00 8 | 9 | 10 | 11 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/table-data/forecast/ 12 | 2021-03-09T14:31:16+09:00 13 | 14 | 15 | 16 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/images/rekognition/ 17 | 2021-03-08T17:41:31+09:00 18 | 19 | 20 | 21 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/voice/transcribe/ 22 | 2021-03-09T14:31:16+09:00 23 | 24 | 25 | 26 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/images/ 27 | 2021-03-08T11:24:33+09:00 28 | 29 | 30 | 31 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/usecases/table-data/ 32 | 2021-03-04T13:20:16+09:00 33 | 34 | 35 | 36 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ml-services/sagemaker-tutorial/ 37 | 2021-03-04T13:20:16+09:00 38 | 39 | 40 | 41 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/usecases/ 42 | 2021-03-04T13:20:16+09:00 43 | 44 | 45 | 46 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/ 47 | 2021-03-09T14:31:16+09:00 48 | 49 | 50 | 51 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/table-data/personalize/ 52 | 2021-03-09T14:31:16+09:00 53 | 54 | 55 | 56 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/images/textract/ 57 | 2021-03-08T19:18:54+09:00 58 | 59 | 60 | 61 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/text/translate/ 62 | 2021-03-09T14:31:16+09:00 63 | 64 | 65 | 66 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/text/ 67 | 2021-03-08T19:18:54+09:00 68 | 69 | 70 | 71 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/usecases/image/ 72 | 2021-03-04T13:20:16+09:00 73 | 74 | 75 | 76 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/usecases/nlp/ 77 | 2021-03-04T13:20:16+09:00 78 | 79 | 80 | 81 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ml-services/sagemaker-dive-deep/ 82 | 2021-03-04T13:20:16+09:00 83 | 84 | 85 | 86 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/table-data/fraud/ 87 | 2021-03-09T14:39:53+09:00 88 | 89 | 90 | 91 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/text/polly/ 92 | 2021-03-09T14:31:16+09:00 93 | 94 | 95 | 96 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/table-data/ 97 | 2021-03-08T19:18:54+09:00 98 | 99 | 100 | 101 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ml-services/ 102 | 2021-03-04T13:20:16+09:00 103 | 104 | 105 | 106 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ai-services/voice/ 107 | 2021-03-08T19:18:54+09:00 108 | 109 | 110 | 111 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/docs/ 112 | 2021-03-04T13:20:16+09:00 113 | 114 | 115 | 116 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/blog/news/ 117 | 2021-02-06T20:24:17+09:00 118 | 119 | 120 | 121 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/blog/releases/ 122 | 2021-02-06T20:24:17+09:00 123 | 124 | 125 | 126 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/blog/2018/10/06/easy-documentation-with-docsy/ 127 | 2021-02-06T20:24:17+09:00 128 | 129 | 130 | 131 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/blog/2018/10/06/the-second-blog-post/ 132 | 2021-02-06T20:24:17+09:00 133 | 134 | 135 | 136 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/blog/2018/01/04/another-great-release/ 137 | 2021-02-06T20:24:17+09:00 138 | 139 | 140 | 141 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/about/ 142 | 2021-02-06T20:24:17+09:00 143 | 144 | 145 | 146 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/ 147 | 2021-03-08T11:24:33+09:00 148 | 149 | 150 | 151 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/blog/ 152 | 2021-03-04T13:20:16+09:00 153 | 154 | 155 | 156 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/examples/ 157 | 2021-03-04T13:20:16+09:00 158 | 159 | 160 | 161 | https://aws-samples.github.io/amazon-sagemaker-examples-jp/search/ 162 | 2021-02-06T20:24:17+09:00 163 | 164 | 165 | 166 | -------------------------------------------------------------------------------- /docs/webfonts/fa-brands-400.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/webfonts/fa-brands-400.eot -------------------------------------------------------------------------------- /docs/webfonts/fa-brands-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/webfonts/fa-brands-400.ttf -------------------------------------------------------------------------------- /docs/webfonts/fa-brands-400.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/webfonts/fa-brands-400.woff -------------------------------------------------------------------------------- /docs/webfonts/fa-brands-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/webfonts/fa-brands-400.woff2 -------------------------------------------------------------------------------- /docs/webfonts/fa-regular-400.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/webfonts/fa-regular-400.eot -------------------------------------------------------------------------------- /docs/webfonts/fa-regular-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/webfonts/fa-regular-400.ttf -------------------------------------------------------------------------------- /docs/webfonts/fa-regular-400.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/webfonts/fa-regular-400.woff -------------------------------------------------------------------------------- /docs/webfonts/fa-regular-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/webfonts/fa-regular-400.woff2 -------------------------------------------------------------------------------- /docs/webfonts/fa-solid-900.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/webfonts/fa-solid-900.eot -------------------------------------------------------------------------------- /docs/webfonts/fa-solid-900.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/webfonts/fa-solid-900.ttf -------------------------------------------------------------------------------- /docs/webfonts/fa-solid-900.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/webfonts/fa-solid-900.woff -------------------------------------------------------------------------------- /docs/webfonts/fa-solid-900.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/docs/webfonts/fa-solid-900.woff2 -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | ec2-key-pair.pem 131 | *.npy 132 | 133 | -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/README.md: -------------------------------------------------------------------------------- 1 | # greengrass-ml-inference 2 | * 本コンテンツは AWS IoT Greengrass(v2) 上で動く ML 推論アプリを開発、デプロイするハンズオンです。 3 | * EC2 を 4 台、それぞれ開発機、ステージング機、本番機 2 台をエッジデバイスとして扱います。 4 | * ML モデルの作成は [build_mnist_dcgan_and_classifier.ipynb](./build_mnist_dcgan_and_classifier.ipynb) で行っていますが、モデル自体はこのリポジトリにすでに含まれているため改めて動かす必要はありません。 5 | * [greengrass_ml_inference.ipynb](greengrass_ml_inference.ipynb)ですべて動作が完結するように記載しているため、詳細は[greengrass_ml_inference.ipynb](greengrass_ml_inference.ipynb)をご参照ください。 6 | * notebook を実行する環境には下記ポリシーがアタッチされていることを前提としています。 7 | * AmazonEC2FullAccess 8 | * AmazonSSMFullAccess 9 | * AmazonS3FullAccess 10 | * IAMFullAccess 11 | * AWSGreengrassFullAccess 12 | * AWSIoTFullAccess 13 | * AmazonEC2ContainerRegistryFullAccess -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/AWS-Architecture-Icons-Deck_For-Dark-BG_20200911.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/AWS-Architecture-Icons-Deck_For-Dark-BG_20200911.pptx -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image01.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image02.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image03.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image04.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image05.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image06.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image06.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image07.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image07.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image08.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image08.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image09.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image09.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image10.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image11.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image12.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image13.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image14.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/image/image15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/image/image15.png -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/classifier_train.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import os 4 | from keras.models import Model,Sequential,load_model 5 | from keras.layers import Input,Conv2D,LeakyReLU,Dense,Reshape,Flatten,Dropout,BatchNormalization,Conv2DTranspose,MaxPool2D 6 | from keras.activations import sigmoid 7 | from keras.datasets import fashion_mnist 8 | from keras.optimizers import Adam 9 | import argparse, json 10 | from distutils.util import strtobool 11 | 12 | MODEL_FILE_NAME = 'classifier.h5' 13 | 14 | def classifier(): 15 | inputs = Input(shape=(28,28,1)) 16 | x = Conv2D(64, (3,3),padding='same')(inputs) 17 | x = BatchNormalization()(x) 18 | x = LeakyReLU(0.2)(x) 19 | x = MaxPool2D(pool_size=(2, 2))(x) # 14x14 20 | x = Conv2D(64, (3,3),padding='same')(x) 21 | x = BatchNormalization()(x) 22 | x = LeakyReLU(0.2)(x) 23 | x = MaxPool2D(pool_size=(2, 2))(x) # 7x7 24 | x = Conv2D(128, (3,3),padding='same')(x) 25 | x = BatchNormalization()(x) 26 | x = LeakyReLU(0.2)(x) 27 | x = MaxPool2D(pool_size=(2, 2))(x) 28 | x = Conv2D(128, (3,3),padding='same')(x) 29 | x = BatchNormalization()(x) 30 | x = LeakyReLU(0.2)(x) 31 | x = MaxPool2D(pool_size=(2, 2))(x) 32 | x = Flatten()(x) 33 | x = Dense(128)(x) 34 | x = LeakyReLU(0.2)(x) 35 | x = BatchNormalization()(x) 36 | x = Dropout(0.5)(x) 37 | x = Dense(10, activation='softmax')(x) 38 | model = Model(inputs=inputs, outputs=x) 39 | model.summary() 40 | return model 41 | 42 | def train(train_x,train_y,valid_x,valid_y,epochs,model_dir,increment,base_dir): 43 | if increment: 44 | model = load_model(os.path.join(base_dir, MODEL_FILE_NAME)) 45 | else: 46 | model = classifier() 47 | model.compile(optimizer=Adam(lr=0.0001),metrics=['accuracy'],loss="categorical_crossentropy") 48 | model.fit(train_x,train_y,batch_size=16,epochs=epochs,validation_data=(valid_x,valid_y)) 49 | 50 | save_model_path = os.path.join(model_dir, MODEL_FILE_NAME) 51 | model.save(save_model_path) 52 | 53 | return model 54 | 55 | def _parse_args(): 56 | parser = argparse.ArgumentParser() 57 | parser.add_argument('--model_dir', type=str) 58 | parser.add_argument('--sm-model-dir', type=str, default=os.environ.get('SM_MODEL_DIR')) 59 | parser.add_argument('--train', type=str, default=os.environ.get('SM_CHANNEL_TRAIN')) 60 | parser.add_argument('--hosts', type=list, default=json.loads(os.environ.get('SM_HOSTS'))) 61 | parser.add_argument('--current-host', type=str, default=os.environ.get('SM_CURRENT_HOST')) 62 | parser.add_argument('--epochs', type=int, default=2) 63 | parser.add_argument('--increment', type=strtobool ,default=False) 64 | 65 | return parser.parse_known_args() 66 | 67 | def load_training_data(base_dir): 68 | X = np.load(os.path.join(base_dir, 'train_X.npy')) 69 | y = np.load(os.path.join(base_dir, 'train_y.npy')) 70 | # shuffle and split 71 | shuffle_index = np.random.choice(np.arange(X.shape[0]), X.shape[0], replace=False) 72 | train_X = X[shuffle_index[0:50000]] 73 | train_y = y[shuffle_index[0:50000]] 74 | valid_X = X[shuffle_index[50000:]] 75 | valid_y = y[shuffle_index[50000:]] 76 | 77 | return train_X, train_y, valid_X, valid_y 78 | 79 | if __name__ == "__main__": 80 | args, unknown = _parse_args() 81 | print(args) 82 | train_X, train_y, valid_X, valid_y = load_training_data(args.train) 83 | model = train(train_X, train_y, valid_X, valid_y,args.epochs,args.sm_model_dir,args.increment,args.train) 84 | 85 | exit() -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.IoTPublisher/1.0.0/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8.10-slim-buster 2 | 3 | RUN apt-get update; \ 4 | apt-get install -y cmake \ 5 | ; \ 6 | rm -rf /var/lib/apt/lists/* 7 | 8 | RUN python3 -m pip install awsiotsdk numpy tensorflow-cpu==2.4.1 Pillow 9 | WORKDIR /app 10 | 11 | COPY ./IoTPublisher.py /app/IoTPublisher.py 12 | COPY ./test_X.npy /app/test_X.npy 13 | COPY ./classifier.h5 /app/classifier.h5 14 | 15 | CMD ["python3", "/app/IoTPublisher.py"] 16 | -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.IoTPublisher/1.0.0/IoTPublisher.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | import datetime,json 3 | from awsiot.greengrasscoreipc import connect 4 | from awsiot.greengrasscoreipc.model import ( 5 | QOS, 6 | PublishToIoTCoreRequest 7 | ) 8 | import tensorflow as tf 9 | from PIL import Image 10 | import os, sys 11 | import numpy as np 12 | import signal 13 | from logging import getLogger 14 | logger = getLogger(__name__) 15 | 16 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 17 | 18 | TIMEOUT = 10 19 | INTERVAL = 60 20 | 21 | ipc_client = connect() 22 | 23 | test_X = np.load('/app/test_X.npy') 24 | 25 | classifier_model = tf.keras.models.load_model('/app/classifier.h5') 26 | 27 | topic = "inference/result" 28 | 29 | def signal_handler(signal, frame): 30 | logger.info(f"Received {signal}, exiting") 31 | sys.exit(0) 32 | 33 | # Register SIGTERM for shutdown of container 34 | signal.signal(signal.SIGTERM, signal_handler) 35 | signal.signal(signal.SIGINT, signal_handler) 36 | 37 | cnt = 0 38 | while True: 39 | 40 | idx = np.random.randint(0,test_X.shape[0]) 41 | img_array = test_X[idx:idx+1,:,:,:] 42 | 43 | pred_y = np.argmax(classifier_model.predict(img_array)) 44 | 45 | result = 'anomaly' if pred_y % 2 == 0 else 'normal' 46 | 47 | cnt = cnt + 1 48 | message = { 49 | "timestamp": str(datetime.datetime.now()), 50 | "message": result, 51 | "counter": str(cnt), 52 | "component_version" : "1.0.0" 53 | } 54 | 55 | request = PublishToIoTCoreRequest(topic_name=topic, qos=QOS.AT_LEAST_ONCE, payload=bytes(json.dumps(message), "utf-8")) 56 | operation = ipc_client.new_publish_to_iot_core() 57 | operation.activate(request) 58 | future = operation.get_response() 59 | future.result(TIMEOUT) 60 | 61 | logger.info("publish") 62 | sleep(INTERVAL) 63 | -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.IoTPublisher/1.0.1/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8.10-slim-buster 2 | 3 | RUN apt-get update; \ 4 | apt-get install -y cmake traceroute\ 5 | ; \ 6 | rm -rf /var/lib/apt/lists/* 7 | 8 | RUN python3 -m pip install awsiotsdk numpy dlr Pillow 9 | WORKDIR /app 10 | 11 | COPY ./IoTPublisher.py /app/IoTPublisher.py 12 | COPY ./test_X.npy /app/test_X.npy 13 | COPY ./classifier/ /app/classifier/ 14 | 15 | CMD ["python3", "/app/IoTPublisher.py"] 16 | -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.IoTPublisher/1.0.1/IoTPublisher.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | import dlr, datetime, json, os, sys, signal 3 | from awsiot.greengrasscoreipc import connect 4 | from awsiot.greengrasscoreipc.model import ( 5 | QOS, 6 | PublishToIoTCoreRequest 7 | ) 8 | import numpy as np 9 | from logging import getLogger 10 | logger = getLogger(__name__) 11 | 12 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 13 | 14 | # メッセージに thing_name を入れるために thing_name を環境変数から取得する 15 | THING_NAME = os.environ.get('AWS_IOT_THING_NAME') 16 | 17 | TIMEOUT = 10 18 | 19 | ipc_client = connect() 20 | 21 | INTERVAL = 60 22 | 23 | neo_dir = '/app/classifier' 24 | classifier_neo = dlr.DLRModel(neo_dir, 'cpu', 0) 25 | 26 | test_X = np.load('/app/test_X.npy') 27 | 28 | topic = "inference/result" 29 | 30 | def signal_handler(signal, frame): 31 | logger.info(f"Received {signal}, exiting") 32 | sys.exit(0) 33 | 34 | # Register SIGTERM for shutdown of container 35 | signal.signal(signal.SIGTERM, signal_handler) 36 | signal.signal(signal.SIGINT, signal_handler) 37 | 38 | cnt = 0 39 | while True: 40 | idx = np.random.randint(0,test_X.shape[0]) 41 | img_array = test_X[idx:idx+1,:,:,:] 42 | 43 | pred_y = np.argmax(classifier_neo.run(test_X[0,:,:,:].reshape(1,1,28,28))[0]) 44 | 45 | result = 'anomaly' if pred_y % 2 == 0 else 'normal' 46 | 47 | cnt = cnt + 1 48 | message = { 49 | "timestamp": str(datetime.datetime.now()), 50 | "message": result, 51 | "counter": str(cnt), 52 | "component_version" : "1.0.1", 53 | "thing_name" : THING_NAME 54 | } 55 | 56 | request = PublishToIoTCoreRequest(topic_name=topic, qos=QOS.AT_LEAST_ONCE, payload=bytes(json.dumps(message), "utf-8")) 57 | operation = ipc_client.new_publish_to_iot_core() 58 | operation.activate(request) 59 | future = operation.get_response() 60 | future.result(TIMEOUT) 61 | 62 | logger.info("publish") 63 | sleep(INTERVAL) 64 | -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.IoTPublisher/1.0.1/classifier/compiled.meta: -------------------------------------------------------------------------------- 1 | { 2 | "Requirements": { 3 | "TargetDevice": "ML_C5", 4 | "TargetDeviceType": "cpu" 5 | }, 6 | "Compilation": { 7 | "CreatedTime": 1628746271.076939 8 | }, 9 | "Model": { 10 | "Inputs": [ 11 | { 12 | "name": "input_1", 13 | "shape": [ 14 | 1, 15 | 1, 16 | 28, 17 | 28 18 | ], 19 | "dtype": "float32" 20 | } 21 | ], 22 | "Outputs": [ 23 | { 24 | "dtype": "float32", 25 | "shape": [ 26 | 1, 27 | 10 28 | ], 29 | "name": "output_0" 30 | } 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.IoTPublisher/1.0.1/classifier/compiled.params: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.IoTPublisher/1.0.1/classifier/compiled.params -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.IoTPublisher/1.0.1/classifier/compiled.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.IoTPublisher/1.0.1/classifier/compiled.so -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.IoTPublisher/1.0.1/classifier/libdlr.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.IoTPublisher/1.0.1/classifier/libdlr.so -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.IoTPublisher/1.0.1/classifier/manifest: -------------------------------------------------------------------------------- 1 | {"algorithm": "blake2s", "files": [{"name": "compiled.meta", "hash": "e41fb375831ba8663cf904fefa79af3bd2572c252ba68cb108366ccf17ce176d"}, {"name": "compiled.params", "hash": "a64bce4a955cd89a6e930c76f7e86c996a91605d12dd37c7e85ed4e4b72b4efd"}, {"name": "compiled.so", "hash": "22118fffbddc22059cbb5f343267ab130ef0d89d31fe07c5b691748b5eef724a"}, {"name": "compiled_model.json", "hash": "303da711e43da5a46afc86b8b906dd94211c500dd054dcd44d069b6ce29f295a"}, {"name": "dlr.h", "hash": "32244b9d4939516f5fcb29d01569f6040320174f220a2fa40f96210ad7cc8ec0"}, {"name": "libdlr.so", "hash": "291cb1f05749cc203bdebb03eb06a4d904780791c6c9f5ded6c6f2a21f9eda3e"}]} -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.Publisher/1.0.0/publisher.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | import datetime,json 3 | import awsiot.greengrasscoreipc 4 | from awsiot.greengrasscoreipc.model import ( 5 | PublishToTopicRequest, 6 | PublishMessage, 7 | JsonMessage 8 | ) 9 | from PIL import Image 10 | import os, sys 11 | import numpy as np 12 | from logging import getLogger 13 | logger = getLogger(__name__) 14 | 15 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 16 | 17 | logger.info(f'argv:{sys.argv}') 18 | 19 | test_X_path = os.path.join(sys.argv[1],'test_X.npy') 20 | test_X = np.load(test_X_path) 21 | 22 | logger.info('start publisher...') 23 | 24 | TIMEOUT = 10 25 | interval = 60 26 | 27 | ipc_client = awsiot.greengrasscoreipc.connect() 28 | 29 | topic = "my/topic" 30 | 31 | logger.info('start loop') 32 | 33 | while True: 34 | # generate and save image file 35 | idx = np.random.randint(0,test_X.shape[0]) 36 | file_name = '/tmp/' + datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=9))).strftime('%Y%m%d%H%M%S') + '.png' 37 | Image.fromarray(((test_X[idx,:,:,0]*127.5)+127.5).astype(np.uint8)).save(file_name) 38 | 39 | message = {"file_name": file_name } 40 | message_json = json.dumps(message).encode('utf-8') 41 | 42 | request = PublishToTopicRequest() 43 | request.topic = topic 44 | publish_message = PublishMessage() 45 | publish_message.json_message = JsonMessage() 46 | publish_message.json_message.message = message 47 | request.publish_message = publish_message 48 | operation = ipc_client.new_publish_to_topic() 49 | operation.activate(request) 50 | future = operation.get_response() 51 | future.result(TIMEOUT) 52 | 53 | logger.info(f'publish message: {message_json}') 54 | sleep(interval) 55 | -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.Subscriber/1.0.0/classifier.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.Subscriber/1.0.0/classifier.h5 -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/artifacts/com.example.Subscriber/1.0.0/subscriber.py: -------------------------------------------------------------------------------- 1 | import time, json, sys, os 2 | import awsiot.greengrasscoreipc 3 | import awsiot.greengrasscoreipc.client as client 4 | from awsiot.greengrasscoreipc.model import ( 5 | SubscribeToTopicRequest, 6 | SubscriptionResponseMessage 7 | ) 8 | import tensorflow as tf 9 | import numpy as np 10 | from PIL import Image 11 | from logging import getLogger 12 | logger = getLogger(__name__) 13 | 14 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 15 | 16 | logger.info(f'argv:{sys.argv}') 17 | 18 | model_path = os.path.join(sys.argv[1],'classifier.h5') 19 | 20 | logger.info('start subscriber...') 21 | 22 | TIMEOUT = 10 23 | 24 | logger.info('start to load model') 25 | model = tf.keras.models.load_model(model_path) 26 | 27 | ipc_client = awsiot.greengrasscoreipc.connect() 28 | 29 | class StreamHandler(client.SubscribeToTopicStreamHandler): 30 | def __init__(self): 31 | super().__init__() 32 | 33 | def on_stream_event(self, event: SubscriptionResponseMessage) -> None: 34 | message_string = event.json_message.message 35 | file_path = message_string['file_name'] 36 | logger.info(f'recieved message: {file_path}') 37 | img = Image.open(file_path) 38 | img = (np.array(img).reshape(1,28,28,1)-127.5)/127.5 39 | pred_y = np.argmax(model.predict(img)) 40 | with open('/tmp/Greengrass_Subscriber.log', 'a') as f: 41 | f.write(f'{file_path}\n') 42 | f.write(f'{img.shape}\n') 43 | f.write(f'{pred_y}\n') 44 | f.write('even\n') if pred_y % 2 == 0 else f.write('odd\n') 45 | # 処理済ファイルを削除 46 | os.remove(file_path) 47 | 48 | def on_stream_error(self, error: Exception) -> bool: 49 | return True 50 | 51 | def on_stream_closed(self) -> None: 52 | pass 53 | 54 | topic = "my/topic" 55 | 56 | request = SubscribeToTopicRequest() 57 | request.topic = topic 58 | handler = StreamHandler() 59 | operation = ipc_client.new_subscribe_to_topic(handler) 60 | future = operation.activate(request) 61 | while True: 62 | time.sleep(1) 63 | 64 | operation.close() 65 | -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/recipes/com.example.Publisher-1.0.0.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | RecipeFormatVersion: '2020-01-25' 3 | ComponentName: com.example.Publisher 4 | ComponentVersion: '1.0.0' 5 | ComponentDescription: A component that publishes messages. 6 | ComponentPublisher: Amazon 7 | ComponentConfiguration: 8 | DefaultConfiguration: 9 | accessControl: 10 | aws.greengrass.ipc.pubsub: 11 | 'com.example.Publisher:pubsub:1': 12 | policyDescription: Allows access to publish to all topics. 13 | operations: 14 | - 'aws.greengrass#PublishToTopic' 15 | resources: 16 | - '*' 17 | Manifests: 18 | - Lifecycle: 19 | Install: 20 | Timeout: 600 21 | Script: python3 -m pip install pip & pip3 install awsiotsdk numpy Pillow -U 22 | Run: python3 {artifacts:path}/publisher.py {artifacts:path} 23 | -------------------------------------------------------------------------------- /edge_inference/greengrass-ml-inference/src/ggv2/components/recipes/com.example.Subscriber-1.0.0.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | RecipeFormatVersion: '2020-01-25' 3 | ComponentName: com.example.Subscriber 4 | ComponentVersion: '1.0.0' 5 | ComponentDescription: A component that subscribes to messages. 6 | ComponentPublisher: Amazon 7 | ComponentConfiguration: 8 | DefaultConfiguration: 9 | accessControl: 10 | aws.greengrass.ipc.pubsub: 11 | 'com.example.Subscriber:pubsub:1': 12 | policyDescription: Allows access to publish to all topics. 13 | operations: 14 | - 'aws.greengrass#SubscribeToTopic' 15 | resources: 16 | - '*' 17 | Manifests: 18 | - Lifecycle: 19 | Install: 20 | Timeout: 600 21 | Script: python3 -m pip install pip & pip3 install awsiotsdk numpy tensorflow-cpu==2.4.1 Pillow -U 22 | Run: python3 {artifacts:path}/subscriber.py {artifacts:path} 23 | -------------------------------------------------------------------------------- /hpo_pytorch_mnist/mnist.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import logging 4 | import os 5 | import sys 6 | import torch 7 | import torch.distributed as dist 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | import torch.optim as optim 11 | import torch.utils.data 12 | import torch.utils.data.distributed 13 | from torchvision import datasets, transforms 14 | 15 | logger = logging.getLogger(__name__) 16 | logger.setLevel(logging.DEBUG) 17 | logger.addHandler(logging.StreamHandler(sys.stdout)) 18 | 19 | 20 | # Based on https://github.com/pytorch/examples/blob/master/mnist/main.py 21 | class Net(nn.Module): 22 | def __init__(self): 23 | super(Net, self).__init__() 24 | self.conv1 = nn.Conv2d(1, 10, kernel_size=5) 25 | self.conv2 = nn.Conv2d(10, 20, kernel_size=5) 26 | self.conv2_drop = nn.Dropout2d() 27 | self.fc1 = nn.Linear(320, 50) 28 | self.fc2 = nn.Linear(50, 10) 29 | 30 | def forward(self, x): 31 | x = F.relu(F.max_pool2d(self.conv1(x), 2)) 32 | x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2)) 33 | x = x.view(-1, 320) 34 | x = F.relu(self.fc1(x)) 35 | x = F.dropout(x, training=self.training) 36 | x = self.fc2(x) 37 | return F.log_softmax(x, dim=1) 38 | 39 | def _get_train_data_loader(batch_size, training_dir, is_distributed, **kwargs): 40 | logger.info("Get train data loader") 41 | train_tensor = torch.load(os.path.join(training_dir, 'training.pt')) 42 | dataset = torch.utils.data.TensorDataset(train_tensor[0], train_tensor[1]) 43 | 44 | train_sampler = torch.utils.data.distributed.DistributedSampler(dataset) if is_distributed else None 45 | return torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=train_sampler is None, 46 | sampler=train_sampler, **kwargs) 47 | 48 | def _get_test_data_loader(test_batch_size, training_dir, **kwargs): 49 | logger.info("Get test data loader") 50 | test_tensor = torch.load(os.path.join(training_dir, 'test.pt')) 51 | dataset = torch.utils.data.TensorDataset(test_tensor[0], test_tensor[1]) 52 | return torch.utils.data.DataLoader( 53 | dataset, 54 | batch_size=test_batch_size, 55 | shuffle=True, **kwargs) 56 | 57 | 58 | def _average_gradients(model): 59 | # Gradient averaging. 60 | size = float(dist.get_world_size()) 61 | for param in model.parameters(): 62 | dist.all_reduce(param.grad.data, op=dist.reduce_op.SUM) 63 | param.grad.data /= size 64 | 65 | 66 | def train(args): 67 | is_distributed = len(args.hosts) > 1 and args.backend is not None 68 | logger.debug("Distributed training - {}".format(is_distributed)) 69 | use_cuda = args.num_gpus > 0 70 | logger.debug("Number of gpus available - {}".format(args.num_gpus)) 71 | kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} 72 | device = torch.device("cuda" if use_cuda else "cpu") 73 | 74 | if is_distributed: 75 | # Initialize the distributed environment. 76 | world_size = len(args.hosts) 77 | os.environ['WORLD_SIZE'] = str(world_size) 78 | host_rank = args.hosts.index(args.current_host) 79 | os.environ['RANK'] = str(host_rank) 80 | dist.init_process_group(backend=args.backend, rank=host_rank, world_size=world_size) 81 | logger.info('Initialized the distributed environment: \'{}\' backend on {} nodes. '.format( 82 | args.backend, dist.get_world_size()) + 'Current host rank is {}. Number of gpus: {}'.format( 83 | dist.get_rank(), args.num_gpus)) 84 | 85 | # set the seed for generating random numbers 86 | torch.manual_seed(args.seed) 87 | if use_cuda: 88 | torch.cuda.manual_seed(args.seed) 89 | 90 | train_loader = _get_train_data_loader(args.batch_size, args.data_dir, is_distributed, **kwargs) 91 | test_loader = _get_test_data_loader(args.test_batch_size, args.data_dir, **kwargs) 92 | 93 | logger.debug("Processes {}/{} ({:.0f}%) of train data".format( 94 | len(train_loader.sampler), len(train_loader.dataset), 95 | 100. * len(train_loader.sampler) / len(train_loader.dataset) 96 | )) 97 | 98 | logger.debug("Processes {}/{} ({:.0f}%) of test data".format( 99 | len(test_loader.sampler), len(test_loader.dataset), 100 | 100. * len(test_loader.sampler) / len(test_loader.dataset) 101 | )) 102 | 103 | model = Net().to(device) 104 | if is_distributed and use_cuda: 105 | # multi-machine multi-gpu case 106 | model = torch.nn.parallel.DistributedDataParallel(model) 107 | else: 108 | # single-machine multi-gpu case or single-machine or multi-machine cpu case 109 | model = torch.nn.DataParallel(model) 110 | 111 | optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) 112 | 113 | for epoch in range(1, args.epochs + 1): 114 | model.train() 115 | for batch_idx, (data, target) in enumerate(train_loader, 1): 116 | data, target = data.to(device), target.to(device) 117 | optimizer.zero_grad() 118 | output = model(data) 119 | loss = F.nll_loss(output, target) 120 | loss.backward() 121 | if is_distributed and not use_cuda: 122 | # average gradients manually for multi-machine cpu case only 123 | _average_gradients(model) 124 | optimizer.step() 125 | if batch_idx % args.log_interval == 0: 126 | logger.info('Train Epoch: {} [{}/{} ({:.0f}%)] Loss: {:.6f}'.format( 127 | epoch, batch_idx * len(data), len(train_loader.sampler), 128 | 100. * batch_idx / len(train_loader), loss.item())) 129 | test(model, test_loader, device) 130 | save_model(model, args.model_dir) 131 | 132 | 133 | def test(model, test_loader, device): 134 | model.eval() 135 | test_loss = 0 136 | correct = 0 137 | with torch.no_grad(): 138 | for data, target in test_loader: 139 | data, target = data.to(device), target.to(device) 140 | output = model(data) 141 | test_loss += F.nll_loss(output, target, size_average=False).item() # sum up batch loss 142 | pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability 143 | correct += pred.eq(target.view_as(pred)).sum().item() 144 | 145 | test_loss /= len(test_loader.dataset) 146 | logger.info('Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( 147 | test_loss, correct, len(test_loader.dataset), 148 | 100. * correct / len(test_loader.dataset))) 149 | 150 | 151 | def model_fn(model_dir): 152 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 153 | model = torch.nn.DataParallel(Net()) 154 | with open(os.path.join(model_dir, 'model.pth'), 'rb') as f: 155 | model.load_state_dict(torch.load(f)) 156 | return model.to(device) 157 | 158 | 159 | def save_model(model, model_dir): 160 | logger.info("Saving the model.") 161 | path = os.path.join(model_dir, 'model.pth') 162 | # recommended way from http://pytorch.org/docs/master/notes/serialization.html 163 | torch.save(model.cpu().state_dict(), path) 164 | 165 | 166 | if __name__ == '__main__': 167 | parser = argparse.ArgumentParser() 168 | 169 | # Data and model checkpoints directories 170 | parser.add_argument('--batch-size', type=int, default=64, metavar='N', 171 | help='input batch size for training (default: 64)') 172 | parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', 173 | help='input batch size for testing (default: 1000)') 174 | parser.add_argument('--epochs', type=int, default=10, metavar='N', 175 | help='number of epochs to train (default: 10)') 176 | parser.add_argument('--lr', type=float, default=0.01, metavar='LR', 177 | help='learning rate (default: 0.01)') 178 | parser.add_argument('--momentum', type=float, default=0.5, metavar='M', 179 | help='SGD momentum (default: 0.5)') 180 | parser.add_argument('--seed', type=int, default=1, metavar='S', 181 | help='random seed (default: 1)') 182 | parser.add_argument('--log-interval', type=int, default=100, metavar='N', 183 | help='how many batches to wait before logging training status') 184 | parser.add_argument('--backend', type=str, default=None, 185 | help='backend for distributed training (tcp, gloo on cpu and gloo, nccl on gpu)') 186 | 187 | # Container environment 188 | parser.add_argument('--hosts', type=list, default=json.loads(os.environ['SM_HOSTS'])) 189 | parser.add_argument('--current-host', type=str, default=os.environ['SM_CURRENT_HOST']) 190 | parser.add_argument('--model-dir', type=str, default=os.environ['SM_MODEL_DIR']) 191 | parser.add_argument('--data-dir', type=str, default=os.environ['SM_CHANNEL_TRAINING']) 192 | parser.add_argument('--num-gpus', type=int, default=os.environ['SM_NUM_GPUS']) 193 | 194 | train(parser.parse_args()) 195 | -------------------------------------------------------------------------------- /mlops/sagemaker-pipelines/sagemaker-pipelines-sample/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | 132 | DKD2e_data_sets.zip 133 | Data sets/* 134 | raw_data/* 135 | -------------------------------------------------------------------------------- /mlops/sagemaker-pipelines/sagemaker-pipelines-sample/README.md: -------------------------------------------------------------------------------- 1 | # sagemaker-pipelines-sample 2 | 3 | ※ SageMaker **Studio** Notebook を前提としており、カーネルは Python3(Data Science) をご利用ください。 4 | ※ confirms this notebook works under data science kernel on SageMaker **Studio** notebook 5 | 6 | SageMaker Pipelines のサンプルコードです。 7 | [sagemaker-pipelines-sample.ipynb](./sagemaker-pipelines-sample.ipynb)を開いてご利用ください。 8 | 9 | SageMaker Pipelines の一通りの機能を利用することができます。 10 | -------------------------------------------------------------------------------- /mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/1.png -------------------------------------------------------------------------------- /mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/2.png -------------------------------------------------------------------------------- /mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/3.png -------------------------------------------------------------------------------- /mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/4.png -------------------------------------------------------------------------------- /mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/5.png -------------------------------------------------------------------------------- /mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/6.png -------------------------------------------------------------------------------- /mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/mlops/sagemaker-pipelines/sagemaker-pipelines-sample/media/7.png -------------------------------------------------------------------------------- /mlops/sagemaker-pipelines/sagemaker-pipelines-sample/postprocess/postprocess.py: -------------------------------------------------------------------------------- 1 | import json, pickle, tarfile, os, argparse 2 | import numpy as np 3 | import xgboost 4 | 5 | from sklearn.metrics import roc_auc_score 6 | 7 | def arg_parse(): 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument('--thistime-train-model-dir', type=str, default=None) 10 | parser.add_argument('--lasttime-train-model-dir', type=str, default=None) 11 | parser.add_argument('--input-data-dir', type=str, default=None) 12 | parser.add_argument('--output-dir', type=str, default=None) 13 | parser.add_argument('--output-file', type=str, default='evaluation.json') 14 | 15 | args, _ = parser.parse_known_args() 16 | return args 17 | 18 | def predict(input_model_dir:str, test_x): 19 | model_path = os.path.join(input_model_dir, 'model.tar.gz') 20 | with tarfile.open(model_path) as tar: 21 | 22 | def is_within_directory(directory, target): 23 | 24 | abs_directory = os.path.abspath(directory) 25 | abs_target = os.path.abspath(target) 26 | 27 | prefix = os.path.commonprefix([abs_directory, abs_target]) 28 | 29 | return prefix == abs_directory 30 | 31 | def safe_extract(tar, path=".", members=None, *, numeric_owner=False): 32 | 33 | for member in tar.getmembers(): 34 | member_path = os.path.join(path, member.name) 35 | if not is_within_directory(path, member_path): 36 | raise Exception("Attempted Path Traversal in Tar File") 37 | 38 | tar.extractall(path, members, numeric_owner=numeric_owner) 39 | 40 | 41 | safe_extract(tar, path=".") 42 | model = pickle.load(open("xgboost-model", "rb")) 43 | pred_y = model.predict(test_x) 44 | return pred_y 45 | 46 | if __name__ == "__main__": 47 | args = arg_parse() 48 | 49 | test_csv_path = os.path.join(args.input_data_dir,'test.csv') 50 | test = np.loadtxt(test_csv_path, delimiter=',') 51 | test_x = xgboost.DMatrix(test[:,1:]) 52 | answer_y = test[:,0] 53 | 54 | thistime_train_model_pred_y = predict(args.thistime_train_model_dir, test_x) 55 | lasttime_train_model_pred_y = predict(args.lasttime_train_model_dir, test_x) 56 | 57 | thistime_train_model_auc = roc_auc_score(answer_y,thistime_train_model_pred_y) 58 | lasttime_train_model_auc = roc_auc_score(answer_y,lasttime_train_model_pred_y) 59 | 60 | 61 | report_dict = { 62 | 'classification_metrics': { 63 | 'thistime_train_model_auc': { 64 | 'value': thistime_train_model_auc, 65 | }, 66 | 'lasttime_train_model_auc': { 67 | 'value': lasttime_train_model_auc, 68 | }, 69 | 'model_change':1 if thistime_train_model_auc > lasttime_train_model_auc else 0 70 | }, 71 | 'detail_result':{ 72 | 'thistime_train_model_pred_y': thistime_train_model_pred_y.tolist(), 73 | 'lasttime_train_model_pred_y': lasttime_train_model_pred_y.tolist(), 74 | 'answer_y': answer_y.tolist() 75 | } 76 | } 77 | print(report_dict) 78 | 79 | eval_result_path = os.path.join(args.output_dir, args.output_file) 80 | print(f'output save : {eval_result_path}') 81 | with open(eval_result_path, "w") as f: 82 | f.write(json.dumps(report_dict)) 83 | print('post process done.') 84 | exit() 85 | -------------------------------------------------------------------------------- /mlops/sagemaker-pipelines/sagemaker-pipelines-sample/preprocess/preprocess.py: -------------------------------------------------------------------------------- 1 | # churn データの前処理スクリプト 2 | # 前処理内容は下記と同じ 3 | # https://github.com/aws-samples/amazon-sagemaker-examples-jp/blob/master/xgboost_customer_churn/xgboost_customer_churn.ipynb 4 | # State, Area Code, VMail Plan などをダミー変数化するところは、全ての週が分けたデータに入っているか不透明だったため、ハードコーディングでダミー変数化する 5 | 6 | import pandas as pd 7 | import numpy as np 8 | import argparse 9 | from sklearn.model_selection import train_test_split 10 | import os 11 | from glob import glob 12 | 13 | def arg_parse(): 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument('--raw-data-input-dir', type=str, default=None) 16 | parser.add_argument('--pre-processed-train-data-input-dir', type=str, default=None) 17 | parser.add_argument('--pre-processed-valid-data-input-dir', type=str, default=None) 18 | parser.add_argument('--pre-processed-test-data-input-dir', type=str, default=None) 19 | parser.add_argument('--train-output-dir', type=str, default=None) 20 | parser.add_argument('--valid-output-dir', type=str, default=None) 21 | parser.add_argument('--test-output-dir', type=str, default=None) 22 | parser.add_argument('--merge' , action='store_true') 23 | args, _ = parser.parse_known_args() 24 | return args 25 | 26 | if __name__=='__main__': 27 | args = arg_parse() 28 | # csv ファイルの読み込み 29 | # 複数ファイルが raw_input_dir に配置されたら全て縦に concat する 30 | df_list = [] 31 | for csv_file in glob(args.raw_data_input_dir+'/*.csv'): 32 | tmp_df = pd.read_csv(csv_file) 33 | df_list.append(tmp_df) 34 | df = pd.concat(df_list,axis=0) 35 | 36 | # 不要カラムの除去 37 | df = df.drop(['Phone','Day Charge', 'Eve Charge', 'Night Charge', 'Intl Charge'], axis=1) 38 | 39 | # Area Code のダミー変数化 40 | df['Area Code'] = df['Area Code'].astype(object) 41 | area_code_list = ['415', '408', '510'] 42 | for area_code in area_code_list: 43 | df[area_code] = df['Area Code'].map(lambda x: 1 if x==area_code else 0) 44 | df = df.drop(['Area Code'],axis=1) 45 | 46 | # State のダミー変数化 47 | states_list = ['KS', 'OH', 'NJ', 'OK', 'AL', 'MA', 'MO', 'LA', 'WV', 'IN', 'RI', 'IA', 'MT', 'NY', 'ID', 'VT', 'VA', 'TX', 'FL', 'CO', 'AZ', 'SC', 'NE', 'WY', 'HI', 'IL', 'NH', 'GA', 'AK', 'MD', 'AR', 'WI', 'OR', 'MI', 'DE', 'UT', 'CA', 'MN', 'SD', 'NC', 'WA', 'NM', 'NV', 'DC', 'KY', 'ME', 'MS', 'TN', 'PA', 'CT', 'ND'] 48 | for state in states_list: 49 | df[state] = df['State'].map(lambda x: 1 if x==state else 0) 50 | df = df.drop(['State'],axis=1) 51 | 52 | # Int'l Plan のダミー変数化 53 | intl_plan_list = ['yes','no'] 54 | for intl_plan in intl_plan_list: 55 | df[f'intl_plan{intl_plan}'] = df["Int'l Plan"].map(lambda x: 1 if x==intl_plan else 0) 56 | df = df.drop(["Int'l Plan"],axis=1) 57 | 58 | # Vmail Plan のダミー変数化 59 | vmail_plan_list = ['yes','no'] 60 | for vmail_plan in vmail_plan_list: 61 | df[f'vmail_plan{vmail_plan}'] = df["VMail Plan"].map(lambda x: 1 if x==vmail_plan else 0) 62 | df = df.drop(["VMail Plan"],axis=1) 63 | 64 | # churn? の 1/0 化 (xgboost のデータ入力仕様) 65 | df['y'] = df['Churn?'].map(lambda x: 1 if x=='True.' else 0) 66 | df = df.drop(['Churn?'], axis=1) 67 | 68 | # 予測したいカラムを最初に持ってくる (xgboost のデータ入力仕様) 69 | df = pd.concat([df.iloc[:,-1],df.iloc[:,:-1]],axis=1) 70 | train_df, valid_df, test_df = np.split(df.sample(frac=1, random_state=42), [int(0.7 * len(df)), int(0.9 * len(df))]) 71 | 72 | # 過去の前処理済データをマージ 73 | if args.merge: 74 | # train 75 | lasttime_train_input_path = os.path.join(args.pre_processed_train_data_input_dir, 'train.csv') 76 | lasttime_train_df = pd.read_csv(lasttime_train_input_path, header=None, names=df.columns.tolist()) 77 | train_df = pd.concat([train_df,lasttime_train_df],axis=0) 78 | # valid 79 | lasttime_valid_input_path = os.path.join(args.pre_processed_valid_data_input_dir, 'valid.csv') 80 | lasttime_valid_df = pd.read_csv(lasttime_valid_input_path, header=None, names=df.columns.tolist()) 81 | valid_df = pd.concat([valid_df,lasttime_valid_df],axis=0) 82 | # test 83 | lasttime_test_input_path = os.path.join(args.pre_processed_test_data_input_dir, 'test.csv') 84 | lasttime_test_df = pd.read_csv(lasttime_test_input_path, header=None, names=df.columns.tolist()) 85 | test_df = pd.concat([test_df,lasttime_test_df],axis=0) 86 | 87 | train_output_path = os.path.join(args.train_output_dir, 'train.csv') 88 | valid_output_path = os.path.join(args.valid_output_dir, 'valid.csv') 89 | test_output_path = os.path.join(args.test_output_dir, 'test.csv') 90 | 91 | train_df.to_csv(train_output_path, header=False, index=False) 92 | valid_df.to_csv(valid_output_path, header=False, index=False) 93 | test_df.to_csv(test_output_path, header=False, index=False) 94 | 95 | exit() -------------------------------------------------------------------------------- /model-deployment/pytorch-deeplab.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# PyTorch DeepLab の学習済みモデルを SageMaker でデプロイする\n", 8 | "\n", 9 | "PyTorch Hub で公開されている [DeepLab V3 のモデル](https://pytorch.org/hub/pytorch_vision_deeplabv3_resnet101/)をダウンロードしてデプロイします。このノートブックでは、モデルのSageMaker への持ち込み方法を知るため、以下のステップでデプロイします。\n", 10 | "\n", 11 | "1. PyTorch Hub からモデルをダウンロードし、S3 に保存します。\n", 12 | "1. ダウンロードしたモデルで推論を行うためのコードを作成します。\n", 13 | "1. S3 に保存したモデルを指定して、SageMaker にデプロイします。\n", 14 | "\n", 15 | "実際には、推論コードの中でPyTorch Hub からモデルをダウンロードできるため、1をスキップする方法も可能です。\n", 16 | "\n", 17 | "\n", 18 | "## 1. PyTorch Hub からのモデルダウンロード\n", 19 | "\n", 20 | "`torch.hub`でモデルをダウンロードし、パラメータの情報のみ保存します。保存したファイルは `tar.gz` の形式にして S3 にアップロードします。" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "import torch\n", 30 | "import os\n", 31 | "os.makedirs('model',exist_ok=True)\n", 32 | "model = torch.hub.load('pytorch/vision:v0.10.0', 'deeplabv3_resnet101', pretrained=True, progress=False)\n", 33 | "path = './model/model.pth'\n", 34 | "torch.save(model.cpu().state_dict(), path)" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "!tar cvzf model.tar.gz -C ./model ." 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [ 52 | "import sagemaker\n", 53 | "sagemaker_session = sagemaker.Session()\n", 54 | "model_path = sagemaker_session.upload_data(\"model.tar.gz\", key_prefix =\"pytorch_deeplab_model\")" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "model_path" 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "metadata": {}, 69 | "source": [ 70 | "## 2. 推論コードの作成\n", 71 | "\n", 72 | "アップロードしたモデルを読み込んで推論を実行するコードを作成します。モデルの読み込みは `model_fn` で、推論の実行は `transform_fn`で実装します。\n", 73 | "PyTorch ではモデルのパラメータ以外にシンボルの情報が必要なので、PyTorch Hub から呼び出して利用します。各関数の実装は[公式の利用方法](https://pytorch.org/hub/pytorch_vision_deeplabv3_resnet101/)を参考にしています。" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": null, 79 | "metadata": {}, 80 | "outputs": [], 81 | "source": [ 82 | "%%writefile deploy.py\n", 83 | "\n", 84 | "from io import BytesIO\n", 85 | "import json\n", 86 | "import numpy as np\n", 87 | "import os\n", 88 | "import torch\n", 89 | "import torch.nn as nn\n", 90 | "import torch.nn.functional as F\n", 91 | "\n", 92 | "from PIL import Image\n", 93 | "from torchvision import transforms \n", 94 | "\n", 95 | "def model_fn(model_dir):\n", 96 | " model = torch.hub.load('pytorch/vision:v0.10.0', 'deeplabv3_resnet101', pretrained=False, progress=False)\n", 97 | " with open(os.path.join(model_dir, \"model.pth\"), \"rb\") as f:\n", 98 | " model.load_state_dict(torch.load(f), strict=False)\n", 99 | " model.eval() # for inference\n", 100 | " return model\n", 101 | "\n", 102 | "def transform_fn(model, request_body, request_content_type, response_content_type):\n", 103 | " \n", 104 | " input_data = np.load(BytesIO(request_body))\n", 105 | " preprocess = transforms.Compose([\n", 106 | " transforms.ToTensor(),\n", 107 | " transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n", 108 | " ])\n", 109 | " \n", 110 | " input_tensor = preprocess(input_data)\n", 111 | " input_batch = input_tensor.unsqueeze(0)\n", 112 | " prediction = model(input_batch)\n", 113 | " return json.dumps(prediction['out'].tolist())" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": {}, 119 | "source": [ 120 | "## 3. デプロイと推論" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": null, 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "from sagemaker.pytorch.model import PyTorchModel\n", 130 | "\n", 131 | "deeplab_model=PyTorchModel(model_data=model_path, \n", 132 | " role=sagemaker.get_execution_role(), \n", 133 | " entry_point='deploy.py', \n", 134 | " framework_version='1.8.1',\n", 135 | " py_version='py3')" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": null, 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [ 144 | "predictor=deeplab_model.deploy(instance_type='ml.m4.xlarge', initial_instance_count=1)" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": null, 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "!wget https://github.com/pytorch/hub/raw/master/images/dog.jpg" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": null, 159 | "metadata": {}, 160 | "outputs": [], 161 | "source": [ 162 | "import matplotlib.pyplot as plt\n", 163 | "from PIL import Image\n", 164 | "import numpy as np\n", 165 | "from torchvision import transforms\n", 166 | "from skimage.segmentation import mark_boundaries\n", 167 | "\n", 168 | "input_image = Image.open(\"dog.jpg\").convert('RGB')\n", 169 | "w, h = input_image.size\n", 170 | "\n", 171 | "input_image = input_image.resize((150, 100))\n", 172 | "np_input_image = np.array(input_image)\n", 173 | "predictor.deserializer = sagemaker.deserializers.JSONDeserializer()\n", 174 | "predictions = predictor.predict(np_input_image)\n", 175 | "\n", 176 | "# create a color pallette, selecting a color for each class\n", 177 | "palette = torch.tensor([2 ** 25 - 1, 2 ** 15 - 1, 2 ** 21 - 1])\n", 178 | "colors = torch.as_tensor([i for i in range(21)])[:, None] * palette\n", 179 | "colors = (colors % 255).numpy().astype(\"uint8\")\n", 180 | "label_map = np.array(predictions[0]).argmax(0)\n", 181 | "\n", 182 | "# # plot the semantic segmentation predictions of 21 classes in each color\n", 183 | "r = Image.fromarray(label_map.astype(np.uint8))\n", 184 | "r.putpalette(colors)\n", 185 | "r = Image.blend(r.convert('RGBA'), input_image.convert('RGBA'), 0.5) \n", 186 | "\n", 187 | "plt.rcParams['figure.figsize'] = [12, 8]\n", 188 | "plt.imshow(r)" 189 | ] 190 | }, 191 | { 192 | "cell_type": "markdown", 193 | "metadata": {}, 194 | "source": [ 195 | "最後に不要なエンドポイントを削除します。" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": null, 201 | "metadata": {}, 202 | "outputs": [], 203 | "source": [ 204 | "predictor.delete_endpoint()" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": null, 210 | "metadata": {}, 211 | "outputs": [], 212 | "source": [] 213 | } 214 | ], 215 | "metadata": { 216 | "instance_type": "ml.t3.medium", 217 | "kernelspec": { 218 | "display_name": "Python 3 (PyTorch 1.6 Python 3.6 CPU Optimized)", 219 | "language": "python", 220 | "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/pytorch-1.6-cpu-py36-ubuntu16.04-v1" 221 | }, 222 | "language_info": { 223 | "codemirror_mode": { 224 | "name": "ipython", 225 | "version": 3 226 | }, 227 | "file_extension": ".py", 228 | "mimetype": "text/x-python", 229 | "name": "python", 230 | "nbconvert_exporter": "python", 231 | "pygments_lexer": "ipython3", 232 | "version": "3.6.13" 233 | } 234 | }, 235 | "nbformat": 4, 236 | "nbformat_minor": 4 237 | } 238 | -------------------------------------------------------------------------------- /nlp_amazon_review/GluonNLP_BERT/src/bert/data/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | # pylint: disable=wildcard-import 21 | """BERT data.""" 22 | from . import transform 23 | -------------------------------------------------------------------------------- /nlp_amazon_review/GluonNLP_BERT/src/bert/data/transform.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors and DMLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """BERT dataset transform.""" 16 | from __future__ import absolute_import 17 | 18 | __all__ = ['BERTDatasetTransform'] 19 | 20 | import numpy as np 21 | 22 | import os 23 | from gluonnlp.data import BERTSentenceTransform 24 | 25 | class BERTDatasetTransform: 26 | """Dataset transformation for BERT-style sentence classification or regression. 27 | 28 | Parameters 29 | ---------- 30 | tokenizer : BERTTokenizer. 31 | Tokenizer for the sentences. 32 | max_seq_length : int. 33 | Maximum sequence length of the sentences. 34 | labels : list of int , float or None. defaults None 35 | List of all label ids for the classification task and regressing task. 36 | If labels is None, the default task is regression 37 | pad : bool, default True 38 | Whether to pad the sentences to maximum length. 39 | pair : bool, default True 40 | Whether to transform sentences or sentence pairs. 41 | label_dtype: int32 or float32, default float32 42 | label_dtype = int32 for classification task 43 | label_dtype = float32 for regression task 44 | """ 45 | 46 | def __init__(self, 47 | tokenizer, 48 | max_seq_length, 49 | class_labels=None, 50 | label_alias=None, 51 | pad=True, 52 | pair=True, 53 | has_label=True): 54 | self.class_labels = class_labels 55 | self.has_label = has_label 56 | self._label_dtype = 'int32' if class_labels else 'float32' 57 | if has_label and class_labels: 58 | self._label_map = {} 59 | for (i, label) in enumerate(class_labels): 60 | self._label_map[label] = i 61 | if label_alias: 62 | for key in label_alias: 63 | self._label_map[key] = self._label_map[label_alias[key]] 64 | self._bert_xform = BERTSentenceTransform( 65 | tokenizer, max_seq_length, pad=pad, pair=pair) 66 | 67 | def __call__(self, line): 68 | """Perform transformation for sequence pairs or single sequences. 69 | 70 | The transformation is processed in the following steps: 71 | - tokenize the input sequences 72 | - insert [CLS], [SEP] as necessary 73 | - generate type ids to indicate whether a token belongs to the first 74 | sequence or the second sequence. 75 | - generate valid length 76 | 77 | For sequence pairs, the input is a tuple of 3 strings: 78 | text_a, text_b and label. 79 | 80 | Inputs: 81 | text_a: 'is this jacksonville ?' 82 | text_b: 'no it is not' 83 | label: '0' 84 | Tokenization: 85 | text_a: 'is this jack ##son ##ville ?' 86 | text_b: 'no it is not .' 87 | Processed: 88 | tokens: '[CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]' 89 | type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 90 | valid_length: 14 91 | label: 0 92 | 93 | For single sequences, the input is a tuple of 2 strings: text_a and label. 94 | Inputs: 95 | text_a: 'the dog is hairy .' 96 | label: '1' 97 | Tokenization: 98 | text_a: 'the dog is hairy .' 99 | Processed: 100 | text_a: '[CLS] the dog is hairy . [SEP]' 101 | type_ids: 0 0 0 0 0 0 0 102 | valid_length: 7 103 | label: 1 104 | 105 | Parameters 106 | ---------- 107 | line: tuple of str 108 | Input strings. For sequence pairs, the input is a tuple of 3 strings: 109 | (text_a, text_b, label). For single sequences, the input is a tuple 110 | of 2 strings: (text_a, label). 111 | 112 | Returns 113 | ------- 114 | np.array: input token ids in 'int32', shape (batch_size, seq_length) 115 | np.array: valid length in 'int32', shape (batch_size,) 116 | np.array: input token type ids in 'int32', shape (batch_size, seq_length) 117 | np.array: classification task: label id in 'int32', shape (batch_size, 1), 118 | regression task: label in 'float32', shape (batch_size, 1) 119 | """ 120 | if self.has_label: 121 | input_ids, valid_length, segment_ids = self._bert_xform(line[:-1]) 122 | label = line[-1] 123 | # map to int if class labels are available 124 | if self.class_labels: 125 | label = self._label_map[label] 126 | label = np.array([label], dtype=self._label_dtype) 127 | return input_ids, valid_length, segment_ids, label 128 | else: 129 | return self._bert_xform(line) 130 | -------------------------------------------------------------------------------- /nlp_amazon_review/GluonNLP_BERT/src/bert/model/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | # pylint: disable=wildcard-import 21 | """BERT model.""" 22 | from . import classification 23 | -------------------------------------------------------------------------------- /nlp_amazon_review/GluonNLP_BERT/src/bert/model/classification.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | """BERT models.""" 20 | 21 | __all__ = ['BERTClassifier', 'BERTRegression'] 22 | 23 | from mxnet.gluon import HybridBlock 24 | from mxnet.gluon import nn 25 | 26 | class BERTRegression(HybridBlock): 27 | """Model for sentence (pair) regression task with BERT. 28 | 29 | The model feeds token ids and token type ids into BERT to get the 30 | pooled BERT sequence representation, then apply a Dense layer for 31 | regression. 32 | 33 | Parameters 34 | ---------- 35 | bert: BERTModel 36 | Bidirectional encoder with transformer. 37 | dropout : float or None, default 0.0. 38 | Dropout probability for the bert output. 39 | prefix : str or None 40 | See document of `mx.gluon.Block`. 41 | params : ParameterDict or None 42 | See document of `mx.gluon.Block`. 43 | """ 44 | 45 | def __init__(self, bert, dropout=0.0, prefix=None, params=None): 46 | super(BERTRegression, self).__init__(prefix=prefix, params=params) 47 | self.bert = bert 48 | with self.name_scope(): 49 | self.regression = nn.HybridSequential(prefix=prefix) 50 | if dropout: 51 | self.regression.add(nn.Dropout(rate=dropout)) 52 | self.regression.add(nn.Dense(1)) 53 | 54 | def hybrid_forward(self, F, inputs, token_types, valid_length=None): 55 | # pylint: disable=arguments-differ 56 | """Generate the unnormalized score for the given the input sequences. 57 | 58 | Parameters 59 | ---------- 60 | inputs : NDArray, shape (batch_size, seq_length) 61 | Input words for the sequences. 62 | token_types : NDArray, shape (batch_size, seq_length) 63 | Token types for the sequences, used to indicate whether the word belongs to the 64 | first sentence or the second one. 65 | valid_length : NDArray or None, shape (batch_size) 66 | Valid length of the sequence. This is used to mask the padded tokens. 67 | 68 | Returns 69 | ------- 70 | outputs : NDArray 71 | Shape (batch_size, num_classes) 72 | """ 73 | _, pooler_out = self.bert(inputs, token_types, valid_length) 74 | return self.regression(pooler_out) 75 | 76 | 77 | class BERTClassifier(HybridBlock): 78 | """Model for sentence (pair) classification task with BERT. 79 | 80 | The model feeds token ids and token type ids into BERT to get the 81 | pooled BERT sequence representation, then apply a Dense layer for 82 | classification. 83 | 84 | Parameters 85 | ---------- 86 | bert: BERTModel 87 | Bidirectional encoder with transformer. 88 | num_classes : int, default is 2 89 | The number of target classes. 90 | dropout : float or None, default 0.0. 91 | Dropout probability for the bert output. 92 | prefix : str or None 93 | See document of `mx.gluon.Block`. 94 | params : ParameterDict or None 95 | See document of `mx.gluon.Block`. 96 | """ 97 | 98 | def __init__(self, 99 | bert, 100 | num_classes=2, 101 | dropout=0.0, 102 | prefix=None, 103 | params=None): 104 | super(BERTClassifier, self).__init__(prefix=prefix, params=params) 105 | self.bert = bert 106 | with self.name_scope(): 107 | self.classifier = nn.HybridSequential(prefix=prefix) 108 | if dropout: 109 | self.classifier.add(nn.Dropout(rate=dropout)) 110 | self.classifier.add(nn.Dense(units=num_classes)) 111 | 112 | def hybrid_forward(self, F, inputs, token_types, valid_length=None): 113 | # pylint: disable=arguments-differ 114 | """Generate the unnormalized score for the given the input sequences. 115 | 116 | Parameters 117 | ---------- 118 | inputs : NDArray, shape (batch_size, seq_length) 119 | Input words for the sequences. 120 | token_types : NDArray, shape (batch_size, seq_length) 121 | Token types for the sequences, used to indicate whether the word belongs to the 122 | first sentence or the second one. 123 | valid_length : NDArray or None, shape (batch_size) 124 | Valid length of the sequence. This is used to mask the padded tokens. 125 | 126 | Returns 127 | ------- 128 | outputs : NDArray 129 | Shape (batch_size, num_classes) 130 | """ 131 | _, pooler_out = self.bert(inputs, token_types, valid_length) 132 | return self.classifier(pooler_out) 133 | -------------------------------------------------------------------------------- /nlp_amazon_review/GluonNLP_BERT/src/train_and_deploy.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import mxnet as mx 3 | import json 4 | import sys 5 | import os 6 | import gluonnlp as nlp 7 | from mxnet import gluon, autograd 8 | from mxnet.gluon import nn 9 | from bert import data, model 10 | import sys 11 | 12 | import logging 13 | logging.basicConfig(level=logging.INFO) 14 | 15 | 16 | try: 17 | num_cpus = int(os.environ['SM_NUM_CPUS']) 18 | num_gpus = int(os.environ['SM_NUM_GPUS']) 19 | except KeyError: 20 | num_gpus = 0 21 | 22 | ctx = mx.gpu() if num_gpus > 0 else mx.cpu() 23 | bert_base, vocabulary = nlp.model.get_model('bert_12_768_12', 24 | dataset_name='wiki_multilingual_uncased', 25 | pretrained=True, ctx=ctx, use_pooler=True, 26 | use_decoder=False, use_classifier=False) 27 | 28 | # The maximum length of an input sequence 29 | max_len = 128 30 | 31 | 32 | if __name__ == '__main__': 33 | 34 | 35 | # Receive hyperparameters passed via create-training-job API 36 | parser = argparse.ArgumentParser() 37 | 38 | parser.add_argument('--batch-size', type=int, default=32) 39 | parser.add_argument('--epochs', type=int, default=1) 40 | parser.add_argument('--learning-rate', type=float, default=5e-6) 41 | parser.add_argument('--log-interval', type=float, default=4) 42 | 43 | parser.add_argument('--model-dir', type=str, default=os.environ['SM_MODEL_DIR']) 44 | parser.add_argument('--train', type=str, default=os.environ['SM_CHANNEL_TRAIN']) 45 | parser.add_argument('--current-host', type=str, default=os.environ['SM_CURRENT_HOST']) 46 | parser.add_argument('--hosts', type=list, default=json.loads(os.environ['SM_HOSTS'])) 47 | 48 | args = parser.parse_args() 49 | 50 | # Set hyperparameters after parsing the arguments 51 | batch_size = args.batch_size 52 | lr = args.learning_rate 53 | log_interval = args.log_interval 54 | num_epochs = args.epochs 55 | current_host = args.current_host 56 | hosts = args.hosts 57 | model_dir = args.model_dir 58 | training_dir = args.train 59 | 60 | # Setting for distributed training 61 | if len(hosts) == 1: 62 | kvstore = 'device' if num_gpus > 0 else 'local' 63 | else: 64 | kvstore = 'dist_device_sync' if num_gpus > 0 else 'dist_sync' 65 | 66 | 67 | 68 | # Load pre-trained bert model and vocabulary 69 | # and define the classification model (add classifier and loss function on bert) 70 | bert_classifier = model.classification.BERTClassifier(bert_base, num_classes=2, dropout=0.1) 71 | # only need to initialize the classifier layer. 72 | bert_classifier.classifier.initialize(init=mx.init.Normal(0.02), ctx=ctx) 73 | bert_classifier.hybridize(static_alloc=True) 74 | loss_function = mx.gluon.loss.SoftmaxCELoss() 75 | loss_function.hybridize(static_alloc=True) 76 | 77 | # Data loading 78 | field_separator = nlp.data.Splitter('\t') 79 | data_train_raw = nlp.data.TSVDataset(filename=os.path.join(training_dir,'train.tsv'), 80 | field_separator=field_separator) 81 | 82 | # Use the vocabulary from pre-trained model for tokenization 83 | bert_tokenizer = nlp.data.BERTTokenizer(vocabulary, lower=True) 84 | 85 | # The labels for the two classes [(0 : Negative) or (1 : Poistive)] 86 | all_labels = ["0", "1"] 87 | 88 | # whether to transform the data as sentence pairs. 89 | # for single sentence classification, set pair=False 90 | # for regression task, set class_labels=None 91 | # for inference without label available, set has_label=False 92 | pair = False 93 | transform = data.transform.BERTDatasetTransform(bert_tokenizer, max_len, 94 | class_labels=all_labels, 95 | has_label=True, 96 | pad=True, 97 | pair=pair) 98 | data_train = data_train_raw.transform(transform) 99 | 100 | 101 | # The FixedBucketSampler and the DataLoader for making the mini-batches 102 | train_sampler = nlp.data.FixedBucketSampler(lengths=[int(item[1]) for item in data_train], 103 | batch_size=batch_size, 104 | shuffle=True) 105 | bert_dataloader = mx.gluon.data.DataLoader(data_train, batch_sampler=train_sampler) 106 | 107 | trainer = mx.gluon.Trainer(bert_classifier.collect_params(), 'adam', 108 | {'learning_rate': lr, 'epsilon': 1e-9}, 109 | kvstore=kvstore) 110 | 111 | # Collect all differentiable parameters 112 | # `grad_req == 'null'` indicates no gradients are calculated (e.g. constant parameters) 113 | # The gradients for these params are clipped later 114 | params = [p for p in bert_classifier.collect_params().values() if p.grad_req != 'null'] 115 | grad_clip = 1 116 | 117 | # For evaluation with accuracy 118 | metric = mx.metric.Accuracy() 119 | 120 | # Training loop 121 | for epoch_id in range(num_epochs): 122 | metric.reset() 123 | step_loss = 0 124 | for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(bert_dataloader): 125 | with mx.autograd.record(): 126 | 127 | # Load the data to the CPU or GPU 128 | token_ids = token_ids.as_in_context(ctx) 129 | valid_length = valid_length.as_in_context(ctx) 130 | segment_ids = segment_ids.as_in_context(ctx) 131 | label = label.as_in_context(ctx) 132 | 133 | # Forward computation 134 | # Loss is weighte by 10 for negaive (0) and 1 for positive(1) 135 | out = bert_classifier(token_ids, segment_ids, valid_length.astype('float32')) 136 | ls = loss_function(out, label).mean() 137 | 138 | # And backwards computation 139 | ls.backward() 140 | 141 | # Gradient clipping 142 | trainer.allreduce_grads() 143 | nlp.utils.clip_grad_global_norm(params, 1) 144 | trainer.update(1) 145 | 146 | step_loss += ls.asscalar() 147 | metric.update([label], [out]) 148 | 149 | # Printing vital information 150 | if (batch_id + 1) % (log_interval) == 0: 151 | print('[Epoch {} Batch {}/{}] loss={:.4f}, lr={:.7f}, acc={:.3f}' 152 | .format(epoch_id, batch_id + 1, len(bert_dataloader), 153 | step_loss / log_interval, 154 | trainer.learning_rate, metric.get()[1])) 155 | step_loss = 0 156 | 157 | 158 | if current_host == hosts[0]: 159 | bert_classifier.export('%s/model'% model_dir) 160 | 161 | 162 | def model_fn(model_dir): 163 | """ 164 | Load the gluon model. Called once when hosting service starts. 165 | 166 | :param: model_dir The directory where model files are stored. 167 | :return: a model (in this case a Gluon network) 168 | """ 169 | 170 | bert_tokenizer = nlp.data.BERTTokenizer(vocabulary, lower=True) 171 | bert_classifier = gluon.SymbolBlock.imports( 172 | '%s/model-symbol.json' % model_dir, 173 | ['data0', 'data1', 'data2'], 174 | '%s/model-0000.params' % model_dir, 175 | ) 176 | return {"net": bert_classifier, "tokenizer": bert_tokenizer} 177 | 178 | 179 | def transform_fn(net, data, input_content_type, output_content_type): 180 | """ 181 | Transform a request using the Gluon model. Called once per request. 182 | 183 | :param net: The Gluon model. 184 | :param data: The request payload. 185 | :param input_content_type: The request content type. 186 | :param output_content_type: The (desired) response content type. 187 | :return: response payload and content type. 188 | """ 189 | # we can use content types to vary input/output handling, but 190 | # here we just assume json for both 191 | bert_classifier = net["net"] 192 | bert_tokenizer = net["tokenizer"] 193 | 194 | # Assume one line of text 195 | parsed = json.loads(data) 196 | logging.info("Received_data: {}".format(parsed)) 197 | tokens = bert_tokenizer(parsed) 198 | logging.info("Tokens: {}".format(tokens)) 199 | token_ids = bert_tokenizer.convert_tokens_to_ids(tokens) 200 | valid_length = len(token_ids) 201 | segment_ids = mx.nd.zeros([1, valid_length]) 202 | 203 | output = bert_classifier(mx.nd.array([token_ids]), 204 | segment_ids, 205 | mx.nd.array([valid_length]).astype('float32')) 206 | response_body = json.dumps(output.asnumpy().tolist()[0]) 207 | return response_body, output_content_type 208 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "amazon-sagemaker-examples-jp", 3 | "lockfileVersion": 2, 4 | "requires": true, 5 | "packages": {} 6 | } 7 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /sagemaker-experiments/pytorch_mnist/figures/experiments-overview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/sagemaker-experiments/pytorch_mnist/figures/experiments-overview.jpg -------------------------------------------------------------------------------- /sagemaker-experiments/pytorch_mnist/src/mnist_deploy.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import os 3 | 4 | import sys 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | # Based on https://github.com/pytorch/examples/blob/master/mnist/main.py 11 | class Net(nn.Module): 12 | def __init__(self): 13 | super(Net, self).__init__() 14 | self.conv1 = nn.Conv2d(1, 10, kernel_size=5) 15 | self.conv2 = nn.Conv2d(10, 20, kernel_size=5) 16 | self.conv2_drop = nn.Dropout2d() 17 | self.fc1 = nn.Linear(320, 50) 18 | self.fc2 = nn.Linear(50, 10) 19 | 20 | def forward(self, x): 21 | x = F.relu(F.max_pool2d(self.conv1(x), 2)) 22 | x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2)) 23 | x = x.view(-1, 320) 24 | x = F.relu(self.fc1(x)) 25 | x = F.dropout(x, training=self.training) 26 | x = self.fc2(x) 27 | return F.log_softmax(x, dim=1) 28 | 29 | 30 | def model_fn(model_dir): 31 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 32 | model = torch.nn.DataParallel(Net()) 33 | with open(os.path.join(model_dir, 'model.pth'), 'rb') as f: 34 | model.load_state_dict(torch.load(f)) 35 | return model.to(device) 36 | -------------------------------------------------------------------------------- /sagemaker-experiments/pytorch_mnist/src/mnist_train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import logging 4 | import os 5 | import sys 6 | import torch 7 | import torch.distributed as dist 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | import torch.optim as optim 11 | import torch.utils.data 12 | import torch.utils.data.distributed 13 | from torchvision import datasets, transforms 14 | 15 | logger = logging.getLogger(__name__) 16 | logger.setLevel(logging.DEBUG) 17 | logger.addHandler(logging.StreamHandler(sys.stdout)) 18 | 19 | 20 | # Based on https://github.com/pytorch/examples/blob/master/mnist/main.py 21 | class Net(nn.Module): 22 | def __init__(self): 23 | super(Net, self).__init__() 24 | self.conv1 = nn.Conv2d(1, 10, kernel_size=5) 25 | self.conv2 = nn.Conv2d(10, 20, kernel_size=5) 26 | self.conv2_drop = nn.Dropout2d() 27 | self.fc1 = nn.Linear(320, 50) 28 | self.fc2 = nn.Linear(50, 10) 29 | 30 | def forward(self, x): 31 | x = F.relu(F.max_pool2d(self.conv1(x), 2)) 32 | x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2)) 33 | x = x.view(-1, 320) 34 | x = F.relu(self.fc1(x)) 35 | x = F.dropout(x, training=self.training) 36 | x = self.fc2(x) 37 | return F.log_softmax(x, dim=1) 38 | 39 | def _get_train_data_loader(batch_size, training_dir, is_distributed, **kwargs): 40 | logger.info("Get train data loader") 41 | train_tensor = torch.load(os.path.join(training_dir, 'training.pt')) 42 | dataset = torch.utils.data.TensorDataset(train_tensor[0], train_tensor[1]) 43 | train_sampler = torch.utils.data.distributed.DistributedSampler(dataset) if is_distributed else None 44 | return torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=train_sampler is None, 45 | sampler=train_sampler, **kwargs) 46 | 47 | def _get_test_data_loader(test_batch_size, training_dir, **kwargs): 48 | logger.info("Get test data loader") 49 | test_tensor = torch.load(os.path.join(training_dir, 'test.pt')) 50 | dataset = torch.utils.data.TensorDataset(test_tensor[0], test_tensor[1]) 51 | return torch.utils.data.DataLoader( 52 | dataset, 53 | batch_size=test_batch_size, 54 | shuffle=True, **kwargs) 55 | 56 | 57 | def _average_gradients(model): 58 | # Gradient averaging. 59 | size = float(dist.get_world_size()) 60 | for param in model.parameters(): 61 | dist.all_reduce(param.grad.data, op=dist.reduce_op.SUM) 62 | param.grad.data /= size 63 | 64 | 65 | def train(args): 66 | is_distributed = len(args.hosts) > 1 and args.backend is not None 67 | logger.debug("Distributed training - {}".format(is_distributed)) 68 | use_cuda = args.num_gpus > 0 69 | logger.debug("Number of gpus available - {}".format(args.num_gpus)) 70 | kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} 71 | device = torch.device("cuda" if use_cuda else "cpu") 72 | 73 | if is_distributed: 74 | # Initialize the distributed environment. 75 | world_size = len(args.hosts) 76 | os.environ['WORLD_SIZE'] = str(world_size) 77 | host_rank = args.hosts.index(args.current_host) 78 | os.environ['RANK'] = str(host_rank) 79 | dist.init_process_group(backend=args.backend, rank=host_rank, world_size=world_size) 80 | logger.info('Initialized the distributed environment: \'{}\' backend on {} nodes. '.format( 81 | args.backend, dist.get_world_size()) + 'Current host rank is {}. Number of gpus: {}'.format( 82 | dist.get_rank(), args.num_gpus)) 83 | 84 | # set the seed for generating random numbers 85 | torch.manual_seed(args.seed) 86 | if use_cuda: 87 | torch.cuda.manual_seed(args.seed) 88 | 89 | train_loader = _get_train_data_loader(args.batch_size, args.data_dir, is_distributed, **kwargs) 90 | test_loader = _get_test_data_loader(args.test_batch_size, args.data_dir, **kwargs) 91 | 92 | logger.debug("Processes {}/{} ({:.0f}%) of train data".format( 93 | len(train_loader.sampler), len(train_loader.dataset), 94 | 100. * len(train_loader.sampler) / len(train_loader.dataset) 95 | )) 96 | 97 | logger.debug("Processes {}/{} ({:.0f}%) of test data".format( 98 | len(test_loader.sampler), len(test_loader.dataset), 99 | 100. * len(test_loader.sampler) / len(test_loader.dataset) 100 | )) 101 | 102 | model = Net().to(device) 103 | if is_distributed and use_cuda: 104 | # multi-machine multi-gpu case 105 | model = torch.nn.parallel.DistributedDataParallel(model) 106 | else: 107 | # single-machine multi-gpu case or single-machine or multi-machine cpu case 108 | model = torch.nn.DataParallel(model) 109 | 110 | optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) 111 | 112 | for epoch in range(1, args.epochs + 1): 113 | model.train() 114 | for batch_idx, (data, target) in enumerate(train_loader, 1): 115 | data, target = data.to(device), target.to(device) 116 | optimizer.zero_grad() 117 | output = model(data) 118 | loss = F.nll_loss(output, target) 119 | loss.backward() 120 | if is_distributed and not use_cuda: 121 | # average gradients manually for multi-machine cpu case only 122 | _average_gradients(model) 123 | optimizer.step() 124 | if batch_idx % args.log_interval == 0: 125 | logger.info('Train Epoch: {} [{}/{} ({:.0f}%)] Loss: {:.6f}'.format( 126 | epoch, batch_idx * len(data), len(train_loader.sampler), 127 | 100. * batch_idx / len(train_loader), loss.item())) 128 | test(model, test_loader, device) 129 | save_model(model, args.model_dir) 130 | 131 | 132 | def test(model, test_loader, device): 133 | model.eval() 134 | test_loss = 0 135 | correct = 0 136 | with torch.no_grad(): 137 | for data, target in test_loader: 138 | data, target = data.to(device), target.to(device) 139 | output = model(data) 140 | test_loss += F.nll_loss(output, target, size_average=False).item() # sum up batch loss 141 | pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability 142 | correct += pred.eq(target.view_as(pred)).sum().item() 143 | 144 | test_loss /= len(test_loader.dataset) 145 | logger.info('Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( 146 | test_loss, correct, len(test_loader.dataset), 147 | 100. * correct / len(test_loader.dataset))) 148 | 149 | 150 | def model_fn(model_dir): 151 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 152 | model = torch.nn.DataParallel(Net()) 153 | with open(os.path.join(model_dir, 'model.pth'), 'rb') as f: 154 | model.load_state_dict(torch.load(f)) 155 | return model.to(device) 156 | 157 | 158 | def save_model(model, model_dir): 159 | logger.info("Saving the model.") 160 | path = os.path.join(model_dir, 'model.pth') 161 | # recommended way from http://pytorch.org/docs/master/notes/serialization.html 162 | torch.save(model.cpu().state_dict(), path) 163 | 164 | 165 | if __name__ == '__main__': 166 | parser = argparse.ArgumentParser() 167 | 168 | # Data and model checkpoints directories 169 | parser.add_argument('--batch-size', type=int, default=64, metavar='N', 170 | help='input batch size for training (default: 64)') 171 | parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', 172 | help='input batch size for testing (default: 1000)') 173 | parser.add_argument('--epochs', type=int, default=10, metavar='N', 174 | help='number of epochs to train (default: 10)') 175 | parser.add_argument('--lr', type=float, default=0.01, metavar='LR', 176 | help='learning rate (default: 0.01)') 177 | parser.add_argument('--momentum', type=float, default=0.5, metavar='M', 178 | help='SGD momentum (default: 0.5)') 179 | parser.add_argument('--seed', type=int, default=1, metavar='S', 180 | help='random seed (default: 1)') 181 | parser.add_argument('--log-interval', type=int, default=100, metavar='N', 182 | help='how many batches to wait before logging training status') 183 | parser.add_argument('--backend', type=str, default=None, 184 | help='backend for distributed training (tcp, gloo on cpu and gloo, nccl on gpu)') 185 | 186 | # Container environment 187 | parser.add_argument('--hosts', type=list, default=json.loads(os.environ['SM_HOSTS'])) 188 | parser.add_argument('--current-host', type=str, default=os.environ['SM_CURRENT_HOST']) 189 | parser.add_argument('--model-dir', type=str, default=os.environ['SM_MODEL_DIR']) 190 | parser.add_argument('--data-dir', type=str, default=os.environ['SM_CHANNEL_TRAINING']) 191 | parser.add_argument('--num-gpus', type=int, default=os.environ['SM_NUM_GPUS']) 192 | 193 | train(parser.parse_args()) 194 | -------------------------------------------------------------------------------- /sagemaker_processing/Feature_engineering_with_RAPIDS_on_SageMaker_Processing/container/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM continuumio/miniconda3:4.8.2 2 | 3 | RUN conda install -c rapidsai -c nvidia -c conda-forge -c defaults rapids=0.16 python=3.7 cudatoolkit=10.1 boto3 4 | 5 | ENV PYTHONUNBUFFERED=TRUE 6 | ENTRYPOINT ["python3"] 7 | -------------------------------------------------------------------------------- /sagemaker_processing/processing_with_bring_your_own_container_for_beginner/0_data_preparation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# 画像で機械学習するための準備（処理時間1分以内）\n", 8 | "* Amazon SageMaker はあまり関係ありません\n", 9 | "\n", 10 | "\n", 11 | "* このノートブックで行うこと\n", 12 | " 1. yan lecun 様の[サイト](http://yann.lecun.com/exdb/mnist/)より mnist のデータセットをノートブックインスタンスにダウンロード\n", 13 | " 2. バイナリデータから画像データを１枚ずつ png ファイルに出力\n", 14 | " 3. バイナリデータからラベルデータを npy ファイルに出力\n", 15 | " 4. zip ファイルに固めて S3 にアップロード \n", 16 | " zip ファイルに固める理由はバラバラの画像ファイルだと転送速度が遅くなるため、固めて転送して演算するコンピューティングリソースで解凍する" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "![](media/0_data_preparation.png)" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "import sagemaker\n", 33 | "print(f'Current sagemaker Version ={sagemaker.__version__}')" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "上記セルを実行して、SageMaker Python SDK Version が 1.xx.x の場合、以下のセルのコメントアウトを解除してから実行してください。実行が完了したら、上にあるメニューから [Kernel] -> [Restart] を選択してカーネルを再起動してください。\n", 41 | "\n", 42 | "再起動が完了したら、このノートブックの一番上のセルから再度実行してください。その場合、以下のセルを実行する必要はありません。" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "# pip install -U --quiet \"sagemaker==2.20.0\"" 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "metadata": {}, 57 | "source": [ 58 | "## 各種設定" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "name = ''" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "import urllib.request, gzip, numpy as np, sagemaker, datetime, yaml, os, shutil\n", 77 | "from matplotlib import pyplot as plt\n", 78 | "from PIL import Image\n", 79 | "from tqdm import tqdm\n", 80 | "\n", 81 | "url_base = 'http://yann.lecun.com/exdb/mnist/'\n", 82 | "key_file = {\n", 83 | " 'train_img':'train-images-idx3-ubyte.gz',\n", 84 | " 'train_label':'train-labels-idx1-ubyte.gz',\n", 85 | " 'test_img':'t10k-images-idx3-ubyte.gz',\n", 86 | " 'test_label':'t10k-labels-idx1-ubyte.gz'\n", 87 | "}\n", 88 | "# 様々な識別子を一意にするためにタイムスタンプを利用する\n", 89 | "timestamp = datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=9))).strftime('%Y%m%d%H%M%S')\n", 90 | "print(f'timestamp: {timestamp}')" 91 | ] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "metadata": {}, 96 | "source": [ 97 | "## データをノートブックインスタンスにダウンロード" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": null, 103 | "metadata": {}, 104 | "outputs": [], 105 | "source": [ 106 | "%%time\n", 107 | "\n", 108 | "dataset_dir = './mnist/' #画像ファイルとラベルを保存するディレクトリ\n", 109 | "\n", 110 | "os.makedirs(dataset_dir, exist_ok=True)\n", 111 | "\n", 112 | "for v in key_file.values():\n", 113 | " file_path = dataset_dir + '/' + v\n", 114 | " # データのダウンロード\n", 115 | " urllib.request.urlretrieve(url_base + v, file_path)" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "!ls {dataset_dir}" 125 | ] 126 | }, 127 | { 128 | "cell_type": "markdown", 129 | "metadata": {}, 130 | "source": [ 131 | "## mnistのバイナリデータを numpy 配列にする\n", 132 | "ついでにラベルデータを事前に one-hot encoding しておく" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": null, 138 | "metadata": {}, 139 | "outputs": [], 140 | "source": [ 141 | "%%time\n", 142 | "\n", 143 | "file_path = dataset_dir + key_file['train_img']\n", 144 | "with gzip.open(file_path, 'rb') as f:\n", 145 | " train_x = np.frombuffer(f.read(), np.uint8, offset=16).reshape(-1,28,28)\n", 146 | "file_path = dataset_dir + key_file['train_label']\n", 147 | "with gzip.open(file_path, 'rb') as f:\n", 148 | " train_y = np.frombuffer(f.read(), np.uint8, offset=8)\n", 149 | "train_y = np.identity(10)[train_y]\n", 150 | "\n", 151 | "\n", 152 | "file_path = dataset_dir + key_file['test_img']\n", 153 | "with gzip.open(file_path, 'rb') as f:\n", 154 | " test_x = np.frombuffer(f.read(), np.uint8, offset=16).reshape(-1,28,28)\n", 155 | "file_path = dataset_dir + key_file['test_label']\n", 156 | "with gzip.open(file_path, 'rb') as f:\n", 157 | " test_y = np.frombuffer(f.read(), np.uint8, offset=8)\n", 158 | "test_y = np.identity(10)[test_y]" 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "metadata": {}, 164 | "source": [ 165 | "## numpy array を 1 枚ずつ png ファイルに出力する\n", 166 | "* [前処理](1_preprocess_kick.ipynb)で hist 平坦化したのち、再度 npy ファイルに変換するため" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": null, 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "%%time\n", 176 | "\n", 177 | "# ローカルへ保存\n", 178 | "base_dir = './dataset/'\n", 179 | "train_x_dir = base_dir + 'train_x/'\n", 180 | "test_x_dir = base_dir + 'test_x/'\n", 181 | "\n", 182 | "os.makedirs(train_x_dir, exist_ok=True)\n", 183 | "os.makedirs(test_x_dir, exist_ok=True)\n", 184 | "\n", 185 | "for i in tqdm(range(train_x.shape[0])):\n", 186 | " Image.fromarray(train_x[i,:,:]).save(train_x_dir + str(i).zfill(5) + \".png\")\n", 187 | "\n", 188 | "for i in tqdm(range(test_x.shape[0])):\n", 189 | " Image.fromarray(test_x[i,:,:]).save(test_x_dir + str(i).zfill(5) + \".png\")\n", 190 | "\n", 191 | "np.save(base_dir + 'train_y.npy',train_y)\n", 192 | "np.save(base_dir + 'test_y.npy',test_y)" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": null, 198 | "metadata": {}, 199 | "outputs": [], 200 | "source": [ 201 | "!ls -l {base_dir}" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": null, 207 | "metadata": {}, 208 | "outputs": [], 209 | "source": [ 210 | "!ls -l {base_dir}train_x | tail -n5" 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": null, 216 | "metadata": {}, 217 | "outputs": [], 218 | "source": [ 219 | "!ls -l {base_dir}test_x | tail -n5" 220 | ] 221 | }, 222 | { 223 | "cell_type": "markdown", 224 | "metadata": {}, 225 | "source": [ 226 | "## zip ファイルに固める\n", 227 | "* 前処理をする際に zip ファイルに固めたほうが S3 から前処理を行うコンテナへの転送効率が高い（細切れのファイルだと転送に時間がかかる）\n", 228 | "* 解凍処理も前処理に含める" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": null, 234 | "metadata": {}, 235 | "outputs": [], 236 | "source": [ 237 | "zip_file = shutil.make_archive('./dataset', 'zip', root_dir='./dataset/')\n", 238 | "print(zip_file)" 239 | ] 240 | }, 241 | { 242 | "cell_type": "markdown", 243 | "metadata": {}, 244 | "source": [ 245 | "## S3 にアップロードする際、一意の URI を指定したいため、設定ファイルの名前を取得" 246 | ] 247 | }, 248 | { 249 | "cell_type": "markdown", 250 | "metadata": {}, 251 | "source": [ 252 | "## SageMaker SDK を利用して 作成した dataset.zip を S3 にアップロード\n", 253 | "* 通常 S3 にアップロードをする際は [boto3](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#client) や [aws cli](https://aws.amazon.com/jp/cli/) を利用する\n", 254 | "* SageMaker SDK はデータサイエンティストのための SDK で boto3 や aws cli などの学習コストを増やさないよう, SageMaker で完結させられる = S3 にアップロードできる機能も持っている\n", 255 | "* upload_data メソッドを利用することで SageMaker のデフォルトバケット(sagemaker-{region}-{account} に 1 行のコードでアップロードできる" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": null, 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [ 264 | "# S3 に dataset.zip をアップロード\n", 265 | "prefix = f'sagemaker-handson-{name}/dataset-{timestamp}'\n", 266 | "zip_dataset_s3_uri = sagemaker.session.Session().upload_data(path=zip_file, key_prefix=prefix)" 267 | ] 268 | }, 269 | { 270 | "cell_type": "code", 271 | "execution_count": null, 272 | "metadata": {}, 273 | "outputs": [], 274 | "source": [ 275 | "# アップロード先などを次のノートブックに引き継ぐため yaml に出力\n", 276 | "with open(\"./setting.yaml\", mode='w') as f:\n", 277 | " f.write('name: ' + name +'\\n')\n", 278 | " f.write('zip_dataset_s3_uri: ' + zip_dataset_s3_uri + '\\n')\n", 279 | " f.write('timestamp: ' + timestamp + '\\n')\n", 280 | "\n", 281 | "with open('./setting.yaml', 'r') as yml:\n", 282 | " config = yaml.load(yml)\n", 283 | "print(config)" 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "execution_count": null, 289 | "metadata": {}, 290 | "outputs": [], 291 | "source": [] 292 | } 293 | ], 294 | "metadata": { 295 | "instance_type": "ml.t3.medium", 296 | "kernelspec": { 297 | "display_name": "conda_python3", 298 | "language": "python", 299 | "name": "conda_python3" 300 | }, 301 | "language_info": { 302 | "codemirror_mode": { 303 | "name": "ipython", 304 | "version": 3 305 | }, 306 | "file_extension": ".py", 307 | "mimetype": "text/x-python", 308 | "name": "python", 309 | "nbconvert_exporter": "python", 310 | "pygments_lexer": "ipython3", 311 | "version": "3.6.10" 312 | } 313 | }, 314 | "nbformat": 4, 315 | "nbformat_minor": 4 316 | } 317 | -------------------------------------------------------------------------------- /sagemaker_processing/processing_with_bring_your_own_container_for_beginner/container/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-slim-buster 2 | 3 | RUN pip3 install scikit-image numpy 4 | ENV PYTHONUNBUFFERED=TRUE 5 | 6 | ENTRYPOINT ["python3"] -------------------------------------------------------------------------------- /sagemaker_processing/processing_with_bring_your_own_container_for_beginner/container/preprocess_code/preprocess.py: -------------------------------------------------------------------------------- 1 | import argparse, os, zipfile 2 | from glob import glob 3 | from PIL import Image 4 | import pandas as pd 5 | import numpy as np 6 | 7 | 8 | def hist_flatten(nparray): 9 | hist,bins = np.histogram(nparray.flatten(),256,[0,256]) 10 | cdf = hist.cumsum() 11 | cdf_normalized = cdf * hist.max()/ cdf.max() 12 | cdf_m = np.ma.masked_equal(cdf,0) 13 | cdf_m = (cdf_m - cdf_m.min())*255/(cdf_m.max()-cdf_m.min()) 14 | cdf = np.ma.filled(cdf_m,0).astype('uint8') 15 | return cdf[nparray] 16 | 17 | cdf = hist.cumsum() 18 | 19 | if __name__=='__main__': 20 | parser = argparse.ArgumentParser() 21 | parser.add_argument('--hist-flatten', type=bool, default=False) 22 | args, _ = parser.parse_known_args() 23 | 24 | print('Received arguments {}'.format(args)) 25 | 26 | input_data_path = os.path.join('/opt/ml/processing/input', 'img.zip') 27 | 28 | print(f'decompress {input_data_path}') 29 | with zipfile.ZipFile('data/temp/new_comp.zip') as existing_zip: 30 | existing_zip.extractall() 31 | train_x = np.zeros((len(glob('/opt/ml/processing/input/img/train_x/*.png')),28,28)) 32 | test_x = np.zeros((len(glob('/opt/ml/processing/input/img/test_x/*.png')),28,28)) 33 | 34 | for i,img_file in enumerate(sorted(glob('/opt/ml/processing/input/img/train_x/*.png'))): 35 | img = np.array(Image.open(img_file)) 36 | if args.hist_flatten: 37 | img = hist_flatten(img) 38 | img = img/255. 39 | train_x[i,:,:] = img 40 | for i,img_file in enumerate(sorted(glob('/opt/ml/processing/input/img/test_x/*.png'))): 41 | img = np.array(Image.open(img_file)) 42 | if args.hist_flatten: 43 | img = hist_flatten(img) 44 | img = img/255. 45 | test_x[i,:,:] = img 46 | 47 | train_features_output_path = '/opt/ml/processing/train/train_x.npy' 48 | test_features_output_path = '/opt/ml/processing/test/train_x.npy' 49 | 50 | print(f'train_x save: {train_features_output_path}') 51 | np.save('/opt/ml/processing/train/train_x.npy', train_x) 52 | print(f'test_x save: {test_features_output_path}') 53 | np.save('/opt/ml/processing/test/test_x.npy', test_x) 54 | -------------------------------------------------------------------------------- /sagemaker_processing/processing_with_bring_your_own_container_for_beginner/media/0_data_preparation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/sagemaker_processing/processing_with_bring_your_own_container_for_beginner/media/0_data_preparation.png -------------------------------------------------------------------------------- /sagemaker_processing/processing_with_bring_your_own_container_for_beginner/media/1_preprocess.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/sagemaker_processing/processing_with_bring_your_own_container_for_beginner/media/1_preprocess.png -------------------------------------------------------------------------------- /sagemaker_processing/processing_with_bring_your_own_container_for_beginner/media/README.md: -------------------------------------------------------------------------------- 1 | # Amazon SageMaker Processing を独自コンテナで動かすハンズオン 2 | 3 | ## 概要 4 | 本ハンズオンは Amazon SageMaker Processing を独自コンテナで動かすハンズオンです。 5 | 6 | Amazon SageMaker Processing には Scikit-learn や Apache Spark が利用可能な 組み込みコンテナを予め用意していますが、 7 | 組み込みコンテナだけでは不足する時のために、自身で Docker イメージを用意しして Amazon SageMaker Processing を動かすための仕組みも用意しており、 8 | その独自の Docker イメージを準備して、動かすことが体験できるハンズオンです。 9 | 10 | ## 構成 11 | 12 | このハンズオンは二段階の構成を取ってます。 13 | 14 | 1. [0_data_preparation.ipynb](./0_data_preparation.ipynb) 15 | このノートブックは SageMaker があまり関係ありませんが、[mnist のデータ](http://yann.lecun.com/exdb/mnist/)を画像化(png)し、zip 圧縮したものを Amazon S3 に保存する処理が入っています。次のノートブックを実行するためのデータ準備処理のみが記載されています。 16 | 2. [1_preprocess.ipynb](./1_preprocess.ipynb) 17 | Amazon SageMaker Processing を実際に動かすノートブックです。実行する処理は、先のノートブックで Amazon S3 に保存したデータを、zip 解凍し、輝度のヒストグラム平坦化の前処理を行ったあと、numpy ファイル( `.npyファイル`)に保存する処理を行います。これらの処理を行うために、画像を扱うために scikit-image をインストールした Docker イメージをビルドし、 Amazon Elastic Container Registry にプッシュし、 Amazon SageMaker Processing でビルドした Docker Image を用いた処理を行います。 -------------------------------------------------------------------------------- /sagemaker_processing/processing_with_bring_your_own_container_for_beginner/preprocess_script/preprocess.py: -------------------------------------------------------------------------------- 1 | import argparse, os, zipfile 2 | from glob import glob 3 | from skimage import io 4 | import numpy as np 5 | import shutil 6 | 7 | def hist_flatten(nparray): 8 | hist,bins = np.histogram(nparray.flatten(),256,[0,256]) 9 | cdf = hist.cumsum() 10 | cdf_normalized = cdf * hist.max()/ cdf.max() 11 | cdf_m = np.ma.masked_equal(cdf,0) 12 | cdf_m = (cdf_m - cdf_m.min())*255/(cdf_m.max()-cdf_m.min()) 13 | cdf = np.ma.filled(cdf_m,0).astype('uint8') 14 | return cdf[nparray] 15 | 16 | if __name__=='__main__': 17 | parser = argparse.ArgumentParser() 18 | parser.add_argument('--hist-flatten', type=bool, default=False) 19 | parser.add_argument('--input-dir', type=str, default=None) 20 | parser.add_argument('--output-train-dir', type=str, default=None) 21 | parser.add_argument('--output-test-dir', type=str, default=None) 22 | 23 | args, _ = parser.parse_known_args() 24 | 25 | print('Received arguments {}'.format(args)) 26 | 27 | input_data_path = os.path.join(args.input_dir, 'dataset.zip') 28 | 29 | print(f'decompress {input_data_path}') 30 | with zipfile.ZipFile(input_data_path) as existing_zip: 31 | existing_zip.extractall(args.input_dir) 32 | train_x = np.zeros((len(glob(args.input_dir + '/train_x/*.png')),28,28,1)).astype('float32') 33 | test_x = np.zeros((len(glob(args.input_dir + '/test_x/*.png')),28,28,1)).astype('float32') 34 | 35 | for i,img_file in enumerate(sorted(glob(args.input_dir + '/train_x/*.png'))): 36 | img = io.imread(img_file) 37 | if args.hist_flatten: 38 | img = hist_flatten(img) 39 | img = img/255. 40 | train_x[i,:,:,0] = img 41 | for i,img_file in enumerate(sorted(glob(args.input_dir + '/test_x/*.png'))): 42 | img = io.imread(img_file) 43 | if args.hist_flatten: 44 | img = hist_flatten(img) 45 | img = img/255. 46 | test_x[i,:,:,0] = img 47 | 48 | train_x_output_path = os.path.join(args.output_train_dir, 'train_x.npy') 49 | train_y_output_path = os.path.join(args.output_train_dir, 'train_y.npy') 50 | test_x_output_path = os.path.join(args.output_test_dir, 'test_x.npy') 51 | test_y_output_path = os.path.join(args.output_test_dir, 'test_y.npy') 52 | 53 | print(f'train_x save: {train_x_output_path}') 54 | np.save(train_x_output_path, train_x) 55 | print(f'train_y save: {train_y_output_path}') 56 | shutil.copyfile(args.input_dir + '/train_y.npy', train_y_output_path) 57 | 58 | print(f'test_x save: {test_x_output_path}') 59 | np.save(test_x_output_path, test_x) 60 | print(f'train_y save: {test_y_output_path}') 61 | shutil.copyfile(args.input_dir + '/test_y.npy', test_y_output_path) 62 | -------------------------------------------------------------------------------- /tensorflow2_training_and_serving/mnist.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | """Convolutional Neural Network Estimator for MNIST, built with tensorflow.keras.layers.""" 14 | 15 | import numpy as np 16 | import tensorflow as tf 17 | import os,json,argparse 18 | from tensorflow.keras.layers import * 19 | from tensorflow.keras.models import * 20 | from tensorflow.keras.optimizers import Adam 21 | from tensorflow.keras.losses import categorical_crossentropy 22 | 23 | 24 | def model(): 25 | model = Sequential() 26 | model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1))) 27 | model.add(MaxPooling2D((2, 2))) 28 | model.add(Conv2D(64, (3, 3), activation='relu')) 29 | model.add(MaxPooling2D((2, 2))) 30 | model.add(Conv2D(64, (3, 3), activation='relu')) 31 | model.add(Flatten()) 32 | model.add(Dense(64, activation='relu')) 33 | model.add(Dense(10, activation='softmax')) 34 | model.compile(optimizer=Adam(lr=0.0001),metrics=['accuracy'],loss="categorical_crossentropy") 35 | return model 36 | 37 | def _load_mnist_data(base_dir, x, y): 38 | x_data = np.load(os.path.join(base_dir, x)).reshape(-1,28,28,1) 39 | y_data = np.identity(10)[np.load(os.path.join(base_dir, y))] 40 | return x_data, y_data 41 | 42 | def _parse_args(): 43 | 44 | parser = argparse.ArgumentParser() 45 | parser.add_argument('--model_dir', type=str) 46 | parser.add_argument('--sm-model-dir', type=str, default=os.environ.get('SM_MODEL_DIR')) 47 | parser.add_argument('--training', type=str, default=os.environ.get('SM_CHANNEL_TRAINING')) 48 | parser.add_argument('--hosts', type=list, default=json.loads(os.environ.get('SM_HOSTS'))) 49 | parser.add_argument('--current-host', type=str, default=os.environ.get('SM_CURRENT_HOST')) 50 | parser.add_argument('--epochs', type=int, default=os.environ.get('SM_CURRENT_HOST')) 51 | parser.add_argument('--batch-size', type=int, default=os.environ.get('SM_CURRENT_HOST')) 52 | 53 | 54 | return parser.parse_known_args() 55 | 56 | 57 | if __name__ == "__main__": 58 | args, unknown = _parse_args() 59 | 60 | x_train, y_train = _load_mnist_data(args.training, 'train_data.npy', 'train_labels.npy') 61 | x_valid, y_valid = _load_mnist_data(args.training,'eval_data.npy', 'eval_labels.npy' ) 62 | 63 | # Create the Estimator 64 | mnist_classifier = model() 65 | # training 66 | mnist_classifier.fit( 67 | x_train, 68 | y_train, 69 | batch_size=args.batch_size, 70 | epochs=args.epochs, 71 | validation_data=(x_valid,y_valid) 72 | ) 73 | 74 | save_model_path = os.path.join(args.sm_model_dir, '000000001') 75 | 76 | if args.current_host == args.hosts[0]: 77 | mnist_classifier.save(save_model_path) 78 | -------------------------------------------------------------------------------- /workshop/lab_bring-your-own-model/img/sagemaker-data-model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-examples-jp/5b5c15433de3add68919f26016d744cc119e363e/workshop/lab_bring-your-own-model/img/sagemaker-data-model.png -------------------------------------------------------------------------------- /workshop/lab_bring-your-own-model/tensorflow/cnn_mnist_after.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Convolutional Neural Network Estimator for MNIST, built with tf.layers.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import numpy as np 21 | import tensorflow as tf 22 | 23 | tf.logging.set_verbosity(tf.logging.INFO) 24 | 25 | def serving_input_fn(): 26 | inputs = {'x': tf.placeholder(tf.float32, [None, 28, 28], name="input_layer")} 27 | return tf.estimator.export.ServingInputReceiver(inputs, inputs) 28 | 29 | def parse_args(): 30 | import argparse, os 31 | parser = argparse.ArgumentParser() 32 | parser.add_argument('--train', type=str, default=os.environ['SM_CHANNEL_TRAIN']) 33 | parser.add_argument('--test', type=str, default=os.environ['SM_CHANNEL_TEST']) 34 | parser.add_argument('--model_dir', type=str) 35 | parser.add_argument('--sm-model-dir', type=str, default=os.environ['SM_MODEL_DIR']) 36 | parser.add_argument('--training-steps', type=int, default=20) 37 | args, _ = parser.parse_known_args() 38 | return args 39 | 40 | 41 | def cnn_model_fn(features, labels, mode): 42 | """Model function for CNN.""" 43 | # Input Layer 44 | # Reshape X to 4-D tensor: [batch_size, width, height, channels] 45 | # MNIST images are 28x28 pixels, and have one color channel 46 | input_layer = tf.reshape(features["x"], [-1, 28, 28, 1]) 47 | 48 | # Convolutional Layer #1 49 | # Computes 32 features using a 5x5 filter with ReLU activation. 50 | # Padding is added to preserve width and height. 51 | # Input Tensor Shape: [batch_size, 28, 28, 1] 52 | # Output Tensor Shape: [batch_size, 28, 28, 32] 53 | conv1 = tf.layers.conv2d( 54 | inputs=input_layer, 55 | filters=32, 56 | kernel_size=[5, 5], 57 | padding="same", 58 | activation=tf.nn.relu) 59 | 60 | # Pooling Layer #1 61 | # First max pooling layer with a 2x2 filter and stride of 2 62 | # Input Tensor Shape: [batch_size, 28, 28, 32] 63 | # Output Tensor Shape: [batch_size, 14, 14, 32] 64 | pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2) 65 | 66 | # Convolutional Layer #2 67 | # Computes 64 features using a 5x5 filter. 68 | # Padding is added to preserve width and height. 69 | # Input Tensor Shape: [batch_size, 14, 14, 32] 70 | # Output Tensor Shape: [batch_size, 14, 14, 64] 71 | conv2 = tf.layers.conv2d( 72 | inputs=pool1, 73 | filters=64, 74 | kernel_size=[5, 5], 75 | padding="same", 76 | activation=tf.nn.relu) 77 | 78 | # Pooling Layer #2 79 | # Second max pooling layer with a 2x2 filter and stride of 2 80 | # Input Tensor Shape: [batch_size, 14, 14, 64] 81 | # Output Tensor Shape: [batch_size, 7, 7, 64] 82 | pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2) 83 | 84 | # Flatten tensor into a batch of vectors 85 | # Input Tensor Shape: [batch_size, 7, 7, 64] 86 | # Output Tensor Shape: [batch_size, 7 * 7 * 64] 87 | pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64]) 88 | 89 | # Dense Layer 90 | # Densely connected layer with 1024 neurons 91 | # Input Tensor Shape: [batch_size, 7 * 7 * 64] 92 | # Output Tensor Shape: [batch_size, 1024] 93 | dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu) 94 | 95 | # Add dropout operation; 0.6 probability that element will be kept 96 | dropout = tf.layers.dropout( 97 | inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN) 98 | 99 | # Logits layer 100 | # Input Tensor Shape: [batch_size, 1024] 101 | # Output Tensor Shape: [batch_size, 10] 102 | logits = tf.layers.dense(inputs=dropout, units=10) 103 | 104 | predictions = { 105 | # Generate predictions (for PREDICT and EVAL mode) 106 | "classes": tf.argmax(input=logits, axis=1), 107 | # Add `softmax_tensor` to the graph. It is used for PREDICT and by the 108 | # `logging_hook`. 109 | "probabilities": tf.nn.softmax(logits, name="softmax_tensor") 110 | } 111 | if mode == tf.estimator.ModeKeys.PREDICT: 112 | return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) 113 | 114 | # Calculate Loss (for both TRAIN and EVAL modes) 115 | loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) 116 | 117 | # Configure the Training Op (for TRAIN mode) 118 | if mode == tf.estimator.ModeKeys.TRAIN: 119 | optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001) 120 | train_op = optimizer.minimize( 121 | loss=loss, 122 | global_step=tf.train.get_global_step()) 123 | return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) 124 | 125 | # Add evaluation metrics (for EVAL mode) 126 | eval_metric_ops = { 127 | "accuracy": tf.metrics.accuracy( 128 | labels=labels, predictions=predictions["classes"])} 129 | return tf.estimator.EstimatorSpec( 130 | mode=mode, loss=loss, eval_metric_ops=eval_metric_ops) 131 | 132 | def main(unused_argv): 133 | args = parse_args() 134 | train_dir = args.train 135 | test_dir = args.test 136 | model_dir = args.model_dir 137 | sm_model_dir = args.sm_model_dir 138 | training_steps = args.training_steps 139 | 140 | #def main(unused_argv): 141 | # Load training and eval data 142 | #mnist = tf.contrib.learn.datasets.load_dataset("mnist") 143 | #train_data = mnist.train.images # Returns np.array 144 | #train_labels = np.asarray(mnist.train.labels, dtype=np.int32) 145 | #eval_data = mnist.test.images # Returns np.array 146 | #eval_labels = np.asarray(mnist.test.labels, dtype=np.int32) 147 | 148 | import os 149 | train_data = np.load(os.path.join(train_dir, 'image.npy')).astype(np.float32) * 1./255 150 | train_labels = np.load(os.path.join(train_dir, 'label.npy')).astype(np.int32) 151 | eval_data = np.load(os.path.join(test_dir, 'image.npy')).astype(np.float32) * 1./255 152 | eval_labels = np.load(os.path.join(test_dir, 'label.npy')).astype(np.int32) 153 | # Create the Estimator 154 | 155 | mnist_classifier = tf.estimator.Estimator( 156 | model_fn=cnn_model_fn, model_dir=model_dir) 157 | 158 | #mnist_classifier = tf.estimator.Estimator( 159 | # model_fn=cnn_model_fn, model_dir="/tmp/mnist_convnet_model") 160 | 161 | # Set up logging for predictions 162 | # Log the values in the "Softmax" tensor with label "probabilities" 163 | tensors_to_log = {"probabilities": "softmax_tensor"} 164 | logging_hook = tf.train.LoggingTensorHook( 165 | tensors=tensors_to_log, every_n_iter=50) 166 | 167 | # Train the model 168 | train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( 169 | x={"x": train_data}, 170 | y=train_labels, 171 | batch_size=100, 172 | num_epochs=None, 173 | shuffle=True) 174 | mnist_classifier.train( 175 | input_fn=train_input_fn, 176 | steps=training_steps, #default:20000 177 | hooks=[logging_hook]) 178 | 179 | # Evaluate the model and print results 180 | #eval_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( 181 | # x={"x": eval_data}, y=eval_labels, num_epochs=1, shuffle=False) 182 | #eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn) 183 | #print(eval_results) 184 | 185 | # Evaluate the model and print results 186 | eval_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( 187 | x={"x": eval_data}, y=eval_labels, num_epochs=1, shuffle=False) 188 | eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn) 189 | print(eval_results) 190 | 191 | mnist_classifier.export_savedmodel(sm_model_dir, serving_input_fn) 192 | 193 | if __name__ == "__main__": 194 | tf.app.run() 195 | -------------------------------------------------------------------------------- /workshop/lab_bring-your-own-model/tensorflow/cnn_mnist_before.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Convolutional Neural Network Estimator for MNIST, built with tf.layers.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import numpy as np 21 | import tensorflow as tf 22 | 23 | tf.logging.set_verbosity(tf.logging.INFO) 24 | 25 | 26 | def cnn_model_fn(features, labels, mode): 27 | """Model function for CNN.""" 28 | # Input Layer 29 | # Reshape X to 4-D tensor: [batch_size, width, height, channels] 30 | # MNIST images are 28x28 pixels, and have one color channel 31 | input_layer = tf.reshape(features["x"], [-1, 28, 28, 1]) 32 | 33 | # Convolutional Layer #1 34 | # Computes 32 features using a 5x5 filter with ReLU activation. 35 | # Padding is added to preserve width and height. 36 | # Input Tensor Shape: [batch_size, 28, 28, 1] 37 | # Output Tensor Shape: [batch_size, 28, 28, 32] 38 | conv1 = tf.layers.conv2d( 39 | inputs=input_layer, 40 | filters=32, 41 | kernel_size=[5, 5], 42 | padding="same", 43 | activation=tf.nn.relu) 44 | 45 | # Pooling Layer #1 46 | # First max pooling layer with a 2x2 filter and stride of 2 47 | # Input Tensor Shape: [batch_size, 28, 28, 32] 48 | # Output Tensor Shape: [batch_size, 14, 14, 32] 49 | pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2) 50 | 51 | # Convolutional Layer #2 52 | # Computes 64 features using a 5x5 filter. 53 | # Padding is added to preserve width and height. 54 | # Input Tensor Shape: [batch_size, 14, 14, 32] 55 | # Output Tensor Shape: [batch_size, 14, 14, 64] 56 | conv2 = tf.layers.conv2d( 57 | inputs=pool1, 58 | filters=64, 59 | kernel_size=[5, 5], 60 | padding="same", 61 | activation=tf.nn.relu) 62 | 63 | # Pooling Layer #2 64 | # Second max pooling layer with a 2x2 filter and stride of 2 65 | # Input Tensor Shape: [batch_size, 14, 14, 64] 66 | # Output Tensor Shape: [batch_size, 7, 7, 64] 67 | pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2) 68 | 69 | # Flatten tensor into a batch of vectors 70 | # Input Tensor Shape: [batch_size, 7, 7, 64] 71 | # Output Tensor Shape: [batch_size, 7 * 7 * 64] 72 | pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64]) 73 | 74 | # Dense Layer 75 | # Densely connected layer with 1024 neurons 76 | # Input Tensor Shape: [batch_size, 7 * 7 * 64] 77 | # Output Tensor Shape: [batch_size, 1024] 78 | dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu) 79 | 80 | # Add dropout operation; 0.6 probability that element will be kept 81 | dropout = tf.layers.dropout( 82 | inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN) 83 | 84 | # Logits layer 85 | # Input Tensor Shape: [batch_size, 1024] 86 | # Output Tensor Shape: [batch_size, 10] 87 | logits = tf.layers.dense(inputs=dropout, units=10) 88 | 89 | predictions = { 90 | # Generate predictions (for PREDICT and EVAL mode) 91 | "classes": tf.argmax(input=logits, axis=1), 92 | # Add `softmax_tensor` to the graph. It is used for PREDICT and by the 93 | # `logging_hook`. 94 | "probabilities": tf.nn.softmax(logits, name="softmax_tensor") 95 | } 96 | if mode == tf.estimator.ModeKeys.PREDICT: 97 | return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) 98 | 99 | # Calculate Loss (for both TRAIN and EVAL modes) 100 | loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) 101 | 102 | # Configure the Training Op (for TRAIN mode) 103 | if mode == tf.estimator.ModeKeys.TRAIN: 104 | optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001) 105 | train_op = optimizer.minimize( 106 | loss=loss, 107 | global_step=tf.train.get_global_step()) 108 | return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) 109 | 110 | # Add evaluation metrics (for EVAL mode) 111 | eval_metric_ops = { 112 | "accuracy": tf.metrics.accuracy( 113 | labels=labels, predictions=predictions["classes"])} 114 | return tf.estimator.EstimatorSpec( 115 | mode=mode, loss=loss, eval_metric_ops=eval_metric_ops) 116 | 117 | 118 | def main(unused_argv): 119 | # Load training and eval data 120 | mnist = tf.contrib.learn.datasets.load_dataset("mnist") 121 | train_data = mnist.train.images # Returns np.array 122 | train_labels = np.asarray(mnist.train.labels, dtype=np.int32) 123 | eval_data = mnist.test.images # Returns np.array 124 | eval_labels = np.asarray(mnist.test.labels, dtype=np.int32) 125 | 126 | # Create the Estimator 127 | mnist_classifier = tf.estimator.Estimator( 128 | model_fn=cnn_model_fn, model_dir="/tmp/mnist_convnet_model") 129 | 130 | # Set up logging for predictions 131 | # Log the values in the "Softmax" tensor with label "probabilities" 132 | tensors_to_log = {"probabilities": "softmax_tensor"} 133 | logging_hook = tf.train.LoggingTensorHook( 134 | tensors=tensors_to_log, every_n_iter=50) 135 | 136 | # Train the model 137 | train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( 138 | x={"x": train_data}, 139 | y=train_labels, 140 | batch_size=100, 141 | num_epochs=None, 142 | shuffle=True) 143 | mnist_classifier.train( 144 | input_fn=train_input_fn, 145 | steps=20000, 146 | hooks=[logging_hook]) 147 | 148 | # Evaluate the model and print results 149 | eval_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( 150 | x={"x": eval_data}, y=eval_labels, num_epochs=1, shuffle=False) 151 | eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn) 152 | print(eval_results) 153 | 154 | 155 | if __name__ == "__main__": 156 | tf.app.run() 157 | --------------------------------------------------------------------------------