├── .gitignore ├── .kokoro ├── example_zoo │ ├── common.cfg │ ├── periodic.cfg │ └── run_tests.sh └── trampoline.sh ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── dlvm ├── gcp-gpu-utilization-metrics │ ├── LICENSE │ ├── README.md │ ├── create_gpu_metrics.py │ ├── report_gpu_metrics.py │ └── requirements.txt ├── nvidia │ ├── README.md │ ├── __init__.py │ ├── cat.jpg │ ├── image.jpg │ ├── inference.py │ └── webapp │ │ ├── __init__.py │ │ ├── requirements.txt │ │ ├── server.py │ │ ├── static │ │ └── images │ │ │ └── README.md │ │ └── templates │ │ └── index.html └── tools │ ├── convert_b64.py │ ├── convert_b64_api.py │ ├── convert_to_base64_resize.py │ ├── convert_to_rt.py │ ├── predict.py │ ├── scripts │ ├── gpu_utilization_agent.service │ ├── notebook_executor.sh │ ├── setup.sh │ ├── start_agent_and_inf_server.sh │ └── tfserve.service │ └── tf_serve.py ├── example_zoo ├── README.md ├── noxfile.py ├── tensorflow │ ├── models │ │ ├── keras_cifar_main │ │ │ ├── README.md │ │ │ ├── cmle_keras_cifar_main_test.py │ │ │ ├── config.yaml │ │ │ ├── official │ │ │ │ ├── __init__.py │ │ │ │ ├── resnet │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── cifar10_main.py │ │ │ │ │ ├── imagenet_preprocessing.py │ │ │ │ │ ├── keras │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── keras_cifar_main.py │ │ │ │ │ │ ├── keras_common.py │ │ │ │ │ │ └── resnet_cifar_model.py │ │ │ │ │ ├── resnet_model.py │ │ │ │ │ └── resnet_run_loop.py │ │ │ │ └── utils │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── export │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── export.py │ │ │ │ │ ├── flags │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── _base.py │ │ │ │ │ ├── _benchmark.py │ │ │ │ │ ├── _conventions.py │ │ │ │ │ ├── _device.py │ │ │ │ │ ├── _misc.py │ │ │ │ │ ├── _performance.py │ │ │ │ │ └── core.py │ │ │ │ │ ├── logs │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── cloud_lib.py │ │ │ │ │ ├── hooks.py │ │ │ │ │ ├── hooks_helper.py │ │ │ │ │ ├── logger.py │ │ │ │ │ └── metric_hook.py │ │ │ │ │ └── misc │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── distribution_utils.py │ │ │ │ │ └── model_helpers.py │ │ │ ├── setup.py │ │ │ ├── submit_27.sh │ │ │ └── submit_35.sh │ │ ├── keras_imagenet_main │ │ │ ├── README.md │ │ │ ├── cmle_keras_imagenet_main_test.py │ │ │ ├── config.yaml │ │ │ ├── official │ │ │ │ ├── __init__.py │ │ │ │ ├── resnet │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── imagenet_main.py │ │ │ │ │ ├── imagenet_preprocessing.py │ │ │ │ │ ├── keras │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── keras_common.py │ │ │ │ │ │ ├── keras_imagenet_main.py │ │ │ │ │ │ └── resnet_model.py │ │ │ │ │ ├── resnet_model.py │ │ │ │ │ └── resnet_run_loop.py │ │ │ │ └── utils │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── export │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── export.py │ │ │ │ │ ├── flags │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── _base.py │ │ │ │ │ ├── _benchmark.py │ │ │ │ │ ├── _conventions.py │ │ │ │ │ ├── _device.py │ │ │ │ │ ├── _misc.py │ │ │ │ │ ├── _performance.py │ │ │ │ │ └── core.py │ │ │ │ │ ├── logs │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── cloud_lib.py │ │ │ │ │ ├── hooks.py │ │ │ │ │ ├── hooks_helper.py │ │ │ │ │ ├── logger.py │ │ │ │ │ └── metric_hook.py │ │ │ │ │ └── misc │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── distribution_utils.py │ │ │ │ │ └── model_helpers.py │ │ │ ├── setup.py │ │ │ ├── submit_27.sh │ │ │ └── submit_35.sh │ │ ├── mnist │ │ │ ├── README.md │ │ │ ├── cmle_mnist_test.py │ │ │ ├── config.yaml │ │ │ ├── official │ │ │ │ ├── __init__.py │ │ │ │ ├── mnist │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── dataset.py │ │ │ │ │ └── mnist.py │ │ │ │ └── utils │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── flags │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── _base.py │ │ │ │ │ ├── _benchmark.py │ │ │ │ │ ├── _conventions.py │ │ │ │ │ ├── _device.py │ │ │ │ │ ├── _misc.py │ │ │ │ │ ├── _performance.py │ │ │ │ │ └── core.py │ │ │ │ │ ├── logs │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── cloud_lib.py │ │ │ │ │ ├── hooks.py │ │ │ │ │ ├── hooks_helper.py │ │ │ │ │ ├── logger.py │ │ │ │ │ └── metric_hook.py │ │ │ │ │ └── misc │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── distribution_utils.py │ │ │ │ │ └── model_helpers.py │ │ │ ├── setup.py │ │ │ ├── submit_27.sh │ │ │ └── submit_35.sh │ │ └── ncf_main │ │ │ ├── README.md │ │ │ ├── cmle_ncf_main_test.py │ │ │ ├── config.yaml │ │ │ ├── official │ │ │ ├── __init__.py │ │ │ ├── datasets │ │ │ │ ├── __init__.py │ │ │ │ └── movielens.py │ │ │ ├── recommendation │ │ │ │ ├── __init__.py │ │ │ │ ├── constants.py │ │ │ │ ├── data_pipeline.py │ │ │ │ ├── data_preprocessing.py │ │ │ │ ├── ncf_main.py │ │ │ │ ├── neumf_model.py │ │ │ │ ├── popen_helper.py │ │ │ │ └── stat_utils.py │ │ │ └── utils │ │ │ │ ├── __init__.py │ │ │ │ ├── flags │ │ │ │ ├── __init__.py │ │ │ │ ├── _base.py │ │ │ │ ├── _benchmark.py │ │ │ │ ├── _conventions.py │ │ │ │ ├── _device.py │ │ │ │ ├── _misc.py │ │ │ │ ├── _performance.py │ │ │ │ └── core.py │ │ │ │ ├── logs │ │ │ │ ├── __init__.py │ │ │ │ ├── cloud_lib.py │ │ │ │ ├── hooks.py │ │ │ │ ├── hooks_helper.py │ │ │ │ ├── logger.py │ │ │ │ ├── metric_hook.py │ │ │ │ └── mlperf_helper.py │ │ │ │ └── misc │ │ │ │ ├── __init__.py │ │ │ │ ├── distribution_utils.py │ │ │ │ └── model_helpers.py │ │ │ ├── setup.py │ │ │ ├── submit_27.sh │ │ │ └── submit_35.sh │ └── probability │ │ ├── bayesian_neural_network │ │ ├── README.md │ │ ├── cmle_bayesian_neural_network_test.py │ │ ├── config.yaml │ │ ├── setup.py │ │ ├── submit_27.sh │ │ ├── submit_35.sh │ │ └── trainer │ │ │ ├── __init__.py │ │ │ ├── bayesian_neural_network.py │ │ │ └── tfgfile_wrapper.py │ │ ├── deep_exponential_family │ │ ├── README.md │ │ ├── cmle_deep_exponential_family_test.py │ │ ├── config.yaml │ │ ├── setup.py │ │ ├── submit_27.sh │ │ ├── submit_35.sh │ │ └── trainer │ │ │ ├── __init__.py │ │ │ └── deep_exponential_family.py │ │ ├── generative_adversarial_network │ │ ├── README.md │ │ ├── cmle_generative_adversarial_network_test.py │ │ ├── config.yaml │ │ ├── setup.py │ │ ├── submit_27.sh │ │ ├── submit_35.sh │ │ └── trainer │ │ │ ├── __init__.py │ │ │ ├── generative_adversarial_network.py │ │ │ └── tfgfile_wrapper.py │ │ ├── grammar_vae │ │ ├── README.md │ │ ├── cmle_grammar_vae_test.py │ │ ├── config.yaml │ │ ├── setup.py │ │ ├── submit_27.sh │ │ ├── submit_35.sh │ │ └── trainer │ │ │ ├── __init__.py │ │ │ └── grammar_vae.py │ │ ├── latent_dirichlet_allocation_distributions │ │ ├── README.md │ │ ├── cmle_latent_dirichlet_allocation_distributions_test.py │ │ ├── config.yaml │ │ ├── setup.py │ │ ├── submit_27.sh │ │ ├── submit_35.sh │ │ └── trainer │ │ │ ├── __init__.py │ │ │ └── latent_dirichlet_allocation_distributions.py │ │ ├── latent_dirichlet_allocation_edward2 │ │ ├── README.md │ │ ├── cmle_latent_dirichlet_allocation_edward2_test.py │ │ ├── config.yaml │ │ ├── setup.py │ │ ├── submit_27.sh │ │ ├── submit_35.sh │ │ └── trainer │ │ │ ├── __init__.py │ │ │ └── latent_dirichlet_allocation_edward2.py │ │ ├── logistic_regression │ │ ├── README.md │ │ ├── cmle_logistic_regression_test.py │ │ ├── config.yaml │ │ ├── setup.py │ │ ├── submit_27.sh │ │ ├── submit_35.sh │ │ └── trainer │ │ │ ├── __init__.py │ │ │ ├── logistic_regression.py │ │ │ └── tfgfile_wrapper.py │ │ ├── vae │ │ ├── README.md │ │ ├── cmle_vae_test.py │ │ ├── config.yaml │ │ ├── setup.py │ │ ├── submit_27.sh │ │ ├── submit_35.sh │ │ └── trainer │ │ │ ├── __init__.py │ │ │ └── vae.py │ │ └── vq_vae │ │ ├── README.md │ │ ├── cmle_vq_vae_test.py │ │ ├── config.yaml │ │ ├── setup.py │ │ ├── submit_27.sh │ │ ├── submit_35.sh │ │ └── trainer │ │ ├── __init__.py │ │ ├── tfgfile_wrapper.py │ │ └── vq_vae.py └── tools │ ├── README.md │ ├── cmle_package.py │ ├── process.py │ ├── source_finder.py │ ├── templates │ ├── README.md │ ├── __init__.py │ ├── cmle_test.py │ ├── config.yaml │ ├── setup.py │ ├── submit_27.sh │ ├── submit_35.sh │ └── tfgfile_wrapper.py │ ├── tf_models_samples.yaml │ └── tf_probability_samples.yaml ├── gce ├── burst-training │ ├── README.md │ ├── build-image.sh │ ├── census-analysis.py │ ├── census-startup.sh │ ├── image.sh │ ├── requirements.txt │ └── train.sh └── survival-training │ ├── README-sklearn.md │ ├── README-tf-estimator.md │ ├── README.md │ ├── gce │ ├── startup.sh │ ├── tf-estimator-instance-template.sh │ ├── tf-estimator-set-metadata.sh │ └── tf-estimator-startup.sh │ ├── img │ ├── cuda.png │ ├── nvidia-smi.png │ ├── pvm-training.png │ ├── tf-estimator-instance-creation.png │ └── tf-proof.png │ └── wrapper │ ├── __init__.py │ ├── train.py │ └── train_test.py ├── gcf └── gcf-ai-platform-example │ ├── .gcloudignore │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ ├── main.py │ └── requirements.txt ├── third_party ├── gpu-burn │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── compare.cu │ └── gpu_burn-drv.cpp └── rapids │ ├── README.md │ ├── distributed │ ├── README.md │ ├── start-dask-cuda-worker.sh │ ├── start-dask-worker.sh │ ├── start-remote-workers.sh │ └── stop-remote-workers.sh │ ├── run.sh │ └── sum.py └── tutorials ├── R ├── 01_EDA-with-R-and-BigQuery.ipynb ├── 02-Training-Serving-TF-Models.ipynb ├── 03-Training-Serving-CARET-Models.ipynb ├── README.md └── src │ ├── caret │ ├── .ipynb_checkpoints │ │ └── Dockerfile-checkpoint │ ├── Dockerfile │ ├── serving │ │ ├── Dockerfile │ │ ├── model_api.R │ │ └── model_prediction.R │ └── training │ │ ├── Dockerfile │ │ └── model_trainer.R │ └── tensorflow │ └── model_trainer.R ├── explanations ├── README.md ├── ai-explanations-image.ipynb └── ai-explanations-tabular.ipynb ├── pytorch └── jasper │ ├── JasperTRT.ipynb │ ├── README.MD │ └── build_asset.sh ├── sklearn ├── gae_serve │ ├── README.md │ ├── __init__.py │ ├── app.yaml │ ├── client.py │ ├── default │ │ ├── README.md │ │ ├── app.yaml │ │ └── modelserve.yaml │ ├── lr.pkl │ ├── main.py │ ├── modelserve.yaml │ └── requirements.txt ├── hpsearch │ ├── README.md │ ├── __init__.py │ ├── get_cluster_credentials.sh │ ├── gke_bayes_search.ipynb │ ├── gke_grid_search.ipynb │ ├── gke_parallel.py │ ├── gke_randomized_search.ipynb │ ├── helpers │ │ ├── __init__.py │ │ ├── cloudbuild_helper.py │ │ ├── gcs_helper.py │ │ ├── gke_helper.py │ │ └── kubernetes_helper.py │ ├── requirements.txt │ └── source │ │ ├── Dockerfile │ │ ├── gcs_helper.py │ │ ├── requirements.txt │ │ └── worker.py └── titanic │ ├── README.md │ ├── requirements.txt │ └── titanic.py └── tensorflow ├── bert ├── BERT_TRT.ipynb ├── Dockerfile ├── README.md └── start.sh ├── bert_finetuning ├── README.MD ├── bert_squad_tf_finetuning.ipynb └── build_asset.sh ├── estimators └── tf-estimators.ipynb └── mlflow_gcp ├── README.md ├── conda ├── MLproject └── conda.yaml ├── docker ├── Dockerfile └── MLproject ├── hp_config.yaml ├── requirements.txt ├── setup.py └── trainer ├── __init__.py ├── model.py ├── model_deployment.py ├── task.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | .*~ 3 | .nox 4 | *.pyc 5 | __pycache__ 6 | -------------------------------------------------------------------------------- /.kokoro/example_zoo/common.cfg: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Format: //devtools/kokoro/config/proto/build.proto 16 | 17 | # Download trampoline resources. 18 | gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" 19 | 20 | gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/ml-on-gcp" 21 | 22 | # All builds use the trampoline script to run in docker. 23 | build_file: "ml-on-gcp/.kokoro/trampoline.sh" 24 | 25 | # Configure the docker image for kokoro-trampoline. 26 | env_vars: { 27 | key: "TRAMPOLINE_IMAGE" 28 | value: "gcr.io/cloud-devrel-kokoro-resources/python@sha256:06a4133f77d22bfd3151b59ceae0683bdebd6d098dcfa2fd1c4e57e4de6c2eae" 29 | } 30 | -------------------------------------------------------------------------------- /.kokoro/example_zoo/periodic.cfg: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | # Format: //devtools/kokoro/config/proto/build.proto 17 | 18 | env_vars: { 19 | key: "TRAMPOLINE_BUILD_FILE" 20 | value: "github/ml-on-gcp/.kokoro/example_zoo/run_tests.sh" 21 | } 22 | 23 | env_vars: { 24 | key: "EXAMPLE_ZOO_ARTIFACTS_BUCKET" 25 | value: "caip_example_zoo" 26 | } 27 | 28 | env_vars: { 29 | key: "EXAMPLE_ZOO_PROJECT_ID" 30 | value: "caip-samples" 31 | } 32 | 33 | -------------------------------------------------------------------------------- /.kokoro/example_zoo/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -eo pipefail 17 | 18 | cd github/ml-on-gcp/example_zoo 19 | 20 | export GOOGLE_CLOUD_PROJECT=caip-samples 21 | export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/caip-samples-ml-on-gcp-example-zoo-kokoro-testing.json 22 | 23 | # Run tests 24 | nox || ret_code=$? 25 | 26 | exit ${ret_code} -------------------------------------------------------------------------------- /.kokoro/trampoline.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -eo pipefail 16 | 17 | # Always run the cleanup script, regardless of the success of bouncing into 18 | # the container. 19 | 20 | function cleanup() { 21 | chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh 22 | ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh 23 | echo "cleanup"; 24 | } 25 | trap cleanup EXIT 26 | 27 | python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution, 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | -------------------------------------------------------------------------------- /dlvm/gcp-gpu-utilization-metrics/requirements.txt: -------------------------------------------------------------------------------- 1 | google-cloud-monitoring==2.0 2 | -------------------------------------------------------------------------------- /dlvm/nvidia/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Blog setup for Nvidia T4 3 | 4 | ## Overview 5 | 6 | This script `ml-on-gcp/dlvm/tools/script/setup.sh` installs a demo cluster of 2 Compute instances 7 | running TensorFlow serving. 8 | Original setup from this [GCP blog](https://cloud.google.com/blog/products/ai-machine-learning/running-tensorflow-inference-workloads-at-scale-with-tensorrt-5-and-nvidia-t4-gpus) 9 | 10 | ### Instructions 11 | 12 | 1. Enable Cloud Compute Engine API in your Project. 13 | 14 | 2. Verify you have enough Nvidia GPU processors in your project. 15 | 16 | - Products > IAM & admin > Quotas 17 | 18 | 3. Define your Project in setup.sh under ```export PROJECT_NAME=""``` 19 | 20 | Deploy instances: 21 | 22 | ```bash 23 | setup.sh install 24 | ``` 25 | Delete demo: 26 | 27 | ```bash 28 | setup.h cleanup 29 | ``` 30 | 31 | Enable firewall: 32 | 33 | ```bash 34 | setup.h enable_firewall 35 | ``` 36 | 37 | Check firewall status: 38 | 39 | ```bash 40 | setup.h firewall_status 41 | ``` 42 | 43 | 44 | ### Testing prediction 45 | 46 | ```bash 47 | 48 | curl -X POST $IP/v1/models/default:predict -d @/tmp/out.json 49 | ``` 50 | 51 | ```bash 52 | python inference.py 53 | ``` 54 | 55 | ```bash 56 | apt-get install apache2-utils -y 57 | ab -n 30000 -c 150 -t 600 -g t4.tsv -H "Accept-Encoding: gzip,deflate" -p /tmp/out.json http://$IP/v1/models/default:predict 58 | ``` 59 | 60 | ```bash 61 | apt-get install gnuplot 62 | gnuplot 63 | gnuplot> set terminal dumb 64 | gnuplot> plot "out.data" using 9 w l 65 | ``` 66 | 67 | 68 | ### Troubleshooting 69 | 70 | Error: 71 | 72 | ```bash 73 | Required 'compute.instanceTemplates.create' permission for 'projects//global/instanceTemplates/tf-inference-template' 74 | ``` 75 | 76 | Solution: 77 | 78 | ```bash 79 | gcloud auth login 80 | ``` 81 | 82 | ```bash 83 | gcloud auth activate-service-account --key-file= 84 | ``` 85 | 86 | Error: 87 | 88 | ```bash 89 | NAME ZONE STATUS ACTION LAST_ERROR 90 | deeplearning-instances-6gck us-central1-b CREATING Error QUOTA_EXCEEDED: Instance 'deeplearning-instances-6gck' creation failed: Quota 'NVIDIA_T4_GPUS' exceeded. Limit: 1.0 in region us-central1. 91 | deeplearning-instances-rz61 us-central1-b CREATING Error QUOTA_EXCEEDED: Instance 'deeplearning-instances-rz61' creation failed: Quota 'NVIDIA_T4_GPUS' exceeded. Limit: 1.0 in region us-central1. 92 | ``` 93 | 94 | Solution: 95 | 96 | Update your Quota to allow more than 1.0 NVIDIA T4 -------------------------------------------------------------------------------- /dlvm/nvidia/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/dlvm/nvidia/__init__.py -------------------------------------------------------------------------------- /dlvm/nvidia/cat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/dlvm/nvidia/cat.jpg -------------------------------------------------------------------------------- /dlvm/nvidia/image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/dlvm/nvidia/image.jpg -------------------------------------------------------------------------------- /dlvm/nvidia/webapp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/dlvm/nvidia/webapp/__init__.py -------------------------------------------------------------------------------- /dlvm/nvidia/webapp/requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.7.0 2 | astor==0.7.1 3 | cachetools==3.0.0 4 | certifi==2018.11.29 5 | chardet==3.0.4 6 | Click==7.0 7 | Flask==1.0.2 8 | gast==0.2.2 9 | google-api-core==1.6.0 10 | google-auth==1.6.1 11 | google-cloud-monitoring==0.31.0 12 | googleapis-common-protos==1.5.5 13 | grpcio==1.17.1 14 | h5py==2.9.0 15 | idna==2.8 16 | itsdangerous==1.1.0 17 | Jinja2>=2.10.1 18 | Markdown==3.0.1 19 | MarkupSafe==1.1.0 20 | numpy==1.16.0 21 | Pillow==6.2.0 22 | protobuf==3.6.1 23 | pyasn1==0.4.4 24 | pyasn1-modules==0.2.2 25 | pytz==2018.7 26 | pyyaml>=4.2b1 27 | requests==2.21.0 28 | rsa==4.0 29 | scipy==1.2.0 30 | six==1.12.0 31 | termcolor==1.1.0 32 | urllib3==1.24.2 33 | Werkzeug==0.15.3 34 | -------------------------------------------------------------------------------- /dlvm/nvidia/webapp/static/images/README.md: -------------------------------------------------------------------------------- 1 | ## Images of predictions will be stored here. -------------------------------------------------------------------------------- /dlvm/tools/convert_b64.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import base64 15 | 16 | INPUT_FILE = 'image.jpg' 17 | OUTPUT_FILE = '/tmp/image_b64.json' 18 | 19 | """Open image and convert it to Base64""" 20 | with open(INPUT_FILE, 'rb') as input_file: 21 | jpeg_bytes = base64.b64encode(input_file.read()).decode('utf-8') 22 | predict_request = '{"image_bytes": {"b64": "%s"}}' % jpeg_bytes 23 | # Write JSON to file 24 | with open(OUTPUT_FILE, 'w') as output_file: 25 | output_file.write(predict_request) 26 | -------------------------------------------------------------------------------- /dlvm/tools/convert_b64_api.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Converts image to Base64 for AI Platform ML API.""" 15 | 16 | import base64 17 | 18 | INPUT_FILE = 'image.jpg' 19 | OUTPUT_FILE = 'image_b64.json' 20 | 21 | def convert_to_base64(image_file): 22 | """Open image and convert it to base64""" 23 | with open(image_file, 'rb') as input_file: 24 | jpeg_bytes = base64.b64encode(input_file.read()).decode('utf-8') 25 | predict_request = '{"instances" : [{"b64": "%s"}]}' % jpeg_bytes 26 | # Write JSON to file 27 | with open(OUTPUT_FILE, 'w') as output_file: 28 | output_file.write(predict_request) 29 | return predict_request 30 | 31 | convert_to_base64(INPUT_FILE) 32 | -------------------------------------------------------------------------------- /dlvm/tools/convert_to_base64_resize.py: -------------------------------------------------------------------------------- 1 | """Converts image file to JSON array""" 2 | 3 | import base64 4 | from PIL import Image 5 | 6 | INPUT_FILE = 'image.jpg' 7 | OUTPUT_FILE = 'image_b64.json' 8 | 9 | 10 | def convert_to_base64_resize(image_file): 11 | """Open image, resize it, encode it to b64 and save in JSON file""" 12 | img = Image.open(image_file).resize((240, 240)) 13 | img.save(image_file) 14 | with open(image_file, 'rb') as f: 15 | jpeg_bytes = base64.b64encode(f.read()).decode('utf-8') 16 | predict_request = '{"instances" : [{"b64": "%s"}]}' % jpeg_bytes 17 | # Write JSON to file 18 | with open(OUTPUT_FILE, 'w') as f: 19 | f.write(predict_request) 20 | return predict_request 21 | 22 | convert_to_base64_resize(INPUT_FILE) 23 | -------------------------------------------------------------------------------- /dlvm/tools/convert_to_rt.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Converts TF SavedModel to the TensorRT enabled graph""" 15 | 16 | import argparse 17 | import tensorflow.contrib.tensorrt as trt 18 | 19 | parser = argparse.ArgumentParser( 20 | description="Converts SavedModel to the TensorRT enabled graph.") 21 | 22 | parser.add_argument("--input_model_dir", required=True) 23 | parser.add_argument("--output_model_dir", required=True) 24 | parser.add_argument("--batch_size", required=True, type=int) 25 | parser.add_argument("--precision_mode", required=True, choices=["FP32", "FP16", "INT8", "INT4"]) 26 | args = parser.parse_args() 27 | 28 | trt.create_inference_graph( 29 | None, 30 | None, 31 | max_batch_size=args.batch_size, 32 | input_saved_model_dir=args.input_model_dir, 33 | output_saved_model_dir=args.output_model_dir, 34 | precision_mode=args.precision_mode) 35 | -------------------------------------------------------------------------------- /dlvm/tools/scripts/gpu_utilization_agent.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=GPU Utilization Metric Agent 3 | [Service] 4 | Type=simple 5 | PIDFile=/run/gpu_agent.pid 6 | ExecStart=/bin/bash --login -c '/usr/bin/python /root/report_gpu_metrics.py' 7 | User=root 8 | Group=root 9 | WorkingDirectory=/ 10 | Restart=always 11 | [Install] 12 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /dlvm/tools/scripts/start_agent_and_inf_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS-IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # Install NVIDIA driver and create GPU utilization service. 16 | 17 | # Install NVIDIA driver 18 | sh /opt/deeplearning/install-driver.sh 19 | 20 | # GPU Agent 21 | git clone https://github.com/GoogleCloudPlatform/ml-on-gcp.git 22 | cd ml-on-gcp/dlvm/gcp-gpu-utilization-metrics 23 | # Install Python dependencies. 24 | pip install -r ./requirements.txt 25 | cp ./report_gpu_metrics.py /root/report_gpu_metrics.py 26 | 27 | # Generate GPU service. 28 | cat <<-EOH > /lib/systemd/system/gpu_utilization_agent.service 29 | [Unit] 30 | Description=GPU Utilization Metric Agent 31 | [Service] 32 | Type=simple 33 | PIDFile=/run/gpu_agent.pid 34 | ExecStart=/bin/bash --login -c '/usr/bin/python /root/report_gpu_metrics.py' 35 | User=root 36 | Group=root 37 | WorkingDirectory=/ 38 | Restart=always 39 | [Install] 40 | WantedBy=multi-user.target 41 | EOH 42 | # Reload systemd manager configuration 43 | systemctl daemon-reload 44 | # Enable gpu_utilization_agent service 45 | systemctl --no-reload --now enable /lib/systemd/system/gpu_utilization_agent.service 46 | 47 | # Generate TF Service service. 48 | cat <<-EOH > /lib/systemd/system/tfserve.service 49 | [Unit] 50 | Description=Inf Logic 51 | [Service] 52 | Type=simple 53 | PIDFile=/run/tfserve_agent.pid 54 | ExecStart=/bin/bash --login -c '/usr/local/bin/tensorflow_model_server --model_base_path=/root/resnet_v2_int8_NCHW/ --rest_api_port=8888' 55 | User=root 56 | Group=root 57 | WorkingDirectory=/ 58 | Restart=always 59 | [Install] 60 | WantedBy=multi-user.target 61 | EOH 62 | 63 | gsutil cp gs://cloud-samples-data/dlvm/t4/model.tar.gz /root/model.tar.gz 64 | tar -xzvf /root/model.tar.gz -C /root 65 | # Reload systemd manager configuration 66 | systemctl daemon-reload 67 | # Enable tfserve service 68 | systemctl --no-reload --now enable /lib/systemd/system/tfserve.service 69 | -------------------------------------------------------------------------------- /dlvm/tools/scripts/tfserve.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Inf Logic 3 | [Service] 4 | Type=simple 5 | PIDFile=/run/tfserve_agent.pid 6 | ExecStart=/bin/bash --login -c '/usr/local/bin/tensorflow_model_server --model_base_path=/root/resnet_v2_int8_NCHW/ --rest_api_port=8888' 7 | User=root 8 | Group=root 9 | WorkingDirectory=/ 10 | Restart=always 11 | [Install] 12 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /dlvm/tools/tf_serve.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Inference logic.""" 15 | 16 | import tempfile 17 | import numpy as np 18 | import PIL 19 | 20 | from tfserve import TFServeApp 21 | 22 | RESNET_MODEL = '/root/tftrt_int8_resnetv2_imagenet_frozen_graph.pb' 23 | INPUT_TENSOR = 'import/input_tensor:0' 24 | SOFTMAX_TENSOR = 'import/softmax_tensor:0' 25 | 26 | 27 | def encode(request_data): 28 | with tempfile.NamedTemporaryFile(mode='wb', suffix='.jpg') as f: 29 | f.write(request_data) 30 | img = PIL.Image.open(f.name).resize((224, 224)) 31 | img = np.asarray(img) / 255. 32 | return {INPUT_TENSOR: img} 33 | 34 | 35 | def decode(outputs): 36 | p = outputs[SOFTMAX_TENSOR] 37 | index = np.argmax(p) 38 | return {'class': str(index), 'prob': str(float(p[index]))} 39 | 40 | 41 | app = TFServeApp(RESNET_MODEL, [INPUT_TENSOR], [SOFTMAX_TENSOR], encode, decode) 42 | app.run('0.0.0.0', 5000) 43 | -------------------------------------------------------------------------------- /example_zoo/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo 2 | 3 | Collections of examples adapted to be runnable on [AI Platform](https://cloud.google.com/ai-platform/). 4 | 5 | 1. [tensorflow-probability examples](/example_zoo/tensorflow/probability). 6 | 7 | 1. [tensorflow-models examples](/example_zoo/tensorflow/models). 8 | -------------------------------------------------------------------------------- /example_zoo/noxfile.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import glob 16 | import os 17 | 18 | import nox 19 | 20 | 21 | # Registered example sources to be tested. 22 | SOURCES = [ 23 | 'tensorflow/probability', 24 | 'tensorflow/models' 25 | ] 26 | 27 | # Each example is represented by its top-level directory. 28 | EXAMPLES = [] 29 | for source in SOURCES: 30 | EXAMPLES.extend(glob.glob(os.path.join(source, '*'))) 31 | 32 | 33 | @nox.session(python='2.7') 34 | @nox.parametrize('example', EXAMPLES) 35 | def example_zoo_tests(session, example): 36 | session.install('pytest', 'google-cloud-storage') 37 | 38 | session.chdir(example) 39 | session.run('pytest', '-x') 40 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: keras_cifar_main 2 | 3 | This is an automatically created sample based on [tensorflow/models/official/resnet/keras/keras_cifar_main.py](https://github.com/tensorflow/models/blob/r1.13.0/official/resnet/keras/keras_cifar_main.py). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/official/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_cifar_main/official/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/official/resnet/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_cifar_main/official/resnet/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/official/resnet/keras/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_cifar_main/official/resnet/keras/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/official/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_cifar_main/official/utils/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/official/utils/export/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_cifar_main/official/utils/export/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/official/utils/export/export.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Convenience functions for exporting models as SavedModels or other types.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import tensorflow as tf 22 | 23 | 24 | def build_tensor_serving_input_receiver_fn(shape, dtype=tf.float32, 25 | batch_size=1): 26 | """Returns a input_receiver_fn that can be used during serving. 27 | 28 | This expects examples to come through as float tensors, and simply 29 | wraps them as TensorServingInputReceivers. 30 | 31 | Arguably, this should live in tf.estimator.export. Testing here first. 32 | 33 | Args: 34 | shape: list representing target size of a single example. 35 | dtype: the expected datatype for the input example 36 | batch_size: number of input tensors that will be passed for prediction 37 | 38 | Returns: 39 | A function that itself returns a TensorServingInputReceiver. 40 | """ 41 | def serving_input_receiver_fn(): 42 | # Prep a placeholder where the input example will be fed in 43 | features = tf.placeholder( 44 | dtype=dtype, shape=[batch_size] + shape, name='input_tensor') 45 | 46 | return tf.estimator.export.TensorServingInputReceiver( 47 | features=features, receiver_tensors=features) 48 | 49 | return serving_input_receiver_fn 50 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/official/utils/flags/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_cifar_main/official/utils/flags/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/official/utils/flags/_conventions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Central location for shared arparse convention definitions.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import codecs 22 | import functools 23 | 24 | from absl import app as absl_app 25 | from absl import flags 26 | 27 | 28 | # This codifies help string conventions and makes it easy to update them if 29 | # necessary. Currently the only major effect is that help bodies start on the 30 | # line after flags are listed. All flag definitions should wrap the text bodies 31 | # with help wrap when calling DEFINE_*. 32 | _help_wrap = functools.partial(flags.text_wrap, length=80, indent="", 33 | firstline_indent="\n") 34 | 35 | 36 | # Pretty formatting causes issues when utf-8 is not installed on a system. 37 | try: 38 | codecs.lookup("utf-8") 39 | help_wrap = _help_wrap 40 | except LookupError: 41 | def help_wrap(text, *args, **kwargs): 42 | return _help_wrap(text, *args, **kwargs).replace("\ufeff", "") 43 | 44 | 45 | # Replace None with h to also allow -h 46 | absl_app.HelpshortFlag.SHORT_NAME = "h" 47 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/official/utils/flags/_misc.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Misc flags.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | from absl import flags 22 | 23 | from official.utils.flags._conventions import help_wrap 24 | 25 | 26 | def define_image(data_format=True): 27 | """Register image specific flags. 28 | 29 | Args: 30 | data_format: Create a flag to specify image axis convention. 31 | 32 | Returns: 33 | A list of flags for core.py to marks as key flags. 34 | """ 35 | 36 | key_flags = [] 37 | 38 | if data_format: 39 | flags.DEFINE_enum( 40 | name="data_format", short_name="df", default=None, 41 | enum_values=["channels_first", "channels_last"], 42 | help=help_wrap( 43 | "A flag to override the data format used in the model. " 44 | "channels_first provides a performance boost on GPU but is not " 45 | "always compatible with CPU. If left unspecified, the data format " 46 | "will be chosen automatically based on whether TensorFlow was " 47 | "built for CPU or GPU.")) 48 | key_flags.append("data_format") 49 | 50 | return key_flags 51 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/official/utils/logs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_cifar_main/official/utils/logs/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/official/utils/logs/cloud_lib.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Utilities that interact with cloud service. 17 | """ 18 | 19 | import requests 20 | 21 | GCP_METADATA_URL = "http://metadata/computeMetadata/v1/instance/hostname" 22 | GCP_METADATA_HEADER = {"Metadata-Flavor": "Google"} 23 | 24 | 25 | def on_gcp(): 26 | """Detect whether the current running environment is on GCP.""" 27 | try: 28 | # Timeout in 5 seconds, in case the test environment has connectivity issue. 29 | # There is not default timeout, which means it might block forever. 30 | response = requests.get( 31 | GCP_METADATA_URL, headers=GCP_METADATA_HEADER, timeout=5) 32 | return response.status_code == 200 33 | except requests.exceptions.RequestException: 34 | return False 35 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/official/utils/misc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_cifar_main/official/utils/misc/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = [] 19 | 20 | setuptools.setup(name='tensorflow_models_keras_cifar_main', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="official" 20 | MODULE_NAME="official.resnet.keras.keras_cifar_main" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="keras_cifar_main_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.13 \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --export_dir=$JOB_DIR \ 41 | --train_steps=100 \ 42 | --skip_eval \ 43 | --use_synthetic_data 44 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_cifar_main/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="official" 20 | MODULE_NAME="official.resnet.keras.keras_cifar_main" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="keras_cifar_main_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.13 \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --export_dir=$JOB_DIR \ 41 | --train_steps=100 \ 42 | --skip_eval \ 43 | --use_synthetic_data 44 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: keras_imagenet_main 2 | 3 | This is an automatically created sample based on [tensorflow/models/official/resnet/keras/keras_imagenet_main.py](https://github.com/tensorflow/models/blob/r1.13.0/official/resnet/keras/keras_imagenet_main.py). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/official/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_imagenet_main/official/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/official/resnet/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_imagenet_main/official/resnet/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/official/resnet/keras/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_imagenet_main/official/resnet/keras/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/official/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_imagenet_main/official/utils/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/official/utils/export/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_imagenet_main/official/utils/export/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/official/utils/export/export.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Convenience functions for exporting models as SavedModels or other types.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import tensorflow as tf 22 | 23 | 24 | def build_tensor_serving_input_receiver_fn(shape, dtype=tf.float32, 25 | batch_size=1): 26 | """Returns a input_receiver_fn that can be used during serving. 27 | 28 | This expects examples to come through as float tensors, and simply 29 | wraps them as TensorServingInputReceivers. 30 | 31 | Arguably, this should live in tf.estimator.export. Testing here first. 32 | 33 | Args: 34 | shape: list representing target size of a single example. 35 | dtype: the expected datatype for the input example 36 | batch_size: number of input tensors that will be passed for prediction 37 | 38 | Returns: 39 | A function that itself returns a TensorServingInputReceiver. 40 | """ 41 | def serving_input_receiver_fn(): 42 | # Prep a placeholder where the input example will be fed in 43 | features = tf.placeholder( 44 | dtype=dtype, shape=[batch_size] + shape, name='input_tensor') 45 | 46 | return tf.estimator.export.TensorServingInputReceiver( 47 | features=features, receiver_tensors=features) 48 | 49 | return serving_input_receiver_fn 50 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/official/utils/flags/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_imagenet_main/official/utils/flags/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/official/utils/flags/_conventions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Central location for shared arparse convention definitions.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import codecs 22 | import functools 23 | 24 | from absl import app as absl_app 25 | from absl import flags 26 | 27 | 28 | # This codifies help string conventions and makes it easy to update them if 29 | # necessary. Currently the only major effect is that help bodies start on the 30 | # line after flags are listed. All flag definitions should wrap the text bodies 31 | # with help wrap when calling DEFINE_*. 32 | _help_wrap = functools.partial(flags.text_wrap, length=80, indent="", 33 | firstline_indent="\n") 34 | 35 | 36 | # Pretty formatting causes issues when utf-8 is not installed on a system. 37 | try: 38 | codecs.lookup("utf-8") 39 | help_wrap = _help_wrap 40 | except LookupError: 41 | def help_wrap(text, *args, **kwargs): 42 | return _help_wrap(text, *args, **kwargs).replace("\ufeff", "") 43 | 44 | 45 | # Replace None with h to also allow -h 46 | absl_app.HelpshortFlag.SHORT_NAME = "h" 47 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/official/utils/flags/_misc.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Misc flags.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | from absl import flags 22 | 23 | from official.utils.flags._conventions import help_wrap 24 | 25 | 26 | def define_image(data_format=True): 27 | """Register image specific flags. 28 | 29 | Args: 30 | data_format: Create a flag to specify image axis convention. 31 | 32 | Returns: 33 | A list of flags for core.py to marks as key flags. 34 | """ 35 | 36 | key_flags = [] 37 | 38 | if data_format: 39 | flags.DEFINE_enum( 40 | name="data_format", short_name="df", default=None, 41 | enum_values=["channels_first", "channels_last"], 42 | help=help_wrap( 43 | "A flag to override the data format used in the model. " 44 | "channels_first provides a performance boost on GPU but is not " 45 | "always compatible with CPU. If left unspecified, the data format " 46 | "will be chosen automatically based on whether TensorFlow was " 47 | "built for CPU or GPU.")) 48 | key_flags.append("data_format") 49 | 50 | return key_flags 51 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/official/utils/logs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_imagenet_main/official/utils/logs/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/official/utils/logs/cloud_lib.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Utilities that interact with cloud service. 17 | """ 18 | 19 | import requests 20 | 21 | GCP_METADATA_URL = "http://metadata/computeMetadata/v1/instance/hostname" 22 | GCP_METADATA_HEADER = {"Metadata-Flavor": "Google"} 23 | 24 | 25 | def on_gcp(): 26 | """Detect whether the current running environment is on GCP.""" 27 | try: 28 | # Timeout in 5 seconds, in case the test environment has connectivity issue. 29 | # There is not default timeout, which means it might block forever. 30 | response = requests.get( 31 | GCP_METADATA_URL, headers=GCP_METADATA_HEADER, timeout=5) 32 | return response.status_code == 200 33 | except requests.exceptions.RequestException: 34 | return False 35 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/official/utils/misc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/keras_imagenet_main/official/utils/misc/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = [] 19 | 20 | setuptools.setup(name='tensorflow_models_keras_imagenet_main', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="official" 20 | MODULE_NAME="official.resnet.keras.keras_imagenet_main" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="keras_imagenet_main_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.13 \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --export_dir=$JOB_DIR \ 41 | --train_steps=100 \ 42 | --skip_eval \ 43 | --use_synthetic_data 44 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/keras_imagenet_main/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="official" 20 | MODULE_NAME="official.resnet.keras.keras_imagenet_main" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="keras_imagenet_main_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.13 \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --export_dir=$JOB_DIR \ 41 | --train_steps=100 \ 42 | --skip_eval \ 43 | --use_synthetic_data 44 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: mnist 2 | 3 | This is an automatically created sample based on [tensorflow/models/official/mnist/mnist.py](https://github.com/tensorflow/models/blob/r1.13.0/official/mnist/mnist.py). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/official/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/mnist/official/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/official/mnist/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/mnist/official/mnist/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/official/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/mnist/official/utils/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/official/utils/flags/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/mnist/official/utils/flags/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/official/utils/flags/_conventions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Central location for shared arparse convention definitions.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import codecs 22 | import functools 23 | 24 | from absl import app as absl_app 25 | from absl import flags 26 | 27 | 28 | # This codifies help string conventions and makes it easy to update them if 29 | # necessary. Currently the only major effect is that help bodies start on the 30 | # line after flags are listed. All flag definitions should wrap the text bodies 31 | # with help wrap when calling DEFINE_*. 32 | _help_wrap = functools.partial(flags.text_wrap, length=80, indent="", 33 | firstline_indent="\n") 34 | 35 | 36 | # Pretty formatting causes issues when utf-8 is not installed on a system. 37 | try: 38 | codecs.lookup("utf-8") 39 | help_wrap = _help_wrap 40 | except LookupError: 41 | def help_wrap(text, *args, **kwargs): 42 | return _help_wrap(text, *args, **kwargs).replace("\ufeff", "") 43 | 44 | 45 | # Replace None with h to also allow -h 46 | absl_app.HelpshortFlag.SHORT_NAME = "h" 47 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/official/utils/flags/_misc.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Misc flags.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | from absl import flags 22 | 23 | from official.utils.flags._conventions import help_wrap 24 | 25 | 26 | def define_image(data_format=True): 27 | """Register image specific flags. 28 | 29 | Args: 30 | data_format: Create a flag to specify image axis convention. 31 | 32 | Returns: 33 | A list of flags for core.py to marks as key flags. 34 | """ 35 | 36 | key_flags = [] 37 | 38 | if data_format: 39 | flags.DEFINE_enum( 40 | name="data_format", short_name="df", default=None, 41 | enum_values=["channels_first", "channels_last"], 42 | help=help_wrap( 43 | "A flag to override the data format used in the model. " 44 | "channels_first provides a performance boost on GPU but is not " 45 | "always compatible with CPU. If left unspecified, the data format " 46 | "will be chosen automatically based on whether TensorFlow was " 47 | "built for CPU or GPU.")) 48 | key_flags.append("data_format") 49 | 50 | return key_flags 51 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/official/utils/logs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/mnist/official/utils/logs/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/official/utils/logs/cloud_lib.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Utilities that interact with cloud service. 17 | """ 18 | 19 | import requests 20 | 21 | GCP_METADATA_URL = "http://metadata/computeMetadata/v1/instance/hostname" 22 | GCP_METADATA_HEADER = {"Metadata-Flavor": "Google"} 23 | 24 | 25 | def on_gcp(): 26 | """Detect whether the current running environment is on GCP.""" 27 | try: 28 | # Timeout in 5 seconds, in case the test environment has connectivity issue. 29 | # There is not default timeout, which means it might block forever. 30 | response = requests.get( 31 | GCP_METADATA_URL, headers=GCP_METADATA_HEADER, timeout=5) 32 | return response.status_code == 200 33 | except requests.exceptions.RequestException: 34 | return False 35 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/official/utils/misc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/mnist/official/utils/misc/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = [] 19 | 20 | setuptools.setup(name='tensorflow_models_mnist', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="official" 20 | MODULE_NAME="official.mnist.mnist" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="mnist_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.13 \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --export_dir=$JOB_DIR \ 41 | --train_epochs=1 42 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/mnist/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="official" 20 | MODULE_NAME="official.mnist.mnist" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="mnist_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.13 \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --export_dir=$JOB_DIR \ 41 | --train_epochs=1 42 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: ncf_main 2 | 3 | This is an automatically created sample based on [tensorflow/models/official/recommendation/ncf_main.py](https://github.com/tensorflow/models/blob/r1.13.0/official/recommendation/ncf_main.py). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/official/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/ncf_main/official/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/official/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/ncf_main/official/datasets/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/official/recommendation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/ncf_main/official/recommendation/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/official/recommendation/popen_helper.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Helper file for running the async data generation process in OSS.""" 16 | 17 | import contextlib 18 | import multiprocessing 19 | import multiprocessing.pool 20 | 21 | 22 | def get_forkpool(num_workers, init_worker=None, closing=True): 23 | pool = multiprocessing.Pool(processes=num_workers, initializer=init_worker) 24 | return contextlib.closing(pool) if closing else pool 25 | 26 | 27 | def get_threadpool(num_workers, init_worker=None, closing=True): 28 | pool = multiprocessing.pool.ThreadPool(processes=num_workers, 29 | initializer=init_worker) 30 | return contextlib.closing(pool) if closing else pool 31 | 32 | 33 | class FauxPool(object): 34 | """Mimic a pool using for loops. 35 | 36 | This class is used in place of proper pools when true determinism is desired 37 | for testing or debugging. 38 | """ 39 | def __init__(self, *args, **kwargs): 40 | pass 41 | 42 | def map(self, func, iterable, chunksize=None): 43 | return [func(i) for i in iterable] 44 | 45 | def imap(self, func, iterable, chunksize=1): 46 | for i in iterable: 47 | yield func(i) 48 | 49 | def close(self): 50 | pass 51 | 52 | def terminate(self): 53 | pass 54 | 55 | def join(self): 56 | pass 57 | 58 | def get_fauxpool(num_workers, init_worker=None, closing=True): 59 | pool = FauxPool(processes=num_workers, initializer=init_worker) 60 | return contextlib.closing(pool) if closing else pool 61 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/official/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/ncf_main/official/utils/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/official/utils/flags/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/ncf_main/official/utils/flags/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/official/utils/flags/_conventions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Central location for shared arparse convention definitions.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import codecs 22 | import functools 23 | 24 | from absl import app as absl_app 25 | from absl import flags 26 | 27 | 28 | # This codifies help string conventions and makes it easy to update them if 29 | # necessary. Currently the only major effect is that help bodies start on the 30 | # line after flags are listed. All flag definitions should wrap the text bodies 31 | # with help wrap when calling DEFINE_*. 32 | _help_wrap = functools.partial(flags.text_wrap, length=80, indent="", 33 | firstline_indent="\n") 34 | 35 | 36 | # Pretty formatting causes issues when utf-8 is not installed on a system. 37 | try: 38 | codecs.lookup("utf-8") 39 | help_wrap = _help_wrap 40 | except LookupError: 41 | def help_wrap(text, *args, **kwargs): 42 | return _help_wrap(text, *args, **kwargs).replace("\ufeff", "") 43 | 44 | 45 | # Replace None with h to also allow -h 46 | absl_app.HelpshortFlag.SHORT_NAME = "h" 47 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/official/utils/flags/_misc.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Misc flags.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | from absl import flags 22 | 23 | from official.utils.flags._conventions import help_wrap 24 | 25 | 26 | def define_image(data_format=True): 27 | """Register image specific flags. 28 | 29 | Args: 30 | data_format: Create a flag to specify image axis convention. 31 | 32 | Returns: 33 | A list of flags for core.py to marks as key flags. 34 | """ 35 | 36 | key_flags = [] 37 | 38 | if data_format: 39 | flags.DEFINE_enum( 40 | name="data_format", short_name="df", default=None, 41 | enum_values=["channels_first", "channels_last"], 42 | help=help_wrap( 43 | "A flag to override the data format used in the model. " 44 | "channels_first provides a performance boost on GPU but is not " 45 | "always compatible with CPU. If left unspecified, the data format " 46 | "will be chosen automatically based on whether TensorFlow was " 47 | "built for CPU or GPU.")) 48 | key_flags.append("data_format") 49 | 50 | return key_flags 51 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/official/utils/logs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/ncf_main/official/utils/logs/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/official/utils/logs/cloud_lib.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Utilities that interact with cloud service. 17 | """ 18 | 19 | import requests 20 | 21 | GCP_METADATA_URL = "http://metadata/computeMetadata/v1/instance/hostname" 22 | GCP_METADATA_HEADER = {"Metadata-Flavor": "Google"} 23 | 24 | 25 | def on_gcp(): 26 | """Detect whether the current running environment is on GCP.""" 27 | try: 28 | # Timeout in 5 seconds, in case the test environment has connectivity issue. 29 | # There is not default timeout, which means it might block forever. 30 | response = requests.get( 31 | GCP_METADATA_URL, headers=GCP_METADATA_HEADER, timeout=5) 32 | return response.status_code == 200 33 | except requests.exceptions.RequestException: 34 | return False 35 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/official/utils/misc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/models/ncf_main/official/utils/misc/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = ['typing==3.7.4'] 19 | 20 | setuptools.setup(name='tensorflow_models_ncf_main', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="official" 20 | MODULE_NAME="official.recommendation.ncf_main" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="ncf_main_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.13 \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --train_epochs=1 \ 41 | --use_synthetic_data 42 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/models/ncf_main/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="official" 20 | MODULE_NAME="official.recommendation.ncf_main" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="ncf_main_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.13 \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --train_epochs=1 \ 41 | --use_synthetic_data 42 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/bayesian_neural_network/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: bayesian_neural_network 2 | 3 | This is an automatically created sample based on [tensorflow/probability/tensorflow_probability/examples/bayesian_neural_network.py](https://github.com/tensorflow/probability/blob/r0.7/tensorflow_probability/examples/bayesian_neural_network.py). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/bayesian_neural_network/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/bayesian_neural_network/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = ['seaborn==0.9.0','tensorflow_probability==0.7'] 19 | 20 | setuptools.setup(name='tensorflow_probability_bayesian_neural_network', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/bayesian_neural_network/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.bayesian_neural_network" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="bayesian_neural_network_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --max_steps=3 \ 41 | --viz_steps=3 42 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/bayesian_neural_network/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.bayesian_neural_network" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="bayesian_neural_network_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --max_steps=3 \ 41 | --viz_steps=3 42 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/bayesian_neural_network/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/probability/bayesian_neural_network/trainer/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/bayesian_neural_network/trainer/tfgfile_wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from functools import wraps 16 | import tensorflow as tf 17 | 18 | 19 | def tfgfile_wrapper(f): 20 | # When an example writes to local disk, change it to write to GCS. This assumes the filename argument is called 'fname' and is passed in either as a keyword argument or as the last non-keyword argument. 21 | @wraps(f) 22 | def wrapper(*args, **kwargs): 23 | if 'fname' in kwargs: 24 | fname = kwargs.pop('fname') 25 | else: 26 | args = list(args) 27 | fname = args.pop(-1) 28 | with tf.gfile.GFile(fname, 'w') as fobj: 29 | kwargs['fname'] = fobj 30 | return_value = f(*args, **kwargs) 31 | return return_value 32 | 33 | return wrapper 34 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/deep_exponential_family/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: deep_exponential_family 2 | 3 | This is an automatically created sample based on [tensorflow/probability/tensorflow_probability/examples/deep_exponential_family.py](https://github.com/tensorflow/probability/blob/r0.7/tensorflow_probability/examples/deep_exponential_family.py). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/deep_exponential_family/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/deep_exponential_family/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = ['tensorflow_probability==0.7'] 19 | 20 | setuptools.setup(name='tensorflow_probability_deep_exponential_family', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/deep_exponential_family/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.deep_exponential_family" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="deep_exponential_family_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --fake_data \ 41 | --max_steps=1000 \ 42 | --layer_sizes=5,3,2 43 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/deep_exponential_family/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.deep_exponential_family" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="deep_exponential_family_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --fake_data \ 41 | --max_steps=1000 \ 42 | --layer_sizes=5,3,2 43 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/deep_exponential_family/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/probability/deep_exponential_family/trainer/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/generative_adversarial_network/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: generative_adversarial_network 2 | 3 | This is an automatically created sample based on [tensorflow/probability/tensorflow_probability/examples/generative_adversarial_network.py](https://github.com/tensorflow/probability/blob/r0.7/tensorflow_probability/examples/generative_adversarial_network.py). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/generative_adversarial_network/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/generative_adversarial_network/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = ['matplotlib==2.2.4','tensorflow_probability==0.7'] 19 | 20 | setuptools.setup(name='tensorflow_probability_generative_adversarial_network', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/generative_adversarial_network/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.generative_adversarial_network" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="generative_adversarial_network_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --fake_data \ 41 | --max_steps=5 \ 42 | --viz_steps=5 43 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/generative_adversarial_network/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.generative_adversarial_network" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="generative_adversarial_network_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --fake_data \ 41 | --max_steps=5 \ 42 | --viz_steps=5 43 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/generative_adversarial_network/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/probability/generative_adversarial_network/trainer/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/generative_adversarial_network/trainer/tfgfile_wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from functools import wraps 16 | import tensorflow as tf 17 | 18 | 19 | def tfgfile_wrapper(f): 20 | # When an example writes to local disk, change it to write to GCS. This assumes the filename argument is called 'fname' and is passed in either as a keyword argument or as the last non-keyword argument. 21 | @wraps(f) 22 | def wrapper(*args, **kwargs): 23 | if 'fname' in kwargs: 24 | fname = kwargs.pop('fname') 25 | else: 26 | args = list(args) 27 | fname = args.pop(-1) 28 | with tf.gfile.GFile(fname, 'w') as fobj: 29 | kwargs['fname'] = fobj 30 | return_value = f(*args, **kwargs) 31 | return return_value 32 | 33 | return wrapper 34 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/grammar_vae/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: grammar_vae 2 | 3 | This is an automatically created sample based on [tensorflow/probability/tensorflow_probability/examples/grammar_vae.py](https://github.com/tensorflow/probability/blob/r0.7/tensorflow_probability/examples/grammar_vae.py). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/grammar_vae/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/grammar_vae/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = ['tensorflow_probability==0.7'] 19 | 20 | setuptools.setup(name='tensorflow_probability_grammar_vae', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/grammar_vae/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.grammar_vae" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="grammar_vae_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --max_steps=5 \ 41 | --latent_size=2 \ 42 | --num_units=3 43 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/grammar_vae/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.grammar_vae" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="grammar_vae_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --max_steps=5 \ 41 | --latent_size=2 \ 42 | --num_units=3 43 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/grammar_vae/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/probability/grammar_vae/trainer/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/latent_dirichlet_allocation_distributions/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: latent_dirichlet_allocation_distributions 2 | 3 | This is an automatically created sample based on [tensorflow/probability/tensorflow_probability/examples/latent_dirichlet_allocation_distributions.py](https://github.com/tensorflow/probability/blob/r0.7/tensorflow_probability/examples/latent_dirichlet_allocation_distributions.py). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/latent_dirichlet_allocation_distributions/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/latent_dirichlet_allocation_distributions/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = ['tensorflow_probability==0.7'] 19 | 20 | setuptools.setup(name='tensorflow_probability_latent_dirichlet_allocation_distributions', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/latent_dirichlet_allocation_distributions/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.latent_dirichlet_allocation_distributions" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="latent_dirichlet_allocation_distributions_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --fake_data \ 41 | --max_steps=5 \ 42 | --delete_existing \ 43 | --viz_steps=5 44 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/latent_dirichlet_allocation_distributions/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.latent_dirichlet_allocation_distributions" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="latent_dirichlet_allocation_distributions_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --fake_data \ 41 | --max_steps=5 \ 42 | --delete_existing \ 43 | --viz_steps=5 44 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/latent_dirichlet_allocation_distributions/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/probability/latent_dirichlet_allocation_distributions/trainer/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/latent_dirichlet_allocation_edward2/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: latent_dirichlet_allocation_edward2 2 | 3 | This is an automatically created sample based on [tensorflow/probability/tensorflow_probability/examples/latent_dirichlet_allocation_edward2.py](https://github.com/tensorflow/probability/blob/r0.7/tensorflow_probability/examples/latent_dirichlet_allocation_edward2.py). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/latent_dirichlet_allocation_edward2/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/latent_dirichlet_allocation_edward2/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = ['tensorflow_probability==0.7'] 19 | 20 | setuptools.setup(name='tensorflow_probability_latent_dirichlet_allocation_edward2', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/latent_dirichlet_allocation_edward2/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.latent_dirichlet_allocation_edward2" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="latent_dirichlet_allocation_edward2_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --fake_data \ 41 | --max_steps=5 \ 42 | --delete_existing \ 43 | --viz_steps=5 44 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/latent_dirichlet_allocation_edward2/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.latent_dirichlet_allocation_edward2" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="latent_dirichlet_allocation_edward2_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --fake_data \ 41 | --max_steps=5 \ 42 | --delete_existing \ 43 | --viz_steps=5 44 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/latent_dirichlet_allocation_edward2/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/probability/latent_dirichlet_allocation_edward2/trainer/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/logistic_regression/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: logistic_regression 2 | 3 | This is an automatically created sample based on [tensorflow/probability/tensorflow_probability/examples/logistic_regression.py](https://github.com/tensorflow/probability/blob/r0.7/tensorflow_probability/examples/logistic_regression.py). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/logistic_regression/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/logistic_regression/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = ['matplotlib==2.2.4','tensorflow_probability==0.7'] 19 | 20 | setuptools.setup(name='tensorflow_probability_logistic_regression', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/logistic_regression/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.logistic_regression" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="logistic_regression_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --num_examples=32 \ 41 | --batch_size=8 \ 42 | --max_steps=50 43 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/logistic_regression/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.logistic_regression" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="logistic_regression_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --num_examples=32 \ 41 | --batch_size=8 \ 42 | --max_steps=50 43 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/logistic_regression/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/probability/logistic_regression/trainer/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/logistic_regression/trainer/tfgfile_wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from functools import wraps 16 | import tensorflow as tf 17 | 18 | 19 | def tfgfile_wrapper(f): 20 | # When an example writes to local disk, change it to write to GCS. This assumes the filename argument is called 'fname' and is passed in either as a keyword argument or as the last non-keyword argument. 21 | @wraps(f) 22 | def wrapper(*args, **kwargs): 23 | if 'fname' in kwargs: 24 | fname = kwargs.pop('fname') 25 | else: 26 | args = list(args) 27 | fname = args.pop(-1) 28 | with tf.gfile.GFile(fname, 'w') as fobj: 29 | kwargs['fname'] = fobj 30 | return_value = f(*args, **kwargs) 31 | return return_value 32 | 33 | return wrapper 34 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/vae/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: vae 2 | 3 | This is an automatically created sample based on [tensorflow/probability/tensorflow_probability/examples/vae.py](https://github.com/tensorflow/probability/blob/r0.7/tensorflow_probability/examples/vae.py). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/vae/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/vae/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = ['tensorflow_probability==0.7'] 19 | 20 | setuptools.setup(name='tensorflow_probability_vae', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/vae/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.vae" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="vae_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --fake_data \ 41 | --max_steps=5 \ 42 | --delete_existing \ 43 | --viz_steps=5 44 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/vae/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.vae" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="vae_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --fake_data \ 41 | --max_steps=5 \ 42 | --delete_existing \ 43 | --viz_steps=5 44 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/vae/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/probability/vae/trainer/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/vq_vae/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: vq_vae 2 | 3 | This is an automatically created sample based on [tensorflow/probability/tensorflow_probability/examples/vq_vae.py](https://github.com/tensorflow/probability/blob/r0.7/tensorflow_probability/examples/vq_vae.py). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/vq_vae/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/vq_vae/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = ['matplotlib==2.2.4','tensorflow_probability==0.7'] 19 | 20 | setuptools.setup(name='tensorflow_probability_vq_vae', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/vq_vae/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.vq_vae" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="vq_vae_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --max_steps=2 \ 41 | --base_depth=2 42 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/vq_vae/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="trainer" 20 | MODULE_NAME="trainer.vq_vae" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="vq_vae_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version 1.14 \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR \ 40 | --max_steps=2 \ 41 | --base_depth=2 42 | -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/vq_vae/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tensorflow/probability/vq_vae/trainer/__init__.py -------------------------------------------------------------------------------- /example_zoo/tensorflow/probability/vq_vae/trainer/tfgfile_wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from functools import wraps 16 | import tensorflow as tf 17 | 18 | 19 | def tfgfile_wrapper(f): 20 | # When an example writes to local disk, change it to write to GCS. This assumes the filename argument is called 'fname' and is passed in either as a keyword argument or as the last non-keyword argument. 21 | @wraps(f) 22 | def wrapper(*args, **kwargs): 23 | if 'fname' in kwargs: 24 | fname = kwargs.pop('fname') 25 | else: 26 | args = list(args) 27 | fname = args.pop(-1) 28 | with tf.gfile.GFile(fname, 'w') as fobj: 29 | kwargs['fname'] = fobj 30 | return_value = f(*args, **kwargs) 31 | return return_value 32 | 33 | return wrapper 34 | -------------------------------------------------------------------------------- /example_zoo/tools/templates/README.md: -------------------------------------------------------------------------------- 1 | # AI Platform Example Zoo: {name} 2 | 3 | This is an automatically created sample based on [{org}/{repository}/{full_path}]({web_url}). 4 | 5 | To run the sample: 6 | 7 | 8 | 1. Update [submit_27.sh](submit_27.sh) (or [submit_35.sh](submit_35.sh) for Python 3.5) with GCS bucket and GCP project ID: 9 | 10 | ``` 11 | # in submit_27.sh 12 | 13 | BUCKET=gs://your-bucket/your- 14 | PROJECT_ID=your-gcp-project-id 15 | ``` 16 | 17 | 1. Submit the job: 18 | 19 | ``` 20 | bash submit_27.sh 21 | ``` 22 | -------------------------------------------------------------------------------- /example_zoo/tools/templates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/example_zoo/tools/templates/__init__.py -------------------------------------------------------------------------------- /example_zoo/tools/templates/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trainingInput: 16 | scaleTier: BASIC 17 | -------------------------------------------------------------------------------- /example_zoo/tools/templates/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import setuptools 17 | 18 | REQUIRES = [{requires}] 19 | 20 | setuptools.setup(name='{org}_{repository}_{name}', 21 | version='0.0.1', 22 | install_requires=REQUIRES, 23 | packages=setuptools.find_packages() 24 | ) 25 | -------------------------------------------------------------------------------- /example_zoo/tools/templates/submit_27.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="{output_package_path}" 20 | MODULE_NAME="{module_name}" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="{name}_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version {runtime_version} \ 36 | --python-version 2.7 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR{args} 40 | -------------------------------------------------------------------------------- /example_zoo/tools/templates/submit_35.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | BUCKET=$EXAMPLE_ZOO_ARTIFACTS_BUCKET 17 | PROJECT_ID=$EXAMPLE_ZOO_PROJECT_ID 18 | 19 | PACKAGE_PATH="{output_package_path}" 20 | MODULE_NAME="{module_name}" 21 | 22 | now=$(date +"%Y%m%d_%H%M%S") 23 | JOB_NAME="{name}_$now" 24 | 25 | JOB_DIR=$BUCKET"/"$JOB_NAME"/" 26 | 27 | gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS 28 | 29 | gcloud ai-platform jobs submit training $JOB_NAME \ 30 | --job-dir $JOB_DIR \ 31 | --package-path $PACKAGE_PATH \ 32 | --module-name $MODULE_NAME \ 33 | --region us-central1 \ 34 | --config config.yaml \ 35 | --runtime-version {runtime_version} \ 36 | --python-version 3.5 \ 37 | --project $PROJECT_ID \ 38 | -- \ 39 | --model_dir=$JOB_DIR{args} 40 | -------------------------------------------------------------------------------- /example_zoo/tools/templates/tfgfile_wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from functools import wraps 16 | import tensorflow as tf 17 | 18 | 19 | def tfgfile_wrapper(f): 20 | # When an example writes to local disk, change it to write to GCS. This assumes the filename argument is called 'fname' and is passed in either as a keyword argument or as the last non-keyword argument. 21 | @wraps(f) 22 | def wrapper(*args, **kwargs): 23 | if 'fname' in kwargs: 24 | fname = kwargs.pop('fname') 25 | else: 26 | args = list(args) 27 | fname = args.pop(-1) 28 | with tf.gfile.GFile(fname, 'w') as fobj: 29 | kwargs['fname'] = fobj 30 | return_value = f(*args, **kwargs) 31 | return return_value 32 | 33 | return wrapper 34 | -------------------------------------------------------------------------------- /example_zoo/tools/tf_models_samples.yaml: -------------------------------------------------------------------------------- 1 | org: tensorflow 2 | repository: models 3 | branch: "r1.13.0" 4 | runtime_version: "1.13" 5 | samples: 6 | - script_path: official/mnist 7 | script_name: mnist.py 8 | args: 9 | - "--export_dir=$JOB_DIR" 10 | - "--train_epochs=1" 11 | artifact: saved_model.pb 12 | wait_time: 300 13 | 14 | - script_path: official/resnet/keras 15 | script_name: keras_cifar_main.py 16 | replace: 17 | - - "return run(flags.FLAGS)" 18 | - "run(flags.FLAGS)" 19 | args: 20 | - "--export_dir=$JOB_DIR" 21 | - "--train_steps=100" 22 | - "--skip_eval" 23 | - "--use_synthetic_data" 24 | artifact: events 25 | wait_time: 180 26 | 27 | - script_path: official/resnet/keras 28 | script_name: keras_imagenet_main.py 29 | replace: 30 | - - "return run(flags.FLAGS)" 31 | - "run(flags.FLAGS)" 32 | args: 33 | - "--export_dir=$JOB_DIR" 34 | - "--train_steps=100" 35 | - "--skip_eval" 36 | - "--use_synthetic_data" 37 | artifact: events 38 | wait_time: 180 39 | 40 | - script_path: official/recommendation 41 | script_name: ncf_main.py 42 | requires: 43 | - "typing==3.7.4" 44 | args: 45 | - "--train_epochs=1" 46 | - "--use_synthetic_data" 47 | artifact: timeline-1000.json 48 | wait_time: 180 49 | -------------------------------------------------------------------------------- /gce/burst-training/build-image.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | #!/bin/bash 16 | 17 | IMAGE_INSTANCE_NAME=$1 18 | IMAGE_FAMILY=$2 19 | IMAGE_NAME="${IMAGE_FAMILY}-$(date +%s)" 20 | 21 | gcloud compute instances create --image-family $IMAGE_FAMILY --metadata-from-file startup-script=image.sh $IMAGE_INSTANCE_NAME 22 | 23 | while : 24 | do 25 | sleep 30 26 | echo "Polling for status of $IMAGE_INSTANCE_NAME ..." 27 | STATUS=$(gcloud compute instances list --filter $IMAGE_INSTANCE_NAME --format 'value(STATUS)') 28 | echo "Status: $STATUS" 29 | if [[ $STATUS == "TERMINATED" ]]; then 30 | break 31 | fi 32 | done 33 | 34 | gcloud compute images create --source-disk $IMAGE_INSTANCE_NAME --family $IMAGE_FAMILY $IMAGE_NAME 35 | 36 | gcloud compute instances delete $IMAGE_INSTANCE_NAME --quiet 37 | 38 | -------------------------------------------------------------------------------- /gce/burst-training/census-startup.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | #!/bin/bash 16 | 17 | ### Metadata specification 18 | # All this metadata is pulled from the Compute Engine instance metadata server 19 | 20 | # GCS path to training script 21 | TRAINING_SCRIPT_DIR=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/TRAINING_SCRIPT_DIR -H "Metadata-Flavor: Google") 22 | TRAINING_SCRIPT_FILE=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/TRAINING_SCRIPT_FILE -H "Metadata-Flavor: Google") 23 | 24 | # GCS path to training and test data 25 | CENSUS_DATA_PATH=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/CENSUS_DATA_PATH -H "Metadata-Flavor: Google") 26 | 27 | # GCS path where the model should be stored 28 | MODEL_OUTPUT_PATH=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/MODEL_OUTPUT_PATH -H "Metadata-Flavor: Google") 29 | 30 | # Number of hyperparameter search iterations 31 | CV_ITERATIONS=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/CV_ITERATIONS -H "Metadata-Flavor: Google") 32 | 33 | 34 | ### Download training script 35 | gsutil cp "$TRAINING_SCRIPT_DIR/$TRAINING_SCRIPT_FILE" . 36 | 37 | ### Run training script 38 | python $TRAINING_SCRIPT_FILE \ 39 | --census-data-path $CENSUS_DATA_PATH \ 40 | --model-output-path $MODEL_OUTPUT_PATH \ 41 | --cv-iterations $CV_ITERATIONS 42 | 43 | 44 | ### Shutdown GCE instance 45 | sudo shutdown -h now 46 | 47 | -------------------------------------------------------------------------------- /gce/burst-training/image.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | #!/bin/bash 16 | 17 | sudo apt-get update && apt-get install -y python-pip 18 | 19 | sudo pip install numpy pandas scipy scikit-learn tensorflow xgboost 20 | 21 | sudo shutdown -h now 22 | 23 | -------------------------------------------------------------------------------- /gce/burst-training/requirements.txt: -------------------------------------------------------------------------------- 1 | backports.weakref==1.0.post1 2 | bleach==3.1.4 3 | enum34==1.1.6 4 | funcsigs==1.0.2 5 | futures==3.2.0 6 | html5lib==0.9999999 7 | Markdown==2.6.10 8 | mock==2.0.0 9 | numpy==1.13.3 10 | pandas==0.21.1 11 | pbr==3.1.1 12 | protobuf==3.5.0.post1 13 | python-dateutil==2.6.1 14 | pytz==2017.3 15 | scikit-learn==0.19.1 16 | scipy==1.0.0 17 | six==1.11.0 18 | tensorflow==1.15.2 19 | tensorflow-tensorboard==0.4.0rc3 20 | Werkzeug==0.15.3 21 | -------------------------------------------------------------------------------- /gce/burst-training/train.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | #/bin/bash 16 | 17 | ### Arguments 18 | INSTANCE_NAME=$1 19 | BUCKET_NAME=$2 20 | IMAGE_FAMILY=$3 21 | 22 | TIMESTAMP=$(date +%s) 23 | 24 | gsutil cp census-analysis.py "$BUCKET_NAME/" 25 | 26 | gcloud compute instances create \ 27 | --machine-type=n1-standard-64 \ 28 | --image-family=$IMAGE_FAMILY \ 29 | --metadata-from-file startup-script=census-startup.sh \ 30 | --metadata TRAINING_SCRIPT_DIR=$BUCKET_NAME,TRAINING_SCRIPT_FILE=census-analysis.py,CENSUS_DATA_PATH=$BUCKET_NAME/census,MODEL_OUTPUT_PATH=$BUCKET_NAME/census-$TIMESTAMP.model,CV_ITERATIONS=300 \ 31 | --scopes=cloud-platform \ 32 | --preemptible \ 33 | $INSTANCE_NAME 34 | -------------------------------------------------------------------------------- /gce/survival-training/README-sklearn.md: -------------------------------------------------------------------------------- 1 | # GCE survival training: sklearn 2 | -------------------------------------------------------------------------------- /gce/survival-training/gce/tf-estimator-instance-template.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 Google Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | ### Metadata specification 19 | 20 | GCE_USER=$GCE_USER 21 | DATA_DIR=$DATA_DIR 22 | JOB_DIR=$JOB_DIR 23 | 24 | TRAINER_GIT_PATH="https://github.com/tensorflow/models.git" 25 | TRAINER_MODULE="tutorials.image.cifar10_estimator.cifar10_main" 26 | 27 | TRAIN_STEPS=99999999 28 | NUM_GPUS=4 29 | 30 | # Hyperparameters 31 | MOMENTUM=0.9 32 | WEIGHT_DECAY=0.0002 33 | LEARNING_RATE=0.1 34 | BATCH_NORM_DECAY=0.997 35 | BATCH_NORM_EPSILON=0.00001 36 | 37 | 38 | ### Creates an instance template for specific CIFAR10 training jobs 39 | gcloud beta compute instance-templates create cifar10-default-params \ 40 | --machine-type n1-standard-16 \ 41 | --boot-disk-size 100GB \ 42 | --accelerator type=nvidia-tesla-k80,count=4 \ 43 | --image gpu-tensorflow --image-project $(gcloud config get-value project) \ 44 | --maintenance-policy TERMINATE --restart-on-failure \ 45 | --metadata "gceUser=$GCE_USER,trainerGitPath=$TRAINER_GIT_PATH,trainerModule=$TRAINER_MODULE,dataDir=$DATA_DIR,jobDir=$JOB_DIR,trainSteps=$TRAIN_STEPS,numGpus=$NUM_GPUS,momentum=$MOMENTUM,weightDecay=$WEIGHT_DECAY,learningRate=$LEARNING_RATE,batchNormDecay=$BATCH_NORM_DECAY,batchNormEpsilon=$BATCH_NORM_EPSILON" \ 46 | --metadata-from-file startup-script=./gce/tf-estimator-startup.sh \ 47 | --scopes https://www.googleapis.com/auth/cloud-platform \ 48 | --preemptible 49 | -------------------------------------------------------------------------------- /gce/survival-training/gce/tf-estimator-set-metadata.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 Google Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | ### Metadata specification 19 | 20 | GCE_USER=$GCE_USER 21 | DATA_DIR=$DATA_DIR 22 | JOB_DIR=$JOB_DIR 23 | 24 | TRAINER_GIT_PATH="https://github.com/tensorflow/models.git" 25 | TRAINER_MODULE="tutorials.image.cifar10_estimator.cifar10_main" 26 | 27 | TRAIN_STEPS=99999999 28 | NUM_GPUS=4 29 | 30 | # Hyperparameters 31 | MOMENTUM=0.9 32 | WEIGHT_DECAY=0.0002 33 | LEARNING_RATE=0.1 34 | BATCH_NORM_DECAY=0.997 35 | BATCH_NORM_EPSILON=0.00001 36 | 37 | 38 | ### Push metadata to GCE instance 39 | 40 | gcloud compute instances add-metadata cifar10-estimator --zone us-west1-b \ 41 | --metadata-from-file startup-script=./gce/tf-estimator-startup.sh \ 42 | --metadata "gceUser=$GCE_USER,trainerGitPath=$TRAINER_GIT_PATH,trainerModule=$TRAINER_MODULE,dataDir=$DATA_DIR,jobDir=$JOB_DIR,trainSteps=$TRAIN_STEPS,numGpus=$NUM_GPUS,momentum=$MOMENTUM,weightDecay=$WEIGHT_DECAY,learningRate=$LEARNING_RATE,batchNormDecay=$BATCH_NORM_DECAY,batchNormEpsilon=$BATCH_NORM_EPSILON" 43 | -------------------------------------------------------------------------------- /gce/survival-training/img/cuda.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/gce/survival-training/img/cuda.png -------------------------------------------------------------------------------- /gce/survival-training/img/nvidia-smi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/gce/survival-training/img/nvidia-smi.png -------------------------------------------------------------------------------- /gce/survival-training/img/pvm-training.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/gce/survival-training/img/pvm-training.png -------------------------------------------------------------------------------- /gce/survival-training/img/tf-estimator-instance-creation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/gce/survival-training/img/tf-estimator-instance-creation.png -------------------------------------------------------------------------------- /gce/survival-training/img/tf-proof.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/gce/survival-training/img/tf-proof.png -------------------------------------------------------------------------------- /gce/survival-training/wrapper/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /gcf/gcf-ai-platform-example/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | #!include:.gitignore 18 | -------------------------------------------------------------------------------- /gcf/gcf-ai-platform-example/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /gcf/gcf-ai-platform-example/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Caravel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /gcf/gcf-ai-platform-example/README.md: -------------------------------------------------------------------------------- 1 | # Google Cloud Function Examples 2 | 3 | Example code to infer Google Cloud AI Platform models from Google Cloud Functions 4 | 5 | ### Example Endpoints 6 | 7 | `main.py` contains the example code for two endpoints: 8 | 9 | * Inference Endpoint: get_demo_inference_endpoint 10 | * Model Meta Information Endpoint: get_demo_inference_meta_endpoint 11 | 12 | 13 | ### Deployment 14 | 15 | Set your project as default GCP project with 16 | ``` 17 | $ gcloud config set project 18 | ``` 19 | 20 | and then deploy the endpoints with 21 | ``` 22 | $ gcloud functions deploy --entry-point --runtime python37 --trigger-http 23 | ``` 24 | 25 | #### Deployment of the Inference Endpoint 26 | 27 | ``` 28 | $ gcloud functions deploy demo_inference --entry-point get_demo_inference_endpoint --runtime python37 --trigger-http 29 | ``` 30 | 31 | #### Deployment of the Model Meta Information Endpoint 32 | 33 | ``` 34 | $ gcloud functions deploy demo_inference_meta --entry-point get_demo_inference_meta_endpoint --runtime python37 --trigger-http 35 | ``` -------------------------------------------------------------------------------- /gcf/gcf-ai-platform-example/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==1.0.2 2 | google-api-python-client==1.7.8 3 | -------------------------------------------------------------------------------- /third_party/gpu-burn/LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2017, Ville Timonen 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /third_party/gpu-burn/Makefile: -------------------------------------------------------------------------------- 1 | CUDAPATH=/usr/local/cuda 2 | 3 | # Have this point to an old enough gcc (for nvcc) 4 | GCCPATH=/usr 5 | 6 | NVCC=nvcc 7 | CCPATH=${GCCPATH}/bin 8 | 9 | drv: 10 | PATH=${PATH}:.:${CCPATH}:${PATH} ${NVCC} -I${CUDAPATH}/include -arch=compute_30 -ptx compare.cu -o compare.ptx 11 | g++ -O3 -Wno-unused-result -I${CUDAPATH}/include -c gpu_burn-drv.cpp 12 | g++ -o gpu_burn gpu_burn-drv.o -O3 -lcuda -L${CUDAPATH}/lib64 -L${CUDAPATH}/lib -Wl,-rpath=${CUDAPATH}/lib64 -Wl,-rpath=${CUDAPATH}/lib -lcublas -lcudart -o gpu_burn 13 | -------------------------------------------------------------------------------- /third_party/gpu-burn/README.md: -------------------------------------------------------------------------------- 1 | # gpu-burn 2 | Multi-GPU CUDA stress test 3 | http://wili.cc/blog/gpu-burn.html 4 | -------------------------------------------------------------------------------- /third_party/rapids/README.md: -------------------------------------------------------------------------------- 1 | # Parallel sum reduction 2 | Python script to perform parallel sum reduction test 3 | http://matthewrocklin.com/blog/work/2019/01/03/dask-array-gpus-first-steps -------------------------------------------------------------------------------- /third_party/rapids/distributed/start-dask-cuda-worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS-IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # This script starts Dask cuda worker. 16 | trap "rm .f 2> /dev/null; exit" 0 1 3 17 | 18 | export CUDA_ROOT=/usr/local/cuda 19 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_ROOT/lib64 20 | export NUMBAPRO_NVVM=$CUDA_ROOT/nvvm/lib64/libnvvm.so 21 | export NUMBAPRO_LIBDEVICE=$CUDA_ROOT/nvvm/libdevice 22 | 23 | function err() { 24 | # Handle error 25 | echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $@" >&2 26 | exit 1 27 | } 28 | 29 | function start_dask_cuda() { 30 | local SCHEDULER_IPADDR=$1 31 | local SCHEDULER_PORT=$2 32 | /opt/anaconda3/bin/dask-cuda-worker ${SCHEDULER_IPADDR}:${SCHEDULER_PORT} 33 | } 34 | 35 | function main() { 36 | start_dask_cuda $1 $2 || err "Unable to start CUDA worker" 37 | } 38 | 39 | if [ -z "$1" ] || [ -z "$2" ]; then 40 | echo "Improper arguments supplied..." 41 | echo "Exiting..." 42 | exit 43 | fi 44 | 45 | main $1 $2 -------------------------------------------------------------------------------- /third_party/rapids/distributed/start-dask-worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS-IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # This script starts Dask worker. 16 | trap "rm .f 2> /dev/null; exit" 0 1 3 17 | 18 | source /opt/anaconda3/bin/activate base 19 | 20 | export CUDA_ROOT=/usr/local/cuda 21 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_ROOT/lib64 22 | export NUMBAPRO_NVVM=$CUDA_ROOT/nvvm/lib64/libnvvm.so 23 | export NUMBAPRO_LIBDEVICE=$CUDA_ROOT/nvvm/libdevice 24 | 25 | function err() { 26 | # Handle error 27 | echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $@" >&2 28 | exit 1 29 | } 30 | 31 | function start_dask() { 32 | local SCHEDULER_IPADDR=$1 33 | local SCHEDULER_PORT=$2 34 | /opt/anaconda3/bin/dask-worker ${SCHEDULER_IPADDR}:${SCHEDULER_PORT} 35 | } 36 | 37 | function main() { 38 | start_dask $1 $2 || err "Unable to start CUDA worker" 39 | } 40 | 41 | if [ -z "$1" ] || [ -z "$2" ]; then 42 | echo "Improper arguments supplied..." 43 | echo "Exiting..." 44 | exit 45 | fi 46 | 47 | main $1 $2 -------------------------------------------------------------------------------- /third_party/rapids/distributed/start-remote-workers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS-IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # This script starts cuda in remote machines 16 | trap "rm .f 2> /dev/null; exit" 0 1 3 17 | 18 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_ROOT/lib64 19 | export NUMBAPRO_NVVM=$CUDA_ROOT/nvvm/lib64/libnvvm.so 20 | export NUMBAPRO_LIBDEVICE=$CUDA_ROOT/nvvm/libdevice 21 | 22 | export WORKERS_FILE="workers.txt" 23 | export ZONE="us-central1-b" 24 | 25 | 26 | while getopts ":cg" opt; do 27 | case ${opt} in 28 | c) 29 | WORKER=dask-worker 30 | ;; 31 | g) 32 | WORKER=dask-cuda-worker 33 | ;; 34 | \?) echo "Usage: cmd [-c] [-g]" 35 | exit 36 | ;; 37 | :) 38 | echo "Invalid option: $OPTARG requires an argument" 1>&2 39 | ;; 40 | esac 41 | done 42 | shift $((OPTIND -1)) 43 | 44 | function err() { 45 | # Handle error 46 | echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $@" >&2 47 | exit 1 48 | } 49 | 50 | function gssh() { 51 | # Connect to remote GCP instance 52 | gcloud compute ssh $1 --internal-ip --zone ${ZONE} --command "$2" 53 | } 54 | 55 | function start_cuda() { 56 | # Connect to remote GCP instance and start CUDA 57 | local SCHEDULER_IP=`hostname --all-ip-addresses | awk ' { print ( $1 ) } '` 58 | local SCHEDULER_PORT=8786 59 | 60 | # Start locally 61 | echo "Starting ${WORKER} locally" 62 | sh /home/jupyter/start-${WORKER}.sh ${SCHEDULER_IP} ${SCHEDULER_PORT} & 63 | 64 | # Start in remote nodes 65 | for i in `tail -n +2 workers.txt`; do 66 | echo "Starting ${WORKER} on node: $i" 67 | gssh $i "/home/jupyter/start-${WORKER}.sh ${SCHEDULER_IP} ${SCHEDULER_PORT}" & 68 | done 69 | } 70 | 71 | function main() { 72 | start_cuda || err "Unable to start CUDA in remote workers" 73 | } 74 | 75 | main 76 | 77 | -------------------------------------------------------------------------------- /third_party/rapids/distributed/stop-remote-workers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS-IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # This script stop cuda in remote machines 16 | trap "rm .f 2> /dev/null; exit" 0 1 3 17 | 18 | export CUDA_ROOT=/usr/local/cuda 19 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_ROOT/lib64 20 | export NUMBAPRO_NVVM=$CUDA_ROOT/nvvm/lib64/libnvvm.so 21 | export NUMBAPRO_LIBDEVICE=$CUDA_ROOT/nvvm/libdevice 22 | 23 | export WORKERS_FILE="workers.txt" 24 | export ZONE="us-central1-b" 25 | 26 | function err() { 27 | # Handle error 28 | echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $@" >&2 29 | exit 1 30 | } 31 | 32 | function gssh() { 33 | # Connect to remote GCP instance 34 | gcloud compute ssh $1 --internal-ip --zone ${ZONE} --command "$2" 35 | } 36 | 37 | function stop_cuda() { 38 | # Connect to remote GCP instance and start cuda 39 | local SCHEDULER_IP=`hostname --all-ip-addresses | awk ' { print ( $1 ) } '` 40 | local SCHEDULER_PORT=8786 41 | 42 | for i in `cat ${WORKERS_FILE}`; do 43 | echo "Stopping dask-cuda-worker on node: $i" 44 | gssh $i "pkill dask" 45 | done 46 | } 47 | 48 | function main() { 49 | stop_cuda || err "Unable to stop CUDA in remote workers" 50 | } 51 | 52 | main -------------------------------------------------------------------------------- /tutorials/R/src/caret/.ipynb_checkpoints/Dockerfile-checkpoint: -------------------------------------------------------------------------------- 1 | FROM gcr.io/deeplearning-platform-release/r-cpu 2 | 3 | RUN R -e "install.packages(c('readr', 'caret', 'xgboost', 'rjson', 'plumber'), repos='http://cran.rstudio.com/')" -------------------------------------------------------------------------------- /tutorials/R/src/caret/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM gcr.io/deeplearning-platform-release/r-cpu 2 | 3 | RUN R -e "install.packages(c('readr', 'caret', 'xgboost', 'rjson', 'plumber'), repos='http://cran.rstudio.com/')" -------------------------------------------------------------------------------- /tutorials/R/src/caret/serving/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM gcr.io/r-on-gcp/caret_base 2 | 3 | RUN mkdir -p /root 4 | COPY model_prediction.R /root 5 | COPY model_api.R /root 6 | WORKDIR /root 7 | 8 | ENV PORT 8080 9 | 10 | EXPOSE 8080 11 | 12 | ENTRYPOINT ["Rscript", "model_api.R"] 13 | -------------------------------------------------------------------------------- /tutorials/R/src/caret/serving/model_api.R: -------------------------------------------------------------------------------- 1 | library(plumber) 2 | 3 | api <- plumb("model_prediction.R") 4 | api$run(host = "0.0.0.0", port = 8080) -------------------------------------------------------------------------------- /tutorials/R/src/caret/serving/model_prediction.R: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Set the project id 16 | project_id <- "your-project-id" 17 | 18 | # Set yout GCS bucket 19 | bucket <- "your_bucket_id" 20 | 21 | model_name <- 'caret_babyweight_estimator' 22 | model_dir <- paste0('models/', model_name) 23 | gcs_model_dir <- paste0("gs://", bucket, "/models/", model_name) 24 | 25 | print("Downloading model file from GCS...") 26 | command <- paste("gsutil cp -r", gcs_model_dir, ".") 27 | system(command) 28 | print("model files downloaded.") 29 | 30 | print("Loading model ...") 31 | model <- readRDS(paste0(model_name,'/trained_model.rds')) 32 | print("Model is loaded.") 33 | 34 | #* @post /estimate 35 | estimate_babyweights <- function(req){ 36 | library("rjson") 37 | instances_json <- req$postBody 38 | instances <- jsonlite::fromJSON(instances_json) 39 | df_instances <- data.frame(instances) 40 | # fix data types 41 | boolean_columns <- c("is_male", "mother_married", "cigarette_use", "alcohol_use") 42 | for(col in boolean_columns){ 43 | df_instances[[col]] <- as.logical(df_instances[[col]]) 44 | } 45 | 46 | estimates <- predict(model, df_instances) 47 | return(estimates) 48 | 49 | } 50 | 51 | 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /tutorials/R/src/caret/training/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM gcr.io/r-on-gcp/caret_base 2 | 3 | RUN mkdir -p /root 4 | COPY model_trainer.R /root 5 | WORKDIR /root 6 | 7 | CMD ["Rscript", "model_trainer.R"] -------------------------------------------------------------------------------- /tutorials/explanations/README.md: -------------------------------------------------------------------------------- 1 | # AI Explanations samples 2 | 3 | NOTE: Future updates to these samples will be made in the [ai-platform-samples](https://github.com/GoogleCloudPlatform/ai-platform-samples/tree/master/notebooks/samples/explanations) repo. 4 | 5 | This directory contains samples for using Explanations on Cloud AI Platform. The following notebooks are available: 6 | 7 | * [Training, deploying, and explaining a tabular data model](https://github.com/GoogleCloudPlatform/ml-on-gcp/tree/master/tutorials/explanations/ai-explanations-tabular.ipynb) 8 | * [Training, deploying, and explaining an image model](https://github.com/GoogleCloudPlatform/ml-on-gcp/tree/master/tutorials/explanations/ai-explanations-image.ipynb) 9 | -------------------------------------------------------------------------------- /tutorials/pytorch/jasper/README.MD: -------------------------------------------------------------------------------- 1 | ``` 2 | # Licensed under the Apache License, Version 2.0 (the "License") 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 7 | # See the License for the specific language governing permissions and limitations under the License. 8 | ``` 9 | 10 | 11 | 12 | # Jasper Inference For TensorRT 13 | 14 | 15 | ## Create Google Cloud Compute Engine instance 16 | 17 | Create a Compute Engine with at least 100 GB Hard Disk 18 | 19 | ``` 20 | export IMAGE_FAMILY="common-cu101" 21 | export ZONE="us-central1-b" 22 | export INSTANCE_NAME="bert-experiment" 23 | export INSTANCE_TYPE="n1-standard-16" 24 | gcloud compute instances create $INSTANCE_NAME \ 25 | --zone=$ZONE \ 26 | --image-family=$IMAGE_FAMILY \ 27 | --image-project=deeplearning-platform-release \ 28 | --machine-type=$INSTANCE_TYPE \ 29 | --boot-disk-size=120GB \ 30 | --maintenance-policy TERMINATE --restart-on-failure \ 31 | --accelerator="type=nvidia-tesla-t4,count=2" \ 32 | --scopes=https://www.googleapis.com/auth/cloud-platform \ 33 | --metadata="install-nvidia-driver=True,proxy-mode=project_editors" \ 34 | --tags http-server,https-server 35 | ``` 36 | 37 | ## Login to instance 38 | 39 | Login via SSH to the compute instance: 40 | 41 | ``` 42 | function gssh() { 43 | gcloud compute ssh $1 --zone ${ZONE}; 44 | } 45 | 46 | gssh $INSTANCE_NAME 47 | ``` 48 | 49 | ## Stop Jupyter 50 | 51 | ``` 52 | sudo systemctl stop jupyter.service 53 | ``` 54 | 55 | ## From terminal 56 | 57 | Copy script `build_asset.sh` 58 | 59 | ``` 60 | gsutil cp gs://aihub/assets/script/build_asset.sh . 61 | 62 | ``` 63 | 64 | ## Run Script 65 | 66 | Run the script to: 67 | - download NGC container, 68 | - clone the DeepLearningExample repository from github 69 | - download the data and the pre-trained models 70 | - launch jupyter lab 71 | 72 | ``` 73 | sh build_asset.sh 74 | ``` 75 | 76 | 77 | _Note: Verify you open the firewall, when accessing the notebook for external. Open TCP port 8888._ 78 | -------------------------------------------------------------------------------- /tutorials/pytorch/jasper/build_asset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | echo "NVIDIA Jasper Tutorial" 19 | shopt -s expand_aliases 20 | 21 | alias jasper_folder='cd ./DeepLearningExamples/PyTorch/SpeechRecognition/Jasper' 22 | 23 | echo "Cloning NVIDIA DeepLearningExamples from github" 24 | git clone https://github.com/NVIDIA/DeepLearningExamples.git 25 | 26 | jasper_folder 27 | 28 | 29 | echo "Building the Jasper PyTorch with TRT 6 container" 30 | sh ./trt/scripts/docker/trt_build.sh 31 | 32 | 33 | mkdir data checkpoint result 34 | 35 | echo "Copying the notebook to the root directory of Jasper" 36 | cp notebooks/JasperTRT.ipynb . 37 | 38 | 39 | echo "Downloading pretrained Jasper model" 40 | wget -nc -q --show-progress -O ./checkpoint/jasper_model.zip https://api.ngc.nvidia.com/v2/models/nvidia/jasperpyt_fp16/versions/1/zip 41 | unzip -o ./checkpoint/jasper_model.zip -d ./checkpoint/ 42 | 43 | 44 | echo "Serving Jasper notebook" 45 | jupyter notebook --ip=0.0.0.0 --allow-root JasperTRT.ipynb -------------------------------------------------------------------------------- /tutorials/sklearn/gae_serve/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/tutorials/sklearn/gae_serve/__init__.py -------------------------------------------------------------------------------- /tutorials/sklearn/gae_serve/app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Google, Inc. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | --- 14 | # If this is the first Google App Engine Service you are deploying, 15 | # change 'modelserve' to 'default', since the first service must be 16 | # named 'default'. 17 | service: modelserve 18 | runtime: python 19 | env: flex 20 | entrypoint: gunicorn -b :$PORT main:app 21 | 22 | runtime_config: 23 | python_version: 3 24 | 25 | manual_scaling: 26 | instances: 1 27 | 28 | endpoints_api_service: 29 | name: modelserve-dot-PROJECT_ID.appspot.com 30 | config_id: CONFIG_ID 31 | 32 | env_variables: 33 | # The app will look for the model file at: gs://MODEL_BUCKET/MODEL_FILENAME 34 | MODEL_BUCKET: BUCKET_NAME 35 | MODEL_FILENAME: lr.pkl 36 | -------------------------------------------------------------------------------- /tutorials/sklearn/gae_serve/client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import requests 16 | import numpy as np 17 | 18 | class ModelServiceClient(object): 19 | def __init__(self, host, api_key): 20 | self.host = host 21 | self.api_key = api_key 22 | 23 | def predict(self, X): 24 | if type(X) is np.ndarray: 25 | X = X.tolist() 26 | 27 | response = requests.post(self.host + '/predict', json={'X': X}, 28 | params={'key': self.api_key}) 29 | 30 | response_json = response.json() 31 | if 'y' in response_json: 32 | return response_json['y'] 33 | else: 34 | print(response_json) 35 | -------------------------------------------------------------------------------- /tutorials/sklearn/gae_serve/default/README.md: -------------------------------------------------------------------------------- 1 | Copy over the yaml files in the root directory of this sample if you want/need to deploy the service as `default`. 2 | -------------------------------------------------------------------------------- /tutorials/sklearn/gae_serve/default/app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Google, Inc. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | --- 14 | service: default 15 | runtime: python 16 | env: flex 17 | entrypoint: gunicorn -b :$PORT main:app 18 | 19 | runtime_config: 20 | python_version: 3 21 | 22 | manual_scaling: 23 | instances: 1 24 | 25 | endpoints_api_service: 26 | name: PROJECT_ID.appspot.com 27 | config_id: CONFIG_ID 28 | 29 | env_variables: 30 | # The app will look for the model file at: gs://BUCKET_NAME/lr.pkl 31 | MODEL_BUCKET: BUCKET_NAME 32 | MODEL_FILENAME: lr.pkl 33 | -------------------------------------------------------------------------------- /tutorials/sklearn/gae_serve/default/modelserve.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | swagger: "2.0" 16 | info: 17 | title: "modelserve" 18 | description: "Get prediction from sklearn model." 19 | version: "1.0.0" 20 | host: "PROJECT_ID.appspot.com" 21 | schemes: 22 | - "https" 23 | security: 24 | - api_key: [] 25 | securityDefinitions: 26 | api_key: 27 | type: "apiKey" 28 | name: "key" 29 | in: "query" 30 | paths: 31 | "/predict": 32 | post: 33 | description: "Get prediction given X." 34 | operationId: "predict" 35 | consumes: 36 | - "application/json" 37 | produces: 38 | - "application/json" 39 | parameters: 40 | - 41 | name: "body" 42 | in: "body" 43 | required: true 44 | schema: 45 | $ref: "#/definitions/X" 46 | responses: 47 | 200: 48 | description: "Success." 49 | schema: 50 | $ref: "#/definitions/y" 51 | 400: 52 | description: "Missing X." 53 | definitions: 54 | X: 55 | type: "object" 56 | required: 57 | - "X" 58 | properties: 59 | X: 60 | type: "array" 61 | items: 62 | type: "array" 63 | items: 64 | type: "number" 65 | format: "float" 66 | y: 67 | type: "object" 68 | required: 69 | - "y" 70 | properties: 71 | y: 72 | type: "array" 73 | items: 74 | type: "number" 75 | format: "float" 76 | -------------------------------------------------------------------------------- /tutorials/sklearn/gae_serve/lr.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/tutorials/sklearn/gae_serve/lr.pkl -------------------------------------------------------------------------------- /tutorials/sklearn/gae_serve/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # [START app] 16 | import json 17 | import logging 18 | import os 19 | 20 | import pickle 21 | from flask import Flask, request 22 | from google.cloud import storage 23 | 24 | MODEL_BUCKET = os.environ['MODEL_BUCKET'] 25 | MODEL_FILENAME = os.environ['MODEL_FILENAME'] 26 | MODEL = None 27 | 28 | app = Flask(__name__) 29 | 30 | 31 | @app.before_first_request 32 | def _load_model(): 33 | global MODEL 34 | client = storage.Client() 35 | bucket = client.get_bucket(MODEL_BUCKET) 36 | blob = bucket.get_blob(MODEL_FILENAME) 37 | s = blob.download_as_string() 38 | 39 | # Note: Change the save/load mechanism according to the framework 40 | # used to build the model. 41 | MODEL = pickle.loads(s) 42 | 43 | 44 | @app.route('/', methods=['GET']) 45 | def index(): 46 | return str(MODEL), 200 47 | 48 | 49 | @app.route('/predict', methods=['POST']) 50 | def predict(): 51 | X = request.get_json()['X'] 52 | y = MODEL.predict(X).tolist() 53 | return json.dumps({'y': y}), 200 54 | 55 | 56 | @app.errorhandler(500) 57 | def server_error(e): 58 | logging.exception('An error occurred during a request.') 59 | return """ 60 | An internal error occurred: 
{}
61 | See logs for full stacktrace. 62 | """.format(e), 500 63 | 64 | 65 | if __name__ == '__main__': 66 | app.run(host='127.0.0.1', port=8080, debug=True) 67 | # [END app] 68 | -------------------------------------------------------------------------------- /tutorials/sklearn/gae_serve/modelserve.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | swagger: "2.0" 16 | info: 17 | title: "modelserve" 18 | description: "Get prediction from sklearn model." 19 | version: "1.0.0" 20 | host: "modelserve-dot-PROJECT_ID.appspot.com" 21 | schemes: 22 | - "https" 23 | security: 24 | - api_key: [] 25 | securityDefinitions: 26 | api_key: 27 | type: "apiKey" 28 | name: "key" 29 | in: "query" 30 | x-google-management: 31 | metrics: 32 | - name: "modelserve-predict" 33 | displayName: "modelserve predict" 34 | valueType: INT64 35 | metricKind: DELTA 36 | quota: 37 | limits: 38 | - name: "modelserve-predict-limit" 39 | metric: "modelserve-predict" 40 | unit: "1/min/{project}" 41 | values: 42 | STANDARD: 1000 43 | paths: 44 | "/predict": 45 | post: 46 | description: "Get prediction given X." 47 | operationId: "predict" 48 | consumes: 49 | - "application/json" 50 | produces: 51 | - "application/json" 52 | parameters: 53 | - 54 | name: "body" 55 | in: "body" 56 | required: true 57 | schema: 58 | $ref: "#/definitions/X" 59 | responses: 60 | 200: 61 | description: "Success." 62 | schema: 63 | $ref: "#/definitions/y" 64 | 400: 65 | description: "Missing X." 66 | x-google-quota: 67 | metricCosts: 68 | modelserve-predict: 1 69 | definitions: 70 | X: 71 | type: "object" 72 | required: 73 | - "X" 74 | properties: 75 | X: 76 | type: "array" 77 | items: 78 | type: "array" 79 | items: 80 | type: "number" 81 | format: "float" 82 | y: 83 | type: "object" 84 | required: 85 | - "y" 86 | properties: 87 | y: 88 | type: "array" 89 | items: 90 | type: "number" 91 | format: "float" 92 | -------------------------------------------------------------------------------- /tutorials/sklearn/gae_serve/requirements.txt: -------------------------------------------------------------------------------- 1 | flask>=1.0.0 2 | gunicorn==19.7.1 3 | 4 | scikit-learn==0.19.0 5 | numpy==1.13.1 6 | scipy==0.19.1 7 | 8 | google-cloud-storage==1.2.0 9 | -------------------------------------------------------------------------------- /tutorials/sklearn/hpsearch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/tutorials/sklearn/hpsearch/__init__.py -------------------------------------------------------------------------------- /tutorials/sklearn/hpsearch/get_cluster_credentials.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | CLUSTER_ID=$1 4 | ZONE=$2 5 | gcloud container clusters get-credentials --zone $ZONE $CLUSTER_ID 6 | 7 | CLUSTER_NAME=`kubectl config get-clusters | grep $CLUSTER_ID` 8 | 9 | kubectl config set-context $CLUSTER_NAME 10 | kubectl get nodes 11 | -------------------------------------------------------------------------------- /tutorials/sklearn/hpsearch/helpers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/tutorials/sklearn/hpsearch/helpers/__init__.py -------------------------------------------------------------------------------- /tutorials/sklearn/hpsearch/helpers/cloudbuild_helper.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Helpers for accessing Google Cloud Container Builder in Python code. 16 | 17 | `build`: Builds a Docker image with the content and Dockerfile in 18 | `source_dir`, and pushes the image to Google Cloud Container Registry. 19 | 20 | For more information: 21 | https://cloud.google.com/container-builder/ 22 | https://cloud.google.com/container-registry/ 23 | """ 24 | 25 | from googleapiclient import discovery 26 | from oauth2client.client import GoogleCredentials 27 | 28 | from gcs_helper import archive_and_upload 29 | 30 | credentials = GoogleCredentials.get_application_default() 31 | 32 | 33 | def _make_body(source_bucket_name, source_object_name, image_name): 34 | body = { 35 | 'source': { 36 | 'storageSource': { 37 | 'bucket': source_bucket_name, 38 | 'object': source_object_name 39 | } 40 | }, 41 | 'steps': [ 42 | { 43 | 'name': 'gcr.io/cloud-builders/docker', 44 | 'args': ['build', '-t', 'gcr.io/$PROJECT_ID/{}'.format(image_name), '.'] 45 | } 46 | ], 47 | 'images': [ 48 | 'gcr.io/$PROJECT_ID/{}'.format(image_name) 49 | ] 50 | } 51 | return body 52 | 53 | 54 | def build(project_id, source_dir, bucket_name, image_name): 55 | """This uses the provided bucket (must be writable by the project) 56 | to store the intermediate source archive, and then builds an image using 57 | the Dockerfile in source_dir. 58 | 59 | If the build is successful the image will be pushed to container registry. 60 | """ 61 | archive_and_upload(bucket_name, source_dir) 62 | 63 | body = _make_body(bucket_name, '{}.zip'.format(source_dir), image_name) 64 | 65 | service = discovery.build('cloudbuild', 'v1', credentials=credentials) 66 | build = service.projects().builds().create(projectId=project_id, body=body).execute() 67 | 68 | return build 69 | 70 | -------------------------------------------------------------------------------- /tutorials/sklearn/hpsearch/requirements.txt: -------------------------------------------------------------------------------- 1 | google-api-python-client==1.6.3 2 | google-cloud-storage==1.4.0 3 | kubernetes==3.0.0 4 | 5 | numpy==1.13.1 6 | scipy==0.19.1 7 | pyyaml>=4.2b1 8 | scikit-learn==0.19.0 9 | scikit-optimize==0.4 10 | -------------------------------------------------------------------------------- /tutorials/sklearn/hpsearch/source/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2.7.13 2 | 3 | COPY requirements.txt ./ 4 | 5 | # This is required for installing scikit-optimize 6 | RUN pip install numpy==1.13.1 7 | 8 | RUN pip install --no-cache-dir -r requirements.txt 9 | 10 | COPY worker.py ./ 11 | 12 | COPY gcs_helper.py ./ 13 | 14 | # The Command is specified in `../gke_parallel.py` at job deployment time in 15 | # order to inject data location and other metadata. -------------------------------------------------------------------------------- /tutorials/sklearn/hpsearch/source/requirements.txt: -------------------------------------------------------------------------------- 1 | google-cloud-storage==1.4.0 2 | google-api-python-client==1.6.3 3 | 4 | numpy==1.13.1 5 | scipy==0.19.1 6 | 7 | scikit-learn==0.19.0 8 | scikit-optimize==0.4 -------------------------------------------------------------------------------- /tutorials/sklearn/titanic/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.14.0 2 | pandas==0.22.0 3 | scikit-learn==0.19.1 4 | scipy==1.0.0 5 | tensorflow==1.15.2 6 | -------------------------------------------------------------------------------- /tutorials/tensorflow/bert/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS-IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | # Build Docker container: 13 | # docker build --build-arg LD_LIBRARY_PATH=$LD_LIBRARY_PATH -t gcr.io//tensorrt_bert_sample:staging ./ 14 | # 15 | FROM gcr.io/deeplearning-platform-release/base-cpu 16 | FROM gcr.io/dpe-cloud-mle/tensorrt_bert_sample:dev 17 | 18 | # AI Hub Plugin 19 | COPY --from=0 /root/miniconda3/lib/python3.7/site-packages/jupyter_aihub_deploy_extension /usr/local/lib/python3.6/dist-packages/jupyter_aihub_deploy_extension 20 | RUN pip install requests 21 | 22 | COPY ./script.sh /usr/local/bin/start.sh 23 | ARG LD_LIBRARY_PATH 24 | ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/tensorrt/lib 25 | 26 | EXPOSE 8080 27 | ENTRYPOINT /usr/local/bin/start.sh 28 | 29 | 30 | -------------------------------------------------------------------------------- /tutorials/tensorflow/bert/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright 2019 Google LLC. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | cd /workspace/TensorRT/demo/BERT 17 | jupyter serverextension enable --py jupyter_aihub_deploy_extension 18 | jupyter lab --ip=0.0.0.0 --port=8080 --allow-root --no-browser --NotebookApp.token='' --NotebookApp.password='' 19 | -------------------------------------------------------------------------------- /tutorials/tensorflow/bert_finetuning/README.MD: -------------------------------------------------------------------------------- 1 | ``` 2 | # Licensed under the Apache License, Version 2.0 (the "License") 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 7 | # See the License for the specific language governing permissions and limitations under the License. 8 | ``` 9 | 10 | 11 | 12 | # BERT Question Answering Inference/Fine-Tuning with Mixed Precision 13 | 14 | 15 | ## Create Google Cloud Compute Engine instance 16 | 17 | Create a Compute Engine with at least 100 GB Hard Disk 18 | 19 | ``` 20 | export IMAGE_FAMILY="common-cu101" 21 | export ZONE="us-central1-b" 22 | export INSTANCE_NAME="bert-experiment" 23 | export INSTANCE_TYPE="n1-standard-16" 24 | gcloud compute instances create $INSTANCE_NAME \ 25 | --zone=$ZONE \ 26 | --image-family=$IMAGE_FAMILY \ 27 | --image-project=deeplearning-platform-release \ 28 | --machine-type=$INSTANCE_TYPE \ 29 | --boot-disk-size=120GB \ 30 | --maintenance-policy TERMINATE --restart-on-failure \ 31 | --accelerator="type=nvidia-tesla-t4,count=2" \ 32 | --scopes=https://www.googleapis.com/auth/cloud-platform \ 33 | --metadata="install-nvidia-driver=True,proxy-mode=project_editors" \ 34 | --tags http-server,https-server 35 | ``` 36 | 37 | ## Login to instance 38 | 39 | Login via SSH to the compute instance: 40 | 41 | ``` 42 | function gssh() { 43 | gcloud compute ssh $1 --zone ${ZONE}; 44 | } 45 | 46 | gssh $INSTANCE_NAME 47 | ``` 48 | 49 | ## Stop Jupyter 50 | 51 | ``` 52 | sudo systemctl stop jupyter.service 53 | ``` 54 | 55 | ## From terminal 56 | 57 | Copy script `build_asset.sh` 58 | 59 | ``` 60 | gsutil cp gs://aihub/assets/script/build_asset.sh . 61 | 62 | ``` 63 | 64 | ## Run Script 65 | 66 | Run the script to: 67 | - download NGC container, 68 | - clone the DeepLearningExample repository from github 69 | - download the data and the pre-trained models 70 | - launch jupyter lab 71 | 72 | ``` 73 | sh build_asset.sh 74 | ``` 75 | 76 | 77 | _Note: Verify you open the firewall, when accessing the notebook for external. Open TCP port 8888._ 78 | -------------------------------------------------------------------------------- /tutorials/tensorflow/bert_finetuning/build_asset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | shopt -s expand_aliases 18 | 19 | alias data_folder='cd ./DeepLearningExamples/TensorFlow/LanguageModeling/BERT/data' 20 | alias bert_folder='cd ..' 21 | 22 | 23 | # NVIDIA-DOCKER is a prerequisite to launch this script 24 | CONTAINER_VERSION='19.09-py3' 25 | 26 | echo "Downloading NVIDIA container from NGC" 27 | docker pull nvcr.io/nvidia/tensorflow:${CONTAINER_VERSION} 28 | 29 | echo "Cloning NVIDIA DeepLearningExamples from github" 30 | git clone https://github.com/NVIDIA/DeepLearningExamples.git 31 | 32 | data_folder 33 | 34 | echo "Downloading pre-trained models" 35 | python3 -c 'from GooglePretrainedWeightDownloader import GooglePretrainedWeightDownloader 36 | downloader = GooglePretrainedWeightDownloader("./download") 37 | downloader.download()' 38 | 39 | echo "Downloading SQuaD" 40 | python3 -c 'from SquadDownloader import SquadDownloader 41 | downloader = SquadDownloader("./download") 42 | downloader.download()' 43 | 44 | bert_folder 45 | 46 | mv ./notebooks/bert_squad_tf_finetuning.ipynb . 47 | 48 | echo "Running NGC container and Jupyter Lab" 49 | nvidia-docker run \ 50 | -v $PWD:/workspace/bert \ 51 | -v $PWD/results:/results \ 52 | --shm-size=1g \ 53 | --ulimit memlock=-1 \ 54 | --ulimit stack=67108864 \ 55 | --publish 0.0.0.0:8888:8888 \ 56 | -it nvcr.io/nvidia/tensorflow:${CONTAINER_VERSION} /bin/sh -c 'cd /workspace/bert; jupyter lab --ip=0.0.0.0 --port=8888 --allow-root --no-browser --NotebookApp.token="" --NotebookApp.password=""' 57 | 58 | -------------------------------------------------------------------------------- /tutorials/tensorflow/mlflow_gcp/conda/MLproject: -------------------------------------------------------------------------------- 1 | name: mlflow_gcp 2 | conda_env: 3 | conda_env: conda.yaml 4 | entry_points: 5 | main: 6 | parameters: 7 | job_dir: 8 | type: string 9 | default: '/tmp/' 10 | num_epochs: 11 | type: int 12 | default: 10 13 | train_steps: 14 | type: int 15 | default: 1000 16 | eval_steps: 17 | type: int 18 | default: 1 19 | batch_size: 20 | type: int 21 | default: 64 22 | train_files: 23 | type: string 24 | default: 'gs://cloud-samples-data/ml-engine/census/data/adult.data.csv' 25 | eval_files: 26 | type: string 27 | default: 'gs://cloud-samples-data/ml-engine/census/data/adult.test.csv' 28 | mlflow_tracking_uri: 29 | type: uri 30 | default: '' 31 | 32 | command: | 33 | python -m trainer.task --job-dir {job_dir} \ 34 | --num-epochs {num_epochs} \ 35 | --train-steps {train_steps} \ 36 | --eval-steps {eval_steps} \ 37 | --train-files {train_files} \ 38 | --eval-files {eval_files} \ 39 | --batch-size {batch_size} \ 40 | --mlflow-tracking-uri {mlflow_tracking_uri} 41 | 42 | # -------------------------------------------------------------------------------- /tutorials/tensorflow/mlflow_gcp/conda/conda.yaml: -------------------------------------------------------------------------------- 1 | name: mlflow-gcp 2 | channels: 3 | - defaults 4 | - anaconda 5 | dependencies: 6 | - python==3.7 7 | - tensorflow==1.15.0 8 | - pip: 9 | - mlflow 10 | -------------------------------------------------------------------------------- /tutorials/tensorflow/mlflow_gcp/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS-IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | # Build Docker container: 13 | # docker build -t mlflow-gcp-example -f Dockerfile . 14 | 15 | FROM gcr.io/deeplearning-platform-release/tf-cpu 16 | RUN git clone https://github.com/GoogleCloudPlatform/ml-on-gcp.git 17 | WORKDIR ml-on-gcp/tutorials/tensorflow/mlflow_gcp 18 | RUN pip install -r requirements.txt -------------------------------------------------------------------------------- /tutorials/tensorflow/mlflow_gcp/docker/MLproject: -------------------------------------------------------------------------------- 1 | name: mlflow_gcp 2 | docker_env: 3 | image: mlflow-gcp-example 4 | entry_points: 5 | main: 6 | parameters: 7 | job_dir: 8 | type: string 9 | default: '/tmp/' 10 | num_epochs: 11 | type: int 12 | default: 10 13 | train_steps: 14 | type: int 15 | default: 1000 16 | eval_steps: 17 | type: int 18 | default: 1 19 | batch_size: 20 | type: int 21 | default: 64 22 | train_files: 23 | type: string 24 | default: 'gs://cloud-samples-data/ml-engine/census/data/adult.data.csv' 25 | eval_files: 26 | type: string 27 | default: 'gs://cloud-samples-data/ml-engine/census/data/adult.test.csv' 28 | mlflow_tracking_uri: 29 | type: uri 30 | default: '' 31 | 32 | command: | 33 | python -m trainer.task --job-dir {job_dir} \ 34 | --num-epochs {num_epochs} \ 35 | --train-steps {train_steps} \ 36 | --eval-steps {eval_steps} \ 37 | --train-files {train_files} \ 38 | --eval-files {eval_files} \ 39 | --batch-size {batch_size} \ 40 | --mlflow-tracking-uri {mlflow_tracking_uri} 41 | -------------------------------------------------------------------------------- /tutorials/tensorflow/mlflow_gcp/hp_config.yaml: -------------------------------------------------------------------------------- 1 | trainingInput: 2 | hyperparameters: 3 | goal: MAXIMIZE 4 | maxTrials: 4 5 | maxParallelTrials: 2 6 | hyperparameterMetricTag: epoch_acc 7 | params: 8 | - parameterName: batch-size 9 | type: INTEGER 10 | minValue: 8 11 | maxValue: 256 12 | scaleType: UNIT_LINEAR_SCALE 13 | - parameterName: learning-rate 14 | type: DOUBLE 15 | minValue: 0.01 16 | maxValue: 0.1 17 | scaleType: UNIT_LOG_SCALE 18 | -------------------------------------------------------------------------------- /tutorials/tensorflow/mlflow_gcp/requirements.txt: -------------------------------------------------------------------------------- 1 | # The pip syntax below allows us to not repeat 2 | # In order to not maintain two separate dependency 3 | # lists in setup.py vs requirements.txt 4 | # See https://caremad.io/posts/2013/07/setup-vs-requirement/ 5 | 6 | --index-url https://pypi.python.org/simple/ 7 | 8 | -e . 9 | -------------------------------------------------------------------------------- /tutorials/tensorflow/mlflow_gcp/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2019 Google LLC. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from setuptools import find_packages 17 | from setuptools import setup 18 | 19 | REQUIRED_PACKAGES = [ 20 | 'numpy>=1.14', 21 | 'pandas>=0.22', 22 | 'six>=1.11', 23 | 'google-api-python-client', 24 | 'google-cloud-storage', 25 | 'tensorflow>=1.15,<2', 26 | 'mlflow>1.0,<2' 27 | ] 28 | 29 | setup( 30 | name='trainer', 31 | version='0.1', 32 | install_requires=REQUIRED_PACKAGES, 33 | packages=find_packages(), 34 | include_package_data=True, 35 | description='AI Platform trainer' 36 | ) 37 | -------------------------------------------------------------------------------- /tutorials/tensorflow/mlflow_gcp/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/ml-on-gcp/2e181d6f575abda12375d59e16d2d93e06700fed/tutorials/tensorflow/mlflow_gcp/trainer/__init__.py --------------------------------------------------------------------------------