├── .darglint ├── .flake8 ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── CONTRIBUTING.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── README.us.md ├── chapters ├── adv_tfserving │ ├── monitoring_config.txt │ └── prometheus.yml ├── adv_tfx │ └── Custom_TFX_Components.ipynb ├── appendix_a │ ├── app.yaml │ ├── nodeport.yaml │ └── pvc.yaml ├── appendix_c │ ├── .gitignore │ ├── README.md │ └── tfx_template_example │ │ ├── __init__.py │ │ ├── beam_dag_runner.py │ │ ├── data_validation.ipynb │ │ ├── kubeflow_dag_runner.py │ │ ├── model_analysis.ipynb │ │ ├── models │ │ ├── __init__.py │ │ ├── features.py │ │ ├── features_test.py │ │ ├── keras │ │ │ ├── __init__.py │ │ │ ├── constants.py │ │ │ ├── model.py │ │ │ └── model_test.py │ │ ├── preprocessing.py │ │ └── preprocessing_test.py │ │ └── pipeline │ │ ├── __init__.py │ │ ├── configs.py │ │ └── pipeline.py ├── data_ingestion │ ├── convert_data_to_tfrecords.py │ ├── data_ingestion.ipynb │ └── tfrecord_sample.py ├── data_privacy │ └── differential_privacy.ipynb ├── data_validation │ ├── data_validation.ipynb │ └── gcp_dataflow.py ├── intro_tfx │ └── Apache_beam_example_notebook.ipynb ├── model_analysis │ └── model_analysis.ipynb └── tfserving │ └── sample_request.py ├── components ├── __init__.py ├── keras_trainer.py ├── module.py ├── module_test.py └── transform.py ├── interactive-pipeline └── interactive_pipeline.ipynb ├── pipelines ├── __init__.py ├── apache_airflow │ ├── Dockerfile │ ├── launch_airflow.sh │ ├── pipeline_airflow.py │ ├── setup_airflow.sh │ └── setup_env.sh ├── apache_beam │ ├── __init__.py │ └── pipeline_beam.py ├── base_pipeline.py ├── gcp_cloud_ai │ └── pipeline_gcp_cloud_ai.py └── kubeflow_pipelines │ ├── __init__.py │ ├── argo_pipeline_files │ └── .gitkeep │ ├── kubeflow-config │ ├── storage-access-pod.yaml │ ├── storage-claim.yaml │ └── storage.yaml │ ├── pipeline_kubeflow.py │ ├── pipeline_kubeflow_gcp_buckets.py │ └── tfx-docker-image │ └── Dockerfile ├── pre-experiment-pipeline ├── experiment_6Mar.ipynb └── make_final_dataset.ipynb ├── pyproject.toml ├── requirements ├── requirements.txt └── test_requirements.txt └── utils ├── __init__.py └── download_dataset.py /.darglint: -------------------------------------------------------------------------------- 1 | [darglint] 2 | strictness = short 3 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/.flake8 -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/.github/workflows/ci.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/.gitignore -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/CONTRIBUTING.rst -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/README.md -------------------------------------------------------------------------------- /README.us.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/README.us.md -------------------------------------------------------------------------------- /chapters/adv_tfserving/monitoring_config.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/adv_tfserving/monitoring_config.txt -------------------------------------------------------------------------------- /chapters/adv_tfserving/prometheus.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/adv_tfserving/prometheus.yml -------------------------------------------------------------------------------- /chapters/adv_tfx/Custom_TFX_Components.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/adv_tfx/Custom_TFX_Components.ipynb -------------------------------------------------------------------------------- /chapters/appendix_a/app.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_a/app.yaml -------------------------------------------------------------------------------- /chapters/appendix_a/nodeport.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_a/nodeport.yaml -------------------------------------------------------------------------------- /chapters/appendix_a/pvc.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_a/pvc.yaml -------------------------------------------------------------------------------- /chapters/appendix_c/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/.gitignore -------------------------------------------------------------------------------- /chapters/appendix_c/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/README.md -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/__init__.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/beam_dag_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/beam_dag_runner.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/data_validation.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/data_validation.ipynb -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/kubeflow_dag_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/kubeflow_dag_runner.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/model_analysis.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/model_analysis.ipynb -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/models/__init__.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/models/features.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/models/features.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/models/features_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/models/features_test.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/models/keras/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/models/keras/__init__.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/models/keras/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/models/keras/constants.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/models/keras/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/models/keras/model.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/models/keras/model_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/models/keras/model_test.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/models/preprocessing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/models/preprocessing.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/models/preprocessing_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/models/preprocessing_test.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/pipeline/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/pipeline/__init__.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/pipeline/configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/pipeline/configs.py -------------------------------------------------------------------------------- /chapters/appendix_c/tfx_template_example/pipeline/pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/appendix_c/tfx_template_example/pipeline/pipeline.py -------------------------------------------------------------------------------- /chapters/data_ingestion/convert_data_to_tfrecords.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/data_ingestion/convert_data_to_tfrecords.py -------------------------------------------------------------------------------- /chapters/data_ingestion/data_ingestion.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/data_ingestion/data_ingestion.ipynb -------------------------------------------------------------------------------- /chapters/data_ingestion/tfrecord_sample.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/data_ingestion/tfrecord_sample.py -------------------------------------------------------------------------------- /chapters/data_privacy/differential_privacy.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/data_privacy/differential_privacy.ipynb -------------------------------------------------------------------------------- /chapters/data_validation/data_validation.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/data_validation/data_validation.ipynb -------------------------------------------------------------------------------- /chapters/data_validation/gcp_dataflow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/data_validation/gcp_dataflow.py -------------------------------------------------------------------------------- /chapters/intro_tfx/Apache_beam_example_notebook.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/intro_tfx/Apache_beam_example_notebook.ipynb -------------------------------------------------------------------------------- /chapters/model_analysis/model_analysis.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/model_analysis/model_analysis.ipynb -------------------------------------------------------------------------------- /chapters/tfserving/sample_request.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/chapters/tfserving/sample_request.py -------------------------------------------------------------------------------- /components/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /components/keras_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/components/keras_trainer.py -------------------------------------------------------------------------------- /components/module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/components/module.py -------------------------------------------------------------------------------- /components/module_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/components/module_test.py -------------------------------------------------------------------------------- /components/transform.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/components/transform.py -------------------------------------------------------------------------------- /interactive-pipeline/interactive_pipeline.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/interactive-pipeline/interactive_pipeline.ipynb -------------------------------------------------------------------------------- /pipelines/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipelines/apache_airflow/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/apache_airflow/Dockerfile -------------------------------------------------------------------------------- /pipelines/apache_airflow/launch_airflow.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/apache_airflow/launch_airflow.sh -------------------------------------------------------------------------------- /pipelines/apache_airflow/pipeline_airflow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/apache_airflow/pipeline_airflow.py -------------------------------------------------------------------------------- /pipelines/apache_airflow/setup_airflow.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/apache_airflow/setup_airflow.sh -------------------------------------------------------------------------------- /pipelines/apache_airflow/setup_env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/apache_airflow/setup_env.sh -------------------------------------------------------------------------------- /pipelines/apache_beam/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipelines/apache_beam/pipeline_beam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/apache_beam/pipeline_beam.py -------------------------------------------------------------------------------- /pipelines/base_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/base_pipeline.py -------------------------------------------------------------------------------- /pipelines/gcp_cloud_ai/pipeline_gcp_cloud_ai.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/gcp_cloud_ai/pipeline_gcp_cloud_ai.py -------------------------------------------------------------------------------- /pipelines/kubeflow_pipelines/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipelines/kubeflow_pipelines/argo_pipeline_files/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipelines/kubeflow_pipelines/kubeflow-config/storage-access-pod.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/kubeflow_pipelines/kubeflow-config/storage-access-pod.yaml -------------------------------------------------------------------------------- /pipelines/kubeflow_pipelines/kubeflow-config/storage-claim.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/kubeflow_pipelines/kubeflow-config/storage-claim.yaml -------------------------------------------------------------------------------- /pipelines/kubeflow_pipelines/kubeflow-config/storage.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/kubeflow_pipelines/kubeflow-config/storage.yaml -------------------------------------------------------------------------------- /pipelines/kubeflow_pipelines/pipeline_kubeflow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/kubeflow_pipelines/pipeline_kubeflow.py -------------------------------------------------------------------------------- /pipelines/kubeflow_pipelines/pipeline_kubeflow_gcp_buckets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/kubeflow_pipelines/pipeline_kubeflow_gcp_buckets.py -------------------------------------------------------------------------------- /pipelines/kubeflow_pipelines/tfx-docker-image/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pipelines/kubeflow_pipelines/tfx-docker-image/Dockerfile -------------------------------------------------------------------------------- /pre-experiment-pipeline/experiment_6Mar.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pre-experiment-pipeline/experiment_6Mar.ipynb -------------------------------------------------------------------------------- /pre-experiment-pipeline/make_final_dataset.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pre-experiment-pipeline/make_final_dataset.ipynb -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/pyproject.toml -------------------------------------------------------------------------------- /requirements/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/requirements/requirements.txt -------------------------------------------------------------------------------- /requirements/test_requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/requirements/test_requirements.txt -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/download_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oreilly-japan/building-ml-pipelines-ja/HEAD/utils/download_dataset.py --------------------------------------------------------------------------------