├── .gitignore ├── README.md ├── dockers ├── deploy │ ├── Dockerfile │ ├── deploy.sh │ └── Dockerize.ipynb ├── test │ ├── Dockerfile │ ├── test.py │ └── Dockerize.ipynb ├── train │ ├── Dockerfile │ ├── train.py │ └── Dockerize-train.ipynb ├── upload │ ├── Dockerfile │ ├── upload.py │ └── Dockerize.ipynb └── preprocess │ ├── Dockerfile │ ├── preprocess.py │ └── Dockerize.ipynb └── BostonHousing.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kubeflow-basic-example 2 | -------------------------------------------------------------------------------- /dockers/deploy/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM google/cloud-sdk:latest 2 | ADD deploy.sh . 3 | RUN chmod 755 /deploy.sh 4 | ENTRYPOINT ["bash","deploy.sh"] -------------------------------------------------------------------------------- /dockers/test/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-slim 2 | 3 | WORKDIR /app 4 | 5 | RUN pip install -U scikit-learn numpy 6 | 7 | COPY test.py ./test.py 8 | 9 | ENTRYPOINT [ "python", "test.py" ] -------------------------------------------------------------------------------- /dockers/train/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-slim 2 | 3 | WORKDIR /app 4 | 5 | RUN pip install -U scikit-learn numpy 6 | 7 | COPY train.py ./train.py 8 | 9 | ENTRYPOINT [ "python", "train.py" ] -------------------------------------------------------------------------------- /dockers/upload/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-slim 2 | 3 | WORKDIR /app 4 | 5 | RUN pip install -U google-cloud-storage 6 | 7 | COPY upload.py ./upload.py 8 | 9 | ENTRYPOINT [ "python", "upload.py" ] -------------------------------------------------------------------------------- /dockers/preprocess/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-slim 2 | 3 | WORKDIR /app 4 | 5 | RUN pip install -U scikit-learn numpy 6 | 7 | COPY preprocess.py ./preprocess.py 8 | 9 | ENTRYPOINT [ "python", "preprocess.py" ] -------------------------------------------------------------------------------- /dockers/deploy/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "19th of Nov" 3 | echo "Using gcloud to deploy model to ai-platform" 4 | 5 | MODEL_DIR="gs://tutorialwsc2/" 6 | VERSION_NAME="v8" 7 | MODEL_NAME="boston" 8 | FRAMEWORK="SCIKIT_LEARN" 9 | 10 | gcloud ai-platform models list 11 | 12 | gcloud ai-platform versions create $VERSION_NAME \ 13 | --model=$MODEL_NAME \ 14 | --origin=$MODEL_DIR \ 15 | --runtime-version=1.14 \ 16 | --framework=$FRAMEWORK \ 17 | --python-version=3.5 \ 18 | --region=us-central1 \ 19 | --machine-type=n1-standard-4 20 | 21 | echo "Done" 22 | -------------------------------------------------------------------------------- /dockers/train/train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import joblib 3 | import numpy as np 4 | from sklearn.linear_model import SGDRegressor 5 | 6 | def train_model(x_train, y_train): 7 | x_train_data = np.load(x_train) 8 | y_train_data = np.load(y_train) 9 | 10 | model = SGDRegressor(verbose=1) 11 | model.fit(x_train_data, y_train_data) 12 | 13 | joblib.dump(model, 'model.pkl') 14 | 15 | if __name__ == '__main__': 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument('--x_train') 18 | parser.add_argument('--y_train') 19 | args = parser.parse_args() 20 | train_model(args.x_train, args.y_train) -------------------------------------------------------------------------------- /dockers/preprocess/preprocess.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn import datasets 3 | from sklearn.model_selection import train_test_split 4 | 5 | def _preprocess_data(): 6 | X, y = datasets.load_boston(return_X_y=True) 7 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33) 8 | np.save('x_train.npy', X_train) 9 | np.save('x_test.npy', X_test) 10 | np.save('y_train.npy', y_train) 11 | np.save('y_test.npy', y_test) 12 | 13 | if __name__ == '__main__': 14 | print("Hello the 19th of Nov") 15 | print('Preprocessing data...') 16 | _preprocess_data() 17 | print('Done preprocess') -------------------------------------------------------------------------------- /dockers/upload/upload.py: -------------------------------------------------------------------------------- 1 | from google.cloud import storage 2 | import argparse 3 | 4 | def upload_blob(model): 5 | 6 | bucket_name = "tutorialwsc" 7 | source_file_name = model 8 | destination_blob_name = "model.pkl" 9 | 10 | storage_client = storage.Client() 11 | bucket = storage_client.bucket(bucket_name) 12 | print("bucket: " + str(bucket)) 13 | blob = bucket.blob(destination_blob_name) 14 | print("blob: " + str(blob)) 15 | blob.upload_from_filename(source_file_name) 16 | 17 | print( 18 | "File {} uploaded to {}.".format( 19 | source_file_name, destination_blob_name 20 | ) 21 | ) 22 | if __name__ == '__main__': 23 | parser = argparse.ArgumentParser() 24 | parser.add_argument('--model') 25 | args = parser.parse_args() 26 | upload_blob(args.model) -------------------------------------------------------------------------------- /dockers/test/test.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import joblib 4 | from sklearn.metrics import mean_squared_error 5 | import json 6 | import io 7 | 8 | def test_model(model,x_test, y_test): 9 | x_test_data = np.load(x_test) 10 | y_test_data = np.load(y_test) 11 | 12 | model = joblib.load(model) 13 | y_pred = model.predict(x_test_data) 14 | 15 | err = mean_squared_error(y_test_data, y_pred) 16 | print(str(err)) 17 | 18 | with open('output.txt', 'a') as f: 19 | f.write(str(err)) 20 | 21 | # write out metrics 22 | metrics = { 23 | 'metrics': [{ 24 | 'name': 'accuracy-score', 25 | 'numberValue': err, 26 | 'format': "PERCENTAGE", 27 | }] 28 | } 29 | 30 | with open('/mlpipeline-metrics.json', 'w') as fo: 31 | json.dump(metrics, fo) 32 | 33 | if __name__ == '__main__': 34 | parser = argparse.ArgumentParser() 35 | parser.add_argument('--model') 36 | parser.add_argument('--x_test') 37 | parser.add_argument('--y_test') 38 | args = parser.parse_args() 39 | test_model(args.model, args.x_test, args.y_test) 40 | -------------------------------------------------------------------------------- /dockers/test/Dockerize.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 21, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "Sending build context to Docker daemon 12.8kB\n", 13 | "Step 1/5 : FROM python:3.7-slim\n", 14 | " ---> 4d4a9832278b\n", 15 | "Step 2/5 : WORKDIR /app\n", 16 | " ---> Using cache\n", 17 | " ---> a44a317c92dc\n", 18 | "Step 3/5 : RUN pip install -U scikit-learn numpy\n", 19 | " ---> Using cache\n", 20 | " ---> b22be2675084\n", 21 | "Step 4/5 : COPY test.py ./test.py\n", 22 | " ---> Using cache\n", 23 | " ---> d8923136060d\n", 24 | "Step 5/5 : ENTRYPOINT [ \"python\", \"test.py\" ]\n", 25 | " ---> Using cache\n", 26 | " ---> f52c75ced916\n", 27 | "Successfully built f52c75ced916\n", 28 | "Successfully tagged test:latest\n", 29 | "The push refers to repository [gcr.io/kubeflow-demos/test]\n", 30 | "\n", 31 | "\u001b[1B9a3dd8f7: Preparing \n", 32 | "\u001b[1B18d350e5: Preparing \n", 33 | "\u001b[1Bab7313b3: Preparing \n", 34 | "\u001b[1Be5632dc8: Preparing \n", 35 | "\u001b[1B857805ec: Preparing \n", 36 | "\u001b[1B87503449: Preparing \n", 37 | "\u001b[1B6688d36c: Preparing \n", 38 | "\u001b[1Bb4339852: Layer already exists \u001b[3A\u001b[2Klatest: digest: sha256:721c54f48e329cef813e85894f47fa1c1be5e0045c4a1da4e5d0a3d3a4fa07e3 size: 1996\n" 39 | ] 40 | } 41 | ], 42 | "source": [ 43 | "!docker build -t test .\n", 44 | "!docker tag test gcr.io/kubeflow-demos/test:latest\n", 45 | "!docker push gcr.io/kubeflow-demos/test:latest" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [] 54 | } 55 | ], 56 | "metadata": { 57 | "environment": { 58 | "name": "tf2-gpu.2-1.m56", 59 | "type": "gcloud", 60 | "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-1:m56" 61 | }, 62 | "kernelspec": { 63 | "display_name": "Python 3", 64 | "language": "python", 65 | "name": "python3" 66 | }, 67 | "language_info": { 68 | "codemirror_mode": { 69 | "name": "ipython", 70 | "version": 3 71 | }, 72 | "file_extension": ".py", 73 | "mimetype": "text/x-python", 74 | "name": "python", 75 | "nbconvert_exporter": "python", 76 | "pygments_lexer": "ipython3", 77 | "version": "3.7.8" 78 | } 79 | }, 80 | "nbformat": 4, 81 | "nbformat_minor": 4 82 | } 83 | -------------------------------------------------------------------------------- /dockers/deploy/Dockerize.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 21, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "Sending build context to Docker daemon 13.82kB\n", 13 | "Step 1/4 : FROM google/cloud-sdk:latest\n", 14 | " ---> 7d28e60605c4\n", 15 | "Step 2/4 : ADD deploy.sh .\n", 16 | " ---> 451fa645edf8\n", 17 | "Step 3/4 : RUN chmod 755 /deploy.sh\n", 18 | " ---> Running in 61d319400d83\n", 19 | "Removing intermediate container 61d319400d83\n", 20 | " ---> c683ee3089f6\n", 21 | "Step 4/4 : ENTRYPOINT [\"bash\",\"deploy.sh\"]\n", 22 | " ---> Running in 068afbeffaaa\n", 23 | "Removing intermediate container 068afbeffaaa\n", 24 | " ---> a4ea575c4787\n", 25 | "Successfully built a4ea575c4787\n", 26 | "Successfully tagged deploy:latest\n", 27 | "The push refers to repository [gcr.io/kubeflow-demos/deploy]\n", 28 | "\n", 29 | "\u001b[1Bb5154d3a: Preparing \n", 30 | "\u001b[1B06e1d025: Preparing \n", 31 | "\u001b[1Bae3a0192: Preparing \n", 32 | "\u001b[1B0d22e711: Preparing \n", 33 | "\u001b[1B1dbdbff9: Preparing \n", 34 | "\u001b[1B53f4885c: Preparing \n", 35 | "\u001b[6B06e1d025: Pushed lready exists A\u001b[2K\u001b[2A\u001b[2K\u001b[1A\u001b[2K\u001b[7A\u001b[2K\u001b[6A\u001b[2Klatest: digest: sha256:ac6579d1f501cc93984edbd8437774061bff0ab36dd757f481e7b37ccfc2cfb0 size: 1787\n" 36 | ] 37 | } 38 | ], 39 | "source": [ 40 | "!docker build -t deploy .\n", 41 | "!docker tag deploy gcr.io/kubeflow-demos/deploy:latest\n", 42 | "!docker push gcr.io/kubeflow-demos/deploy:latest" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [] 51 | } 52 | ], 53 | "metadata": { 54 | "environment": { 55 | "name": "tf2-gpu.2-1.m56", 56 | "type": "gcloud", 57 | "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-1:m56" 58 | }, 59 | "kernelspec": { 60 | "display_name": "Python 3", 61 | "language": "python", 62 | "name": "python3" 63 | }, 64 | "language_info": { 65 | "codemirror_mode": { 66 | "name": "ipython", 67 | "version": 3 68 | }, 69 | "file_extension": ".py", 70 | "mimetype": "text/x-python", 71 | "name": "python", 72 | "nbconvert_exporter": "python", 73 | "pygments_lexer": "ipython3", 74 | "version": "3.7.8" 75 | } 76 | }, 77 | "nbformat": 4, 78 | "nbformat_minor": 4 79 | } 80 | -------------------------------------------------------------------------------- /dockers/train/Dockerize-train.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 8, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "Sending build context to Docker daemon 12.8kB\n", 13 | "Step 1/5 : FROM python:3.7-slim\n", 14 | " ---> 4d4a9832278b\n", 15 | "Step 2/5 : WORKDIR /app\n", 16 | " ---> Using cache\n", 17 | " ---> a44a317c92dc\n", 18 | "Step 3/5 : RUN pip install -U scikit-learn numpy\n", 19 | " ---> Using cache\n", 20 | " ---> b22be2675084\n", 21 | "Step 4/5 : COPY train.py ./train.py\n", 22 | " ---> Using cache\n", 23 | " ---> a9d7367d87a2\n", 24 | "Step 5/5 : ENTRYPOINT [ \"python\", \"train.py\" ]\n", 25 | " ---> Using cache\n", 26 | " ---> 3cd22eec8b01\n", 27 | "Successfully built 3cd22eec8b01\n", 28 | "Successfully tagged train-2:latest\n", 29 | "The push refers to repository [gcr.io/kubeflow-demos/train-2]\n", 30 | "\n", 31 | "\u001b[1Bb1f1085b: Preparing \n", 32 | "\u001b[1B18d350e5: Preparing \n", 33 | "\u001b[1Bab7313b3: Preparing \n", 34 | "\u001b[1Be5632dc8: Preparing \n", 35 | "\u001b[1B857805ec: Preparing \n", 36 | "\u001b[1B87503449: Preparing \n", 37 | "\u001b[1B6688d36c: Preparing \n", 38 | "\u001b[1Bb4339852: Layer already exists \u001b[5A\u001b[2K\u001b[3A\u001b[2Klatest: digest: sha256:bb1af61358564ff2a8d668c6b9da1fb3129d24e877b0ddf6fbfea6cf809bb835 size: 1996\n" 39 | ] 40 | } 41 | ], 42 | "source": [ 43 | "!docker build -t train-2 .\n", 44 | "!docker tag train-2 gcr.io/kubeflow-demos/train-2:latest\n", 45 | "!docker push gcr.io/kubeflow-demos/train-2:latest" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [] 54 | } 55 | ], 56 | "metadata": { 57 | "environment": { 58 | "name": "tf2-gpu.2-1.m56", 59 | "type": "gcloud", 60 | "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-1:m56" 61 | }, 62 | "kernelspec": { 63 | "display_name": "Python 3", 64 | "language": "python", 65 | "name": "python3" 66 | }, 67 | "language_info": { 68 | "codemirror_mode": { 69 | "name": "ipython", 70 | "version": 3 71 | }, 72 | "file_extension": ".py", 73 | "mimetype": "text/x-python", 74 | "name": "python", 75 | "nbconvert_exporter": "python", 76 | "pygments_lexer": "ipython3", 77 | "version": "3.7.8" 78 | } 79 | }, 80 | "nbformat": 4, 81 | "nbformat_minor": 4 82 | } 83 | -------------------------------------------------------------------------------- /dockers/upload/Dockerize.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 8, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "Sending build context to Docker daemon 12.8kB\n", 13 | "Step 1/5 : FROM python:3.7-slim\n", 14 | " ---> 4d4a9832278b\n", 15 | "Step 2/5 : WORKDIR /app\n", 16 | " ---> Using cache\n", 17 | " ---> a44a317c92dc\n", 18 | "Step 3/5 : RUN pip install -U google-cloud-storage\n", 19 | " ---> Using cache\n", 20 | " ---> 7b5cca3e54b1\n", 21 | "Step 4/5 : COPY upload.py ./upload.py\n", 22 | " ---> a2d31ba24a6e\n", 23 | "Step 5/5 : ENTRYPOINT [ \"python\", \"upload.py\" ]\n", 24 | " ---> Running in 281f9f50e979\n", 25 | "Removing intermediate container 281f9f50e979\n", 26 | " ---> 7fee823038e0\n", 27 | "Successfully built 7fee823038e0\n", 28 | "Successfully tagged upload:latest\n", 29 | "The push refers to repository [gcr.io/kubeflow-demos/upload]\n", 30 | "\n", 31 | "\u001b[1Bf76e7bd6: Preparing \n", 32 | "\u001b[1Bf6c678bb: Preparing \n", 33 | "\u001b[1Bab7313b3: Preparing \n", 34 | "\u001b[1Be5632dc8: Preparing \n", 35 | "\u001b[1B857805ec: Preparing \n", 36 | "\u001b[1B87503449: Preparing \n", 37 | "\u001b[1B6688d36c: Preparing \n", 38 | "\u001b[8Bf76e7bd6: Pushed lready exists \u001b[7A\u001b[2K\u001b[3A\u001b[2K\u001b[8A\u001b[2Klatest: digest: sha256:387b3eee8e080cb29427890da196944345f55e4aaf89cc66dfd8d9654ad0e87e size: 1995\n" 39 | ] 40 | } 41 | ], 42 | "source": [ 43 | "!docker build -t upload .\n", 44 | "!docker tag upload gcr.io/kubeflow-demos/upload:latest\n", 45 | "!docker push gcr.io/kubeflow-demos/upload:latest" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [] 54 | } 55 | ], 56 | "metadata": { 57 | "environment": { 58 | "name": "tf2-gpu.2-1.m56", 59 | "type": "gcloud", 60 | "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-1:m56" 61 | }, 62 | "kernelspec": { 63 | "display_name": "Python 3", 64 | "language": "python", 65 | "name": "python3" 66 | }, 67 | "language_info": { 68 | "codemirror_mode": { 69 | "name": "ipython", 70 | "version": 3 71 | }, 72 | "file_extension": ".py", 73 | "mimetype": "text/x-python", 74 | "name": "python", 75 | "nbconvert_exporter": "python", 76 | "pygments_lexer": "ipython3", 77 | "version": "3.7.8" 78 | } 79 | }, 80 | "nbformat": 4, 81 | "nbformat_minor": 4 82 | } 83 | -------------------------------------------------------------------------------- /dockers/preprocess/Dockerize.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "Sending build context to Docker daemon 11.26kB\n", 13 | "Step 1/5 : FROM python:3.7-slim\n", 14 | " ---> 4d4a9832278b\n", 15 | "Step 2/5 : WORKDIR /app\n", 16 | " ---> Using cache\n", 17 | " ---> a44a317c92dc\n", 18 | "Step 3/5 : RUN pip install -U scikit-learn numpy\n", 19 | " ---> Using cache\n", 20 | " ---> b22be2675084\n", 21 | "Step 4/5 : COPY preprocess.py ./preprocess.py\n", 22 | " ---> 5ec6d26e78d9\n", 23 | "Step 5/5 : ENTRYPOINT [ \"python\", \"preprocess.py\" ]\n", 24 | " ---> Running in ff34aea2d190\n", 25 | "Removing intermediate container ff34aea2d190\n", 26 | " ---> 81f21a7a18de\n", 27 | "Successfully built 81f21a7a18de\n", 28 | "Successfully tagged preprocess:latest\n", 29 | "The push refers to repository [gcr.io/kubeflow-demos/preprocess]\n", 30 | "\n", 31 | "\u001b[1Beae4292e: Preparing \n", 32 | "\u001b[1B18d350e5: Preparing \n", 33 | "\u001b[1Bab7313b3: Preparing \n", 34 | "\u001b[1Be5632dc8: Preparing \n", 35 | "\u001b[1B857805ec: Preparing \n", 36 | "\u001b[1B87503449: Preparing \n", 37 | "\u001b[1B6688d36c: Preparing \n", 38 | "\u001b[8Beae4292e: Pushed lready exists \u001b[8A\u001b[2K\u001b[5A\u001b[2K\u001b[2A\u001b[2K\u001b[8A\u001b[2Klatest: digest: sha256:8fca06e45f829003198a359a38b7f8719eb380ebf44608ccd091852a9d936735 size: 1996\n" 39 | ] 40 | } 41 | ], 42 | "source": [ 43 | "!docker build -t preprocess .\n", 44 | "!docker tag preprocess gcr.io/kubeflow-demos/preprocess:latest\n", 45 | "!docker push gcr.io/kubeflow-demos/preprocess:latest" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [] 54 | } 55 | ], 56 | "metadata": { 57 | "environment": { 58 | "name": "tf2-gpu.2-1.m56", 59 | "type": "gcloud", 60 | "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-1:m56" 61 | }, 62 | "kernelspec": { 63 | "display_name": "Python 3", 64 | "language": "python", 65 | "name": "python3" 66 | }, 67 | "language_info": { 68 | "codemirror_mode": { 69 | "name": "ipython", 70 | "version": 3 71 | }, 72 | "file_extension": ".py", 73 | "mimetype": "text/x-python", 74 | "name": "python", 75 | "nbconvert_exporter": "python", 76 | "pygments_lexer": "ipython3", 77 | "version": "3.7.8" 78 | } 79 | }, 80 | "nbformat": 4, 81 | "nbformat_minor": 4 82 | } 83 | -------------------------------------------------------------------------------- /BostonHousing.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 87, 6 | "metadata": { 7 | "jupyter": { 8 | "source_hidden": true 9 | } 10 | }, 11 | "outputs": [ 12 | { 13 | "name": "stdout", 14 | "output_type": "stream", 15 | "text": [ 16 | "Requirement already satisfied: kfp in /opt/conda/lib/python3.7/site-packages (1.0.1)\n", 17 | "Requirement already satisfied: tabulate in /opt/conda/lib/python3.7/site-packages (from kfp) (0.8.7)\n", 18 | "Requirement already satisfied: google-cloud-storage>=1.13.0 in /opt/conda/lib/python3.7/site-packages (from kfp) (1.30.0)\n", 19 | "Requirement already satisfied: PyYAML in /opt/conda/lib/python3.7/site-packages (from kfp) (5.3.1)\n", 20 | "Requirement already satisfied: google-auth>=1.6.1 in /opt/conda/lib/python3.7/site-packages (from kfp) (1.21.2)\n", 21 | "Requirement already satisfied: cloudpickle in /opt/conda/lib/python3.7/site-packages (from kfp) (1.6.0)\n", 22 | "Requirement already satisfied: kfp-server-api<2.0.0,>=0.2.5 in /opt/conda/lib/python3.7/site-packages (from kfp) (1.0.1)\n", 23 | "Requirement already satisfied: jsonschema>=3.0.1 in /opt/conda/lib/python3.7/site-packages (from kfp) (3.2.0)\n", 24 | "Requirement already satisfied: requests-toolbelt>=0.8.0 in /opt/conda/lib/python3.7/site-packages (from kfp) (0.9.1)\n", 25 | "Requirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from kfp) (7.1.2)\n", 26 | "Requirement already satisfied: Deprecated in /opt/conda/lib/python3.7/site-packages (from kfp) (1.2.10)\n", 27 | "Requirement already satisfied: strip-hints in /opt/conda/lib/python3.7/site-packages (from kfp) (0.1.9)\n", 28 | "Requirement already satisfied: kubernetes<12.0.0,>=8.0.0 in /opt/conda/lib/python3.7/site-packages (from kfp) (11.0.0)\n", 29 | "Requirement already satisfied: google-resumable-media<2.0dev,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-storage>=1.13.0->kfp) (1.0.0)\n", 30 | "Requirement already satisfied: google-cloud-core<2.0dev,>=1.2.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-storage>=1.13.0->kfp) (1.3.0)\n", 31 | "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from google-auth>=1.6.1->kfp) (4.1.1)\n", 32 | "Requirement already satisfied: setuptools>=40.3.0 in /opt/conda/lib/python3.7/site-packages (from google-auth>=1.6.1->kfp) (50.3.0)\n", 33 | "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from google-auth>=1.6.1->kfp) (0.2.8)\n", 34 | "Requirement already satisfied: six>=1.9.0 in /opt/conda/lib/python3.7/site-packages (from google-auth>=1.6.1->kfp) (1.15.0)\n", 35 | "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.5\" in /opt/conda/lib/python3.7/site-packages (from google-auth>=1.6.1->kfp) (4.6)\n", 36 | "Requirement already satisfied: certifi in /opt/conda/lib/python3.7/site-packages (from kfp-server-api<2.0.0,>=0.2.5->kfp) (2020.6.20)\n", 37 | "Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.7/site-packages (from kfp-server-api<2.0.0,>=0.2.5->kfp) (2.8.1)\n", 38 | "Requirement already satisfied: urllib3>=1.15 in /opt/conda/lib/python3.7/site-packages (from kfp-server-api<2.0.0,>=0.2.5->kfp) (1.25.10)\n", 39 | "Requirement already satisfied: attrs>=17.4.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema>=3.0.1->kfp) (20.2.0)\n", 40 | "Requirement already satisfied: pyrsistent>=0.14.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema>=3.0.1->kfp) (0.17.3)\n", 41 | "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /opt/conda/lib/python3.7/site-packages (from jsonschema>=3.0.1->kfp) (1.7.0)\n", 42 | "Requirement already satisfied: requests<3.0.0,>=2.0.1 in /opt/conda/lib/python3.7/site-packages (from requests-toolbelt>=0.8.0->kfp) (2.24.0)\n", 43 | "Requirement already satisfied: wrapt<2,>=1.10 in /opt/conda/lib/python3.7/site-packages (from Deprecated->kfp) (1.12.1)\n", 44 | "Requirement already satisfied: wheel in /opt/conda/lib/python3.7/site-packages (from strip-hints->kfp) (0.35.1)\n", 45 | "Requirement already satisfied: requests-oauthlib in /opt/conda/lib/python3.7/site-packages (from kubernetes<12.0.0,>=8.0.0->kfp) (1.3.0)\n", 46 | "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /opt/conda/lib/python3.7/site-packages (from kubernetes<12.0.0,>=8.0.0->kfp) (0.57.0)\n", 47 | "Requirement already satisfied: google-crc32c<2.0dev,>=1.0; python_version >= \"3.5\" in /opt/conda/lib/python3.7/site-packages (from google-resumable-media<2.0dev,>=0.6.0->google-cloud-storage>=1.13.0->kfp) (1.0.0)\n", 48 | "Requirement already satisfied: google-api-core<2.0.0dev,>=1.16.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp) (1.22.2)\n", 49 | "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.6.1->kfp) (0.4.8)\n", 50 | "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata; python_version < \"3.8\"->jsonschema>=3.0.1->kfp) (3.1.0)\n", 51 | "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0,>=2.0.1->requests-toolbelt>=0.8.0->kfp) (3.0.4)\n", 52 | "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0,>=2.0.1->requests-toolbelt>=0.8.0->kfp) (2.10)\n", 53 | "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from requests-oauthlib->kubernetes<12.0.0,>=8.0.0->kfp) (3.1.0)\n", 54 | "Requirement already satisfied: cffi>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from google-crc32c<2.0dev,>=1.0; python_version >= \"3.5\"->google-resumable-media<2.0dev,>=0.6.0->google-cloud-storage>=1.13.0->kfp) (1.14.3)\n", 55 | "Requirement already satisfied: protobuf>=3.12.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp) (3.13.0)\n", 56 | "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp) (1.52.0)\n", 57 | "Requirement already satisfied: pytz in /opt/conda/lib/python3.7/site-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp) (2020.1)\n", 58 | "Requirement already satisfied: pycparser in /opt/conda/lib/python3.7/site-packages (from cffi>=1.0.0->google-crc32c<2.0dev,>=1.0; python_version >= \"3.5\"->google-resumable-media<2.0dev,>=0.6.0->google-cloud-storage>=1.13.0->kfp) (2.20)\n" 59 | ] 60 | } 61 | ], 62 | "source": [ 63 | "!pip3 install kfp" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 199, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "import kfp\n", 73 | "from kfp import dsl\n", 74 | "from kfp import compiler\n", 75 | "import kfp.components as comp\n", 76 | "from kfp import gcp\n", 77 | "\n", 78 | "def preprocess_op():\n", 79 | "\n", 80 | " return dsl.ContainerOp(\n", 81 | " name='Preprocess Data',\n", 82 | " image='gcr.io/kubeflow-demos/preprocess:latest',\n", 83 | " arguments=[],\n", 84 | " file_outputs={\n", 85 | " 'x_train': '/app/x_train.npy',\n", 86 | " 'x_test': '/app/x_test.npy',\n", 87 | " 'y_train': '/app/y_train.npy',\n", 88 | " 'y_test': '/app/y_test.npy',\n", 89 | " }\n", 90 | " )\n", 91 | "\n", 92 | "\n", 93 | "def train_op(x_train, y_train):\n", 94 | "\n", 95 | " return dsl.ContainerOp(\n", 96 | " name='Train Model',\n", 97 | " image='gcr.io/kubeflow-demos/train-2:latest',\n", 98 | " arguments=[\n", 99 | " '--x_train', x_train,\n", 100 | " '--y_train', y_train\n", 101 | " ],\n", 102 | " file_outputs={\n", 103 | " 'model': '/app/model.pkl'\n", 104 | " }\n", 105 | " )\n", 106 | "\n", 107 | "def upload_op(model):\n", 108 | " \n", 109 | " return dsl.ContainerOp(\n", 110 | " name='upload to GCS',\n", 111 | " image='gcr.io/kubeflow-demos/upload:latest',\n", 112 | " arguments=[\n", 113 | " '--model', model\n", 114 | " ]\n", 115 | " )\n", 116 | "\n", 117 | "def test_op(x_test, y_test, model):\n", 118 | "\n", 119 | " return dsl.ContainerOp(\n", 120 | " name='Test Model',\n", 121 | " image='gcr.io/kubeflow-demos/test:latest',\n", 122 | " arguments=[\n", 123 | " '--x_test', x_test,\n", 124 | " '--y_test', y_test,\n", 125 | " '--model', model\n", 126 | " ],\n", 127 | " file_outputs={\n", 128 | " 'mean_squared_error': '/app/output.txt',\n", 129 | " 'mlpipeline-metrics': '/mlpipeline-metrics.json'\n", 130 | " }\n", 131 | " )\n", 132 | "\n", 133 | "def deploy_model_op(model):\n", 134 | "\n", 135 | " return dsl.ContainerOp(\n", 136 | " name='Deploy Model',\n", 137 | " image='gcr.io/kubeflow-demos/deploy:latest',\n", 138 | " arguments=[\n", 139 | " '--model', model\n", 140 | " ]\n", 141 | " )" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": 200, 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "@dsl.pipeline(\n", 151 | " name='Boston Housing Pipeline',\n", 152 | " description='Build Scikit model to predict house prices'\n", 153 | ")\n", 154 | "def boston_pipeline(model_version: str):\n", 155 | " _preprocess_op = preprocess_op()\n", 156 | " _preprocess_op.execution_options.caching_strategy.max_cache_staleness = \"P0D\"\n", 157 | "\n", 158 | " _train_op = train_op(\n", 159 | " dsl.InputArgumentPath(_preprocess_op.outputs['x_train']),\n", 160 | " dsl.InputArgumentPath(_preprocess_op.outputs['y_train'])\n", 161 | " ).after(_preprocess_op)\n", 162 | " _train_op.execution_options.caching_strategy.max_cache_staleness = \"P0D\"\n", 163 | "\n", 164 | " \n", 165 | " _upload_op = upload_op(\n", 166 | " dsl.InputArgumentPath(_train_op.outputs['model'])\n", 167 | " ).after(_train_op)\n", 168 | " _upload_op.execution_options.caching_strategy.max_cache_staleness = \"P0D\"\n", 169 | "\n", 170 | " _test_op = test_op(\n", 171 | " dsl.InputArgumentPath(_preprocess_op.outputs['x_test']),\n", 172 | " dsl.InputArgumentPath(_preprocess_op.outputs['y_test']),\n", 173 | " dsl.InputArgumentPath(_train_op.outputs['model'])\n", 174 | " ).after(_train_op)\n", 175 | " _test_op.execution_options.caching_strategy.max_cache_staleness = \"P0D\"\n", 176 | "\n", 177 | " deploy_op =deploy_model_op(\n", 178 | " dsl.InputArgumentPath(_train_op.outputs['model'])\n", 179 | " ).after(_test_op)\n", 180 | " deploy_op.execution_options.caching_strategy.max_cache_staleness = \"P0D\"" 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": 204, 186 | "metadata": {}, 187 | "outputs": [ 188 | { 189 | "data": { 190 | "text/html": [ 191 | "Experiment link here" 192 | ], 193 | "text/plain": [ 194 | "" 195 | ] 196 | }, 197 | "metadata": {}, 198 | "output_type": "display_data" 199 | }, 200 | { 201 | "data": { 202 | "text/html": [ 203 | "Run link here" 204 | ], 205 | "text/plain": [ 206 | "" 207 | ] 208 | }, 209 | "metadata": {}, 210 | "output_type": "display_data" 211 | }, 212 | { 213 | "data": { 214 | "text/plain": [ 215 | "RunPipelineResult(run_id=c191cc67-e682-48f1-9b83-d3dd08ec3b47)" 216 | ] 217 | }, 218 | "execution_count": 204, 219 | "metadata": {}, 220 | "output_type": "execute_result" 221 | } 222 | ], 223 | "source": [ 224 | "args = {\n", 225 | " \"model_version\": \"11\"\n", 226 | "}\n", 227 | "\n", 228 | "client = kfp.Client(host='c4e37c713669144-dot-us-central2.pipelines.googleusercontent.com')\n", 229 | "client.create_run_from_pipeline_func(boston_pipeline, experiment_name='user-group-demo-experiment', arguments=args)" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": null, 235 | "metadata": {}, 236 | "outputs": [], 237 | "source": [] 238 | } 239 | ], 240 | "metadata": { 241 | "environment": { 242 | "name": "tf2-gpu.2-1.m56", 243 | "type": "gcloud", 244 | "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-1:m56" 245 | }, 246 | "kernelspec": { 247 | "display_name": "Python 3", 248 | "language": "python", 249 | "name": "python3" 250 | }, 251 | "language_info": { 252 | "codemirror_mode": { 253 | "name": "ipython", 254 | "version": 3 255 | }, 256 | "file_extension": ".py", 257 | "mimetype": "text/x-python", 258 | "name": "python", 259 | "nbconvert_exporter": "python", 260 | "pygments_lexer": "ipython3", 261 | "version": "3.7.8" 262 | } 263 | }, 264 | "nbformat": 4, 265 | "nbformat_minor": 4 266 | } 267 | --------------------------------------------------------------------------------