├── .gitignore ├── 1.intro ├── chapter1 │ ├── intro-demo │ │ ├── notebook.ipynb │ │ └── xor.csv │ └── intro-text-generation │ │ ├── my_model │ │ ├── config.json │ │ ├── generation_config.json │ │ └── training_args.bin │ │ ├── notebook.ipynb │ │ └── odh-merged-docs.adoc ├── chapter2 │ └── lab │ │ ├── group.yaml │ │ ├── test_connection.ipynb │ │ └── test_env_vars.ipynb └── chapter3 │ ├── customize │ ├── notebooks-customize.ipynb │ └── requirements.txt │ ├── intro │ ├── Bongo_sound.wav │ ├── notebooks-intro.ipynb │ ├── redhat.png │ └── test.mp4 │ └── lab │ ├── .gitignore │ ├── db.json │ ├── notebooks-lab.ipynb │ └── solution │ ├── db.json │ └── notebooks-lab-solution.ipynb ├── 3.create-model └── pytorch-lab │ ├── data │ └── diabetes.csv │ ├── model │ └── README.md │ └── notebooks │ ├── 01-data-exploration.ipynb │ └── 02-model-development.ipynb ├── 4.rhods-deploy ├── chapter1 │ ├── purchase-amount.ipynb │ ├── requirements.txt │ └── use-purchase-amount.ipynb ├── chapter2 │ ├── iris_to_onnx.ipynb │ ├── minio.yml │ └── requirements.txt └── custom-runtime-example │ ├── README.adoc │ ├── example.ipynb │ └── ml-server-runtime.yaml ├── 5.pipelines └── elyra │ ├── data │ └── .gitkeep │ ├── data_ingestion.py │ ├── model_loading.py │ ├── offline-scoring.ipynb │ ├── preprocessing.py │ ├── results_upload.py │ └── scoring.py ├── 6.rhod-admin └── custom-images │ ├── Containerfile │ └── requirements.txt ├── 7.hands-on-lab ├── .gitignore ├── 1.collect.ipynb ├── 2.exploration.ipynb ├── 3.preprocessing.ipynb ├── 4.model_training.ipynb ├── 5.model_upload.ipynb ├── 6.test.ipynb ├── README.md ├── data │ └── .gitignore └── solutions │ ├── 1.collect.ipynb │ ├── 2.exploration.ipynb │ ├── 3.preprocessing.ipynb │ ├── 4.model_training.ipynb │ ├── 5.model_upload.ipynb │ ├── 6.test.ipynb │ └── mypipeline.pipeline └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | .DS_Store 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | .pybuilder/ 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | # For a library or package, you might want to ignore these files since the code is 88 | # intended to run in multiple environments; otherwise, check them in: 89 | # .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # poetry 99 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 100 | # This is especially recommended for binary packages to ensure reproducibility, and is more 101 | # commonly ignored for libraries. 102 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 103 | #poetry.lock 104 | 105 | # pdm 106 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 107 | #pdm.lock 108 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 109 | # in version control. 110 | # https://pdm.fming.dev/#use-with-ide 111 | .pdm.toml 112 | 113 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 114 | __pypackages__/ 115 | 116 | # Celery stuff 117 | celerybeat-schedule 118 | celerybeat.pid 119 | 120 | # SageMath parsed files 121 | *.sage.py 122 | 123 | # Environments 124 | .env 125 | .venv 126 | env/ 127 | venv/ 128 | ENV/ 129 | env.bak/ 130 | venv.bak/ 131 | 132 | # Spyder project settings 133 | .spyderproject 134 | .spyproject 135 | 136 | # Rope project settings 137 | .ropeproject 138 | 139 | # mkdocs documentation 140 | /site 141 | 142 | # mypy 143 | .mypy_cache/ 144 | .dmypy.json 145 | dmypy.json 146 | 147 | # Pyre type checker 148 | .pyre/ 149 | 150 | # pytype static type analyzer 151 | .pytype/ 152 | 153 | # Cython debug symbols 154 | cython_debug/ 155 | 156 | # PyCharm 157 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 158 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 159 | # and can be added to the global gitignore or merged into this file. For a more nuclear 160 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 161 | #.idea/ 162 | .vscode 163 | -------------------------------------------------------------------------------- /1.intro/chapter1/intro-demo/notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Getting Started: Training a Neural Network to Act as an XOR gate\n", 8 | "\n", 9 | "This is a minimal example about how you can use common data science libraries in RHOAI to train an AI model." 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "## Data Loading\n", 17 | "\n", 18 | "Read the data from a CSV file by using Pandas.\n", 19 | "Pandas loads the data into a `DataFrame` object:" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "metadata": { 26 | "tags": [] 27 | }, 28 | "outputs": [], 29 | "source": [ 30 | "import pandas as pd\n", 31 | "\n", 32 | "data = pd.read_csv(\"xor.csv\")" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "Inspect the data.\n", 40 | "\n", 41 | "The XOR gate receives 2 input values and returns one output.\n", 42 | "The gate returns `1` (or `true`) when the 2 input values are different.\n", 43 | "Othewise, the gate returns `0` (or `false`).\n", 44 | "\n", 45 | "The first unnamed column is the dataframe index. You can ignore the index in this example." 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": { 52 | "tags": [] 53 | }, 54 | "outputs": [], 55 | "source": [ 56 | "data" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": {}, 62 | "source": [ 63 | "## Data Inspection\n", 64 | "\n", 65 | "After loading the data, you can explore the cases, for example, by gathering basic metrics:" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": null, 71 | "metadata": { 72 | "tags": [] 73 | }, 74 | "outputs": [], 75 | "source": [ 76 | "data.describe()" 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "You can also create data visualizations with libraries such as `matplotlib` to discover patterns and correlations:" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "metadata": { 90 | "tags": [] 91 | }, 92 | "outputs": [], 93 | "source": [ 94 | "%pip install seaborn==0.13.0\n", 95 | "\n", 96 | "import seaborn as sns\n", 97 | "import matplotlib.pyplot as plt\n", 98 | "\n", 99 | "# Set graph style\n", 100 | "sns.set(style=\"whitegrid\")\n", 101 | "\n", 102 | "# Create the plot\n", 103 | "plt.figure(figsize=(4, 4))\n", 104 | "plot = sns.scatterplot(\n", 105 | " x='Input 1',\n", 106 | " y='Input 2',\n", 107 | " hue='Output',\n", 108 | " data=data,\n", 109 | " palette={0: \"red\", 1: \"green\"},\n", 110 | " s=200\n", 111 | ")\n", 112 | "plt.xticks([0, 1])\n", 113 | "plt.yticks([0, 1])\n", 114 | "plt.title('XOR Gate Cases')\n", 115 | "plt.show()" 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": {}, 121 | "source": [ 122 | "## Data Preparation\n", 123 | "\n", 124 | "Typically, you must prepocess, clean, normalize, and prepare the data in a format that is suitable for the library and the model that you are training.\n", 125 | "You must also split the data into input and output data.\n", 126 | "\n", 127 | "In most cases, you would also split the data into training, test, validation subsets, so that you can evaluate the performance of your model after training.\n", 128 | "\n", 129 | "In this case, for the sake of simplicity, just split the data into inputs and output.\n", 130 | "\n", 131 | "To select the inputs, select all rows (`:`) and the first two columns (`:2`):" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "metadata": { 138 | "tags": [] 139 | }, 140 | "outputs": [], 141 | "source": [ 142 | "inputs = data.iloc[:, :2]\n", 143 | "inputs" 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": {}, 149 | "source": [ 150 | "To select the output, pick the last column:" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": { 157 | "tags": [] 158 | }, 159 | "outputs": [], 160 | "source": [ 161 | "output = data.iloc[:, -1:]\n", 162 | "output" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "metadata": {}, 168 | "source": [ 169 | "## Training\n", 170 | "\n", 171 | "After your data is clean and ready, you can create and train your model.\n", 172 | "\n", 173 | "In this case, the exercise uses a simple neural network by using the `tensorflow` and `keras` libraries." 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": null, 179 | "metadata": { 180 | "tags": [] 181 | }, 182 | "outputs": [], 183 | "source": [ 184 | "from keras.layers import Dense\n", 185 | "from keras.models import Sequential\n", 186 | "\n", 187 | "# Define a sequential neural network\n", 188 | "model = Sequential([\n", 189 | " Dense(32, input_dim=2, activation=\"relu\"),\n", 190 | " Dense(1, activation=\"sigmoid\")\n", 191 | "])\n", 192 | "\n", 193 | "# Compile the model\n", 194 | "model.compile(loss=\"mean_squared_error\", optimizer='rmsprop', metrics=['accuracy'])\n", 195 | "\n", 196 | "# Train the model until it reaches 100% accuracy. Given the simple XOR use case and that our training\n", 197 | "# data encompasses the entire problem space of possible inputs and outputs, we know we can get it to 100%.\n", 198 | "epochs = 0\n", 199 | "while model.fit(inputs, output, epochs=epochs+10, initial_epoch=epochs).history['accuracy'][-1] != 1.0:\n", 200 | " epochs += 10" 201 | ] 202 | }, 203 | { 204 | "cell_type": "markdown", 205 | "metadata": {}, 206 | "source": [ 207 | "The model is now trained." 208 | ] 209 | }, 210 | { 211 | "cell_type": "markdown", 212 | "metadata": {}, 213 | "source": [ 214 | "## Evaluation\n", 215 | "\n", 216 | "After training, evaluate your model.\n", 217 | "Typically you should use a dedicated test subset to evaluate the model, but this simple case uses the same training data for testing." 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": null, 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [ 226 | "loss, accuracy = model.evaluate(inputs, output)\n", 227 | "print(\"Model accuracy:\", accuracy)" 228 | ] 229 | }, 230 | { 231 | "cell_type": "markdown", 232 | "metadata": {}, 233 | "source": [ 234 | "Finally, you can also feed inputs into the trained model and verify the result." 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "execution_count": null, 240 | "metadata": { 241 | "tags": [] 242 | }, 243 | "outputs": [], 244 | "source": [ 245 | "predictions = model.predict(inputs)\n", 246 | "expected_predictions = output.to_numpy()\n", 247 | "rounded_predictions = [int(round(p[0])) for p in predictions]\n", 248 | "\n", 249 | "# Print the predicted values\n", 250 | "print(\"Predictions:\")\n", 251 | "print(\"Expected\\tPredicted\")\n", 252 | "print(\"-------------------------\")\n", 253 | "for input_val, output_val in zip(expected_predictions, rounded_predictions):\n", 254 | " print(f\"{input_val[0]}\\t\\t{output_val}\")" 255 | ] 256 | } 257 | ], 258 | "metadata": { 259 | "kernelspec": { 260 | "display_name": "Python 3.9", 261 | "language": "python", 262 | "name": "python3" 263 | }, 264 | "language_info": { 265 | "codemirror_mode": { 266 | "name": "ipython", 267 | "version": 3 268 | }, 269 | "file_extension": ".py", 270 | "mimetype": "text/x-python", 271 | "name": "python", 272 | "nbconvert_exporter": "python", 273 | "pygments_lexer": "ipython3", 274 | "version": "3.9.18" 275 | } 276 | }, 277 | "nbformat": 4, 278 | "nbformat_minor": 4 279 | } 280 | -------------------------------------------------------------------------------- /1.intro/chapter1/intro-demo/xor.csv: -------------------------------------------------------------------------------- 1 | Input 1,Input 2,Output 2 | 0,0,0 3 | 0,1,1 4 | 1,0,1 5 | 1,1,0 6 | -------------------------------------------------------------------------------- /1.intro/chapter1/intro-text-generation/my_model/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "distilgpt2", 3 | "_num_labels": 1, 4 | "activation_function": "gelu_new", 5 | "architectures": [ 6 | "GPT2LMHeadModel" 7 | ], 8 | "attn_pdrop": 0.1, 9 | "bos_token_id": 50256, 10 | "embd_pdrop": 0.1, 11 | "eos_token_id": 50256, 12 | "id2label": { 13 | "0": "LABEL_0" 14 | }, 15 | "initializer_range": 0.02, 16 | "label2id": { 17 | "LABEL_0": 0 18 | }, 19 | "layer_norm_epsilon": 1e-05, 20 | "model_type": "gpt2", 21 | "n_ctx": 1024, 22 | "n_embd": 768, 23 | "n_head": 12, 24 | "n_inner": null, 25 | "n_layer": 6, 26 | "n_positions": 1024, 27 | "reorder_and_upcast_attn": false, 28 | "resid_pdrop": 0.1, 29 | "scale_attn_by_inverse_layer_idx": false, 30 | "scale_attn_weights": true, 31 | "summary_activation": null, 32 | "summary_first_dropout": 0.1, 33 | "summary_proj_to_labels": true, 34 | "summary_type": "cls_index", 35 | "summary_use_proj": true, 36 | "task_specific_params": { 37 | "text-generation": { 38 | "do_sample": true, 39 | "max_length": 50 40 | } 41 | }, 42 | "torch_dtype": "float32", 43 | "transformers_version": "4.34.0", 44 | "use_cache": true, 45 | "vocab_size": 50257 46 | } 47 | -------------------------------------------------------------------------------- /1.intro/chapter1/intro-text-generation/my_model/generation_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_from_model_config": true, 3 | "bos_token_id": 50256, 4 | "eos_token_id": 50256, 5 | "transformers_version": "4.34.0" 6 | } 7 | -------------------------------------------------------------------------------- /1.intro/chapter1/intro-text-generation/my_model/training_args.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RedHatQuickCourses/rhods-qc-apps/60776f6fadc901c0ede9c0eab9a7d6c7198f4ba1/1.intro/chapter1/intro-text-generation/my_model/training_args.bin -------------------------------------------------------------------------------- /1.intro/chapter2/lab/group.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: user.openshift.io/v1 2 | kind: Group 3 | metadata: 4 | name: data-scientists 5 | users: 6 | - user4 7 | - user5 -------------------------------------------------------------------------------- /1.intro/chapter2/lab/test_connection.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "624661a1-b0cb-43f7-b94f-cae2ec6e07c6", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import os\n", 11 | "import boto3" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "id": "0a21fa71-11eb-4d10-82aa-95020c2f2e4a", 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "key_id = os.getenv(\"AWS_ACCESS_KEY_ID\")\n", 22 | "secret_key = os.getenv(\"AWS_SECRET_ACCESS_KEY\")\n", 23 | "region = os.getenv(\"AWS_DEFAULT_REGION\")\n", 24 | "endpoint = os.getenv(\"AWS_S3_ENDPOINT\")\n", 25 | "bucket_name = os.getenv(\"AWS_S3_BUCKET\")" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": null, 31 | "id": "5ad4179f-cbd4-4c52-8403-0556dd3bf255", 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [ 35 | "s3 = boto3.client(\n", 36 | " \"s3\",\n", 37 | " region,\n", 38 | " aws_access_key_id=key_id,\n", 39 | " aws_secret_access_key=secret_key,\n", 40 | " endpoint_url=endpoint,\n", 41 | " use_ssl=True\n", 42 | ")" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "id": "b647d612-3665-4de2-a948-35b3aa553c78", 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [ 52 | "response = s3.list_buckets()\n", 53 | "response[\"Buckets\"]" 54 | ] 55 | } 56 | ], 57 | "metadata": { 58 | "kernelspec": { 59 | "display_name": "Python 3.9", 60 | "language": "python", 61 | "name": "python3" 62 | }, 63 | "language_info": { 64 | "codemirror_mode": { 65 | "name": "ipython", 66 | "version": 3 67 | }, 68 | "file_extension": ".py", 69 | "mimetype": "text/x-python", 70 | "name": "python", 71 | "nbconvert_exporter": "python", 72 | "pygments_lexer": "ipython3", 73 | "version": "3.9.16" 74 | } 75 | }, 76 | "nbformat": 4, 77 | "nbformat_minor": 5 78 | } 79 | -------------------------------------------------------------------------------- /1.intro/chapter2/lab/test_env_vars.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "4f59bca9-fc56-4f0f-84d1-d064d30d87ec", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import os" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 2, 16 | "id": "fa5456fa-d261-4b15-9f9d-492cb88fd4f3", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "name = os.getenv(\"STUDENT_NAME\")" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "id": "c58a03d5-14f1-4e0f-b5ec-095a34032ea3", 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "print(f\"Hello {name}\")" 31 | ] 32 | } 33 | ], 34 | "metadata": { 35 | "kernelspec": { 36 | "display_name": "Python 3.9", 37 | "language": "python", 38 | "name": "python3" 39 | }, 40 | "language_info": { 41 | "codemirror_mode": { 42 | "name": "ipython", 43 | "version": 3 44 | }, 45 | "file_extension": ".py", 46 | "mimetype": "text/x-python", 47 | "name": "python", 48 | "nbconvert_exporter": "python", 49 | "pygments_lexer": "ipython3", 50 | "version": "3.9.16" 51 | } 52 | }, 53 | "nbformat": 4, 54 | "nbformat_minor": 5 55 | } 56 | -------------------------------------------------------------------------------- /1.intro/chapter3/customize/requirements.txt: -------------------------------------------------------------------------------- 1 | pytest==7.4.3 2 | tomli==2.0.1 -------------------------------------------------------------------------------- /1.intro/chapter3/intro/Bongo_sound.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RedHatQuickCourses/rhods-qc-apps/60776f6fadc901c0ede9c0eab9a7d6c7198f4ba1/1.intro/chapter3/intro/Bongo_sound.wav -------------------------------------------------------------------------------- /1.intro/chapter3/intro/redhat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RedHatQuickCourses/rhods-qc-apps/60776f6fadc901c0ede9c0eab9a7d6c7198f4ba1/1.intro/chapter3/intro/redhat.png -------------------------------------------------------------------------------- /1.intro/chapter3/intro/test.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RedHatQuickCourses/rhods-qc-apps/60776f6fadc901c0ede9c0eab9a7d6c7198f4ba1/1.intro/chapter3/intro/test.mp4 -------------------------------------------------------------------------------- /1.intro/chapter3/lab/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RedHatQuickCourses/rhods-qc-apps/60776f6fadc901c0ede9c0eab9a7d6c7198f4ba1/1.intro/chapter3/lab/.gitignore -------------------------------------------------------------------------------- /1.intro/chapter3/lab/db.json: -------------------------------------------------------------------------------- 1 | {"_default": {"1": {"name": "Ravi"}, "2": {"name": "Alex"}}} -------------------------------------------------------------------------------- /1.intro/chapter3/lab/notebooks-lab.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "01dc34ad-610f-4706-8ad7-4dc598538ba4", 6 | "metadata": {}, 7 | "source": [ 8 | "# Lab: Jupyter Notebooks\n", 9 | "\n", 10 | "This notebook uses the TinyDB python package. TinyDB is a lightweight database that supports JSON and in-memory storage.\n", 11 | "\n", 12 | "Follow the instructions that this notebook provides.\n", 13 | "Add, edit, and complete cells as necessary to complete the exercise.\n", 14 | "\n", 15 | "1. Verify that the list of installed packages does not include `tinydb`" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "id": "c6de9d25-e85c-44ab-a0dc-358aa6a7255a", 21 | "metadata": {}, 22 | "source": [ 23 | "2. Install the `4.8.0` version of the `tinydb` package." 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "id": "074cdebc-e143-4e3a-adea-7431483bc6cf", 29 | "metadata": {}, 30 | "source": [ 31 | "3. Import the `tinydb` package into the notebook." 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "id": "9c67d3ab-2872-49b0-85d1-88cb07e83e04", 37 | "metadata": {}, 38 | "source": [ 39 | "4. Verify that the following code works.\n", 40 | "The cell should display the two users contained in the TinyDB database." 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "id": "be12b74a-a22a-4803-a346-3a29adb3b21d", 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "users = tinydb.TinyDB(\"db.json\")\n", 51 | "\n", 52 | "users.all()" 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "id": "639bee01-d775-4ca3-a7bf-6b255a9ea81a", 58 | "metadata": {}, 59 | "source": [ 60 | "5. Fix the following cell." 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "id": "f87a1f5d-4190-44f2-98da-ae5450937b7f", 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "This cell contains text but the notebook is trying to execute the cell in a kernel." 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "id": "899456fa-050c-4eab-afd6-75d5a170a78d", 76 | "metadata": {}, 77 | "source": [ 78 | "6. Debug the following cell.\n", 79 | "This cell should return the `Alex` user, but it is returning a different user.\n", 80 | "Enable the debuger, find the problem, and fix the code." 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "id": "0d30f1e8-f633-4451-8155-71626bf60a95", 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "def find_user_by_name(name):\n", 91 | " User = tinydb.Query()\n", 92 | " return users.search(User.name == name)\n", 93 | "\n", 94 | "\n", 95 | "# Find Alex\n", 96 | "usernames = [\"Ravi\", \"Alex\", \"John\"]\n", 97 | "username = usernames[0]\n", 98 | "find_user_by_name(username)" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "id": "f2e67868-6c90-4596-b86c-b3009ca1b0f2", 104 | "metadata": {}, 105 | "source": [ 106 | "7. Remove the following cell" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "id": "e7bbea40-da71-42e8-8dea-9911479582d1", 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "# This cell contains deprecated comments and can be removed" 117 | ] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "id": "492e512b-8abd-468b-a811-c4ef67f5fe6b", 122 | "metadata": {}, 123 | "source": [ 124 | "## Finish\n", 125 | "\n", 126 | "You have completed the exercise.\n", 127 | "You can clean up this environment by removing the workbench." 128 | ] 129 | } 130 | ], 131 | "metadata": { 132 | "kernelspec": { 133 | "display_name": "Python 3.9", 134 | "language": "python", 135 | "name": "python3" 136 | }, 137 | "language_info": { 138 | "codemirror_mode": { 139 | "name": "ipython", 140 | "version": 3 141 | }, 142 | "file_extension": ".py", 143 | "mimetype": "text/x-python", 144 | "name": "python", 145 | "nbconvert_exporter": "python", 146 | "pygments_lexer": "ipython3", 147 | "version": "3.9.16" 148 | } 149 | }, 150 | "nbformat": 4, 151 | "nbformat_minor": 5 152 | } 153 | -------------------------------------------------------------------------------- /1.intro/chapter3/lab/solution/db.json: -------------------------------------------------------------------------------- 1 | {"_default": {"1": {"name": "Ravi"}, "2": {"name": "Alex"}}} -------------------------------------------------------------------------------- /1.intro/chapter3/lab/solution/notebooks-lab-solution.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "01dc34ad-610f-4706-8ad7-4dc598538ba4", 6 | "metadata": {}, 7 | "source": [ 8 | "# Lab: Jupyter Notebooks\n", 9 | "\n", 10 | "This notebook uses the TinyDB python package. TinyDB is a lightweight database that supports JSON and in-memory storage.\n", 11 | "\n", 12 | "Follow the instructions that this notebook provides.\n", 13 | "Add, edit, and complete cells as necessary to complete the exercise.\n", 14 | "\n", 15 | "1. Verify that the list of installed packages does not include `tinydb`" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 1, 21 | "id": "b69f5d8a-61f5-4335-8b65-1ec1cfd83d71", 22 | "metadata": {}, 23 | "outputs": [ 24 | { 25 | "name": "stdout", 26 | "output_type": "stream", 27 | "text": [ 28 | "\n", 29 | "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.2.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n", 30 | "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", 31 | "Note: you may need to restart the kernel to use updated packages.\n" 32 | ] 33 | } 34 | ], 35 | "source": [ 36 | "%pip list | grep tinydb" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "id": "c6de9d25-e85c-44ab-a0dc-358aa6a7255a", 42 | "metadata": {}, 43 | "source": [ 44 | "2. Install the `4.8.0` version of the `tinydb` package." 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 2, 50 | "id": "173bc071-be79-497e-b9e9-b9c850bf2690", 51 | "metadata": {}, 52 | "outputs": [ 53 | { 54 | "name": "stdout", 55 | "output_type": "stream", 56 | "text": [ 57 | "Collecting tinydb==4.8.0\n", 58 | " Downloading tinydb-4.8.0-py3-none-any.whl (24 kB)\n", 59 | "Installing collected packages: tinydb\n", 60 | "Successfully installed tinydb-4.8.0\n", 61 | "\n", 62 | "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.2.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n", 63 | "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", 64 | "Note: you may need to restart the kernel to use updated packages.\n" 65 | ] 66 | } 67 | ], 68 | "source": [ 69 | "%pip install tinydb==4.8.0" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "id": "074cdebc-e143-4e3a-adea-7431483bc6cf", 75 | "metadata": {}, 76 | "source": [ 77 | "3. Import the `tinydb` package into the notebook." 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 3, 83 | "id": "b81643d9-8eca-4f36-9c6a-b569d35a53f6", 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "import tinydb" 88 | ] 89 | }, 90 | { 91 | "cell_type": "markdown", 92 | "id": "9c67d3ab-2872-49b0-85d1-88cb07e83e04", 93 | "metadata": {}, 94 | "source": [ 95 | "4. Verify that the following code works.\n", 96 | "The cell should display the two users contained in the TinyDB database." 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 4, 102 | "id": "be12b74a-a22a-4803-a346-3a29adb3b21d", 103 | "metadata": {}, 104 | "outputs": [ 105 | { 106 | "data": { 107 | "text/plain": [ 108 | "[{'name': 'Ravi'}, {'name': 'Alex'}]" 109 | ] 110 | }, 111 | "execution_count": 4, 112 | "metadata": {}, 113 | "output_type": "execute_result" 114 | } 115 | ], 116 | "source": [ 117 | "users = tinydb.TinyDB(\"db.json\")\n", 118 | "\n", 119 | "users.all()" 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "id": "639bee01-d775-4ca3-a7bf-6b255a9ea81a", 125 | "metadata": {}, 126 | "source": [ 127 | "5. Fix the following cell." 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "id": "08ab6171-b8b6-4f17-882d-a54d801eeecc", 133 | "metadata": {}, 134 | "source": [ 135 | "This cell contains text but the notebook is trying to execute the cell in a kernel." 136 | ] 137 | }, 138 | { 139 | "cell_type": "markdown", 140 | "id": "899456fa-050c-4eab-afd6-75d5a170a78d", 141 | "metadata": {}, 142 | "source": [ 143 | "6. Debug the following cell.\n", 144 | "This cell should return the `Alex` user, but it is returning a different user.\n", 145 | "Enable the debuger, find the problem, and fix the code." 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 7, 151 | "id": "0d30f1e8-f633-4451-8155-71626bf60a95", 152 | "metadata": {}, 153 | "outputs": [ 154 | { 155 | "data": { 156 | "text/plain": [ 157 | "[{'name': 'Alex'}]" 158 | ] 159 | }, 160 | "execution_count": 7, 161 | "metadata": {}, 162 | "output_type": "execute_result" 163 | } 164 | ], 165 | "source": [ 166 | "def find_user_by_name(name):\n", 167 | " User = tinydb.Query()\n", 168 | " return users.search(User.name == name)\n", 169 | "\n", 170 | "\n", 171 | "# Find Alex\n", 172 | "usernames = [\"Ravi\", \"Alex\", \"John\"]\n", 173 | "username = usernames[1]\n", 174 | "find_user_by_name(username)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "id": "f2e67868-6c90-4596-b86c-b3009ca1b0f2", 180 | "metadata": {}, 181 | "source": [ 182 | "7. Remove the following cell" 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "id": "8343c930-515f-430c-9b5a-3e7f392d4fef", 188 | "metadata": {}, 189 | "source": [ 190 | "## Finish\n", 191 | "\n", 192 | "You have completed the exercise.\n", 193 | "You can clean up this environment by removing the workbench." 194 | ] 195 | } 196 | ], 197 | "metadata": { 198 | "kernelspec": { 199 | "display_name": "Python 3.9", 200 | "language": "python", 201 | "name": "python3" 202 | }, 203 | "language_info": { 204 | "codemirror_mode": { 205 | "name": "ipython", 206 | "version": 3 207 | }, 208 | "file_extension": ".py", 209 | "mimetype": "text/x-python", 210 | "name": "python", 211 | "nbconvert_exporter": "python", 212 | "pygments_lexer": "ipython3", 213 | "version": "3.9.16" 214 | } 215 | }, 216 | "nbformat": 4, 217 | "nbformat_minor": 5 218 | } 219 | -------------------------------------------------------------------------------- /3.create-model/pytorch-lab/data/diabetes.csv: -------------------------------------------------------------------------------- 1 | Pregnancies,Glucose,BloodPressure,SkinThickness,Insulin,BMI,DiabetesPedigreeFunction,Age,Outcome 2 | 6,148,72,35,0,33.6,0.627,50,1 3 | 1,85,66,29,0,26.6,0.351,31,0 4 | 8,183,64,0,0,23.3,0.672,32,1 5 | 1,89,66,23,94,28.1,0.167,21,0 6 | 0,137,40,35,168,43.1,2.288,33,1 7 | 5,116,74,0,0,25.6,0.201,30,0 8 | 3,78,50,32,88,31,0.248,26,1 9 | 10,115,0,0,0,35.3,0.134,29,0 10 | 2,197,70,45,543,30.5,0.158,53,1 11 | 8,125,96,0,0,0,0.232,54,1 12 | 4,110,92,0,0,37.6,0.191,30,0 13 | 10,168,74,0,0,38,0.537,34,1 14 | 10,139,80,0,0,27.1,1.441,57,0 15 | 1,189,60,23,846,30.1,0.398,59,1 16 | 5,166,72,19,175,25.8,0.587,51,1 17 | 7,100,0,0,0,30,0.484,32,1 18 | 0,118,84,47,230,45.8,0.551,31,1 19 | 7,107,74,0,0,29.6,0.254,31,1 20 | 1,103,30,38,83,43.3,0.183,33,0 21 | 1,115,70,30,96,34.6,0.529,32,1 22 | 3,126,88,41,235,39.3,0.704,27,0 23 | 8,99,84,0,0,35.4,0.388,50,0 24 | 7,196,90,0,0,39.8,0.451,41,1 25 | 9,119,80,35,0,29,0.263,29,1 26 | 11,143,94,33,146,36.6,0.254,51,1 27 | 10,125,70,26,115,31.1,0.205,41,1 28 | 7,147,76,0,0,39.4,0.257,43,1 29 | 1,97,66,15,140,23.2,0.487,22,0 30 | 13,145,82,19,110,22.2,0.245,57,0 31 | 5,117,92,0,0,34.1,0.337,38,0 32 | 5,109,75,26,0,36,0.546,60,0 33 | 3,158,76,36,245,31.6,0.851,28,1 34 | 3,88,58,11,54,24.8,0.267,22,0 35 | 6,92,92,0,0,19.9,0.188,28,0 36 | 10,122,78,31,0,27.6,0.512,45,0 37 | 4,103,60,33,192,24,0.966,33,0 38 | 11,138,76,0,0,33.2,0.42,35,0 39 | 9,102,76,37,0,32.9,0.665,46,1 40 | 2,90,68,42,0,38.2,0.503,27,1 41 | 4,111,72,47,207,37.1,1.39,56,1 42 | 3,180,64,25,70,34,0.271,26,0 43 | 7,133,84,0,0,40.2,0.696,37,0 44 | 7,106,92,18,0,22.7,0.235,48,0 45 | 9,171,110,24,240,45.4,0.721,54,1 46 | 7,159,64,0,0,27.4,0.294,40,0 47 | 0,180,66,39,0,42,1.893,25,1 48 | 1,146,56,0,0,29.7,0.564,29,0 49 | 2,71,70,27,0,28,0.586,22,0 50 | 7,103,66,32,0,39.1,0.344,31,1 51 | 7,105,0,0,0,0,0.305,24,0 52 | 1,103,80,11,82,19.4,0.491,22,0 53 | 1,101,50,15,36,24.2,0.526,26,0 54 | 5,88,66,21,23,24.4,0.342,30,0 55 | 8,176,90,34,300,33.7,0.467,58,1 56 | 7,150,66,42,342,34.7,0.718,42,0 57 | 1,73,50,10,0,23,0.248,21,0 58 | 7,187,68,39,304,37.7,0.254,41,1 59 | 0,100,88,60,110,46.8,0.962,31,0 60 | 0,146,82,0,0,40.5,1.781,44,0 61 | 0,105,64,41,142,41.5,0.173,22,0 62 | 2,84,0,0,0,0,0.304,21,0 63 | 8,133,72,0,0,32.9,0.27,39,1 64 | 5,44,62,0,0,25,0.587,36,0 65 | 2,141,58,34,128,25.4,0.699,24,0 66 | 7,114,66,0,0,32.8,0.258,42,1 67 | 5,99,74,27,0,29,0.203,32,0 68 | 0,109,88,30,0,32.5,0.855,38,1 69 | 2,109,92,0,0,42.7,0.845,54,0 70 | 1,95,66,13,38,19.6,0.334,25,0 71 | 4,146,85,27,100,28.9,0.189,27,0 72 | 2,100,66,20,90,32.9,0.867,28,1 73 | 5,139,64,35,140,28.6,0.411,26,0 74 | 13,126,90,0,0,43.4,0.583,42,1 75 | 4,129,86,20,270,35.1,0.231,23,0 76 | 1,79,75,30,0,32,0.396,22,0 77 | 1,0,48,20,0,24.7,0.14,22,0 78 | 7,62,78,0,0,32.6,0.391,41,0 79 | 5,95,72,33,0,37.7,0.37,27,0 80 | 0,131,0,0,0,43.2,0.27,26,1 81 | 2,112,66,22,0,25,0.307,24,0 82 | 3,113,44,13,0,22.4,0.14,22,0 83 | 2,74,0,0,0,0,0.102,22,0 84 | 7,83,78,26,71,29.3,0.767,36,0 85 | 0,101,65,28,0,24.6,0.237,22,0 86 | 5,137,108,0,0,48.8,0.227,37,1 87 | 2,110,74,29,125,32.4,0.698,27,0 88 | 13,106,72,54,0,36.6,0.178,45,0 89 | 2,100,68,25,71,38.5,0.324,26,0 90 | 15,136,70,32,110,37.1,0.153,43,1 91 | 1,107,68,19,0,26.5,0.165,24,0 92 | 1,80,55,0,0,19.1,0.258,21,0 93 | 4,123,80,15,176,32,0.443,34,0 94 | 7,81,78,40,48,46.7,0.261,42,0 95 | 4,134,72,0,0,23.8,0.277,60,1 96 | 2,142,82,18,64,24.7,0.761,21,0 97 | 6,144,72,27,228,33.9,0.255,40,0 98 | 2,92,62,28,0,31.6,0.13,24,0 99 | 1,71,48,18,76,20.4,0.323,22,0 100 | 6,93,50,30,64,28.7,0.356,23,0 101 | 1,122,90,51,220,49.7,0.325,31,1 102 | 1,163,72,0,0,39,1.222,33,1 103 | 1,151,60,0,0,26.1,0.179,22,0 104 | 0,125,96,0,0,22.5,0.262,21,0 105 | 1,81,72,18,40,26.6,0.283,24,0 106 | 2,85,65,0,0,39.6,0.93,27,0 107 | 1,126,56,29,152,28.7,0.801,21,0 108 | 1,96,122,0,0,22.4,0.207,27,0 109 | 4,144,58,28,140,29.5,0.287,37,0 110 | 3,83,58,31,18,34.3,0.336,25,0 111 | 0,95,85,25,36,37.4,0.247,24,1 112 | 3,171,72,33,135,33.3,0.199,24,1 113 | 8,155,62,26,495,34,0.543,46,1 114 | 1,89,76,34,37,31.2,0.192,23,0 115 | 4,76,62,0,0,34,0.391,25,0 116 | 7,160,54,32,175,30.5,0.588,39,1 117 | 4,146,92,0,0,31.2,0.539,61,1 118 | 5,124,74,0,0,34,0.22,38,1 119 | 5,78,48,0,0,33.7,0.654,25,0 120 | 4,97,60,23,0,28.2,0.443,22,0 121 | 4,99,76,15,51,23.2,0.223,21,0 122 | 0,162,76,56,100,53.2,0.759,25,1 123 | 6,111,64,39,0,34.2,0.26,24,0 124 | 2,107,74,30,100,33.6,0.404,23,0 125 | 5,132,80,0,0,26.8,0.186,69,0 126 | 0,113,76,0,0,33.3,0.278,23,1 127 | 1,88,30,42,99,55,0.496,26,1 128 | 3,120,70,30,135,42.9,0.452,30,0 129 | 1,118,58,36,94,33.3,0.261,23,0 130 | 1,117,88,24,145,34.5,0.403,40,1 131 | 0,105,84,0,0,27.9,0.741,62,1 132 | 4,173,70,14,168,29.7,0.361,33,1 133 | 9,122,56,0,0,33.3,1.114,33,1 134 | 3,170,64,37,225,34.5,0.356,30,1 135 | 8,84,74,31,0,38.3,0.457,39,0 136 | 2,96,68,13,49,21.1,0.647,26,0 137 | 2,125,60,20,140,33.8,0.088,31,0 138 | 0,100,70,26,50,30.8,0.597,21,0 139 | 0,93,60,25,92,28.7,0.532,22,0 140 | 0,129,80,0,0,31.2,0.703,29,0 141 | 5,105,72,29,325,36.9,0.159,28,0 142 | 3,128,78,0,0,21.1,0.268,55,0 143 | 5,106,82,30,0,39.5,0.286,38,0 144 | 2,108,52,26,63,32.5,0.318,22,0 145 | 10,108,66,0,0,32.4,0.272,42,1 146 | 4,154,62,31,284,32.8,0.237,23,0 147 | 0,102,75,23,0,0,0.572,21,0 148 | 9,57,80,37,0,32.8,0.096,41,0 149 | 2,106,64,35,119,30.5,1.4,34,0 150 | 5,147,78,0,0,33.7,0.218,65,0 151 | 2,90,70,17,0,27.3,0.085,22,0 152 | 1,136,74,50,204,37.4,0.399,24,0 153 | 4,114,65,0,0,21.9,0.432,37,0 154 | 9,156,86,28,155,34.3,1.189,42,1 155 | 1,153,82,42,485,40.6,0.687,23,0 156 | 8,188,78,0,0,47.9,0.137,43,1 157 | 7,152,88,44,0,50,0.337,36,1 158 | 2,99,52,15,94,24.6,0.637,21,0 159 | 1,109,56,21,135,25.2,0.833,23,0 160 | 2,88,74,19,53,29,0.229,22,0 161 | 17,163,72,41,114,40.9,0.817,47,1 162 | 4,151,90,38,0,29.7,0.294,36,0 163 | 7,102,74,40,105,37.2,0.204,45,0 164 | 0,114,80,34,285,44.2,0.167,27,0 165 | 2,100,64,23,0,29.7,0.368,21,0 166 | 0,131,88,0,0,31.6,0.743,32,1 167 | 6,104,74,18,156,29.9,0.722,41,1 168 | 3,148,66,25,0,32.5,0.256,22,0 169 | 4,120,68,0,0,29.6,0.709,34,0 170 | 4,110,66,0,0,31.9,0.471,29,0 171 | 3,111,90,12,78,28.4,0.495,29,0 172 | 6,102,82,0,0,30.8,0.18,36,1 173 | 6,134,70,23,130,35.4,0.542,29,1 174 | 2,87,0,23,0,28.9,0.773,25,0 175 | 1,79,60,42,48,43.5,0.678,23,0 176 | 2,75,64,24,55,29.7,0.37,33,0 177 | 8,179,72,42,130,32.7,0.719,36,1 178 | 6,85,78,0,0,31.2,0.382,42,0 179 | 0,129,110,46,130,67.1,0.319,26,1 180 | 5,143,78,0,0,45,0.19,47,0 181 | 5,130,82,0,0,39.1,0.956,37,1 182 | 6,87,80,0,0,23.2,0.084,32,0 183 | 0,119,64,18,92,34.9,0.725,23,0 184 | 1,0,74,20,23,27.7,0.299,21,0 185 | 5,73,60,0,0,26.8,0.268,27,0 186 | 4,141,74,0,0,27.6,0.244,40,0 187 | 7,194,68,28,0,35.9,0.745,41,1 188 | 8,181,68,36,495,30.1,0.615,60,1 189 | 1,128,98,41,58,32,1.321,33,1 190 | 8,109,76,39,114,27.9,0.64,31,1 191 | 5,139,80,35,160,31.6,0.361,25,1 192 | 3,111,62,0,0,22.6,0.142,21,0 193 | 9,123,70,44,94,33.1,0.374,40,0 194 | 7,159,66,0,0,30.4,0.383,36,1 195 | 11,135,0,0,0,52.3,0.578,40,1 196 | 8,85,55,20,0,24.4,0.136,42,0 197 | 5,158,84,41,210,39.4,0.395,29,1 198 | 1,105,58,0,0,24.3,0.187,21,0 199 | 3,107,62,13,48,22.9,0.678,23,1 200 | 4,109,64,44,99,34.8,0.905,26,1 201 | 4,148,60,27,318,30.9,0.15,29,1 202 | 0,113,80,16,0,31,0.874,21,0 203 | 1,138,82,0,0,40.1,0.236,28,0 204 | 0,108,68,20,0,27.3,0.787,32,0 205 | 2,99,70,16,44,20.4,0.235,27,0 206 | 6,103,72,32,190,37.7,0.324,55,0 207 | 5,111,72,28,0,23.9,0.407,27,0 208 | 8,196,76,29,280,37.5,0.605,57,1 209 | 5,162,104,0,0,37.7,0.151,52,1 210 | 1,96,64,27,87,33.2,0.289,21,0 211 | 7,184,84,33,0,35.5,0.355,41,1 212 | 2,81,60,22,0,27.7,0.29,25,0 213 | 0,147,85,54,0,42.8,0.375,24,0 214 | 7,179,95,31,0,34.2,0.164,60,0 215 | 0,140,65,26,130,42.6,0.431,24,1 216 | 9,112,82,32,175,34.2,0.26,36,1 217 | 12,151,70,40,271,41.8,0.742,38,1 218 | 5,109,62,41,129,35.8,0.514,25,1 219 | 6,125,68,30,120,30,0.464,32,0 220 | 5,85,74,22,0,29,1.224,32,1 221 | 5,112,66,0,0,37.8,0.261,41,1 222 | 0,177,60,29,478,34.6,1.072,21,1 223 | 2,158,90,0,0,31.6,0.805,66,1 224 | 7,119,0,0,0,25.2,0.209,37,0 225 | 7,142,60,33,190,28.8,0.687,61,0 226 | 1,100,66,15,56,23.6,0.666,26,0 227 | 1,87,78,27,32,34.6,0.101,22,0 228 | 0,101,76,0,0,35.7,0.198,26,0 229 | 3,162,52,38,0,37.2,0.652,24,1 230 | 4,197,70,39,744,36.7,2.329,31,0 231 | 0,117,80,31,53,45.2,0.089,24,0 232 | 4,142,86,0,0,44,0.645,22,1 233 | 6,134,80,37,370,46.2,0.238,46,1 234 | 1,79,80,25,37,25.4,0.583,22,0 235 | 4,122,68,0,0,35,0.394,29,0 236 | 3,74,68,28,45,29.7,0.293,23,0 237 | 4,171,72,0,0,43.6,0.479,26,1 238 | 7,181,84,21,192,35.9,0.586,51,1 239 | 0,179,90,27,0,44.1,0.686,23,1 240 | 9,164,84,21,0,30.8,0.831,32,1 241 | 0,104,76,0,0,18.4,0.582,27,0 242 | 1,91,64,24,0,29.2,0.192,21,0 243 | 4,91,70,32,88,33.1,0.446,22,0 244 | 3,139,54,0,0,25.6,0.402,22,1 245 | 6,119,50,22,176,27.1,1.318,33,1 246 | 2,146,76,35,194,38.2,0.329,29,0 247 | 9,184,85,15,0,30,1.213,49,1 248 | 10,122,68,0,0,31.2,0.258,41,0 249 | 0,165,90,33,680,52.3,0.427,23,0 250 | 9,124,70,33,402,35.4,0.282,34,0 251 | 1,111,86,19,0,30.1,0.143,23,0 252 | 9,106,52,0,0,31.2,0.38,42,0 253 | 2,129,84,0,0,28,0.284,27,0 254 | 2,90,80,14,55,24.4,0.249,24,0 255 | 0,86,68,32,0,35.8,0.238,25,0 256 | 12,92,62,7,258,27.6,0.926,44,1 257 | 1,113,64,35,0,33.6,0.543,21,1 258 | 3,111,56,39,0,30.1,0.557,30,0 259 | 2,114,68,22,0,28.7,0.092,25,0 260 | 1,193,50,16,375,25.9,0.655,24,0 261 | 11,155,76,28,150,33.3,1.353,51,1 262 | 3,191,68,15,130,30.9,0.299,34,0 263 | 3,141,0,0,0,30,0.761,27,1 264 | 4,95,70,32,0,32.1,0.612,24,0 265 | 3,142,80,15,0,32.4,0.2,63,0 266 | 4,123,62,0,0,32,0.226,35,1 267 | 5,96,74,18,67,33.6,0.997,43,0 268 | 0,138,0,0,0,36.3,0.933,25,1 269 | 2,128,64,42,0,40,1.101,24,0 270 | 0,102,52,0,0,25.1,0.078,21,0 271 | 2,146,0,0,0,27.5,0.24,28,1 272 | 10,101,86,37,0,45.6,1.136,38,1 273 | 2,108,62,32,56,25.2,0.128,21,0 274 | 3,122,78,0,0,23,0.254,40,0 275 | 1,71,78,50,45,33.2,0.422,21,0 276 | 13,106,70,0,0,34.2,0.251,52,0 277 | 2,100,70,52,57,40.5,0.677,25,0 278 | 7,106,60,24,0,26.5,0.296,29,1 279 | 0,104,64,23,116,27.8,0.454,23,0 280 | 5,114,74,0,0,24.9,0.744,57,0 281 | 2,108,62,10,278,25.3,0.881,22,0 282 | 0,146,70,0,0,37.9,0.334,28,1 283 | 10,129,76,28,122,35.9,0.28,39,0 284 | 7,133,88,15,155,32.4,0.262,37,0 285 | 7,161,86,0,0,30.4,0.165,47,1 286 | 2,108,80,0,0,27,0.259,52,1 287 | 7,136,74,26,135,26,0.647,51,0 288 | 5,155,84,44,545,38.7,0.619,34,0 289 | 1,119,86,39,220,45.6,0.808,29,1 290 | 4,96,56,17,49,20.8,0.34,26,0 291 | 5,108,72,43,75,36.1,0.263,33,0 292 | 0,78,88,29,40,36.9,0.434,21,0 293 | 0,107,62,30,74,36.6,0.757,25,1 294 | 2,128,78,37,182,43.3,1.224,31,1 295 | 1,128,48,45,194,40.5,0.613,24,1 296 | 0,161,50,0,0,21.9,0.254,65,0 297 | 6,151,62,31,120,35.5,0.692,28,0 298 | 2,146,70,38,360,28,0.337,29,1 299 | 0,126,84,29,215,30.7,0.52,24,0 300 | 14,100,78,25,184,36.6,0.412,46,1 301 | 8,112,72,0,0,23.6,0.84,58,0 302 | 0,167,0,0,0,32.3,0.839,30,1 303 | 2,144,58,33,135,31.6,0.422,25,1 304 | 5,77,82,41,42,35.8,0.156,35,0 305 | 5,115,98,0,0,52.9,0.209,28,1 306 | 3,150,76,0,0,21,0.207,37,0 307 | 2,120,76,37,105,39.7,0.215,29,0 308 | 10,161,68,23,132,25.5,0.326,47,1 309 | 0,137,68,14,148,24.8,0.143,21,0 310 | 0,128,68,19,180,30.5,1.391,25,1 311 | 2,124,68,28,205,32.9,0.875,30,1 312 | 6,80,66,30,0,26.2,0.313,41,0 313 | 0,106,70,37,148,39.4,0.605,22,0 314 | 2,155,74,17,96,26.6,0.433,27,1 315 | 3,113,50,10,85,29.5,0.626,25,0 316 | 7,109,80,31,0,35.9,1.127,43,1 317 | 2,112,68,22,94,34.1,0.315,26,0 318 | 3,99,80,11,64,19.3,0.284,30,0 319 | 3,182,74,0,0,30.5,0.345,29,1 320 | 3,115,66,39,140,38.1,0.15,28,0 321 | 6,194,78,0,0,23.5,0.129,59,1 322 | 4,129,60,12,231,27.5,0.527,31,0 323 | 3,112,74,30,0,31.6,0.197,25,1 324 | 0,124,70,20,0,27.4,0.254,36,1 325 | 13,152,90,33,29,26.8,0.731,43,1 326 | 2,112,75,32,0,35.7,0.148,21,0 327 | 1,157,72,21,168,25.6,0.123,24,0 328 | 1,122,64,32,156,35.1,0.692,30,1 329 | 10,179,70,0,0,35.1,0.2,37,0 330 | 2,102,86,36,120,45.5,0.127,23,1 331 | 6,105,70,32,68,30.8,0.122,37,0 332 | 8,118,72,19,0,23.1,1.476,46,0 333 | 2,87,58,16,52,32.7,0.166,25,0 334 | 1,180,0,0,0,43.3,0.282,41,1 335 | 12,106,80,0,0,23.6,0.137,44,0 336 | 1,95,60,18,58,23.9,0.26,22,0 337 | 0,165,76,43,255,47.9,0.259,26,0 338 | 0,117,0,0,0,33.8,0.932,44,0 339 | 5,115,76,0,0,31.2,0.343,44,1 340 | 9,152,78,34,171,34.2,0.893,33,1 341 | 7,178,84,0,0,39.9,0.331,41,1 342 | 1,130,70,13,105,25.9,0.472,22,0 343 | 1,95,74,21,73,25.9,0.673,36,0 344 | 1,0,68,35,0,32,0.389,22,0 345 | 5,122,86,0,0,34.7,0.29,33,0 346 | 8,95,72,0,0,36.8,0.485,57,0 347 | 8,126,88,36,108,38.5,0.349,49,0 348 | 1,139,46,19,83,28.7,0.654,22,0 349 | 3,116,0,0,0,23.5,0.187,23,0 350 | 3,99,62,19,74,21.8,0.279,26,0 351 | 5,0,80,32,0,41,0.346,37,1 352 | 4,92,80,0,0,42.2,0.237,29,0 353 | 4,137,84,0,0,31.2,0.252,30,0 354 | 3,61,82,28,0,34.4,0.243,46,0 355 | 1,90,62,12,43,27.2,0.58,24,0 356 | 3,90,78,0,0,42.7,0.559,21,0 357 | 9,165,88,0,0,30.4,0.302,49,1 358 | 1,125,50,40,167,33.3,0.962,28,1 359 | 13,129,0,30,0,39.9,0.569,44,1 360 | 12,88,74,40,54,35.3,0.378,48,0 361 | 1,196,76,36,249,36.5,0.875,29,1 362 | 5,189,64,33,325,31.2,0.583,29,1 363 | 5,158,70,0,0,29.8,0.207,63,0 364 | 5,103,108,37,0,39.2,0.305,65,0 365 | 4,146,78,0,0,38.5,0.52,67,1 366 | 4,147,74,25,293,34.9,0.385,30,0 367 | 5,99,54,28,83,34,0.499,30,0 368 | 6,124,72,0,0,27.6,0.368,29,1 369 | 0,101,64,17,0,21,0.252,21,0 370 | 3,81,86,16,66,27.5,0.306,22,0 371 | 1,133,102,28,140,32.8,0.234,45,1 372 | 3,173,82,48,465,38.4,2.137,25,1 373 | 0,118,64,23,89,0,1.731,21,0 374 | 0,84,64,22,66,35.8,0.545,21,0 375 | 2,105,58,40,94,34.9,0.225,25,0 376 | 2,122,52,43,158,36.2,0.816,28,0 377 | 12,140,82,43,325,39.2,0.528,58,1 378 | 0,98,82,15,84,25.2,0.299,22,0 379 | 1,87,60,37,75,37.2,0.509,22,0 380 | 4,156,75,0,0,48.3,0.238,32,1 381 | 0,93,100,39,72,43.4,1.021,35,0 382 | 1,107,72,30,82,30.8,0.821,24,0 383 | 0,105,68,22,0,20,0.236,22,0 384 | 1,109,60,8,182,25.4,0.947,21,0 385 | 1,90,62,18,59,25.1,1.268,25,0 386 | 1,125,70,24,110,24.3,0.221,25,0 387 | 1,119,54,13,50,22.3,0.205,24,0 388 | 5,116,74,29,0,32.3,0.66,35,1 389 | 8,105,100,36,0,43.3,0.239,45,1 390 | 5,144,82,26,285,32,0.452,58,1 391 | 3,100,68,23,81,31.6,0.949,28,0 392 | 1,100,66,29,196,32,0.444,42,0 393 | 5,166,76,0,0,45.7,0.34,27,1 394 | 1,131,64,14,415,23.7,0.389,21,0 395 | 4,116,72,12,87,22.1,0.463,37,0 396 | 4,158,78,0,0,32.9,0.803,31,1 397 | 2,127,58,24,275,27.7,1.6,25,0 398 | 3,96,56,34,115,24.7,0.944,39,0 399 | 0,131,66,40,0,34.3,0.196,22,1 400 | 3,82,70,0,0,21.1,0.389,25,0 401 | 3,193,70,31,0,34.9,0.241,25,1 402 | 4,95,64,0,0,32,0.161,31,1 403 | 6,137,61,0,0,24.2,0.151,55,0 404 | 5,136,84,41,88,35,0.286,35,1 405 | 9,72,78,25,0,31.6,0.28,38,0 406 | 5,168,64,0,0,32.9,0.135,41,1 407 | 2,123,48,32,165,42.1,0.52,26,0 408 | 4,115,72,0,0,28.9,0.376,46,1 409 | 0,101,62,0,0,21.9,0.336,25,0 410 | 8,197,74,0,0,25.9,1.191,39,1 411 | 1,172,68,49,579,42.4,0.702,28,1 412 | 6,102,90,39,0,35.7,0.674,28,0 413 | 1,112,72,30,176,34.4,0.528,25,0 414 | 1,143,84,23,310,42.4,1.076,22,0 415 | 1,143,74,22,61,26.2,0.256,21,0 416 | 0,138,60,35,167,34.6,0.534,21,1 417 | 3,173,84,33,474,35.7,0.258,22,1 418 | 1,97,68,21,0,27.2,1.095,22,0 419 | 4,144,82,32,0,38.5,0.554,37,1 420 | 1,83,68,0,0,18.2,0.624,27,0 421 | 3,129,64,29,115,26.4,0.219,28,1 422 | 1,119,88,41,170,45.3,0.507,26,0 423 | 2,94,68,18,76,26,0.561,21,0 424 | 0,102,64,46,78,40.6,0.496,21,0 425 | 2,115,64,22,0,30.8,0.421,21,0 426 | 8,151,78,32,210,42.9,0.516,36,1 427 | 4,184,78,39,277,37,0.264,31,1 428 | 0,94,0,0,0,0,0.256,25,0 429 | 1,181,64,30,180,34.1,0.328,38,1 430 | 0,135,94,46,145,40.6,0.284,26,0 431 | 1,95,82,25,180,35,0.233,43,1 432 | 2,99,0,0,0,22.2,0.108,23,0 433 | 3,89,74,16,85,30.4,0.551,38,0 434 | 1,80,74,11,60,30,0.527,22,0 435 | 2,139,75,0,0,25.6,0.167,29,0 436 | 1,90,68,8,0,24.5,1.138,36,0 437 | 0,141,0,0,0,42.4,0.205,29,1 438 | 12,140,85,33,0,37.4,0.244,41,0 439 | 5,147,75,0,0,29.9,0.434,28,0 440 | 1,97,70,15,0,18.2,0.147,21,0 441 | 6,107,88,0,0,36.8,0.727,31,0 442 | 0,189,104,25,0,34.3,0.435,41,1 443 | 2,83,66,23,50,32.2,0.497,22,0 444 | 4,117,64,27,120,33.2,0.23,24,0 445 | 8,108,70,0,0,30.5,0.955,33,1 446 | 4,117,62,12,0,29.7,0.38,30,1 447 | 0,180,78,63,14,59.4,2.42,25,1 448 | 1,100,72,12,70,25.3,0.658,28,0 449 | 0,95,80,45,92,36.5,0.33,26,0 450 | 0,104,64,37,64,33.6,0.51,22,1 451 | 0,120,74,18,63,30.5,0.285,26,0 452 | 1,82,64,13,95,21.2,0.415,23,0 453 | 2,134,70,0,0,28.9,0.542,23,1 454 | 0,91,68,32,210,39.9,0.381,25,0 455 | 2,119,0,0,0,19.6,0.832,72,0 456 | 2,100,54,28,105,37.8,0.498,24,0 457 | 14,175,62,30,0,33.6,0.212,38,1 458 | 1,135,54,0,0,26.7,0.687,62,0 459 | 5,86,68,28,71,30.2,0.364,24,0 460 | 10,148,84,48,237,37.6,1.001,51,1 461 | 9,134,74,33,60,25.9,0.46,81,0 462 | 9,120,72,22,56,20.8,0.733,48,0 463 | 1,71,62,0,0,21.8,0.416,26,0 464 | 8,74,70,40,49,35.3,0.705,39,0 465 | 5,88,78,30,0,27.6,0.258,37,0 466 | 10,115,98,0,0,24,1.022,34,0 467 | 0,124,56,13,105,21.8,0.452,21,0 468 | 0,74,52,10,36,27.8,0.269,22,0 469 | 0,97,64,36,100,36.8,0.6,25,0 470 | 8,120,0,0,0,30,0.183,38,1 471 | 6,154,78,41,140,46.1,0.571,27,0 472 | 1,144,82,40,0,41.3,0.607,28,0 473 | 0,137,70,38,0,33.2,0.17,22,0 474 | 0,119,66,27,0,38.8,0.259,22,0 475 | 7,136,90,0,0,29.9,0.21,50,0 476 | 4,114,64,0,0,28.9,0.126,24,0 477 | 0,137,84,27,0,27.3,0.231,59,0 478 | 2,105,80,45,191,33.7,0.711,29,1 479 | 7,114,76,17,110,23.8,0.466,31,0 480 | 8,126,74,38,75,25.9,0.162,39,0 481 | 4,132,86,31,0,28,0.419,63,0 482 | 3,158,70,30,328,35.5,0.344,35,1 483 | 0,123,88,37,0,35.2,0.197,29,0 484 | 4,85,58,22,49,27.8,0.306,28,0 485 | 0,84,82,31,125,38.2,0.233,23,0 486 | 0,145,0,0,0,44.2,0.63,31,1 487 | 0,135,68,42,250,42.3,0.365,24,1 488 | 1,139,62,41,480,40.7,0.536,21,0 489 | 0,173,78,32,265,46.5,1.159,58,0 490 | 4,99,72,17,0,25.6,0.294,28,0 491 | 8,194,80,0,0,26.1,0.551,67,0 492 | 2,83,65,28,66,36.8,0.629,24,0 493 | 2,89,90,30,0,33.5,0.292,42,0 494 | 4,99,68,38,0,32.8,0.145,33,0 495 | 4,125,70,18,122,28.9,1.144,45,1 496 | 3,80,0,0,0,0,0.174,22,0 497 | 6,166,74,0,0,26.6,0.304,66,0 498 | 5,110,68,0,0,26,0.292,30,0 499 | 2,81,72,15,76,30.1,0.547,25,0 500 | 7,195,70,33,145,25.1,0.163,55,1 501 | 6,154,74,32,193,29.3,0.839,39,0 502 | 2,117,90,19,71,25.2,0.313,21,0 503 | 3,84,72,32,0,37.2,0.267,28,0 504 | 6,0,68,41,0,39,0.727,41,1 505 | 7,94,64,25,79,33.3,0.738,41,0 506 | 3,96,78,39,0,37.3,0.238,40,0 507 | 10,75,82,0,0,33.3,0.263,38,0 508 | 0,180,90,26,90,36.5,0.314,35,1 509 | 1,130,60,23,170,28.6,0.692,21,0 510 | 2,84,50,23,76,30.4,0.968,21,0 511 | 8,120,78,0,0,25,0.409,64,0 512 | 12,84,72,31,0,29.7,0.297,46,1 513 | 0,139,62,17,210,22.1,0.207,21,0 514 | 9,91,68,0,0,24.2,0.2,58,0 515 | 2,91,62,0,0,27.3,0.525,22,0 516 | 3,99,54,19,86,25.6,0.154,24,0 517 | 3,163,70,18,105,31.6,0.268,28,1 518 | 9,145,88,34,165,30.3,0.771,53,1 519 | 7,125,86,0,0,37.6,0.304,51,0 520 | 13,76,60,0,0,32.8,0.18,41,0 521 | 6,129,90,7,326,19.6,0.582,60,0 522 | 2,68,70,32,66,25,0.187,25,0 523 | 3,124,80,33,130,33.2,0.305,26,0 524 | 6,114,0,0,0,0,0.189,26,0 525 | 9,130,70,0,0,34.2,0.652,45,1 526 | 3,125,58,0,0,31.6,0.151,24,0 527 | 3,87,60,18,0,21.8,0.444,21,0 528 | 1,97,64,19,82,18.2,0.299,21,0 529 | 3,116,74,15,105,26.3,0.107,24,0 530 | 0,117,66,31,188,30.8,0.493,22,0 531 | 0,111,65,0,0,24.6,0.66,31,0 532 | 2,122,60,18,106,29.8,0.717,22,0 533 | 0,107,76,0,0,45.3,0.686,24,0 534 | 1,86,66,52,65,41.3,0.917,29,0 535 | 6,91,0,0,0,29.8,0.501,31,0 536 | 1,77,56,30,56,33.3,1.251,24,0 537 | 4,132,0,0,0,32.9,0.302,23,1 538 | 0,105,90,0,0,29.6,0.197,46,0 539 | 0,57,60,0,0,21.7,0.735,67,0 540 | 0,127,80,37,210,36.3,0.804,23,0 541 | 3,129,92,49,155,36.4,0.968,32,1 542 | 8,100,74,40,215,39.4,0.661,43,1 543 | 3,128,72,25,190,32.4,0.549,27,1 544 | 10,90,85,32,0,34.9,0.825,56,1 545 | 4,84,90,23,56,39.5,0.159,25,0 546 | 1,88,78,29,76,32,0.365,29,0 547 | 8,186,90,35,225,34.5,0.423,37,1 548 | 5,187,76,27,207,43.6,1.034,53,1 549 | 4,131,68,21,166,33.1,0.16,28,0 550 | 1,164,82,43,67,32.8,0.341,50,0 551 | 4,189,110,31,0,28.5,0.68,37,0 552 | 1,116,70,28,0,27.4,0.204,21,0 553 | 3,84,68,30,106,31.9,0.591,25,0 554 | 6,114,88,0,0,27.8,0.247,66,0 555 | 1,88,62,24,44,29.9,0.422,23,0 556 | 1,84,64,23,115,36.9,0.471,28,0 557 | 7,124,70,33,215,25.5,0.161,37,0 558 | 1,97,70,40,0,38.1,0.218,30,0 559 | 8,110,76,0,0,27.8,0.237,58,0 560 | 11,103,68,40,0,46.2,0.126,42,0 561 | 11,85,74,0,0,30.1,0.3,35,0 562 | 6,125,76,0,0,33.8,0.121,54,1 563 | 0,198,66,32,274,41.3,0.502,28,1 564 | 1,87,68,34,77,37.6,0.401,24,0 565 | 6,99,60,19,54,26.9,0.497,32,0 566 | 0,91,80,0,0,32.4,0.601,27,0 567 | 2,95,54,14,88,26.1,0.748,22,0 568 | 1,99,72,30,18,38.6,0.412,21,0 569 | 6,92,62,32,126,32,0.085,46,0 570 | 4,154,72,29,126,31.3,0.338,37,0 571 | 0,121,66,30,165,34.3,0.203,33,1 572 | 3,78,70,0,0,32.5,0.27,39,0 573 | 2,130,96,0,0,22.6,0.268,21,0 574 | 3,111,58,31,44,29.5,0.43,22,0 575 | 2,98,60,17,120,34.7,0.198,22,0 576 | 1,143,86,30,330,30.1,0.892,23,0 577 | 1,119,44,47,63,35.5,0.28,25,0 578 | 6,108,44,20,130,24,0.813,35,0 579 | 2,118,80,0,0,42.9,0.693,21,1 580 | 10,133,68,0,0,27,0.245,36,0 581 | 2,197,70,99,0,34.7,0.575,62,1 582 | 0,151,90,46,0,42.1,0.371,21,1 583 | 6,109,60,27,0,25,0.206,27,0 584 | 12,121,78,17,0,26.5,0.259,62,0 585 | 8,100,76,0,0,38.7,0.19,42,0 586 | 8,124,76,24,600,28.7,0.687,52,1 587 | 1,93,56,11,0,22.5,0.417,22,0 588 | 8,143,66,0,0,34.9,0.129,41,1 589 | 6,103,66,0,0,24.3,0.249,29,0 590 | 3,176,86,27,156,33.3,1.154,52,1 591 | 0,73,0,0,0,21.1,0.342,25,0 592 | 11,111,84,40,0,46.8,0.925,45,1 593 | 2,112,78,50,140,39.4,0.175,24,0 594 | 3,132,80,0,0,34.4,0.402,44,1 595 | 2,82,52,22,115,28.5,1.699,25,0 596 | 6,123,72,45,230,33.6,0.733,34,0 597 | 0,188,82,14,185,32,0.682,22,1 598 | 0,67,76,0,0,45.3,0.194,46,0 599 | 1,89,24,19,25,27.8,0.559,21,0 600 | 1,173,74,0,0,36.8,0.088,38,1 601 | 1,109,38,18,120,23.1,0.407,26,0 602 | 1,108,88,19,0,27.1,0.4,24,0 603 | 6,96,0,0,0,23.7,0.19,28,0 604 | 1,124,74,36,0,27.8,0.1,30,0 605 | 7,150,78,29,126,35.2,0.692,54,1 606 | 4,183,0,0,0,28.4,0.212,36,1 607 | 1,124,60,32,0,35.8,0.514,21,0 608 | 1,181,78,42,293,40,1.258,22,1 609 | 1,92,62,25,41,19.5,0.482,25,0 610 | 0,152,82,39,272,41.5,0.27,27,0 611 | 1,111,62,13,182,24,0.138,23,0 612 | 3,106,54,21,158,30.9,0.292,24,0 613 | 3,174,58,22,194,32.9,0.593,36,1 614 | 7,168,88,42,321,38.2,0.787,40,1 615 | 6,105,80,28,0,32.5,0.878,26,0 616 | 11,138,74,26,144,36.1,0.557,50,1 617 | 3,106,72,0,0,25.8,0.207,27,0 618 | 6,117,96,0,0,28.7,0.157,30,0 619 | 2,68,62,13,15,20.1,0.257,23,0 620 | 9,112,82,24,0,28.2,1.282,50,1 621 | 0,119,0,0,0,32.4,0.141,24,1 622 | 2,112,86,42,160,38.4,0.246,28,0 623 | 2,92,76,20,0,24.2,1.698,28,0 624 | 6,183,94,0,0,40.8,1.461,45,0 625 | 0,94,70,27,115,43.5,0.347,21,0 626 | 2,108,64,0,0,30.8,0.158,21,0 627 | 4,90,88,47,54,37.7,0.362,29,0 628 | 0,125,68,0,0,24.7,0.206,21,0 629 | 0,132,78,0,0,32.4,0.393,21,0 630 | 5,128,80,0,0,34.6,0.144,45,0 631 | 4,94,65,22,0,24.7,0.148,21,0 632 | 7,114,64,0,0,27.4,0.732,34,1 633 | 0,102,78,40,90,34.5,0.238,24,0 634 | 2,111,60,0,0,26.2,0.343,23,0 635 | 1,128,82,17,183,27.5,0.115,22,0 636 | 10,92,62,0,0,25.9,0.167,31,0 637 | 13,104,72,0,0,31.2,0.465,38,1 638 | 5,104,74,0,0,28.8,0.153,48,0 639 | 2,94,76,18,66,31.6,0.649,23,0 640 | 7,97,76,32,91,40.9,0.871,32,1 641 | 1,100,74,12,46,19.5,0.149,28,0 642 | 0,102,86,17,105,29.3,0.695,27,0 643 | 4,128,70,0,0,34.3,0.303,24,0 644 | 6,147,80,0,0,29.5,0.178,50,1 645 | 4,90,0,0,0,28,0.61,31,0 646 | 3,103,72,30,152,27.6,0.73,27,0 647 | 2,157,74,35,440,39.4,0.134,30,0 648 | 1,167,74,17,144,23.4,0.447,33,1 649 | 0,179,50,36,159,37.8,0.455,22,1 650 | 11,136,84,35,130,28.3,0.26,42,1 651 | 0,107,60,25,0,26.4,0.133,23,0 652 | 1,91,54,25,100,25.2,0.234,23,0 653 | 1,117,60,23,106,33.8,0.466,27,0 654 | 5,123,74,40,77,34.1,0.269,28,0 655 | 2,120,54,0,0,26.8,0.455,27,0 656 | 1,106,70,28,135,34.2,0.142,22,0 657 | 2,155,52,27,540,38.7,0.24,25,1 658 | 2,101,58,35,90,21.8,0.155,22,0 659 | 1,120,80,48,200,38.9,1.162,41,0 660 | 11,127,106,0,0,39,0.19,51,0 661 | 3,80,82,31,70,34.2,1.292,27,1 662 | 10,162,84,0,0,27.7,0.182,54,0 663 | 1,199,76,43,0,42.9,1.394,22,1 664 | 8,167,106,46,231,37.6,0.165,43,1 665 | 9,145,80,46,130,37.9,0.637,40,1 666 | 6,115,60,39,0,33.7,0.245,40,1 667 | 1,112,80,45,132,34.8,0.217,24,0 668 | 4,145,82,18,0,32.5,0.235,70,1 669 | 10,111,70,27,0,27.5,0.141,40,1 670 | 6,98,58,33,190,34,0.43,43,0 671 | 9,154,78,30,100,30.9,0.164,45,0 672 | 6,165,68,26,168,33.6,0.631,49,0 673 | 1,99,58,10,0,25.4,0.551,21,0 674 | 10,68,106,23,49,35.5,0.285,47,0 675 | 3,123,100,35,240,57.3,0.88,22,0 676 | 8,91,82,0,0,35.6,0.587,68,0 677 | 6,195,70,0,0,30.9,0.328,31,1 678 | 9,156,86,0,0,24.8,0.23,53,1 679 | 0,93,60,0,0,35.3,0.263,25,0 680 | 3,121,52,0,0,36,0.127,25,1 681 | 2,101,58,17,265,24.2,0.614,23,0 682 | 2,56,56,28,45,24.2,0.332,22,0 683 | 0,162,76,36,0,49.6,0.364,26,1 684 | 0,95,64,39,105,44.6,0.366,22,0 685 | 4,125,80,0,0,32.3,0.536,27,1 686 | 5,136,82,0,0,0,0.64,69,0 687 | 2,129,74,26,205,33.2,0.591,25,0 688 | 3,130,64,0,0,23.1,0.314,22,0 689 | 1,107,50,19,0,28.3,0.181,29,0 690 | 1,140,74,26,180,24.1,0.828,23,0 691 | 1,144,82,46,180,46.1,0.335,46,1 692 | 8,107,80,0,0,24.6,0.856,34,0 693 | 13,158,114,0,0,42.3,0.257,44,1 694 | 2,121,70,32,95,39.1,0.886,23,0 695 | 7,129,68,49,125,38.5,0.439,43,1 696 | 2,90,60,0,0,23.5,0.191,25,0 697 | 7,142,90,24,480,30.4,0.128,43,1 698 | 3,169,74,19,125,29.9,0.268,31,1 699 | 0,99,0,0,0,25,0.253,22,0 700 | 4,127,88,11,155,34.5,0.598,28,0 701 | 4,118,70,0,0,44.5,0.904,26,0 702 | 2,122,76,27,200,35.9,0.483,26,0 703 | 6,125,78,31,0,27.6,0.565,49,1 704 | 1,168,88,29,0,35,0.905,52,1 705 | 2,129,0,0,0,38.5,0.304,41,0 706 | 4,110,76,20,100,28.4,0.118,27,0 707 | 6,80,80,36,0,39.8,0.177,28,0 708 | 10,115,0,0,0,0,0.261,30,1 709 | 2,127,46,21,335,34.4,0.176,22,0 710 | 9,164,78,0,0,32.8,0.148,45,1 711 | 2,93,64,32,160,38,0.674,23,1 712 | 3,158,64,13,387,31.2,0.295,24,0 713 | 5,126,78,27,22,29.6,0.439,40,0 714 | 10,129,62,36,0,41.2,0.441,38,1 715 | 0,134,58,20,291,26.4,0.352,21,0 716 | 3,102,74,0,0,29.5,0.121,32,0 717 | 7,187,50,33,392,33.9,0.826,34,1 718 | 3,173,78,39,185,33.8,0.97,31,1 719 | 10,94,72,18,0,23.1,0.595,56,0 720 | 1,108,60,46,178,35.5,0.415,24,0 721 | 5,97,76,27,0,35.6,0.378,52,1 722 | 4,83,86,19,0,29.3,0.317,34,0 723 | 1,114,66,36,200,38.1,0.289,21,0 724 | 1,149,68,29,127,29.3,0.349,42,1 725 | 5,117,86,30,105,39.1,0.251,42,0 726 | 1,111,94,0,0,32.8,0.265,45,0 727 | 4,112,78,40,0,39.4,0.236,38,0 728 | 1,116,78,29,180,36.1,0.496,25,0 729 | 0,141,84,26,0,32.4,0.433,22,0 730 | 2,175,88,0,0,22.9,0.326,22,0 731 | 2,92,52,0,0,30.1,0.141,22,0 732 | 3,130,78,23,79,28.4,0.323,34,1 733 | 8,120,86,0,0,28.4,0.259,22,1 734 | 2,174,88,37,120,44.5,0.646,24,1 735 | 2,106,56,27,165,29,0.426,22,0 736 | 2,105,75,0,0,23.3,0.56,53,0 737 | 4,95,60,32,0,35.4,0.284,28,0 738 | 0,126,86,27,120,27.4,0.515,21,0 739 | 8,65,72,23,0,32,0.6,42,0 740 | 2,99,60,17,160,36.6,0.453,21,0 741 | 1,102,74,0,0,39.5,0.293,42,1 742 | 11,120,80,37,150,42.3,0.785,48,1 743 | 3,102,44,20,94,30.8,0.4,26,0 744 | 1,109,58,18,116,28.5,0.219,22,0 745 | 9,140,94,0,0,32.7,0.734,45,1 746 | 13,153,88,37,140,40.6,1.174,39,0 747 | 12,100,84,33,105,30,0.488,46,0 748 | 1,147,94,41,0,49.3,0.358,27,1 749 | 1,81,74,41,57,46.3,1.096,32,0 750 | 3,187,70,22,200,36.4,0.408,36,1 751 | 6,162,62,0,0,24.3,0.178,50,1 752 | 4,136,70,0,0,31.2,1.182,22,1 753 | 1,121,78,39,74,39,0.261,28,0 754 | 3,108,62,24,0,26,0.223,25,0 755 | 0,181,88,44,510,43.3,0.222,26,1 756 | 8,154,78,32,0,32.4,0.443,45,1 757 | 1,128,88,39,110,36.5,1.057,37,1 758 | 7,137,90,41,0,32,0.391,39,0 759 | 0,123,72,0,0,36.3,0.258,52,1 760 | 1,106,76,0,0,37.5,0.197,26,0 761 | 6,190,92,0,0,35.5,0.278,66,1 762 | 2,88,58,26,16,28.4,0.766,22,0 763 | 9,170,74,31,0,44,0.403,43,1 764 | 9,89,62,0,0,22.5,0.142,33,0 765 | 10,101,76,48,180,32.9,0.171,63,0 766 | 2,122,70,27,0,36.8,0.34,27,0 767 | 5,121,72,23,112,26.2,0.245,30,0 768 | 1,126,60,0,0,30.1,0.349,47,1 769 | 1,93,70,31,0,30.4,0.315,23,0 -------------------------------------------------------------------------------- /3.create-model/pytorch-lab/model/README.md: -------------------------------------------------------------------------------- 1 | Created models will be saved in this folder -------------------------------------------------------------------------------- /3.create-model/pytorch-lab/notebooks/01-data-exploration.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Data Exploration\n", 8 | "\n", 9 | "In this notebook, we'll take the opportunity to explore the data we will be using. Data exploration and understanding is a fundamental step in Machine Learning.\n", 10 | "\n", 11 | "In this learning path, the data we'll be looking at is the Diabetes data set. It consists of ~800 rows of medical data (e.g. blood pressure, heart rate etc...) for female patients with and without diabetes. It is worth mentioning that in ML, 800 examples is considered a very very small dataset. We'll be creating a model that will help us determine which of our patients, based on their medical data, may have diabetes.\n", 12 | "\n", 13 | "Reference to the dataset - https://www.kaggle.com/datasets/mathchi/diabetes-data-set\n", 14 | "\n", 15 | "It is helpful to know a few characteristics about the dataset in this lab. Specifically how it is laid out. In the Diabetes dataset we have provided, we have done some preprocessing of the data. The data you will be provided with is in CSV format. There are no patient names provided in this dataset.\n", 16 | "\n", 17 | "This dataset will be used to train the model to recognize which medical data (e.g. blood pressure, blood values) are common in patients who have diabetes.\n", 18 | "\n", 19 | "At the end of this notebook, we'll have a better understanding of what our data looks like, and that will help us when constructing the AI in the future notebook.\n", 20 | "\n", 21 | "To start, let's load the data!" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "# import numpy as np\n", 31 | "import pandas as pd\n", 32 | "import matplotlib.pyplot as plt" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": null, 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "# Load diabetes dataset into a dataframe called 'df'\n", 42 | "df = pd.read_csv(\"../data/diabetes.csv\")\n", 43 | "\n", 44 | "# Obtain the length (rows) and shape (columns) of our dataset\n", 45 | "len(df)\n", 46 | "df.shape" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "We see that there are 769 rows and 9 columns. " 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "## Data Analysis\n", 61 | "Let's analyze our data closer using some standard data analysis methods available in python. A useful method is info() which will show us the column names and data types in our dataset." 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "df.info()" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "We can see the column names and associated data types. Pregnancies, Glucose, BloodPressure, SkinThickness, Insulin, Age and Outcome are of data type integer (e.g. 34, 2, 98).\n", 78 | "\n", 79 | "BMI and Diabetes PedigreeFuntion are of data type float (e.g. 1.3, 4.56)\n", 80 | "\n", 81 | "Let's use the describe() function to learn statistical information (e.g. mean, standard deviation, min, max...) for each of our columns." 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "df.describe()" 91 | ] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "metadata": {}, 96 | "source": [ 97 | "We see that there are 769 entries in our dataset. There are 9 columns and they are all non-null." 98 | ] 99 | }, 100 | { 101 | "cell_type": "markdown", 102 | "metadata": {}, 103 | "source": [ 104 | "The dataset consists of several medical predictor (independent) variables and one target (dependent) variable, Outcome. Independent variables include the number of pregnancies the patient has had, their BMI, insulin level, age, and so on.\n", 105 | "\n", 106 | "* Pregnancies - Number of times pregnant\n", 107 | "* Glucose - Plasma glucose concentration a 2 hours in an oral glucose tolerance test\n", 108 | "* Blood Pressure - Diastolic blood pressure (mm Hg)\n", 109 | "* SkinThickness - Triceps skin fold thickness (mm)\n", 110 | "* Insulin - 2-Hour serum insulin (mu U/ml)\n", 111 | "* BMI - Body mass index (weight in kg/(height in m)^2)\n", 112 | "* DiabetesPedigreeFunction - Diabetes pedigree function\n", 113 | "* Age - Age (years)\n", 114 | "* Outcome - Class variable (0 or 1) 268 of 768 are 1, the others are 0" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "\n", 122 | "Let's now look at our data using a series of histograms and plots in order to see how 'distributed' our data appears. We can do this using the hist() function to visually see how our data is distributed for each column in our data set. This will allow us to determine if any of the columns have 'outliers'. \n", 123 | "\n", 124 | "Outliers are data points that differ significantly from other data points. They can be due to a variability in measurement or may even indicate a measurement error. It is up to the Data Engineer (or Data Scientist) to determine if the outlier(s) are considered abnormal and should be removed or kept within the dataset. \n" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "# Plot histograms of the columns on multiple subplots\n", 134 | "plt.close('all')\n", 135 | "df.hist(color=\"k\", alpha=0.5, bins=20, figsize=(15, 10))" 136 | ] 137 | }, 138 | { 139 | "cell_type": "markdown", 140 | "metadata": {}, 141 | "source": [ 142 | "From the above histograms we can see that our dataset is evenly distributed except for some outliers in BMI, Blood Pressure, Outcomes and DiabetesPedigreeFunction. For now we will keep the outliers in our dataset.\n" 143 | ] 144 | }, 145 | { 146 | "cell_type": "markdown", 147 | "metadata": {}, 148 | "source": [ 149 | "Let's take a closer look at the actual values in our dataset. A method we can use to examine the first 20 rows of data (plus the column names) is the head() function. Using df.head() we can see the column names of our dataframe: Pregnancies, Glucose, BloodPressure, Skinthickness, Insulin, BMI, \n", 150 | "DiabetespedigreeFunction, Age and Outcome" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [ 159 | "df.head(20)" 160 | ] 161 | }, 162 | { 163 | "cell_type": "markdown", 164 | "metadata": {}, 165 | "source": [ 166 | "If we look closely we see that there are '0' values found for Insulin levels and skin thickness. The 0 values indicate that we have missing or null data. Let's check the last 20 rows of our dataset and determine if there are any '0' values present." 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": null, 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "# Let's examine the last 20 rows of our dataframe\n", 176 | "df.tail(20)" 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": {}, 182 | "source": [ 183 | "In the last 20 rows of our dataset, there are also '0' values present for Insulin and SkinThickness. Therefore, it is likely that there are more '0' values present throughout our dataset.\n", 184 | "\n", 185 | "Let's determine how many '0' values are present in our dataset. " 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "metadata": {}, 192 | "outputs": [], 193 | "source": [ 194 | "# Determine the number of '0' values\n", 195 | "df2 = df.iloc[:, :-1]\n", 196 | "\n", 197 | "# Number & percent of '0's for each attribute\n", 198 | "numZero = (df2[:] == 0).sum()\n", 199 | "perZero = ((df2[:] == 0).sum())/768*100\n", 200 | "\n", 201 | "print(\"\\nRows, Columns: \",df2.shape)\n", 202 | "print(\"\\nNumber of 0's:\")\n", 203 | "print(numZero)\n", 204 | "print(\"\\nPercentage of 0's:\")\n", 205 | "print(perZero)" 206 | ] 207 | }, 208 | { 209 | "cell_type": "markdown", 210 | "metadata": {}, 211 | "source": [ 212 | "\n", 213 | "There are 227 Zero values for SkinThickness and 374 Zero values (or ~50%) for Insulin. If we want to build and train a model we need to address these '0' values. We need to ask ourselves if the missing data values are in some way related (correlated) to each other. Let's look for correlations in our dataset." 214 | ] 215 | }, 216 | { 217 | "cell_type": "markdown", 218 | "metadata": {}, 219 | "source": [ 220 | "## Looking for Correlations\n", 221 | "\n", 222 | "Because our dataset is small we can compute the 'standard correlation coefficient' between every pair of attributes using corr()" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": null, 228 | "metadata": {}, 229 | "outputs": [], 230 | "source": [ 231 | "corrM = df.corr()\n", 232 | "corrM.style.background_gradient().format(precision=3)" 233 | ] 234 | }, 235 | { 236 | "cell_type": "markdown", 237 | "metadata": {}, 238 | "source": [ 239 | "When the coefficient is close to 1 it means there is a strong positive correlation. When we take a look at our data we don't see any attributes that are highly correleated, even our 0 value attributes Insulin & SkinThickness only have a correlation value of 0.437. Even though there is not high correleation, the zero values are still eraneous and we shouldn't include them in our model. Is this a common problem faced? And how is it solved? \n", 240 | "\n", 241 | "This is a common problem. And when dealing with missing and erraneous data, data scientists can use two primary methods to solve the error: imputation or the removal of data. Imputation develops reasonable guesses for missing data such as replacing these values with a median measurement. But we don't want to do this before we break the dataset into training and testing sets for our model. Let's leave this issue until then. In the meantime, since we are fairly happy with our Data Analysis, we can go to the next step which is model development.\n", 242 | "\n", 243 | "Open notebook 02-model-development.ipynb to continue.\n", 244 | "\n" 245 | ] 246 | } 247 | ], 248 | "metadata": { 249 | "kernelspec": { 250 | "display_name": "Python 3 (ipykernel)", 251 | "language": "python", 252 | "name": "python3" 253 | }, 254 | "language_info": { 255 | "codemirror_mode": { 256 | "name": "ipython", 257 | "version": 3 258 | }, 259 | "file_extension": ".py", 260 | "mimetype": "text/x-python", 261 | "name": "python", 262 | "nbconvert_exporter": "python", 263 | "pygments_lexer": "ipython3", 264 | "version": "3.8.8" 265 | } 266 | }, 267 | "nbformat": 4, 268 | "nbformat_minor": 4 269 | } 270 | -------------------------------------------------------------------------------- /3.create-model/pytorch-lab/notebooks/02-model-development.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Model Development\n", 8 | "\n", 9 | "We are ready to create a Diabetes model (using PyTorch) which will predict whether or not a patient has diabetes based on current medical readings. \n", 10 | "\n", 11 | "To start we will need to import our required libraries and packages. We will load our diabetes data set, create test and training sets and then start developing our model." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 8, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "# Diabetes model using PyTorch\n", 21 | "# Uses the data file: diabetes.csv\n", 22 | "import pandas as pd\n", 23 | "import torch\n", 24 | "import torch.nn as nn\n", 25 | "import torch.nn.functional as F\n", 26 | "\n", 27 | "from sklearn.model_selection import train_test_split\n", 28 | "from sklearn.metrics import accuracy_score" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "After we have imported our required libraries and packages we load the data into a dataframe (df)." 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 9, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "# This code assumes that the file is in a data folder up one dir from this file.\n", 45 | "data_file_name = '../data/diabetes.csv'\n", 46 | "model_name = '../model/PytorchDiabetesModel.pt'\n", 47 | "\n", 48 | "df = pd.read_csv(data_file_name)\n", 49 | "X = df.drop('Outcome', axis=1) # Independent Feature\n", 50 | "y = df['Outcome'] # Dependent Feature" 51 | ] 52 | }, 53 | { 54 | "cell_type": "markdown", 55 | "metadata": {}, 56 | "source": [ 57 | "Before we can train the model, we need to divide the data into 'training' and 'testing' datasets. We will use sklearn's train_test_split method to split the dataset into random train and test subsets.\n", 58 | "\n", 59 | "Once we have done this, we create tensors. Tensors are specialized data structures that are similar to arrays and matrices. In PyTorch, we use tensors to encode the inputs and outputs of a model, as well as the model's parameters. Below we are initializing the tensors directly from the data." 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": 10, 65 | "metadata": {}, 66 | "outputs": [], 67 | "source": [ 68 | "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)\n", 69 | "\n", 70 | "# Creating Tensors (multidimensional matrix) x-input data y-output data\n", 71 | "X_train = torch.FloatTensor(X_train.values)\n", 72 | "X_test = torch.FloatTensor(X_test.values)\n", 73 | "y_train = torch.LongTensor(y_train.values)\n", 74 | "y_test = torch.LongTensor(y_test.values)" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "Now we can create our model. We will need to eventually create a python file to house our model and api code. Therefore let's put our model into a class called \"ANN_model\" which we can re-use later in our Python (.py) file." 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 11, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "class ANN_model(nn.Module):\n", 91 | " def __init__(self, input_features=8, hidden1=20, hidden2=10, out_features=2):\n", 92 | " super().__init__()\n", 93 | " self.f_connected1 = nn.Linear(input_features, hidden1)\n", 94 | " self.f_connected2 = nn.Linear(hidden1, hidden2)\n", 95 | " self.out = nn.Linear(hidden2, out_features)\n", 96 | " \n", 97 | " def forward(self, x):\n", 98 | " x = F.relu(self.f_connected1(x))\n", 99 | " x = F.relu(self.f_connected2(x))\n", 100 | " x = self.out(x)\n", 101 | " return x\n", 102 | "\n", 103 | " def save(self, model_path):\n", 104 | " torch.save(model.state_dict(), model_path)\n", 105 | "\n", 106 | " def load(self, model_path):\n", 107 | " self.load_state_dict(torch.load(model_path))\n", 108 | " self.eval()\n", 109 | "\n", 110 | "\n", 111 | "torch.manual_seed(20)\n", 112 | "\n", 113 | "model = ANN_model()\n", 114 | "\n", 115 | "\n", 116 | "# Backward Propagation - loss and optimizer\n", 117 | "loss_function = nn.CrossEntropyLoss() # CrossEntropyLoss also used in Tensorflow\n", 118 | "optimizer = torch.optim.Adam(model.parameters(), lr=0.01) # Tensorflow also uses Adam\n", 119 | "\n", 120 | "epochs = 500\n", 121 | "final_losses = []\n", 122 | "for i in range(epochs):\n", 123 | " i = i+1\n", 124 | " y_pred = model.forward(X_train)\n", 125 | " loss = loss_function(y_pred, y_train)\n", 126 | " final_losses.append(loss)\n", 127 | " optimizer.zero_grad()\n", 128 | " loss.backward()\n", 129 | " optimizer.step()" 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "metadata": {}, 135 | "source": [ 136 | "Once our model is created we should test the model's accuracy. We can do this by comparing the results from the test data set." 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": 12, 142 | "metadata": {}, 143 | "outputs": [ 144 | { 145 | "name": "stdout", 146 | "output_type": "stream", 147 | "text": [ 148 | "0.7922077922077922\n" 149 | ] 150 | } 151 | ], 152 | "source": [ 153 | "# Accuracy - comparing the results from the test data\n", 154 | "\n", 155 | "predictions = []\n", 156 | "with torch.no_grad():\n", 157 | " for i, data in enumerate(X_test):\n", 158 | " y_pred = model(data)\n", 159 | " predictions.append(y_pred.argmax())\n", 160 | " \n", 161 | "score = accuracy_score(y_test, predictions) # Simply calculates number of hits / length of y_test\n", 162 | "print(score)" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "metadata": {}, 168 | "source": [ 169 | "We see that our model has an accuracy of nearly 80% This is a pretty decent score. We can improve our score by retraining the model with better data or more features or by tweaking the hyper parameters. We will see that in the next chapter." 170 | ] 171 | }, 172 | { 173 | "cell_type": "markdown", 174 | "metadata": {}, 175 | "source": [ 176 | "Now that we have built a model, let's test it with some data from 2 patients: one patient with diabetes and one patient without diabetes." 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": {}, 182 | "source": [ 183 | "To make our model testing easier, let's create a prediction function." 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": 13, 189 | "metadata": {}, 190 | "outputs": [ 191 | { 192 | "name": "stdout", 193 | "output_type": "stream", 194 | "text": [ 195 | "Diabetes Predicted\n", 196 | "No diabetes\n" 197 | ] 198 | } 199 | ], 200 | "source": [ 201 | "# Create prediction function\n", 202 | "\n", 203 | "def predict(dataset):\n", 204 | " predict_data = dataset\n", 205 | " predict_data_tensor = torch.tensor(predict_data) # Convert input array to tensor\n", 206 | " prediction_value = model(predict_data_tensor) # This is a tensor\n", 207 | "\n", 208 | " # Dict for textual display of prediction\n", 209 | " outcomes = {0: 'No diabetes', 1: 'Diabetes Predicted'}\n", 210 | "\n", 211 | " # From the prediction tensor, get the index of the max value ( Either 0 or 1)\n", 212 | " prediction_index = prediction_value.argmax().item()\n", 213 | " prediction = outcomes[prediction_index]\n", 214 | " return prediction\n", 215 | "\n", 216 | "\n", 217 | "# Test our prediction function\n", 218 | "# Pregnancies, Glucose, BloodPressure, SkinThickness, Insulin, BMI, DiabetesPedigreeFunction, Age\n", 219 | "dataset = [6.0, 110.0, 65.0, 15.0, 1.0, 45.7, 0.627, 50.0] # Has diabetes\n", 220 | "dataset_nb = [0, 88.0, 60.0, 35.0, 1.0, 45.7, 0.27, 20.0] # No diabetes\n", 221 | "\n", 222 | "diabetes_prediction = predict(dataset)\n", 223 | "print(diabetes_prediction)\n", 224 | "\n", 225 | "no_diabetes_prediction = predict(dataset_nb)\n", 226 | "print(no_diabetes_prediction)" 227 | ] 228 | }, 229 | { 230 | "cell_type": "markdown", 231 | "metadata": {}, 232 | "source": [ 233 | "Now that we are satisfied with the accuracy of the model and the predictions that it is able to make, we can save the model for using it in our applications. The following python code saves the model locally to the specified path" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": null, 239 | "metadata": {}, 240 | "outputs": [], 241 | "source": [ 242 | "# Save model\n", 243 | "# For more information on saving and loading PyTorch models: https://pytorch.org/tutorials/beginner/saving_loading_models.html\n", 244 | "# We are saving the model as a 'pt'. Another file format we could use is a Pickle file. Following article describes this process\n", 245 | "# https://machinelearningmastery.com/save-load-machine-learning-models-python-scikit-learn/\n", 246 | "\n", 247 | "model.save(model_name)" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": null, 253 | "metadata": {}, 254 | "outputs": [], 255 | "source": [] 256 | } 257 | ], 258 | "metadata": { 259 | "kernelspec": { 260 | "display_name": "Python 3.9", 261 | "language": "python", 262 | "name": "python3" 263 | }, 264 | "language_info": { 265 | "codemirror_mode": { 266 | "name": "ipython", 267 | "version": 3 268 | }, 269 | "file_extension": ".py", 270 | "mimetype": "text/x-python", 271 | "name": "python", 272 | "nbconvert_exporter": "python", 273 | "pygments_lexer": "ipython3", 274 | "version": "3.9.16" 275 | } 276 | }, 277 | "nbformat": 4, 278 | "nbformat_minor": 4 279 | } 280 | -------------------------------------------------------------------------------- /4.rhods-deploy/chapter1/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.24.4 2 | matplotlib==3.6.3 3 | scikit-learn==1.2.2 -------------------------------------------------------------------------------- /4.rhods-deploy/chapter1/use-purchase-amount.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "44b098ec-838d-49c9-bdbe-1192e99d908e", 7 | "metadata": {}, 8 | "outputs": [ 9 | { 10 | "name": "stdout", 11 | "output_type": "stream", 12 | "text": [ 13 | "22.879625918124702\n" 14 | ] 15 | } 16 | ], 17 | "source": [ 18 | "# Include required packages for execution\n", 19 | "import pickle\n", 20 | "\n", 21 | "# Load models\n", 22 | "with open('mymodel.pkl', 'rb') as f:\n", 23 | " model = pickle.load(f)\n", 24 | "\n", 25 | "# perform a prediction using the model\n", 26 | "print(model(5))" 27 | ] 28 | } 29 | ], 30 | "metadata": { 31 | "kernelspec": { 32 | "display_name": "Python 3.9", 33 | "language": "python", 34 | "name": "python3" 35 | }, 36 | "language_info": { 37 | "codemirror_mode": { 38 | "name": "ipython", 39 | "version": 3 40 | }, 41 | "file_extension": ".py", 42 | "mimetype": "text/x-python", 43 | "name": "python", 44 | "nbconvert_exporter": "python", 45 | "pygments_lexer": "ipython3", 46 | "version": "3.9.16" 47 | } 48 | }, 49 | "nbformat": 4, 50 | "nbformat_minor": 5 51 | } 52 | -------------------------------------------------------------------------------- /4.rhods-deploy/chapter2/iris_to_onnx.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "51c38d9f-53b8-4bd7-9785-15a88f427e75", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import numpy as np\n", 11 | "from sklearn.datasets import load_iris\n", 12 | "from sklearn.model_selection import train_test_split\n", 13 | "from skl2onnx import to_onnx\n", 14 | "from sklearn.cluster import KMeans\n", 15 | "\n", 16 | "iris = load_iris()\n", 17 | "X, y = iris.data, iris.target\n", 18 | "X = X.astype(np.float32)\n", 19 | "X_train, X_test, y_train, y_test = train_test_split(X, y)\n", 20 | "\n", 21 | "kmeans = KMeans(n_clusters=3, init='k-means++', max_iter=300, n_init=10,\n", 22 | " random_state=0)\n", 23 | "y_kmeans = kmeans.fit_predict(X_train, y_train)\n", 24 | "\n", 25 | "# Convert into ONNX format.\n", 26 | "onx = to_onnx(kmeans, X[:1])\n", 27 | "with open(\"rf_iris.onnx\", \"wb\") as f:\n", 28 | " f.write(onx.SerializeToString())" 29 | ] 30 | } 31 | ], 32 | "metadata": { 33 | "kernelspec": { 34 | "display_name": "Python 3.9", 35 | "language": "python", 36 | "name": "python3" 37 | }, 38 | "language_info": { 39 | "codemirror_mode": { 40 | "name": "ipython", 41 | "version": 3 42 | }, 43 | "file_extension": ".py", 44 | "mimetype": "text/x-python", 45 | "name": "python", 46 | "nbconvert_exporter": "python", 47 | "pygments_lexer": "ipython3", 48 | "version": "3.9.16" 49 | } 50 | }, 51 | "nbformat": 4, 52 | "nbformat_minor": 5 53 | } 54 | -------------------------------------------------------------------------------- /4.rhods-deploy/chapter2/minio.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: PersistentVolumeClaim 3 | apiVersion: v1 4 | metadata: 5 | name: minio-pvc 6 | spec: 7 | accessModes: 8 | - ReadWriteOnce 9 | resources: 10 | requests: 11 | storage: 20Gi 12 | volumeMode: Filesystem 13 | --- 14 | kind: Secret 15 | apiVersion: v1 16 | metadata: 17 | name: minio-secret 18 | stringData: 19 | minio_root_user: minio 20 | minio_root_password: minio123 21 | --- 22 | kind: Deployment 23 | apiVersion: apps/v1 24 | metadata: 25 | name: minio 26 | spec: 27 | replicas: 1 28 | selector: 29 | matchLabels: 30 | app: minio 31 | template: 32 | metadata: 33 | labels: 34 | app: minio 35 | spec: 36 | volumes: 37 | - name: data 38 | persistentVolumeClaim: 39 | claimName: minio-pvc 40 | containers: 41 | - resources: 42 | limits: 43 | cpu: 250m 44 | memory: 1Gi 45 | requests: 46 | cpu: 20m 47 | memory: 100Mi 48 | readinessProbe: 49 | tcpSocket: 50 | port: 9000 51 | initialDelaySeconds: 5 52 | timeoutSeconds: 1 53 | periodSeconds: 5 54 | successThreshold: 1 55 | failureThreshold: 3 56 | terminationMessagePath: /dev/termination-log 57 | name: minio 58 | livenessProbe: 59 | tcpSocket: 60 | port: 9000 61 | initialDelaySeconds: 30 62 | timeoutSeconds: 1 63 | periodSeconds: 5 64 | successThreshold: 1 65 | failureThreshold: 3 66 | env: 67 | - name: MINIO_ROOT_USER 68 | valueFrom: 69 | secretKeyRef: 70 | name: minio-secret 71 | key: minio_root_user 72 | - name: MINIO_ROOT_PASSWORD 73 | valueFrom: 74 | secretKeyRef: 75 | name: minio-secret 76 | key: minio_root_password 77 | ports: 78 | - containerPort: 9000 79 | protocol: TCP 80 | - containerPort: 9090 81 | protocol: TCP 82 | imagePullPolicy: IfNotPresent 83 | volumeMounts: 84 | - name: data 85 | mountPath: /data 86 | subPath: minio 87 | terminationMessagePolicy: File 88 | image: >- 89 | quay.io/minio/minio:RELEASE.2023-06-19T19-52-50Z 90 | args: 91 | - server 92 | - /data 93 | - --console-address 94 | - :9090 95 | restartPolicy: Always 96 | terminationGracePeriodSeconds: 30 97 | dnsPolicy: ClusterFirst 98 | securityContext: {} 99 | schedulerName: default-scheduler 100 | strategy: 101 | type: Recreate 102 | revisionHistoryLimit: 10 103 | progressDeadlineSeconds: 600 104 | --- 105 | kind: Service 106 | apiVersion: v1 107 | metadata: 108 | name: minio-service 109 | spec: 110 | ipFamilies: 111 | - IPv4 112 | ports: 113 | - name: api 114 | protocol: TCP 115 | port: 9000 116 | targetPort: 9000 117 | - name: ui 118 | protocol: TCP 119 | port: 9090 120 | targetPort: 9090 121 | internalTrafficPolicy: Cluster 122 | type: ClusterIP 123 | ipFamilyPolicy: SingleStack 124 | sessionAffinity: None 125 | selector: 126 | app: minio 127 | --- 128 | kind: Route 129 | apiVersion: route.openshift.io/v1 130 | metadata: 131 | name: minio-api 132 | spec: 133 | to: 134 | kind: Service 135 | name: minio-service 136 | weight: 100 137 | port: 138 | targetPort: api 139 | wildcardPolicy: None 140 | tls: 141 | termination: edge 142 | insecureEdgeTerminationPolicy: Redirect 143 | --- 144 | kind: Route 145 | apiVersion: route.openshift.io/v1 146 | metadata: 147 | name: minio-ui 148 | spec: 149 | to: 150 | kind: Service 151 | name: minio-service 152 | weight: 100 153 | port: 154 | targetPort: ui 155 | wildcardPolicy: None 156 | tls: 157 | termination: edge 158 | insecureEdgeTerminationPolicy: Redirect -------------------------------------------------------------------------------- /4.rhods-deploy/chapter2/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.24.4 2 | matplotlib==3.6.3 3 | scikit-learn==1.2.2 4 | skl2onnx==1.15.0 -------------------------------------------------------------------------------- /4.rhods-deploy/custom-runtime-example/README.adoc: -------------------------------------------------------------------------------- 1 | [NOTE] 2 | ==== 3 | This example is supplemental material and is not yet included in the https://redhatquickcourses.github.io/rhods-deploy[RHOAI models deployment course]. 4 | ==== -------------------------------------------------------------------------------- /4.rhods-deploy/custom-runtime-example/example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# MLServer Runtime Demo\n", 8 | "\n", 9 | "In this demo, you train a simple model to recognize hand-written digits and serve the model with the [Seldon MLServer runtime](https://github.com/SeldonIO/MLServer).\n", 10 | "\n", 11 | "Train the model with the Scikit-learn library." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "id": "27e01bfb-c9e0-4f06-8134-563b50dde8c0", 18 | "metadata": {}, 19 | "outputs": [ 20 | { 21 | "data": { 22 | "text/html": [ 23 | "
SVC(gamma=0.001)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" 24 | ], 25 | "text/plain": [ 26 | "SVC(gamma=0.001)" 27 | ] 28 | }, 29 | "execution_count": 1, 30 | "metadata": {}, 31 | "output_type": "execute_result" 32 | } 33 | ], 34 | "source": [ 35 | "# Original source code and more details can be found in:\n", 36 | "# https://scikit-learn.org/stable/auto_examples/classification/plot_digits_classification.html\n", 37 | "\n", 38 | "# Import datasets, classifiers and performance metrics\n", 39 | "from sklearn import datasets, svm, metrics\n", 40 | "from sklearn.model_selection import train_test_split\n", 41 | "\n", 42 | "# The digits dataset\n", 43 | "digits = datasets.load_digits()\n", 44 | "\n", 45 | "# To apply a classifier on this data, we need to flatten the image, to\n", 46 | "# turn the data in a (samples, feature) matrix:\n", 47 | "n_samples = len(digits.images)\n", 48 | "data = digits.images.reshape((n_samples, -1))\n", 49 | "\n", 50 | "# Create a classifier: a support vector classifier\n", 51 | "classifier = svm.SVC(gamma=0.001)\n", 52 | "\n", 53 | "# Split data into train and test subsets\n", 54 | "X_train, X_test, y_train, y_test = train_test_split(\n", 55 | " data, digits.target, test_size=0.5, shuffle=False)\n", 56 | "\n", 57 | "# We learn the digits on the first half of the digits\n", 58 | "classifier.fit(X_train, y_train)" 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "metadata": {}, 64 | "source": [ 65 | "\n", 66 | "Export the model with the Joblib library.\n", 67 | "Joblib is a popular option for model persistence in Scikit-learn.\n", 68 | "\n", 69 | "Note that exporting the model with Joblib serializes the model into a file format different from ONNX.\n", 70 | "OpenVino does not support the joblib format, so you must serve this model with a different runtime." 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 2, 76 | "id": "ee30440a-2491-4852-b7b7-f8922c3bccff", 77 | "metadata": {}, 78 | "outputs": [ 79 | { 80 | "data": { 81 | "text/plain": [ 82 | "['mnist-svm.joblib']" 83 | ] 84 | }, 85 | "execution_count": 2, 86 | "metadata": {}, 87 | "output_type": "execute_result" 88 | } 89 | ], 90 | "source": [ 91 | "import joblib\n", 92 | "\n", 93 | "model_file_name = \"mnist-svm.joblib\"\n", 94 | "joblib.dump(classifier, model_file_name)" 95 | ] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "metadata": {}, 100 | "source": [ 101 | "In the RHODS dashboard, add the MLServer runtime in the `Settings > Serving runtimes` section.\n", 102 | "Add the contents of the `ml-server-runtime.yaml` manifest to add the runtime.\n", 103 | "\n", 104 | "After you have added the MLServer runtime, return to your data science project.\n", 105 | "Add a new `MLServer` model server by selecting the `MLServer` serving runtime in the model server creation form.\n", 106 | "\n", 107 | "Next, upload the joblib model file to your S3 instance and create a data connection to connect to that instance.\n", 108 | "\n", 109 | "Finally, deploy the model by using the `sklearn` model framework and your S3 data connection.\n", 110 | "\n", 111 | "Use the following cells to test your deployed model:" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": 35, 117 | "id": "605b6844-f1d1-4c94-8175-381465579e3b", 118 | "metadata": {}, 119 | "outputs": [ 120 | { 121 | "data": { 122 | "text/plain": [ 123 | "" 124 | ] 125 | }, 126 | "execution_count": 35, 127 | "metadata": {}, 128 | "output_type": "execute_result" 129 | }, 130 | { 131 | "data": { 132 | "text/plain": [ 133 | "
" 134 | ] 135 | }, 136 | "metadata": {}, 137 | "output_type": "display_data" 138 | }, 139 | { 140 | "data": { 141 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAGkCAYAAAAIduO+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAY6klEQVR4nO3df2zUhf3H8dfRrgdqe/yQQjtKi4oiP3oiBcKqQwUxDRLZH4wQzAq4JZJjUBoT038GyzKO/TGD20gFxloTx2BbVnAm0AGTkkU6SrELaIKgCEWEzkXu2i45TO/z/eObdXZA28/RNx8+1+cj+WTe8TnuFUZ48rkevYDjOI4AADAyxOsBAID0RmgAAKYIDQDAFKEBAJgiNAAAU4QGAGCK0AAATBEaAIApQgMAMEVoAACm0iY0W7duVVFRkYYOHarZs2fr+PHjXk/q09GjR7Vo0SLl5+crEAho7969Xk/ql2g0qpkzZyo7O1u5ublavHixzpw54/WsfqmurlZxcbFycnKUk5OjOXPmaP/+/V7Pcm3z5s0KBAKqqKjwekqfNm7cqEAg0OOYNGmS17P65bPPPtOLL76oUaNGadiwYZo2bZpOnDjh9aw+FRUV3fBrHggEFIlEPNmTFqHZs2ePKisrtWHDBp08eVLhcFjPPfec2travJ7Wq87OToXDYW3dutXrKa40NDQoEomosbFRBw8e1FdffaUFCxaos7PT62l9GjdunDZv3qzm5madOHFCzzzzjF544QV98MEHXk/rt6amJm3btk3FxcVeT+m3KVOm6PPPP+8+/va3v3k9qU9ffvmlSktL9Y1vfEP79+/Xhx9+qJ///OcaMWKE19P61NTU1OPX++DBg5KkJUuWeDPISQOzZs1yIpFI9+2uri4nPz/fiUajHq5yR5JTV1fn9YyUtLW1OZKchoYGr6ekZMSIEc6vf/1rr2f0S3t7uzNx4kTn4MGDzty5c51169Z5PalPGzZscMLhsNczXHv11VedJ554wusZA2LdunXOgw8+6CSTSU+e3/dXNNevX1dzc7Pmz5/ffd+QIUM0f/58HTt2zMNlg0csFpMkjRw50uMl7nR1dWn37t3q7OzUnDlzvJ7TL5FIRAsXLuzx+90Pzp49q/z8fD3wwANavny5Ll686PWkPr399tsqKSnRkiVLlJubq+nTp2vHjh1ez3Lt+vXreuutt7Rq1SoFAgFPNvg+NF988YW6uro0ZsyYHvePGTNGV65c8WjV4JFMJlVRUaHS0lJNnTrV6zn9curUKd13330KBoN6+eWXVVdXp8mTJ3s9q0+7d+/WyZMnFY1GvZ7iyuzZs1VbW6sDBw6ourpa58+f15NPPqn29navp/Xqk08+UXV1tSZOnKj6+nqtXr1aa9eu1Ztvvun1NFf27t2ra9euacWKFZ5tyPTsmZEWIpGITp8+7YvX3P/jkUceUUtLi2KxmP74xz+qvLxcDQ0Nd3VsWltbtW7dOh08eFBDhw71eo4rZWVl3f9dXFys2bNnq7CwUL///e/10ksvebisd8lkUiUlJdq0aZMkafr06Tp9+rTeeOMNlZeXe7yu/3bu3KmysjLl5+d7tsH3VzT333+/MjIydPXq1R73X716VWPHjvVo1eCwZs0avfPOO3r33Xc1btw4r+f0W1ZWlh566CHNmDFD0WhU4XBYr7/+utezetXc3Ky2tjY9/vjjyszMVGZmphoaGvSLX/xCmZmZ6urq8npivw0fPlwPP/ywzp075/WUXuXl5d3wl49HH33UFy/7/ceFCxd06NAhff/73/d0h+9Dk5WVpRkzZujw4cPd9yWTSR0+fNg3r7v7jeM4WrNmjerq6vTXv/5VEyZM8HrSbUkmk0okEl7P6NW8efN06tQptbS0dB8lJSVavny5WlpalJGR4fXEfuvo6NDHH3+svLw8r6f0qrS09Ia37X/00UcqLCz0aJF7NTU1ys3N1cKFCz3dkRYvnVVWVqq8vFwlJSWaNWuWtmzZos7OTq1cudLrab3q6Ojo8be68+fPq6WlRSNHjtT48eM9XNa7SCSiXbt2ad++fcrOzu7+WlgoFNKwYcM8Xte7qqoqlZWVafz48Wpvb9euXbt05MgR1dfXez2tV9nZ2Td8Dezee+/VqFGj7vqvjb3yyitatGiRCgsLdfnyZW3YsEEZGRlatmyZ19N6tX79en3rW9/Spk2b9N3vflfHjx/X9u3btX37dq+n9UsymVRNTY3Ky8uVmenxH/WevNfNwC9/+Utn/PjxTlZWljNr1iynsbHR60l9evfddx1JNxzl5eVeT+vVzTZLcmpqarye1qdVq1Y5hYWFTlZWljN69Ghn3rx5zl/+8hevZ6XEL29vXrp0qZOXl+dkZWU53/zmN52lS5c6586d83pWv/z5z392pk6d6gSDQWfSpEnO9u3bvZ7Ub/X19Y4k58yZM15PcQKO4zjeJA4AMBj4/ms0AIC7G6EBAJgiNAAAU4QGAGCK0AAATBEaAICptApNIpHQxo0b7/p/5f2//Lpb8u92v+6W/Lvdr7sl/26/W3an1b+jicfjCoVCisViysnJ8XpOv/l1t+Tf7X7dLfl3u193S/7dfrfsTqsrGgDA3YfQAABM3fHvtJZMJnX58mVlZ2cP+Ke9xePxHv/rF37dLfl3u193S/7d7tfdkn+3W+92HEft7e3Kz8/XkCG3vm6541+juXTpkgoKCu7kUwIADLW2tvb6mVR3/IomOzv7Tj8l9P/fwt+PqqurvZ6QsmnTpnk9ISV+/b0iyfPPXUnVqVOnvJ5wW/r6c/2Oh2agXy5D//j11/2ee+7xekLK/PqXKj+9q+p/+ekD4NJJX3++8GYAAIApQgMAMEVoAACmCA0AwBShAQCYIjQAAFOEBgBgitAAAEwRGgCAKUIDADBFaAAApggNAMAUoQEAmCI0AABThAYAYIrQAABMpRSarVu3qqioSEOHDtXs2bN1/Pjxgd4FAEgTrkOzZ88eVVZWasOGDTp58qTC4bCee+45tbW1WewDAPic69C89tpr+sEPfqCVK1dq8uTJeuONN3TPPffoN7/5jcU+AIDPuQrN9evX1dzcrPnz5//3JxgyRPPnz9exY8du+phEIqF4PN7jAAAMHq5C88UXX6irq0tjxozpcf+YMWN05cqVmz4mGo0qFAp1HwUFBamvBQD4jvm7zqqqqhSLxbqP1tZW66cEANxFMt2cfP/99ysjI0NXr17tcf/Vq1c1duzYmz4mGAwqGAymvhAA4GuurmiysrI0Y8YMHT58uPu+ZDKpw4cPa86cOQM+DgDgf66uaCSpsrJS5eXlKikp0axZs7RlyxZ1dnZq5cqVFvsAAD7nOjRLly7VP//5T/3oRz/SlStX9Nhjj+nAgQM3vEEAAAAphdBI0po1a7RmzZqB3gIASEN8rzMAgClCAwAwRWgAAKYIDQDAFKEBAJgiNAAAU4QGAGCK0AAATBEaAIApQgMAMEVoAACmCA0AwBShAQCYIjQAAFOEBgBgitAAAEyl9MFng9Xw4cO9npCyI0eOeD0hJeFw2OsJKWtoaPB6Qkrmzp3r9YSULV682OsJKWlpafF6gimuaAAApggNAMAUoQEAmCI0AABThAYAYIrQAABMERoAgClCAwAwRWgAAKYIDQDAFKEBAJgiNAAAU4QGAGCK0AAATBEaAIApQgMAMEVoAACmCA0AwBShAQCYch2ao0ePatGiRcrPz1cgENDevXsNZgEA0oXr0HR2diocDmvr1q0WewAAaSbT7QPKyspUVlZmsQUAkIZch8atRCKhRCLRfTsej1s/JQDgLmL+ZoBoNKpQKNR9FBQUWD8lAOAuYh6aqqoqxWKx7qO1tdX6KQEAdxHzl86CwaCCwaD10wAA7lL8OxoAgCnXVzQdHR06d+5c9+3z58+rpaVFI0eO1Pjx4wd0HADA/1yH5sSJE3r66ae7b1dWVkqSysvLVVtbO2DDAADpwXVonnrqKTmOY7EFAJCG+BoNAMAUoQEAmCI0AABThAYAYIrQAABMERoAgClCAwAwRWgAAKYIDQDAFKEBAJgiNAAAU4QGAGCK0AAATBEaAIApQgMAMEVoAACmXH/w2WBWUVHh9YSUhcNhryek5Ouf5uo3RUVFXk9Iydy5c72ekLL333/f6wm4Ca5oAACmCA0AwBShAQCYIjQAAFOEBgBgitAAAEwRGgCAKUIDADBFaAAApggNAMAUoQEAmCI0AABThAYAYIrQAABMERoAgClCAwAwRWgAAKYIDQDAlKvQRKNRzZw5U9nZ2crNzdXixYt15swZq20AgDTgKjQNDQ2KRCJqbGzUwYMH9dVXX2nBggXq7Oy02gcA8LlMNycfOHCgx+3a2lrl5uaqublZ3/72twd0GAAgPbgKzf+KxWKSpJEjR97ynEQioUQi0X07Ho/fzlMCAHwm5TcDJJNJVVRUqLS0VFOnTr3ledFoVKFQqPsoKChI9SkBAD6UcmgikYhOnz6t3bt393peVVWVYrFY99Ha2prqUwIAfCill87WrFmjd955R0ePHtW4ceN6PTcYDCoYDKY0DgDgf65C4ziOfvjDH6qurk5HjhzRhAkTrHYBANKEq9BEIhHt2rVL+/btU3Z2tq5cuSJJCoVCGjZsmMlAAIC/ufoaTXV1tWKxmJ566inl5eV1H3v27LHaBwDwOdcvnQEA4Abf6wwAYIrQAABMERoAgClCAwAwRWgAAKYIDQDAFKEBAJgiNAAAU4QGAGCK0AAATBEaAIApQgMAMEVoAACmCA0AwBShAQCYIjQAAFOuPvhssHv//fe9npCyWCzm9YSUVFRUeD0hZUVFRV5PSMmFCxe8npCyffv2eT0BN8EVDQDAFKEBAJgiNAAAU4QGAGCK0AAATBEaAIApQgMAMEVoAACmCA0AwBShAQCYIjQAAFOEBgBgitAAAEwRGgCAKUIDADBFaAAApggNAMAUoQEAmCI0AABTrkJTXV2t4uJi5eTkKCcnR3PmzNH+/futtgEA0oCr0IwbN06bN29Wc3OzTpw4oWeeeUYvvPCCPvjgA6t9AACfy3Rz8qJFi3rc/ulPf6rq6mo1NjZqypQpAzoMAJAeXIXm67q6uvSHP/xBnZ2dmjNnzi3PSyQSSiQS3bfj8XiqTwkA8CHXbwY4deqU7rvvPgWDQb388suqq6vT5MmTb3l+NBpVKBTqPgoKCm5rMADAX1yH5pFHHlFLS4v+/ve/a/Xq1SovL9eHH354y/OrqqoUi8W6j9bW1tsaDADwF9cvnWVlZemhhx6SJM2YMUNNTU16/fXXtW3btpueHwwGFQwGb28lAMC3bvvf0SSTyR5fgwEA4OtcXdFUVVWprKxM48ePV3t7u3bt2qUjR46ovr7eah8AwOdchaatrU3f+9739PnnnysUCqm4uFj19fV69tlnrfYBAHzOVWh27txptQMAkKb4XmcAAFOEBgBgitAAAEwRGgCAKUIDADBFaAAApggNAMAUoQEAmCI0AABThAYAYIrQAABMERoAgClCAwAwRWgAAKYIDQDAFKEBAJhy9cFng92+ffu8npCyxx57zOsJKamtrfV6QsrC4bDXE1Lyj3/8w+sJSDNc0QAATBEaAIApQgMAMEVoAACmCA0AwBShAQCYIjQAAFOEBgBgitAAAEwRGgCAKUIDADBFaAAApggNAMAUoQEAmCI0AABThAYAYIrQAABMERoAgKnbCs3mzZsVCARUUVExQHMAAOkm5dA0NTVp27ZtKi4uHsg9AIA0k1JoOjo6tHz5cu3YsUMjRowY6E0AgDSSUmgikYgWLlyo+fPn93luIpFQPB7vcQAABo9Mtw/YvXu3Tp48qaampn6dH41G9eMf/9j1MABAenB1RdPa2qp169bpt7/9rYYOHdqvx1RVVSkWi3Ufra2tKQ0FAPiTqyua5uZmtbW16fHHH+++r6urS0ePHtWvfvUrJRIJZWRk9HhMMBhUMBgcmLUAAN9xFZp58+bp1KlTPe5buXKlJk2apFdfffWGyAAA4Co02dnZmjp1ao/77r33Xo0aNeqG+wEAkPjOAAAAY67fdfa/jhw5MgAzAADpiisaAIApQgMAMEVoAACmCA0AwBShAQCYIjQAAFOEBgBgitAAAEwRGgCAKUIDADBFaAAApggNAMAUoQEAmCI0AABThAYAYIrQAABM3fYHn8EfPv30U68npGT48OFeTxh0wuGw1xNStmLFCq8npKS2ttbrCaa4ogEAmCI0AABThAYAYIrQAABMERoAgClCAwAwRWgAAKYIDQDAFKEBAJgiNAAAU4QGAGCK0AAATBEaAIApQgMAMEVoAACmCA0AwBShAQCYIjQAAFOEBgBgylVoNm7cqEAg0OOYNGmS1TYAQBrIdPuAKVOm6NChQ//9CTJd/xQAgEHEdSUyMzM1duxYiy0AgDTk+ms0Z8+eVX5+vh544AEtX75cFy9e7PX8RCKheDze4wAADB6uQjN79mzV1tbqwIEDqq6u1vnz5/Xkk0+qvb39lo+JRqMKhULdR0FBwW2PBgD4R8BxHCfVB1+7dk2FhYV67bXX9NJLL930nEQioUQi0X07Ho8TG/RbS0uL1xNSFg6HvZ4w6KxcudLrCSmpra31esJticViysnJueWP39ZX8ocPH66HH35Y586du+U5wWBQwWDwdp4GAOBjt/XvaDo6OvTxxx8rLy9voPYAANKMq9C88soramho0Keffqr33ntP3/nOd5SRkaFly5ZZ7QMA+Jyrl84uXbqkZcuW6V//+pdGjx6tJ554Qo2NjRo9erTVPgCAz7kKze7du612AADSFN/rDABgitAAAEwRGgCAKUIDADBFaAAApggNAMAUoQEAmCI0AABThAYAYIrQAABMERoAgClCAwAwRWgAAKYIDQDAFKEBAJgiNAAAU64++Ay408LhsNcT4CPDhw/3egJugisaAIApQgMAMEVoAACmCA0AwBShAQCYIjQAAFOEBgBgitAAAEwRGgCAKUIDADBFaAAApggNAMAUoQEAmCI0AABThAYAYIrQAABMERoAgClCAwAw5To0n332mV588UWNGjVKw4YN07Rp03TixAmLbQCANJDp5uQvv/xSpaWlevrpp7V//36NHj1aZ8+e1YgRI6z2AQB8zlVofvazn6mgoEA1NTXd902YMGHARwEA0oerl87efvttlZSUaMmSJcrNzdX06dO1Y8eOXh+TSCQUj8d7HACAwcNVaD755BNVV1dr4sSJqq+v1+rVq7V27Vq9+eabt3xMNBpVKBTqPgoKCm57NADAPwKO4zj9PTkrK0slJSV67733uu9bu3atmpqadOzYsZs+JpFIKJFIdN+Ox+PEBv3m4rcnoPXr13s9ISVbtmzxesJticViysnJueWPu7qiycvL0+TJk3vc9+ijj+rixYu3fEwwGFROTk6PAwAweLgKTWlpqc6cOdPjvo8++kiFhYUDOgoAkD5chWb9+vVqbGzUpk2bdO7cOe3atUvbt29XJBKx2gcA8DlXoZk5c6bq6ur0u9/9TlOnTtVPfvITbdmyRcuXL7faBwDwOVf/jkaSnn/+eT3//PMWWwAAaYjvdQYAMEVoAACmCA0AwBShAQCYIjQAAFOEBgBgitAAAEwRGgCAKUIDADBFaAAApggNAMAUoQEAmCI0AABThAYAYIrQAABMERoAgCnXH3wG3En79u3zekLKioqKvJ6QkmvXrnk9IWW1tbVeT8BNcEUDADBFaAAApggNAMAUoQEAmCI0AABThAYAYIrQAABMERoAgClCAwAwRWgAAKYIDQDAFKEBAJgiNAAAU4QGAGCK0AAATBEaAIApQgMAMEVoAACmCA0AwJSr0BQVFSkQCNxwRCIRq30AAJ/LdHNyU1OTurq6um+fPn1azz77rJYsWTLgwwAA6cFVaEaPHt3j9ubNm/Xggw9q7ty5AzoKAJA+XIXm665fv6633npLlZWVCgQCtzwvkUgokUh0347H46k+JQDAh1J+M8DevXt17do1rVixotfzotGoQqFQ91FQUJDqUwIAfCjl0OzcuVNlZWXKz8/v9byqqirFYrHuo7W1NdWnBAD4UEovnV24cEGHDh3Sn/70pz7PDQaDCgaDqTwNACANpHRFU1NTo9zcXC1cuHCg9wAA0ozr0CSTSdXU1Ki8vFyZmSm/lwAAMEi4Ds2hQ4d08eJFrVq1ymIPACDNuL4kWbBggRzHsdgCAEhDfK8zAIApQgMAMEVoAACmCA0AwBShAQCYIjQAAFOEBgBgitAAAEwRGgCAKUIDADBFaAAApggNAMAUoQEAmCI0AABThAYAYOqOf0Qmn2UDN/797397PSFlHR0dXk9ISWdnp9cTUsafL97o69c94Nzh/2cuXbqkgoKCO/mUAABDra2tGjdu3C1//I6HJplM6vLly8rOzlYgEBjQnzsej6ugoECtra3KyckZ0J/bkl93S/7d7tfdkn+3+3W35N/t1rsdx1F7e7vy8/M1ZMitvxJzx186GzJkSK/lGwg5OTm++s3wH37dLfl3u193S/7d7tfdkn+3W+4OhUJ9nsObAQAApggNAMBUWoUmGAxqw4YNCgaDXk9xxa+7Jf9u9+tuyb/b/bpb8u/2u2X3HX8zAABgcEmrKxoAwN2H0AAATBEaAIApQgMAMEVoAACmCA0AwBShAQCYIjQAAFP/B9G3S9t6/nf8AAAAAElFTkSuQmCC", 142 | "text/plain": [ 143 | "
" 144 | ] 145 | }, 146 | "metadata": {}, 147 | "output_type": "display_data" 148 | } 149 | ], 150 | "source": [ 151 | "import matplotlib.pyplot as plt\n", 152 | "\n", 153 | "x_0 = X_test[0:1]\n", 154 | "\n", 155 | "plt.gray()\n", 156 | "plt.matshow(x_0.reshape((8, 8)))" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": 32, 162 | "id": "3758fb97-6dc7-457e-9dd5-d33cbe789540", 163 | "metadata": {}, 164 | "outputs": [ 165 | { 166 | "data": { 167 | "text/plain": [ 168 | "{'model_name': 'digits-model__isvc-16547a49a0',\n", 169 | " 'outputs': [{'name': 'predict',\n", 170 | " 'datatype': 'INT64',\n", 171 | " 'shape': [1, 1],\n", 172 | " 'parameters': {'content_type': 'np'},\n", 173 | " 'data': [8]}]}" 174 | ] 175 | }, 176 | "execution_count": 32, 177 | "metadata": {}, 178 | "output_type": "execute_result" 179 | } 180 | ], 181 | "source": [ 182 | "import requests\n", 183 | "\n", 184 | "inference_request = {\n", 185 | " \"inputs\": [\n", 186 | " {\n", 187 | " \"name\": \"predict\",\n", 188 | " \"shape\": x_0.shape,\n", 189 | " \"datatype\": \"FP32\",\n", 190 | " \"data\": x_0.tolist()\n", 191 | " }\n", 192 | " ]\n", 193 | "}\n", 194 | "\n", 195 | "endpoint = \"https://YOUR-MODEL-INFENRENCE-URL\"\n", 196 | "response = requests.post(endpoint, json=inference_request)\n", 197 | "\n", 198 | "result = response.json()\n", 199 | "result" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": 33, 205 | "id": "4a39e9dc-5247-499e-97c1-1f3a44bda970", 206 | "metadata": {}, 207 | "outputs": [ 208 | { 209 | "name": "stdout", 210 | "output_type": "stream", 211 | "text": [ 212 | "\n", 213 | "Predicted digit: 8\n" 214 | ] 215 | } 216 | ], 217 | "source": [ 218 | "print(\"\\nPredicted digit:\", result[\"outputs\"][0][\"data\"][0])" 219 | ] 220 | } 221 | ], 222 | "metadata": { 223 | "kernelspec": { 224 | "display_name": "Python 3.9", 225 | "language": "python", 226 | "name": "python3" 227 | }, 228 | "language_info": { 229 | "codemirror_mode": { 230 | "name": "ipython", 231 | "version": 3 232 | }, 233 | "file_extension": ".py", 234 | "mimetype": "text/x-python", 235 | "name": "python", 236 | "nbconvert_exporter": "python", 237 | "pygments_lexer": "ipython3", 238 | "version": "3.9.16" 239 | } 240 | }, 241 | "nbformat": 4, 242 | "nbformat_minor": 5 243 | } 244 | -------------------------------------------------------------------------------- /4.rhods-deploy/custom-runtime-example/ml-server-runtime.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: serving.kserve.io/v1alpha1 2 | kind: ServingRuntime 3 | metadata: 4 | name: mlserver-1.x 5 | annotations: 6 | openshift.io/display-name: MLServer 7 | labels: 8 | name: modelmesh-serving-mlserver-1.x-SR 9 | spec: 10 | supportedModelFormats: 11 | - name: sklearn 12 | version: "0" 13 | autoSelect: true 14 | - name: xgboost 15 | version: "1" 16 | autoSelect: true 17 | - name: lightgbm 18 | version: "3" 19 | autoSelect: true 20 | protocolVersions: 21 | - grpc-v2 22 | multiModel: true 23 | grpcEndpoint: port:8085 24 | grpcDataEndpoint: port:8001 25 | containers: 26 | - name: mlserver 27 | image: quay.io/jramcast/seldonio-mlserver:1.3.5 28 | env: 29 | - name: MLSERVER_MODELS_DIR 30 | value: /models/_mlserver_models/ 31 | - name: MLSERVER_GRPC_PORT 32 | value: "8001" 33 | - name: MLSERVER_HTTP_PORT 34 | value: "8002" 35 | - name: MLSERVER_LOAD_MODELS_AT_STARTUP 36 | value: "false" 37 | - name: MLSERVER_MODEL_NAME 38 | value: dummy-model-fixme 39 | - name: MLSERVER_HOST 40 | value: 127.0.0.1 41 | - name: MLSERVER_GRPC_MAX_MESSAGE_LENGTH 42 | value: "-1" 43 | resources: 44 | requests: 45 | cpu: 500m 46 | memory: 1Gi 47 | limits: 48 | cpu: "5" 49 | memory: 1Gi 50 | builtInAdapter: 51 | serverType: mlserver 52 | runtimeManagementPort: 8001 53 | memBufferBytes: 134217728 54 | modelLoadingTimeoutMillis: 90000 55 | -------------------------------------------------------------------------------- /5.pipelines/elyra/data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RedHatQuickCourses/rhods-qc-apps/60776f6fadc901c0ede9c0eab9a7d6c7198f4ba1/5.pipelines/elyra/data/.gitkeep -------------------------------------------------------------------------------- /5.pipelines/elyra/data_ingestion.py: -------------------------------------------------------------------------------- 1 | from os import environ 2 | 3 | import boto3 4 | 5 | 6 | def ingest_data(data_object_name='', data_folder='./data'): 7 | print('Commencing data ingestion.') 8 | 9 | s3_endpoint_url = environ.get('AWS_S3_ENDPOINT') 10 | s3_access_key = environ.get('AWS_ACCESS_KEY_ID') 11 | s3_secret_key = environ.get('AWS_SECRET_ACCESS_KEY') 12 | s3_bucket_name = environ.get('AWS_S3_BUCKET') 13 | data_object_name = data_object_name or environ.get( 14 | 'data_object_name', 'live-data.csv' 15 | ) 16 | 17 | print(f'Downloading data "{data_object_name}" ' 18 | f'from bucket "{s3_bucket_name}" ' 19 | f'from S3 storage at {s3_endpoint_url}') 20 | 21 | s3_client = boto3.client( 22 | 's3', endpoint_url=s3_endpoint_url, 23 | aws_access_key_id=s3_access_key, aws_secret_access_key=s3_secret_key 24 | ) 25 | 26 | s3_client.download_file( 27 | s3_bucket_name, 28 | data_object_name, 29 | f'{data_folder}/data.csv' 30 | ) 31 | print('Finished data ingestion.') 32 | 33 | 34 | if __name__ == '__main__': 35 | ingest_data(data_folder='/data') 36 | -------------------------------------------------------------------------------- /5.pipelines/elyra/model_loading.py: -------------------------------------------------------------------------------- 1 | from os import environ 2 | 3 | from boto3 import client 4 | 5 | 6 | def load_model(model_file_name=''): 7 | print('Commencing model loading.') 8 | 9 | s3_endpoint_url = environ.get('AWS_S3_ENDPOINT') 10 | s3_access_key = environ.get('AWS_ACCESS_KEY_ID') 11 | s3_secret_key = environ.get('AWS_SECRET_ACCESS_KEY') 12 | s3_bucket_name = environ.get('AWS_S3_BUCKET') 13 | model_object_name = model_file_name or environ.get( 14 | 'model_object_name', 'model-latest.onnx' 15 | ) 16 | 17 | print(f'Downloading model "{model_object_name}" ' 18 | f'from bucket {s3_bucket_name} ' 19 | f'from S3 storage at {s3_endpoint_url}') 20 | 21 | s3_client = client( 22 | 's3', endpoint_url=s3_endpoint_url, 23 | aws_access_key_id=s3_access_key, aws_secret_access_key=s3_secret_key 24 | ) 25 | 26 | s3_client.download_file( 27 | s3_bucket_name, model_object_name, 'model.onnx' 28 | ) 29 | 30 | print('Finished model loading.') 31 | 32 | 33 | if __name__ == '__main__': 34 | load_model() 35 | -------------------------------------------------------------------------------- /5.pipelines/elyra/offline-scoring.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "66db5c01-e2c7-4251-b3f6-30f89c946c2b", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "from data_ingestion import ingest_data\n", 11 | "from preprocessing import preprocess\n", 12 | "from model_loading import load_model\n", 13 | "from scoring import predict\n", 14 | "from results_upload import upload_results" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "id": "4d7accea-e620-4e80-aed9-6dd8821799b5", 21 | "metadata": { 22 | "tags": [] 23 | }, 24 | "outputs": [], 25 | "source": [ 26 | "ingest_data('live-data.csv')" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "id": "bf82360c-fb68-4407-adc2-86e8815e1391", 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "df = preprocess()" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "id": "f6280f28-babc-4e99-b3b5-e66b5c0cf839", 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "load_model('model-latest.onnx')" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": null, 52 | "id": "23291014-609f-4504-abc1-acfefc21c6ad", 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "predict()" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "id": "a7e1f76a-a969-4a10-b3d5-22c9d157ee8e", 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "upload_results()" 67 | ] 68 | } 69 | ], 70 | "metadata": { 71 | "kernelspec": { 72 | "display_name": "Python 3.9.16", 73 | "language": "python", 74 | "name": "python3" 75 | }, 76 | "language_info": { 77 | "codemirror_mode": { 78 | "name": "ipython", 79 | "version": 3 80 | }, 81 | "file_extension": ".py", 82 | "mimetype": "text/x-python", 83 | "name": "python", 84 | "nbconvert_exporter": "python", 85 | "pygments_lexer": "ipython3", 86 | "version": "3.9.16" 87 | } 88 | }, 89 | "nbformat": 4, 90 | "nbformat_minor": 5 91 | } 92 | -------------------------------------------------------------------------------- /5.pipelines/elyra/preprocessing.py: -------------------------------------------------------------------------------- 1 | from numpy import save 2 | from pandas import read_csv 3 | from sklearn.preprocessing import RobustScaler 4 | 5 | 6 | def preprocess(data_folder='./data'): 7 | print('preprocessing data') 8 | 9 | df = read_csv(f'{data_folder}/data.csv', index_col=0) 10 | 11 | rob_scaler = RobustScaler() 12 | 13 | df['scaled_amount'] = rob_scaler.fit_transform( 14 | df['Amount'].values.reshape(-1, 1) 15 | ) 16 | df['scaled_time'] = rob_scaler.fit_transform( 17 | df['Time'].values.reshape(-1, 1) 18 | ) 19 | df.drop(['Time', 'Amount'], axis=1, inplace=True) 20 | scaled_amount = df['scaled_amount'] 21 | scaled_time = df['scaled_time'] 22 | 23 | df.drop(['scaled_amount', 'scaled_time'], axis=1, inplace=True) 24 | df.insert(0, 'scaled_amount', scaled_amount) 25 | df.insert(1, 'scaled_time', scaled_time) 26 | 27 | save(f'{data_folder}/samples.npy', df.values) 28 | 29 | print('data processing done') 30 | 31 | 32 | if __name__ == '__main__': 33 | preprocess(data_folder='/data') 34 | -------------------------------------------------------------------------------- /5.pipelines/elyra/results_upload.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from os import environ 3 | 4 | from boto3 import client 5 | 6 | 7 | def upload_results(data_folder='./data'): 8 | print('Commencing results upload.') 9 | 10 | s3_endpoint_url = environ.get('AWS_S3_ENDPOINT') 11 | s3_access_key = environ.get('AWS_ACCESS_KEY_ID') 12 | s3_secret_key = environ.get('AWS_SECRET_ACCESS_KEY') 13 | s3_bucket_name = environ.get('AWS_S3_BUCKET') 14 | 15 | timestamp = datetime.now().strftime('%y%m%d%H%M') 16 | results_name = f'predictions-{timestamp}.csv' 17 | 18 | print(f'Uploading predictions to bucket {s3_bucket_name} ' 19 | f'to S3 storage at {s3_endpoint_url}') 20 | 21 | s3_client = client( 22 | 's3', endpoint_url=s3_endpoint_url, 23 | aws_access_key_id=s3_access_key, aws_secret_access_key=s3_secret_key 24 | ) 25 | 26 | with open(f'{data_folder}/predictions.csv', 'rb') as results_file: 27 | s3_client.upload_fileobj(results_file, s3_bucket_name, results_name) 28 | 29 | print('Finished uploading results.') 30 | 31 | 32 | if __name__ == '__main__': 33 | upload_results(data_folder='/data') 34 | -------------------------------------------------------------------------------- /5.pipelines/elyra/scoring.py: -------------------------------------------------------------------------------- 1 | from numpy import argmax, array, load 2 | from onnxruntime import InferenceSession 3 | from pandas import DataFrame 4 | 5 | 6 | def predict(data_folder='./data'): 7 | print('Commencing offline scoring.') 8 | 9 | X = load(f'{data_folder}/samples.npy').astype('float32') 10 | 11 | session = InferenceSession('model.onnx') 12 | raw_results = session.run([], {'dense_3_input': X})[0] 13 | 14 | results = argmax(raw_results, axis=1) 15 | class_map_array = array(['no fraud', 'fraud']) 16 | mapped_results = class_map_array[results] 17 | 18 | print(f'Scored data set. Writing report.') 19 | 20 | column_names = [f'V{i}' for i in range(1, 31)] 21 | report = DataFrame(X, columns=column_names) 22 | report.insert(0, 'Prediction', mapped_results) 23 | 24 | report.to_csv(f'{data_folder}/predictions.csv') 25 | 26 | print('Wrote report. Offline scoring complete.') 27 | 28 | 29 | if __name__ == '__main__': 30 | predict(data_folder='/data') 31 | -------------------------------------------------------------------------------- /6.rhod-admin/custom-images/Containerfile: -------------------------------------------------------------------------------- 1 | #BASE_IMAGE 2 | #This is the Jupyter Data Science Workbench image 3 | #################################################################### 4 | # TODO - Add the command to use the Jupyter Data Science Workbench # 5 | #################################################################### 6 | 7 | 8 | # Copying custom packages 9 | # The requirements.txt file contains additional package dependencies we want installed 10 | ########################################################### 11 | # TODO - Add the command to copy the requirements file # 12 | ########################################################### 13 | 14 | # Installing custom packages 15 | ########################################################### 16 | # TODO - Add the command to install the requirements # 17 | ########################################################### 18 | 19 | 20 | 21 | # Fix permissions to support pip in Openshift environments \ 22 | RUN chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \ 23 | fix-permissions /opt/app-root -P 24 | -------------------------------------------------------------------------------- /6.rhod-admin/custom-images/requirements.txt: -------------------------------------------------------------------------------- 1 | seaborn==0.13.0 2 | -------------------------------------------------------------------------------- /7.hands-on-lab/.gitignore: -------------------------------------------------------------------------------- 1 | *.onnx -------------------------------------------------------------------------------- /7.hands-on-lab/1.collect.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "1ce8ea35-5013-40aa-83d3-e949b50ad22f", 6 | "metadata": {}, 7 | "source": [ 8 | "# Collect the Dataset\n", 9 | "\n", 10 | "To collect the dataset, download the `creditcard.csv` file from your S3 bucket.\n", 11 | "Save the file as `data/credictcard.csv`." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "id": "286bfe3b-257a-4bdc-a2ec-d44d425b6b90", 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "# Make sure that the data directory exists\n", 22 | "!mkdir -p data" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 1, 28 | "id": "9d9bf7f7-97d8-4ec7-8de5-2d7d88baed5d", 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "# TODO..." 33 | ] 34 | } 35 | ], 36 | "metadata": { 37 | "kernelspec": { 38 | "display_name": "Python 3.9", 39 | "language": "python", 40 | "name": "python3" 41 | }, 42 | "language_info": { 43 | "codemirror_mode": { 44 | "name": "ipython", 45 | "version": 3 46 | }, 47 | "file_extension": ".py", 48 | "mimetype": "text/x-python", 49 | "name": "python", 50 | "nbconvert_exporter": "python", 51 | "pygments_lexer": "ipython3", 52 | "version": "3.9.16" 53 | } 54 | }, 55 | "nbformat": 4, 56 | "nbformat_minor": 5 57 | } 58 | -------------------------------------------------------------------------------- /7.hands-on-lab/2.exploration.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "7dbaa1af-dcda-4460-8756-7b8611ae8302", 6 | "metadata": {}, 7 | "source": [ 8 | "# Data Exploration\n", 9 | "\n", 10 | "Execute this notebook to explore the data.\n", 11 | "\n", 12 | "The data used in this exercise contains records of normal and fraudulent credit card payments.\n", 13 | "\n", 14 | "You must use the data to train a machine learning model for detecting whether a transaction is a normal payment or a fraud." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "id": "41bd14ef-8bd5-4cd3-9921-0b37be68ca85", 20 | "metadata": {}, 21 | "source": [ 22 | "First of all, load the data and inspect a few records:" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "id": "e0823c66-8012-474e-aadc-3e6179617f9b", 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "import pandas as pd\n", 33 | "\n", 34 | "df = pd.read_csv('data/creditcard.csv')\n", 35 | "df.head()" 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "id": "c37f5a6b-0171-438f-b7b7-c7595bc3aa5e", 41 | "metadata": {}, 42 | "source": [ 43 | "The dataset contains the following columns:\n", 44 | "\n", 45 | "* **Time**: the transaction time.\n", 46 | "* **Columns V1..V28**: Anonymized values for privacy reasons. You can assume that these columns describe the details of each transaction.\n", 47 | "* **Amount**: The amount of the transaction.\n", 48 | "* **Class**: Whether the transaction is fraudlent(`1`) or not (`0`)." 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "id": "ffa09973-e140-449c-985e-0f4dd1aaacaf", 54 | "metadata": {}, 55 | "source": [ 56 | "Count the number of rows (samples) of the dataset:" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "id": "f6292765-a3d8-4012-9da8-6450cc848609", 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "len(df)" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "id": "5b5af588-5628-467e-9e88-d9248133a76f", 72 | "metadata": {}, 73 | "source": [ 74 | "You can take a look at descriptive statistics for each column" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "id": "4b66d42d-e00c-4467-94ca-6d817bcbb6c9", 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "df.describe()" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "id": "1361e4a4-2241-4b04-abd9-17975f7f760b", 90 | "metadata": {}, 91 | "source": [ 92 | "Verify whether the dataset contains null values" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "id": "f88948fa-8042-40a2-a52d-3c6cbaaa0d93", 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "print(\n", 103 | " f\"No Frauds {round(df['Class'].value_counts()[0]/len(df) * 100, 2)}\"\n", 104 | " f' % of the dataset'\n", 105 | ")\n", 106 | "print(\n", 107 | " f\"Frauds {round(df['Class'].value_counts()[1]/len(df) * 100, 2)}\"\n", 108 | " f'% of the dataset'\n", 109 | ")" 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "id": "03f1fca7-00d5-406b-91a1-3b1c0d4a20dc", 115 | "metadata": {}, 116 | "source": [ 117 | "As you can see, most of the transactions are non-fraud. If we use this datase as the base for training our model, then we might get a lot of classification errors and the model will probably assume that most transactions are not fraud. But we don't want our model to assume based on class probabilities, we want our model to detect patterns that give signs of fraud!\n", 118 | "\n", 119 | "The preprocessing step will take care of this problem by balancing the dataset." 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "id": "6cf6e0e8-d645-441f-9e71-373fe9cae24e", 125 | "metadata": {}, 126 | "source": [ 127 | "## References\n", 128 | "\n", 129 | "* The code used for this exercise is inspired in the following Kaggle notebook https://www.kaggle.com/code/janiobachmann/credit-fraud-dealing-with-imbalanced-datasets\n", 130 | "\n", 131 | "* Data https://www.kaggle.com/code/janiobachmann/credit-fraud-dealing-with-imbalanced-datasets/input" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "id": "de68bbdf-f7af-4add-9272-80871203fb2d", 138 | "metadata": {}, 139 | "outputs": [], 140 | "source": [] 141 | } 142 | ], 143 | "metadata": { 144 | "kernelspec": { 145 | "display_name": "Python 3.9", 146 | "language": "python", 147 | "name": "python3" 148 | }, 149 | "language_info": { 150 | "codemirror_mode": { 151 | "name": "ipython", 152 | "version": 3 153 | }, 154 | "file_extension": ".py", 155 | "mimetype": "text/x-python", 156 | "name": "python", 157 | "nbconvert_exporter": "python", 158 | "pygments_lexer": "ipython3", 159 | "version": "3.9.16" 160 | } 161 | }, 162 | "nbformat": 4, 163 | "nbformat_minor": 5 164 | } 165 | -------------------------------------------------------------------------------- /7.hands-on-lab/3.preprocessing.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "c912c1f9-2279-4c6c-b989-9b292e49a504", 6 | "metadata": {}, 7 | "source": [ 8 | "# Data Preprocessing\n", 9 | "\n", 10 | "Run this notebook to preprocess the data.\n", 11 | "This notebook scales the data, deals with data imbalance and generates a training subset.\n", 12 | "\n", 13 | "> **NOTE**: This notebook performs all the necessary steps to preprocess the data before training.\n", 14 | "You do not need to develop additional preprocessing steps.\n", 15 | "\n" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "id": "31da1256-2896-4f06-89fd-717d528698b3", 21 | "metadata": {}, 22 | "source": [ 23 | "Import dependencies:" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "id": "c952c8f9-5eb2-4435-9e24-0b288b0035c4", 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "from imblearn.over_sampling import SMOTE\n", 34 | "from numpy import save, count_nonzero\n", 35 | "from pandas import read_csv\n", 36 | "from sklearn.model_selection import StratifiedKFold\n", 37 | "from sklearn.preprocessing import RobustScaler" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "4a3a7e18-4a26-4ade-a5c3-4ad563cdd4d7", 43 | "metadata": {}, 44 | "source": [ 45 | "The following code scales the `Amount` and `Time` columns, then builds a training subset that contains a balanced number of each class (`Fraud`(1), `No Fraud`(0)).\n", 46 | "\n", 47 | "The result of preprocessing the data is the `data/training_samples.npy` file, which contains all columns except for the class, and `data/training_labels.npy`, which contains only the class column." 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "id": "67f70658-4f72-45eb-b0fc-69c172c16b91", 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "df = read_csv('data/creditcard.csv')\n", 58 | "\n", 59 | "rob_scaler = RobustScaler()\n", 60 | "\n", 61 | "df['scaled_amount'] = rob_scaler.fit_transform(\n", 62 | " df['Amount'].values.reshape(-1, 1)\n", 63 | ")\n", 64 | "df['scaled_time'] = rob_scaler.fit_transform(\n", 65 | " df['Time'].values.reshape(-1, 1)\n", 66 | ")\n", 67 | "df.drop(['Time', 'Amount'], axis=1, inplace=True)\n", 68 | "scaled_amount = df['scaled_amount']\n", 69 | "scaled_time = df['scaled_time']\n", 70 | "\n", 71 | "df.drop(['scaled_amount', 'scaled_time'], axis=1, inplace=True)\n", 72 | "df.insert(0, 'scaled_amount', scaled_amount)\n", 73 | "df.insert(1, 'scaled_time', scaled_time)\n", 74 | "\n", 75 | "X = df.drop('Class', axis=1)\n", 76 | "y = df['Class']\n", 77 | "sss = StratifiedKFold(n_splits=5, random_state=None, shuffle=False)\n", 78 | "\n", 79 | "for train_index, test_index in sss.split(X, y):\n", 80 | " original_Xtrain = X.iloc[train_index]\n", 81 | " original_ytrain = y.iloc[train_index]\n", 82 | "\n", 83 | "original_Xtrain = original_Xtrain.values\n", 84 | "original_ytrain = original_ytrain.values\n", 85 | "\n", 86 | "sm = SMOTE(sampling_strategy='minority', random_state=42)\n", 87 | "Xsm_train, ysm_train = sm.fit_resample(original_Xtrain, original_ytrain)\n", 88 | "\n", 89 | "save('data/training_samples.npy', Xsm_train)\n", 90 | "save('data/training_labels.npy', ysm_train)\n", 91 | "\n", 92 | "print('Data processing done!')" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "id": "57e83635-73d4-4ebb-a2f0-d8e7cd9a60a2", 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "num_frauds = count_nonzero(ysm_train)\n", 103 | "\n", 104 | "print(\"Fraud cases\", num_frauds)\n", 105 | "print(\"No fraud cases\", ysm_train.size - num_frauds)" 106 | ] 107 | } 108 | ], 109 | "metadata": { 110 | "kernelspec": { 111 | "display_name": "Python 3.9", 112 | "language": "python", 113 | "name": "python3" 114 | }, 115 | "language_info": { 116 | "codemirror_mode": { 117 | "name": "ipython", 118 | "version": 3 119 | }, 120 | "file_extension": ".py", 121 | "mimetype": "text/x-python", 122 | "name": "python", 123 | "nbconvert_exporter": "python", 124 | "pygments_lexer": "ipython3", 125 | "version": "3.9.16" 126 | } 127 | }, 128 | "nbformat": 4, 129 | "nbformat_minor": 5 130 | } 131 | -------------------------------------------------------------------------------- /7.hands-on-lab/4.model_training.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "6621e85f-cb72-4a58-acb5-1675fe6e819a", 6 | "metadata": {}, 7 | "source": [ 8 | "# Model Training\n", 9 | "\n", 10 | "Run this notebook to train a basic neural network by using the preprocessed data and the Keras high-level TensorFlow API.\n", 11 | "\n", 12 | "At the bottom of the notebook, after model training, add the code to convert the trained model to ONNX format." 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "7db270f7-7965-44a9-98cb-7b4cfd122af8", 18 | "metadata": {}, 19 | "source": [ 20 | "First, import the necessary libraries and deactivate GPU acceleration." 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 2, 26 | "id": "e40e8d5c-f870-43df-8968-905f8370c515", 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "from os import environ\n", 31 | "\n", 32 | "environ['CUDA_VISIBLE_DEVICES'] = '-1'\n", 33 | "\n", 34 | "from keras.models import Sequential\n", 35 | "from keras.layers.core import Dense\n", 36 | "from keras.optimizers import Adam\n", 37 | "from numpy import load\n", 38 | "from onnx import save\n", 39 | "from tf2onnx import convert" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "id": "1cffb9fd-8297-4f60-89df-935ebf50fa08", 45 | "metadata": {}, 46 | "source": [ 47 | "Next, create a basic neural network, and train the network by using the preprocessed data." 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "id": "993bd9e1-04af-480a-bd22-c3ba9a23c4d3", 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "print('training model')\n", 58 | "\n", 59 | "epoch_count = int(environ.get('epoch_count', '20'))\n", 60 | "learning_rate = float(environ.get('learning_rate', '0.001'))\n", 61 | "\n", 62 | "Xsm_train = load('data/training_samples.npy')\n", 63 | "ysm_train = load('data/training_labels.npy')\n", 64 | "n_inputs = Xsm_train.shape[1]\n", 65 | "\n", 66 | "model = Sequential([\n", 67 | " Dense(n_inputs, input_shape=(n_inputs, ), activation='relu'),\n", 68 | " Dense(32, activation='relu'),\n", 69 | " Dense(2, activation='softmax'),\n", 70 | "])\n", 71 | "model.compile(\n", 72 | " Adam(learning_rate=learning_rate),\n", 73 | " loss='sparse_categorical_crossentropy',\n", 74 | " metrics=['accuracy'],\n", 75 | ")\n", 76 | "model.fit(\n", 77 | " Xsm_train,\n", 78 | " ysm_train,\n", 79 | " validation_split=0.2,\n", 80 | " batch_size=300,\n", 81 | " epochs=epoch_count,\n", 82 | " shuffle=True,\n", 83 | " verbose=2,\n", 84 | ")" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "id": "16cf108f-c4b7-4113-95a8-401105e7d09e", 90 | "metadata": {}, 91 | "source": [ 92 | "# Export the Model to ONNX Format" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 2, 98 | "id": "e70bb8cc-c9e8-4fcc-9cd7-9e5d288b564e", 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "# TODO" 103 | ] 104 | } 105 | ], 106 | "metadata": { 107 | "kernelspec": { 108 | "display_name": "Python 3.9", 109 | "language": "python", 110 | "name": "python3" 111 | }, 112 | "language_info": { 113 | "codemirror_mode": { 114 | "name": "ipython", 115 | "version": 3 116 | }, 117 | "file_extension": ".py", 118 | "mimetype": "text/x-python", 119 | "name": "python", 120 | "nbconvert_exporter": "python", 121 | "pygments_lexer": "ipython3", 122 | "version": "3.9.16" 123 | } 124 | }, 125 | "nbformat": 4, 126 | "nbformat_minor": 5 127 | } 128 | -------------------------------------------------------------------------------- /7.hands-on-lab/5.model_upload.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "58dcdaab-02d0-4234-873f-90e98c3d7f4f", 6 | "metadata": {}, 7 | "source": [ 8 | "# Model Upload\n", 9 | "\n", 10 | "Complete and run this notebook to upload the trained model in ONNX format to your S3 bucket.\n", 11 | "You will use the uploaded model in a Model Server." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "id": "1bc75ddb-0662-4218-a047-90a4dde8d0ae", 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "# TODO" 22 | ] 23 | } 24 | ], 25 | "metadata": { 26 | "kernelspec": { 27 | "display_name": "Python 3.9", 28 | "language": "python", 29 | "name": "python3" 30 | }, 31 | "language_info": { 32 | "codemirror_mode": { 33 | "name": "ipython", 34 | "version": 3 35 | }, 36 | "file_extension": ".py", 37 | "mimetype": "text/x-python", 38 | "name": "python", 39 | "nbconvert_exporter": "python", 40 | "pygments_lexer": "ipython3", 41 | "version": "3.9.16" 42 | } 43 | }, 44 | "nbformat": 4, 45 | "nbformat_minor": 5 46 | } 47 | -------------------------------------------------------------------------------- /7.hands-on-lab/6.test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "4f925f11-3643-46b7-8bf7-656ac5b64f3d", 6 | "metadata": {}, 7 | "source": [ 8 | "# Test the Uploaded Model\n", 9 | "\n", 10 | "Complete and run this notebook to test your deployed model." 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "id": "f7690e15-b191-4899-a250-502a2b066e53", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "import requests\n", 21 | "from numpy import argmax" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "id": "4033e87d-fe10-4870-8bb1-e3a1c0b758af", 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "INFERENCE_ENDPOINT = 'TODO'" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": null, 37 | "id": "16bd03d7-73f8-4be3-8bf8-80e59fd6ab89", 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "# No fraud sample\n", 42 | "sample = [\n", 43 | " -0.15384616, -0.9909186, 1.0770786, 0.2849802, 0.0077313827, 1.657073, 0.05202024, 0.44638866, \n", 44 | " -0.40703616, 0.3557039, 0.6260392, -0.92908716, 1.0941651, 0.57956475, -0.8621889, -2.2244275, \n", 45 | " -1.1098708, 0.17238183, 1.7235482, 0.6979903, -0.15358274, -0.14279902, -0.17433698, -0.1741605, \n", 46 | " -0.15337533, -0.46633127, 0.6110009, -0.25287056, 0.090374656, 0.054820385\n", 47 | "]" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "id": "30a55457-b313-4af8-9827-c5dbaebdd01a", 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "payload = {\n", 58 | " # TODO\n", 59 | "}\n", 60 | "\n", 61 | "response = requests.post(INFERENCE_ENDPOINT, json=payload)\n", 62 | "result = response.json()\n", 63 | "result" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "id": "5bfbaf0d-b14e-44cd-af51-cfb0911a406d", 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "labels = ['No Fraud', 'Fraud']\n", 74 | "model_output = result['outputs'][0]['data']\n", 75 | "classification_result = labels[argmax(model_output)]\n", 76 | "\n", 77 | "print(f'The sample was classified as \"{classification_result}\".')" 78 | ] 79 | } 80 | ], 81 | "metadata": { 82 | "kernelspec": { 83 | "display_name": "Python 3.9", 84 | "language": "python", 85 | "name": "python3" 86 | }, 87 | "language_info": { 88 | "codemirror_mode": { 89 | "name": "ipython", 90 | "version": 3 91 | }, 92 | "file_extension": ".py", 93 | "mimetype": "text/x-python", 94 | "name": "python", 95 | "nbconvert_exporter": "python", 96 | "pygments_lexer": "ipython3", 97 | "version": "3.9.16" 98 | } 99 | }, 100 | "nbformat": 4, 101 | "nbformat_minor": 5 102 | } 103 | -------------------------------------------------------------------------------- /7.hands-on-lab/README.md: -------------------------------------------------------------------------------- 1 | # Hands-on Lab: Hit the RHODS 2 | 3 | In this exercise, you will complete the data science project for detecting fraudulent credit card transactions. In the [previous exercise](https://redhatquickcourses.github.io/rhods-pipelines/rhods-pipelines/1.33/chapter1/section2.html#_exercise_offline_scoring_for_fraud_detection), you have seen how to run pipelines for detecting fraudulent transactions using a pre-trained model. Let's now create a new version of this model and deploy it! 4 | 5 | The lab materials include notebooks that implement parts of the project workflow. 6 | Your task is to complete the missing parts of those notebooks and create a data science pipeline and serve the model to meet the lab specifications. 7 | 8 | ## Prerequisites 9 | 10 | 11 | > **❗ IMPORTANT** 12 | > 13 | > Before working on this exercise, make sure that you have taken the following courses: 14 | > 15 | > * [Introduction to Red Hat OpenShift AI](https://redhatquickcourses.github.io/rhods-intro) 16 | > * [Red Hat OpenShift AI Administration](https://redhatquickcourses.github.io/rhods-admin) 17 | > * [Creating Machine Learning Models with Red Hat OpenShift AI](https://redhatquickcourses.github.io/rhods-model) 18 | > * [Deploying Machine Learning Models with Red Hat OpenShift AI](https://redhatquickcourses.github.io/rhods-deploy) 19 | > * [Automation using Data Science Pipelines](https://redhatquickcourses.github.io/rhods-pipelines) 20 | > 21 | > Use the same cluster that you have been using for the preceding courses. 22 | > 23 | > If you still want to proceed with this lab without taking the courses, you can use the [Base RHODS on AWS demo environment](https://demo.redhat.com/catalog?search=rhods+aws&item=babylon-catalog-prod%2Fsandboxes-gpte.ocp4-workshop-rhods-base-aws.prod). 24 | 25 | The lab scenario assumes that the dataset is available in the same `fraud-detection` S3 bucket that you have used in the [RHODS pipelines quick course](https://redhatquickcourses.github.io/rhods-pipelines/rhods-pipelines/1.33/chapter1/section2.html#_data_science_pipeline_hands_on_example). 26 | 27 | You will also use the same bucket to store the trained model and pipeline output reports. 28 | 29 | The dataset is a CSV file called `live-data.csv`. 30 | 31 | 32 | > **NOTE** 33 | > 34 | > If you have not initialized your S3 store in the [administration course](https://redhatquickcourses.github.io/rhods-admin), then you can follow these steps: 35 | > 36 | > * Create the S3 object store. 37 | > You can create an s3 object store with Minio, by using this guide: https://ai-on-openshift.io/tools-and-applications/minio/minio/. 38 | > 39 | > * Download the dataset file from https://drive.google.com/file/d/1Gd-ENv1SdqcXYFf81KKIWAsSFwe0Z6ch/view?usp=sharing. 40 | Alternatively, you can download the dataset from Kaggle (https://www.kaggle.com/code/janiobachmann/credit-fraud-dealing-with-imbalanced-datasets/input). 41 | > 42 | > * Upload the dataset into the root of your S3 bucket. 43 | 44 | 45 | ## Specifications 46 | 47 | * Perform your exercise in the fraud detection workbench (`quay.io/mmurakam/workbenches:fraud-detection-v1.0.1`). 48 | Make sure that the workbench is associated to a data connection that includes your S3 connection settings. 49 | Make sure that the workbench size is at least `Small`. 50 | 51 | * Clone the https://github.com/RedHatQuickCourses/rhods-qc-apps repository into your workbench. 52 | The materials for this lab are located in the `7.hands-on-lab/` directory. 53 | 54 | * Collect the data. 55 | Use the `1.collect` notebook to download the `live-data.csv` file into `data/creditcard.csv`. 56 | 57 | * Explore and preprocess data. 58 | Use the `2.exploration` and `3.preprocessing` notebooks. 59 | You do not need to make changes to these notebooks. 60 | 61 | * Train the model. 62 | Use the `4.model_training` notebook. 63 | After training, add the code to convert the model to ONNX format. 64 | 65 | 66 | * Upload the model to S3. 67 | Use the `5.model_upload` notebook. 68 | 69 | * Create a data science pipeline. 70 | Note that, in the [pipelines quick course](https://redhatquickcourses.github.io/rhods-pipelines/rhods-pipelines/1.33/index.html) you already created a pipeline for offline scoring, that is, for classification of unlabeled transactions given a pretrained model file. 71 | 72 | In this case, you must build a pipeline to train a model. 73 | This pipeline must collect training data, preprocess the data, train the model, and upload the trained model. 74 | You can use the `quay.io/mmurakam/runtimes:fraud-detection-v0.1.0` runtime image for this pipeline. 75 | 76 | * Create a model server and serve the model that you uploaded to S3. 77 | Expose the model via an external route. 78 | 79 | * Test that you can consume the deployed model by using the `6.test` notebook. 80 | To consume the model, you must send a request that uses the [KServe inference API format](https://github.com/kserve/kserve/blob/master/docs/predict-api/v2/required_api.md#inference). 81 | Use `dense_input` as the inputs name. 82 | 83 | 84 | You can find the solutions in the `solutions` directory. 85 | -------------------------------------------------------------------------------- /7.hands-on-lab/data/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /7.hands-on-lab/solutions/1.collect.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "dad5d363-8e5b-4098-a603-984122e65e24", 6 | "metadata": {}, 7 | "source": [ 8 | "# Collect the Dataset\n", 9 | "\n", 10 | "To collect the dataset, download the `creditcard.csv` file from your S3 bucket.\n", 11 | "Save the file as `data/credictcard.csv`." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 2, 17 | "id": "7965ffbf-5b64-49c1-866e-43c6a7bed802", 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "# Make sure that the data directory exists\n", 22 | "!mkdir -p data" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 3, 28 | "id": "9d9bf7f7-97d8-4ec7-8de5-2d7d88baed5d", 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "import os\n", 33 | "import boto3" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 4, 39 | "id": "4e55804e-c172-45e0-ab13-f43a5239f121", 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "s3_data_path = \"creditcard.csv\"" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 5, 49 | "id": "60b6c03b-b5c6-4db8-880c-230b13f0deef", 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "key_id = os.getenv(\"AWS_ACCESS_KEY_ID\")\n", 54 | "secret_key = os.getenv(\"AWS_SECRET_ACCESS_KEY\")\n", 55 | "endpoint = os.getenv(\"AWS_S3_ENDPOINT\")\n", 56 | "bucket_name = os.getenv(\"AWS_S3_BUCKET\")\n", 57 | "\n", 58 | "s3 = boto3.client(\n", 59 | " \"s3\",\n", 60 | " aws_access_key_id=key_id,\n", 61 | " aws_secret_access_key=secret_key,\n", 62 | " endpoint_url=endpoint,\n", 63 | " use_ssl=True\n", 64 | ")\n", 65 | "\n", 66 | "s3.download_file(bucket_name, s3_data_path, \"data/creditcard.csv\")" 67 | ] 68 | } 69 | ], 70 | "metadata": { 71 | "kernelspec": { 72 | "display_name": "Python 3.9.16", 73 | "language": "python", 74 | "name": "python3" 75 | }, 76 | "language_info": { 77 | "codemirror_mode": { 78 | "name": "ipython", 79 | "version": 3 80 | }, 81 | "file_extension": ".py", 82 | "mimetype": "text/x-python", 83 | "name": "python", 84 | "nbconvert_exporter": "python", 85 | "pygments_lexer": "ipython3", 86 | "version": "3.9.16" 87 | } 88 | }, 89 | "nbformat": 4, 90 | "nbformat_minor": 5 91 | } 92 | -------------------------------------------------------------------------------- /7.hands-on-lab/solutions/2.exploration.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "7dbaa1af-dcda-4460-8756-7b8611ae8302", 6 | "metadata": {}, 7 | "source": [ 8 | "# Data Exploration\n", 9 | "\n", 10 | "Execute this notebook to explore the data.\n", 11 | "\n", 12 | "The data used in this exercise contains records of normal and fraudulent credit card payments.\n", 13 | "\n", 14 | "You must use the data to train a machine learning model for detecting whether a transaction is a normal payment or a fraud." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "id": "41bd14ef-8bd5-4cd3-9921-0b37be68ca85", 20 | "metadata": {}, 21 | "source": [ 22 | "First of all, load the data and inspect a few records:" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 1, 28 | "id": "e0823c66-8012-474e-aadc-3e6179617f9b", 29 | "metadata": {}, 30 | "outputs": [ 31 | { 32 | "data": { 33 | "text/html": [ 34 | "
\n", 35 | "\n", 48 | "\n", 49 | " \n", 50 | " \n", 51 | " \n", 52 | " \n", 53 | " \n", 54 | " \n", 55 | " \n", 56 | " \n", 57 | " \n", 58 | " \n", 59 | " \n", 60 | " \n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | "
TimeV1V2V3V4V5V6V7V8V9...V21V22V23V24V25V26V27V28AmountClass
00.0-1.359807-0.0727812.5363471.378155-0.3383210.4623880.2395990.0986980.363787...-0.0183070.277838-0.1104740.0669280.128539-0.1891150.133558-0.021053149.620
10.01.1918570.2661510.1664800.4481540.060018-0.082361-0.0788030.085102-0.255425...-0.225775-0.6386720.101288-0.3398460.1671700.125895-0.0089830.0147242.690
21.0-1.358354-1.3401631.7732090.379780-0.5031981.8004990.7914610.247676-1.514654...0.2479980.7716790.909412-0.689281-0.327642-0.139097-0.055353-0.059752378.660
31.0-0.966272-0.1852261.792993-0.863291-0.0103091.2472030.2376090.377436-1.387024...-0.1083000.005274-0.190321-1.1755750.647376-0.2219290.0627230.061458123.500
42.0-1.1582330.8777371.5487180.403034-0.4071930.0959210.592941-0.2705330.817739...-0.0094310.798278-0.1374580.141267-0.2060100.5022920.2194220.21515369.990
\n", 198 | "

5 rows × 31 columns

\n", 199 | "
" 200 | ], 201 | "text/plain": [ 202 | " Time V1 V2 V3 V4 V5 V6 V7 \\\n", 203 | "0 0.0 -1.359807 -0.072781 2.536347 1.378155 -0.338321 0.462388 0.239599 \n", 204 | "1 0.0 1.191857 0.266151 0.166480 0.448154 0.060018 -0.082361 -0.078803 \n", 205 | "2 1.0 -1.358354 -1.340163 1.773209 0.379780 -0.503198 1.800499 0.791461 \n", 206 | "3 1.0 -0.966272 -0.185226 1.792993 -0.863291 -0.010309 1.247203 0.237609 \n", 207 | "4 2.0 -1.158233 0.877737 1.548718 0.403034 -0.407193 0.095921 0.592941 \n", 208 | "\n", 209 | " V8 V9 ... V21 V22 V23 V24 V25 \\\n", 210 | "0 0.098698 0.363787 ... -0.018307 0.277838 -0.110474 0.066928 0.128539 \n", 211 | "1 0.085102 -0.255425 ... -0.225775 -0.638672 0.101288 -0.339846 0.167170 \n", 212 | "2 0.247676 -1.514654 ... 0.247998 0.771679 0.909412 -0.689281 -0.327642 \n", 213 | "3 0.377436 -1.387024 ... -0.108300 0.005274 -0.190321 -1.175575 0.647376 \n", 214 | "4 -0.270533 0.817739 ... -0.009431 0.798278 -0.137458 0.141267 -0.206010 \n", 215 | "\n", 216 | " V26 V27 V28 Amount Class \n", 217 | "0 -0.189115 0.133558 -0.021053 149.62 0 \n", 218 | "1 0.125895 -0.008983 0.014724 2.69 0 \n", 219 | "2 -0.139097 -0.055353 -0.059752 378.66 0 \n", 220 | "3 -0.221929 0.062723 0.061458 123.50 0 \n", 221 | "4 0.502292 0.219422 0.215153 69.99 0 \n", 222 | "\n", 223 | "[5 rows x 31 columns]" 224 | ] 225 | }, 226 | "execution_count": 1, 227 | "metadata": {}, 228 | "output_type": "execute_result" 229 | } 230 | ], 231 | "source": [ 232 | "import pandas as pd\n", 233 | "\n", 234 | "df = pd.read_csv('data/creditcard.csv')\n", 235 | "df.head()" 236 | ] 237 | }, 238 | { 239 | "cell_type": "markdown", 240 | "id": "c37f5a6b-0171-438f-b7b7-c7595bc3aa5e", 241 | "metadata": {}, 242 | "source": [ 243 | "The dataset contains the following columns:\n", 244 | "\n", 245 | "* **Time**: the transaction time.\n", 246 | "* **Columns V1..V28**: Anonymized values for privacy reasons. You can assume that these columns describe the details of each transaction.\n", 247 | "* **Amount**: The amount of the transaction.\n", 248 | "* **Class**: Whether the transaction is fraudlent(`1`) or not (`0`)." 249 | ] 250 | }, 251 | { 252 | "cell_type": "markdown", 253 | "id": "ffa09973-e140-449c-985e-0f4dd1aaacaf", 254 | "metadata": {}, 255 | "source": [ 256 | "Count the number of rows (samples) of the dataset:" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": 2, 262 | "id": "f6292765-a3d8-4012-9da8-6450cc848609", 263 | "metadata": {}, 264 | "outputs": [ 265 | { 266 | "data": { 267 | "text/plain": [ 268 | "284807" 269 | ] 270 | }, 271 | "execution_count": 2, 272 | "metadata": {}, 273 | "output_type": "execute_result" 274 | } 275 | ], 276 | "source": [ 277 | "len(df)" 278 | ] 279 | }, 280 | { 281 | "cell_type": "markdown", 282 | "id": "5b5af588-5628-467e-9e88-d9248133a76f", 283 | "metadata": {}, 284 | "source": [ 285 | "You can take a look at descriptive statistics for each column" 286 | ] 287 | }, 288 | { 289 | "cell_type": "code", 290 | "execution_count": 3, 291 | "id": "4b66d42d-e00c-4467-94ca-6d817bcbb6c9", 292 | "metadata": {}, 293 | "outputs": [ 294 | { 295 | "data": { 296 | "text/html": [ 297 | "
\n", 298 | "\n", 311 | "\n", 312 | " \n", 313 | " \n", 314 | " \n", 315 | " \n", 316 | " \n", 317 | " \n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | " \n", 403 | " \n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | "
TimeV1V2V3V4V5V6V7V8V9...V21V22V23V24V25V26V27V28AmountClass
count284807.0000002.848070e+052.848070e+052.848070e+052.848070e+052.848070e+052.848070e+052.848070e+052.848070e+052.848070e+05...2.848070e+052.848070e+052.848070e+052.848070e+052.848070e+052.848070e+052.848070e+052.848070e+05284807.000000284807.000000
mean94813.8595751.168375e-153.416908e-16-1.379537e-152.074095e-159.604066e-161.487313e-15-5.556467e-161.213481e-16-2.406331e-15...1.654067e-16-3.568593e-162.578648e-164.473266e-155.340915e-161.683437e-15-3.660091e-16-1.227390e-1688.3496190.001727
std47488.1459551.958696e+001.651309e+001.516255e+001.415869e+001.380247e+001.332271e+001.237094e+001.194353e+001.098632e+00...7.345240e-017.257016e-016.244603e-016.056471e-015.212781e-014.822270e-014.036325e-013.300833e-01250.1201090.041527
min0.000000-5.640751e+01-7.271573e+01-4.832559e+01-5.683171e+00-1.137433e+02-2.616051e+01-4.355724e+01-7.321672e+01-1.343407e+01...-3.483038e+01-1.093314e+01-4.480774e+01-2.836627e+00-1.029540e+01-2.604551e+00-2.256568e+01-1.543008e+010.0000000.000000
25%54201.500000-9.203734e-01-5.985499e-01-8.903648e-01-8.486401e-01-6.915971e-01-7.682956e-01-5.540759e-01-2.086297e-01-6.430976e-01...-2.283949e-01-5.423504e-01-1.618463e-01-3.545861e-01-3.171451e-01-3.269839e-01-7.083953e-02-5.295979e-025.6000000.000000
50%84692.0000001.810880e-026.548556e-021.798463e-01-1.984653e-02-5.433583e-02-2.741871e-014.010308e-022.235804e-02-5.142873e-02...-2.945017e-026.781943e-03-1.119293e-024.097606e-021.659350e-02-5.213911e-021.342146e-031.124383e-0222.0000000.000000
75%139320.5000001.315642e+008.037239e-011.027196e+007.433413e-016.119264e-013.985649e-015.704361e-013.273459e-015.971390e-01...1.863772e-015.285536e-011.476421e-014.395266e-013.507156e-012.409522e-019.104512e-027.827995e-0277.1650000.000000
max172792.0000002.454930e+002.205773e+019.382558e+001.687534e+013.480167e+017.330163e+011.205895e+022.000721e+011.559499e+01...2.720284e+011.050309e+012.252841e+014.584549e+007.519589e+003.517346e+003.161220e+013.384781e+0125691.1600001.000000
\n", 533 | "

8 rows × 31 columns

\n", 534 | "
" 535 | ], 536 | "text/plain": [ 537 | " Time V1 V2 V3 V4 \\\n", 538 | "count 284807.000000 2.848070e+05 2.848070e+05 2.848070e+05 2.848070e+05 \n", 539 | "mean 94813.859575 1.168375e-15 3.416908e-16 -1.379537e-15 2.074095e-15 \n", 540 | "std 47488.145955 1.958696e+00 1.651309e+00 1.516255e+00 1.415869e+00 \n", 541 | "min 0.000000 -5.640751e+01 -7.271573e+01 -4.832559e+01 -5.683171e+00 \n", 542 | "25% 54201.500000 -9.203734e-01 -5.985499e-01 -8.903648e-01 -8.486401e-01 \n", 543 | "50% 84692.000000 1.810880e-02 6.548556e-02 1.798463e-01 -1.984653e-02 \n", 544 | "75% 139320.500000 1.315642e+00 8.037239e-01 1.027196e+00 7.433413e-01 \n", 545 | "max 172792.000000 2.454930e+00 2.205773e+01 9.382558e+00 1.687534e+01 \n", 546 | "\n", 547 | " V5 V6 V7 V8 V9 \\\n", 548 | "count 2.848070e+05 2.848070e+05 2.848070e+05 2.848070e+05 2.848070e+05 \n", 549 | "mean 9.604066e-16 1.487313e-15 -5.556467e-16 1.213481e-16 -2.406331e-15 \n", 550 | "std 1.380247e+00 1.332271e+00 1.237094e+00 1.194353e+00 1.098632e+00 \n", 551 | "min -1.137433e+02 -2.616051e+01 -4.355724e+01 -7.321672e+01 -1.343407e+01 \n", 552 | "25% -6.915971e-01 -7.682956e-01 -5.540759e-01 -2.086297e-01 -6.430976e-01 \n", 553 | "50% -5.433583e-02 -2.741871e-01 4.010308e-02 2.235804e-02 -5.142873e-02 \n", 554 | "75% 6.119264e-01 3.985649e-01 5.704361e-01 3.273459e-01 5.971390e-01 \n", 555 | "max 3.480167e+01 7.330163e+01 1.205895e+02 2.000721e+01 1.559499e+01 \n", 556 | "\n", 557 | " ... V21 V22 V23 V24 \\\n", 558 | "count ... 2.848070e+05 2.848070e+05 2.848070e+05 2.848070e+05 \n", 559 | "mean ... 1.654067e-16 -3.568593e-16 2.578648e-16 4.473266e-15 \n", 560 | "std ... 7.345240e-01 7.257016e-01 6.244603e-01 6.056471e-01 \n", 561 | "min ... -3.483038e+01 -1.093314e+01 -4.480774e+01 -2.836627e+00 \n", 562 | "25% ... -2.283949e-01 -5.423504e-01 -1.618463e-01 -3.545861e-01 \n", 563 | "50% ... -2.945017e-02 6.781943e-03 -1.119293e-02 4.097606e-02 \n", 564 | "75% ... 1.863772e-01 5.285536e-01 1.476421e-01 4.395266e-01 \n", 565 | "max ... 2.720284e+01 1.050309e+01 2.252841e+01 4.584549e+00 \n", 566 | "\n", 567 | " V25 V26 V27 V28 Amount \\\n", 568 | "count 2.848070e+05 2.848070e+05 2.848070e+05 2.848070e+05 284807.000000 \n", 569 | "mean 5.340915e-16 1.683437e-15 -3.660091e-16 -1.227390e-16 88.349619 \n", 570 | "std 5.212781e-01 4.822270e-01 4.036325e-01 3.300833e-01 250.120109 \n", 571 | "min -1.029540e+01 -2.604551e+00 -2.256568e+01 -1.543008e+01 0.000000 \n", 572 | "25% -3.171451e-01 -3.269839e-01 -7.083953e-02 -5.295979e-02 5.600000 \n", 573 | "50% 1.659350e-02 -5.213911e-02 1.342146e-03 1.124383e-02 22.000000 \n", 574 | "75% 3.507156e-01 2.409522e-01 9.104512e-02 7.827995e-02 77.165000 \n", 575 | "max 7.519589e+00 3.517346e+00 3.161220e+01 3.384781e+01 25691.160000 \n", 576 | "\n", 577 | " Class \n", 578 | "count 284807.000000 \n", 579 | "mean 0.001727 \n", 580 | "std 0.041527 \n", 581 | "min 0.000000 \n", 582 | "25% 0.000000 \n", 583 | "50% 0.000000 \n", 584 | "75% 0.000000 \n", 585 | "max 1.000000 \n", 586 | "\n", 587 | "[8 rows x 31 columns]" 588 | ] 589 | }, 590 | "execution_count": 3, 591 | "metadata": {}, 592 | "output_type": "execute_result" 593 | } 594 | ], 595 | "source": [ 596 | "df.describe()" 597 | ] 598 | }, 599 | { 600 | "cell_type": "markdown", 601 | "id": "1361e4a4-2241-4b04-abd9-17975f7f760b", 602 | "metadata": {}, 603 | "source": [ 604 | "Verify whether the dataset contains null values" 605 | ] 606 | }, 607 | { 608 | "cell_type": "code", 609 | "execution_count": 4, 610 | "id": "f88948fa-8042-40a2-a52d-3c6cbaaa0d93", 611 | "metadata": {}, 612 | "outputs": [ 613 | { 614 | "name": "stdout", 615 | "output_type": "stream", 616 | "text": [ 617 | "No Frauds 99.83 % of the dataset\n", 618 | "Frauds 0.17% of the dataset\n" 619 | ] 620 | } 621 | ], 622 | "source": [ 623 | "print(\n", 624 | " f\"No Frauds {round(df['Class'].value_counts()[0]/len(df) * 100, 2)}\"\n", 625 | " f' % of the dataset'\n", 626 | ")\n", 627 | "print(\n", 628 | " f\"Frauds {round(df['Class'].value_counts()[1]/len(df) * 100, 2)}\"\n", 629 | " f'% of the dataset'\n", 630 | ")" 631 | ] 632 | }, 633 | { 634 | "cell_type": "markdown", 635 | "id": "03f1fca7-00d5-406b-91a1-3b1c0d4a20dc", 636 | "metadata": {}, 637 | "source": [ 638 | "As you can see, most of the transactions are non-fraud. If we use this datase as the base for training our model, then we might get a lot of classification errors and the model will probably assume that most transactions are not fraud. But we don't want our model to assume based on class probabilities, we want our model to detect patterns that give signs of fraud!\n", 639 | "\n", 640 | "The preprocessing step will take care of this problem by balancing the dataset." 641 | ] 642 | }, 643 | { 644 | "cell_type": "markdown", 645 | "id": "6cf6e0e8-d645-441f-9e71-373fe9cae24e", 646 | "metadata": {}, 647 | "source": [ 648 | "## References\n", 649 | "\n", 650 | "* The code used for this exercise is inspired in the following Kaggle notebook https://www.kaggle.com/code/janiobachmann/credit-fraud-dealing-with-imbalanced-datasets\n", 651 | "\n", 652 | "* Data https://www.kaggle.com/code/janiobachmann/credit-fraud-dealing-with-imbalanced-datasets/input" 653 | ] 654 | } 655 | ], 656 | "metadata": { 657 | "kernelspec": { 658 | "display_name": "Python 3.9", 659 | "language": "python", 660 | "name": "python3" 661 | }, 662 | "language_info": { 663 | "codemirror_mode": { 664 | "name": "ipython", 665 | "version": 3 666 | }, 667 | "file_extension": ".py", 668 | "mimetype": "text/x-python", 669 | "name": "python", 670 | "nbconvert_exporter": "python", 671 | "pygments_lexer": "ipython3", 672 | "version": "3.9.16" 673 | } 674 | }, 675 | "nbformat": 4, 676 | "nbformat_minor": 5 677 | } 678 | -------------------------------------------------------------------------------- /7.hands-on-lab/solutions/3.preprocessing.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "c912c1f9-2279-4c6c-b989-9b292e49a504", 6 | "metadata": {}, 7 | "source": [ 8 | "# Data Preprocessing\n", 9 | "\n", 10 | "Run this notebook to preprocess the data.\n", 11 | "This notebook scales the data, deals with data imbalance and generates a training subset.\n", 12 | "\n", 13 | "> **NOTE**: This notebook performs all the necessary steps to preprocess the data before training.\n", 14 | "You do not need to develop additional preprocessing steps.\n", 15 | "\n" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "id": "31da1256-2896-4f06-89fd-717d528698b3", 21 | "metadata": {}, 22 | "source": [ 23 | "Import dependencies:" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 2, 29 | "id": "c952c8f9-5eb2-4435-9e24-0b288b0035c4", 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "from imblearn.over_sampling import SMOTE\n", 34 | "from numpy import save, count_nonzero\n", 35 | "from pandas import read_csv\n", 36 | "from sklearn.model_selection import StratifiedKFold\n", 37 | "from sklearn.preprocessing import RobustScaler" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "4a3a7e18-4a26-4ade-a5c3-4ad563cdd4d7", 43 | "metadata": {}, 44 | "source": [ 45 | "The following code scales the `Amount` and `Time` columns, then builds a training subset that contains a balanced number of each class (`Fraud`(1), `No Fraud`(0)).\n", 46 | "\n", 47 | "The result of preprocessing the data is the `data/training_samples.npy` file, which contains all columns except for the class, and `data/training_labels.npy`, which contains only the class column." 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 3, 53 | "id": "67f70658-4f72-45eb-b0fc-69c172c16b91", 54 | "metadata": {}, 55 | "outputs": [ 56 | { 57 | "name": "stdout", 58 | "output_type": "stream", 59 | "text": [ 60 | "Data processing done!\n" 61 | ] 62 | } 63 | ], 64 | "source": [ 65 | "df = read_csv('data/creditcard.csv')\n", 66 | "\n", 67 | "rob_scaler = RobustScaler()\n", 68 | "\n", 69 | "df['scaled_amount'] = rob_scaler.fit_transform(\n", 70 | " df['Amount'].values.reshape(-1, 1)\n", 71 | ")\n", 72 | "df['scaled_time'] = rob_scaler.fit_transform(\n", 73 | " df['Time'].values.reshape(-1, 1)\n", 74 | ")\n", 75 | "df.drop(['Time', 'Amount'], axis=1, inplace=True)\n", 76 | "scaled_amount = df['scaled_amount']\n", 77 | "scaled_time = df['scaled_time']\n", 78 | "\n", 79 | "df.drop(['scaled_amount', 'scaled_time'], axis=1, inplace=True)\n", 80 | "df.insert(0, 'scaled_amount', scaled_amount)\n", 81 | "df.insert(1, 'scaled_time', scaled_time)\n", 82 | "\n", 83 | "X = df.drop('Class', axis=1)\n", 84 | "y = df['Class']\n", 85 | "sss = StratifiedKFold(n_splits=5, random_state=None, shuffle=False)\n", 86 | "\n", 87 | "for train_index, test_index in sss.split(X, y):\n", 88 | " original_Xtrain = X.iloc[train_index]\n", 89 | " original_ytrain = y.iloc[train_index]\n", 90 | "\n", 91 | "original_Xtrain = original_Xtrain.values\n", 92 | "original_ytrain = original_ytrain.values\n", 93 | "\n", 94 | "sm = SMOTE(sampling_strategy='minority', random_state=42)\n", 95 | "Xsm_train, ysm_train = sm.fit_resample(original_Xtrain, original_ytrain)\n", 96 | "\n", 97 | "save('data/training_samples.npy', Xsm_train)\n", 98 | "save('data/training_labels.npy', ysm_train)\n", 99 | "\n", 100 | "print('Data processing done!')" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 4, 106 | "id": "57e83635-73d4-4ebb-a2f0-d8e7cd9a60a2", 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "name": "stdout", 111 | "output_type": "stream", 112 | "text": [ 113 | "Fraud cases 227452\n", 114 | "No fraud cases 227452\n" 115 | ] 116 | } 117 | ], 118 | "source": [ 119 | "num_frauds = count_nonzero(ysm_train)\n", 120 | "\n", 121 | "print(\"Fraud cases\", num_frauds)\n", 122 | "print(\"No fraud cases\", ysm_train.size - num_frauds)" 123 | ] 124 | } 125 | ], 126 | "metadata": { 127 | "kernelspec": { 128 | "display_name": "Python 3.9", 129 | "language": "python", 130 | "name": "python3" 131 | }, 132 | "language_info": { 133 | "codemirror_mode": { 134 | "name": "ipython", 135 | "version": 3 136 | }, 137 | "file_extension": ".py", 138 | "mimetype": "text/x-python", 139 | "name": "python", 140 | "nbconvert_exporter": "python", 141 | "pygments_lexer": "ipython3", 142 | "version": "3.9.16" 143 | } 144 | }, 145 | "nbformat": 4, 146 | "nbformat_minor": 5 147 | } 148 | -------------------------------------------------------------------------------- /7.hands-on-lab/solutions/4.model_training.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "97b1d6d2-1a32-4a8e-b125-5ea8b389b85d", 6 | "metadata": {}, 7 | "source": [ 8 | "# Model Training\n", 9 | "\n", 10 | "Run this notebook to train a basic neural network by using the preprocessed data and the Keras high-level TensorFlow API.\n", 11 | "\n", 12 | "At the bottom of the notebook, after model training, add the code to convert the trained model to ONNX format." 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "9fdb7bbc-5cc5-4045-b2c6-b98afce790fc", 18 | "metadata": {}, 19 | "source": [ 20 | "First, import the necessary libraries and deactivate GPU acceleration." 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 1, 26 | "id": "e40e8d5c-f870-43df-8968-905f8370c515", 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "name": "stderr", 31 | "output_type": "stream", 32 | "text": [ 33 | "2023-11-08 15:16:28.425790: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", 34 | "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", 35 | "2023-11-08 15:16:30.218302: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64\n", 36 | "2023-11-08 15:16:30.218385: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64\n", 37 | "2023-11-08 15:16:30.218395: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n" 38 | ] 39 | } 40 | ], 41 | "source": [ 42 | "from os import environ\n", 43 | "\n", 44 | "environ['CUDA_VISIBLE_DEVICES'] = '-1'\n", 45 | "\n", 46 | "from keras.models import Sequential\n", 47 | "from keras.layers.core import Dense\n", 48 | "from keras.optimizers import Adam\n", 49 | "from numpy import load\n", 50 | "from onnx import save\n", 51 | "from tf2onnx import convert" 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "id": "698b05ea-47b2-4b2b-8e12-de857916ec14", 57 | "metadata": {}, 58 | "source": [ 59 | "Next, create a basic neural network, and train the network by using the preprocessed data." 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": 2, 65 | "id": "993bd9e1-04af-480a-bd22-c3ba9a23c4d3", 66 | "metadata": {}, 67 | "outputs": [ 68 | { 69 | "name": "stdout", 70 | "output_type": "stream", 71 | "text": [ 72 | "training model\n", 73 | "Epoch 1/20\n", 74 | "1214/1214 - 2s - loss: 0.0726 - accuracy: 0.9718 - val_loss: 0.0263 - val_accuracy: 0.9920 - 2s/epoch - 2ms/step\n", 75 | "Epoch 2/20\n", 76 | "1214/1214 - 2s - loss: 0.0136 - accuracy: 0.9968 - val_loss: 0.0053 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 77 | "Epoch 3/20\n", 78 | "1214/1214 - 2s - loss: 0.0070 - accuracy: 0.9986 - val_loss: 0.0097 - val_accuracy: 0.9995 - 2s/epoch - 1ms/step\n", 79 | "Epoch 4/20\n", 80 | "1214/1214 - 2s - loss: 0.0046 - accuracy: 0.9991 - val_loss: 0.0019 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 81 | "Epoch 5/20\n", 82 | "1214/1214 - 2s - loss: 0.0037 - accuracy: 0.9993 - val_loss: 0.0029 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 83 | "Epoch 6/20\n", 84 | "1214/1214 - 2s - loss: 0.0030 - accuracy: 0.9994 - val_loss: 0.0018 - val_accuracy: 0.9997 - 2s/epoch - 1ms/step\n", 85 | "Epoch 7/20\n", 86 | "1214/1214 - 2s - loss: 0.0025 - accuracy: 0.9995 - val_loss: 0.0027 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 87 | "Epoch 8/20\n", 88 | "1214/1214 - 2s - loss: 0.0023 - accuracy: 0.9996 - val_loss: 8.7773e-04 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 89 | "Epoch 9/20\n", 90 | "1214/1214 - 2s - loss: 0.0020 - accuracy: 0.9996 - val_loss: 0.0013 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 91 | "Epoch 10/20\n", 92 | "1214/1214 - 2s - loss: 0.0019 - accuracy: 0.9996 - val_loss: 2.2378e-04 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 93 | "Epoch 11/20\n", 94 | "1214/1214 - 2s - loss: 0.0015 - accuracy: 0.9997 - val_loss: 0.0097 - val_accuracy: 0.9968 - 2s/epoch - 1ms/step\n", 95 | "Epoch 12/20\n", 96 | "1214/1214 - 2s - loss: 0.0016 - accuracy: 0.9997 - val_loss: 3.1088e-04 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 97 | "Epoch 13/20\n", 98 | "1214/1214 - 2s - loss: 0.0014 - accuracy: 0.9998 - val_loss: 7.1644e-04 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 99 | "Epoch 14/20\n", 100 | "1214/1214 - 2s - loss: 0.0012 - accuracy: 0.9997 - val_loss: 2.2249e-04 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 101 | "Epoch 15/20\n", 102 | "1214/1214 - 2s - loss: 0.0012 - accuracy: 0.9998 - val_loss: 2.4286e-04 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 103 | "Epoch 16/20\n", 104 | "1214/1214 - 2s - loss: 0.0012 - accuracy: 0.9998 - val_loss: 0.0025 - val_accuracy: 0.9993 - 2s/epoch - 1ms/step\n", 105 | "Epoch 17/20\n", 106 | "1214/1214 - 2s - loss: 9.9806e-04 - accuracy: 0.9998 - val_loss: 3.8181e-04 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 107 | "Epoch 18/20\n", 108 | "1214/1214 - 2s - loss: 0.0011 - accuracy: 0.9998 - val_loss: 3.5925e-04 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 109 | "Epoch 19/20\n", 110 | "1214/1214 - 2s - loss: 7.9881e-04 - accuracy: 0.9999 - val_loss: 1.0618e-04 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n", 111 | "Epoch 20/20\n", 112 | "1214/1214 - 2s - loss: 8.3637e-04 - accuracy: 0.9998 - val_loss: 2.0100e-04 - val_accuracy: 1.0000 - 2s/epoch - 1ms/step\n" 113 | ] 114 | }, 115 | { 116 | "data": { 117 | "text/plain": [ 118 | "" 119 | ] 120 | }, 121 | "execution_count": 2, 122 | "metadata": {}, 123 | "output_type": "execute_result" 124 | } 125 | ], 126 | "source": [ 127 | "print('training model')\n", 128 | "\n", 129 | "epoch_count = int(environ.get('epoch_count', '20'))\n", 130 | "learning_rate = float(environ.get('learning_rate', '0.001'))\n", 131 | "\n", 132 | "Xsm_train = load('data/training_samples.npy')\n", 133 | "ysm_train = load('data/training_labels.npy')\n", 134 | "n_inputs = Xsm_train.shape[1]\n", 135 | "\n", 136 | "model = Sequential([\n", 137 | " Dense(n_inputs, input_shape=(n_inputs, ), activation='relu'),\n", 138 | " Dense(32, activation='relu'),\n", 139 | " Dense(2, activation='softmax'),\n", 140 | "])\n", 141 | "model.compile(\n", 142 | " Adam(learning_rate=learning_rate),\n", 143 | " loss='sparse_categorical_crossentropy',\n", 144 | " metrics=['accuracy'],\n", 145 | ")\n", 146 | "model.fit(\n", 147 | " Xsm_train,\n", 148 | " ysm_train,\n", 149 | " validation_split=0.2,\n", 150 | " batch_size=300,\n", 151 | " epochs=epoch_count,\n", 152 | " shuffle=True,\n", 153 | " verbose=2,\n", 154 | ")" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "id": "16cf108f-c4b7-4113-95a8-401105e7d09e", 160 | "metadata": {}, 161 | "source": [ 162 | "# Export the model to ONNX" 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": 3, 168 | "id": "e70bb8cc-c9e8-4fcc-9cd7-9e5d288b564e", 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "onnx_model, _ = convert.from_keras(model)\n", 173 | "save(onnx_model, 'model.onnx')" 174 | ] 175 | } 176 | ], 177 | "metadata": { 178 | "kernelspec": { 179 | "display_name": "Python 3.9", 180 | "language": "python", 181 | "name": "python3" 182 | }, 183 | "language_info": { 184 | "codemirror_mode": { 185 | "name": "ipython", 186 | "version": 3 187 | }, 188 | "file_extension": ".py", 189 | "mimetype": "text/x-python", 190 | "name": "python", 191 | "nbconvert_exporter": "python", 192 | "pygments_lexer": "ipython3", 193 | "version": "3.9.16" 194 | } 195 | }, 196 | "nbformat": 4, 197 | "nbformat_minor": 5 198 | } 199 | -------------------------------------------------------------------------------- /7.hands-on-lab/solutions/5.model_upload.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "9061faa9-aba9-4a5e-a7b6-42273dcb2e4c", 6 | "metadata": {}, 7 | "source": [ 8 | "# Model Upload\n", 9 | "\n", 10 | "Complete and run this notebook to upload the trained model in ONNX format to your S3 bucket.\n", 11 | "You will use the uploaded model in a Model Server." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 2, 17 | "id": "c80315bb-0f4f-4957-b3e1-ce8d306b7926", 18 | "metadata": {}, 19 | "outputs": [ 20 | { 21 | "name": "stdout", 22 | "output_type": "stream", 23 | "text": [ 24 | "File model.onnx uploaded to S3!\n" 25 | ] 26 | } 27 | ], 28 | "source": [ 29 | "import os\n", 30 | "import boto3\n", 31 | "\n", 32 | "onnx_filename = \"model.onnx\"\n", 33 | "\n", 34 | "key_id = os.getenv(\"AWS_ACCESS_KEY_ID\")\n", 35 | "secret_key = os.getenv(\"AWS_SECRET_ACCESS_KEY\")\n", 36 | "endpoint = os.getenv(\"AWS_S3_ENDPOINT\")\n", 37 | "bucket_name = os.getenv(\"AWS_S3_BUCKET\")\n", 38 | "\n", 39 | "s3_client = boto3.client(\n", 40 | " \"s3\",\n", 41 | " aws_access_key_id=key_id,\n", 42 | " aws_secret_access_key=secret_key,\n", 43 | " endpoint_url=endpoint,\n", 44 | " use_ssl=True\n", 45 | ")\n", 46 | "\n", 47 | "s3_client.upload_file(onnx_filename, bucket_name, Key=onnx_filename)\n", 48 | "\n", 49 | "print(f\"File {onnx_filename} uploaded to S3!\")" 50 | ] 51 | } 52 | ], 53 | "metadata": { 54 | "kernelspec": { 55 | "display_name": "Python 3.9.16", 56 | "language": "python", 57 | "name": "python3" 58 | }, 59 | "language_info": { 60 | "codemirror_mode": { 61 | "name": "ipython", 62 | "version": 3 63 | }, 64 | "file_extension": ".py", 65 | "mimetype": "text/x-python", 66 | "name": "python", 67 | "nbconvert_exporter": "python", 68 | "pygments_lexer": "ipython3", 69 | "version": "3.9.16" 70 | } 71 | }, 72 | "nbformat": 4, 73 | "nbformat_minor": 5 74 | } 75 | -------------------------------------------------------------------------------- /7.hands-on-lab/solutions/6.test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "8f556f87-129a-4227-8f8b-43c7412e74d2", 6 | "metadata": {}, 7 | "source": [ 8 | "# Test the Uploaded Model\n", 9 | "\n", 10 | "Complete and run this notebook to test your deployed model." 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 23, 16 | "id": "f7690e15-b191-4899-a250-502a2b066e53", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "import requests\n", 21 | "from numpy import argmax" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 24, 27 | "id": "4033e87d-fe10-4870-8bb1-e3a1c0b758af", 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "INFERENCE_ENDPOINT = \"https://fraud-detection-rhods-intro-s2.apps.rhods-internal.61tk.p1.openshiftapps.com/v2/models/fraud-detection/infer\"" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 33, 37 | "id": "30a55457-b313-4af8-9827-c5dbaebdd01a", 38 | "metadata": {}, 39 | "outputs": [ 40 | { 41 | "data": { 42 | "text/plain": [ 43 | "{'model_name': 'fraud-detection__isvc-97ce9545a4',\n", 44 | " 'model_version': '1',\n", 45 | " 'outputs': [{'name': 'dense_2',\n", 46 | " 'datatype': 'FP32',\n", 47 | " 'shape': [1, 2],\n", 48 | " 'data': [0.9999962, 3.8042233e-06]}]}" 49 | ] 50 | }, 51 | "execution_count": 33, 52 | "metadata": {}, 53 | "output_type": "execute_result" 54 | } 55 | ], 56 | "source": [ 57 | "sample = [\n", 58 | " -0.15384616, -0.9909186, 1.0770786, 0.2849802, 0.0077313827, 1.657073, 0.05202024, 0.44638866, \n", 59 | " -0.40703616, 0.3557039, 0.6260392, -0.92908716, 1.0941651, 0.57956475, -0.8621889, -2.2244275, \n", 60 | " -1.1098708, 0.17238183, 1.7235482, 0.6979903, -0.15358274, -0.14279902, -0.17433698, -0.1741605, \n", 61 | " -0.15337533, -0.46633127, 0.6110009, -0.25287056, 0.090374656, 0.054820385\n", 62 | "]\n", 63 | "\n", 64 | "\n", 65 | "payload = {\n", 66 | " \"inputs\": [\n", 67 | " {\n", 68 | " # The ONNX model requires this name\n", 69 | " \"name\": \"dense_input\",\n", 70 | " \"shape\": [1, 30],\n", 71 | " \"datatype\": \"FP32\",\n", 72 | " \"data\": sample\n", 73 | " }\n", 74 | " ]\n", 75 | "}\n", 76 | "\n", 77 | "response = requests.post(INFERENCE_ENDPOINT, json=payload)\n", 78 | "result = response.json()\n", 79 | "result" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 30, 85 | "id": "5bfbaf0d-b14e-44cd-af51-cfb0911a406d", 86 | "metadata": {}, 87 | "outputs": [ 88 | { 89 | "name": "stdout", 90 | "output_type": "stream", 91 | "text": [ 92 | "The sample was classified as \"No Fraud\".\n" 93 | ] 94 | } 95 | ], 96 | "source": [ 97 | "labels = ['No Fraud', 'Fraud']\n", 98 | "model_output = result['outputs'][0]['data']\n", 99 | "classification_result = labels[argmax(model_output)]\n", 100 | "\n", 101 | "print(f'The sample was classified as \"{classification_result}\".')" 102 | ] 103 | } 104 | ], 105 | "metadata": { 106 | "kernelspec": { 107 | "display_name": "Python 3.9", 108 | "language": "python", 109 | "name": "python3" 110 | }, 111 | "language_info": { 112 | "codemirror_mode": { 113 | "name": "ipython", 114 | "version": 3 115 | }, 116 | "file_extension": ".py", 117 | "mimetype": "text/x-python", 118 | "name": "python", 119 | "nbconvert_exporter": "python", 120 | "pygments_lexer": "ipython3", 121 | "version": "3.9.16" 122 | } 123 | }, 124 | "nbformat": 4, 125 | "nbformat_minor": 5 126 | } 127 | -------------------------------------------------------------------------------- /7.hands-on-lab/solutions/mypipeline.pipeline: -------------------------------------------------------------------------------- 1 | { 2 | "doc_type": "pipeline", 3 | "version": "3.0", 4 | "json_schema": "http://api.dataplatform.ibm.com/schemas/common-pipeline/pipeline-flow/pipeline-flow-v3-schema.json", 5 | "id": "elyra-auto-generated-pipeline", 6 | "primary_pipeline": "primary", 7 | "pipelines": [ 8 | { 9 | "id": "primary", 10 | "nodes": [ 11 | { 12 | "id": "0f504a05-f5d6-4403-8ecc-2156ce392dd1", 13 | "type": "execution_node", 14 | "op": "execute-notebook-node", 15 | "app_data": { 16 | "component_parameters": { 17 | "dependencies": [], 18 | "include_subdirectories": false, 19 | "outputs": [ 20 | "data/training_labels.npy", 21 | "data/training_samples.npy" 22 | ], 23 | "env_vars": [], 24 | "kubernetes_pod_annotations": [], 25 | "kubernetes_pod_labels": [], 26 | "kubernetes_secrets": [], 27 | "kubernetes_shared_mem_size": {}, 28 | "kubernetes_tolerations": [], 29 | "mounted_volumes": [], 30 | "filename": "3.preprocessing.ipynb", 31 | "runtime_image": "quay.io/mmurakam/runtimes:fraud-detection-v0.1.0" 32 | }, 33 | "label": "", 34 | "ui_data": { 35 | "label": "3.preprocessing.ipynb", 36 | "image": "/notebook/rhods-intro-s2/hit-roads/static/elyra/notebook.svg", 37 | "x_pos": 243, 38 | "y_pos": 159, 39 | "description": "Run notebook file" 40 | } 41 | }, 42 | "inputs": [ 43 | { 44 | "id": "inPort", 45 | "app_data": { 46 | "ui_data": { 47 | "cardinality": { 48 | "min": 0, 49 | "max": -1 50 | }, 51 | "label": "Input Port" 52 | } 53 | }, 54 | "links": [ 55 | { 56 | "id": "70e59903-ad67-433b-b9d7-83394ce8dd48", 57 | "node_id_ref": "80ff689d-da82-44ad-9056-048ddc77948e", 58 | "port_id_ref": "outPort" 59 | } 60 | ] 61 | } 62 | ], 63 | "outputs": [ 64 | { 65 | "id": "outPort", 66 | "app_data": { 67 | "ui_data": { 68 | "cardinality": { 69 | "min": 0, 70 | "max": -1 71 | }, 72 | "label": "Output Port" 73 | } 74 | } 75 | } 76 | ] 77 | }, 78 | { 79 | "id": "b0a53172-431d-4df3-969b-f191e2eb63b4", 80 | "type": "execution_node", 81 | "op": "execute-notebook-node", 82 | "app_data": { 83 | "component_parameters": { 84 | "dependencies": [], 85 | "include_subdirectories": false, 86 | "outputs": [ 87 | "model.onnx" 88 | ], 89 | "env_vars": [], 90 | "kubernetes_pod_annotations": [], 91 | "kubernetes_pod_labels": [], 92 | "kubernetes_secrets": [], 93 | "kubernetes_shared_mem_size": {}, 94 | "kubernetes_tolerations": [], 95 | "mounted_volumes": [], 96 | "filename": "4.model_training.ipynb", 97 | "runtime_image": "quay.io/mmurakam/runtimes:fraud-detection-v0.1.0" 98 | }, 99 | "label": "", 100 | "ui_data": { 101 | "label": "4.model_training.ipynb", 102 | "image": "/notebook/rhods-intro-s2/hit-roads/static/elyra/notebook.svg", 103 | "x_pos": 491, 104 | "y_pos": 160, 105 | "description": "Run notebook file" 106 | } 107 | }, 108 | "inputs": [ 109 | { 110 | "id": "inPort", 111 | "app_data": { 112 | "ui_data": { 113 | "cardinality": { 114 | "min": 0, 115 | "max": -1 116 | }, 117 | "label": "Input Port" 118 | } 119 | }, 120 | "links": [ 121 | { 122 | "id": "1c1f6343-575f-47fa-a88a-be7c52abd80b", 123 | "node_id_ref": "0f504a05-f5d6-4403-8ecc-2156ce392dd1", 124 | "port_id_ref": "outPort" 125 | } 126 | ] 127 | } 128 | ], 129 | "outputs": [ 130 | { 131 | "id": "outPort", 132 | "app_data": { 133 | "ui_data": { 134 | "cardinality": { 135 | "min": 0, 136 | "max": -1 137 | }, 138 | "label": "Output Port" 139 | } 140 | } 141 | } 142 | ] 143 | }, 144 | { 145 | "id": "bddf0237-d9fd-4f76-8c46-36b9dd975610", 146 | "type": "execution_node", 147 | "op": "execute-notebook-node", 148 | "app_data": { 149 | "component_parameters": { 150 | "dependencies": [], 151 | "include_subdirectories": false, 152 | "outputs": [], 153 | "env_vars": [], 154 | "kubernetes_pod_annotations": [], 155 | "kubernetes_pod_labels": [], 156 | "kubernetes_secrets": [], 157 | "kubernetes_shared_mem_size": {}, 158 | "kubernetes_tolerations": [], 159 | "mounted_volumes": [], 160 | "filename": "5.model_upload.ipynb", 161 | "runtime_image": "quay.io/mmurakam/runtimes:fraud-detection-v0.1.0" 162 | }, 163 | "label": "", 164 | "ui_data": { 165 | "label": "5.model_upload.ipynb", 166 | "image": "/notebook/rhods-intro-s2/hit-roads/static/elyra/notebook.svg", 167 | "x_pos": 724, 168 | "y_pos": 161, 169 | "description": "Run notebook file" 170 | } 171 | }, 172 | "inputs": [ 173 | { 174 | "id": "inPort", 175 | "app_data": { 176 | "ui_data": { 177 | "cardinality": { 178 | "min": 0, 179 | "max": -1 180 | }, 181 | "label": "Input Port" 182 | } 183 | }, 184 | "links": [ 185 | { 186 | "id": "8ac2b35c-e15d-4537-b122-bdd3b2d655b9", 187 | "node_id_ref": "b0a53172-431d-4df3-969b-f191e2eb63b4", 188 | "port_id_ref": "outPort" 189 | } 190 | ] 191 | } 192 | ], 193 | "outputs": [ 194 | { 195 | "id": "outPort", 196 | "app_data": { 197 | "ui_data": { 198 | "cardinality": { 199 | "min": 0, 200 | "max": -1 201 | }, 202 | "label": "Output Port" 203 | } 204 | } 205 | } 206 | ] 207 | }, 208 | { 209 | "id": "80ff689d-da82-44ad-9056-048ddc77948e", 210 | "type": "execution_node", 211 | "op": "execute-notebook-node", 212 | "app_data": { 213 | "component_parameters": { 214 | "dependencies": [], 215 | "include_subdirectories": false, 216 | "outputs": [ 217 | "data/creditcard.csv" 218 | ], 219 | "env_vars": [], 220 | "kubernetes_pod_annotations": [], 221 | "kubernetes_pod_labels": [], 222 | "kubernetes_secrets": [], 223 | "kubernetes_shared_mem_size": {}, 224 | "kubernetes_tolerations": [], 225 | "mounted_volumes": [], 226 | "filename": "1.collect.ipynb", 227 | "runtime_image": "quay.io/mmurakam/runtimes:fraud-detection-v0.1.0" 228 | }, 229 | "label": "", 230 | "ui_data": { 231 | "label": "1.collect.ipynb", 232 | "image": "/notebook/rhods-intro-s2/hit-roads/static/elyra/notebook.svg", 233 | "x_pos": 4, 234 | "y_pos": 163, 235 | "description": "Run notebook file" 236 | } 237 | }, 238 | "inputs": [ 239 | { 240 | "id": "inPort", 241 | "app_data": { 242 | "ui_data": { 243 | "cardinality": { 244 | "min": 0, 245 | "max": -1 246 | }, 247 | "label": "Input Port" 248 | } 249 | } 250 | } 251 | ], 252 | "outputs": [ 253 | { 254 | "id": "outPort", 255 | "app_data": { 256 | "ui_data": { 257 | "cardinality": { 258 | "min": 0, 259 | "max": -1 260 | }, 261 | "label": "Output Port" 262 | } 263 | } 264 | } 265 | ] 266 | } 267 | ], 268 | "app_data": { 269 | "ui_data": { 270 | "comments": [] 271 | }, 272 | "version": 8, 273 | "runtime_type": "KUBEFLOW_PIPELINES", 274 | "properties": { 275 | "name": "mypipeline", 276 | "runtime": "Data Science Pipelines", 277 | "pipeline_defaults": { 278 | "kubernetes_shared_mem_size": {}, 279 | "mounted_volumes": [], 280 | "kubernetes_pod_annotations": [], 281 | "kubernetes_tolerations": [], 282 | "kubernetes_pod_labels": [], 283 | "env_vars": [], 284 | "kubernetes_secrets": [ 285 | { 286 | "env_var": "AWS_ACCESS_KEY_ID", 287 | "name": "aws-connection-minio", 288 | "key": "AWS_ACCESS_KEY_ID" 289 | }, 290 | { 291 | "env_var": "AWS_S3_BUCKET", 292 | "name": "aws-connection-minio", 293 | "key": "AWS_S3_BUCKET" 294 | }, 295 | { 296 | "env_var": "AWS_S3_ENDPOINT", 297 | "name": "aws-connection-minio", 298 | "key": "AWS_S3_ENDPOINT" 299 | }, 300 | { 301 | "env_var": "AWS_SECRET_ACCESS_KEY", 302 | "name": "aws-connection-minio", 303 | "key": "AWS_SECRET_ACCESS_KEY" 304 | } 305 | ], 306 | "runtime_image": "quay.io/mmurakam/runtimes:fraud-detection-v0.1.0" 307 | } 308 | } 309 | }, 310 | "runtime_ref": "" 311 | } 312 | ], 313 | "schemas": [] 314 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This repository contains the source code for the 6 RHODS quick courses. Create appropriately named folder here and then put your notebooks, code and snippets in this repo and refer to them in your hands-on labs 2 | --------------------------------------------------------------------------------