├── .gitignore ├── README.md ├── images ├── pt_bert.png ├── pt_gpu_vs_cpu.png ├── pt_samples_sec.png └── tf_resnet50_results.png ├── pytorch ├── M1Pro_pytorch.ipynb ├── README.md ├── pets.py ├── setup.sh ├── train_bert.py ├── train_pets.py ├── train_pets_no_wandb.py └── utils.py └── tensorflow ├── Basic_Performance.ipynb ├── README.md ├── keras_cvp.py ├── tf_apple.yml ├── tf_linux.yml ├── train_bert.py ├── train_pets.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | Gemfile.lock 2 | collect_env.py 3 | tmp* 4 | Untitled*.ipynb 5 | *.bak 6 | token 7 | .idea/ 8 | docs/ 9 | conda/ 10 | tmp/ 11 | 12 | tags 13 | *~ 14 | ~* 15 | *.swp 16 | .gitconfig 17 | # Byte-compiled / optimized / DLL files 18 | __pycache__/ 19 | *.py[cod] 20 | *$py.class 21 | 22 | # C extensions 23 | *.so 24 | 25 | # Distribution / packaging 26 | .Python 27 | build/ 28 | develop-eggs/ 29 | dist/ 30 | downloads/ 31 | eggs/ 32 | .eggs/ 33 | lib/ 34 | lib64/ 35 | parts/ 36 | sdist/ 37 | var/ 38 | wheels/ 39 | *.egg-info/ 40 | .installed.cfg 41 | *.egg 42 | MANIFEST 43 | 44 | # PyInstaller 45 | # Usually these files are written by a python script from a template 46 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 47 | *.manifest 48 | *.spec 49 | 50 | # Installer logs 51 | pip-log.txt 52 | pip-delete-this-directory.txt 53 | 54 | # Unit test / coverage reports 55 | htmlcov/ 56 | .tox/ 57 | .coverage 58 | .coverage.* 59 | .cache 60 | nosetests.xml 61 | coverage.xml 62 | *.cover 63 | .hypothesis/ 64 | .pytest_cache/ 65 | 66 | # Translations 67 | *.mo 68 | *.pot 69 | 70 | # Django stuff: 71 | *.log 72 | local_settings.py 73 | db.sqlite3 74 | 75 | # Flask stuff: 76 | instance/ 77 | .webassets-cache 78 | 79 | # Scrapy stuff: 80 | .scrapy 81 | 82 | # Sphinx documentation 83 | docs/_build/ 84 | 85 | # PyBuilder 86 | target/ 87 | 88 | # Jupyter Notebook 89 | .ipynb_checkpoints 90 | 91 | # pyenv 92 | .python-version 93 | 94 | .vscode/ 95 | 96 | # celery beat schedule file 97 | celerybeat-schedule 98 | 99 | # SageMath parsed files 100 | *.sage.py 101 | 102 | # Environments 103 | .env 104 | .venv 105 | env/ 106 | venv/ 107 | ENV/ 108 | env.bak/ 109 | venv.bak/ 110 | 111 | # Spyder project settings 112 | .spyderproject 113 | .spyproject 114 | 115 | # Rope project settings 116 | .ropeproject 117 | 118 | # mkdocs documentation 119 | /site 120 | 121 | # mypy 122 | .mypy_cache/ 123 | 124 | # Mac stuff 125 | .DS_Store 126 | 127 | # Wandb 128 | wandb 129 | MNIST 130 | data 131 | *.onnx 132 | *.pt 133 | artifacts 134 | tensorflow/test_tf.py -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Apple Silicon DL benchmarks 2 | 3 | Currently we have PyTorch and Tensorflow that have Metal backend. 4 | 5 | ## Results 6 | Varied results across frameworks: 7 | - [Apple M1Pro Pytorch Training Results](https://wandb.me/pytorch_m1) 8 | - [Apple M1Pro Tensorflow Training Results](https://wandb.me/m1pro) 9 | 10 | ### Tensorflow Resnet50: 11 | ![tf_resnet_50results.png](images/tf_resnet50_results.png) 12 | 13 | ### PyTorch Resnet50: 14 | - Difference between CPU and GPU 15 | ![gpu_vs_cpu.png](images/pt_gpu_vs_cpu.png) 16 | - Comparing with Nvidia 17 | ![samples_sec.png](images/pt_samples_sec.png) 18 | 19 | ### PyTorch Bert 20 | - Running a Bert from Huggingface 21 | ![pt_bert.png](images/pt_bert.png) 22 | 23 | 24 | ## Pytorch 25 | We have official PyTorch support! check [pytorch](pytorch) folder to start running your benchmarks 26 | 27 | 28 | 29 | ## Tensorflow 30 | You can run tensorflow benchmarks by going to the [tensorflow](tensorflow) folder. 31 | -------------------------------------------------------------------------------- /images/pt_bert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tcapelle/apple_m1_pro_python/a8e6462b1a6b82beb8d8c22a9157435126fe6ed7/images/pt_bert.png -------------------------------------------------------------------------------- /images/pt_gpu_vs_cpu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tcapelle/apple_m1_pro_python/a8e6462b1a6b82beb8d8c22a9157435126fe6ed7/images/pt_gpu_vs_cpu.png -------------------------------------------------------------------------------- /images/pt_samples_sec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tcapelle/apple_m1_pro_python/a8e6462b1a6b82beb8d8c22a9157435126fe6ed7/images/pt_samples_sec.png -------------------------------------------------------------------------------- /images/tf_resnet50_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tcapelle/apple_m1_pro_python/a8e6462b1a6b82beb8d8c22a9157435126fe6ed7/images/tf_resnet50_results.png -------------------------------------------------------------------------------- /pytorch/M1Pro_pytorch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "bbbc3c4b", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "graphics_output = \"\"\"\n", 11 | "Graphics/Displays:\n", 12 | "\n", 13 | " Apple M1 Pro:\n", 14 | "\n", 15 | " Chipset Model: Apple M1 Pro\n", 16 | " Type: GPU\n", 17 | " Bus: Built-In\n", 18 | " Total Number of Cores: 16\n", 19 | " Vendor: Apple (0x106b)\n", 20 | " Metal Support: Metal 3\n", 21 | " Displays:\n", 22 | " Color LCD:\n", 23 | " Display Type: Built-in Liquid Retina XDR Display\n", 24 | " Resolution: 3024 x 1964 Retina\n", 25 | " Main Display: Yes\n", 26 | " Mirror: Off\n", 27 | " Online: Yes\n", 28 | " Automatically Adjust Brightness: Yes\n", 29 | " Connection Type: Internal\n", 30 | "\"\"\"" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 6, 36 | "id": "6a0b1c82", 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "import subprocess" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 48, 46 | "id": "c854f94f", 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "import re, subprocess\n", 51 | "\n", 52 | "cpu_info = subprocess.run([\"system_profiler\",\"SPHardwareDataType\"], stdout=subprocess.PIPE).stdout.decode(\"utf-8\")\n", 53 | "gpu_info = subprocess.run([\"system_profiler\",\"SPDisplaysDataType\"], stdout=subprocess.PIPE).stdout.decode(\"utf-8\") " 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 46, 59 | "id": "331e1f19", 60 | "metadata": {}, 61 | "outputs": [ 62 | { 63 | "name": "stdout", 64 | "output_type": "stream", 65 | "text": [ 66 | "Apple M1 Pro 10 16\n" 67 | ] 68 | } 69 | ], 70 | "source": [ 71 | "cpu = re.search(r'Chip:\\s+(.+)', cpu_info).group(1)\n", 72 | "cpu_cores = re.search(r'Number of Cores:\\s+(\\d+)', cpu_info).group(1)\n", 73 | "memory = re.search(r'Memory:\\s+(\\d+)\\s+GB', cpu_info).group(1)\n", 74 | "\n", 75 | "print(cpu, cpu_cores, memory)\n" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 49, 81 | "id": "a473e3b2", 82 | "metadata": {}, 83 | "outputs": [ 84 | { 85 | "name": "stdout", 86 | "output_type": "stream", 87 | "text": [ 88 | "Apple M1 Pro 16\n" 89 | ] 90 | } 91 | ], 92 | "source": [ 93 | "\n", 94 | "\n", 95 | "gpu = re.search(r'Chipset Model:\\s+(.+)', gpu_info).group(1)\n", 96 | "gpu_cores = re.search(r'Total Number of Cores:\\s+(\\d+)', gpu_info).group(1)\n", 97 | "\n", 98 | "print(gpu, gpu_cores)" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "id": "3ecc2bbc", 105 | "metadata": {}, 106 | "outputs": [], 107 | "source": [] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "id": "fb1d57be", 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": null, 120 | "id": "2b55750d-d78c-489d-a8f3-0615853605bf", 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "import torchvision as tv" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "id": "fea3b49c-33a9-479d-b9d2-d7304eed50e3", 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "from torchvision.io import read_image" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": 23, 140 | "id": "419cc07c-1072-4602-9095-28d9b29ecc1d", 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [ 144 | "from tqdm.notebook import tqdm\n", 145 | "from train_pets import get_pets, get_dataloader, Pets" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 24, 151 | "id": "52a77d25-55c5-48d0-9a18-b9b4c13fe28f", 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "NUM_WORKERS = 0\n", 156 | "BATCH_SIZE = 64\n", 157 | "IMAGE_SIZE = 128" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 25, 163 | "id": "ccfcef3b-2ef8-4a64-8189-898d43d8ab9b", 164 | "metadata": {}, 165 | "outputs": [ 166 | { 167 | "name": "stderr", 168 | "output_type": "stream", 169 | "text": [ 170 | "\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact PETS:v1, 783.01MB. 25864 files... Done. 0:0:0\n" 171 | ] 172 | } 173 | ], 174 | "source": [ 175 | "dataset_path = get_pets()" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": null, 181 | "id": "f31adfbc-d9cd-429e-8d4f-cc4ce92c62d3", 182 | "metadata": {}, 183 | "outputs": [], 184 | "source": [ 185 | "ds = Pets(dataset_path, 128)" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": 36, 191 | "id": "789271c3-0bce-4c03-8abd-e4beadb0476c", 192 | "metadata": {}, 193 | "outputs": [], 194 | "source": [ 195 | "file = str(ds.files[0])" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": null, 201 | "id": "bfb61077-c0bf-46b8-a475-674d96b34698", 202 | "metadata": {}, 203 | "outputs": [ 204 | { 205 | "ename": "RuntimeError", 206 | "evalue": "isString()INTERNAL ASSERT FAILED at \"/Users/malfet/miniconda3/envs/whl-py310-torch-1.11.0/lib/python3.10/site-packages/torch/include/ATen/core/ivalue_inl.h\":2088, please report a bug to PyTorch. Expected String but got Blob", 207 | "output_type": "error", 208 | "traceback": [ 209 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 210 | "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", 211 | "Input \u001b[0;32mIn [32]\u001b[0m, in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mread_image\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n", 212 | "File \u001b[0;32m~/miniforge3/envs/pytorch/lib/python3.10/site-packages/torchvision/io/image.py:245\u001b[0m, in \u001b[0;36mread_image\u001b[0;34m(path, mode)\u001b[0m\n\u001b[1;32m 243\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mjit\u001b[38;5;241m.\u001b[39mis_scripting() \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mjit\u001b[38;5;241m.\u001b[39mis_tracing():\n\u001b[1;32m 244\u001b[0m _log_api_usage_once(read_image)\n\u001b[0;32m--> 245\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[43mread_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 246\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m decode_image(data, mode)\n", 213 | "File \u001b[0;32m~/miniforge3/envs/pytorch/lib/python3.10/site-packages/torchvision/io/image.py:47\u001b[0m, in \u001b[0;36mread_file\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mjit\u001b[38;5;241m.\u001b[39mis_scripting() \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mjit\u001b[38;5;241m.\u001b[39mis_tracing():\n\u001b[1;32m 46\u001b[0m _log_api_usage_once(read_file)\n\u001b[0;32m---> 47\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mops\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mimage\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 48\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m data\n", 214 | "File \u001b[0;32m~/miniforge3/envs/pytorch/lib/python3.10/site-packages/torch/_ops.py:143\u001b[0m, in \u001b[0;36mOpOverloadPacket.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 138\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 139\u001b[0m \u001b[38;5;66;03m# overloading __call__ to ensure torch.ops.foo.bar()\u001b[39;00m\n\u001b[1;32m 140\u001b[0m \u001b[38;5;66;03m# is still callable from JIT\u001b[39;00m\n\u001b[1;32m 141\u001b[0m \u001b[38;5;66;03m# We save the function ptr as the `op` attribute on\u001b[39;00m\n\u001b[1;32m 142\u001b[0m \u001b[38;5;66;03m# OpOverloadPacket to access it here.\u001b[39;00m\n\u001b[0;32m--> 143\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_op\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\n", 215 | "\u001b[0;31mRuntimeError\u001b[0m: isString()INTERNAL ASSERT FAILED at \"/Users/malfet/miniconda3/envs/whl-py310-torch-1.11.0/lib/python3.10/site-packages/torch/include/ATen/core/ivalue_inl.h\":2088, please report a bug to PyTorch. Expected String but got Blob" 216 | ] 217 | } 218 | ], 219 | "source": [ 220 | "read_image()" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": 4, 226 | "id": "01bcdc90-7e93-4d7b-85fb-785e1cf39edf", 227 | "metadata": {}, 228 | "outputs": [], 229 | "source": [ 230 | "dl = get_dataloader(dataset_path, batch_size=BATCH_SIZE, image_size=IMAGE_SIZE, num_workers=NUM_WORKERS)" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": 10, 236 | "id": "365d8717-40c2-44e5-93db-ebd060f4980e", 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [ 240 | "def cycle_dl(dl, n=1):\n", 241 | " for _ in range(n):\n", 242 | " for b in tqdm(dl):\n", 243 | " pass" 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "execution_count": 13, 249 | "id": "77e44c7c-b684-42e4-a0bb-87a37510333e", 250 | "metadata": {}, 251 | "outputs": [ 252 | { 253 | "data": { 254 | "application/vnd.jupyter.widget-view+json": { 255 | "model_id": "df30374fc1be4afa827bc7ef32dd657f", 256 | "version_major": 2, 257 | "version_minor": 0 258 | }, 259 | "text/plain": [ 260 | " 0%| | 0/116 [00:00 The `setup.sh` script takes care of installing all the above 38 | 39 | ```bash 40 | $ sh setup.sh 41 | ``` 42 | 43 | > For Linux, replace the MiniForge binaries with the ones for Linux: 44 | https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh 45 | 46 | ## Verify your Install 47 | Run the following in python 48 | 49 | ```python 50 | import torch 51 | 52 | 53 | torch.__version__ 54 | >>> '1.13.1' 55 | 56 | 57 | torch.tensor([1,2,3], device="mps") 58 | ``` 59 | 60 | ## Running the Benchmark 61 | In your terminal run: 62 | 63 | ```bash 64 | $ python train_pets.py --gpu_name="M1Pro GPU 16 Cores" #replace with your GPU name 65 | ``` 66 | 67 | - Pass the `--gpu_name` flag to group the runs, I am not able to detect this automatically on Apple. 68 | - To run on cpu pass `--device="cpu"` or for CUDA `--device="cuda"` (you need a linux PC with an Nvidia GPU) 69 | - You can also pass other params, and play with different `batch_size` and `model_name`. 70 | 71 | 72 | ## Bert Benchmark 73 | 74 | In your terminal run: 75 | 76 | ```bash 77 | $ python train_bert.py --gpu_name="M1Pro GPU 16 Cores" #replace with your GPU name 78 | ``` 79 | 80 | ## Notes 81 | 82 | You have many parameters available to override, call the script with the flag `--help` to see them all. To extract the maximum performance of you GPU you may need to adjust `batch_size` and maybe run for longer than 1 `epochs`. 83 | ``` 84 | (pt) thomascapelle@mac-mini pytorch % python train_pets.py --help 85 | usage: train_pets.py [-h] [--entity ENTITY] [--batch_size BATCH_SIZE] [--epochs EPOCHS] 86 | [--num_experiments NUM_EXPERIMENTS] [--learning_rate LEARNING_RATE] 87 | [--image_size IMAGE_SIZE] [--model_name MODEL_NAME] [--dataset DATASET] 88 | [--device DEVICE] [--gpu_name GPU_NAME] [--num_workers NUM_WORKERS] 89 | [--mixed_precision] [--channels_last] [--optimizer OPTIMIZER] 90 | 91 | options: 92 | -h, --help show this help message and exit 93 | --entity ENTITY 94 | --batch_size BATCH_SIZE 95 | --epochs EPOCHS 96 | --num_experiments NUM_EXPERIMENTS 97 | --learning_rate LEARNING_RATE 98 | --image_size IMAGE_SIZE 99 | --model_name MODEL_NAME 100 | --dataset DATASET 101 | --device DEVICE 102 | --gpu_name GPU_NAME 103 | --num_workers NUM_WORKERS 104 | --mixed_precision 105 | --channels_last 106 | --optimizer OPTIMIZER 107 | ``` -------------------------------------------------------------------------------- /pytorch/pets.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from pathlib import Path 4 | 5 | import wandb 6 | 7 | 8 | from PIL import Image 9 | 10 | import torch 11 | import torchvision.transforms as T 12 | 13 | 14 | def get_pets(version="v3"): 15 | api = wandb.Api() 16 | at = api.artifact(f'capecape/pytorch-M1Pro/PETS:{version}', type='dataset') 17 | dataset_path = at.download() 18 | return dataset_path 19 | 20 | class Pets(torch.utils.data.Dataset): 21 | pat = r'(^[a-zA-Z]+_*[a-zA-Z]+)' 22 | vocab = ['Abyssinian', 'Bengal', 'Birman', 'Bombay', 'British_Shorthair', 'Egyptian_Mau', 'Maine_Coon', 23 | 'Persian', 'Ragdoll', 'Russian_Blue', 'Siamese', 'Sphynx', 'american_bulldog', 'american_pit', 24 | 'basset_hound', 'beagle', 'boxer', 'chihuahua', 'english_cocker', 'english_setter', 'german_shorthaired', 25 | 'great_pyrenees', 'havanese', 'japanese_chin', 'keeshond', 'leonberger', 'miniature_pinscher', 'newfoundland', 26 | 'pomeranian', 'pug', 'saint_bernard', 'samoyed', 'scottish_terrier', 'shiba_inu', 'staffordshire_bull', 27 | 'wheaten_terrier', 'yorkshire_terrier'] 28 | vocab_map = {v:i for i,v in enumerate(vocab)} 29 | 30 | 31 | def __init__(self, pets_path, image_size=224): 32 | self.path = Path(pets_path) 33 | self.files = list(self.path.glob("images/*.jpg")) 34 | self.tfms =T.Compose([T.Resize((image_size, image_size)), T.ToTensor()]) 35 | self.vocab_map = {v:i for i, v in enumerate(self.vocab)} 36 | 37 | @staticmethod 38 | def load_image(fn, mode="RGB"): 39 | "Open and load a `PIL.Image` and convert to `mode`" 40 | im = Image.open(fn) 41 | im.load() 42 | im = im._new(im.im) 43 | return im.convert(mode) if mode else im 44 | 45 | def __getitem__(self, idx): 46 | file = self.files[idx] 47 | return self.tfms(self.load_image(str(file))), self.vocab_map[re.match(self.pat, file.name)[0]] 48 | 49 | def __len__(self): return len(self.files) 50 | 51 | 52 | 53 | def get_pets_dataloader(batch_size, image_size=224, num_workers=0, **kwargs): 54 | "Get a training dataloader" 55 | dataset_path = get_pets() 56 | ds = Pets(dataset_path, image_size=image_size) 57 | loader = torch.utils.data.DataLoader(ds, 58 | batch_size=batch_size, 59 | pin_memory=True, 60 | num_workers=int(num_workers), 61 | **kwargs) 62 | return loader 63 | 64 | class OneBatchDataLoader: 65 | def __init__(self, dl, N=100): 66 | self.dl = dl 67 | self.batch = next(iter(dl)) 68 | self.N = N 69 | self.dataset = dl.dataset 70 | 71 | def __iter__(self): 72 | for i in range(self.N): 73 | yield self.batch 74 | 75 | def __len__(self): 76 | return self.N 77 | 78 | 79 | def get_fast_pets_dataloader(batch_size, image_size=224, num_workers=0, N=100, **kwargs): 80 | "Get a training dataloader" 81 | dl = get_pets_dataloader(batch_size, image_size=image_size, num_workers=num_workers, **kwargs) 82 | return OneBatchDataLoader(dl, N=N) -------------------------------------------------------------------------------- /pytorch/setup.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | echo "Setting up your Mac for Pytorch" 4 | echo "===============================" 5 | 6 | echo "Downloading MiniForge" 7 | curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh 8 | sh Miniforge3-MacOSX-arm64.sh 9 | 10 | 11 | echo "Installing PyTorch" 12 | conda create --name="pt" "python<3.11" 13 | conda activate pt 14 | conda install pytorch torchvision torchaudio -c pytorch 15 | pip install wandb tqdm 16 | 17 | echo "Installing Huggingface Stack" 18 | pip install transformers datasets 19 | -------------------------------------------------------------------------------- /pytorch/train_bert.py: -------------------------------------------------------------------------------- 1 | ## Author: Thomas Capelle, Soumik Rakshit 2 | ## Mail: tcapelle@wandb.com, soumik.rakshit@wandb.com 3 | 4 | """"Benchmarking apple M1Pro with Tensorflow 5 | @wandbcode{apple_m1_pro}""" 6 | 7 | 8 | import torch, wandb, argparse 9 | from types import SimpleNamespace 10 | from torch.utils.data import DataLoader 11 | from transformers import AutoModelForSequenceClassification, AutoTokenizer, default_data_collator 12 | from datasets import load_dataset 13 | 14 | from utils import MicroTrainer, get_gpu_name 15 | 16 | 17 | PROJECT = "pytorch-M1Pro" 18 | ENTITY = "capecape" 19 | GROUP = "pytorch" 20 | 21 | 22 | config_defaults = SimpleNamespace( 23 | batch_size=16, 24 | epochs=1, 25 | num_experiments=1, 26 | learning_rate=1e-3, 27 | model_name="bert-base-cased", 28 | dataset="yelp_review_full", 29 | device="mps", 30 | gpu_name=get_gpu_name(), 31 | num_workers=8, 32 | mixed_precision=False, 33 | syncro=False, 34 | inference_only=False, 35 | compile=False, 36 | ) 37 | 38 | def parse_args(): 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument('--batch_size', type=int, default=config_defaults.batch_size) 41 | parser.add_argument('--epochs', type=int, default=config_defaults.epochs) 42 | parser.add_argument('--num_experiments', type=int, default=config_defaults.num_experiments) 43 | parser.add_argument('--learning_rate', type=float, default=config_defaults.learning_rate) 44 | parser.add_argument('--model_name', type=str, default=config_defaults.model_name) 45 | parser.add_argument('--dataset', type=str, default=config_defaults.dataset) 46 | parser.add_argument('--device', type=str, default=config_defaults.device) 47 | parser.add_argument('--gpu_name', type=str, default=config_defaults.gpu_name) 48 | parser.add_argument('--num_workers', type=int, default=config_defaults.num_workers) 49 | parser.add_argument('--inference_only', action="store_true") 50 | parser.add_argument('--mixed_precision', action="store_true") 51 | parser.add_argument('--compile', action="store_true") 52 | parser.add_argument('--tags', type=str, default=None) 53 | parser.add_argument('--syncro', action="store_true") 54 | return parser.parse_args() 55 | 56 | 57 | def get_dls(model_name="bert-base-cased", dataset_name="yelp_review_full", batch_size=8, num_workers=0, sample_size=2048): 58 | 59 | # download and prepare cc_news dataset 60 | dataset = load_dataset(dataset_name) 61 | 62 | # get bert and tokenizer 63 | tokenizer = AutoTokenizer.from_pretrained(model_name) 64 | 65 | def tokenize_function(examples): 66 | return tokenizer(examples["text"], padding="max_length", truncation=True) 67 | 68 | # tokenize the dataset 69 | tokenized_datasets = dataset.map(tokenize_function, batched=True) 70 | small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(sample_size)) 71 | small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(sample_size)) 72 | 73 | train_loader = DataLoader( 74 | small_train_dataset, 75 | batch_size=batch_size, 76 | num_workers=num_workers, 77 | pin_memory=True, 78 | collate_fn=default_data_collator, 79 | ) 80 | 81 | test_loader = DataLoader( 82 | small_eval_dataset, 83 | batch_size=batch_size, 84 | num_workers=num_workers, 85 | pin_memory=True, 86 | collate_fn=default_data_collator, 87 | ) 88 | 89 | return train_loader, test_loader 90 | 91 | 92 | def get_model(model_name="bert-base-cased", num_labels=5): 93 | return AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=num_labels) 94 | 95 | def check_cuda(config): 96 | if torch.cuda.is_available(): 97 | config.device = "cuda" 98 | config.mixed_precision = True 99 | config.pt_version = torch.__version__ 100 | config.cuda_version = torch.version.cuda 101 | return config 102 | 103 | def train_bert(config): 104 | train_dl, test_loader = get_dls( 105 | model_name=config.model_name, 106 | batch_size=config.batch_size, 107 | num_workers=config.num_workers) 108 | 109 | model = get_model(config.model_name).to(config.device) 110 | if torch.__version__ >= "2.0" and config.compile: 111 | print("Compiling model...") 112 | model = torch.compile(model) 113 | 114 | trainer = MicroTrainer(model, train_dl, device=config.device, mixed_precision=config.mixed_precision, syncro=config.syncro) 115 | tags = [f"pt{torch.__version__}", f"cuda{torch.version.cuda}"] + (config.tags.split(",") if config.tags is not None else []) 116 | with wandb.init(project=PROJECT, entity=ENTITY, group=GROUP, tags=tags, config=config): 117 | config = wandb.config 118 | if not config.inference_only: 119 | trainer.fit(config.epochs) 120 | trainer.inference(test_loader) 121 | 122 | if __name__ == "__main__": 123 | args = parse_args() 124 | args = check_cuda(args) 125 | for _ in range(args.num_experiments): 126 | train_bert(config=args) 127 | -------------------------------------------------------------------------------- /pytorch/train_pets.py: -------------------------------------------------------------------------------- 1 | ## Author: Thomas Capelle, Soumik Rakshit 2 | ## Mail: tcapelle@wandb.com, soumik.rakshit@wandb.com 3 | 4 | """"Benchmarking apple M1Pro with Tensorflow 5 | @wandbcode{apple_m1_pro}""" 6 | 7 | import re, math, argparse 8 | from types import SimpleNamespace 9 | from pathlib import Path 10 | from time import perf_counter 11 | 12 | import wandb 13 | from PIL import Image 14 | from tqdm import tqdm 15 | 16 | 17 | import torch 18 | import torch.nn as nn 19 | import torchvision as tv 20 | import torchvision.transforms as T 21 | from torch.cuda.amp import autocast 22 | 23 | from utils import get_gpu_name 24 | from pets import get_pets_dataloader, get_fast_pets_dataloader 25 | 26 | PROJECT = "pytorch-M1Pro" 27 | ENTITY = "capecape" 28 | GROUP = "pytorch" 29 | 30 | config_defaults = SimpleNamespace( 31 | batch_size=128, 32 | device="cuda", 33 | epochs=1, 34 | num_experiments=1, 35 | learning_rate=1e-3, 36 | image_size=512, 37 | model_name="resnet50", 38 | dataset="PETS", 39 | num_workers=0, 40 | gpu_name=get_gpu_name(), 41 | mixed_precision=False, 42 | channels_last=False, 43 | optimizer="Adam", 44 | compile=False, 45 | tags=None, 46 | dl="full", 47 | syncro=False, 48 | ) 49 | 50 | def parse_args(): 51 | parser = argparse.ArgumentParser() 52 | parser.add_argument("--entity", type=str, default=ENTITY) 53 | parser.add_argument('--batch_size', type=int, default=config_defaults.batch_size) 54 | parser.add_argument('--epochs', type=int, default=config_defaults.epochs) 55 | parser.add_argument('--num_experiments', type=int, default=config_defaults.num_experiments) 56 | parser.add_argument('--learning_rate', type=float, default=config_defaults.learning_rate) 57 | parser.add_argument('--image_size', type=int, default=config_defaults.image_size) 58 | parser.add_argument('--model_name', type=str, default=config_defaults.model_name) 59 | parser.add_argument('--dataset', type=str, default=config_defaults.dataset) 60 | parser.add_argument('--device', type=str, default=config_defaults.device) 61 | parser.add_argument('--gpu_name', type=str, default=config_defaults.gpu_name) 62 | parser.add_argument('--num_workers', type=int, default=config_defaults.num_workers) 63 | parser.add_argument('--mixed_precision', action="store_true") 64 | parser.add_argument('--channels_last', action="store_true") 65 | parser.add_argument('--optimizer', type=str, default=config_defaults.optimizer) 66 | parser.add_argument('--compile', action="store_true") 67 | parser.add_argument('--tags', type=str, default=None) 68 | parser.add_argument('--dl', type=str, default=config_defaults.dl) 69 | parser.add_argument('--syncro', action="store_true") 70 | return parser.parse_args() 71 | 72 | 73 | def get_model(n_out, arch="resnet50"): 74 | model = getattr(tv.models, arch)(weights=tv.models.ResNet50_Weights.IMAGENET1K_V1) 75 | model.fc = nn.Linear(model.fc.in_features, n_out) 76 | return model 77 | 78 | 79 | def check_cuda(config): 80 | if torch.cuda.is_available(): 81 | config.device = "cuda" 82 | config.mixed_precision = True 83 | config.pt_version = torch.__version__ 84 | config.cuda_version = torch.version.cuda 85 | return config 86 | 87 | def train(config): 88 | config = check_cuda(config) 89 | tags = [f"pt{torch.__version__}", f"cuda{torch.version.cuda}"] + (config.tags.split(",") if config.tags is not None else []) 90 | print(tags) 91 | with wandb.init(project=PROJECT, entity=args.entity, group=GROUP, tags=tags, config=config): 92 | 93 | # Copy your config 94 | config = wandb.config 95 | 96 | # Get the data 97 | if not config.dl == "one_batch": 98 | train_dl = get_pets_dataloader(batch_size=config.batch_size, 99 | image_size=config.image_size, 100 | num_workers=config.num_workers) 101 | else: 102 | train_dl = get_fast_pets_dataloader(batch_size=config.batch_size, 103 | image_size=config.image_size, 104 | num_workers=0) 105 | 106 | n_steps_per_epoch = len(train_dl) 107 | 108 | model = get_model(len(train_dl.dataset.vocab), config.model_name) 109 | model.to(config.device) 110 | if config.channels_last: 111 | model.to(memory_format=torch.channels_last) 112 | if torch.__version__ >= "2.0" and config.compile: 113 | print("Compiling model...") 114 | model = torch.compile(model) 115 | 116 | # Make the loss and optimizer 117 | loss_func = nn.CrossEntropyLoss() 118 | optimizer = getattr(torch.optim, config.optimizer) 119 | optimizer = optimizer(model.parameters(), lr=config.learning_rate) 120 | 121 | # Training 122 | example_ct = 0 123 | for epoch in tqdm(range(config.epochs)): 124 | tf = t0 = perf_counter() 125 | model.train() 126 | for step, (images, labels) in enumerate(tqdm(train_dl, leave=False)): 127 | images, labels = images.to(config.device), labels.to(config.device) 128 | if config.channels_last: 129 | images = images.contiguous(memory_format=torch.channels_last) 130 | 131 | # compute the model froward and backward pass time 132 | ti = perf_counter() 133 | if config.mixed_precision: 134 | with autocast(): 135 | outputs = model(images) 136 | train_loss = loss_func(outputs, labels) 137 | else: 138 | outputs = model(images) 139 | train_loss = loss_func(outputs, labels) 140 | train_loss.backward() 141 | optimizer.step() 142 | optimizer.zero_grad() 143 | if config.syncro: 144 | torch.cuda.synchronize(device="cuda") 145 | tf_with_dataloader = perf_counter() - tf 146 | tf = perf_counter() 147 | 148 | 149 | # log the metrics 150 | example_ct += len(images) 151 | metrics = {"train/train_loss": train_loss, 152 | "train/epoch": (step + 1 + (n_steps_per_epoch * epoch)) / n_steps_per_epoch, 153 | "train/example_ct": example_ct, 154 | "samples_per_sec":len(images)/(tf-ti), 155 | "samples_per_sec_dl":len(images)/tf_with_dataloader, 156 | "samples_per_sec_epoch":example_ct/(tf-t0)} 157 | 158 | if step + 1 < n_steps_per_epoch: 159 | # 🐝 Log train metrics to wandb 160 | wandb.log(metrics) 161 | 162 | if __name__ == "__main__": 163 | args = parse_args() 164 | for _ in range(args.num_experiments): 165 | train(config=args) 166 | -------------------------------------------------------------------------------- /pytorch/train_pets_no_wandb.py: -------------------------------------------------------------------------------- 1 | ## Author: Thomas Capelle, Soumik Rakshit 2 | ## Mail: tcapelle@wandb.com, soumik.rakshit@wandb.com 3 | 4 | """"Benchmarking apple M1Pro with Tensorflow 5 | @wandbcode{apple_m1_pro}""" 6 | 7 | import re, math, argparse 8 | from types import SimpleNamespace 9 | from pathlib import Path 10 | from time import perf_counter 11 | import contextlib 12 | 13 | import wandb 14 | from PIL import Image 15 | from tqdm import tqdm 16 | 17 | import numpy as np 18 | 19 | import torch 20 | import torch.nn as nn 21 | import torchvision as tv 22 | import torchvision.transforms as T 23 | from torch import autocast 24 | 25 | from utils import get_gpu_name 26 | 27 | 28 | config_defaults = SimpleNamespace( 29 | batch_size=128, 30 | device="cuda", 31 | epochs=1, 32 | num_experiments=1, 33 | learning_rate=1e-3, 34 | image_size=512, 35 | model_name="resnet50", 36 | dataset="PETS", 37 | num_workers=0, 38 | gpu_name=get_gpu_name(), 39 | mixed_precision=False, 40 | channels_last=False, 41 | optimizer="Adam", 42 | compile=False, 43 | tags=None, 44 | ) 45 | 46 | def parse_args(): 47 | parser = argparse.ArgumentParser() 48 | parser.add_argument('--batch_size', type=int, default=config_defaults.batch_size) 49 | parser.add_argument('--epochs', type=int, default=config_defaults.epochs) 50 | parser.add_argument('--num_experiments', type=int, default=config_defaults.num_experiments) 51 | parser.add_argument('--learning_rate', type=float, default=config_defaults.learning_rate) 52 | parser.add_argument('--image_size', type=int, default=config_defaults.image_size) 53 | parser.add_argument('--model_name', type=str, default=config_defaults.model_name) 54 | parser.add_argument('--dataset', type=str, default=config_defaults.dataset) 55 | parser.add_argument('--device', type=str, default=config_defaults.device) 56 | parser.add_argument('--gpu_name', type=str, default=config_defaults.gpu_name) 57 | parser.add_argument('--num_workers', type=int, default=config_defaults.num_workers) 58 | parser.add_argument('--mixed_precision', action="store_true") 59 | parser.add_argument('--channels_last', action="store_true") 60 | parser.add_argument('--optimizer', type=str, default=config_defaults.optimizer) 61 | parser.add_argument('--compile', action="store_true") 62 | parser.add_argument('--tags', type=str, default=None) 63 | return parser.parse_args() 64 | 65 | def get_pets(version="v3"): 66 | api = wandb.Api() 67 | at = api.artifact(f'capecape/pytorch-M1Pro/PETS:{version}', type='dataset') 68 | dataset_path = at.download() 69 | return dataset_path 70 | 71 | class Pets(torch.utils.data.Dataset): 72 | pat = r'(^[a-zA-Z]+_*[a-zA-Z]+)' 73 | vocab = ['Abyssinian', 'Bengal', 'Birman', 'Bombay', 'British_Shorthair', 'Egyptian_Mau', 'Maine_Coon', 74 | 'Persian', 'Ragdoll', 'Russian_Blue', 'Siamese', 'Sphynx', 'american_bulldog', 'american_pit', 75 | 'basset_hound', 'beagle', 'boxer', 'chihuahua', 'english_cocker', 'english_setter', 'german_shorthaired', 76 | 'great_pyrenees', 'havanese', 'japanese_chin', 'keeshond', 'leonberger', 'miniature_pinscher', 'newfoundland', 77 | 'pomeranian', 'pug', 'saint_bernard', 'samoyed', 'scottish_terrier', 'shiba_inu', 'staffordshire_bull', 78 | 'wheaten_terrier', 'yorkshire_terrier'] 79 | vocab_map = {v:i for i,v in enumerate(vocab)} 80 | 81 | 82 | def __init__(self, pets_path="artifacts/PETS:v3", image_size=224): 83 | self.path = Path(pets_path) 84 | print(f"Dataset path: {self.path}") 85 | self.files = list(self.path.glob("images/*.jpg")) 86 | self.tfms =T.Compose([T.Resize((image_size, image_size)), T.ToTensor()]) 87 | self.vocab_map = {v:i for i, v in enumerate(self.vocab)} 88 | 89 | @staticmethod 90 | def load_image(fn, mode="RGB"): 91 | "Open and load a `PIL.Image` and convert to `mode`" 92 | im = Image.open(fn) 93 | im.load() 94 | im = im._new(im.im) 95 | return im.convert(mode) if mode else im 96 | 97 | def __getitem__(self, idx): 98 | file = self.files[idx] 99 | return self.tfms(self.load_image(str(file))), self.vocab_map[re.match(self.pat, file.name)[0]] 100 | 101 | def __len__(self): return len(self.files) 102 | 103 | 104 | def get_dataloader(dataset_path, batch_size, image_size=224, num_workers=0, **kwargs): 105 | "Get a training dataloader" 106 | ds = Pets(dataset_path, image_size=image_size) 107 | loader = torch.utils.data.DataLoader(ds, 108 | batch_size=batch_size, 109 | pin_memory=True, 110 | multiprocessing_context='spawn', 111 | num_workers=num_workers, 112 | **kwargs) 113 | return loader 114 | 115 | 116 | def get_model(n_out, arch="resnet50"): 117 | model = getattr(tv.models, arch)(weights=tv.models.ResNet50_Weights.IMAGENET1K_V1) 118 | model.fc = nn.Linear(model.fc.in_features, n_out) 119 | return model 120 | 121 | 122 | def check_cuda(config): 123 | if torch.cuda.is_available(): 124 | config.device = "cuda" 125 | config.mixed_precision = True 126 | return config 127 | 128 | @contextlib.contextmanager 129 | def timeit(s="func_name"): 130 | start = perf_counter() 131 | yield 132 | end = perf_counter() 133 | print(f"Elapsed in {s}: {end-start:.2f}s") 134 | 135 | 136 | def train(config): 137 | config = check_cuda(config) 138 | tags = [f"pt{torch.__version__}", f"cuda{torch.version.cuda}"] + (config.tags.split(",") if config.tags is not None else []) 139 | print(tags) 140 | 141 | # Get the data 142 | with timeit("get_dataloader"): 143 | train_dl = get_dataloader(dataset_path="artifacts/PETS:v3", 144 | batch_size=config.batch_size, 145 | image_size=config.image_size, 146 | num_workers=config.num_workers) 147 | with timeit("get_model"): 148 | model = get_model(len(train_dl.dataset.vocab), config.model_name) 149 | model.to(config.device) 150 | model.train() 151 | if config.channels_last: 152 | model.to(memory_format=torch.channels_last) 153 | with timeit("compile"): 154 | if torch.__version__ >= "2.0" and config.compile: 155 | print("Compiling model...") 156 | model = torch.compile(model) 157 | 158 | # Make the loss and optimizer 159 | with timeit("loss"): 160 | loss_func = nn.CrossEntropyLoss() 161 | optimizer = getattr(torch.optim, config.optimizer) 162 | optimizer = optimizer(model.parameters(), lr=config.learning_rate) 163 | 164 | # Training 165 | example_ct = 0 166 | samples_per_sec = 0. 167 | times_per_batch = [] 168 | for epoch in tqdm(range(config.epochs)): 169 | t0 = perf_counter() 170 | for step, (images, labels) in enumerate(tqdm(train_dl, leave=False)): 171 | images, labels = images.to(config.device), labels.to(config.device) 172 | if config.channels_last: 173 | images = images.contiguous(memory_format=torch.channels_last) 174 | 175 | # compute the model froward and backward pass time 176 | ti = perf_counter() 177 | if config.mixed_precision: 178 | with autocast("cuda"): 179 | outputs = model(images) 180 | train_loss = loss_func(outputs, labels) 181 | else: 182 | outputs = model(images) 183 | train_loss = loss_func(outputs, labels) 184 | train_loss.backward() 185 | optimizer.step() 186 | optimizer.zero_grad() 187 | tf = perf_counter() 188 | times_per_batch.append(tf-ti) 189 | # log the metrics 190 | example_ct += len(images) 191 | print(f"\nEpoch {epoch} took {perf_counter() - t0:.2f}s") 192 | print(f"Average samples/sec: {example_ct / (perf_counter() - t0):.2f}") 193 | print(f"Average batch time: {np.mean(times_per_batch):.2f}s") 194 | 195 | if __name__ == "__main__": 196 | args = parse_args() 197 | for _ in range(args.num_experiments): 198 | train(config=args) 199 | -------------------------------------------------------------------------------- /pytorch/utils.py: -------------------------------------------------------------------------------- 1 | import math, re, subprocess 2 | from sys import platform 3 | from types import SimpleNamespace 4 | from dataclasses import dataclass 5 | from time import perf_counter 6 | 7 | import wandb 8 | from tqdm import tqdm 9 | 10 | import torch 11 | from torch.optim import AdamW 12 | from torch.utils.data import DataLoader 13 | from torch.cuda.amp import autocast 14 | 15 | config_defaults = SimpleNamespace( 16 | learning_rate=1e-3, 17 | ) 18 | 19 | def get_apple_hardware(): 20 | "Get apple hardware info" 21 | cpu_info = subprocess.run(["system_profiler","SPHardwareDataType"], stdout=subprocess.PIPE).stdout.decode("utf-8") 22 | gpu_info = subprocess.run(["system_profiler","SPDisplaysDataType"], stdout=subprocess.PIPE).stdout.decode("utf-8") 23 | system_info = dict( 24 | cpu = re.search(r'Chip:\s+(.+)', cpu_info).group(1), 25 | cpu_cores = re.search(r'Number of Cores:\s+(\d+)', cpu_info).group(1), 26 | memory = re.search(r'Memory:\s+(\d+)\s+GB', cpu_info).group(1), 27 | gpu = re.search(r'Chipset Model:\s+(.+)', gpu_info).group(1), 28 | gpu_cores = re.search(r'Total Number of Cores:\s+(\d+)', gpu_info).group(1), 29 | ) 30 | return system_info 31 | 32 | def get_gpu_name(): 33 | if platform == "darwin": 34 | system_info = get_apple_hardware() 35 | return f"{system_info['gpu']} GPU {system_info['gpu_cores']} Cores" 36 | elif torch.cuda.is_available(): 37 | return torch.cuda.get_device_name() 38 | return None 39 | 40 | def to_device(batch, device="cpu"): 41 | "Move tensors to device" 42 | if isinstance(batch, torch.Tensor): 43 | batch.to(device) 44 | elif isinstance(batch, dict): 45 | for k,v in batch.items(): 46 | batch[k] = v.to(device) 47 | else: 48 | raise Exception(f"Can't put your batch of type {type(batch)} into device: {device}") 49 | return batch 50 | 51 | 52 | @dataclass 53 | class MicroTrainer: 54 | model: torch.nn.Module 55 | train_dl: DataLoader 56 | test_dl: DataLoader=None 57 | device: str="mps" 58 | mixed_precision: bool=False 59 | syncro: bool=False 60 | 61 | def __post_init__(self): 62 | self.model = self.model.to(self.device) 63 | self.optimizer = AdamW(self.model.parameters(), lr=config_defaults.learning_rate) 64 | self.n_steps_per_epoch = math.ceil(len(self.train_dl.dataset) / self.train_dl.batch_size) 65 | 66 | def do_one_batch(self, batch): 67 | batch = to_device(batch, device=self.device) 68 | if self.mixed_precision: 69 | with autocast(): 70 | outputs = self.model(**batch) 71 | loss = outputs.loss 72 | else: 73 | outputs = self.model(**batch) 74 | loss = outputs.loss 75 | return loss 76 | 77 | def do_one_epoch(self, dl, epoch): 78 | tf=perf_counter() 79 | for step, batch in enumerate(tqdm(dl, leave=False)): 80 | ti = perf_counter() 81 | self.optimizer.zero_grad() 82 | loss = self.do_one_batch(batch) 83 | loss.backward() 84 | self.optimizer.step() 85 | if self.syncro: 86 | torch.cuda.synchronize(device="cuda") 87 | tf_with_dataloader = perf_counter() - tf 88 | tf = perf_counter() 89 | self.example_ct += len(batch["labels"]) 90 | metrics = {"train/train_loss": loss, 91 | "train/epoch": (step + 1 + (self.n_steps_per_epoch * epoch)) / self.n_steps_per_epoch, 92 | "train/example_ct": self.example_ct, 93 | "sqe_per_sec":len(batch["labels"])/(tf-ti), 94 | "seq_per_sec_dl":len(batch["labels"])/tf_with_dataloader,} 95 | if step + 1 < self.n_steps_per_epoch: 96 | # 🐝 Log train metrics to wandb 97 | wandb.log(metrics) 98 | 99 | self.step_ct += 1 100 | 101 | def fit(self, epochs): 102 | self.example_ct = 0 103 | self.step_ct = 0 104 | self.model.train() 105 | for epoch in tqdm(range(epochs)): 106 | self.do_one_epoch(self.train_dl, epoch) 107 | 108 | def inference(self, dl, repeat=10): 109 | self.model.eval() 110 | batch = next(iter(dl)) 111 | N = len(batch["labels"]) 112 | inference_times = [] 113 | for _ in tqdm(range(repeat)): 114 | with torch.no_grad(): 115 | ti = perf_counter() 116 | _ = self.do_one_batch(batch) 117 | tf = perf_counter() 118 | inference_times.append(N/(tf-ti)) 119 | wandb.log({"inference_seq_per_sec": sum(inference_times)/repeat}) -------------------------------------------------------------------------------- /tensorflow/Basic_Performance.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 5, 6 | "metadata": { 7 | "colab": { 8 | "base_uri": "https://localhost:8080/", 9 | "height": 1000, 10 | "referenced_widgets": [ 11 | "97651ef84a5d4f11a7e75670ce8300e7", 12 | "b479a3899aba49fd99294d9df2eec701", 13 | "7d2be1b24f2841fa93c57aae78f31a15", 14 | "4c181783977f4ebf9e1a2a49c3ec0120", 15 | "60170335e6514507bd87c6a4dac3d623", 16 | "058bb2545d34464b9ca0718261778026", 17 | "c418d6652e43445da8e239f0ca859846", 18 | "623e060e7e514a30aadda61cb7c1674b", 19 | "7d2279098f384e2eb2dbeaf53cde52c1", 20 | "a0273718b661442b8d1334943460ede6", 21 | "8b5f7d4f32c345de90ed6d1875bd6bd1", 22 | "f9129b445ddf4aee859db77ea883c8fe", 23 | "5adc2846a13b4a3d981cbd7fc0769cab", 24 | "ca45fa9c101d4051860f176bf1bab1da", 25 | "bf7a7a2133994732bfa850f7eb51fdf0", 26 | "7cbfd13104c44eba8aa1e8d1fb2fc7f1", 27 | "7b6f698700794454bcdd0cb4e5881b3e", 28 | "f580aa5ff4ec41c6ae48419661333f7f", 29 | "066346a946674cb7bd869ae6feb86b5e", 30 | "bd8c6c0cd07a4c4782db78b6f3c529cb", 31 | "ab368a038d174a9f979fcbb9f470b8c5", 32 | "53f3956ad0f945499c3daad7a104e392", 33 | "a8eb7ef889934f5ba61591bc338aafcd", 34 | "49c58f268866499c8ea5b5e86e7b629e", 35 | "7d62fc41e1a94a5eb9df853f7a2f16c2", 36 | "93f911153ab54aa0b625794462a91ad5", 37 | "e3cfed5314bb4867a4241cff2e0adac8", 38 | "46da9173be364dafaaaae3123ed61653", 39 | "bf9c745dd3784c2ab2562c5a6e1e002e", 40 | "9584b9a224444b7685e3eb883fa71015", 41 | "a08692e01d504d3fbf711c61f04f6ee6", 42 | "e4354c4c8da74a1ab1e44b987b221947" 43 | ] 44 | }, 45 | "id": "Ka4FcloCqnH_", 46 | "outputId": "a698a483-d3d6-464b-9e25-7fe62f9741d6" 47 | }, 48 | "outputs": [], 49 | "source": [ 50 | "import os\n", 51 | "import random\n", 52 | "\n", 53 | "from pathlib import Path\n", 54 | "import matplotlib.pyplot as plt\n", 55 | "import numpy as np\n", 56 | "import pandas as pd\n", 57 | "import tensorflow as tf\n", 58 | "from tensorflow import keras\n", 59 | "from tensorflow.keras import layers\n", 60 | "from tensorflow.keras.datasets import cifar10\n", 61 | "\n", 62 | "\n", 63 | "PROJECT = 'apple_m1_pro'\n", 64 | "GROUP = 'keras'\n", 65 | "JOB_TYPE = 'M1Pro'\n", 66 | "\n", 67 | "# Set the random seeds\n", 68 | "os.environ['TF_CUDNN_DETERMINISTIC'] = '1' \n", 69 | "random.seed(hash(\"setting random seeds\") % 2**32 - 1)\n", 70 | "np.random.seed(hash(\"improves reproducibility\") % 2**32 - 1)\n", 71 | "tf.random.set_seed(hash(\"by removing stochasticity\") % 2**32 - 1)\n", 72 | "\n", 73 | "\n", 74 | "IMG_SIZE = 128\n", 75 | "BS = 64\n", 76 | "EPOCHS = 10" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 6, 82 | "metadata": { 83 | "colab": { 84 | "base_uri": "https://localhost:8080/", 85 | "height": 68 86 | }, 87 | "id": "FBqlJEVarXP6", 88 | "outputId": "75b1d12d-c181-4327-c438-2b64ff1829ab" 89 | }, 90 | "outputs": [], 91 | "source": [ 92 | "import wandb" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 9, 98 | "metadata": {}, 99 | "outputs": [ 100 | { 101 | "name": "stdout", 102 | "output_type": "stream", 103 | "text": [ 104 | "WARNING:tensorflow:From /var/folders/sf/tgv7vcv96x557p38bvvp1ms40000gn/T/ipykernel_1357/337460670.py:1: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.\n", 105 | "Instructions for updating:\n", 106 | "Use `tf.config.list_physical_devices('GPU')` instead.\n", 107 | "Metal device set to: Apple M1 Pro\n" 108 | ] 109 | }, 110 | { 111 | "name": "stderr", 112 | "output_type": "stream", 113 | "text": [ 114 | "2021-11-22 09:51:58.136042: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n", 115 | "2021-11-22 09:51:58.136414: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: )\n" 116 | ] 117 | }, 118 | { 119 | "data": { 120 | "text/plain": [ 121 | "True" 122 | ] 123 | }, 124 | "execution_count": 9, 125 | "metadata": {}, 126 | "output_type": "execute_result" 127 | } 128 | ], 129 | "source": [ 130 | "tf.test.is_gpu_available()" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 10, 136 | "metadata": {}, 137 | "outputs": [ 138 | { 139 | "name": "stderr", 140 | "output_type": "stream", 141 | "text": [ 142 | "2021-11-22 09:52:25.257768: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n", 143 | "2021-11-22 09:52:25.257802: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: )\n" 144 | ] 145 | }, 146 | { 147 | "data": { 148 | "text/plain": [ 149 | "'/device:GPU:0'" 150 | ] 151 | }, 152 | "execution_count": 10, 153 | "metadata": {}, 154 | "output_type": "execute_result" 155 | } 156 | ], 157 | "source": [ 158 | "tf.test.gpu_device_name()" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": 12, 164 | "metadata": {}, 165 | "outputs": [ 166 | { 167 | "name": "stdout", 168 | "output_type": "stream", 169 | "text": [ 170 | "Name: /physical_device:GPU:0 Type: GPU\n" 171 | ] 172 | } 173 | ], 174 | "source": [ 175 | "gpus = tf.config.list_physical_devices('GPU')\n", 176 | "for gpu in gpus:\n", 177 | " print(\"Name:\", gpu.name, \" Type:\", gpu.device_type)" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": 7, 183 | "metadata": {}, 184 | "outputs": [], 185 | "source": [ 186 | "def hardware():\n", 187 | " \"\"\"Detect platform and accelerator\"\"\"\n", 188 | " if platform.system() == \"Darwin\":\n", 189 | " if platform.processor() == \"arm\":\n", 190 | " return \"Apple M1\"\n", 191 | " else:\n", 192 | " return \"Apple Intel\"\n", 193 | " elif len(tf.config.list_physical_devices('GPU')) > 0:\n", 194 | " return \"Nvidia\"\n", 195 | " return \"Linux Intel\"" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 8, 201 | "metadata": {}, 202 | "outputs": [ 203 | { 204 | "ename": "NameError", 205 | "evalue": "name 'platform' is not defined", 206 | "output_type": "error", 207 | "traceback": [ 208 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 209 | "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", 210 | "\u001b[0;32m/var/folders/sf/tgv7vcv96x557p38bvvp1ms40000gn/T/ipykernel_1357/2333675197.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhardware\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 211 | "\u001b[0;32m/var/folders/sf/tgv7vcv96x557p38bvvp1ms40000gn/T/ipykernel_1357/3459158288.py\u001b[0m in \u001b[0;36mhardware\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mhardware\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\"\"\"Detect platform and accelerator\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mplatform\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"Darwin\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mplatform\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocessor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"arm\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;34m\"Apple M1\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 212 | "\u001b[0;31mNameError\u001b[0m: name 'platform' is not defined" 213 | ] 214 | } 215 | ], 216 | "source": [ 217 | "hardware()" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": 15, 223 | "metadata": {}, 224 | "outputs": [ 225 | { 226 | "data": { 227 | "text/html": [ 228 | "Finishing last run (ID:18ul76ty) before initializing another..." 229 | ], 230 | "text/plain": [ 231 | "" 232 | ] 233 | }, 234 | "metadata": {}, 235 | "output_type": "display_data" 236 | }, 237 | { 238 | "data": { 239 | "text/html": [ 240 | "
Waiting for W&B process to finish, PID 94249... (success)." 241 | ], 242 | "text/plain": [ 243 | "" 244 | ] 245 | }, 246 | "metadata": {}, 247 | "output_type": "display_data" 248 | }, 249 | { 250 | "data": { 251 | "application/vnd.jupyter.widget-view+json": { 252 | "model_id": "", 253 | "version_major": 2, 254 | "version_minor": 0 255 | }, 256 | "text/plain": [ 257 | "VBox(children=(Label(value=' 0.00MB of 0.00MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…" 258 | ] 259 | }, 260 | "metadata": {}, 261 | "output_type": "display_data" 262 | }, 263 | { 264 | "data": { 265 | "text/html": [ 266 | "\n", 271 | "
\n", 272 | "
\n", 273 | "
\n", 274 | "Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n", 275 | "
Synced feasible-oath-19: https://wandb.ai/tcapelle/apple_m1_pro/runs/18ul76ty
\n", 276 | "Find logs at: ./wandb/run-20211119_192019-18ul76ty/logs
\n" 277 | ], 278 | "text/plain": [ 279 | "" 280 | ] 281 | }, 282 | "metadata": {}, 283 | "output_type": "display_data" 284 | }, 285 | { 286 | "data": { 287 | "text/html": [ 288 | "Successfully finished last run (ID:18ul76ty). Initializing new run:
" 289 | ], 290 | "text/plain": [ 291 | "" 292 | ] 293 | }, 294 | "metadata": {}, 295 | "output_type": "display_data" 296 | }, 297 | { 298 | "data": { 299 | "text/html": [ 300 | "\n", 301 | " Syncing run brisk-violet-20 to Weights & Biases (docs).
\n", 302 | "\n", 303 | " " 304 | ], 305 | "text/plain": [ 306 | "" 307 | ] 308 | }, 309 | "metadata": {}, 310 | "output_type": "display_data" 311 | } 312 | ], 313 | "source": [ 314 | "run = wandb.init(project=PROJECT, group=GROUP, job_type=JOB_TYPE)" 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "execution_count": 15, 320 | "metadata": {}, 321 | "outputs": [ 322 | { 323 | "ename": "NameError", 324 | "evalue": "name 'run' is not defined", 325 | "output_type": "error", 326 | "traceback": [ 327 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 328 | "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", 329 | "\u001b[0;32m/var/folders/sf/tgv7vcv96x557p38bvvp1ms40000gn/T/ipykernel_1357/2789942315.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0martifact\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_artifact\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'tcapelle/apple_m1_pro/PetImages:latest'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'dataset'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0martifact_dir\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0martifact\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 330 | "\u001b[0;31mNameError\u001b[0m: name 'run' is not defined" 331 | ] 332 | } 333 | ], 334 | "source": [ 335 | "artifact = run.use_artifact('tcapelle/apple_m1_pro/PetImages:latest', type='dataset')\n", 336 | "artifact_dir = artifact.download()" 337 | ] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "execution_count": 35, 342 | "metadata": {}, 343 | "outputs": [ 344 | { 345 | "name": "stdout", 346 | "output_type": "stream", 347 | "text": [ 348 | "Deleted 0 images\n" 349 | ] 350 | } 351 | ], 352 | "source": [ 353 | "import os\n", 354 | "from PIL import Image\n", 355 | "\n", 356 | "num_skipped = 0\n", 357 | "for folder_name in (\"Cat\", \"Dog\"):\n", 358 | " folder_path = os.path.join(artifact_dir, folder_name)\n", 359 | " for fname in os.listdir(folder_path):\n", 360 | " fpath = os.path.join(folder_path, fname)\n", 361 | " try:\n", 362 | " fobj = open(fpath, \"rb\")\n", 363 | " is_jfif = tf.compat.as_bytes(\"JFIF\") in fobj.peek(10)\n", 364 | " finally:\n", 365 | " fobj.close()\n", 366 | "\n", 367 | " if not is_jfif:\n", 368 | " num_skipped += 1\n", 369 | " # Delete corrupted image\n", 370 | " os.remove(fpath)\n", 371 | "\n", 372 | "print(\"Deleted %d images\" % num_skipped)" 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "execution_count": 13, 378 | "metadata": {}, 379 | "outputs": [], 380 | "source": [ 381 | "def get_data(data_dir, batch_size):\n", 382 | " print(f'Reading from : {Path(data_dir).absolute()}')\n", 383 | " train_ds = tf.keras.preprocessing.image_dataset_from_directory(\n", 384 | " Path(data_dir),\n", 385 | " validation_split=0.2,\n", 386 | " subset=\"training\",\n", 387 | " seed=1337,\n", 388 | " image_size=(IMG_SIZE,IMG_SIZE),\n", 389 | " batch_size=BS,\n", 390 | " )\n", 391 | " val_ds = tf.keras.preprocessing.image_dataset_from_directory(\n", 392 | " Path(data_dir),\n", 393 | " validation_split=0.2,\n", 394 | " subset=\"validation\",\n", 395 | " seed=1337,\n", 396 | " image_size=(IMG_SIZE,IMG_SIZE),\n", 397 | " batch_size=BS,\n", 398 | " )\n", 399 | " return train_ds, val_ds" 400 | ] 401 | }, 402 | { 403 | "cell_type": "code", 404 | "execution_count": 14, 405 | "metadata": {}, 406 | "outputs": [ 407 | { 408 | "ename": "NameError", 409 | "evalue": "name 'artifact_dir' is not defined", 410 | "output_type": "error", 411 | "traceback": [ 412 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 413 | "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", 414 | "\u001b[0;32m/var/folders/sf/tgv7vcv96x557p38bvvp1ms40000gn/T/ipykernel_1357/1649941138.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain_ds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0martifact_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 415 | "\u001b[0;31mNameError\u001b[0m: name 'artifact_dir' is not defined" 416 | ] 417 | } 418 | ], 419 | "source": [ 420 | "train_ds, val_ds = get_data(artifact_dir, 32)" 421 | ] 422 | }, 423 | { 424 | "cell_type": "code", 425 | "execution_count": 19, 426 | "metadata": {}, 427 | "outputs": [], 428 | "source": [ 429 | "#not supported on GPU in M1\n", 430 | "data_augmentation = keras.Sequential(\n", 431 | " [\n", 432 | " layers.RandomFlip(\"horizontal\"),\n", 433 | " layers.RandomRotation(0.1),\n", 434 | " ]\n", 435 | ")\n", 436 | "data_augmentation = lambda x: x" 437 | ] 438 | }, 439 | { 440 | "cell_type": "code", 441 | "execution_count": 20, 442 | "metadata": {}, 443 | "outputs": [], 444 | "source": [ 445 | "train_ds = train_ds.prefetch(buffer_size=32)\n", 446 | "val_ds = val_ds.prefetch(buffer_size=32)" 447 | ] 448 | }, 449 | { 450 | "cell_type": "code", 451 | "execution_count": 21, 452 | "metadata": {}, 453 | "outputs": [], 454 | "source": [ 455 | "def make_model(input_shape, num_classes):\n", 456 | " inputs = keras.Input(shape=input_shape)\n", 457 | " # Image augmentation block\n", 458 | " x = data_augmentation(inputs)\n", 459 | "\n", 460 | " # Entry block\n", 461 | " x = layers.Rescaling(1.0 / 255)(x)\n", 462 | " x = layers.Conv2D(32, 3, strides=2, padding=\"same\")(x)\n", 463 | " x = layers.BatchNormalization()(x)\n", 464 | " x = layers.Activation(\"relu\")(x)\n", 465 | "\n", 466 | " x = layers.Conv2D(64, 3, padding=\"same\")(x)\n", 467 | " x = layers.BatchNormalization()(x)\n", 468 | " x = layers.Activation(\"relu\")(x)\n", 469 | "\n", 470 | " previous_block_activation = x # Set aside residual\n", 471 | "\n", 472 | " for size in [128, 256, 512, 728]:\n", 473 | " x = layers.Activation(\"relu\")(x)\n", 474 | " x = layers.SeparableConv2D(size, 3, padding=\"same\")(x)\n", 475 | " x = layers.BatchNormalization()(x)\n", 476 | "\n", 477 | " x = layers.Activation(\"relu\")(x)\n", 478 | " x = layers.SeparableConv2D(size, 3, padding=\"same\")(x)\n", 479 | " x = layers.BatchNormalization()(x)\n", 480 | "\n", 481 | " x = layers.MaxPooling2D(3, strides=2, padding=\"same\")(x)\n", 482 | "\n", 483 | " # Project residual\n", 484 | " residual = layers.Conv2D(size, 1, strides=2, padding=\"same\")(\n", 485 | " previous_block_activation\n", 486 | " )\n", 487 | " x = layers.add([x, residual]) # Add back residual\n", 488 | " previous_block_activation = x # Set aside next residual\n", 489 | "\n", 490 | " x = layers.SeparableConv2D(1024, 3, padding=\"same\")(x)\n", 491 | " x = layers.BatchNormalization()(x)\n", 492 | " x = layers.Activation(\"relu\")(x)\n", 493 | "\n", 494 | " x = layers.GlobalAveragePooling2D()(x)\n", 495 | " if num_classes == 2:\n", 496 | " activation = \"sigmoid\"\n", 497 | " units = 1\n", 498 | " else:\n", 499 | " activation = \"softmax\"\n", 500 | " units = num_classes\n", 501 | "\n", 502 | " x = layers.Dropout(0.5)(x)\n", 503 | " outputs = layers.Dense(units, activation=activation)(x)\n", 504 | " return keras.Model(inputs, outputs)\n", 505 | "\n", 506 | "\n", 507 | "model = make_model(input_shape=(IMG_SIZE, IMG_SIZE) + (3,), num_classes=2)" 508 | ] 509 | }, 510 | { 511 | "cell_type": "code", 512 | "execution_count": 22, 513 | "metadata": {}, 514 | "outputs": [ 515 | { 516 | "name": "stdout", 517 | "output_type": "stream", 518 | "text": [ 519 | "Epoch 1/10\n" 520 | ] 521 | }, 522 | { 523 | "name": "stderr", 524 | "output_type": "stream", 525 | "text": [ 526 | "2021-11-19 19:21:14.311909: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n", 527 | "2021-11-19 19:21:14.315356: W tensorflow/core/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n", 528 | "2021-11-19 19:21:14.315483: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n", 529 | "Corrupt JPEG data: 2226 extraneous bytes before marker 0xd9\n" 530 | ] 531 | }, 532 | { 533 | "name": "stdout", 534 | "output_type": "stream", 535 | "text": [ 536 | " 6/293 [..............................] - ETA: 2:28 - loss: 1.0128 - accuracy: 0.5208WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1761s vs `on_train_batch_end` time: 0.3246s). Check your callbacks.\n", 537 | "126/293 [===========>..................] - ETA: 1:24 - loss: 0.6724 - accuracy: 0.6336" 538 | ] 539 | }, 540 | { 541 | "name": "stderr", 542 | "output_type": "stream", 543 | "text": [ 544 | "Corrupt JPEG data: 228 extraneous bytes before marker 0xd9\n" 545 | ] 546 | }, 547 | { 548 | "name": "stdout", 549 | "output_type": "stream", 550 | "text": [ 551 | "141/293 [=============>................] - ETA: 1:17 - loss: 0.6654 - accuracy: 0.6395" 552 | ] 553 | }, 554 | { 555 | "name": "stderr", 556 | "output_type": "stream", 557 | "text": [ 558 | "Warning: unknown JFIF revision number 0.00\n" 559 | ] 560 | }, 561 | { 562 | "name": "stdout", 563 | "output_type": "stream", 564 | "text": [ 565 | "180/293 [=================>............] - ETA: 57s - loss: 0.6376 - accuracy: 0.6632" 566 | ] 567 | }, 568 | { 569 | "name": "stderr", 570 | "output_type": "stream", 571 | "text": [ 572 | "Corrupt JPEG data: 128 extraneous bytes before marker 0xd9\n" 573 | ] 574 | }, 575 | { 576 | "name": "stdout", 577 | "output_type": "stream", 578 | "text": [ 579 | "185/293 [=================>............] - ETA: 54s - loss: 0.6344 - accuracy: 0.6653" 580 | ] 581 | }, 582 | { 583 | "name": "stderr", 584 | "output_type": "stream", 585 | "text": [ 586 | "Corrupt JPEG data: 65 extraneous bytes before marker 0xd9\n" 587 | ] 588 | }, 589 | { 590 | "name": "stdout", 591 | "output_type": "stream", 592 | "text": [ 593 | "191/293 [==================>...........] - ETA: 51s - loss: 0.6316 - accuracy: 0.6673" 594 | ] 595 | }, 596 | { 597 | "name": "stderr", 598 | "output_type": "stream", 599 | "text": [ 600 | "Corrupt JPEG data: 396 extraneous bytes before marker 0xd9\n" 601 | ] 602 | }, 603 | { 604 | "name": "stdout", 605 | "output_type": "stream", 606 | "text": [ 607 | "194/293 [==================>...........] - ETA: 50s - loss: 0.6298 - accuracy: 0.6688" 608 | ] 609 | }, 610 | { 611 | "name": "stderr", 612 | "output_type": "stream", 613 | "text": [ 614 | "Corrupt JPEG data: 239 extraneous bytes before marker 0xd9\n" 615 | ] 616 | }, 617 | { 618 | "name": "stdout", 619 | "output_type": "stream", 620 | "text": [ 621 | "293/293 [==============================] - ETA: 0s - loss: 0.5843 - accuracy: 0.7013" 622 | ] 623 | }, 624 | { 625 | "name": "stderr", 626 | "output_type": "stream", 627 | "text": [ 628 | "2021-11-19 19:23:43.002272: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n", 629 | "Corrupt JPEG data: 252 extraneous bytes before marker 0xd9\n", 630 | "Corrupt JPEG data: 1153 extraneous bytes before marker 0xd9\n", 631 | "Corrupt JPEG data: 162 extraneous bytes before marker 0xd9\n", 632 | "Corrupt JPEG data: 214 extraneous bytes before marker 0xd9\n", 633 | "Corrupt JPEG data: 99 extraneous bytes before marker 0xd9\n", 634 | "Corrupt JPEG data: 1403 extraneous bytes before marker 0xd9\n" 635 | ] 636 | }, 637 | { 638 | "name": "stdout", 639 | "output_type": "stream", 640 | "text": [ 641 | "293/293 [==============================] - 159s 538ms/step - loss: 0.5843 - accuracy: 0.7013 - val_loss: 0.8676 - val_accuracy: 0.4957\n", 642 | "Epoch 2/10\n" 643 | ] 644 | }, 645 | { 646 | "name": "stderr", 647 | "output_type": "stream", 648 | "text": [ 649 | "Corrupt JPEG data: 2226 extraneous bytes before marker 0xd9\n" 650 | ] 651 | }, 652 | { 653 | "name": "stdout", 654 | "output_type": "stream", 655 | "text": [ 656 | "126/293 [===========>..................] - ETA: 1:25 - loss: 0.4167 - accuracy: 0.8140" 657 | ] 658 | }, 659 | { 660 | "name": "stderr", 661 | "output_type": "stream", 662 | "text": [ 663 | "Corrupt JPEG data: 228 extraneous bytes before marker 0xd9\n" 664 | ] 665 | }, 666 | { 667 | "name": "stdout", 668 | "output_type": "stream", 669 | "text": [ 670 | "141/293 [=============>................] - ETA: 1:18 - loss: 0.4122 - accuracy: 0.8162" 671 | ] 672 | }, 673 | { 674 | "name": "stderr", 675 | "output_type": "stream", 676 | "text": [ 677 | "Warning: unknown JFIF revision number 0.00\n" 678 | ] 679 | }, 680 | { 681 | "name": "stdout", 682 | "output_type": "stream", 683 | "text": [ 684 | "180/293 [=================>............] - ETA: 57s - loss: 0.3962 - accuracy: 0.8245" 685 | ] 686 | }, 687 | { 688 | "name": "stderr", 689 | "output_type": "stream", 690 | "text": [ 691 | "Corrupt JPEG data: 128 extraneous bytes before marker 0xd9\n" 692 | ] 693 | }, 694 | { 695 | "name": "stdout", 696 | "output_type": "stream", 697 | "text": [ 698 | "185/293 [=================>............] - ETA: 55s - loss: 0.3934 - accuracy: 0.8258" 699 | ] 700 | }, 701 | { 702 | "name": "stderr", 703 | "output_type": "stream", 704 | "text": [ 705 | "Corrupt JPEG data: 65 extraneous bytes before marker 0xd9\n" 706 | ] 707 | }, 708 | { 709 | "name": "stdout", 710 | "output_type": "stream", 711 | "text": [ 712 | "191/293 [==================>...........] - ETA: 52s - loss: 0.3908 - accuracy: 0.8273" 713 | ] 714 | }, 715 | { 716 | "name": "stderr", 717 | "output_type": "stream", 718 | "text": [ 719 | "Corrupt JPEG data: 396 extraneous bytes before marker 0xd9\n" 720 | ] 721 | }, 722 | { 723 | "name": "stdout", 724 | "output_type": "stream", 725 | "text": [ 726 | "194/293 [==================>...........] - ETA: 50s - loss: 0.3901 - accuracy: 0.8275" 727 | ] 728 | }, 729 | { 730 | "name": "stderr", 731 | "output_type": "stream", 732 | "text": [ 733 | "Corrupt JPEG data: 239 extraneous bytes before marker 0xd9\n" 734 | ] 735 | }, 736 | { 737 | "name": "stdout", 738 | "output_type": "stream", 739 | "text": [ 740 | "293/293 [==============================] - ETA: 0s - loss: 0.3622 - accuracy: 0.8413" 741 | ] 742 | }, 743 | { 744 | "name": "stderr", 745 | "output_type": "stream", 746 | "text": [ 747 | "Corrupt JPEG data: 252 extraneous bytes before marker 0xd9\n", 748 | "Corrupt JPEG data: 1153 extraneous bytes before marker 0xd9\n", 749 | "Corrupt JPEG data: 162 extraneous bytes before marker 0xd9\n", 750 | "Corrupt JPEG data: 214 extraneous bytes before marker 0xd9\n", 751 | "Corrupt JPEG data: 99 extraneous bytes before marker 0xd9\n", 752 | "Corrupt JPEG data: 1403 extraneous bytes before marker 0xd9\n" 753 | ] 754 | }, 755 | { 756 | "name": "stdout", 757 | "output_type": "stream", 758 | "text": [ 759 | "293/293 [==============================] - 159s 543ms/step - loss: 0.3622 - accuracy: 0.8413 - val_loss: 0.8233 - val_accuracy: 0.6025\n", 760 | "Epoch 3/10\n" 761 | ] 762 | }, 763 | { 764 | "name": "stderr", 765 | "output_type": "stream", 766 | "text": [ 767 | "Corrupt JPEG data: 2226 extraneous bytes before marker 0xd9\n" 768 | ] 769 | }, 770 | { 771 | "name": "stdout", 772 | "output_type": "stream", 773 | "text": [ 774 | "126/293 [===========>..................] - ETA: 1:25 - loss: 0.2653 - accuracy: 0.8859" 775 | ] 776 | }, 777 | { 778 | "name": "stderr", 779 | "output_type": "stream", 780 | "text": [ 781 | "Corrupt JPEG data: 228 extraneous bytes before marker 0xd9\n" 782 | ] 783 | }, 784 | { 785 | "name": "stdout", 786 | "output_type": "stream", 787 | "text": [ 788 | "141/293 [=============>................] - ETA: 1:17 - loss: 0.2618 - accuracy: 0.8871" 789 | ] 790 | }, 791 | { 792 | "name": "stderr", 793 | "output_type": "stream", 794 | "text": [ 795 | "Warning: unknown JFIF revision number 0.00\n" 796 | ] 797 | }, 798 | { 799 | "name": "stdout", 800 | "output_type": "stream", 801 | "text": [ 802 | "180/293 [=================>............] - ETA: 57s - loss: 0.2581 - accuracy: 0.8895" 803 | ] 804 | }, 805 | { 806 | "name": "stderr", 807 | "output_type": "stream", 808 | "text": [ 809 | "Corrupt JPEG data: 128 extraneous bytes before marker 0xd9\n" 810 | ] 811 | }, 812 | { 813 | "name": "stdout", 814 | "output_type": "stream", 815 | "text": [ 816 | "185/293 [=================>............] - ETA: 55s - loss: 0.2571 - accuracy: 0.8901" 817 | ] 818 | }, 819 | { 820 | "name": "stderr", 821 | "output_type": "stream", 822 | "text": [ 823 | "Corrupt JPEG data: 65 extraneous bytes before marker 0xd9\n" 824 | ] 825 | }, 826 | { 827 | "name": "stdout", 828 | "output_type": "stream", 829 | "text": [ 830 | "191/293 [==================>...........] - ETA: 52s - loss: 0.2562 - accuracy: 0.8905" 831 | ] 832 | }, 833 | { 834 | "name": "stderr", 835 | "output_type": "stream", 836 | "text": [ 837 | "Corrupt JPEG data: 396 extraneous bytes before marker 0xd9\n" 838 | ] 839 | }, 840 | { 841 | "name": "stdout", 842 | "output_type": "stream", 843 | "text": [ 844 | "194/293 [==================>...........] - ETA: 50s - loss: 0.2549 - accuracy: 0.8912" 845 | ] 846 | }, 847 | { 848 | "name": "stderr", 849 | "output_type": "stream", 850 | "text": [ 851 | "Corrupt JPEG data: 239 extraneous bytes before marker 0xd9\n" 852 | ] 853 | }, 854 | { 855 | "name": "stdout", 856 | "output_type": "stream", 857 | "text": [ 858 | "293/293 [==============================] - ETA: 0s - loss: 0.2412 - accuracy: 0.8982" 859 | ] 860 | }, 861 | { 862 | "name": "stderr", 863 | "output_type": "stream", 864 | "text": [ 865 | "Corrupt JPEG data: 252 extraneous bytes before marker 0xd9\n", 866 | "Corrupt JPEG data: 1153 extraneous bytes before marker 0xd9\n", 867 | "Corrupt JPEG data: 162 extraneous bytes before marker 0xd9\n", 868 | "Corrupt JPEG data: 214 extraneous bytes before marker 0xd9\n", 869 | "Corrupt JPEG data: 99 extraneous bytes before marker 0xd9\n", 870 | "Corrupt JPEG data: 1403 extraneous bytes before marker 0xd9\n" 871 | ] 872 | }, 873 | { 874 | "name": "stdout", 875 | "output_type": "stream", 876 | "text": [ 877 | "293/293 [==============================] - 160s 543ms/step - loss: 0.2412 - accuracy: 0.8982 - val_loss: 0.3012 - val_accuracy: 0.8661\n", 878 | "Epoch 4/10\n" 879 | ] 880 | }, 881 | { 882 | "name": "stderr", 883 | "output_type": "stream", 884 | "text": [ 885 | "Corrupt JPEG data: 2226 extraneous bytes before marker 0xd9\n" 886 | ] 887 | }, 888 | { 889 | "name": "stdout", 890 | "output_type": "stream", 891 | "text": [ 892 | "126/293 [===========>..................] - ETA: 1:25 - loss: 0.1764 - accuracy: 0.9249" 893 | ] 894 | }, 895 | { 896 | "name": "stderr", 897 | "output_type": "stream", 898 | "text": [ 899 | "Corrupt JPEG data: 228 extraneous bytes before marker 0xd9\n" 900 | ] 901 | }, 902 | { 903 | "name": "stdout", 904 | "output_type": "stream", 905 | "text": [ 906 | "141/293 [=============>................] - ETA: 1:17 - loss: 0.1755 - accuracy: 0.9251" 907 | ] 908 | }, 909 | { 910 | "name": "stderr", 911 | "output_type": "stream", 912 | "text": [ 913 | "Warning: unknown JFIF revision number 0.00\n" 914 | ] 915 | }, 916 | { 917 | "name": "stdout", 918 | "output_type": "stream", 919 | "text": [ 920 | "180/293 [=================>............] - ETA: 57s - loss: 0.1758 - accuracy: 0.9262" 921 | ] 922 | }, 923 | { 924 | "name": "stderr", 925 | "output_type": "stream", 926 | "text": [ 927 | "Corrupt JPEG data: 128 extraneous bytes before marker 0xd9\n" 928 | ] 929 | }, 930 | { 931 | "name": "stdout", 932 | "output_type": "stream", 933 | "text": [ 934 | "185/293 [=================>............] - ETA: 55s - loss: 0.1760 - accuracy: 0.9263" 935 | ] 936 | }, 937 | { 938 | "name": "stderr", 939 | "output_type": "stream", 940 | "text": [ 941 | "Corrupt JPEG data: 65 extraneous bytes before marker 0xd9\n" 942 | ] 943 | }, 944 | { 945 | "name": "stdout", 946 | "output_type": "stream", 947 | "text": [ 948 | "191/293 [==================>...........] - ETA: 52s - loss: 0.1762 - accuracy: 0.9268" 949 | ] 950 | }, 951 | { 952 | "name": "stderr", 953 | "output_type": "stream", 954 | "text": [ 955 | "Corrupt JPEG data: 396 extraneous bytes before marker 0xd9\n" 956 | ] 957 | }, 958 | { 959 | "name": "stdout", 960 | "output_type": "stream", 961 | "text": [ 962 | "194/293 [==================>...........] - ETA: 50s - loss: 0.1752 - accuracy: 0.9273" 963 | ] 964 | }, 965 | { 966 | "name": "stderr", 967 | "output_type": "stream", 968 | "text": [ 969 | "Corrupt JPEG data: 239 extraneous bytes before marker 0xd9\n" 970 | ] 971 | }, 972 | { 973 | "name": "stdout", 974 | "output_type": "stream", 975 | "text": [ 976 | "293/293 [==============================] - ETA: 0s - loss: 0.1702 - accuracy: 0.9304" 977 | ] 978 | }, 979 | { 980 | "name": "stderr", 981 | "output_type": "stream", 982 | "text": [ 983 | "Corrupt JPEG data: 252 extraneous bytes before marker 0xd9\n", 984 | "Corrupt JPEG data: 1153 extraneous bytes before marker 0xd9\n", 985 | "Corrupt JPEG data: 162 extraneous bytes before marker 0xd9\n", 986 | "Corrupt JPEG data: 214 extraneous bytes before marker 0xd9\n", 987 | "Corrupt JPEG data: 99 extraneous bytes before marker 0xd9\n", 988 | "Corrupt JPEG data: 1403 extraneous bytes before marker 0xd9\n" 989 | ] 990 | }, 991 | { 992 | "name": "stdout", 993 | "output_type": "stream", 994 | "text": [ 995 | "293/293 [==============================] - 159s 543ms/step - loss: 0.1702 - accuracy: 0.9304 - val_loss: 0.4854 - val_accuracy: 0.8138\n", 996 | "Epoch 5/10\n" 997 | ] 998 | }, 999 | { 1000 | "name": "stderr", 1001 | "output_type": "stream", 1002 | "text": [ 1003 | "Corrupt JPEG data: 2226 extraneous bytes before marker 0xd9\n" 1004 | ] 1005 | }, 1006 | { 1007 | "name": "stdout", 1008 | "output_type": "stream", 1009 | "text": [ 1010 | "126/293 [===========>..................] - ETA: 1:25 - loss: 0.1326 - accuracy: 0.9472" 1011 | ] 1012 | }, 1013 | { 1014 | "name": "stderr", 1015 | "output_type": "stream", 1016 | "text": [ 1017 | "Corrupt JPEG data: 228 extraneous bytes before marker 0xd9\n" 1018 | ] 1019 | }, 1020 | { 1021 | "name": "stdout", 1022 | "output_type": "stream", 1023 | "text": [ 1024 | "141/293 [=============>................] - ETA: 1:17 - loss: 0.1302 - accuracy: 0.9488" 1025 | ] 1026 | }, 1027 | { 1028 | "name": "stderr", 1029 | "output_type": "stream", 1030 | "text": [ 1031 | "Warning: unknown JFIF revision number 0.00\n" 1032 | ] 1033 | }, 1034 | { 1035 | "name": "stdout", 1036 | "output_type": "stream", 1037 | "text": [ 1038 | "180/293 [=================>............] - ETA: 57s - loss: 0.1336 - accuracy: 0.9469" 1039 | ] 1040 | }, 1041 | { 1042 | "name": "stderr", 1043 | "output_type": "stream", 1044 | "text": [ 1045 | "Corrupt JPEG data: 128 extraneous bytes before marker 0xd9\n" 1046 | ] 1047 | }, 1048 | { 1049 | "name": "stdout", 1050 | "output_type": "stream", 1051 | "text": [ 1052 | "185/293 [=================>............] - ETA: 54s - loss: 0.1339 - accuracy: 0.9465" 1053 | ] 1054 | }, 1055 | { 1056 | "name": "stderr", 1057 | "output_type": "stream", 1058 | "text": [ 1059 | "Corrupt JPEG data: 65 extraneous bytes before marker 0xd9\n" 1060 | ] 1061 | }, 1062 | { 1063 | "name": "stdout", 1064 | "output_type": "stream", 1065 | "text": [ 1066 | "191/293 [==================>...........] - ETA: 51s - loss: 0.1338 - accuracy: 0.9466" 1067 | ] 1068 | }, 1069 | { 1070 | "name": "stderr", 1071 | "output_type": "stream", 1072 | "text": [ 1073 | "Corrupt JPEG data: 396 extraneous bytes before marker 0xd9\n" 1074 | ] 1075 | }, 1076 | { 1077 | "name": "stdout", 1078 | "output_type": "stream", 1079 | "text": [ 1080 | "194/293 [==================>...........] - ETA: 50s - loss: 0.1331 - accuracy: 0.9468" 1081 | ] 1082 | }, 1083 | { 1084 | "name": "stderr", 1085 | "output_type": "stream", 1086 | "text": [ 1087 | "Corrupt JPEG data: 239 extraneous bytes before marker 0xd9\n" 1088 | ] 1089 | }, 1090 | { 1091 | "name": "stdout", 1092 | "output_type": "stream", 1093 | "text": [ 1094 | "293/293 [==============================] - ETA: 0s - loss: 0.1272 - accuracy: 0.9491" 1095 | ] 1096 | }, 1097 | { 1098 | "name": "stderr", 1099 | "output_type": "stream", 1100 | "text": [ 1101 | "Corrupt JPEG data: 252 extraneous bytes before marker 0xd9\n", 1102 | "Corrupt JPEG data: 1153 extraneous bytes before marker 0xd9\n", 1103 | "Corrupt JPEG data: 162 extraneous bytes before marker 0xd9\n", 1104 | "Corrupt JPEG data: 214 extraneous bytes before marker 0xd9\n", 1105 | "Corrupt JPEG data: 99 extraneous bytes before marker 0xd9\n", 1106 | "Corrupt JPEG data: 1403 extraneous bytes before marker 0xd9\n" 1107 | ] 1108 | }, 1109 | { 1110 | "name": "stdout", 1111 | "output_type": "stream", 1112 | "text": [ 1113 | "293/293 [==============================] - 159s 541ms/step - loss: 0.1272 - accuracy: 0.9491 - val_loss: 0.3413 - val_accuracy: 0.8971\n", 1114 | "Epoch 6/10\n" 1115 | ] 1116 | }, 1117 | { 1118 | "name": "stderr", 1119 | "output_type": "stream", 1120 | "text": [ 1121 | "Corrupt JPEG data: 2226 extraneous bytes before marker 0xd9\n" 1122 | ] 1123 | }, 1124 | { 1125 | "name": "stdout", 1126 | "output_type": "stream", 1127 | "text": [ 1128 | "125/293 [===========>..................] - ETA: 1:25 - loss: 0.1024 - accuracy: 0.9574" 1129 | ] 1130 | }, 1131 | { 1132 | "name": "stderr", 1133 | "output_type": "stream", 1134 | "text": [ 1135 | "Corrupt JPEG data: 228 extraneous bytes before marker 0xd9\n" 1136 | ] 1137 | }, 1138 | { 1139 | "name": "stdout", 1140 | "output_type": "stream", 1141 | "text": [ 1142 | "140/293 [=============>................] - ETA: 1:18 - loss: 0.1028 - accuracy: 0.9583" 1143 | ] 1144 | }, 1145 | { 1146 | "name": "stderr", 1147 | "output_type": "stream", 1148 | "text": [ 1149 | "Warning: unknown JFIF revision number 0.00\n" 1150 | ] 1151 | }, 1152 | { 1153 | "name": "stdout", 1154 | "output_type": "stream", 1155 | "text": [ 1156 | "179/293 [=================>............] - ETA: 58s - loss: 0.1059 - accuracy: 0.9576" 1157 | ] 1158 | }, 1159 | { 1160 | "name": "stderr", 1161 | "output_type": "stream", 1162 | "text": [ 1163 | "Corrupt JPEG data: 128 extraneous bytes before marker 0xd9\n" 1164 | ] 1165 | }, 1166 | { 1167 | "name": "stdout", 1168 | "output_type": "stream", 1169 | "text": [ 1170 | "184/293 [=================>............] - ETA: 55s - loss: 0.1059 - accuracy: 0.9576" 1171 | ] 1172 | }, 1173 | { 1174 | "name": "stderr", 1175 | "output_type": "stream", 1176 | "text": [ 1177 | "Corrupt JPEG data: 65 extraneous bytes before marker 0xd9\n" 1178 | ] 1179 | }, 1180 | { 1181 | "name": "stdout", 1182 | "output_type": "stream", 1183 | "text": [ 1184 | "190/293 [==================>...........] - ETA: 52s - loss: 0.1073 - accuracy: 0.9567" 1185 | ] 1186 | }, 1187 | { 1188 | "name": "stderr", 1189 | "output_type": "stream", 1190 | "text": [ 1191 | "Corrupt JPEG data: 396 extraneous bytes before marker 0xd9\n" 1192 | ] 1193 | }, 1194 | { 1195 | "name": "stdout", 1196 | "output_type": "stream", 1197 | "text": [ 1198 | "193/293 [==================>...........] - ETA: 50s - loss: 0.1071 - accuracy: 0.9569" 1199 | ] 1200 | }, 1201 | { 1202 | "name": "stderr", 1203 | "output_type": "stream", 1204 | "text": [ 1205 | "Corrupt JPEG data: 239 extraneous bytes before marker 0xd9\n" 1206 | ] 1207 | }, 1208 | { 1209 | "name": "stdout", 1210 | "output_type": "stream", 1211 | "text": [ 1212 | "293/293 [==============================] - ETA: 0s - loss: 0.1076 - accuracy: 0.9570" 1213 | ] 1214 | }, 1215 | { 1216 | "name": "stderr", 1217 | "output_type": "stream", 1218 | "text": [ 1219 | "Corrupt JPEG data: 252 extraneous bytes before marker 0xd9\n", 1220 | "Corrupt JPEG data: 1153 extraneous bytes before marker 0xd9\n", 1221 | "Corrupt JPEG data: 162 extraneous bytes before marker 0xd9\n", 1222 | "Corrupt JPEG data: 214 extraneous bytes before marker 0xd9\n", 1223 | "Corrupt JPEG data: 99 extraneous bytes before marker 0xd9\n", 1224 | "Corrupt JPEG data: 1403 extraneous bytes before marker 0xd9\n" 1225 | ] 1226 | }, 1227 | { 1228 | "name": "stdout", 1229 | "output_type": "stream", 1230 | "text": [ 1231 | "293/293 [==============================] - 159s 542ms/step - loss: 0.1076 - accuracy: 0.9570 - val_loss: 0.3643 - val_accuracy: 0.8889\n", 1232 | "Epoch 7/10\n" 1233 | ] 1234 | }, 1235 | { 1236 | "name": "stderr", 1237 | "output_type": "stream", 1238 | "text": [ 1239 | "Corrupt JPEG data: 2226 extraneous bytes before marker 0xd9\n" 1240 | ] 1241 | }, 1242 | { 1243 | "name": "stdout", 1244 | "output_type": "stream", 1245 | "text": [ 1246 | "125/293 [===========>..................] - ETA: 1:25 - loss: 0.0834 - accuracy: 0.9668" 1247 | ] 1248 | }, 1249 | { 1250 | "name": "stderr", 1251 | "output_type": "stream", 1252 | "text": [ 1253 | "Corrupt JPEG data: 228 extraneous bytes before marker 0xd9\n" 1254 | ] 1255 | }, 1256 | { 1257 | "name": "stdout", 1258 | "output_type": "stream", 1259 | "text": [ 1260 | "140/293 [=============>................] - ETA: 1:17 - loss: 0.0810 - accuracy: 0.9673" 1261 | ] 1262 | }, 1263 | { 1264 | "name": "stderr", 1265 | "output_type": "stream", 1266 | "text": [ 1267 | "Warning: unknown JFIF revision number 0.00\n" 1268 | ] 1269 | }, 1270 | { 1271 | "name": "stdout", 1272 | "output_type": "stream", 1273 | "text": [ 1274 | "179/293 [=================>............] - ETA: 57s - loss: 0.0785 - accuracy: 0.9688" 1275 | ] 1276 | }, 1277 | { 1278 | "name": "stderr", 1279 | "output_type": "stream", 1280 | "text": [ 1281 | "Corrupt JPEG data: 128 extraneous bytes before marker 0xd9\n" 1282 | ] 1283 | }, 1284 | { 1285 | "name": "stdout", 1286 | "output_type": "stream", 1287 | "text": [ 1288 | "184/293 [=================>............] - ETA: 55s - loss: 0.0786 - accuracy: 0.9688" 1289 | ] 1290 | }, 1291 | { 1292 | "name": "stderr", 1293 | "output_type": "stream", 1294 | "text": [ 1295 | "Corrupt JPEG data: 65 extraneous bytes before marker 0xd9\n" 1296 | ] 1297 | }, 1298 | { 1299 | "name": "stdout", 1300 | "output_type": "stream", 1301 | "text": [ 1302 | "190/293 [==================>...........] - ETA: 52s - loss: 0.0782 - accuracy: 0.9689" 1303 | ] 1304 | }, 1305 | { 1306 | "name": "stderr", 1307 | "output_type": "stream", 1308 | "text": [ 1309 | "Corrupt JPEG data: 396 extraneous bytes before marker 0xd9\n" 1310 | ] 1311 | }, 1312 | { 1313 | "name": "stdout", 1314 | "output_type": "stream", 1315 | "text": [ 1316 | "193/293 [==================>...........] - ETA: 50s - loss: 0.0782 - accuracy: 0.9691" 1317 | ] 1318 | }, 1319 | { 1320 | "name": "stderr", 1321 | "output_type": "stream", 1322 | "text": [ 1323 | "Corrupt JPEG data: 239 extraneous bytes before marker 0xd9\n" 1324 | ] 1325 | }, 1326 | { 1327 | "name": "stdout", 1328 | "output_type": "stream", 1329 | "text": [ 1330 | "293/293 [==============================] - ETA: 0s - loss: 0.0812 - accuracy: 0.9687" 1331 | ] 1332 | }, 1333 | { 1334 | "name": "stderr", 1335 | "output_type": "stream", 1336 | "text": [ 1337 | "Corrupt JPEG data: 252 extraneous bytes before marker 0xd9\n", 1338 | "Corrupt JPEG data: 1153 extraneous bytes before marker 0xd9\n", 1339 | "Corrupt JPEG data: 162 extraneous bytes before marker 0xd9\n", 1340 | "Corrupt JPEG data: 214 extraneous bytes before marker 0xd9\n", 1341 | "Corrupt JPEG data: 99 extraneous bytes before marker 0xd9\n", 1342 | "Corrupt JPEG data: 1403 extraneous bytes before marker 0xd9\n" 1343 | ] 1344 | }, 1345 | { 1346 | "name": "stdout", 1347 | "output_type": "stream", 1348 | "text": [ 1349 | "293/293 [==============================] - 159s 542ms/step - loss: 0.0812 - accuracy: 0.9687 - val_loss: 1.1136 - val_accuracy: 0.7774\n", 1350 | "Epoch 8/10\n" 1351 | ] 1352 | }, 1353 | { 1354 | "name": "stderr", 1355 | "output_type": "stream", 1356 | "text": [ 1357 | "Corrupt JPEG data: 2226 extraneous bytes before marker 0xd9\n" 1358 | ] 1359 | }, 1360 | { 1361 | "name": "stdout", 1362 | "output_type": "stream", 1363 | "text": [ 1364 | "126/293 [===========>..................] - ETA: 1:25 - loss: 0.0680 - accuracy: 0.9741" 1365 | ] 1366 | }, 1367 | { 1368 | "name": "stderr", 1369 | "output_type": "stream", 1370 | "text": [ 1371 | "Corrupt JPEG data: 228 extraneous bytes before marker 0xd9\n" 1372 | ] 1373 | }, 1374 | { 1375 | "name": "stdout", 1376 | "output_type": "stream", 1377 | "text": [ 1378 | "141/293 [=============>................] - ETA: 1:17 - loss: 0.0653 - accuracy: 0.9752" 1379 | ] 1380 | }, 1381 | { 1382 | "name": "stderr", 1383 | "output_type": "stream", 1384 | "text": [ 1385 | "Warning: unknown JFIF revision number 0.00\n" 1386 | ] 1387 | }, 1388 | { 1389 | "name": "stdout", 1390 | "output_type": "stream", 1391 | "text": [ 1392 | "180/293 [=================>............] - ETA: 57s - loss: 0.0656 - accuracy: 0.9754" 1393 | ] 1394 | }, 1395 | { 1396 | "name": "stderr", 1397 | "output_type": "stream", 1398 | "text": [ 1399 | "Corrupt JPEG data: 128 extraneous bytes before marker 0xd9\n" 1400 | ] 1401 | }, 1402 | { 1403 | "name": "stdout", 1404 | "output_type": "stream", 1405 | "text": [ 1406 | "185/293 [=================>............] - ETA: 55s - loss: 0.0649 - accuracy: 0.9758" 1407 | ] 1408 | }, 1409 | { 1410 | "name": "stderr", 1411 | "output_type": "stream", 1412 | "text": [ 1413 | "Corrupt JPEG data: 65 extraneous bytes before marker 0xd9\n" 1414 | ] 1415 | }, 1416 | { 1417 | "name": "stdout", 1418 | "output_type": "stream", 1419 | "text": [ 1420 | "191/293 [==================>...........] - ETA: 52s - loss: 0.0647 - accuracy: 0.9759" 1421 | ] 1422 | }, 1423 | { 1424 | "name": "stderr", 1425 | "output_type": "stream", 1426 | "text": [ 1427 | "Corrupt JPEG data: 396 extraneous bytes before marker 0xd9\n" 1428 | ] 1429 | }, 1430 | { 1431 | "name": "stdout", 1432 | "output_type": "stream", 1433 | "text": [ 1434 | "194/293 [==================>...........] - ETA: 50s - loss: 0.0650 - accuracy: 0.9758" 1435 | ] 1436 | }, 1437 | { 1438 | "name": "stderr", 1439 | "output_type": "stream", 1440 | "text": [ 1441 | "Corrupt JPEG data: 239 extraneous bytes before marker 0xd9\n" 1442 | ] 1443 | }, 1444 | { 1445 | "name": "stdout", 1446 | "output_type": "stream", 1447 | "text": [ 1448 | "293/293 [==============================] - ETA: 0s - loss: 0.0642 - accuracy: 0.9754" 1449 | ] 1450 | }, 1451 | { 1452 | "name": "stderr", 1453 | "output_type": "stream", 1454 | "text": [ 1455 | "Corrupt JPEG data: 252 extraneous bytes before marker 0xd9\n", 1456 | "Corrupt JPEG data: 1153 extraneous bytes before marker 0xd9\n", 1457 | "Corrupt JPEG data: 162 extraneous bytes before marker 0xd9\n", 1458 | "Corrupt JPEG data: 214 extraneous bytes before marker 0xd9\n", 1459 | "Corrupt JPEG data: 99 extraneous bytes before marker 0xd9\n", 1460 | "Corrupt JPEG data: 1403 extraneous bytes before marker 0xd9\n" 1461 | ] 1462 | }, 1463 | { 1464 | "name": "stdout", 1465 | "output_type": "stream", 1466 | "text": [ 1467 | "293/293 [==============================] - 159s 543ms/step - loss: 0.0642 - accuracy: 0.9754 - val_loss: 0.2818 - val_accuracy: 0.9020\n", 1468 | "Epoch 9/10\n" 1469 | ] 1470 | }, 1471 | { 1472 | "name": "stderr", 1473 | "output_type": "stream", 1474 | "text": [ 1475 | "Corrupt JPEG data: 2226 extraneous bytes before marker 0xd9\n" 1476 | ] 1477 | }, 1478 | { 1479 | "name": "stdout", 1480 | "output_type": "stream", 1481 | "text": [ 1482 | "125/293 [===========>..................] - ETA: 1:25 - loss: 0.0641 - accuracy: 0.9740" 1483 | ] 1484 | }, 1485 | { 1486 | "name": "stderr", 1487 | "output_type": "stream", 1488 | "text": [ 1489 | "Corrupt JPEG data: 228 extraneous bytes before marker 0xd9\n" 1490 | ] 1491 | }, 1492 | { 1493 | "name": "stdout", 1494 | "output_type": "stream", 1495 | "text": [ 1496 | "140/293 [=============>................] - ETA: 1:18 - loss: 0.0629 - accuracy: 0.9744" 1497 | ] 1498 | }, 1499 | { 1500 | "name": "stderr", 1501 | "output_type": "stream", 1502 | "text": [ 1503 | "Warning: unknown JFIF revision number 0.00\n" 1504 | ] 1505 | }, 1506 | { 1507 | "name": "stdout", 1508 | "output_type": "stream", 1509 | "text": [ 1510 | "179/293 [=================>............] - ETA: 58s - loss: 0.0618 - accuracy: 0.9754" 1511 | ] 1512 | }, 1513 | { 1514 | "name": "stderr", 1515 | "output_type": "stream", 1516 | "text": [ 1517 | "Corrupt JPEG data: 128 extraneous bytes before marker 0xd9\n" 1518 | ] 1519 | }, 1520 | { 1521 | "name": "stdout", 1522 | "output_type": "stream", 1523 | "text": [ 1524 | "184/293 [=================>............] - ETA: 55s - loss: 0.0617 - accuracy: 0.9755" 1525 | ] 1526 | }, 1527 | { 1528 | "name": "stderr", 1529 | "output_type": "stream", 1530 | "text": [ 1531 | "Corrupt JPEG data: 65 extraneous bytes before marker 0xd9\n" 1532 | ] 1533 | }, 1534 | { 1535 | "name": "stdout", 1536 | "output_type": "stream", 1537 | "text": [ 1538 | "190/293 [==================>...........] - ETA: 52s - loss: 0.0624 - accuracy: 0.9752" 1539 | ] 1540 | }, 1541 | { 1542 | "name": "stderr", 1543 | "output_type": "stream", 1544 | "text": [ 1545 | "Corrupt JPEG data: 396 extraneous bytes before marker 0xd9\n" 1546 | ] 1547 | }, 1548 | { 1549 | "name": "stdout", 1550 | "output_type": "stream", 1551 | "text": [ 1552 | "193/293 [==================>...........] - ETA: 51s - loss: 0.0629 - accuracy: 0.9750" 1553 | ] 1554 | }, 1555 | { 1556 | "name": "stderr", 1557 | "output_type": "stream", 1558 | "text": [ 1559 | "Corrupt JPEG data: 239 extraneous bytes before marker 0xd9\n" 1560 | ] 1561 | }, 1562 | { 1563 | "name": "stdout", 1564 | "output_type": "stream", 1565 | "text": [ 1566 | "293/293 [==============================] - ETA: 0s - loss: 0.0613 - accuracy: 0.9760" 1567 | ] 1568 | }, 1569 | { 1570 | "name": "stderr", 1571 | "output_type": "stream", 1572 | "text": [ 1573 | "Corrupt JPEG data: 252 extraneous bytes before marker 0xd9\n", 1574 | "Corrupt JPEG data: 1153 extraneous bytes before marker 0xd9\n", 1575 | "Corrupt JPEG data: 162 extraneous bytes before marker 0xd9\n", 1576 | "Corrupt JPEG data: 214 extraneous bytes before marker 0xd9\n", 1577 | "Corrupt JPEG data: 99 extraneous bytes before marker 0xd9\n", 1578 | "Corrupt JPEG data: 1403 extraneous bytes before marker 0xd9\n" 1579 | ] 1580 | }, 1581 | { 1582 | "name": "stdout", 1583 | "output_type": "stream", 1584 | "text": [ 1585 | "293/293 [==============================] - 159s 542ms/step - loss: 0.0613 - accuracy: 0.9760 - val_loss: 0.3614 - val_accuracy: 0.9065\n", 1586 | "Epoch 10/10\n" 1587 | ] 1588 | }, 1589 | { 1590 | "name": "stderr", 1591 | "output_type": "stream", 1592 | "text": [ 1593 | "Corrupt JPEG data: 2226 extraneous bytes before marker 0xd9\n" 1594 | ] 1595 | }, 1596 | { 1597 | "name": "stdout", 1598 | "output_type": "stream", 1599 | "text": [ 1600 | "126/293 [===========>..................] - ETA: 1:24 - loss: 0.0574 - accuracy: 0.9788" 1601 | ] 1602 | }, 1603 | { 1604 | "name": "stderr", 1605 | "output_type": "stream", 1606 | "text": [ 1607 | "Corrupt JPEG data: 228 extraneous bytes before marker 0xd9\n" 1608 | ] 1609 | }, 1610 | { 1611 | "name": "stdout", 1612 | "output_type": "stream", 1613 | "text": [ 1614 | "141/293 [=============>................] - ETA: 1:17 - loss: 0.0568 - accuracy: 0.9787" 1615 | ] 1616 | }, 1617 | { 1618 | "name": "stderr", 1619 | "output_type": "stream", 1620 | "text": [ 1621 | "Warning: unknown JFIF revision number 0.00\n" 1622 | ] 1623 | }, 1624 | { 1625 | "name": "stdout", 1626 | "output_type": "stream", 1627 | "text": [ 1628 | "180/293 [=================>............] - ETA: 57s - loss: 0.0548 - accuracy: 0.9789" 1629 | ] 1630 | }, 1631 | { 1632 | "name": "stderr", 1633 | "output_type": "stream", 1634 | "text": [ 1635 | "Corrupt JPEG data: 128 extraneous bytes before marker 0xd9\n" 1636 | ] 1637 | }, 1638 | { 1639 | "name": "stdout", 1640 | "output_type": "stream", 1641 | "text": [ 1642 | "185/293 [=================>............] - ETA: 54s - loss: 0.0544 - accuracy: 0.9790" 1643 | ] 1644 | }, 1645 | { 1646 | "name": "stderr", 1647 | "output_type": "stream", 1648 | "text": [ 1649 | "Corrupt JPEG data: 65 extraneous bytes before marker 0xd9\n" 1650 | ] 1651 | }, 1652 | { 1653 | "name": "stdout", 1654 | "output_type": "stream", 1655 | "text": [ 1656 | "191/293 [==================>...........] - ETA: 51s - loss: 0.0553 - accuracy: 0.9785" 1657 | ] 1658 | }, 1659 | { 1660 | "name": "stderr", 1661 | "output_type": "stream", 1662 | "text": [ 1663 | "Corrupt JPEG data: 396 extraneous bytes before marker 0xd9\n" 1664 | ] 1665 | }, 1666 | { 1667 | "name": "stdout", 1668 | "output_type": "stream", 1669 | "text": [ 1670 | "194/293 [==================>...........] - ETA: 50s - loss: 0.0551 - accuracy: 0.9786" 1671 | ] 1672 | }, 1673 | { 1674 | "name": "stderr", 1675 | "output_type": "stream", 1676 | "text": [ 1677 | "Corrupt JPEG data: 239 extraneous bytes before marker 0xd9\n" 1678 | ] 1679 | }, 1680 | { 1681 | "name": "stdout", 1682 | "output_type": "stream", 1683 | "text": [ 1684 | "293/293 [==============================] - ETA: 0s - loss: 0.0542 - accuracy: 0.9793" 1685 | ] 1686 | }, 1687 | { 1688 | "name": "stderr", 1689 | "output_type": "stream", 1690 | "text": [ 1691 | "Corrupt JPEG data: 252 extraneous bytes before marker 0xd9\n", 1692 | "Corrupt JPEG data: 1153 extraneous bytes before marker 0xd9\n", 1693 | "Corrupt JPEG data: 162 extraneous bytes before marker 0xd9\n", 1694 | "Corrupt JPEG data: 214 extraneous bytes before marker 0xd9\n", 1695 | "Corrupt JPEG data: 99 extraneous bytes before marker 0xd9\n", 1696 | "Corrupt JPEG data: 1403 extraneous bytes before marker 0xd9\n" 1697 | ] 1698 | }, 1699 | { 1700 | "name": "stdout", 1701 | "output_type": "stream", 1702 | "text": [ 1703 | "293/293 [==============================] - 159s 541ms/step - loss: 0.0542 - accuracy: 0.9793 - val_loss: 0.4073 - val_accuracy: 0.9052\n" 1704 | ] 1705 | }, 1706 | { 1707 | "data": { 1708 | "text/plain": [ 1709 | "" 1710 | ] 1711 | }, 1712 | "execution_count": 22, 1713 | "metadata": {}, 1714 | "output_type": "execute_result" 1715 | } 1716 | ], 1717 | "source": [ 1718 | "callbacks = [\n", 1719 | " wandb.keras.WandbCallback(save_model=False),\n", 1720 | "]\n", 1721 | "model.compile(\n", 1722 | " optimizer=keras.optimizers.Adam(1e-3),\n", 1723 | " loss=\"binary_crossentropy\",\n", 1724 | " metrics=[\"accuracy\"],\n", 1725 | ")\n", 1726 | "model.fit(\n", 1727 | " train_ds, epochs=EPOCHS, callbacks=callbacks, validation_data=val_ds,\n", 1728 | ")" 1729 | ] 1730 | }, 1731 | { 1732 | "cell_type": "code", 1733 | "execution_count": 36, 1734 | "metadata": {}, 1735 | "outputs": [ 1736 | { 1737 | "data": { 1738 | "text/html": [ 1739 | "
Waiting for W&B process to finish, PID 94302... (success)." 1740 | ], 1741 | "text/plain": [ 1742 | "" 1743 | ] 1744 | }, 1745 | "metadata": {}, 1746 | "output_type": "display_data" 1747 | }, 1748 | { 1749 | "data": { 1750 | "application/vnd.jupyter.widget-view+json": { 1751 | "model_id": "", 1752 | "version_major": 2, 1753 | "version_minor": 0 1754 | }, 1755 | "text/plain": [ 1756 | "VBox(children=(Label(value=' 0.01MB of 0.01MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…" 1757 | ] 1758 | }, 1759 | "metadata": {}, 1760 | "output_type": "display_data" 1761 | }, 1762 | { 1763 | "data": { 1764 | "text/html": [ 1765 | "\n", 1770 | "
\n", 1771 | "

Run history:


accuracy▁▅▆▇▇▇████
epoch▁▂▃▃▄▅▆▆▇█
loss█▅▃▃▂▂▁▁▁▁
val_accuracy▁▃▇▆██▆███
val_loss▆▆▁▃▂▂█▁▂▂

\n", 1772 | "

Run summary:


accuracy0.97928
best_epoch7
best_val_loss0.28175
epoch9
loss0.05418
val_accuracy0.90517
val_loss0.40733
\n", 1773 | "
\n", 1774 | "Synced 6 W&B file(s), 1 media file(s), 0 artifact file(s) and 0 other file(s)\n", 1775 | "
Synced brisk-violet-20: https://wandb.ai/tcapelle/apple_m1_pro/runs/faz28dh6
\n", 1776 | "Find logs at: ./wandb/run-20211119_192058-faz28dh6/logs
\n" 1777 | ], 1778 | "text/plain": [ 1779 | "" 1780 | ] 1781 | }, 1782 | "metadata": {}, 1783 | "output_type": "display_data" 1784 | } 1785 | ], 1786 | "source": [ 1787 | "wandb.finish()" 1788 | ] 1789 | }, 1790 | { 1791 | "cell_type": "markdown", 1792 | "metadata": {}, 1793 | "source": [ 1794 | "# Old code" 1795 | ] 1796 | }, 1797 | { 1798 | "cell_type": "code", 1799 | "execution_count": 23, 1800 | "metadata": {}, 1801 | "outputs": [ 1802 | { 1803 | "ename": "NameError", 1804 | "evalue": "name 'platform' is not defined", 1805 | "output_type": "error", 1806 | "traceback": [ 1807 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 1808 | "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", 1809 | "\u001b[0;32m/var/folders/sf/tgv7vcv96x557p38bvvp1ms40000gn/T/ipykernel_94235/2064899850.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 8\u001b[0m default_config = {\n\u001b[1;32m 9\u001b[0m \u001b[0;34m\"batch_size\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"epochs\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"dropout\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m0.4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"base_model\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mBASE_MODEL\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;34m\"init_lr\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m0.0005\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"decay\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m0.96\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"num_classes\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mN_CLASSES\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"hardware\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mhardware\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 11\u001b[0m \u001b[0;34m\"train_size\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"test_size\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;34m\"dataset\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mDATASET\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"img_dim\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIMG_DIM\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"trainable\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1810 | "\u001b[0;32m/var/folders/sf/tgv7vcv96x557p38bvvp1ms40000gn/T/ipykernel_94235/3459158288.py\u001b[0m in \u001b[0;36mhardware\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mhardware\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\"\"\"Detect platform and accelerator\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mplatform\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"Darwin\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mplatform\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocessor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"arm\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;34m\"Apple M1\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1811 | "\u001b[0;31mNameError\u001b[0m: name 'platform' is not defined" 1812 | ] 1813 | } 1814 | ], 1815 | "source": [ 1816 | "# get data\n", 1817 | "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n", 1818 | "\n", 1819 | "# Subsetting train data and normalizing to [0., 1.]\n", 1820 | "x_train, x_test = x_train / 255., x_test / 255.\n", 1821 | "\n", 1822 | "\n", 1823 | "default_config = {\n", 1824 | " \"batch_size\": 64, \"epochs\": 10, \"dropout\": 0.4, \"base_model\": BASE_MODEL, \n", 1825 | " \"init_lr\": 0.0005, \"decay\": 0.96, \"num_classes\": N_CLASSES, \"hardware\": hardware(), \n", 1826 | " \"train_size\": len(x_train), \"test_size\": len(x_test),\n", 1827 | " \"dataset\": DATASET, \"img_dim\": IMG_DIM, \"trainable\": True,\n", 1828 | "}" 1829 | ] 1830 | }, 1831 | { 1832 | "cell_type": "code", 1833 | "execution_count": null, 1834 | "metadata": {}, 1835 | "outputs": [], 1836 | "source": [ 1837 | "def train():\n", 1838 | " \"\"\"Run transfer learning on the configured model and dataset\"\"\"\n", 1839 | " global IMG_DIM, N_CLASSES, DS_CACHE\n", 1840 | " with wandb.init(project=\"m1-benchmark\", config=default_config) as run:\n", 1841 | " # Set global defaults when running in sweep mode\n", 1842 | " IMG_DIM = run.config.img_dim\n", 1843 | " N_CLASSES = run.config.num_classes\n", 1844 | " DS_CACHE = os.path.join(tempfile.mkdtemp(), str(hash(frozenset(run.config.items()))))\n", 1845 | "\n", 1846 | " # Setup base model to transfer from, optionally fine-tune\n", 1847 | " base_model = getattr(tf.keras.applications, run.config.base_model)(\n", 1848 | " input_shape=(run.config.img_dim, run.config.img_dim, 3),\n", 1849 | " include_top=False, weights='imagenet')\n", 1850 | " base_model.trainable = run.config.trainable\n", 1851 | "\n", 1852 | " # Decay learning rate\n", 1853 | " lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(\n", 1854 | " run.config.init_lr, decay_steps=run.config.train_size, decay_rate=run.config.decay)\n", 1855 | "\n", 1856 | " # Compile model for this dataset\n", 1857 | " model = tf.keras.Sequential([\n", 1858 | " base_model,\n", 1859 | " tf.keras.layers.GlobalAveragePooling2D(),\n", 1860 | " tf.keras.layers.Dropout(run.config.dropout),\n", 1861 | " tf.keras.layers.Dense(run.config.num_classes, activation='softmax')\n", 1862 | " ])\n", 1863 | " model.compile(optimizer=tf.keras.optimizers.Adam(lr_schedule),\n", 1864 | " loss='categorical_crossentropy',\n", 1865 | " metrics=['accuracy', 'top_k_categorical_accuracy'])\n", 1866 | " \n", 1867 | " # Update config and print summary\n", 1868 | " run.config.update({\n", 1869 | " \"total_params\": model.count_params(),\n", 1870 | " \"trainable_params\": trainable_params(model),\n", 1871 | " })\n", 1872 | " print(\"Model {}:\".format(run.config.base_model))\n", 1873 | " print(\" trainable parameters:\", run.config.trainable_params)\n", 1874 | " print(\" total parameters:\", run.config.total_params)\n", 1875 | " print(\"Dataset {}:\".format(run.config.dataset))\n", 1876 | " print(\" training: \", run.config.train_size)\n", 1877 | " print(\" test: \", run.config.test_size)\n", 1878 | " print(\" shape: {}\\n\".format((run.config.img_dim, run.config.img_dim, 3)))\n", 1879 | " \n", 1880 | " # Train the model\n", 1881 | " history = model.fit(x_train, y_train, \n", 1882 | " epochs=run.config.epochs, \n", 1883 | " validation_data=(x_test, y_test),\n", 1884 | " callbacks=[wandb.keras.WandbCallback(save_model=False)])" 1885 | ] 1886 | }, 1887 | { 1888 | "cell_type": "code", 1889 | "execution_count": null, 1890 | "metadata": {}, 1891 | "outputs": [], 1892 | "source": [ 1893 | "train()" 1894 | ] 1895 | }, 1896 | { 1897 | "cell_type": "code", 1898 | "execution_count": null, 1899 | "metadata": {}, 1900 | "outputs": [], 1901 | "source": [] 1902 | } 1903 | ], 1904 | "metadata": { 1905 | "accelerator": "GPU", 1906 | "colab": { 1907 | "collapsed_sections": [], 1908 | "name": "Basic Performance.ipynb", 1909 | "provenance": [] 1910 | }, 1911 | "kernelspec": { 1912 | "display_name": "Python 3 (ipykernel)", 1913 | "language": "python", 1914 | "name": "python3" 1915 | }, 1916 | "language_info": { 1917 | "codemirror_mode": { 1918 | "name": "ipython", 1919 | "version": 3 1920 | }, 1921 | "file_extension": ".py", 1922 | "mimetype": "text/x-python", 1923 | "name": "python", 1924 | "nbconvert_exporter": "python", 1925 | "pygments_lexer": "ipython3", 1926 | "version": "3.9.7" 1927 | }, 1928 | "widgets": { 1929 | "application/vnd.jupyter.widget-state+json": { 1930 | "058bb2545d34464b9ca0718261778026": { 1931 | "model_module": "@jupyter-widgets/base", 1932 | "model_module_version": "1.2.0", 1933 | "model_name": "LayoutModel", 1934 | "state": { 1935 | "_model_module": "@jupyter-widgets/base", 1936 | "_model_module_version": "1.2.0", 1937 | "_model_name": "LayoutModel", 1938 | "_view_count": null, 1939 | "_view_module": "@jupyter-widgets/base", 1940 | "_view_module_version": "1.2.0", 1941 | "_view_name": "LayoutView", 1942 | "align_content": null, 1943 | "align_items": null, 1944 | "align_self": null, 1945 | "border": null, 1946 | "bottom": null, 1947 | "display": null, 1948 | "flex": null, 1949 | "flex_flow": null, 1950 | "grid_area": null, 1951 | "grid_auto_columns": null, 1952 | "grid_auto_flow": null, 1953 | "grid_auto_rows": null, 1954 | "grid_column": null, 1955 | "grid_gap": null, 1956 | "grid_row": null, 1957 | "grid_template_areas": null, 1958 | "grid_template_columns": null, 1959 | "grid_template_rows": null, 1960 | "height": null, 1961 | "justify_content": null, 1962 | "justify_items": null, 1963 | "left": null, 1964 | "margin": null, 1965 | "max_height": null, 1966 | "max_width": null, 1967 | "min_height": null, 1968 | "min_width": null, 1969 | "object_fit": null, 1970 | "object_position": null, 1971 | "order": null, 1972 | "overflow": null, 1973 | "overflow_x": null, 1974 | "overflow_y": null, 1975 | "padding": null, 1976 | "right": null, 1977 | "top": null, 1978 | "visibility": null, 1979 | "width": null 1980 | } 1981 | }, 1982 | "066346a946674cb7bd869ae6feb86b5e": { 1983 | "model_module": "@jupyter-widgets/controls", 1984 | "model_module_version": "1.5.0", 1985 | "model_name": "LabelModel", 1986 | "state": { 1987 | "_dom_classes": [], 1988 | "_model_module": "@jupyter-widgets/controls", 1989 | "_model_module_version": "1.5.0", 1990 | "_model_name": "LabelModel", 1991 | "_view_count": null, 1992 | "_view_module": "@jupyter-widgets/controls", 1993 | "_view_module_version": "1.5.0", 1994 | "_view_name": "LabelView", 1995 | "description": "", 1996 | "description_tooltip": null, 1997 | "layout": "IPY_MODEL_53f3956ad0f945499c3daad7a104e392", 1998 | "placeholder": "​", 1999 | "style": "IPY_MODEL_ab368a038d174a9f979fcbb9f470b8c5", 2000 | "value": " 1.86MB of 1.86MB uploaded (0.00MB deduped)\r" 2001 | } 2002 | }, 2003 | "46da9173be364dafaaaae3123ed61653": { 2004 | "model_module": "@jupyter-widgets/controls", 2005 | "model_module_version": "1.5.0", 2006 | "model_name": "FloatProgressModel", 2007 | "state": { 2008 | "_dom_classes": [], 2009 | "_model_module": "@jupyter-widgets/controls", 2010 | "_model_module_version": "1.5.0", 2011 | "_model_name": "FloatProgressModel", 2012 | "_view_count": null, 2013 | "_view_module": "@jupyter-widgets/controls", 2014 | "_view_module_version": "1.5.0", 2015 | "_view_name": "ProgressView", 2016 | "bar_style": "", 2017 | "description": "", 2018 | "description_tooltip": null, 2019 | "layout": "IPY_MODEL_e4354c4c8da74a1ab1e44b987b221947", 2020 | "max": 1, 2021 | "min": 0, 2022 | "orientation": "horizontal", 2023 | "style": "IPY_MODEL_a08692e01d504d3fbf711c61f04f6ee6", 2024 | "value": 1 2025 | } 2026 | }, 2027 | "49c58f268866499c8ea5b5e86e7b629e": { 2028 | "model_module": "@jupyter-widgets/base", 2029 | "model_module_version": "1.2.0", 2030 | "model_name": "LayoutModel", 2031 | "state": { 2032 | "_model_module": "@jupyter-widgets/base", 2033 | "_model_module_version": "1.2.0", 2034 | "_model_name": "LayoutModel", 2035 | "_view_count": null, 2036 | "_view_module": "@jupyter-widgets/base", 2037 | "_view_module_version": "1.2.0", 2038 | "_view_name": "LayoutView", 2039 | "align_content": null, 2040 | "align_items": null, 2041 | "align_self": null, 2042 | "border": null, 2043 | "bottom": null, 2044 | "display": null, 2045 | "flex": null, 2046 | "flex_flow": null, 2047 | "grid_area": null, 2048 | "grid_auto_columns": null, 2049 | "grid_auto_flow": null, 2050 | "grid_auto_rows": null, 2051 | "grid_column": null, 2052 | "grid_gap": null, 2053 | "grid_row": null, 2054 | "grid_template_areas": null, 2055 | "grid_template_columns": null, 2056 | "grid_template_rows": null, 2057 | "height": null, 2058 | "justify_content": null, 2059 | "justify_items": null, 2060 | "left": null, 2061 | "margin": null, 2062 | "max_height": null, 2063 | "max_width": null, 2064 | "min_height": null, 2065 | "min_width": null, 2066 | "object_fit": null, 2067 | "object_position": null, 2068 | "order": null, 2069 | "overflow": null, 2070 | "overflow_x": null, 2071 | "overflow_y": null, 2072 | "padding": null, 2073 | "right": null, 2074 | "top": null, 2075 | "visibility": null, 2076 | "width": null 2077 | } 2078 | }, 2079 | "4c181783977f4ebf9e1a2a49c3ec0120": { 2080 | "model_module": "@jupyter-widgets/controls", 2081 | "model_module_version": "1.5.0", 2082 | "model_name": "FloatProgressModel", 2083 | "state": { 2084 | "_dom_classes": [], 2085 | "_model_module": "@jupyter-widgets/controls", 2086 | "_model_module_version": "1.5.0", 2087 | "_model_name": "FloatProgressModel", 2088 | "_view_count": null, 2089 | "_view_module": "@jupyter-widgets/controls", 2090 | "_view_module_version": "1.5.0", 2091 | "_view_name": "ProgressView", 2092 | "bar_style": "", 2093 | "description": "", 2094 | "description_tooltip": null, 2095 | "layout": "IPY_MODEL_623e060e7e514a30aadda61cb7c1674b", 2096 | "max": 1, 2097 | "min": 0, 2098 | "orientation": "horizontal", 2099 | "style": "IPY_MODEL_c418d6652e43445da8e239f0ca859846", 2100 | "value": 1 2101 | } 2102 | }, 2103 | "53f3956ad0f945499c3daad7a104e392": { 2104 | "model_module": "@jupyter-widgets/base", 2105 | "model_module_version": "1.2.0", 2106 | "model_name": "LayoutModel", 2107 | "state": { 2108 | "_model_module": "@jupyter-widgets/base", 2109 | "_model_module_version": "1.2.0", 2110 | "_model_name": "LayoutModel", 2111 | "_view_count": null, 2112 | "_view_module": "@jupyter-widgets/base", 2113 | "_view_module_version": "1.2.0", 2114 | "_view_name": "LayoutView", 2115 | "align_content": null, 2116 | "align_items": null, 2117 | "align_self": null, 2118 | "border": null, 2119 | "bottom": null, 2120 | "display": null, 2121 | "flex": null, 2122 | "flex_flow": null, 2123 | "grid_area": null, 2124 | "grid_auto_columns": null, 2125 | "grid_auto_flow": null, 2126 | "grid_auto_rows": null, 2127 | "grid_column": null, 2128 | "grid_gap": null, 2129 | "grid_row": null, 2130 | "grid_template_areas": null, 2131 | "grid_template_columns": null, 2132 | "grid_template_rows": null, 2133 | "height": null, 2134 | "justify_content": null, 2135 | "justify_items": null, 2136 | "left": null, 2137 | "margin": null, 2138 | "max_height": null, 2139 | "max_width": null, 2140 | "min_height": null, 2141 | "min_width": null, 2142 | "object_fit": null, 2143 | "object_position": null, 2144 | "order": null, 2145 | "overflow": null, 2146 | "overflow_x": null, 2147 | "overflow_y": null, 2148 | "padding": null, 2149 | "right": null, 2150 | "top": null, 2151 | "visibility": null, 2152 | "width": null 2153 | } 2154 | }, 2155 | "5adc2846a13b4a3d981cbd7fc0769cab": { 2156 | "model_module": "@jupyter-widgets/controls", 2157 | "model_module_version": "1.5.0", 2158 | "model_name": "DescriptionStyleModel", 2159 | "state": { 2160 | "_model_module": "@jupyter-widgets/controls", 2161 | "_model_module_version": "1.5.0", 2162 | "_model_name": "DescriptionStyleModel", 2163 | "_view_count": null, 2164 | "_view_module": "@jupyter-widgets/base", 2165 | "_view_module_version": "1.2.0", 2166 | "_view_name": "StyleView", 2167 | "description_width": "" 2168 | } 2169 | }, 2170 | "60170335e6514507bd87c6a4dac3d623": { 2171 | "model_module": "@jupyter-widgets/controls", 2172 | "model_module_version": "1.5.0", 2173 | "model_name": "DescriptionStyleModel", 2174 | "state": { 2175 | "_model_module": "@jupyter-widgets/controls", 2176 | "_model_module_version": "1.5.0", 2177 | "_model_name": "DescriptionStyleModel", 2178 | "_view_count": null, 2179 | "_view_module": "@jupyter-widgets/base", 2180 | "_view_module_version": "1.2.0", 2181 | "_view_name": "StyleView", 2182 | "description_width": "" 2183 | } 2184 | }, 2185 | "623e060e7e514a30aadda61cb7c1674b": { 2186 | "model_module": "@jupyter-widgets/base", 2187 | "model_module_version": "1.2.0", 2188 | "model_name": "LayoutModel", 2189 | "state": { 2190 | "_model_module": "@jupyter-widgets/base", 2191 | "_model_module_version": "1.2.0", 2192 | "_model_name": "LayoutModel", 2193 | "_view_count": null, 2194 | "_view_module": "@jupyter-widgets/base", 2195 | "_view_module_version": "1.2.0", 2196 | "_view_name": "LayoutView", 2197 | "align_content": null, 2198 | "align_items": null, 2199 | "align_self": null, 2200 | "border": null, 2201 | "bottom": null, 2202 | "display": null, 2203 | "flex": null, 2204 | "flex_flow": null, 2205 | "grid_area": null, 2206 | "grid_auto_columns": null, 2207 | "grid_auto_flow": null, 2208 | "grid_auto_rows": null, 2209 | "grid_column": null, 2210 | "grid_gap": null, 2211 | "grid_row": null, 2212 | "grid_template_areas": null, 2213 | "grid_template_columns": null, 2214 | "grid_template_rows": null, 2215 | "height": null, 2216 | "justify_content": null, 2217 | "justify_items": null, 2218 | "left": null, 2219 | "margin": null, 2220 | "max_height": null, 2221 | "max_width": null, 2222 | "min_height": null, 2223 | "min_width": null, 2224 | "object_fit": null, 2225 | "object_position": null, 2226 | "order": null, 2227 | "overflow": null, 2228 | "overflow_x": null, 2229 | "overflow_y": null, 2230 | "padding": null, 2231 | "right": null, 2232 | "top": null, 2233 | "visibility": null, 2234 | "width": null 2235 | } 2236 | }, 2237 | "7b6f698700794454bcdd0cb4e5881b3e": { 2238 | "model_module": "@jupyter-widgets/controls", 2239 | "model_module_version": "1.5.0", 2240 | "model_name": "VBoxModel", 2241 | "state": { 2242 | "_dom_classes": [], 2243 | "_model_module": "@jupyter-widgets/controls", 2244 | "_model_module_version": "1.5.0", 2245 | "_model_name": "VBoxModel", 2246 | "_view_count": null, 2247 | "_view_module": "@jupyter-widgets/controls", 2248 | "_view_module_version": "1.5.0", 2249 | "_view_name": "VBoxView", 2250 | "box_style": "", 2251 | "children": [ 2252 | "IPY_MODEL_066346a946674cb7bd869ae6feb86b5e", 2253 | "IPY_MODEL_bd8c6c0cd07a4c4782db78b6f3c529cb" 2254 | ], 2255 | "layout": "IPY_MODEL_f580aa5ff4ec41c6ae48419661333f7f" 2256 | } 2257 | }, 2258 | "7cbfd13104c44eba8aa1e8d1fb2fc7f1": { 2259 | "model_module": "@jupyter-widgets/base", 2260 | "model_module_version": "1.2.0", 2261 | "model_name": "LayoutModel", 2262 | "state": { 2263 | "_model_module": "@jupyter-widgets/base", 2264 | "_model_module_version": "1.2.0", 2265 | "_model_name": "LayoutModel", 2266 | "_view_count": null, 2267 | "_view_module": "@jupyter-widgets/base", 2268 | "_view_module_version": "1.2.0", 2269 | "_view_name": "LayoutView", 2270 | "align_content": null, 2271 | "align_items": null, 2272 | "align_self": null, 2273 | "border": null, 2274 | "bottom": null, 2275 | "display": null, 2276 | "flex": null, 2277 | "flex_flow": null, 2278 | "grid_area": null, 2279 | "grid_auto_columns": null, 2280 | "grid_auto_flow": null, 2281 | "grid_auto_rows": null, 2282 | "grid_column": null, 2283 | "grid_gap": null, 2284 | "grid_row": null, 2285 | "grid_template_areas": null, 2286 | "grid_template_columns": null, 2287 | "grid_template_rows": null, 2288 | "height": null, 2289 | "justify_content": null, 2290 | "justify_items": null, 2291 | "left": null, 2292 | "margin": null, 2293 | "max_height": null, 2294 | "max_width": null, 2295 | "min_height": null, 2296 | "min_width": null, 2297 | "object_fit": null, 2298 | "object_position": null, 2299 | "order": null, 2300 | "overflow": null, 2301 | "overflow_x": null, 2302 | "overflow_y": null, 2303 | "padding": null, 2304 | "right": null, 2305 | "top": null, 2306 | "visibility": null, 2307 | "width": null 2308 | } 2309 | }, 2310 | "7d2279098f384e2eb2dbeaf53cde52c1": { 2311 | "model_module": "@jupyter-widgets/controls", 2312 | "model_module_version": "1.5.0", 2313 | "model_name": "VBoxModel", 2314 | "state": { 2315 | "_dom_classes": [], 2316 | "_model_module": "@jupyter-widgets/controls", 2317 | "_model_module_version": "1.5.0", 2318 | "_model_name": "VBoxModel", 2319 | "_view_count": null, 2320 | "_view_module": "@jupyter-widgets/controls", 2321 | "_view_module_version": "1.5.0", 2322 | "_view_name": "VBoxView", 2323 | "box_style": "", 2324 | "children": [ 2325 | "IPY_MODEL_8b5f7d4f32c345de90ed6d1875bd6bd1", 2326 | "IPY_MODEL_f9129b445ddf4aee859db77ea883c8fe" 2327 | ], 2328 | "layout": "IPY_MODEL_a0273718b661442b8d1334943460ede6" 2329 | } 2330 | }, 2331 | "7d2be1b24f2841fa93c57aae78f31a15": { 2332 | "model_module": "@jupyter-widgets/controls", 2333 | "model_module_version": "1.5.0", 2334 | "model_name": "LabelModel", 2335 | "state": { 2336 | "_dom_classes": [], 2337 | "_model_module": "@jupyter-widgets/controls", 2338 | "_model_module_version": "1.5.0", 2339 | "_model_name": "LabelModel", 2340 | "_view_count": null, 2341 | "_view_module": "@jupyter-widgets/controls", 2342 | "_view_module_version": "1.5.0", 2343 | "_view_name": "LabelView", 2344 | "description": "", 2345 | "description_tooltip": null, 2346 | "layout": "IPY_MODEL_058bb2545d34464b9ca0718261778026", 2347 | "placeholder": "​", 2348 | "style": "IPY_MODEL_60170335e6514507bd87c6a4dac3d623", 2349 | "value": " 1.79MB of 1.79MB uploaded (0.00MB deduped)\r" 2350 | } 2351 | }, 2352 | "7d62fc41e1a94a5eb9df853f7a2f16c2": { 2353 | "model_module": "@jupyter-widgets/controls", 2354 | "model_module_version": "1.5.0", 2355 | "model_name": "VBoxModel", 2356 | "state": { 2357 | "_dom_classes": [], 2358 | "_model_module": "@jupyter-widgets/controls", 2359 | "_model_module_version": "1.5.0", 2360 | "_model_name": "VBoxModel", 2361 | "_view_count": null, 2362 | "_view_module": "@jupyter-widgets/controls", 2363 | "_view_module_version": "1.5.0", 2364 | "_view_name": "VBoxView", 2365 | "box_style": "", 2366 | "children": [ 2367 | "IPY_MODEL_e3cfed5314bb4867a4241cff2e0adac8", 2368 | "IPY_MODEL_46da9173be364dafaaaae3123ed61653" 2369 | ], 2370 | "layout": "IPY_MODEL_93f911153ab54aa0b625794462a91ad5" 2371 | } 2372 | }, 2373 | "8b5f7d4f32c345de90ed6d1875bd6bd1": { 2374 | "model_module": "@jupyter-widgets/controls", 2375 | "model_module_version": "1.5.0", 2376 | "model_name": "LabelModel", 2377 | "state": { 2378 | "_dom_classes": [], 2379 | "_model_module": "@jupyter-widgets/controls", 2380 | "_model_module_version": "1.5.0", 2381 | "_model_name": "LabelModel", 2382 | "_view_count": null, 2383 | "_view_module": "@jupyter-widgets/controls", 2384 | "_view_module_version": "1.5.0", 2385 | "_view_name": "LabelView", 2386 | "description": "", 2387 | "description_tooltip": null, 2388 | "layout": "IPY_MODEL_ca45fa9c101d4051860f176bf1bab1da", 2389 | "placeholder": "​", 2390 | "style": "IPY_MODEL_5adc2846a13b4a3d981cbd7fc0769cab", 2391 | "value": " 0.92MB of 0.92MB uploaded (0.00MB deduped)\r" 2392 | } 2393 | }, 2394 | "93f911153ab54aa0b625794462a91ad5": { 2395 | "model_module": "@jupyter-widgets/base", 2396 | "model_module_version": "1.2.0", 2397 | "model_name": "LayoutModel", 2398 | "state": { 2399 | "_model_module": "@jupyter-widgets/base", 2400 | "_model_module_version": "1.2.0", 2401 | "_model_name": "LayoutModel", 2402 | "_view_count": null, 2403 | "_view_module": "@jupyter-widgets/base", 2404 | "_view_module_version": "1.2.0", 2405 | "_view_name": "LayoutView", 2406 | "align_content": null, 2407 | "align_items": null, 2408 | "align_self": null, 2409 | "border": null, 2410 | "bottom": null, 2411 | "display": null, 2412 | "flex": null, 2413 | "flex_flow": null, 2414 | "grid_area": null, 2415 | "grid_auto_columns": null, 2416 | "grid_auto_flow": null, 2417 | "grid_auto_rows": null, 2418 | "grid_column": null, 2419 | "grid_gap": null, 2420 | "grid_row": null, 2421 | "grid_template_areas": null, 2422 | "grid_template_columns": null, 2423 | "grid_template_rows": null, 2424 | "height": null, 2425 | "justify_content": null, 2426 | "justify_items": null, 2427 | "left": null, 2428 | "margin": null, 2429 | "max_height": null, 2430 | "max_width": null, 2431 | "min_height": null, 2432 | "min_width": null, 2433 | "object_fit": null, 2434 | "object_position": null, 2435 | "order": null, 2436 | "overflow": null, 2437 | "overflow_x": null, 2438 | "overflow_y": null, 2439 | "padding": null, 2440 | "right": null, 2441 | "top": null, 2442 | "visibility": null, 2443 | "width": null 2444 | } 2445 | }, 2446 | "9584b9a224444b7685e3eb883fa71015": { 2447 | "model_module": "@jupyter-widgets/base", 2448 | "model_module_version": "1.2.0", 2449 | "model_name": "LayoutModel", 2450 | "state": { 2451 | "_model_module": "@jupyter-widgets/base", 2452 | "_model_module_version": "1.2.0", 2453 | "_model_name": "LayoutModel", 2454 | "_view_count": null, 2455 | "_view_module": "@jupyter-widgets/base", 2456 | "_view_module_version": "1.2.0", 2457 | "_view_name": "LayoutView", 2458 | "align_content": null, 2459 | "align_items": null, 2460 | "align_self": null, 2461 | "border": null, 2462 | "bottom": null, 2463 | "display": null, 2464 | "flex": null, 2465 | "flex_flow": null, 2466 | "grid_area": null, 2467 | "grid_auto_columns": null, 2468 | "grid_auto_flow": null, 2469 | "grid_auto_rows": null, 2470 | "grid_column": null, 2471 | "grid_gap": null, 2472 | "grid_row": null, 2473 | "grid_template_areas": null, 2474 | "grid_template_columns": null, 2475 | "grid_template_rows": null, 2476 | "height": null, 2477 | "justify_content": null, 2478 | "justify_items": null, 2479 | "left": null, 2480 | "margin": null, 2481 | "max_height": null, 2482 | "max_width": null, 2483 | "min_height": null, 2484 | "min_width": null, 2485 | "object_fit": null, 2486 | "object_position": null, 2487 | "order": null, 2488 | "overflow": null, 2489 | "overflow_x": null, 2490 | "overflow_y": null, 2491 | "padding": null, 2492 | "right": null, 2493 | "top": null, 2494 | "visibility": null, 2495 | "width": null 2496 | } 2497 | }, 2498 | "97651ef84a5d4f11a7e75670ce8300e7": { 2499 | "model_module": "@jupyter-widgets/controls", 2500 | "model_module_version": "1.5.0", 2501 | "model_name": "VBoxModel", 2502 | "state": { 2503 | "_dom_classes": [], 2504 | "_model_module": "@jupyter-widgets/controls", 2505 | "_model_module_version": "1.5.0", 2506 | "_model_name": "VBoxModel", 2507 | "_view_count": null, 2508 | "_view_module": "@jupyter-widgets/controls", 2509 | "_view_module_version": "1.5.0", 2510 | "_view_name": "VBoxView", 2511 | "box_style": "", 2512 | "children": [ 2513 | "IPY_MODEL_7d2be1b24f2841fa93c57aae78f31a15", 2514 | "IPY_MODEL_4c181783977f4ebf9e1a2a49c3ec0120" 2515 | ], 2516 | "layout": "IPY_MODEL_b479a3899aba49fd99294d9df2eec701" 2517 | } 2518 | }, 2519 | "a0273718b661442b8d1334943460ede6": { 2520 | "model_module": "@jupyter-widgets/base", 2521 | "model_module_version": "1.2.0", 2522 | "model_name": "LayoutModel", 2523 | "state": { 2524 | "_model_module": "@jupyter-widgets/base", 2525 | "_model_module_version": "1.2.0", 2526 | "_model_name": "LayoutModel", 2527 | "_view_count": null, 2528 | "_view_module": "@jupyter-widgets/base", 2529 | "_view_module_version": "1.2.0", 2530 | "_view_name": "LayoutView", 2531 | "align_content": null, 2532 | "align_items": null, 2533 | "align_self": null, 2534 | "border": null, 2535 | "bottom": null, 2536 | "display": null, 2537 | "flex": null, 2538 | "flex_flow": null, 2539 | "grid_area": null, 2540 | "grid_auto_columns": null, 2541 | "grid_auto_flow": null, 2542 | "grid_auto_rows": null, 2543 | "grid_column": null, 2544 | "grid_gap": null, 2545 | "grid_row": null, 2546 | "grid_template_areas": null, 2547 | "grid_template_columns": null, 2548 | "grid_template_rows": null, 2549 | "height": null, 2550 | "justify_content": null, 2551 | "justify_items": null, 2552 | "left": null, 2553 | "margin": null, 2554 | "max_height": null, 2555 | "max_width": null, 2556 | "min_height": null, 2557 | "min_width": null, 2558 | "object_fit": null, 2559 | "object_position": null, 2560 | "order": null, 2561 | "overflow": null, 2562 | "overflow_x": null, 2563 | "overflow_y": null, 2564 | "padding": null, 2565 | "right": null, 2566 | "top": null, 2567 | "visibility": null, 2568 | "width": null 2569 | } 2570 | }, 2571 | "a08692e01d504d3fbf711c61f04f6ee6": { 2572 | "model_module": "@jupyter-widgets/controls", 2573 | "model_module_version": "1.5.0", 2574 | "model_name": "ProgressStyleModel", 2575 | "state": { 2576 | "_model_module": "@jupyter-widgets/controls", 2577 | "_model_module_version": "1.5.0", 2578 | "_model_name": "ProgressStyleModel", 2579 | "_view_count": null, 2580 | "_view_module": "@jupyter-widgets/base", 2581 | "_view_module_version": "1.2.0", 2582 | "_view_name": "StyleView", 2583 | "bar_color": null, 2584 | "description_width": "" 2585 | } 2586 | }, 2587 | "a8eb7ef889934f5ba61591bc338aafcd": { 2588 | "model_module": "@jupyter-widgets/controls", 2589 | "model_module_version": "1.5.0", 2590 | "model_name": "ProgressStyleModel", 2591 | "state": { 2592 | "_model_module": "@jupyter-widgets/controls", 2593 | "_model_module_version": "1.5.0", 2594 | "_model_name": "ProgressStyleModel", 2595 | "_view_count": null, 2596 | "_view_module": "@jupyter-widgets/base", 2597 | "_view_module_version": "1.2.0", 2598 | "_view_name": "StyleView", 2599 | "bar_color": null, 2600 | "description_width": "" 2601 | } 2602 | }, 2603 | "ab368a038d174a9f979fcbb9f470b8c5": { 2604 | "model_module": "@jupyter-widgets/controls", 2605 | "model_module_version": "1.5.0", 2606 | "model_name": "DescriptionStyleModel", 2607 | "state": { 2608 | "_model_module": "@jupyter-widgets/controls", 2609 | "_model_module_version": "1.5.0", 2610 | "_model_name": "DescriptionStyleModel", 2611 | "_view_count": null, 2612 | "_view_module": "@jupyter-widgets/base", 2613 | "_view_module_version": "1.2.0", 2614 | "_view_name": "StyleView", 2615 | "description_width": "" 2616 | } 2617 | }, 2618 | "b479a3899aba49fd99294d9df2eec701": { 2619 | "model_module": "@jupyter-widgets/base", 2620 | "model_module_version": "1.2.0", 2621 | "model_name": "LayoutModel", 2622 | "state": { 2623 | "_model_module": "@jupyter-widgets/base", 2624 | "_model_module_version": "1.2.0", 2625 | "_model_name": "LayoutModel", 2626 | "_view_count": null, 2627 | "_view_module": "@jupyter-widgets/base", 2628 | "_view_module_version": "1.2.0", 2629 | "_view_name": "LayoutView", 2630 | "align_content": null, 2631 | "align_items": null, 2632 | "align_self": null, 2633 | "border": null, 2634 | "bottom": null, 2635 | "display": null, 2636 | "flex": null, 2637 | "flex_flow": null, 2638 | "grid_area": null, 2639 | "grid_auto_columns": null, 2640 | "grid_auto_flow": null, 2641 | "grid_auto_rows": null, 2642 | "grid_column": null, 2643 | "grid_gap": null, 2644 | "grid_row": null, 2645 | "grid_template_areas": null, 2646 | "grid_template_columns": null, 2647 | "grid_template_rows": null, 2648 | "height": null, 2649 | "justify_content": null, 2650 | "justify_items": null, 2651 | "left": null, 2652 | "margin": null, 2653 | "max_height": null, 2654 | "max_width": null, 2655 | "min_height": null, 2656 | "min_width": null, 2657 | "object_fit": null, 2658 | "object_position": null, 2659 | "order": null, 2660 | "overflow": null, 2661 | "overflow_x": null, 2662 | "overflow_y": null, 2663 | "padding": null, 2664 | "right": null, 2665 | "top": null, 2666 | "visibility": null, 2667 | "width": null 2668 | } 2669 | }, 2670 | "bd8c6c0cd07a4c4782db78b6f3c529cb": { 2671 | "model_module": "@jupyter-widgets/controls", 2672 | "model_module_version": "1.5.0", 2673 | "model_name": "FloatProgressModel", 2674 | "state": { 2675 | "_dom_classes": [], 2676 | "_model_module": "@jupyter-widgets/controls", 2677 | "_model_module_version": "1.5.0", 2678 | "_model_name": "FloatProgressModel", 2679 | "_view_count": null, 2680 | "_view_module": "@jupyter-widgets/controls", 2681 | "_view_module_version": "1.5.0", 2682 | "_view_name": "ProgressView", 2683 | "bar_style": "", 2684 | "description": "", 2685 | "description_tooltip": null, 2686 | "layout": "IPY_MODEL_49c58f268866499c8ea5b5e86e7b629e", 2687 | "max": 1, 2688 | "min": 0, 2689 | "orientation": "horizontal", 2690 | "style": "IPY_MODEL_a8eb7ef889934f5ba61591bc338aafcd", 2691 | "value": 1 2692 | } 2693 | }, 2694 | "bf7a7a2133994732bfa850f7eb51fdf0": { 2695 | "model_module": "@jupyter-widgets/controls", 2696 | "model_module_version": "1.5.0", 2697 | "model_name": "ProgressStyleModel", 2698 | "state": { 2699 | "_model_module": "@jupyter-widgets/controls", 2700 | "_model_module_version": "1.5.0", 2701 | "_model_name": "ProgressStyleModel", 2702 | "_view_count": null, 2703 | "_view_module": "@jupyter-widgets/base", 2704 | "_view_module_version": "1.2.0", 2705 | "_view_name": "StyleView", 2706 | "bar_color": null, 2707 | "description_width": "" 2708 | } 2709 | }, 2710 | "bf9c745dd3784c2ab2562c5a6e1e002e": { 2711 | "model_module": "@jupyter-widgets/controls", 2712 | "model_module_version": "1.5.0", 2713 | "model_name": "DescriptionStyleModel", 2714 | "state": { 2715 | "_model_module": "@jupyter-widgets/controls", 2716 | "_model_module_version": "1.5.0", 2717 | "_model_name": "DescriptionStyleModel", 2718 | "_view_count": null, 2719 | "_view_module": "@jupyter-widgets/base", 2720 | "_view_module_version": "1.2.0", 2721 | "_view_name": "StyleView", 2722 | "description_width": "" 2723 | } 2724 | }, 2725 | "c418d6652e43445da8e239f0ca859846": { 2726 | "model_module": "@jupyter-widgets/controls", 2727 | "model_module_version": "1.5.0", 2728 | "model_name": "ProgressStyleModel", 2729 | "state": { 2730 | "_model_module": "@jupyter-widgets/controls", 2731 | "_model_module_version": "1.5.0", 2732 | "_model_name": "ProgressStyleModel", 2733 | "_view_count": null, 2734 | "_view_module": "@jupyter-widgets/base", 2735 | "_view_module_version": "1.2.0", 2736 | "_view_name": "StyleView", 2737 | "bar_color": null, 2738 | "description_width": "" 2739 | } 2740 | }, 2741 | "ca45fa9c101d4051860f176bf1bab1da": { 2742 | "model_module": "@jupyter-widgets/base", 2743 | "model_module_version": "1.2.0", 2744 | "model_name": "LayoutModel", 2745 | "state": { 2746 | "_model_module": "@jupyter-widgets/base", 2747 | "_model_module_version": "1.2.0", 2748 | "_model_name": "LayoutModel", 2749 | "_view_count": null, 2750 | "_view_module": "@jupyter-widgets/base", 2751 | "_view_module_version": "1.2.0", 2752 | "_view_name": "LayoutView", 2753 | "align_content": null, 2754 | "align_items": null, 2755 | "align_self": null, 2756 | "border": null, 2757 | "bottom": null, 2758 | "display": null, 2759 | "flex": null, 2760 | "flex_flow": null, 2761 | "grid_area": null, 2762 | "grid_auto_columns": null, 2763 | "grid_auto_flow": null, 2764 | "grid_auto_rows": null, 2765 | "grid_column": null, 2766 | "grid_gap": null, 2767 | "grid_row": null, 2768 | "grid_template_areas": null, 2769 | "grid_template_columns": null, 2770 | "grid_template_rows": null, 2771 | "height": null, 2772 | "justify_content": null, 2773 | "justify_items": null, 2774 | "left": null, 2775 | "margin": null, 2776 | "max_height": null, 2777 | "max_width": null, 2778 | "min_height": null, 2779 | "min_width": null, 2780 | "object_fit": null, 2781 | "object_position": null, 2782 | "order": null, 2783 | "overflow": null, 2784 | "overflow_x": null, 2785 | "overflow_y": null, 2786 | "padding": null, 2787 | "right": null, 2788 | "top": null, 2789 | "visibility": null, 2790 | "width": null 2791 | } 2792 | }, 2793 | "e3cfed5314bb4867a4241cff2e0adac8": { 2794 | "model_module": "@jupyter-widgets/controls", 2795 | "model_module_version": "1.5.0", 2796 | "model_name": "LabelModel", 2797 | "state": { 2798 | "_dom_classes": [], 2799 | "_model_module": "@jupyter-widgets/controls", 2800 | "_model_module_version": "1.5.0", 2801 | "_model_name": "LabelModel", 2802 | "_view_count": null, 2803 | "_view_module": "@jupyter-widgets/controls", 2804 | "_view_module_version": "1.5.0", 2805 | "_view_name": "LabelView", 2806 | "description": "", 2807 | "description_tooltip": null, 2808 | "layout": "IPY_MODEL_9584b9a224444b7685e3eb883fa71015", 2809 | "placeholder": "​", 2810 | "style": "IPY_MODEL_bf9c745dd3784c2ab2562c5a6e1e002e", 2811 | "value": " 1.61MB of 1.61MB uploaded (0.00MB deduped)\r" 2812 | } 2813 | }, 2814 | "e4354c4c8da74a1ab1e44b987b221947": { 2815 | "model_module": "@jupyter-widgets/base", 2816 | "model_module_version": "1.2.0", 2817 | "model_name": "LayoutModel", 2818 | "state": { 2819 | "_model_module": "@jupyter-widgets/base", 2820 | "_model_module_version": "1.2.0", 2821 | "_model_name": "LayoutModel", 2822 | "_view_count": null, 2823 | "_view_module": "@jupyter-widgets/base", 2824 | "_view_module_version": "1.2.0", 2825 | "_view_name": "LayoutView", 2826 | "align_content": null, 2827 | "align_items": null, 2828 | "align_self": null, 2829 | "border": null, 2830 | "bottom": null, 2831 | "display": null, 2832 | "flex": null, 2833 | "flex_flow": null, 2834 | "grid_area": null, 2835 | "grid_auto_columns": null, 2836 | "grid_auto_flow": null, 2837 | "grid_auto_rows": null, 2838 | "grid_column": null, 2839 | "grid_gap": null, 2840 | "grid_row": null, 2841 | "grid_template_areas": null, 2842 | "grid_template_columns": null, 2843 | "grid_template_rows": null, 2844 | "height": null, 2845 | "justify_content": null, 2846 | "justify_items": null, 2847 | "left": null, 2848 | "margin": null, 2849 | "max_height": null, 2850 | "max_width": null, 2851 | "min_height": null, 2852 | "min_width": null, 2853 | "object_fit": null, 2854 | "object_position": null, 2855 | "order": null, 2856 | "overflow": null, 2857 | "overflow_x": null, 2858 | "overflow_y": null, 2859 | "padding": null, 2860 | "right": null, 2861 | "top": null, 2862 | "visibility": null, 2863 | "width": null 2864 | } 2865 | }, 2866 | "f580aa5ff4ec41c6ae48419661333f7f": { 2867 | "model_module": "@jupyter-widgets/base", 2868 | "model_module_version": "1.2.0", 2869 | "model_name": "LayoutModel", 2870 | "state": { 2871 | "_model_module": "@jupyter-widgets/base", 2872 | "_model_module_version": "1.2.0", 2873 | "_model_name": "LayoutModel", 2874 | "_view_count": null, 2875 | "_view_module": "@jupyter-widgets/base", 2876 | "_view_module_version": "1.2.0", 2877 | "_view_name": "LayoutView", 2878 | "align_content": null, 2879 | "align_items": null, 2880 | "align_self": null, 2881 | "border": null, 2882 | "bottom": null, 2883 | "display": null, 2884 | "flex": null, 2885 | "flex_flow": null, 2886 | "grid_area": null, 2887 | "grid_auto_columns": null, 2888 | "grid_auto_flow": null, 2889 | "grid_auto_rows": null, 2890 | "grid_column": null, 2891 | "grid_gap": null, 2892 | "grid_row": null, 2893 | "grid_template_areas": null, 2894 | "grid_template_columns": null, 2895 | "grid_template_rows": null, 2896 | "height": null, 2897 | "justify_content": null, 2898 | "justify_items": null, 2899 | "left": null, 2900 | "margin": null, 2901 | "max_height": null, 2902 | "max_width": null, 2903 | "min_height": null, 2904 | "min_width": null, 2905 | "object_fit": null, 2906 | "object_position": null, 2907 | "order": null, 2908 | "overflow": null, 2909 | "overflow_x": null, 2910 | "overflow_y": null, 2911 | "padding": null, 2912 | "right": null, 2913 | "top": null, 2914 | "visibility": null, 2915 | "width": null 2916 | } 2917 | }, 2918 | "f9129b445ddf4aee859db77ea883c8fe": { 2919 | "model_module": "@jupyter-widgets/controls", 2920 | "model_module_version": "1.5.0", 2921 | "model_name": "FloatProgressModel", 2922 | "state": { 2923 | "_dom_classes": [], 2924 | "_model_module": "@jupyter-widgets/controls", 2925 | "_model_module_version": "1.5.0", 2926 | "_model_name": "FloatProgressModel", 2927 | "_view_count": null, 2928 | "_view_module": "@jupyter-widgets/controls", 2929 | "_view_module_version": "1.5.0", 2930 | "_view_name": "ProgressView", 2931 | "bar_style": "", 2932 | "description": "", 2933 | "description_tooltip": null, 2934 | "layout": "IPY_MODEL_7cbfd13104c44eba8aa1e8d1fb2fc7f1", 2935 | "max": 1, 2936 | "min": 0, 2937 | "orientation": "horizontal", 2938 | "style": "IPY_MODEL_bf7a7a2133994732bfa850f7eb51fdf0", 2939 | "value": 1 2940 | } 2941 | } 2942 | } 2943 | } 2944 | }, 2945 | "nbformat": 4, 2946 | "nbformat_minor": 4 2947 | } 2948 | -------------------------------------------------------------------------------- /tensorflow/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Macbook Pro M1 setup for Tensorflow 3 | > How to setup python and DL libs on your new macbook pro 4 | 5 | The new apple silicon is pretty amazing, it is fast and very power efficient, but does it work for data science? The first thing you need is to install python. ~~To do so, refer to this [video](https://www.youtube.com/watch?v=w2qlou7n7MA&list=RDCMUCR1-GEpyOPzT2AO4D_eifdw&index=1) from the amazing Jeff Heaton.~~ 6 | 7 | ## Install 8 | 9 | - Install apple developer tools: Just git clone something, and the install will be triggered. 10 | - Install [miniforge](https://github.com/conda-forge/miniforge): This is a fork of miniconda that has the default channel `conda-forge` (instead of conda default), this is a good thing as almost every package you want to use is in this channel. 11 | > Note: Don't forget to choose the ARM M1 binaries 12 | 13 | ## Environment setup (TLDR) 14 | 15 | You can now create you environment and start working! 16 | 17 | ```bash 18 | $ curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh 19 | $ sh Miniforge3-MacOSX-arm64.sh 20 | # follow instructions and init conda at the end, restart your terminal 21 | # this will create an environment called `tf` with python 22 | $ conda create --name=tf "python<3.11" 23 | 24 | # activate the environment 25 | $ conda activate tf 26 | $ conda install -c apple tensorflow-deps 27 | $ pip install tensorflow-macos tensorflow-metal 28 | 29 | # install benchmark dependencies 30 | $ pip install wandb transformers datasets tqdm scikit-learn 31 | ``` 32 | 33 | ## Apple M1 Tensorflow 34 | Apple has made binaries for tensorflow 2 that supports the GPU inside the M1 processor. This makes training way faster than CPU. You need can grab tensorflow install intruction from the apple website [here](https://developer.apple.com/metal/tensorflow-plugin/) or use the environment files that are provided here. ([tf_apple.yml](tf_apple.yml)). I also provide a [linux env file](tf_linux.yml) in case you want to try. 35 | 36 | ## Benchmarks 37 | 38 | You can check some runs on this [report](http://wandb.me/m1pro). To run the benchmark yourself in your new macbook pro: 39 | - setup the enviroment: 40 | ```bash 41 | conda env create --file=tf_apple.yml 42 | conda activate tf 43 | ``` 44 | - You will need a [wandb][wandb.ai] account, follow instructions once the script is launched. 45 | - Run the training: 46 | 47 | ## Running the Benchmark on Oxford PETS Resnet50 Training 48 | In your terminal run: 49 | 50 | ```bash 51 | $ python train_pets.py --gpu_name="M1Pro GPU 16 Cores" #replace with your GPU name 52 | ``` 53 | 54 | - Pass the `--gpu_name` flag to group the runs, I am not able to detect this automatically on Apple. 55 | - To run on cpu pass `--device="cpu"` or for CUDA `--device="cuda"` (you need a linux PC with an Nvidia GPU) 56 | - You can also pass other params, and play with different `batch_size` and `model_name`. 57 | 58 | 59 | ## Bert Benchmark 60 | 61 | In your terminal run: 62 | 63 | ```bash 64 | $ python train_bert.py --gpu_name="M1Pro GPU 16 Cores" #replace with your GPU name 65 | ``` 66 | 67 | 68 | ## NGC Docker 69 | 70 | We can also run the benchmarks on linux using nvidia docker [containers](https://docs.nvidia.com/deeplearning/frameworks/user-guide/index.html#runcont):: 71 | 72 | - Install `docker` with [following official nvidia documentation](https://docs.nvidia.com/ai-enterprise/deployment-guide/dg-docker.html). Once the installation of docker and nvidia support is complete, you can run the tensorflow container. You cant test that your setup works correctly by running the dummy cuda container and yo should see the `nvidia-smi` output: 73 | 74 | ```bash 75 | sudo docker run --rm --gpus all nvidia/cuda:11.0-base nvidia-smi 76 | ``` 77 | 78 | Now with the tensorflow container: 79 | 80 | - Pull the container: 81 | 82 | ```bash 83 | sudo docker pull nvcr.io/nvidia/tensorflow:21.11-tf2-py3 84 | ``` 85 | once the download has finished, you can run the container with: 86 | 87 | - Run the containter, replace the `path_to_folder` with the path to this repository. This will link this folder inside the docker container to `/code` path. 88 | 89 | ```bash 90 | sudo docker run --gpus all -it --rm -v path_to_folder/apple_m1_pro_python:/code nvcr.io/nvidia/tensorflow:21.11-tf2-py3 91 | ``` 92 | 93 | once inside the container install the missing libraries: 94 | 95 | ```bash 96 | $ pip install wandb transformers datasets tqdm scikit-learn 97 | ``` 98 | 99 | - And finally run the benchmark, replace `your_gpu_name` with your graphcis card name: `RTX3070m`, `A100`, etc... With modern Nvidia hardware (after RTX cards) you should enable the `--fp16` flag to use the tensor cores on your GPU. 100 | 101 | ```bash 102 | python train_pets.py --gpu_name="My_fancyNvidia_GPU" 103 | ``` 104 | 105 | > Note: You may need `sudo` to run docker. 106 | 107 | > Note2: Using this method is substantially faster than installing the python libs one by one, please use the NV containers. -------------------------------------------------------------------------------- /tensorflow/keras_cvp.py: -------------------------------------------------------------------------------- 1 | ## Author: Thomas Capelle 2 | ## Mail: tcapelle@wandb.com 3 | 4 | """"Benchmarking apple M1Pro with Tensorflow 5 | @wandbcode{apple_m1_pro}""" 6 | 7 | import os 8 | import time 9 | import random 10 | import shutil 11 | import tempfile 12 | 13 | import wandb 14 | from fastcore.script import * 15 | import numpy as np 16 | import tensorflow as tf 17 | from tensorflow import keras as K 18 | from tensorflow.keras.backend import count_params 19 | from tensorflow.keras import mixed_precision 20 | import tensorflow_datasets as tfds 21 | 22 | 23 | # Set the random seeds 24 | os.environ['TF_CUDNN_DETERMINISTIC'] = '1' 25 | random.seed(hash("setting random seeds") % 2**32 - 1) 26 | np.random.seed(hash("improves reproducibility") % 2**32 - 1) 27 | tf.random.set_seed(hash("by removing stochasticity") % 2**32 - 1) 28 | 29 | 30 | PROJECT = "apple_m1_pro" 31 | HW = 'M1Pro' 32 | ENTITY = None #replace with the team id 33 | 34 | N_CLASSES = 10 35 | DATASET = "cifar10" 36 | BASE_MODEL = "ResNet50" 37 | BS = 128 38 | IMG_SIZE = 128 39 | 40 | class SamplesSec(K.callbacks.Callback): 41 | def __init__(self, epochs=1, batch_size=1, drop=5): 42 | self.epochs = epochs 43 | self.batch_size = batch_size 44 | self.drop = drop 45 | 46 | 47 | def on_train_begin(self, logs={}): 48 | self.epoch_times = [] 49 | self.samples_s = 0. 50 | 51 | def on_epoch_begin(self, epoch, logs={}): 52 | self.batch_times = [] 53 | 54 | def on_train_batch_begin(self, batch, logs={}): 55 | self.batch_train_start = time.time() 56 | 57 | def on_train_batch_end(self, batch, logs={}): 58 | t = time.time() - self.batch_train_start 59 | self.batch_times.append(t) 60 | 61 | def on_epoch_end(self, epoch, logs={}): 62 | self.batch_times.sort() 63 | avg_time_per_batch = sum(self.batch_times[0:-self.drop])/(len(self.batch_times)-self.drop) 64 | samples_s_batch = self.batch_size / avg_time_per_batch 65 | wandb.log({"samples_per_batch": samples_s_batch}, step=epoch+1) 66 | self.samples_s += samples_s_batch 67 | 68 | def on_train_end(self, logs={}): 69 | wandb.log({"samples_per_s": self.samples_s/self.epochs}) 70 | 71 | def preprocess(image, label=None): 72 | """Normalize and resize images, one-hot labels 73 | @wandbcode{apple_m1_pro}""" 74 | if label is None: 75 | label = image['label'] 76 | image = image['image'] 77 | image = tf.image.convert_image_dtype(image, dtype=tf.float32) 78 | image = tf.image.resize(image, (IMG_DIM, IMG_DIM), method='nearest') 79 | label = tf.one_hot(label, N_CLASSES) 80 | return image, label 81 | 82 | def prepare(dataset, batch_size=None, cache=True): 83 | """Preprocess, shuffle, batch (opt), cache (opt) and prefetch a tf.Dataset""" 84 | ds = dataset.map(preprocess, num_parallel_calls=tf.data.AUTOTUNE) 85 | if cache: 86 | ds = ds.cache(DS_CACHE) 87 | ds = ds.shuffle(1024) 88 | if batch_size: 89 | ds = ds.batch(batch_size) 90 | ds = ds.prefetch(buffer_size=tf.data.AUTOTUNE) 91 | return ds 92 | 93 | def trainable_params(model): 94 | """Count the number of trainable parameters in a Keras model""" 95 | trainable_count = np.sum([count_params(w) for w in model.trainable_weights]) 96 | non_trainable_count = np.sum([count_params(w) for w in model.non_trainable_weights]) 97 | 98 | print('Total params: {:,}'.format(trainable_count + non_trainable_count)) 99 | print('Trainable params: {:,}'.format(trainable_count)) 100 | print('Non-trainable params: {:,}'.format(non_trainable_count)) 101 | return trainable_count 102 | 103 | def train(train_dataset, test_dataset, default_config, project=PROJECT, hw=HW, team=ENTITY): 104 | """Run transfer learning on the configured model and dataset""" 105 | global IMG_DIM, N_CLASSES, DS_CACHE 106 | with wandb.init(project=project, group=hw, config=default_config, entity=team) as run: 107 | # Set global defaults when running in sweep mode 108 | IMG_DIM = run.config.img_dim 109 | N_CLASSES = run.config.num_classes 110 | DS_CACHE = os.path.join(tempfile.mkdtemp(), str(hash(frozenset(run.config.items())))) 111 | 112 | # Setup base model to transfer from, optionally fine-tune 113 | base_model = getattr(tf.keras.applications, run.config.base_model)( 114 | input_shape=(run.config.img_dim, run.config.img_dim, 3), 115 | include_top=False, weights='imagenet') 116 | base_model.trainable = run.config.trainable 117 | 118 | 119 | # Decay learning rate 120 | lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay( 121 | run.config.init_lr, decay_steps=run.config.train_size, decay_rate=run.config.decay) 122 | 123 | # Compile model for this dataset 124 | tf.keras.backend.clear_session() 125 | model = tf.keras.Sequential([ 126 | base_model, 127 | tf.keras.layers.GlobalAveragePooling2D(), 128 | tf.keras.layers.Dropout(run.config.dropout), 129 | tf.keras.layers.Dense(run.config.num_classes, activation='softmax') 130 | ]) 131 | model.compile(optimizer=tf.keras.optimizers.Adam(lr_schedule), 132 | loss='categorical_crossentropy', 133 | metrics=['accuracy', 'top_k_categorical_accuracy']) 134 | 135 | # Update config and print summary 136 | run.config.update({ 137 | "total_params": model.count_params(), 138 | "trainable_params": trainable_params(model), 139 | }) 140 | print("Model {}:".format(run.config.base_model)) 141 | print(" trainable parameters:", run.config.trainable_params) 142 | print(" total parameters:", run.config.total_params) 143 | print("Dataset {}:".format(run.config.dataset)) 144 | print(" training: ", run.config.train_size) 145 | print(" test: ", run.config.test_size) 146 | print(" shape: {}\n".format((run.config.img_dim, run.config.img_dim, 3))) 147 | print("DS_CACHE: {}\n".format(DS_CACHE)) 148 | # Train the model 149 | train_batches = prepare(train_dataset, batch_size=run.config.batch_size) 150 | test_batches = prepare(test_dataset, batch_size=run.config.batch_size) 151 | 152 | cbs = [ 153 | wandb.keras.WandbCallback(save_model=False), 154 | SamplesSec(run.config.epochs, run.config.batch_size)] 155 | 156 | _ = model.fit(train_batches, epochs=run.config.epochs, validation_data=test_batches, 157 | callbacks=cbs) 158 | shutil.rmtree(os.path.dirname(DS_CACHE)) 159 | 160 | 161 | @call_parse 162 | def main( 163 | project: Param("Name of the wandb Project to log on", str)=PROJECT, 164 | hw: Param("Name of the hardware: V100, M1, M1Pro, etc...", str)=HW, 165 | trainable: Param("Train full model or only head", store_true)=False, 166 | repeat: Param("Number of times to repeat training", int)=1, 167 | epochs: Param("Override epochs", int) = 10, 168 | bs: Param("Override Batch Size", int) = BS, 169 | img_size: Param("Override Image Size", int) = IMG_SIZE, 170 | fp16: Param("Use mixed precision training", store_true)=False, 171 | ): 172 | 173 | wandb.login() 174 | # Default hyper-parameters, potentially overridden in sweep mode 175 | 176 | #mixed prec training for tensor core use 177 | if fp16: 178 | tf.config.optimizer.set_jit(True) 179 | policy = mixed_precision.Policy("mixed_float16") 180 | mixed_precision.set_global_policy(policy) 181 | 182 | train_dataset = tfds.load(name=DATASET, as_supervised=True, split="train") 183 | test_dataset = tfds.load(name=DATASET, as_supervised=True, split="test") 184 | default_config = { 185 | "batch_size": bs, "epochs": epochs, "dropout": 0.4, "base_model": BASE_MODEL, 186 | "init_lr": 0.0005, "decay": 0.96, "num_classes": N_CLASSES, "hardware": hw, 187 | "train_size": len(train_dataset), "test_size": len(test_dataset), 188 | "dataset": DATASET, "img_dim": img_size, "trainable": trainable, 189 | } 190 | 191 | for _ in range(repeat): 192 | train(train_dataset, test_dataset, default_config, project=project, hw=hw) 193 | -------------------------------------------------------------------------------- /tensorflow/tf_apple.yml: -------------------------------------------------------------------------------- 1 | name: tf 2 | channels: 3 | - apple 4 | - conda-forge 5 | dependencies: 6 | - python<3.11 7 | - pip>=19.0 8 | - tensorflow-deps 9 | - scikit-learn 10 | - tqdm 11 | - datasets 12 | - transformers 13 | - pip: 14 | - tensorflow-macos 15 | - tensorflow-metal 16 | - wandb 17 | -------------------------------------------------------------------------------- /tensorflow/tf_linux.yml: -------------------------------------------------------------------------------- 1 | name: tf 2 | 3 | channels: 4 | - conda-forge 5 | 6 | dependencies: 7 | - python<3.11 8 | - pip>=19.0 9 | - jupyter 10 | - tensorflow>=2.6 11 | - tensorflow-datasets 12 | - scikit-learn 13 | - scipy 14 | - pandas 15 | - pandas-datareader 16 | - matplotlib 17 | - pillow 18 | - tqdm 19 | - requests 20 | - h5py 21 | - pyyaml 22 | - transformers 23 | - datasets 24 | - wandb 25 | - pip: 26 | - wandb 27 | - fastcore 28 | -------------------------------------------------------------------------------- /tensorflow/train_bert.py: -------------------------------------------------------------------------------- 1 | ## Author: Thomas Capelle, Soumik Rakshit 2 | ## Mail: tcapelle@wandb.com, soumik.rakshit@wandb.com 3 | 4 | """"Benchmarking apple M1Pro with Tensorflow 5 | @wandbcode{apple_m1_pro}""" 6 | 7 | import wandb, argparse 8 | from types import SimpleNamespace 9 | from time import perf_counter 10 | 11 | import tensorflow as tf 12 | from tensorflow.keras import losses 13 | from tensorflow.keras import mixed_precision 14 | from tensorflow.keras.optimizers import legacy as legacy_optimizers 15 | from transformers import TFAutoModelForSequenceClassification, AutoTokenizer, DefaultDataCollator 16 | from datasets import load_dataset 17 | 18 | import wandb 19 | from wandb.keras import WandbCallback 20 | 21 | from utils import get_apple_gpu_name 22 | 23 | 24 | PROJECT = "pytorch-M1Pro" 25 | ENTITY = "capecape" 26 | GROUP = "tf" 27 | 28 | config_defaults = SimpleNamespace( 29 | batch_size=4, 30 | epochs=1, 31 | num_experiments=1, 32 | learning_rate=1e-3, 33 | model_name="bert-base-cased", 34 | dataset="yelp_review_full", 35 | device="mps", 36 | gpu_name=get_apple_gpu_name(), 37 | num_workers=0, 38 | mixed_precision=False, 39 | ) 40 | 41 | def parse_args(): 42 | parser = argparse.ArgumentParser() 43 | parser.add_argument('--batch_size', type=int, default=config_defaults.batch_size) 44 | parser.add_argument('--epochs', type=int, default=config_defaults.epochs) 45 | parser.add_argument('--num_experiments', type=int, default=config_defaults.num_experiments) 46 | parser.add_argument('--learning_rate', type=float, default=config_defaults.learning_rate) 47 | parser.add_argument('--model_name', type=str, default=config_defaults.model_name) 48 | parser.add_argument('--dataset', type=str, default=config_defaults.dataset) 49 | parser.add_argument('--device', type=str, default=config_defaults.device) 50 | parser.add_argument('--gpu_name', type=str, default=config_defaults.gpu_name) 51 | parser.add_argument('--num_workers', type=int, default=config_defaults.num_workers) 52 | parser.add_argument('--inference_only', action="store_true") 53 | parser.add_argument('--mixed_precision', action="store_true") 54 | return parser.parse_args() 55 | 56 | class SamplesSec(tf.keras.callbacks.Callback): 57 | def __init__(self, batch_size=1, drop=5): 58 | self.batch_size = batch_size 59 | self.drop = drop 60 | 61 | 62 | def on_train_begin(self, logs={}): 63 | self.epoch_times = [] 64 | self.samples_s = 0. 65 | 66 | def on_epoch_begin(self, epoch, logs={}): 67 | self.batch_times = [] 68 | 69 | def on_train_batch_begin(self, batch, logs={}): 70 | self.batch_train_start = perf_counter() 71 | 72 | def on_train_batch_end(self, batch, logs={}): 73 | t = perf_counter() - self.batch_train_start 74 | wandb.log({"samples_per_sec": self.batch_size/t}) 75 | 76 | 77 | def get_dls(model_name="bert-base-cased", dataset_name="yelp_review_full", batch_size=8, num_workers=0, sample_size=100): 78 | 79 | # download and prepare cc_news dataset 80 | dataset = load_dataset(dataset_name) 81 | 82 | # get bert and tokenizer 83 | tokenizer = AutoTokenizer.from_pretrained(model_name) 84 | 85 | def tokenize_function(examples): 86 | return tokenizer(examples["text"], padding="max_length", truncation=True) 87 | 88 | # tokenize the dataset 89 | tokenized_datasets = dataset.map(tokenize_function, batched=True) 90 | small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(sample_size)) 91 | small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(sample_size)) 92 | 93 | default_data_collator = DefaultDataCollator(return_tensors="tf") 94 | 95 | train = small_train_dataset.to_tf_dataset( 96 | columns=["input_ids", "token_type_ids", "attention_mask"], 97 | label_cols=["labels"], 98 | batch_size=batch_size, 99 | shuffle=False, 100 | collate_fn=default_data_collator, 101 | ) 102 | 103 | validation = small_eval_dataset.to_tf_dataset( 104 | columns=["input_ids", "token_type_ids", "attention_mask"], 105 | label_cols=["labels"], 106 | batch_size=batch_size, 107 | shuffle=False, 108 | collate_fn=default_data_collator, 109 | ) 110 | 111 | return train, validation 112 | 113 | 114 | def get_model(model_name="bert-base-cased", num_labels=5): 115 | return TFAutoModelForSequenceClassification.from_pretrained(model_name, num_labels=num_labels) 116 | 117 | def train_bert(config): 118 | train_ds, _ = get_dls( 119 | model_name=config.model_name, 120 | batch_size=config.batch_size, 121 | num_workers=config.num_workers) 122 | 123 | if config.mixed_precision: 124 | mixed_precision.set_global_policy('mixed_float16') 125 | 126 | optimizer = legacy_optimizers.Adam(learning_rate=config.learning_rate) 127 | 128 | model = get_model(config.model_name) 129 | 130 | model.compile( 131 | loss=losses.SparseCategoricalCrossentropy(from_logits=True), 132 | optimizer=optimizer, 133 | metrics=["accuracy"], 134 | ) 135 | 136 | with wandb.init(project=PROJECT, entity=ENTITY, group=GROUP, config=config): 137 | model.fit( 138 | train_ds, 139 | epochs=config.epochs, 140 | callbacks=[WandbCallback(save_model=False), 141 | SamplesSec(config.batch_size)], 142 | ) 143 | 144 | if __name__ == "__main__": 145 | args = parse_args() 146 | for _ in range(args.num_experiments): 147 | train_bert(config=args) -------------------------------------------------------------------------------- /tensorflow/train_pets.py: -------------------------------------------------------------------------------- 1 | ## Author: Thomas Capelle, Soumik Rakshit 2 | ## Mail: tcapelle@wandb.com, soumik.rakshit@wandb.com 3 | 4 | """"Benchmarking apple M1Pro with Tensorflow 5 | @wandbcode{apple_m1_pro}""" 6 | 7 | import re 8 | import subprocess 9 | import os 10 | import platform 11 | import argparse 12 | from time import perf_counter 13 | from glob import glob 14 | from typing import List 15 | from pathlib import Path 16 | from typing import Callable, List 17 | from types import SimpleNamespace 18 | from sklearn.model_selection import train_test_split 19 | 20 | import wandb 21 | from wandb.keras import WandbCallback 22 | 23 | import tensorflow as tf 24 | from tensorflow.keras.optimizers import legacy as legacy_optimizers 25 | from tensorflow.keras import Input, Model 26 | from tensorflow.keras import mixed_precision 27 | from tensorflow.keras import layers, losses, applications 28 | 29 | from utils import get_apple_gpu_name 30 | 31 | 32 | PROJECT = "Pytorch-M1Pro" 33 | ENTITY = "capecape" 34 | GROUP = "tf" 35 | 36 | config_defaults = SimpleNamespace( 37 | batch_size=64, 38 | epochs=1, 39 | num_experiments=1, 40 | learning_rate=1e-3, 41 | validation_split=0.0, 42 | image_size=128, 43 | model_name="resnet50", 44 | dataset="PETS", 45 | artifact_address="capecape/pytorch-M1Pro/PETS:v3", 46 | gpu_name=get_apple_gpu_name(), 47 | mixed_precision=False, 48 | optimizer="Adam", # currently an issue forced to legacy optim 49 | ) 50 | 51 | 52 | def parse_args(): 53 | parser = argparse.ArgumentParser() 54 | parser.add_argument("--entity", type=str, default=ENTITY) 55 | parser.add_argument("--image_size", type=int, default=config_defaults.image_size) 56 | parser.add_argument("--batch_size", type=int, default=config_defaults.batch_size) 57 | parser.add_argument("--epochs", type=int, default=config_defaults.epochs) 58 | parser.add_argument( 59 | "--num_experiments", type=int, default=config_defaults.num_experiments 60 | ) 61 | parser.add_argument( 62 | "--validation_split", type=float, default=config_defaults.validation_split 63 | ) 64 | parser.add_argument( 65 | "--learning_rate", type=float, default=config_defaults.learning_rate 66 | ) 67 | parser.add_argument("--model_name", type=str, default=config_defaults.model_name) 68 | parser.add_argument('--dataset', type=str, default=config_defaults.dataset) 69 | parser.add_argument("--artifact_address", type=str, default=config_defaults.artifact_address) 70 | parser.add_argument("--gpu_name", type=str, default=config_defaults.gpu_name) 71 | parser.add_argument('--optimizer', type=str, default=config_defaults.optimizer) 72 | parser.add_argument("--mixed_precision", type=int, default=config_defaults.mixed_precision) 73 | return parser.parse_args() 74 | 75 | 76 | AUTOTUNE = tf.data.AUTOTUNE 77 | BACKBONE_DICT = { 78 | "resnet50": { 79 | "model": applications.ResNet50, 80 | "preprocess_fn": applications.resnet50.preprocess_input, 81 | } 82 | } 83 | 84 | VOCAB = [ 85 | "Abyssinian", 86 | "Bengal", 87 | "Birman", 88 | "Bombay", 89 | "British_Shorthair", 90 | "Egyptian_Mau", 91 | "Maine_Coon", 92 | "Persian", 93 | "Ragdoll", 94 | "Russian_Blue", 95 | "Siamese", 96 | "Sphynx", 97 | "american_bulldog", 98 | "american_pit", 99 | "basset_hound", 100 | "beagle", 101 | "boxer", 102 | "chihuahua", 103 | "english_cocker", 104 | "english_setter", 105 | "german_shorthaired", 106 | "great_pyrenees", 107 | "havanese", 108 | "japanese_chin", 109 | "keeshond", 110 | "leonberger", 111 | "miniature_pinscher", 112 | "newfoundland", 113 | "pomeranian", 114 | "pug", 115 | "saint_bernard", 116 | "samoyed", 117 | "scottish_terrier", 118 | "shiba_inu", 119 | "staffordshire_bull", 120 | "wheaten_terrier", 121 | "yorkshire_terrier", 122 | ] 123 | 124 | 125 | class SamplesSec(tf.keras.callbacks.Callback): 126 | def __init__(self, batch_size=1, drop=5): 127 | self.batch_size = batch_size 128 | self.drop = drop 129 | 130 | 131 | def on_train_begin(self, logs={}): 132 | self.epoch_times = [] 133 | self.samples_s = 0. 134 | 135 | def on_epoch_begin(self, epoch, logs={}): 136 | self.batch_times = [] 137 | 138 | def on_train_batch_begin(self, batch, logs={}): 139 | self.batch_train_start = perf_counter() 140 | 141 | def on_train_batch_end(self, batch, logs={}): 142 | t = perf_counter() - self.batch_train_start 143 | wandb.log({"samples_per_sec": self.batch_size/t}) 144 | 145 | class PetsDataLoader: 146 | def __init__( 147 | self, 148 | artifact_address: str, 149 | preprocess_fn: Callable, 150 | image_size: int, 151 | batch_size: int, 152 | vocab: List[str]=VOCAB, 153 | ): 154 | self.artifact_address = artifact_address 155 | self.dataset_path = self.get_pets() 156 | self.preprocess_fn = preprocess_fn 157 | print(self.preprocess_fn) 158 | self.image_size = image_size 159 | self.batch_size = batch_size 160 | self.pattern = r"(^[a-zA-Z]+_*[a-zA-Z]+)" 161 | self.vocab_map = {v: i for i, v in enumerate(vocab)} 162 | self.image_files = glob(os.path.join(self.dataset_path, "images", "*.jpg")) 163 | self.labels = [ 164 | self.vocab_map[re.match(self.pattern, Path(image_file).name)[0]] 165 | for image_file in self.image_files 166 | ] 167 | 168 | def __len__(self): 169 | return len(self.image_files) 170 | 171 | def get_pets(self): 172 | api = wandb.Api() 173 | at = api.artifact(self.artifact_address, type="dataset") 174 | dataset_path = at.download() 175 | return dataset_path 176 | 177 | def map_fn(self, image_file, label): 178 | image = tf.io.read_file(image_file) 179 | image = tf.image.decode_png(image, channels=3) 180 | image.set_shape([None, None, 3]) 181 | image = tf.image.resize(image, [self.image_size, self.image_size]) 182 | image = self.preprocess_fn(image) 183 | return image, label 184 | 185 | def build_dataset(self, images, labels): 186 | dataset = tf.data.Dataset.from_tensor_slices((images, labels)) 187 | dataset = dataset.map(self.map_fn, num_parallel_calls=AUTOTUNE) 188 | dataset = dataset.batch(self.batch_size, drop_remainder=True) 189 | return dataset 190 | 191 | def get_datasets(self, val_split: float): 192 | if val_split>0: 193 | train_images, val_images, train_labels, val_labels = train_test_split( 194 | self.image_files, self.labels, test_size=val_split 195 | ) 196 | train_dataset = self.build_dataset(train_images, train_labels) 197 | val_dataset = self.build_dataset(val_images, val_labels) 198 | else: 199 | train_dataset = self.build_dataset(self.image_files, self.labels) 200 | val_dataset = None 201 | return train_dataset, val_dataset 202 | 203 | 204 | def get_model(image_size: int, model_name: str, vocab: List[str]) -> Model: 205 | input_shape = [image_size, image_size, 3] 206 | input_tensor = Input(shape=input_shape) 207 | backbone_out = BACKBONE_DICT[model_name]["model"]( 208 | include_top=False, input_tensor=input_tensor 209 | )(input_tensor) 210 | x = layers.GlobalAveragePooling2D()(backbone_out) 211 | output = layers.Dense(len(vocab))(x) 212 | return Model(input_tensor, output) 213 | 214 | 215 | def train(args): 216 | with wandb.init(project=PROJECT, entity=args.entity, group=GROUP, config=args): 217 | config = wandb.config 218 | if args.mixed_precision: 219 | mixed_precision.set_global_policy('mixed_float16') 220 | loader = PetsDataLoader( 221 | artifact_address=config.artifact_address, 222 | preprocess_fn=BACKBONE_DICT[config.model_name]["preprocess_fn"], 223 | image_size=config.image_size, 224 | batch_size=config.batch_size, 225 | ) 226 | print("Dataset Size:", len(loader)) 227 | 228 | train_dataset, val_dataset = loader.get_datasets( 229 | val_split=config.validation_split 230 | ) 231 | 232 | model = get_model( 233 | image_size=config.image_size, 234 | model_name=config.model_name, 235 | vocab=VOCAB, 236 | ) 237 | model.summary() 238 | 239 | optimizer = getattr(legacy_optimizers, config.optimizer) 240 | 241 | model.compile( 242 | loss=losses.SparseCategoricalCrossentropy(from_logits=True), 243 | optimizer=optimizer(learning_rate=config.learning_rate), 244 | metrics=["accuracy"], 245 | ) 246 | 247 | model.fit( 248 | train_dataset, 249 | validation_data=val_dataset, 250 | epochs=config.epochs, 251 | callbacks=[WandbCallback(save_model=False), 252 | SamplesSec(config.batch_size)], 253 | ) 254 | 255 | 256 | if __name__ == "__main__": 257 | args = parse_args() 258 | for _ in range(args.num_experiments): 259 | train(args=args) 260 | -------------------------------------------------------------------------------- /tensorflow/utils.py: -------------------------------------------------------------------------------- 1 | import re, subprocess 2 | from sys import platform 3 | 4 | def get_apple_hardware(): 5 | "Get apple hardware info" 6 | cpu_info = subprocess.run(["system_profiler","SPHardwareDataType"], stdout=subprocess.PIPE).stdout.decode("utf-8") 7 | gpu_info = subprocess.run(["system_profiler","SPDisplaysDataType"], stdout=subprocess.PIPE).stdout.decode("utf-8") 8 | system_info = dict( 9 | cpu = re.search(r'Chip:\s+(.+)', cpu_info).group(1), 10 | cpu_cores = re.search(r'Number of Cores:\s+(\d+)', cpu_info).group(1), 11 | memory = re.search(r'Memory:\s+(\d+)\s+GB', cpu_info).group(1), 12 | gpu = re.search(r'Chipset Model:\s+(.+)', gpu_info).group(1), 13 | gpu_cores = re.search(r'Total Number of Cores:\s+(\d+)', gpu_info).group(1), 14 | ) 15 | return system_info 16 | 17 | def get_apple_gpu_name(): 18 | if platform == "darwin": 19 | system_info = get_apple_hardware() 20 | return f"{system_info['gpu']} GPU {system_info['gpu_cores']} Cores" 21 | else: 22 | return None --------------------------------------------------------------------------------