├── .gitignore ├── LICENSE ├── README.md ├── benchmarks.ipynb ├── data ├── cityscapes │ ├── LICENSE │ └── cityscapes_data.py └── coco │ ├── Coco164kFull_Stuff_Coarse_7.txt │ ├── LICENSE │ ├── coco_data.py │ └── fine_to_coarse_dict.pickle ├── diffseg.ipynb ├── diffseg ├── LICENSE ├── segmentor.py └── utils.py ├── docs ├── code-of-conduct.md └── contributing.md ├── images └── LICENSE ├── requirements.txt └── third_party └── keras_cv ├── LICENSE.txt ├── diffusion_model.py └── stable_diffusion.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | # Custom folders 163 | images -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2023 Google LLC 4 | 5 | Permission is hereby granted, free of charge, to any 6 | person obtaining a copy of this software and associated 7 | documentation files (the "Software"), to deal in the 8 | Software without restriction, including without 9 | limitation the rights to use, copy, modify, merge, 10 | publish, distribute, sublicense, and/or sell copies of 11 | the Software, and to permit persons to whom the Software 12 | is furnished to do so, subject to the following 13 | conditions: 14 | 15 | The above copyright notice and this permission notice 16 | shall be included in all copies or substantial portions 17 | of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 20 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 21 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 22 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 23 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 24 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 25 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 26 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 27 | DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Diffuse, Attend, and Segment 2 | This repo implements the DiffSeg segmentation method in the paper [``Diffuse, Attend, and Segment: Unsupervised Zero-Shot Segmentation using Stable Diffusion``](https://arxiv.org/abs/2308.12469). 3 | 4 | ``` 5 | @article{tian2023diffuse, 6 | title={Diffuse, Attend, and Segment: Unsupervised Zero-Shot Segmentation using Stable Diffusion}, 7 | author={Tian, Junjiao and Aggarwal, Lavisha and Colaco, Andrea and Kira, Zsolt and Gonzalez-Franco, Mar}, 8 | journal={arXiv preprint arXiv:2308.12469}, 9 | year={2023} 10 | } 11 | ``` 12 | 13 | ## Overview 14 | DiffSeg is an unsupervised zero-shot segmentation method using attention information from a stable-diffusion model. This repo implements the main DiffSeg algorithm and additionally include an experimental feature to add semantic labels to the masks based on a generated caption. 15 | 16 | More details can be found on the project page: https://sites.google.com/corp/view/diffseg/home 17 | 18 | ## Create conda environment 19 | 20 | - The environment uses Ubuntu 18.04 and Tensorflow 2.14 supported on CUDA 11.x and python 3.9. 21 | ``` 22 | cd diffseg 23 | conda create --name diffseg python=3.9 24 | conda activate diffseg 25 | pip install -r path/to/requirements.txt 26 | ``` 27 | 28 | ## Computation Requirement 29 | - We recommend using 2 GPUs with a minimum 11G VRAM each, e.g., RTX2080Ti. 30 | - One GPU is for loading the Stable Diffusion model and the other is for the BLIP captioning model. 31 | 32 | ## DiffSeg Notebook 33 | Please see the instructions in the ``diffseg.ipynb`` for running instructions. 34 | 35 | ## Benchmarks 36 | We benchmark the performance of DiffSeg on [CoCo-Stuff-27](https://github.com/nightrome/cocostuff) and [Cityscapes](https://www.cityscapes-dataset.com/). Please see instructions in ``benchmarks.ipynb``. 37 | * We follow the evaluation protocol in [PiCIE](https://sites.google.com/view/picie-cvpr2021/home) and use the [Hungarian algorithm](https://en.wikipedia.org/wiki/Hungarian_algorithm) for matching predictions and ground truth labels. 38 | 39 | ## Contributors 40 | - **Junjiao Tian (Google and Georgia Tech)** 41 | - **Lavisha Aggarwal (Google)** 42 | - **Andrea Colaco (Google)** 43 | - **Zsolt Kira (Georgia Tech)** 44 | - **Mar Gonzalez-Franco (Google)** 45 | -------------------------------------------------------------------------------- /benchmarks.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "1NJCZtpiWAHX" 7 | }, 8 | "source": [ 9 | "Copyright 2023 Google LLC\n", 10 | "\n", 11 | "Use of this source code is governed by an MIT-style\n", 12 | "license that can be found in the LICENSE file or at\n", 13 | "https://opensource.org/licenses/MIT.\n", 14 | "\n", 15 | "# Instructions\n", 16 | "1. Download CoCo-Stuff [annotations](http://calvin.inf.ed.ac.uk/wp-content/uploads/data/cocostuffdataset/stuffthingmaps_trainval2017.zip) and [val images](http://images.cocodataset.org/zips/val2017.zip).\n", 17 | "* Please first download the [annotations](http://calvin.inf.ed.ac.uk/wp-content/uploads/data/cocostuffdataset/stuffthingmaps_trainval2017.zip) and rename `val2017` to `annotation_val2017`.\n", 18 | "\n", 19 | "\n", 20 | "2. Download [Cityscapes](https://www.cityscapes-dataset.com/).\n", 21 | "* Cityscapes download requires login.\n", 22 | "* Please download `leftImg8bit_trainvaltest.zip` and `gtFine_trainvaltest.zip` to your data folder.\n", 23 | "\n", 24 | "3. Please run the cells in order and choose 2a or 2b, not both.\n", 25 | "* 2a: load CoCo-Stuff data.\n", 26 | "* 2b: load Cityscapes data.\n", 27 | "\n", 28 | "4. Metrics\n", 29 | "* The inference code will return pixel accuracy (ACC) and mean IoU (mIoU)." 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "metadata": { 35 | "id": "Yq63OpI2PCka" 36 | }, 37 | "source": [ 38 | "# Imports" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "metadata": { 45 | "id": "LJia4pK2MGju" 46 | }, 47 | "outputs": [], 48 | "source": [ 49 | "import tensorflow as tf\n", 50 | "import numpy as np\n", 51 | "from tqdm import tqdm\n", 52 | "from diffseg.segmentor import DiffSeg\n", 53 | "from keras_cv.src.models.stable_diffusion.image_encoder import ImageEncoder\n", 54 | "from third_party.keras_cv.stable_diffusion import StableDiffusion \n", 55 | "from data.cityscapes import cityscapes_data\n", 56 | "from data.coco import coco_data\n", 57 | "from diffseg.utils import hungarian_matching\n", 58 | "\n", 59 | "!nvidia-smi" 60 | ] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": { 65 | "id": "u2dXePdUM0SS" 66 | }, 67 | "source": [ 68 | "# 1. Initialize SD Model" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "metadata": { 75 | "id": "aizDc1dRM2-n" 76 | }, 77 | "outputs": [], 78 | "source": [ 79 | "# Initialize Stable Diffusion Model on all GPUs.\n", 80 | "strategy = tf.distribute.MirroredStrategy()\n", 81 | "print('Number of devices: {}'.format(strategy.num_replicas_in_sync))\n", 82 | "with strategy.scope():\n", 83 | " image_encoder = ImageEncoder()\n", 84 | " vae=tf.keras.Model(\n", 85 | " image_encoder.input,\n", 86 | " image_encoder.layers[-1].output,\n", 87 | " )\n", 88 | " model = StableDiffusion(img_width=512, img_height=512)" 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": { 94 | "id": "KxFEgBH-MRqO" 95 | }, 96 | "source": [ 97 | "# 2a. Load COCO-Stuff Data" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": null, 103 | "metadata": { 104 | "id": "FOfz-QHIMY1w" 105 | }, 106 | "outputs": [], 107 | "source": [ 108 | "ROOT = \"../coco_data/\" # Change this directory to your coco data folder.\n", 109 | "FINE_TO_COARSE_PATH = \"./data/coco/fine_to_coarse_dict.pickle\"\n", 110 | "BATCH_SIZE = strategy.num_replicas_in_sync\n", 111 | "\n", 112 | "# Load fine to coarse label mapping.\n", 113 | "fine_to_coarse_map = coco_data.get_fine_to_coarse(FINE_TO_COARSE_PATH)\n", 114 | "\n", 115 | "# Prepare the coco-stuff validation dataset.\n", 116 | "file_list = coco_data.load_imdb(\"./data/coco/Coco164kFull_Stuff_Coarse_7.txt\")\n", 117 | "image_list, label_list = coco_data.create_path(ROOT, file_list)\n", 118 | "val_dataset = coco_data.prepare_dataset(\n", 119 | " image_list, label_list, batch_size=BATCH_SIZE\n", 120 | ")" 121 | ] 122 | }, 123 | { 124 | "cell_type": "markdown", 125 | "metadata": { 126 | "id": "eKmvvYcRMpDW" 127 | }, 128 | "source": [ 129 | "# 2b. Load Cityscapes Data" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": { 136 | "id": "9XtJWMCxMtCX" 137 | }, 138 | "outputs": [], 139 | "source": [ 140 | "ROOT = \"../cityscapes_data/\"\n", 141 | "BATCH_SIZE = strategy.num_replicas_in_sync\n", 142 | "\n", 143 | "# Load fine to coarse label mapping.\n", 144 | "fine_to_coarse_map = cityscapes_data.get_fine_to_coarse()\n", 145 | "\n", 146 | "# Prepare the cityscapes validation dataset.\n", 147 | "image_list, label_list = cityscapes_data.create_path(ROOT)\n", 148 | "val_dataset = cityscapes_data.prepare_dataset(\n", 149 | " image_list, label_list, batch_size=BATCH_SIZE\n", 150 | ")" 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": { 156 | "id": "sjopCVKLNFTp" 157 | }, 158 | "source": [ 159 | "# 3. Run Inference" 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": null, 165 | "metadata": { 166 | "id": "WYtn8g7INKy5" 167 | }, 168 | "outputs": [], 169 | "source": [ 170 | "N_CLASS = 27\n", 171 | "TP = np.zeros(N_CLASS)\n", 172 | "FP = np.zeros(N_CLASS)\n", 173 | "FN = np.zeros(N_CLASS)\n", 174 | "ALL = 0\n", 175 | "\n", 176 | "# Initialize DiffSeg\n", 177 | "KL_THRESHOLD = [1.1]*3 # This controls the merge threshold for masks (1.1 for CoCo-Stuff and 0.9 for Cityscapes)\n", 178 | "NUM_POINTS = 16\n", 179 | "REFINEMENT = False # Whether use K-Means refinement. Increase inference time from 2s to 3s.\n", 180 | "\n", 181 | "with strategy.scope():\n", 182 | " segmentor = DiffSeg(KL_THRESHOLD, REFINEMENT, NUM_POINTS)\n", 183 | "\n", 184 | " for i,batch in enumerate(tqdm(val_dataset)):\n", 185 | " images = batch[\"images\"]\n", 186 | " labels = fine_to_coarse_map(batch[\"labels\"][:,:,:,0])\n", 187 | " latent = vae(images, training=False)\n", 188 | "\n", 189 | " # Extract attention maps from a single iteration of diffusion.\n", 190 | " images, weight_64, weight_32, weight_16, weight_8, _, _, _, _ = model.text_to_image(\n", 191 | " None,\n", 192 | " batch_size=images.shape[0],\n", 193 | " latent=latent,\n", 194 | " timestep=300\n", 195 | " )\n", 196 | "\n", 197 | " # Segment using DiffSeg.\n", 198 | " pred = segmentor.segment(weight_64, weight_32, weight_16, weight_8) # b x 512 x 512\n", 199 | " \n", 200 | " # Run hungarian matching for evaluation.\n", 201 | " tp, fp, fn, all = hungarian_matching(pred, labels, N_CLASS)\n", 202 | " TP += tp\n", 203 | " FP += fp\n", 204 | " FN += fn\n", 205 | " ALL += all\n", 206 | "\n", 207 | " # Print accuracy and mean IoU occasionally.\n", 208 | " if (i+1) % 10 == 0:\n", 209 | " acc = TP.sum()/ALL\n", 210 | " iou = TP / (TP + FP + FN)\n", 211 | " miou = np.nanmean(iou)\n", 212 | " print(\"pixel accuracy:{}, mIoU:{}\".format(acc, miou))\n", 213 | "\n", 214 | "# Print final accuracy and mean IoU.\n", 215 | "acc = TP.sum()/ALL\n", 216 | "iou = TP / (TP + FP + FN)\n", 217 | "miou = np.nanmean(iou)\n", 218 | "print(\"final pixel accuracy:{}, mIoU:{}\".format(acc, miou))" 219 | ] 220 | } 221 | ], 222 | "metadata": { 223 | "colab": { 224 | "collapsed_sections": [ 225 | "KxFEgBH-MRqO", 226 | "eKmvvYcRMpDW" 227 | ], 228 | "last_runtime": { 229 | "build_target": "//learning/grp/tools/ml_python:ml_notebook", 230 | "kind": "private" 231 | }, 232 | "private_outputs": true, 233 | "provenance": [ 234 | { 235 | "file_id": "1MfflAKfkM4uimNb-plpgq8dDJf7-vUUg", 236 | "timestamp": 1691431782950 237 | } 238 | ], 239 | "toc_visible": true 240 | }, 241 | "kernelspec": { 242 | "display_name": "Python 3", 243 | "name": "python3" 244 | }, 245 | "language_info": { 246 | "codemirror_mode": { 247 | "name": "ipython", 248 | "version": 3 249 | }, 250 | "file_extension": ".py", 251 | "mimetype": "text/x-python", 252 | "name": "python", 253 | "nbconvert_exporter": "python", 254 | "pygments_lexer": "ipython3", 255 | "version": "3.9.18" 256 | } 257 | }, 258 | "nbformat": 4, 259 | "nbformat_minor": 0 260 | } 261 | -------------------------------------------------------------------------------- /data/cityscapes/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2023 Google LLC 4 | 5 | Permission is hereby granted, free of charge, to any 6 | person obtaining a copy of this software and associated 7 | documentation files (the "Software"), to deal in the 8 | Software without restriction, including without 9 | limitation the rights to use, copy, modify, merge, 10 | publish, distribute, sublicense, and/or sell copies of 11 | the Software, and to permit persons to whom the Software 12 | is furnished to do so, subject to the following 13 | conditions: 14 | 15 | The above copyright notice and this permission notice 16 | shall be included in all copies or substantial portions 17 | of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 20 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 21 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 22 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 23 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 24 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 25 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 26 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 27 | DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /data/cityscapes/cityscapes_data.py: -------------------------------------------------------------------------------- 1 | #Copyright 2023 Google LLC 2 | 3 | #Use of this source code is governed by an MIT-style 4 | #license that can be found in the LICENSE file or at 5 | #https://opensource.org/licenses/MIT. 6 | 7 | """cityscapes_data.py 8 | Data pre-processing and loading pipeline for Cityscapes. 9 | """ 10 | 11 | import os 12 | import numpy as np 13 | from tensorflow import keras 14 | import tensorflow as tf 15 | import glob 16 | 17 | RESOLUTION = 512 18 | AUTO = tf.data.AUTOTUNE 19 | 20 | augmenter = keras.Sequential( 21 | layers=[ 22 | tf.keras.layers.Rescaling(scale=1.0 / 127.5, offset=-1), 23 | ] 24 | ) 25 | 26 | def get_fine_to_coarse(): 27 | """Map fine label indexing to coarse label indexing.""" 28 | label_dict = { 29 | 0: -1, 30 | 1: -1, 31 | 2: -1, 32 | 3: -1, 33 | 4: -1, 34 | 5: -1, 35 | 6: -1, 36 | 7: 0, 37 | 8: 1, 38 | 9: 2, 39 | 10: 3, 40 | 11: 4, 41 | 12: 5, 42 | 13: 6, 43 | 14: 7, 44 | 15: 8, 45 | 16: 9, 46 | 17: 10, 47 | 18: 11, 48 | 19: 12, 49 | 20: 13, 50 | 21: 14, 51 | 22: 15, 52 | 23: 16, 53 | 24: 17, 54 | 25: 18, 55 | 26: 19, 56 | 27: 20, 57 | 28: 21, 58 | 29: 22, 59 | 30: 23, 60 | 31: 24, 61 | 32: 25, 62 | 33: 26, 63 | -1: -1, 64 | } 65 | cityscape_labelmap = np.vectorize(lambda x: label_dict[x]) 66 | return cityscape_labelmap 67 | 68 | def create_path(root, split="val"): 69 | """This function creates data loading paths.""" 70 | image_path = [] 71 | label_path = [] 72 | 73 | image_folder = "leftImg8bit/" + split 74 | label_folder = "gtFine/" + split 75 | 76 | for folder in os.listdir(os.path.join(root, image_folder)): 77 | for file_path in glob.glob( 78 | os.path.join(root, image_folder, folder, "*.png") 79 | ): 80 | image_path.append(file_path) 81 | label_path.append(file_path.replace("leftImg8bit","gtFine").replace(".png","_labelIds.png")) 82 | return image_path, label_path 83 | 84 | def process_image(image_path, label_path): 85 | """This function reads and resizes images and labels.""" 86 | image = tf.io.read_file(image_path) 87 | image = tf.io.decode_png(image, 3) 88 | label = tf.io.read_file(label_path) 89 | label = tf.io.decode_png(label, 3) 90 | 91 | s = tf.shape(image) 92 | w, h = s[0], s[1] 93 | c = tf.minimum(w, h) 94 | w_start = (w - c) // 2 95 | h_start = (h - c) // 2 96 | image = image[w_start : w_start + c, h_start : h_start + c, :] 97 | label = label[w_start : w_start + c, h_start : h_start + c] 98 | image = tf.image.resize(image, (RESOLUTION, RESOLUTION)) 99 | label = tf.image.resize( 100 | label, 101 | (RESOLUTION, RESOLUTION), 102 | method=tf.image.ResizeMethod.NEAREST_NEIGHBOR, 103 | ) 104 | return image, label 105 | 106 | def apply_augmentation(image_batch, label_batch): 107 | """This function applies image augmentation to batches.""" 108 | return augmenter(image_batch), label_batch 109 | 110 | def prepare_dict(image_batch, label_batch): 111 | return { 112 | "images": image_batch, 113 | "labels": label_batch, 114 | } 115 | 116 | def prepare_dataset(image_paths, label_paths, batch_size=1, train=False): 117 | dataset = tf.data.Dataset.from_tensor_slices((image_paths, label_paths)) 118 | if train: 119 | dataset = dataset.shuffle(batch_size * 10) 120 | dataset = dataset.map(process_image, num_parallel_calls=AUTO).batch( 121 | batch_size 122 | ) 123 | dataset = dataset.map(apply_augmentation, num_parallel_calls=AUTO) 124 | dataset = dataset.map(prepare_dict, num_parallel_calls=AUTO) 125 | return dataset.prefetch(AUTO) -------------------------------------------------------------------------------- /data/coco/Coco164kFull_Stuff_Coarse_7.txt: -------------------------------------------------------------------------------- 1 | 000000000139 2 | 000000000724 3 | 000000000785 4 | 000000000872 5 | 000000000885 6 | 000000001268 7 | 000000001532 8 | 000000001761 9 | 000000002153 10 | 000000002261 11 | 000000002473 12 | 000000002532 13 | 000000002923 14 | 000000003255 15 | 000000003553 16 | 000000005060 17 | 000000005529 18 | 000000006213 19 | 000000006460 20 | 000000006723 21 | 000000006818 22 | 000000007088 23 | 000000007278 24 | 000000007281 25 | 000000007511 26 | 000000007574 27 | 000000007784 28 | 000000007816 29 | 000000007888 30 | 000000007977 31 | 000000008211 32 | 000000008762 33 | 000000008899 34 | 000000009483 35 | 000000009769 36 | 000000009772 37 | 000000010764 38 | 000000010977 39 | 000000011122 40 | 000000011197 41 | 000000011615 42 | 000000012120 43 | 000000012280 44 | 000000012639 45 | 000000012667 46 | 000000013201 47 | 000000013348 48 | 000000013546 49 | 000000013774 50 | 000000014007 51 | 000000014038 52 | 000000014380 53 | 000000014439 54 | 000000015272 55 | 000000015338 56 | 000000015440 57 | 000000015517 58 | 000000015597 59 | 000000015746 60 | 000000015751 61 | 000000015956 62 | 000000016010 63 | 000000016439 64 | 000000016451 65 | 000000016502 66 | 000000016958 67 | 000000017029 68 | 000000017178 69 | 000000017182 70 | 000000017379 71 | 000000017436 72 | 000000017627 73 | 000000017905 74 | 000000017959 75 | 000000018491 76 | 000000018519 77 | 000000018737 78 | 000000018833 79 | 000000019042 80 | 000000019109 81 | 000000019742 82 | 000000020059 83 | 000000020247 84 | 000000020553 85 | 000000021839 86 | 000000021879 87 | 000000021903 88 | 000000022396 89 | 000000022755 90 | 000000022969 91 | 000000023023 92 | 000000023230 93 | 000000023359 94 | 000000023666 95 | 000000023751 96 | 000000023781 97 | 000000023937 98 | 000000024027 99 | 000000024610 100 | 000000024919 101 | 000000025057 102 | 000000025181 103 | 000000025228 104 | 000000025393 105 | 000000025424 106 | 000000025593 107 | 000000025986 108 | 000000026204 109 | 000000026690 110 | 000000027972 111 | 000000028285 112 | 000000028993 113 | 000000029187 114 | 000000029393 115 | 000000029596 116 | 000000029984 117 | 000000030494 118 | 000000030504 119 | 000000030828 120 | 000000031050 121 | 000000031093 122 | 000000031118 123 | 000000031217 124 | 000000031269 125 | 000000031322 126 | 000000031620 127 | 000000031735 128 | 000000031749 129 | 000000032081 130 | 000000032285 131 | 000000032570 132 | 000000032735 133 | 000000032811 134 | 000000032887 135 | 000000032941 136 | 000000033005 137 | 000000033109 138 | 000000033114 139 | 000000033368 140 | 000000033638 141 | 000000033707 142 | 000000034205 143 | 000000034417 144 | 000000034452 145 | 000000034760 146 | 000000034873 147 | 000000035197 148 | 000000035963 149 | 000000036539 150 | 000000036678 151 | 000000036844 152 | 000000036861 153 | 000000037689 154 | 000000037740 155 | 000000037751 156 | 000000037988 157 | 000000038118 158 | 000000038210 159 | 000000039405 160 | 000000039477 161 | 000000039480 162 | 000000039484 163 | 000000039551 164 | 000000039785 165 | 000000039914 166 | 000000039956 167 | 000000040036 168 | 000000040471 169 | 000000041488 170 | 000000041635 171 | 000000041872 172 | 000000041888 173 | 000000041990 174 | 000000042276 175 | 000000042563 176 | 000000042628 177 | 000000042888 178 | 000000043314 179 | 000000043435 180 | 000000043737 181 | 000000043816 182 | 000000044195 183 | 000000044260 184 | 000000044652 185 | 000000044699 186 | 000000045229 187 | 000000045472 188 | 000000045596 189 | 000000045728 190 | 000000046252 191 | 000000046804 192 | 000000047010 193 | 000000047571 194 | 000000047769 195 | 000000047819 196 | 000000048924 197 | 000000049091 198 | 000000049259 199 | 000000049759 200 | 000000049761 201 | 000000050006 202 | 000000050165 203 | 000000050326 204 | 000000050638 205 | 000000050679 206 | 000000051314 207 | 000000051326 208 | 000000051598 209 | 000000051712 210 | 000000051938 211 | 000000051961 212 | 000000051976 213 | 000000052017 214 | 000000052462 215 | 000000052507 216 | 000000052565 217 | 000000053505 218 | 000000053624 219 | 000000053626 220 | 000000053994 221 | 000000054164 222 | 000000054931 223 | 000000054967 224 | 000000055072 225 | 000000055167 226 | 000000055299 227 | 000000055950 228 | 000000056127 229 | 000000057149 230 | 000000057597 231 | 000000057672 232 | 000000057725 233 | 000000058029 234 | 000000058384 235 | 000000058636 236 | 000000058655 237 | 000000059386 238 | 000000059635 239 | 000000059920 240 | 000000060090 241 | 000000060347 242 | 000000060363 243 | 000000060449 244 | 000000060507 245 | 000000060770 246 | 000000061268 247 | 000000061418 248 | 000000061471 249 | 000000061584 250 | 000000061747 251 | 000000062025 252 | 000000062353 253 | 000000062355 254 | 000000062692 255 | 000000063047 256 | 000000063154 257 | 000000063602 258 | 000000064462 259 | 000000064495 260 | 000000064499 261 | 000000064574 262 | 000000064898 263 | 000000065736 264 | 000000066038 265 | 000000066706 266 | 000000067213 267 | 000000067310 268 | 000000067406 269 | 000000067616 270 | 000000067896 271 | 000000068078 272 | 000000068387 273 | 000000068628 274 | 000000068833 275 | 000000069795 276 | 000000070254 277 | 000000070774 278 | 000000071756 279 | 000000071877 280 | 000000072281 281 | 000000072852 282 | 000000074058 283 | 000000074457 284 | 000000074646 285 | 000000074860 286 | 000000075393 287 | 000000076211 288 | 000000076261 289 | 000000077460 290 | 000000078032 291 | 000000078266 292 | 000000078565 293 | 000000078915 294 | 000000078959 295 | 000000079034 296 | 000000079144 297 | 000000079229 298 | 000000079565 299 | 000000079588 300 | 000000079837 301 | 000000080022 302 | 000000080153 303 | 000000080273 304 | 000000080274 305 | 000000080413 306 | 000000080671 307 | 000000081594 308 | 000000081766 309 | 000000081988 310 | 000000082715 311 | 000000082821 312 | 000000082986 313 | 000000083113 314 | 000000083172 315 | 000000083540 316 | 000000084031 317 | 000000084170 318 | 000000084270 319 | 000000084477 320 | 000000084492 321 | 000000084752 322 | 000000085376 323 | 000000085478 324 | 000000085576 325 | 000000085665 326 | 000000085682 327 | 000000085772 328 | 000000085911 329 | 000000086220 330 | 000000086483 331 | 000000086755 332 | 000000087038 333 | 000000087244 334 | 000000087875 335 | 000000088218 336 | 000000088265 337 | 000000088345 338 | 000000088432 339 | 000000088462 340 | 000000088951 341 | 000000089045 342 | 000000089078 343 | 000000089296 344 | 000000089697 345 | 000000089761 346 | 000000090003 347 | 000000090062 348 | 000000090108 349 | 000000090155 350 | 000000090284 351 | 000000090631 352 | 000000090956 353 | 000000091615 354 | 000000091619 355 | 000000091921 356 | 000000092124 357 | 000000093154 358 | 000000093261 359 | 000000094157 360 | 000000094185 361 | 000000094326 362 | 000000094614 363 | 000000094751 364 | 000000094852 365 | 000000094944 366 | 000000095069 367 | 000000095155 368 | 000000096001 369 | 000000096825 370 | 000000096960 371 | 000000097022 372 | 000000097230 373 | 000000097988 374 | 000000097994 375 | 000000098018 376 | 000000098287 377 | 000000098392 378 | 000000098497 379 | 000000098520 380 | 000000099242 381 | 000000100274 382 | 000000100283 383 | 000000100489 384 | 000000100510 385 | 000000101022 386 | 000000101780 387 | 000000101787 388 | 000000101884 389 | 000000102411 390 | 000000102644 391 | 000000103548 392 | 000000103585 393 | 000000103723 394 | 000000104198 395 | 000000104424 396 | 000000104572 397 | 000000104603 398 | 000000104619 399 | 000000104666 400 | 000000104782 401 | 000000105264 402 | 000000105455 403 | 000000105912 404 | 000000105923 405 | 000000106266 406 | 000000106281 407 | 000000106563 408 | 000000106881 409 | 000000106912 410 | 000000107094 411 | 000000107554 412 | 000000107851 413 | 000000108440 414 | 000000108503 415 | 000000108864 416 | 000000109441 417 | 000000109827 418 | 000000109900 419 | 000000109976 420 | 000000109992 421 | 000000110042 422 | 000000110282 423 | 000000110359 424 | 000000110784 425 | 000000111179 426 | 000000111207 427 | 000000111951 428 | 000000112110 429 | 000000112298 430 | 000000112378 431 | 000000113051 432 | 000000113354 433 | 000000113867 434 | 000000115118 435 | 000000115898 436 | 000000115946 437 | 000000116206 438 | 000000117197 439 | 000000117374 440 | 000000117645 441 | 000000118405 442 | 000000118921 443 | 000000119038 444 | 000000119088 445 | 000000119445 446 | 000000119516 447 | 000000119641 448 | 000000119677 449 | 000000119911 450 | 000000119995 451 | 000000120572 452 | 000000120584 453 | 000000120777 454 | 000000121031 455 | 000000121153 456 | 000000121242 457 | 000000121586 458 | 000000121591 459 | 000000121673 460 | 000000122166 461 | 000000122217 462 | 000000122672 463 | 000000122927 464 | 000000123131 465 | 000000123480 466 | 000000123585 467 | 000000124277 468 | 000000124442 469 | 000000124659 470 | 000000124798 471 | 000000125257 472 | 000000125405 473 | 000000125572 474 | 000000126107 475 | 000000126216 476 | 000000126592 477 | 000000127092 478 | 000000127135 479 | 000000127182 480 | 000000127263 481 | 000000127530 482 | 000000127624 483 | 000000128112 484 | 000000128148 485 | 000000128654 486 | 000000128658 487 | 000000128675 488 | 000000129054 489 | 000000129062 490 | 000000129322 491 | 000000129416 492 | 000000129756 493 | 000000129812 494 | 000000130386 495 | 000000130465 496 | 000000130566 497 | 000000130579 498 | 000000130586 499 | 000000130613 500 | 000000130699 501 | 000000131138 502 | 000000131386 503 | 000000131431 504 | 000000131556 505 | 000000132408 506 | 000000132703 507 | 000000132931 508 | 000000133087 509 | 000000133233 510 | 000000133418 511 | 000000133645 512 | 000000134034 513 | 000000134322 514 | 000000134689 515 | 000000134722 516 | 000000134856 517 | 000000134886 518 | 000000135561 519 | 000000135604 520 | 000000135670 521 | 000000135673 522 | 000000135872 523 | 000000135890 524 | 000000136033 525 | 000000136334 526 | 000000136600 527 | 000000137576 528 | 000000137950 529 | 000000138550 530 | 000000138639 531 | 000000138819 532 | 000000139077 533 | 000000139871 534 | 000000139883 535 | 000000140270 536 | 000000140286 537 | 000000140420 538 | 000000140439 539 | 000000140583 540 | 000000140658 541 | 000000142238 542 | 000000142472 543 | 000000142790 544 | 000000142971 545 | 000000143572 546 | 000000144114 547 | 000000144706 548 | 000000144784 549 | 000000144798 550 | 000000144932 551 | 000000145620 552 | 000000145665 553 | 000000146363 554 | 000000146498 555 | 000000146831 556 | 000000147518 557 | 000000147725 558 | 000000148707 559 | 000000148730 560 | 000000148739 561 | 000000149375 562 | 000000149406 563 | 000000149622 564 | 000000150224 565 | 000000150265 566 | 000000150726 567 | 000000150930 568 | 000000151051 569 | 000000151480 570 | 000000151516 571 | 000000151629 572 | 000000151662 573 | 000000152465 574 | 000000152686 575 | 000000152771 576 | 000000152870 577 | 000000153343 578 | 000000153510 579 | 000000153527 580 | 000000153782 581 | 000000153797 582 | 000000154004 583 | 000000154087 584 | 000000154213 585 | 000000154358 586 | 000000154425 587 | 000000154431 588 | 000000154705 589 | 000000154947 590 | 000000155051 591 | 000000155145 592 | 000000155154 593 | 000000155291 594 | 000000155341 595 | 000000155443 596 | 000000155451 597 | 000000156278 598 | 000000156292 599 | 000000157046 600 | 000000157124 601 | 000000157213 602 | 000000157365 603 | 000000157756 604 | 000000157847 605 | 000000158548 606 | 000000158956 607 | 000000159399 608 | 000000159977 609 | 000000160728 610 | 000000160772 611 | 000000161032 612 | 000000161128 613 | 000000161397 614 | 000000161642 615 | 000000161781 616 | 000000161799 617 | 000000161861 618 | 000000161875 619 | 000000161879 620 | 000000161978 621 | 000000162035 622 | 000000162130 623 | 000000162543 624 | 000000162581 625 | 000000162858 626 | 000000163057 627 | 000000163118 628 | 000000163155 629 | 000000163258 630 | 000000163290 631 | 000000163746 632 | 000000164115 633 | 000000164637 634 | 000000164885 635 | 000000165336 636 | 000000165351 637 | 000000165500 638 | 000000165518 639 | 000000165713 640 | 000000166165 641 | 000000166166 642 | 000000166259 643 | 000000166287 644 | 000000166391 645 | 000000166509 646 | 000000166521 647 | 000000166563 648 | 000000166664 649 | 000000166747 650 | 000000167240 651 | 000000167540 652 | 000000167898 653 | 000000167902 654 | 000000168330 655 | 000000168337 656 | 000000168458 657 | 000000168619 658 | 000000169169 659 | 000000169356 660 | 000000169996 661 | 000000170474 662 | 000000170545 663 | 000000170955 664 | 000000171050 665 | 000000171611 666 | 000000171788 667 | 000000172083 668 | 000000172547 669 | 000000172648 670 | 000000172649 671 | 000000172856 672 | 000000172935 673 | 000000173033 674 | 000000173057 675 | 000000173183 676 | 000000173302 677 | 000000173830 678 | 000000175387 679 | 000000175438 680 | 000000175443 681 | 000000176037 682 | 000000176232 683 | 000000176606 684 | 000000176701 685 | 000000176778 686 | 000000176799 687 | 000000176847 688 | 000000177065 689 | 000000177357 690 | 000000177714 691 | 000000177861 692 | 000000177893 693 | 000000178028 694 | 000000178618 695 | 000000178744 696 | 000000178982 697 | 000000179265 698 | 000000179487 699 | 000000179653 700 | 000000180792 701 | 000000180798 702 | 000000181421 703 | 000000181666 704 | 000000181859 705 | 000000182021 706 | 000000182162 707 | 000000182441 708 | 000000183127 709 | 000000183437 710 | 000000183500 711 | 000000184321 712 | 000000184324 713 | 000000184400 714 | 000000184791 715 | 000000184978 716 | 000000185157 717 | 000000185250 718 | 000000185292 719 | 000000185472 720 | 000000186042 721 | 000000186345 722 | 000000186422 723 | 000000186632 724 | 000000186637 725 | 000000186873 726 | 000000186938 727 | 000000187362 728 | 000000187513 729 | 000000187734 730 | 000000187745 731 | 000000187990 732 | 000000188439 733 | 000000189213 734 | 000000189226 735 | 000000189310 736 | 000000189698 737 | 000000189806 738 | 000000189828 739 | 000000190007 740 | 000000190307 741 | 000000190753 742 | 000000190756 743 | 000000190841 744 | 000000190923 745 | 000000191013 746 | 000000191288 747 | 000000191614 748 | 000000191672 749 | 000000192047 750 | 000000192964 751 | 000000193162 752 | 000000193245 753 | 000000193429 754 | 000000193494 755 | 000000193674 756 | 000000193884 757 | 000000194506 758 | 000000195045 759 | 000000195165 760 | 000000195754 761 | 000000195842 762 | 000000196141 763 | 000000196185 764 | 000000196442 765 | 000000196754 766 | 000000197388 767 | 000000197658 768 | 000000197796 769 | 000000197870 770 | 000000198915 771 | 000000198928 772 | 000000198960 773 | 000000199236 774 | 000000199395 775 | 000000199442 776 | 000000199977 777 | 000000200152 778 | 000000200667 779 | 000000200839 780 | 000000200961 781 | 000000201418 782 | 000000201676 783 | 000000201775 784 | 000000202228 785 | 000000203095 786 | 000000203317 787 | 000000203488 788 | 000000203546 789 | 000000203580 790 | 000000203639 791 | 000000203864 792 | 000000204329 793 | 000000205105 794 | 000000205289 795 | 000000205324 796 | 000000205401 797 | 000000205542 798 | 000000205647 799 | 000000206994 800 | 000000207306 801 | 000000208208 802 | 000000208423 803 | 000000208901 804 | 000000209222 805 | 000000209613 806 | 000000209757 807 | 000000209829 808 | 000000210299 809 | 000000210388 810 | 000000210394 811 | 000000210855 812 | 000000210915 813 | 000000211042 814 | 000000211069 815 | 000000212072 816 | 000000212453 817 | 000000212895 818 | 000000213171 819 | 000000213593 820 | 000000213830 821 | 000000214192 822 | 000000214205 823 | 000000214224 824 | 000000214539 825 | 000000214703 826 | 000000214753 827 | 000000215245 828 | 000000215644 829 | 000000216296 830 | 000000216419 831 | 000000216497 832 | 000000216516 833 | 000000216739 834 | 000000217060 835 | 000000217425 836 | 000000217614 837 | 000000217957 838 | 000000218091 839 | 000000218997 840 | 000000219283 841 | 000000219485 842 | 000000220584 843 | 000000220764 844 | 000000220858 845 | 000000221017 846 | 000000221155 847 | 000000221708 848 | 000000221754 849 | 000000222094 850 | 000000222118 851 | 000000222235 852 | 000000222458 853 | 000000222559 854 | 000000222825 855 | 000000222991 856 | 000000223130 857 | 000000223738 858 | 000000224051 859 | 000000224119 860 | 000000224222 861 | 000000224664 862 | 000000224724 863 | 000000225184 864 | 000000225405 865 | 000000225532 866 | 000000225946 867 | 000000226111 868 | 000000226802 869 | 000000226883 870 | 000000226984 871 | 000000227187 872 | 000000227478 873 | 000000227482 874 | 000000227491 875 | 000000227511 876 | 000000227898 877 | 000000228771 878 | 000000228981 879 | 000000229311 880 | 000000229553 881 | 000000229601 882 | 000000229747 883 | 000000229753 884 | 000000229858 885 | 000000229997 886 | 000000230166 887 | 000000230362 888 | 000000230450 889 | 000000230819 890 | 000000230983 891 | 000000231169 892 | 000000231549 893 | 000000231747 894 | 000000232538 895 | 000000232646 896 | 000000232692 897 | 000000233033 898 | 000000233139 899 | 000000233370 900 | 000000233825 901 | 000000234366 902 | 000000234413 903 | 000000234757 904 | 000000234807 905 | 000000235057 906 | 000000235252 907 | 000000235784 908 | 000000235836 909 | 000000236308 910 | 000000236426 911 | 000000236690 912 | 000000237071 913 | 000000237316 914 | 000000237864 915 | 000000237984 916 | 000000238039 917 | 000000239041 918 | 000000239274 919 | 000000239537 920 | 000000239843 921 | 000000240754 922 | 000000240767 923 | 000000241297 924 | 000000241319 925 | 000000241602 926 | 000000241677 927 | 000000242287 928 | 000000242411 929 | 000000242678 930 | 000000242724 931 | 000000243034 932 | 000000243495 933 | 000000244019 934 | 000000244099 935 | 000000244379 936 | 000000244411 937 | 000000245102 938 | 000000245173 939 | 000000245320 940 | 000000245513 941 | 000000246308 942 | 000000246522 943 | 000000246883 944 | 000000246963 945 | 000000246968 946 | 000000247806 947 | 000000247838 948 | 000000248111 949 | 000000248112 950 | 000000248284 951 | 000000248752 952 | 000000248810 953 | 000000248980 954 | 000000249025 955 | 000000249180 956 | 000000249786 957 | 000000250205 958 | 000000250619 959 | 000000250758 960 | 000000251065 961 | 000000252219 962 | 000000252507 963 | 000000252559 964 | 000000252701 965 | 000000252716 966 | 000000253819 967 | 000000253835 968 | 000000254016 969 | 000000255401 970 | 000000255536 971 | 000000255664 972 | 000000255749 973 | 000000255917 974 | 000000256407 975 | 000000256775 976 | 000000256868 977 | 000000257478 978 | 000000257566 979 | 000000257865 980 | 000000258388 981 | 000000258793 982 | 000000259382 983 | 000000259625 984 | 000000259830 985 | 000000259854 986 | 000000260105 987 | 000000260106 988 | 000000260266 989 | 000000261535 990 | 000000261712 991 | 000000261796 992 | 000000261888 993 | 000000262440 994 | 000000262487 995 | 000000262587 996 | 000000262682 997 | 000000263403 998 | 000000263425 999 | 000000263644 1000 | 000000263679 1001 | 000000264335 1002 | 000000264535 1003 | 000000265108 1004 | 000000265518 1005 | 000000265816 1006 | 000000266082 1007 | 000000266206 1008 | 000000266409 1009 | 000000266892 1010 | 000000266981 1011 | 000000267351 1012 | 000000267434 1013 | 000000267670 1014 | 000000267903 1015 | 000000267940 1016 | 000000268000 1017 | 000000268729 1018 | 000000268996 1019 | 000000269113 1020 | 000000269121 1021 | 000000269314 1022 | 000000269316 1023 | 000000269682 1024 | 000000269942 1025 | 000000270122 1026 | 000000270244 1027 | 000000270297 1028 | 000000270386 1029 | 000000270402 1030 | 000000270474 1031 | 000000270677 1032 | 000000272148 1033 | 000000272212 1034 | 000000272416 1035 | 000000273198 1036 | 000000273232 1037 | 000000273420 1038 | 000000273617 1039 | 000000273760 1040 | 000000274066 1041 | 000000274460 1042 | 000000275058 1043 | 000000275727 1044 | 000000276024 1045 | 000000276284 1046 | 000000276707 1047 | 000000276720 1048 | 000000276804 1049 | 000000277005 1050 | 000000277197 1051 | 000000278006 1052 | 000000278463 1053 | 000000278705 1054 | 000000278973 1055 | 000000279541 1056 | 000000279769 1057 | 000000279774 1058 | 000000280325 1059 | 000000280779 1060 | 000000281179 1061 | 000000281414 1062 | 000000281447 1063 | 000000282037 1064 | 000000282912 1065 | 000000283037 1066 | 000000283070 1067 | 000000283318 1068 | 000000283520 1069 | 000000284296 1070 | 000000284445 1071 | 000000284698 1072 | 000000284762 1073 | 000000285788 1074 | 000000286503 1075 | 000000286507 1076 | 000000286907 1077 | 000000286994 1078 | 000000287291 1079 | 000000287545 1080 | 000000287714 1081 | 000000287874 1082 | 000000288042 1083 | 000000288062 1084 | 000000288391 1085 | 000000288430 1086 | 000000288584 1087 | 000000288862 1088 | 000000289343 1089 | 000000289417 1090 | 000000289586 1091 | 000000289594 1092 | 000000289659 1093 | 000000289702 1094 | 000000289741 1095 | 000000289938 1096 | 000000289960 1097 | 000000289992 1098 | 000000290179 1099 | 000000290248 1100 | 000000290619 1101 | 000000290768 1102 | 000000290771 1103 | 000000291551 1104 | 000000291664 1105 | 000000292005 1106 | 000000292060 1107 | 000000292456 1108 | 000000292908 1109 | 000000292997 1110 | 000000293300 1111 | 000000293324 1112 | 000000293390 1113 | 000000294163 1114 | 000000294695 1115 | 000000294783 1116 | 000000294855 1117 | 000000295138 1118 | 000000295316 1119 | 000000295420 1120 | 000000295478 1121 | 000000295797 1122 | 000000295809 1123 | 000000296231 1124 | 000000296317 1125 | 000000296649 1126 | 000000296657 1127 | 000000296969 1128 | 000000297022 1129 | 000000297343 1130 | 000000297595 1131 | 000000297681 1132 | 000000297698 1133 | 000000298697 1134 | 000000298994 1135 | 000000299553 1136 | 000000299609 1137 | 000000299720 1138 | 000000300155 1139 | 000000300659 1140 | 000000300842 1141 | 000000301135 1142 | 000000301563 1143 | 000000301981 1144 | 000000302107 1145 | 000000302760 1146 | 000000302990 1147 | 000000303653 1148 | 000000303818 1149 | 000000304180 1150 | 000000304291 1151 | 000000304365 1152 | 000000304396 1153 | 000000304812 1154 | 000000304817 1155 | 000000305309 1156 | 000000305695 1157 | 000000306437 1158 | 000000306582 1159 | 000000306733 1160 | 000000307598 1161 | 000000308165 1162 | 000000308193 1163 | 000000308466 1164 | 000000308587 1165 | 000000308753 1166 | 000000308793 1167 | 000000308799 1168 | 000000309391 1169 | 000000309452 1170 | 000000309655 1171 | 000000310622 1172 | 000000311081 1173 | 000000311295 1174 | 000000311518 1175 | 000000311928 1176 | 000000311950 1177 | 000000312263 1178 | 000000312340 1179 | 000000312406 1180 | 000000312549 1181 | 000000312586 1182 | 000000312720 1183 | 000000313588 1184 | 000000313783 1185 | 000000314034 1186 | 000000314177 1187 | 000000314251 1188 | 000000314294 1189 | 000000314541 1190 | 000000314709 1191 | 000000314914 1192 | 000000315001 1193 | 000000315187 1194 | 000000316054 1195 | 000000316666 1196 | 000000319369 1197 | 000000319607 1198 | 000000319935 1199 | 000000320232 1200 | 000000320425 1201 | 000000320490 1202 | 000000320696 1203 | 000000320706 1204 | 000000321790 1205 | 000000322724 1206 | 000000322864 1207 | 000000322968 1208 | 000000323263 1209 | 000000323496 1210 | 000000323571 1211 | 000000323751 1212 | 000000323828 1213 | 000000323895 1214 | 000000324818 1215 | 000000325114 1216 | 000000325527 1217 | 000000326082 1218 | 000000326128 1219 | 000000326174 1220 | 000000326248 1221 | 000000326542 1222 | 000000326627 1223 | 000000327306 1224 | 000000327605 1225 | 000000327701 1226 | 000000327769 1227 | 000000327890 1228 | 000000328117 1229 | 000000328238 1230 | 000000328286 1231 | 000000328337 1232 | 000000328430 1233 | 000000328683 1234 | 000000329219 1235 | 000000329827 1236 | 000000330396 1237 | 000000330554 1238 | 000000331317 1239 | 000000331604 1240 | 000000332318 1241 | 000000332351 1242 | 000000333069 1243 | 000000333697 1244 | 000000333956 1245 | 000000334006 1246 | 000000334371 1247 | 000000334521 1248 | 000000334530 1249 | 000000334719 1250 | 000000335177 1251 | 000000335328 1252 | 000000335450 1253 | 000000335529 1254 | 000000335658 1255 | 000000335800 1256 | 000000336209 1257 | 000000336265 1258 | 000000336309 1259 | 000000336587 1260 | 000000336658 1261 | 000000337987 1262 | 000000338325 1263 | 000000338532 1264 | 000000338560 1265 | 000000338624 1266 | 000000338625 1267 | 000000338986 1268 | 000000339870 1269 | 000000340175 1270 | 000000340272 1271 | 000000341094 1272 | 000000341196 1273 | 000000341681 1274 | 000000341719 1275 | 000000341921 1276 | 000000342006 1277 | 000000342397 1278 | 000000343149 1279 | 000000343218 1280 | 000000343315 1281 | 000000343453 1282 | 000000343496 1283 | 000000343524 1284 | 000000343803 1285 | 000000343934 1286 | 000000343937 1287 | 000000343976 1288 | 000000344059 1289 | 000000344611 1290 | 000000344614 1291 | 000000344816 1292 | 000000344888 1293 | 000000345027 1294 | 000000345261 1295 | 000000345466 1296 | 000000347265 1297 | 000000347664 1298 | 000000347693 1299 | 000000348045 1300 | 000000348243 1301 | 000000348481 1302 | 000000348881 1303 | 000000349302 1304 | 000000349678 1305 | 000000349860 1306 | 000000350003 1307 | 000000350023 1308 | 000000350054 1309 | 000000350122 1310 | 000000350488 1311 | 000000350833 1312 | 000000351362 1313 | 000000351823 1314 | 000000352760 1315 | 000000353518 1316 | 000000354753 1317 | 000000354829 1318 | 000000355677 1319 | 000000355817 1320 | 000000355905 1321 | 000000356094 1322 | 000000356169 1323 | 000000356261 1324 | 000000356427 1325 | 000000356505 1326 | 000000356968 1327 | 000000357060 1328 | 000000357238 1329 | 000000357459 1330 | 000000357501 1331 | 000000357567 1332 | 000000357742 1333 | 000000357888 1334 | 000000357978 1335 | 000000358195 1336 | 000000359540 1337 | 000000359677 1338 | 000000359781 1339 | 000000360137 1340 | 000000360564 1341 | 000000360661 1342 | 000000360943 1343 | 000000361142 1344 | 000000361268 1345 | 000000361506 1346 | 000000361551 1347 | 000000361586 1348 | 000000361730 1349 | 000000361919 1350 | 000000362434 1351 | 000000362716 1352 | 000000363072 1353 | 000000363461 1354 | 000000363784 1355 | 000000364102 1356 | 000000364126 1357 | 000000364557 1358 | 000000365095 1359 | 000000365207 1360 | 000000365208 1361 | 000000365521 1362 | 000000365655 1363 | 000000365745 1364 | 000000365766 1365 | 000000366141 1366 | 000000366178 1367 | 000000366611 1368 | 000000366884 1369 | 000000367569 1370 | 000000367818 1371 | 000000368900 1372 | 000000368940 1373 | 000000368982 1374 | 000000369081 1375 | 000000369503 1376 | 000000369541 1377 | 000000369751 1378 | 000000369757 1379 | 000000369812 1380 | 000000370813 1381 | 000000370818 1382 | 000000370999 1383 | 000000371042 1384 | 000000371472 1385 | 000000371552 1386 | 000000371699 1387 | 000000372203 1388 | 000000372260 1389 | 000000372307 1390 | 000000372349 1391 | 000000372577 1392 | 000000372819 1393 | 000000373315 1394 | 000000373353 1395 | 000000373705 1396 | 000000374052 1397 | 000000374369 1398 | 000000374545 1399 | 000000374727 1400 | 000000375078 1401 | 000000375763 1402 | 000000376206 1403 | 000000376310 1404 | 000000376478 1405 | 000000376625 1406 | 000000376856 1407 | 000000376900 1408 | 000000377393 1409 | 000000377486 1410 | 000000377588 1411 | 000000377635 1412 | 000000377723 1413 | 000000377882 1414 | 000000378116 1415 | 000000378139 1416 | 000000378244 1417 | 000000378454 1418 | 000000378673 1419 | 000000378873 1420 | 000000379441 1421 | 000000379453 1422 | 000000379533 1423 | 000000380706 1424 | 000000380711 1425 | 000000381360 1426 | 000000381639 1427 | 000000381971 1428 | 000000382088 1429 | 000000382122 1430 | 000000382696 1431 | 000000382734 1432 | 000000382743 1433 | 000000383339 1434 | 000000383384 1435 | 000000383443 1436 | 000000383606 1437 | 000000383621 1438 | 000000383676 1439 | 000000384350 1440 | 000000384513 1441 | 000000384616 1442 | 000000384670 1443 | 000000384850 1444 | 000000384949 1445 | 000000385190 1446 | 000000385997 1447 | 000000386277 1448 | 000000386457 1449 | 000000387148 1450 | 000000387387 1451 | 000000387916 1452 | 000000388215 1453 | 000000388846 1454 | 000000389197 1455 | 000000389451 1456 | 000000389532 1457 | 000000389804 1458 | 000000390246 1459 | 000000390301 1460 | 000000391290 1461 | 000000391375 1462 | 000000391648 1463 | 000000392228 1464 | 000000392481 1465 | 000000392722 1466 | 000000392933 1467 | 000000393056 1468 | 000000393093 1469 | 000000393226 1470 | 000000393282 1471 | 000000393469 1472 | 000000393569 1473 | 000000394328 1474 | 000000394559 1475 | 000000394611 1476 | 000000395180 1477 | 000000395633 1478 | 000000395801 1479 | 000000395903 1480 | 000000396200 1481 | 000000396205 1482 | 000000396274 1483 | 000000396518 1484 | 000000396568 1485 | 000000396580 1486 | 000000396903 1487 | 000000397279 1488 | 000000397327 1489 | 000000397639 1490 | 000000398237 1491 | 000000398810 1492 | 000000399296 1493 | 000000400044 1494 | 000000400161 1495 | 000000400922 1496 | 000000401244 1497 | 000000401446 1498 | 000000402096 1499 | 000000402118 1500 | 000000402334 1501 | 000000402519 1502 | 000000402774 1503 | 000000403122 1504 | 000000403385 1505 | 000000403565 1506 | 000000403584 1507 | 000000404128 1508 | 000000404249 1509 | 000000404479 1510 | 000000404534 1511 | 000000404568 1512 | 000000404601 1513 | 000000404805 1514 | 000000404923 1515 | 000000405279 1516 | 000000405970 1517 | 000000406611 1518 | 000000407002 1519 | 000000407518 1520 | 000000407614 1521 | 000000407650 1522 | 000000407868 1523 | 000000408112 1524 | 000000408120 1525 | 000000409198 1526 | 000000409358 1527 | 000000409475 1528 | 000000409542 1529 | 000000409867 1530 | 000000410221 1531 | 000000410428 1532 | 000000410456 1533 | 000000410510 1534 | 000000410650 1535 | 000000410712 1536 | 000000410934 1537 | 000000411530 1538 | 000000412240 1539 | 000000412531 1540 | 000000412894 1541 | 000000413404 1542 | 000000413689 1543 | 000000414133 1544 | 000000414170 1545 | 000000414340 1546 | 000000414385 1547 | 000000414676 1548 | 000000415194 1549 | 000000415238 1550 | 000000415536 1551 | 000000415716 1552 | 000000415741 1553 | 000000415748 1554 | 000000415990 1555 | 000000416170 1556 | 000000416745 1557 | 000000416837 1558 | 000000417043 1559 | 000000417085 1560 | 000000417779 1561 | 000000417876 1562 | 000000417911 1563 | 000000418281 1564 | 000000418696 1565 | 000000418959 1566 | 000000418961 1567 | 000000419201 1568 | 000000419408 1569 | 000000420069 1570 | 000000420472 1571 | 000000420916 1572 | 000000421060 1573 | 000000421757 1574 | 000000421834 1575 | 000000421923 1576 | 000000422706 1577 | 000000423104 1578 | 000000423229 1579 | 000000423944 1580 | 000000423971 1581 | 000000424135 1582 | 000000424551 1583 | 000000424642 1584 | 000000424776 1585 | 000000424975 1586 | 000000425221 1587 | 000000425227 1588 | 000000425906 1589 | 000000425925 1590 | 000000426203 1591 | 000000426241 1592 | 000000426376 1593 | 000000426836 1594 | 000000427055 1595 | 000000427160 1596 | 000000427338 1597 | 000000427500 1598 | 000000427649 1599 | 000000428111 1600 | 000000428218 1601 | 000000429718 1602 | 000000429761 1603 | 000000430048 1604 | 000000430073 1605 | 000000430377 1606 | 000000430871 1607 | 000000430875 1608 | 000000431693 1609 | 000000431727 1610 | 000000431848 1611 | 000000431876 1612 | 000000432898 1613 | 000000433204 1614 | 000000433243 1615 | 000000433374 1616 | 000000433515 1617 | 000000433774 1618 | 000000434247 1619 | 000000434548 1620 | 000000435206 1621 | 000000435880 1622 | 000000436738 1623 | 000000437110 1624 | 000000437239 1625 | 000000437331 1626 | 000000437898 1627 | 000000438017 1628 | 000000438269 1629 | 000000438907 1630 | 000000439290 1631 | 000000439715 1632 | 000000440184 1633 | 000000440475 1634 | 000000440507 1635 | 000000440508 1636 | 000000440617 1637 | 000000441286 1638 | 000000441442 1639 | 000000441468 1640 | 000000441586 1641 | 000000442009 1642 | 000000442306 1643 | 000000442323 1644 | 000000442463 1645 | 000000442480 1646 | 000000442661 1647 | 000000442746 1648 | 000000442836 1649 | 000000442993 1650 | 000000443498 1651 | 000000443844 1652 | 000000444275 1653 | 000000444879 1654 | 000000445602 1655 | 000000445675 1656 | 000000445846 1657 | 000000446206 1658 | 000000446574 1659 | 000000447169 1660 | 000000447187 1661 | 000000447313 1662 | 000000447314 1663 | 000000447342 1664 | 000000447465 1665 | 000000447789 1666 | 000000447917 1667 | 000000448076 1668 | 000000448365 1669 | 000000448810 1670 | 000000449579 1671 | 000000449603 1672 | 000000449996 1673 | 000000450303 1674 | 000000450559 1675 | 000000450758 1676 | 000000451043 1677 | 000000451084 1678 | 000000451090 1679 | 000000451144 1680 | 000000451155 1681 | 000000451435 1682 | 000000451714 1683 | 000000452122 1684 | 000000453302 1685 | 000000453341 1686 | 000000453584 1687 | 000000453634 1688 | 000000453722 1689 | 000000454798 1690 | 000000454978 1691 | 000000455219 1692 | 000000455352 1693 | 000000455555 1694 | 000000455872 1695 | 000000456015 1696 | 000000456292 1697 | 000000456496 1698 | 000000456865 1699 | 000000457078 1700 | 000000457884 1701 | 000000458045 1702 | 000000458410 1703 | 000000458663 1704 | 000000458702 1705 | 000000458768 1706 | 000000459153 1707 | 000000459195 1708 | 000000459467 1709 | 000000459500 1710 | 000000459634 1711 | 000000459757 1712 | 000000459809 1713 | 000000460147 1714 | 000000460160 1715 | 000000460333 1716 | 000000460347 1717 | 000000460379 1718 | 000000460841 1719 | 000000460967 1720 | 000000461405 1721 | 000000461573 1722 | 000000462614 1723 | 000000462728 1724 | 000000462756 1725 | 000000463174 1726 | 000000463542 1727 | 000000463647 1728 | 000000463690 1729 | 000000463849 1730 | 000000464089 1731 | 000000464144 1732 | 000000464251 1733 | 000000464689 1734 | 000000464872 1735 | 000000465675 1736 | 000000465836 1737 | 000000466085 1738 | 000000466416 1739 | 000000466567 1740 | 000000466602 1741 | 000000466835 1742 | 000000466986 1743 | 000000467315 1744 | 000000467776 1745 | 000000467848 1746 | 000000468245 1747 | 000000468501 1748 | 000000468632 1749 | 000000469174 1750 | 000000469192 1751 | 000000469828 1752 | 000000470173 1753 | 000000471023 1754 | 000000471450 1755 | 000000471567 1756 | 000000471756 1757 | 000000471789 1758 | 000000471991 1759 | 000000472030 1760 | 000000472046 1761 | 000000472298 1762 | 000000472623 1763 | 000000472678 1764 | 000000473015 1765 | 000000474028 1766 | 000000474078 1767 | 000000474170 1768 | 000000474293 1769 | 000000474344 1770 | 000000474452 1771 | 000000474786 1772 | 000000474881 1773 | 000000475191 1774 | 000000475223 1775 | 000000475484 1776 | 000000475904 1777 | 000000476119 1778 | 000000476258 1779 | 000000477227 1780 | 000000477441 1781 | 000000477955 1782 | 000000478286 1783 | 000000478474 1784 | 000000478721 1785 | 000000479030 1786 | 000000479099 1787 | 000000479912 1788 | 000000479953 1789 | 000000480842 1790 | 000000480944 1791 | 000000481159 1792 | 000000481404 1793 | 000000481413 1794 | 000000481480 1795 | 000000481567 1796 | 000000482100 1797 | 000000482477 1798 | 000000482487 1799 | 000000482735 1800 | 000000482800 1801 | 000000484029 1802 | 000000484296 1803 | 000000484760 1804 | 000000485071 1805 | 000000485424 1806 | 000000485802 1807 | 000000486104 1808 | 000000486573 1809 | 000000487583 1810 | 000000488075 1811 | 000000488270 1812 | 000000488592 1813 | 000000488664 1814 | 000000488736 1815 | 000000489046 1816 | 000000489924 1817 | 000000490171 1818 | 000000490470 1819 | 000000490515 1820 | 000000491071 1821 | 000000491464 1822 | 000000491470 1823 | 000000491613 1824 | 000000491683 1825 | 000000491725 1826 | 000000492282 1827 | 000000492284 1828 | 000000492758 1829 | 000000492968 1830 | 000000493019 1831 | 000000493442 1832 | 000000493613 1833 | 000000493772 1834 | 000000493864 1835 | 000000493905 1836 | 000000494759 1837 | 000000494869 1838 | 000000495054 1839 | 000000496409 1840 | 000000496597 1841 | 000000496722 1842 | 000000498032 1843 | 000000498807 1844 | 000000498857 1845 | 000000499181 1846 | 000000499768 1847 | 000000499775 1848 | 000000500049 1849 | 000000500211 1850 | 000000500270 1851 | 000000500464 1852 | 000000500663 1853 | 000000500826 1854 | 000000501005 1855 | 000000501023 1856 | 000000502136 1857 | 000000502168 1858 | 000000502229 1859 | 000000502336 1860 | 000000502347 1861 | 000000502910 1862 | 000000503841 1863 | 000000503855 1864 | 000000504415 1865 | 000000504580 1866 | 000000504589 1867 | 000000504635 1868 | 000000505169 1869 | 000000505565 1870 | 000000505789 1871 | 000000505942 1872 | 000000506004 1873 | 000000506310 1874 | 000000506454 1875 | 000000507015 1876 | 000000507037 1877 | 000000507081 1878 | 000000507575 1879 | 000000507893 1880 | 000000508101 1881 | 000000508586 1882 | 000000508639 1883 | 000000508917 1884 | 000000509258 1885 | 000000509403 1886 | 000000509656 1887 | 000000509719 1888 | 000000509735 1889 | 000000510095 1890 | 000000510329 1891 | 000000511076 1892 | 000000511321 1893 | 000000511384 1894 | 000000511599 1895 | 000000511647 1896 | 000000512194 1897 | 000000512248 1898 | 000000512476 1899 | 000000512564 1900 | 000000512648 1901 | 000000512985 1902 | 000000513181 1903 | 000000513524 1904 | 000000513580 1905 | 000000514376 1906 | 000000514540 1907 | 000000514797 1908 | 000000514914 1909 | 000000515350 1910 | 000000515445 1911 | 000000515828 1912 | 000000515982 1913 | 000000516038 1914 | 000000516173 1915 | 000000516316 1916 | 000000516318 1917 | 000000516601 1918 | 000000516677 1919 | 000000516708 1920 | 000000516804 1921 | 000000517069 1922 | 000000517523 1923 | 000000518213 1924 | 000000519338 1925 | 000000519491 1926 | 000000519522 1927 | 000000519569 1928 | 000000519611 1929 | 000000520009 1930 | 000000520324 1931 | 000000520659 1932 | 000000520707 1933 | 000000520832 1934 | 000000520910 1935 | 000000521141 1936 | 000000521231 1937 | 000000521282 1938 | 000000521509 1939 | 000000521717 1940 | 000000521819 1941 | 000000522393 1942 | 000000522713 1943 | 000000522889 1944 | 000000523033 1945 | 000000523194 1946 | 000000523811 1947 | 000000523957 1948 | 000000525083 1949 | 000000525286 1950 | 000000525322 1951 | 000000526103 1952 | 000000526256 1953 | 000000526706 1954 | 000000526728 1955 | 000000526751 1956 | 000000527215 1957 | 000000527528 1958 | 000000527616 1959 | 000000527750 1960 | 000000527960 1961 | 000000528314 1962 | 000000528524 1963 | 000000528578 1964 | 000000528705 1965 | 000000528862 1966 | 000000528977 1967 | 000000528980 1968 | 000000529105 1969 | 000000529568 1970 | 000000529939 1971 | 000000530457 1972 | 000000530470 1973 | 000000530820 1974 | 000000530836 1975 | 000000531134 1976 | 000000531135 1977 | 000000531495 1978 | 000000531707 1979 | 000000532071 1980 | 000000532481 1981 | 000000532493 1982 | 000000532530 1983 | 000000532855 1984 | 000000534270 1985 | 000000535523 1986 | 000000535578 1987 | 000000535608 1988 | 000000535858 1989 | 000000536947 1990 | 000000537053 1991 | 000000537153 1992 | 000000537355 1993 | 000000537812 1994 | 000000537827 1995 | 000000537964 1996 | 000000538067 1997 | 000000538458 1998 | 000000539445 1999 | 000000539962 2000 | 000000540414 2001 | 000000540466 2002 | 000000540502 2003 | 000000540932 2004 | 000000540962 2005 | 000000541055 2006 | 000000541291 2007 | 000000541952 2008 | 000000542073 2009 | 000000542089 2010 | 000000542127 2011 | 000000542423 2012 | 000000543300 2013 | 000000543528 2014 | 000000544052 2015 | 000000544306 2016 | 000000544444 2017 | 000000544605 2018 | 000000545007 2019 | 000000545100 2020 | 000000545129 2021 | 000000545407 2022 | 000000545730 2023 | 000000545958 2024 | 000000546659 2025 | 000000546823 2026 | 000000546829 2027 | 000000547502 2028 | 000000547519 2029 | 000000547816 2030 | 000000547886 2031 | 000000548267 2032 | 000000548524 2033 | 000000549136 2034 | 000000549390 2035 | 000000549738 2036 | 000000550084 2037 | 000000550939 2038 | 000000551215 2039 | 000000551304 2040 | 000000551350 2041 | 000000551780 2042 | 000000551794 2043 | 000000551804 2044 | 000000551820 2045 | 000000552842 2046 | 000000552883 2047 | 000000552902 2048 | 000000553094 2049 | 000000553221 2050 | 000000553339 2051 | 000000553669 2052 | 000000553990 2053 | 000000554156 2054 | 000000554595 2055 | 000000555012 2056 | 000000555597 2057 | 000000556158 2058 | 000000556498 2059 | 000000556765 2060 | 000000556873 2061 | 000000557172 2062 | 000000557501 2063 | 000000557884 2064 | 000000558073 2065 | 000000558213 2066 | 000000558421 2067 | 000000558558 2068 | 000000559099 2069 | 000000559160 2070 | 000000559513 2071 | 000000560312 2072 | 000000560371 2073 | 000000560880 2074 | 000000561009 2075 | 000000561223 2076 | 000000561256 2077 | 000000561335 2078 | 000000561366 2079 | 000000561679 2080 | 000000561889 2081 | 000000561958 2082 | 000000562121 2083 | 000000562229 2084 | 000000562448 2085 | 000000562581 2086 | 000000562818 2087 | 000000563281 2088 | 000000563349 2089 | 000000563470 2090 | 000000563603 2091 | 000000563604 2092 | 000000563648 2093 | 000000563653 2094 | 000000563702 2095 | 000000564023 2096 | 000000564127 2097 | 000000564133 2098 | 000000565012 2099 | 000000565153 2100 | 000000565469 2101 | 000000565563 2102 | 000000565776 2103 | 000000565778 2104 | 000000565853 2105 | 000000565989 2106 | 000000566282 2107 | 000000566524 2108 | 000000567011 2109 | 000000567197 2110 | 000000567432 2111 | 000000567740 2112 | 000000568147 2113 | 000000568290 2114 | 000000568439 2115 | 000000568710 2116 | 000000568981 2117 | 000000569030 2118 | 000000569059 2119 | 000000569273 2120 | 000000569565 2121 | 000000569917 2122 | 000000569972 2123 | 000000570448 2124 | 000000570456 2125 | 000000570688 2126 | 000000570736 2127 | 000000570756 2128 | 000000570834 2129 | 000000571264 2130 | 000000571313 2131 | 000000571804 2132 | 000000571857 2133 | 000000571893 2134 | 000000571943 2135 | 000000572408 2136 | 000000572517 2137 | 000000572620 2138 | 000000572900 2139 | 000000573008 2140 | 000000573258 2141 | 000000573391 2142 | 000000573626 2143 | 000000573943 2144 | 000000574297 2145 | 000000574520 2146 | 000000575081 2147 | 000000575187 2148 | 000000575205 2149 | 000000575243 2150 | 000000575357 2151 | 000000575372 2152 | 000000575500 2153 | 000000576031 2154 | 000000576052 2155 | 000000576566 2156 | 000000576654 2157 | 000000576955 2158 | 000000577149 2159 | 000000577584 2160 | 000000577864 2161 | 000000577932 2162 | 000000577959 2163 | 000000577976 2164 | 000000578093 2165 | 000000578236 2166 | 000000579307 2167 | 000000579635 2168 | 000000579818 2169 | 000000580410 2170 | 000000580757 2171 | 000000581062 2172 | 000000581100 2173 | 000000581357 2174 | 000000581482 2175 | 000000581615 2176 | -------------------------------------------------------------------------------- /data/coco/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2023 Google LLC 4 | 5 | Permission is hereby granted, free of charge, to any 6 | person obtaining a copy of this software and associated 7 | documentation files (the "Software"), to deal in the 8 | Software without restriction, including without 9 | limitation the rights to use, copy, modify, merge, 10 | publish, distribute, sublicense, and/or sell copies of 11 | the Software, and to permit persons to whom the Software 12 | is furnished to do so, subject to the following 13 | conditions: 14 | 15 | The above copyright notice and this permission notice 16 | shall be included in all copies or substantial portions 17 | of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 20 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 21 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 22 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 23 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 24 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 25 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 26 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 27 | DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /data/coco/coco_data.py: -------------------------------------------------------------------------------- 1 | #Copyright 2023 Google LLC 2 | 3 | #Use of this source code is governed by an MIT-style 4 | #license that can be found in the LICENSE file or at 5 | #https://opensource.org/licenses/MIT. 6 | 7 | """coco_data.py 8 | Data pre-processing and loading pipeline for COCO-Stuff-27. 9 | """ 10 | 11 | import os 12 | import pickle 13 | import numpy as np 14 | from tensorflow import keras 15 | import tensorflow as tf 16 | 17 | RESOLUTION = 512 18 | AUTO = tf.data.AUTOTUNE 19 | 20 | augmenter = keras.Sequential( 21 | layers=[ 22 | tf.keras.layers.Rescaling(scale=1.0 / 127.5, offset=-1), 23 | ] 24 | ) 25 | 26 | def get_fine_to_coarse(fine_to_coarse_path): 27 | """Map fine label indexing to coarse label indexing.""" 28 | with open(fine_to_coarse_path, "rb") as f: 29 | d = pickle.load(f) 30 | fine_to_coarse_dict = d["fine_index_to_coarse_index"] 31 | fine_to_coarse_dict[255] = -1 32 | fine_to_coarse_map = np.vectorize( 33 | lambda x: fine_to_coarse_dict[x] 34 | ) # not in-place. 35 | return fine_to_coarse_map 36 | 37 | def load_imdb(imdb_path): 38 | with open(imdb_path, "r") as f: 39 | imdb = tuple(f) 40 | imdb = [id_.rstrip() for id_ in imdb] 41 | return imdb 42 | 43 | def create_path(root, file_list, split="val"): 44 | """This function creates data loading paths.""" 45 | image_path = [] 46 | label_path = [] 47 | 48 | image_folder = split + "2017" 49 | label_folder = "annotation_" + split + "2017" 50 | 51 | for file in file_list: 52 | image_path.append(os.path.join(root, image_folder, file + ".jpg")) 53 | label_path.append(os.path.join(root, label_folder, file + ".png")) 54 | return image_path, label_path 55 | 56 | def process_image(image_path, label_path): 57 | """This function reads and resizes images and labels.""" 58 | image = tf.io.read_file(image_path) 59 | image = tf.io.decode_jpeg(image, 3) 60 | label = tf.io.read_file(label_path) 61 | label = tf.io.decode_png(label, 3) 62 | 63 | s = tf.shape(image) 64 | w, h = s[0], s[1] 65 | c = tf.minimum(w, h) 66 | w_start = (w - c) // 2 67 | h_start = (h - c) // 2 68 | image = image[w_start : w_start + c, h_start : h_start + c, :] 69 | label = label[w_start : w_start + c, h_start : h_start + c] 70 | image = tf.image.resize(image, (RESOLUTION, RESOLUTION)) 71 | label = tf.image.resize( 72 | label, 73 | (RESOLUTION, RESOLUTION), 74 | method=tf.image.ResizeMethod.NEAREST_NEIGHBOR, 75 | ) 76 | return image, label 77 | 78 | def apply_augmentation(image_batch, label_batch): 79 | """This function applies image augmentation to batches.""" 80 | return augmenter(image_batch), label_batch 81 | 82 | def prepare_dict(image_batch, label_batch): 83 | return { 84 | "images": image_batch, 85 | "labels": label_batch, 86 | } 87 | 88 | def prepare_dataset(image_paths, label_paths, batch_size=1): 89 | dataset = tf.data.Dataset.from_tensor_slices((image_paths, label_paths)) 90 | dataset = dataset.map(process_image, num_parallel_calls=AUTO).batch( 91 | batch_size 92 | ) 93 | dataset = dataset.map(apply_augmentation, num_parallel_calls=AUTO) 94 | dataset = dataset.map(prepare_dict, num_parallel_calls=AUTO) 95 | return dataset.prefetch(AUTO) -------------------------------------------------------------------------------- /data/coco/fine_to_coarse_dict.pickle: -------------------------------------------------------------------------------- 1 | (dp0 2 | S'fine_name_to_coarse_name' 3 | p1 4 | (dp2 5 | S'teddy bear' 6 | p3 7 | S'indoor-things' 8 | p4 9 | sS'kite' 10 | p5 11 | S'sports-things' 12 | p6 13 | sS'wall-other' 14 | p7 15 | S'wall-stuff' 16 | p8 17 | sS'laptop' 18 | p9 19 | S'electronic-things' 20 | p10 21 | sS'potted plant' 22 | p11 23 | S'furniture-things' 24 | p12 25 | sS'floor-stone' 26 | p13 27 | S'floor-stuff' 28 | p14 29 | sS'paper' 30 | p15 31 | S'rawmaterial-stuff' 32 | p16 33 | sS'motorcycle' 34 | p17 35 | S'vehicle-things' 36 | p18 37 | sS'oven' 38 | p19 39 | S'appliance-things' 40 | p20 41 | sS'keyboard' 42 | p21 43 | g10 44 | sS'fire hydrant' 45 | p22 46 | S'outdoor-things' 47 | p23 48 | sS'blender' 49 | p24 50 | g20 51 | sS'cup' 52 | p25 53 | S'kitchen-things' 54 | p26 55 | sS'tv' 56 | p27 57 | g10 58 | sS'cage' 59 | p28 60 | S'structural-stuff' 61 | p29 62 | sS'backpack' 63 | p30 64 | S'accessory-things' 65 | p31 66 | sS'bench' 67 | p32 68 | g23 69 | sS'platform' 70 | p33 71 | S'ground-stuff' 72 | p34 73 | sS'window' 74 | p35 75 | g12 76 | sS'wood' 77 | p36 78 | S'solid-stuff' 79 | p37 80 | sS'orange' 81 | p38 82 | S'food-things' 83 | p39 84 | sS'gravel' 85 | p40 86 | g34 87 | sS'hat' 88 | p41 89 | g31 90 | sS'skis' 91 | p42 92 | g6 93 | sS'skyscraper' 94 | p43 95 | S'building-stuff' 96 | p44 97 | sS'furniture-other' 98 | p45 99 | S'furniture-stuff' 100 | p46 101 | sS'plate' 102 | p47 103 | g26 104 | sS'wall-stone' 105 | p48 106 | g8 107 | sS'blanket' 108 | p49 109 | S'textile-stuff' 110 | p50 111 | sS'wall-panel' 112 | p51 113 | g8 114 | sS'bear' 115 | p52 116 | S'animal-things' 117 | p53 118 | sS'vase' 119 | p54 120 | g4 121 | sS'vegetable' 122 | p55 123 | S'food-stuff' 124 | p56 125 | sS'spoon' 126 | p57 127 | g26 128 | sS'giraffe' 129 | p58 130 | g53 131 | sS'desk' 132 | p59 133 | g12 134 | sS'towel' 135 | p60 136 | g50 137 | sS'railroad' 138 | p61 139 | g34 140 | sS'banner' 141 | p62 142 | g50 143 | sS'clouds' 144 | p63 145 | S'sky-stuff' 146 | p64 147 | sS'water-other' 148 | p65 149 | S'water-stuff' 150 | p66 151 | sS'wall-tile' 152 | p67 153 | g8 154 | sS'shelf' 155 | p68 156 | g46 157 | sS'baseball glove' 158 | p69 159 | g6 160 | sS'pavement' 161 | p70 162 | g34 163 | sS'rock' 164 | p71 165 | g37 166 | sS'desk-stuff' 167 | p72 168 | g46 169 | sS'river' 170 | p73 171 | g66 172 | sS'shoe' 173 | p74 174 | g31 175 | sS'bicycle' 176 | p75 177 | g18 178 | sS'wall-concrete' 179 | p76 180 | g8 181 | sS'fence' 182 | p77 183 | g29 184 | sS'house' 185 | p78 186 | g44 187 | sS'railing' 188 | p79 189 | g29 190 | sS'ground-other' 191 | p80 192 | g34 193 | sS'frisbee' 194 | p81 195 | g6 196 | sS'zebra' 197 | p82 198 | g53 199 | sS'sea' 200 | p83 201 | g66 202 | sS'mirror' 203 | p84 204 | g12 205 | sS'chair' 206 | p85 207 | g12 208 | sS'window-blind' 209 | p86 210 | S'window-stuff' 211 | p87 212 | sS'mat' 213 | p88 214 | g50 215 | sS'flower' 216 | p89 217 | S'plant-stuff' 218 | p90 219 | sS'clock' 220 | p91 221 | g4 222 | sS'sand' 223 | p92 224 | g34 225 | sS'microwave' 226 | p93 227 | g20 228 | sS'mud' 229 | p94 230 | g34 231 | sS'fog' 232 | p95 233 | g66 234 | sS'hill' 235 | p96 236 | g37 237 | sS'net' 238 | p97 239 | g29 240 | sS'carpet' 241 | p98 242 | g14 243 | sS'bird' 244 | p99 245 | g53 246 | sS'knife' 247 | p100 248 | g26 249 | sS'tent' 250 | p101 251 | g44 252 | sS'eye glasses' 253 | p102 254 | g31 255 | sS'playingfield' 256 | p103 257 | g34 258 | sS'umbrella' 259 | p104 260 | g31 261 | sS'dirt' 262 | p105 263 | g34 264 | sS'rug' 265 | p106 266 | g50 267 | sS'toothbrush' 268 | p107 269 | g4 270 | sS'sink' 271 | p108 272 | g20 273 | sS'floor-wood' 274 | p109 275 | g14 276 | sS'roof' 277 | p110 278 | g44 279 | sS'cardboard' 280 | p111 281 | g16 282 | sS'table' 283 | p112 284 | g46 285 | sS'stone' 286 | p113 287 | g37 288 | sS'wall-wood' 289 | p114 290 | g8 291 | sS'cow' 292 | p115 293 | g53 294 | sS'baseball bat' 295 | p116 296 | g6 297 | sS'leaves' 298 | p117 299 | g90 300 | sS'sports ball' 301 | p118 302 | g6 303 | sS'skateboard' 304 | p119 305 | g6 306 | sS'wall-brick' 307 | p120 308 | g8 309 | sS'plant-other' 310 | p121 311 | g90 312 | sS'clothes' 313 | p122 314 | g50 315 | sS'pillow' 316 | p123 317 | g50 318 | sS'road' 319 | p124 320 | g34 321 | sS'apple' 322 | p125 323 | g39 324 | sS'bridge' 325 | p126 326 | g44 327 | sS'straw' 328 | p127 329 | g90 330 | sS'stop sign' 331 | p128 332 | g23 333 | sS'tennis racket' 334 | p129 335 | g6 336 | sS'floor-marble' 337 | p130 338 | g14 339 | sS'snowboard' 340 | p131 341 | g6 342 | sS'scissors' 343 | p132 344 | g4 345 | sS'door-stuff' 346 | p133 347 | g46 348 | sS'boat' 349 | p134 350 | g18 351 | sS'sheep' 352 | p135 353 | g53 354 | sS'ceiling-other' 355 | p136 356 | S'ceiling-stuff' 357 | p137 358 | sS'horse' 359 | p138 360 | g53 361 | sS'textile-other' 362 | p139 363 | g50 364 | sS'traffic light' 365 | p140 366 | g23 367 | sS'structural-other' 368 | p141 369 | g29 370 | sS'bush' 371 | p142 372 | g90 373 | sS'plastic' 374 | p143 375 | g16 376 | sS'sky-other' 377 | p144 378 | g64 379 | sS'curtain' 380 | p145 381 | g50 382 | sS'ceiling-tile' 383 | p146 384 | g137 385 | sS'banana' 386 | p147 387 | g39 388 | sS'fork' 389 | p148 390 | g26 391 | sS'door' 392 | p149 393 | g12 394 | sS'bus' 395 | p150 396 | g18 397 | sS'cabinet' 398 | p151 399 | g46 400 | sS'cloth' 401 | p152 402 | g50 403 | sS'train' 404 | p153 405 | g18 406 | sS'stairs' 407 | p154 408 | g46 409 | sS'floor-other' 410 | p155 411 | g14 412 | sS'salad' 413 | p156 414 | g56 415 | sS'dining table' 416 | p157 417 | g12 418 | sS'car' 419 | p158 420 | g18 421 | sS'tree' 422 | p159 423 | g90 424 | sS'bed' 425 | p160 426 | g12 427 | sS'cat' 428 | p161 429 | g53 430 | sS'donut' 431 | p162 432 | g39 433 | sS'suitcase' 434 | p163 435 | g31 436 | sS'cake' 437 | p164 438 | g39 439 | sS'wine glass' 440 | p165 441 | g26 442 | sS'grass' 443 | p166 444 | g90 445 | sS'toilet' 446 | p167 447 | g12 448 | sS'napkin' 449 | p168 450 | g50 451 | sS'carrot' 452 | p169 453 | g39 454 | sS'airplane' 455 | p170 456 | g18 457 | sS'mouse' 458 | p171 459 | g10 460 | sS'waterdrops' 461 | p172 462 | g66 463 | sS'hair drier' 464 | p173 465 | g4 466 | sS'mountain' 467 | p174 468 | g37 469 | sS'floor-tile' 470 | p175 471 | g14 472 | sS'toaster' 473 | p176 474 | g20 475 | sS'bowl' 476 | p177 477 | g26 478 | sS'snow' 479 | p178 480 | g34 481 | sS'couch' 482 | p179 483 | g12 484 | sS'book' 485 | p180 486 | g4 487 | sS'branch' 488 | p181 489 | g90 490 | sS'elephant' 491 | p182 492 | g53 493 | sS'moss' 494 | p183 495 | g90 496 | sS'tie' 497 | p184 498 | g31 499 | sS'pizza' 500 | p185 501 | g39 502 | sS'sandwich' 503 | p186 504 | g39 505 | sS'cupboard' 506 | p187 507 | g46 508 | sS'parking meter' 509 | p188 510 | g23 511 | sS'food-other' 512 | p189 513 | g56 514 | sS'solid-other' 515 | p190 516 | g37 517 | sS'fruit' 518 | p191 519 | g56 520 | sS'hair brush' 521 | p192 522 | g4 523 | sS'handbag' 524 | p193 525 | g31 526 | sS'cell phone' 527 | p194 528 | g10 529 | sS'street sign' 530 | p195 531 | g23 532 | sS'broccoli' 533 | p196 534 | g39 535 | sS'refrigerator' 536 | p197 537 | g20 538 | sS'mirror-stuff' 539 | p198 540 | g46 541 | sS'window-other' 542 | p199 543 | g87 544 | sS'remote' 545 | p200 546 | g10 547 | sS'surfboard' 548 | p201 549 | g6 550 | sS'hot dog' 551 | p202 552 | g39 553 | sS'light' 554 | p203 555 | g46 556 | sS'counter' 557 | p204 558 | g46 559 | sS'metal' 560 | p205 561 | g16 562 | sS'dog' 563 | p206 564 | g53 565 | sS'person' 566 | p207 567 | S'person-things' 568 | p208 569 | sS'truck' 570 | p209 571 | g18 572 | sS'bottle' 573 | p210 574 | g26 575 | sS'building-other' 576 | p211 577 | g44 578 | ssS'fine_index_to_coarse_index' 579 | p212 580 | (dp213 581 | I0 582 | I9 583 | sI1 584 | I11 585 | sI2 586 | I11 587 | sI3 588 | I11 589 | sI4 590 | I11 591 | sI5 592 | I11 593 | sI6 594 | I11 595 | sI7 596 | I11 597 | sI8 598 | I11 599 | sI9 600 | I8 601 | sI10 602 | I8 603 | sI11 604 | I8 605 | sI12 606 | I8 607 | sI13 608 | I8 609 | sI14 610 | I8 611 | sI15 612 | I7 613 | sI16 614 | I7 615 | sI17 616 | I7 617 | sI18 618 | I7 619 | sI19 620 | I7 621 | sI20 622 | I7 623 | sI21 624 | I7 625 | sI22 626 | I7 627 | sI23 628 | I7 629 | sI24 630 | I7 631 | sI25 632 | I6 633 | sI26 634 | I6 635 | sI27 636 | I6 637 | sI28 638 | I6 639 | sI29 640 | I6 641 | sI30 642 | I6 643 | sI31 644 | I6 645 | sI32 646 | I6 647 | sI33 648 | I10 649 | sI34 650 | I10 651 | sI35 652 | I10 653 | sI36 654 | I10 655 | sI37 656 | I10 657 | sI38 658 | I10 659 | sI39 660 | I10 661 | sI40 662 | I10 663 | sI41 664 | I10 665 | sI42 666 | I10 667 | sI43 668 | I5 669 | sI44 670 | I5 671 | sI45 672 | I5 673 | sI46 674 | I5 675 | sI47 676 | I5 677 | sI48 678 | I5 679 | sI49 680 | I5 681 | sI50 682 | I5 683 | sI51 684 | I2 685 | sI52 686 | I2 687 | sI53 688 | I2 689 | sI54 690 | I2 691 | sI55 692 | I2 693 | sI56 694 | I2 695 | sI57 696 | I2 697 | sI58 698 | I2 699 | sI59 700 | I2 701 | sI60 702 | I2 703 | sI61 704 | I3 705 | sI62 706 | I3 707 | sI63 708 | I3 709 | sI64 710 | I3 711 | sI65 712 | I3 713 | sI66 714 | I3 715 | sI67 716 | I3 717 | sI68 718 | I3 719 | sI69 720 | I3 721 | sI70 722 | I3 723 | sI71 724 | I0 725 | sI72 726 | I0 727 | sI73 728 | I0 729 | sI74 730 | I0 731 | sI75 732 | I0 733 | sI76 734 | I0 735 | sI77 736 | I1 737 | sI78 738 | I1 739 | sI79 740 | I1 741 | sI80 742 | I1 743 | sI81 744 | I1 745 | sI82 746 | I1 747 | sI83 748 | I4 749 | sI84 750 | I4 751 | sI85 752 | I4 753 | sI86 754 | I4 755 | sI87 756 | I4 757 | sI88 758 | I4 759 | sI89 760 | I4 761 | sI90 762 | I4 763 | sI91 764 | I17 765 | sI92 766 | I17 767 | sI93 768 | I22 769 | sI94 770 | I20 771 | sI95 772 | I20 773 | sI96 774 | I22 775 | sI97 776 | I15 777 | sI98 778 | I25 779 | sI99 780 | I16 781 | sI100 782 | I13 783 | sI101 784 | I12 785 | sI102 786 | I12 787 | sI103 788 | I17 789 | sI104 790 | I17 791 | sI105 792 | I23 793 | sI106 794 | I15 795 | sI107 796 | I15 797 | sI108 798 | I17 799 | sI109 800 | I15 801 | sI110 802 | I21 803 | sI111 804 | I15 805 | sI112 806 | I25 807 | sI113 808 | I13 809 | sI114 810 | I13 811 | sI115 812 | I13 813 | sI116 814 | I13 815 | sI117 816 | I13 817 | sI118 818 | I22 819 | sI119 820 | I26 821 | sI120 822 | I14 823 | sI121 824 | I14 825 | sI122 826 | I15 827 | sI123 828 | I22 829 | sI124 830 | I21 831 | sI125 832 | I21 833 | sI126 834 | I24 835 | sI127 836 | I20 837 | sI128 838 | I22 839 | sI129 840 | I15 841 | sI130 842 | I17 843 | sI131 844 | I16 845 | sI132 846 | I15 847 | sI133 848 | I22 849 | sI134 850 | I24 851 | sI135 852 | I21 853 | sI136 854 | I17 855 | sI137 856 | I25 857 | sI138 858 | I16 859 | sI139 860 | I21 861 | sI140 862 | I17 863 | sI141 864 | I22 865 | sI142 866 | I16 867 | sI143 868 | I21 869 | sI144 870 | I21 871 | sI145 872 | I25 873 | sI146 874 | I21 875 | sI147 876 | I26 877 | sI148 878 | I21 879 | sI149 880 | I24 881 | sI150 882 | I20 883 | sI151 884 | I17 885 | sI152 886 | I14 887 | sI153 888 | I21 889 | sI154 890 | I26 891 | sI155 892 | I15 893 | sI156 894 | I23 895 | sI157 896 | I20 897 | sI158 898 | I21 899 | sI159 900 | I24 901 | sI160 902 | I15 903 | sI161 904 | I24 905 | sI162 906 | I22 907 | sI163 908 | I25 909 | sI164 910 | I15 911 | sI165 912 | I20 913 | sI166 914 | I17 915 | sI167 916 | I17 917 | sI168 918 | I22 919 | sI169 920 | I14 921 | sI170 922 | I18 923 | sI171 924 | I18 925 | sI172 926 | I18 927 | sI173 928 | I18 929 | sI174 930 | I18 931 | sI175 932 | I18 933 | sI176 934 | I18 935 | sI177 936 | I26 937 | sI178 938 | I26 939 | sI179 940 | I19 941 | sI180 942 | I19 943 | sI181 944 | I24 945 | ss. -------------------------------------------------------------------------------- /diffseg.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "Copyright 2023 Google LLC\n", 8 | "\n", 9 | "Use of this source code is governed by an MIT-style\n", 10 | "license that can be found in the LICENSE file or at\n", 11 | "https://opensource.org/licenses/MIT.\n", 12 | "\n", 13 | "# Instructions\n", 14 | "Please run the following cells sequentially\n", 15 | "1. (Optional) Running 1b adds semantic labels and requires addtional resources (default to a second GPU).\n", 16 | "* The function relies on an additonal image captioning model, e.g., BLIP. \n", 17 | "* The labels are nouns, extracted from the generated caption. \n", 18 | "* It merge masks sharing the same label. \n", 19 | "2. Add your own image and update ``image_path`` variable. \n", 20 | "3. Feel free to play with DiffSeg hyper-parameters such as the ``KL_THRESHOLD``." 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "id": "r52zlVDUJl_9" 27 | }, 28 | "source": [ 29 | "# Import" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": { 36 | "id": "3ViBbqfx9un_" 37 | }, 38 | "outputs": [], 39 | "source": [ 40 | "import tensorflow as tf\n", 41 | "from PIL import Image\n", 42 | "import nltk\n", 43 | "from transformers import AutoProcessor, TFBlipForConditionalGeneration\n", 44 | "from keras_cv.src.models.stable_diffusion.image_encoder import ImageEncoder\n", 45 | "from third_party.keras_cv.stable_diffusion import StableDiffusion \n", 46 | "from third_party.keras_cv.diffusion_model import SpatialTransformer\n", 47 | "from diffseg.utils import process_image, augmenter, vis_without_label, semantic_mask\n", 48 | "from diffseg.segmentor import DiffSeg\n", 49 | "\n", 50 | "is_noun = lambda pos: pos[:2] == 'NN'\n", 51 | "!nvidia-smi\n", 52 | "nltk.download('all')" 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "metadata": { 58 | "id": "KfCj5i1be890" 59 | }, 60 | "source": [ 61 | "# 1. Initialize SD Model" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": { 68 | "id": "I8b6beipfCA3" 69 | }, 70 | "outputs": [], 71 | "source": [ 72 | "# Inialize Stable Diffusion Model on GPU:0 \n", 73 | "with tf.device('/GPU:0'):\n", 74 | " image_encoder = ImageEncoder()\n", 75 | " vae=tf.keras.Model(\n", 76 | " image_encoder.input,\n", 77 | " image_encoder.layers[-1].output,\n", 78 | " )\n", 79 | " model = StableDiffusion(img_width=512, img_height=512)\n", 80 | "blip = None" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "# 1b. Initialize BLIP (optional)" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "# Optionally initalize a BLIP captioning model on GPU:1\n", 97 | "with tf.device('/GPU:1'):\n", 98 | " processor = AutoProcessor.from_pretrained(\"Salesforce/blip-image-captioning-base\")\n", 99 | " blip = TFBlipForConditionalGeneration.from_pretrained(\"Salesforce/blip-image-captioning-base\")" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "# 2. Run Inference on Real Images" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "# The first time running this cell will be slow\n", 116 | "# because the model needs to download and loads pre-trained weights.\n", 117 | "\n", 118 | "image_path = \"./images/img5.jpeg\" # Specify the path to your image\n", 119 | "\n", 120 | "if blip is not None:\n", 121 | " with tf.device('/GPU:1'):\n", 122 | " inputs = processor(images=Image.open(image_path), return_tensors=\"tf\")\n", 123 | " out = blip.generate(**inputs)\n", 124 | " prompt = processor.decode(out[0], skip_special_tokens=True)\n", 125 | " print(prompt)\n", 126 | "else:\n", 127 | " prompt = None\n", 128 | "\n", 129 | "with tf.device('/GPU:0'):\n", 130 | " images = process_image(image_path)\n", 131 | " images = augmenter(images)\n", 132 | " latent = vae(tf.expand_dims(images, axis=0), training=False)\n", 133 | " images, weight_64, weight_32, weight_16, weight_8, x_weights_64, x_weights_32, x_weights_16, x_weights_8 = model.text_to_image(\n", 134 | " prompt,\n", 135 | " batch_size=1,\n", 136 | " latent=latent,\n", 137 | " timestep=300\n", 138 | " )\n", 139 | " " 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "# 3. Generate Segementation Masks" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "KL_THRESHOLD = [0.9]*3 # KL_THRESHOLD controls the merging threshold\n", 156 | "NUM_POINTS = 16\n", 157 | "REFINEMENT = True\n", 158 | "\n", 159 | "\n", 160 | "with tf.device('/GPU:0'):\n", 161 | " segmentor = DiffSeg(KL_THRESHOLD, REFINEMENT, NUM_POINTS)\n", 162 | " pred = segmentor.segment(weight_64, weight_32, weight_16, weight_8) # b x 512 x 512\n", 163 | " if blip is not None:\n", 164 | " tokenized = nltk.word_tokenize(prompt)\n", 165 | " nouns = [(i,word) for i,(word, pos) in enumerate(nltk.pos_tag(tokenized)) if is_noun(pos)] \n", 166 | "\n", 167 | " for i in range(len(images)):\n", 168 | " if blip is not None:\n", 169 | " x_weight = segmentor.aggregate_x_weights([x_weights_64[i],x_weights_32[i], x_weights_16[i], x_weights_8[i]],weight_ratio=[1.0,1.0,1.0,1.0])\n", 170 | " label_to_mask = segmentor.get_semantics(pred[i], x_weight[i], nouns,voting=\"mean\")\n", 171 | " semantic_mask(images[i], pred[i], label_to_mask)\n", 172 | " vis_without_label(pred[i],images[i],num_class=len(set(pred[i].flatten())))" 173 | ] 174 | } 175 | ], 176 | "metadata": { 177 | "colab": { 178 | "collapsed_sections": [ 179 | "r52zlVDUJl_9", 180 | "8s6-2GsMiEwi", 181 | "YqhfoJBLPDkj", 182 | "HkDuWW_S6mFK", 183 | "Ir_E8vlvmC3i", 184 | "Jq_UYaw9z9W1", 185 | "AeVPOortS0Xg", 186 | "drmyt0Zq-Yuf", 187 | "F_2rE9z3lJxp", 188 | "gwq2tPWBoUCs", 189 | "ZuURD2pAgE6S", 190 | "MhoxFI32WrsZ", 191 | "KfCj5i1be890", 192 | "i9M17URAGrRT", 193 | "ZMg1NFyON8ve", 194 | "2uO8K_INONGB", 195 | "n4Mmj0o4OYWu", 196 | "25FBAbAWOjQT", 197 | "zdgg-Yirfj1s", 198 | "aiNXdN1UckMN", 199 | "XBW0qHSxlF0Z", 200 | "G4ktTGCsHjZU", 201 | "JTOmRatCdsap", 202 | "mc1OqXkMsik4", 203 | "EVTDEno2spnw" 204 | ], 205 | "last_runtime": { 206 | "build_target": "//experimental/humaninterface/explorations/diffseg/colab_runtime:ml_notebook", 207 | "kind": "private" 208 | }, 209 | "name": "diffusion_inference_coco_cityscapes.ipynb", 210 | "private_outputs": true, 211 | "provenance": [ 212 | { 213 | "file_id": "/piper/depot/google3/experimental/humaninterface/explorations/junjiaot/diffusion/diffusion_inference_coco.ipynb?workspaceId=junjiaot:scene::citc", 214 | "timestamp": 1689088568669 215 | }, 216 | { 217 | "file_id": "/piper/depot/google3/experimental/humaninterface/explorations/junjiaot/diffusion/diffusion_inference_coco.ipynb?workspaceId=junjiaot:scene::citc", 218 | "timestamp": 1688922575491 219 | }, 220 | { 221 | "file_id": "/piper/depot/google3/experimental/humaninterface/explorations/junjiaot/diffusion/diffusion_inference_coco.ipynb?workspaceId=junjiaot:scene::citc", 222 | "timestamp": 1688833060440 223 | }, 224 | { 225 | "file_id": "/piper/depot/google3/experimental/humaninterface/explorations/junjiaot/diffusion/diffusion_inference.ipynb?workspaceId=junjiaot:scene::citc", 226 | "timestamp": 1687892328479 227 | }, 228 | { 229 | "file_id": "/piper/depot/google3/experimental/humaninterface/explorations/junjiaot/diffusion/diffusion_inference.ipynb?workspaceId=junjiaot:scene::citc", 230 | "timestamp": 1687837396445 231 | }, 232 | { 233 | "file_id": "/piper/depot/google3/experimental/humaninterface/explorations/junjiaot/diffusion/diffusion_inference_test.ipynb?workspaceId=junjiaot:scene::citc", 234 | "timestamp": 1687364924074 235 | }, 236 | { 237 | "file_id": "/piper/depot/google3/experimental/humaninterface/explorations/junjiaot/diffusion/diffusion_inference_test.ipynb?workspaceId=junjiaot:scene::citc", 238 | "timestamp": 1687114629456 239 | }, 240 | { 241 | "file_id": "/piper/depot/google3/experimental/humaninterface/explorations/junjiaot/diffusion/diffusion_inference_test.ipynb?workspaceId=junjiaot:scene::citc", 242 | "timestamp": 1686761132459 243 | }, 244 | { 245 | "file_id": "/piper/depot/google3/experimental/humaninterface/explorations/junjiaot/diffusion/diffusion_test.ipynb?workspaceId=junjiaot:scene::citc", 246 | "timestamp": 1686163689880 247 | }, 248 | { 249 | "file_id": "1z1ERc9A5S1u5ci0dHLSoqDgPeXpUI_lY", 250 | "timestamp": 1686159970456 251 | } 252 | ], 253 | "toc_visible": true 254 | }, 255 | "kernelspec": { 256 | "display_name": "Python 3", 257 | "name": "python3" 258 | }, 259 | "language_info": { 260 | "codemirror_mode": { 261 | "name": "ipython", 262 | "version": 3 263 | }, 264 | "file_extension": ".py", 265 | "mimetype": "text/x-python", 266 | "name": "python", 267 | "nbconvert_exporter": "python", 268 | "pygments_lexer": "ipython3", 269 | "version": "3.9.18" 270 | } 271 | }, 272 | "nbformat": 4, 273 | "nbformat_minor": 0 274 | } 275 | -------------------------------------------------------------------------------- /diffseg/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2023 Google LLC 4 | 5 | Permission is hereby granted, free of charge, to any 6 | person obtaining a copy of this software and associated 7 | documentation files (the "Software"), to deal in the 8 | Software without restriction, including without 9 | limitation the rights to use, copy, modify, merge, 10 | publish, distribute, sublicense, and/or sell copies of 11 | the Software, and to permit persons to whom the Software 12 | is furnished to do so, subject to the following 13 | conditions: 14 | 15 | The above copyright notice and this permission notice 16 | shall be included in all copies or substantial portions 17 | of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 20 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 21 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 22 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 23 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 24 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 25 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 26 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 27 | DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /diffseg/segmentor.py: -------------------------------------------------------------------------------- 1 | #Copyright 2023 Google LLC 2 | 3 | #Use of this source code is governed by an MIT-style 4 | #license that can be found in the LICENSE file or at 5 | #https://opensource.org/licenses/MIT. 6 | 7 | 8 | import tensorflow as tf 9 | import copy 10 | from sklearn.cluster import KMeans 11 | import numpy as np 12 | from collections import defaultdict 13 | 14 | class DiffSeg: 15 | def __init__(self, kl_threshold, refine, num_points): 16 | # Generate the gird 17 | self.grid = self.generate_sampling_grid(num_points) 18 | # Inialize other parameters 19 | self.kl_threshold = np.array(kl_threshold) 20 | self.refine = refine 21 | 22 | def generate_sampling_grid(self,num_of_points): 23 | segment_len = 63//(num_of_points-1) 24 | total_len = segment_len*(num_of_points-1) 25 | start_point = (63 - total_len)//2 26 | x_new = np.linspace(start_point, total_len+start_point, num_of_points) 27 | y_new = np.linspace(start_point, total_len+start_point, num_of_points) 28 | x_new,y_new=np.meshgrid(x_new,y_new,indexing='ij') 29 | points = np.concatenate(([x_new.reshape(-1,1),y_new.reshape(-1,1)]),axis=-1).astype(int) 30 | return points 31 | 32 | def get_weight_rato(self, weight_list): 33 | # This function assigns proportional aggergation weight 34 | sizes = [] 35 | for weights in weight_list: 36 | sizes.append(np.sqrt(weights.shape[-2])) 37 | denom = np.sum(sizes) 38 | return sizes / denom 39 | 40 | def aggregate_weights(self, weight_list, weight_ratio=None): 41 | if weight_ratio is None: 42 | weight_ratio = self.get_weight_rato(weight_list) 43 | aggre_weights = np.zeros((64,64,64,64)) 44 | 45 | for index,weights in enumerate(weight_list): 46 | size = int(np.sqrt(weights.shape[-1])) 47 | ratio = int(64/size) 48 | # Average over the multi-head channel 49 | weights = weights.mean(0).reshape(-1,size,size) 50 | 51 | # Upsample the last two dimensions to 64 x 64 52 | weights = tf.keras.layers.UpSampling2D(size=(ratio, ratio), data_format="channels_last", interpolation='bilinear')(tf.expand_dims(weights,axis=-1)) 53 | weights = tf.reshape(weights,(size,size,64,64)) 54 | 55 | # Normalize to make sure each map sums to one 56 | weights = weights/tf.math.reduce_sum(weights,(2,3),keepdims=True) 57 | 58 | # Spatial tiling along the first two dimensions 59 | weights = tf.repeat(weights,repeats=ratio,axis=0) 60 | weights = tf.repeat(weights,repeats=ratio,axis=1) 61 | 62 | # Aggrgate accroding to weight_ratio 63 | aggre_weights += weights*weight_ratio[index] 64 | return aggre_weights.numpy().astype(np.double) 65 | 66 | def aggregate_x_weights(self, weight_list, weight_ratio=None): 67 | # x_weights: 8 x size**2 x 77 68 | # return 512 x 512 x 77 69 | if weight_ratio is None: 70 | weight_ratio = self.get_weight_rato(weight_list) 71 | aggre_weights = np.zeros((512, 512, 77)) 72 | 73 | for index,weights in enumerate(weight_list): 74 | size = int(np.sqrt(weights.shape[-2])) 75 | ratio = int(512/size) 76 | weights = weights.mean(0).reshape(1,size,size,-1) 77 | weights = tf.keras.layers.UpSampling2D(size=(ratio, ratio), data_format="channels_last", interpolation='bilinear')(weights) 78 | weights = weights/tf.math.reduce_sum(weights,axis=-1,keepdims=True) 79 | aggre_weights += weights*weight_ratio[index] 80 | return aggre_weights.numpy().astype(np.double) 81 | 82 | def KL(self,x,Y): 83 | qoutient = tf.math.log(x)-tf.math.log(Y) 84 | kl_1 = tf.math.reduce_sum(tf.math.multiply(x, qoutient),axis=(-2,-1))/2 85 | kl_2 = -tf.math.reduce_sum(tf.math.multiply(Y, qoutient),axis=(-2,-1))/2 86 | return tf.math.add(kl_1,kl_2) 87 | 88 | def mask_merge(self, iter, attns, kl_threshold, grid=None): 89 | if iter == 0: 90 | # The first iteration of merging 91 | anchors = attns[grid[:,0],grid[:,1],:,:] # 256 x 64 x 64 92 | anchors = tf.expand_dims(anchors, axis=(1)) # 256 x 1 x 64 x 64 93 | attns = attns.reshape(1,4096,64,64) 94 | # 256 x 4096 x 64 x 64 is too large for a single gpu, splitting into 16 portions 95 | split = np.sqrt(grid.shape[0]).astype(int) 96 | kl_bin=[] 97 | for i in range(split): 98 | temp = self.KL(tf.cast(anchors[i*split:(i+1)*split],tf.float16),tf.cast(attns,tf.float16)) < kl_threshold[iter] # type cast from tf.float64 to tf.float16 99 | kl_bin.append(temp) 100 | kl_bin = tf.cast(tf.concat(kl_bin, axis=0), tf.float64) # 256 x 4096 101 | new_attns = tf.reshape(tf.matmul(kl_bin,tf.reshape(attns,(-1,4096)))/tf.math.reduce_sum(kl_bin,1,keepdims=True),(-1,64,64)) # 256 x 64 x 64 102 | else: 103 | # The rest of merging iterations, reducing the number of masks 104 | matched = set() 105 | new_attns = [] 106 | for i,point in enumerate(attns): 107 | if i in matched: 108 | continue 109 | matched.add(i) 110 | anchor = point 111 | kl_bin = (self.KL(anchor,attns) < kl_threshold[iter]).numpy() # 64 x 64 112 | if kl_bin.sum() > 0: 113 | matched_idx = np.arange(len(attns))[kl_bin.reshape(-1)] 114 | for idx in matched_idx: matched.add(idx) 115 | aggregated_attn = attns[kl_bin].mean(0) 116 | new_attns.append(aggregated_attn.reshape(1,64,64)) 117 | return np.array(new_attns) 118 | 119 | def generate_masks(self, attns, kl_threshold, grid): 120 | # Iterative Attention Merging 121 | for i in range(len(kl_threshold)): 122 | if i == 0: 123 | attns_merged = self.mask_merge(i, attns, kl_threshold, grid=grid) 124 | else: 125 | attns_merged = self.mask_merge(i, attns_merged, kl_threshold) 126 | attns_merged = attns_merged[:,0,:,:] 127 | 128 | # Kmeans refinement (optional for better visual consistency) 129 | if self.refine: 130 | attns = attns.reshape(-1,64*64) 131 | kmeans = KMeans(n_clusters=attns_merged.shape[0], init=attns_merged.reshape(-1,64*64), n_init=1).fit(attns) 132 | clusters = kmeans.labels_ 133 | attns_merged = [] 134 | for i in range(len(set(clusters))): 135 | cluster = (i == clusters) 136 | attns_merged.append(attns[cluster,:].mean(0).reshape(64,64)) 137 | attns_merged = np.array(attns_merged) 138 | 139 | # Upsampling 140 | self.upsampled = tf.keras.layers.UpSampling2D(size=(8, 8), data_format="channels_last", interpolation='bilinear')(tf.expand_dims(attns_merged,axis=-1)) 141 | 142 | # Non-Maximum Suppression 143 | M_final = tf.reshape(tf.math.argmax(self.upsampled,axis=0),(512,512)).numpy() 144 | 145 | return M_final 146 | 147 | def segment(self, weight_64, weight_32, weight_16, weight_8, weight_ratio = None): 148 | M_list = [] 149 | for i in range(len(weight_64)): 150 | # Step 1: Attention Aggregation 151 | weights = self.aggregate_weights([weight_64[i],weight_32[i], weight_16[i], weight_8[i]],weight_ratio=weight_ratio) 152 | # Step 2 & 3: Iterative Merging & NMS 153 | M_final = self.generate_masks(weights, self.kl_threshold, self.grid) 154 | M_list.append(M_final) 155 | return np.array(M_list) 156 | 157 | def get_semantics(self, pred, x_weight, nouns, voting="majority"): 158 | # This function assigns semantic labels to masks 159 | indices = [item[0]+1 for item in nouns] # Igonore the first BOS token 160 | prompt_list = [item[1] for item in nouns] 161 | x_weight = x_weight[:,:,indices] # size x size x N 162 | x_weight = x_weight.reshape(512*512,-1) 163 | norm = np.linalg.norm(x_weight,axis=0,keepdims=True) 164 | x_weight = x_weight/norm # Normalize the cross-attention maps spatially 165 | pred = pred.reshape(512*512,-1) 166 | 167 | label_to_mask = defaultdict(list) 168 | for i in set(pred.flatten()): 169 | if voting == "majority": 170 | logits = x_weight[(pred==i).flatten(),:] 171 | index = logits.argmax(axis=-1) 172 | category = prompt_list[int(np.median(index))] 173 | else: 174 | logit = x_weight[(pred==i).flatten(),:].mean(0) 175 | category = prompt_list[logit.argmax(axis=-1)] 176 | label_to_mask[category].append(i) 177 | return label_to_mask 178 | -------------------------------------------------------------------------------- /diffseg/utils.py: -------------------------------------------------------------------------------- 1 | #Copyright 2023 Google LLC 2 | 3 | #Use of this source code is governed by an MIT-style 4 | #license that can be found in the LICENSE file or at 5 | #https://opensource.org/licenses/MIT. 6 | 7 | from tensorflow import keras 8 | import matplotlib.pyplot as plt 9 | import tensorflow as tf 10 | import PIL 11 | import numpy as np 12 | from scipy.optimize import linear_sum_assignment as LinearSumAssignment 13 | 14 | def process_image(image_path): 15 | with open(image_path, "rb") as f: 16 | image = np.array(PIL.Image.open(f)) 17 | s = tf.shape(image) 18 | w, h = s[0], s[1] 19 | c = tf.minimum(w, h) 20 | w_start = (w - c) // 2 21 | h_start = (h - c) // 2 22 | image = image[w_start : w_start + c, h_start : h_start + c, :] 23 | image = tf.image.resize(image, (512, 512)) 24 | return image 25 | 26 | augmenter = keras.Sequential( 27 | layers=[ 28 | tf.keras.layers.CenterCrop(512, 512), 29 | tf.keras.layers.Rescaling(scale=1.0 / 127.5, offset=-1), 30 | ] 31 | ) 32 | 33 | def find_edges(M): 34 | edges = np.zeros((512,512)) 35 | m1 = M[1:510,1:510] != M[0:509,1:510] 36 | m2 = M[1:510,1:510] != M[2:511,1:510] 37 | m3 = M[1:510,1:510] != M[1:510,0:509] 38 | m4 = M[1:510,1:510] != M[1:510,2:511] 39 | edges[1:510,1:510] = (m1 | m2 | m3 | m4).astype(int) 40 | x_new = np.linspace(0, 511, 512) 41 | y_new = np.linspace(0, 511, 512) 42 | x_new,y_new=np.meshgrid(x_new,y_new) 43 | x_new = x_new[edges==1] 44 | y_new = y_new[edges==1] 45 | return x_new,y_new 46 | 47 | def vis_without_label(M,image,index=None,save=False,dir=None,num_class=26): 48 | fig = plt.figure(figsize=(20, 20)) 49 | ax = plt.subplot(1, 3, 1) 50 | ax.imshow(image) 51 | ax.set_title("Input",fontdict={"fontsize":30}) 52 | plt.axis("off") 53 | 54 | x,y = find_edges(M) 55 | ax = plt.subplot(1, 3, 2) 56 | ax.imshow(image) 57 | ax.imshow(M,cmap='jet',alpha=0.5, vmin=-1, vmax=num_class) 58 | ax.scatter(x,y,color="blue", s=0.5) 59 | ax.set_title("Overlay",fontdict={"fontsize":30}) 60 | plt.axis("off") 61 | 62 | ax = plt.subplot(1, 3, 3) 63 | ax.imshow(M, cmap='jet',alpha=0.5, vmin=-1, vmax=num_class), 64 | ax.set_title("Segmentation",fontdict={"fontsize":30}) 65 | plt.axis("off") 66 | 67 | if save: 68 | fig.savefig(open(dir+"/example_{}.png".format(index), 'wb'), format='png',bbox_inches='tight', dpi=200) 69 | plt.close(fig) 70 | 71 | 72 | def semantic_mask(image, pred, label_to_mask): 73 | num_fig = len(label_to_mask) 74 | plt.figure(figsize=(20, 20)) 75 | for i,label in enumerate(label_to_mask.keys()): 76 | ax = plt.subplot(1, num_fig, i+1) 77 | image = image.reshape(512*512,-1) 78 | bin_mask = np.zeros_like(image) 79 | for mask in label_to_mask[label]: 80 | bin_mask[(pred.reshape(512*512)==mask).flatten(),:] = 1 81 | ax.imshow((image*bin_mask).reshape(512,512,-1)) 82 | ax.set_title(label,fontdict={"fontsize":30}) 83 | ax.axis("off") 84 | 85 | def _fast_hist(label_true, label_pred, n_class): 86 | # Adapted from https://github.com/janghyuncho/PiCIE/blob/c3aa029283eed7c156bbd23c237c151b19d6a4ad/utils.py#L99 87 | pred_n_class = np.maximum(n_class,label_pred.max()+1) 88 | mask = (label_true >= 0) & (label_true < n_class) # Exclude unlabelled data. 89 | hist = np.bincount(pred_n_class * label_true[mask] + label_pred[mask],\ 90 | minlength=n_class * pred_n_class).reshape(n_class, pred_n_class) 91 | return hist 92 | 93 | def hungarian_matching(pred,label,n_class): 94 | # X,Y: b x 512 x 512 95 | batch_size = pred.shape[0] 96 | tp = np.zeros(n_class) 97 | fp = np.zeros(n_class) 98 | fn = np.zeros(n_class) 99 | all = 0 100 | for i in range(batch_size): 101 | hist = _fast_hist(label[i].flatten(),pred[i].flatten(),n_class) 102 | row_ind, col_ind = LinearSumAssignment(hist,maximize=True) 103 | all += hist.sum() 104 | fn += (np.sum(hist, 1) - hist[row_ind,col_ind]) 105 | tp += hist[row_ind,col_ind] 106 | hist = hist[:, col_ind] # re-order hist to align labels to calculate FP 107 | fp += (np.sum(hist, 0) - np.diag(hist)) 108 | return tp,fp,fn,all -------------------------------------------------------------------------------- /docs/code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of 9 | experience, education, socio-economic status, nationality, personal appearance, 10 | race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or reject 41 | comments, commits, code, wiki edits, issues, and other contributions that are 42 | not aligned to this Code of Conduct, or to ban temporarily or permanently any 43 | contributor for other behaviors that they deem inappropriate, threatening, 44 | offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | This Code of Conduct also applies outside the project spaces when the Project 56 | Steward has a reasonable belief that an individual's behavior may have a 57 | negative impact on the project or its community. 58 | 59 | ## Conflict Resolution 60 | 61 | We do not believe that all conflict is bad; healthy debate and disagreement 62 | often yield positive results. However, it is never okay to be disrespectful or 63 | to engage in behavior that violates the project’s code of conduct. 64 | 65 | If you see someone violating the code of conduct, you are encouraged to address 66 | the behavior directly with those involved. Many issues can be resolved quickly 67 | and easily, and this gives people more control over the outcome of their 68 | dispute. If you are unable to resolve the matter for any reason, or if the 69 | behavior is threatening or harassing, report it. We are dedicated to providing 70 | an environment where participants feel welcome and safe. 71 | 72 | Reports should be directed to *[PROJECT STEWARD NAME(s) AND EMAIL(s)]*, the 73 | Project Steward(s) for *[PROJECT NAME]*. It is the Project Steward’s duty to 74 | receive and address reported violations of the code of conduct. They will then 75 | work with a committee consisting of representatives from the Open Source 76 | Programs Office and the Google Open Source Strategy team. If for any reason you 77 | are uncomfortable reaching out to the Project Steward, please email 78 | opensource@google.com. 79 | 80 | We will investigate every complaint, but you may not receive a direct response. 81 | We will use our discretion in determining when and how to follow up on reported 82 | incidents, which may range from not taking action to permanent expulsion from 83 | the project and project-sponsored spaces. We will notify the accused of the 84 | report and provide them an opportunity to discuss it before any action is taken. 85 | The identity of the reporter will be omitted from the details of the report 86 | supplied to the accused. In potentially harmful situations, such as ongoing 87 | harassment or threats to anyone's safety, we may take action without notice. 88 | 89 | ## Attribution 90 | 91 | This Code of Conduct is adapted from the Contributor Covenant, version 1.4, 92 | available at 93 | https://www.contributor-covenant.org/version/1/4/code-of-conduct/ 94 | -------------------------------------------------------------------------------- /docs/contributing.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We would love to accept your patches and contributions to this project. 4 | 5 | ## Before you begin 6 | 7 | ### Sign our Contributor License Agreement 8 | 9 | Contributions to this project must be accompanied by a 10 | [Contributor License Agreement](https://cla.developers.google.com/about) (CLA). 11 | You (or your employer) retain the copyright to your contribution; this simply 12 | gives us permission to use and redistribute your contributions as part of the 13 | project. 14 | 15 | If you or your current employer have already signed the Google CLA (even if it 16 | was for a different project), you probably don't need to do it again. 17 | 18 | Visit to see your current agreements or to 19 | sign a new one. 20 | 21 | ### Review our Community Guidelines 22 | 23 | This project follows [Google's Open Source Community 24 | Guidelines](https://opensource.google/conduct/). 25 | 26 | ## Contribution process 27 | 28 | ### Code Reviews 29 | 30 | All submissions, including submissions by project members, require review. We 31 | use [GitHub pull requests](https://docs.github.com/articles/about-pull-requests) 32 | for this purpose. 33 | -------------------------------------------------------------------------------- /images/LICENSE: -------------------------------------------------------------------------------- 1 | All images in this folder are under the Creative Commons Attribution (CC BY 2.0) license by Flickr. -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==1.4.0 2 | array-record==0.5.0 3 | asttokens==2.4.1 4 | astunparse==1.6.3 5 | cachetools==5.3.2 6 | certifi==2023.7.22 7 | charset-normalizer==3.3.1 8 | click==8.1.7 9 | contourpy==1.1.1 10 | cycler==0.12.1 11 | decorator==5.1.1 12 | dm-tree==0.1.8 13 | etils==1.5.2 14 | exceptiongroup==1.1.3 15 | executing==2.0.1 16 | filelock==3.13.0 17 | flatbuffers==23.5.26 18 | fonttools==4.43.1 19 | fsspec==2023.10.0 20 | gast==0.5.4 21 | google-auth==2.23.3 22 | google-auth-oauthlib==1.0.0 23 | google-pasta==0.2.0 24 | googleapis-common-protos==1.61.0 25 | grpcio==1.59.0 26 | h5py==3.10.0 27 | huggingface-hub==0.17.3 28 | idna==3.4 29 | importlib-metadata==6.8.0 30 | importlib-resources==6.1.0 31 | ipdb==0.13.13 32 | ipython==8.17.1 33 | jedi==0.19.1 34 | joblib==1.3.2 35 | keras==2.14.0 36 | keras-core==0.1.7 37 | keras-cv==0.6.4 38 | kiwisolver==1.4.5 39 | libclang==16.0.6 40 | Markdown==3.5 41 | markdown-it-py==3.0.0 42 | MarkupSafe==2.1.3 43 | matplotlib==3.8.0 44 | matplotlib-inline==0.1.6 45 | mdurl==0.1.2 46 | ml-dtypes==0.2.0 47 | namex==0.0.7 48 | nltk==3.8.1 49 | numpy==1.26.1 50 | nvidia-cublas-cu11==11.11.3.6 51 | nvidia-cuda-cupti-cu11==11.8.87 52 | nvidia-cuda-nvcc-cu11==11.8.89 53 | nvidia-cuda-runtime-cu11==11.8.89 54 | nvidia-cudnn-cu11==8.7.0.84 55 | nvidia-cufft-cu11==10.9.0.58 56 | nvidia-curand-cu11==10.3.0.86 57 | nvidia-cusolver-cu11==11.4.1.48 58 | nvidia-cusparse-cu11==11.7.5.86 59 | nvidia-nccl-cu11==2.16.5 60 | oauthlib==3.2.2 61 | opencv-python==4.8.1.78 62 | opt-einsum==3.3.0 63 | packaging==23.2 64 | parso==0.8.3 65 | pexpect==4.8.0 66 | Pillow==10.1.0 67 | promise==2.3 68 | prompt-toolkit==3.0.39 69 | protobuf==3.20.3 70 | psutil==5.9.6 71 | ptyprocess==0.7.0 72 | pure-eval==0.2.2 73 | pyasn1==0.5.0 74 | pyasn1-modules==0.3.0 75 | Pygments==2.16.1 76 | pyparsing==3.1.1 77 | python-dateutil==2.8.2 78 | PyYAML==6.0.1 79 | regex==2023.10.3 80 | requests==2.31.0 81 | requests-oauthlib==1.3.1 82 | rich==13.6.0 83 | rsa==4.9 84 | safetensors==0.4.0 85 | scikit-learn==1.3.2 86 | scipy==1.11.3 87 | six==1.16.0 88 | stack-data==0.6.3 89 | tensorboard==2.14.1 90 | tensorboard-data-server==0.7.2 91 | tensorflow==2.14.0 92 | tensorflow-datasets==4.9.3 93 | tensorflow-estimator==2.14.0 94 | tensorflow-io-gcs-filesystem==0.34.0 95 | tensorflow-metadata==1.14.0 96 | tensorrt==8.5.3.1 97 | termcolor==2.3.0 98 | threadpoolctl==3.2.0 99 | tokenizers==0.14.1 100 | toml==0.10.2 101 | tomli==2.0.1 102 | tqdm==4.66.1 103 | traitlets==5.13.0 104 | transformers==4.34.1 105 | typing_extensions==4.8.0 106 | urllib3==2.0.7 107 | wcwidth==0.2.8 108 | Werkzeug==3.0.1 109 | wrapt==1.14.1 110 | zipp==3.17.0 111 | -------------------------------------------------------------------------------- /third_party/keras_cv/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright © 2023 The KerasCV Authors 2 | All code in this repository excluding the code located in 3 | keras_cv/layers/preprocessing_3d/waymo is licensed under the Apache License, 4 | Version 2.0. The code appearing in the keras_cv/layers/preprocessing_3d/waymo 5 | folder is licensed under terms appearing below. 6 | 7 | Apache License 8 | Version 2.0, January 2004 9 | http://www.apache.org/licenses/ 10 | 11 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 12 | 13 | 1. Definitions. 14 | 15 | "License" shall mean the terms and conditions for use, reproduction, 16 | and distribution as defined by Sections 1 through 9 of this document. 17 | 18 | "Licensor" shall mean the copyright owner or entity authorized by 19 | the copyright owner that is granting the License. 20 | 21 | "Legal Entity" shall mean the union of the acting entity and all 22 | other entities that control, are controlled by, or are under common 23 | control with that entity. For the purposes of this definition, 24 | "control" means (i) the power, direct or indirect, to cause the 25 | direction or management of such entity, whether by contract or 26 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 27 | outstanding shares, or (iii) beneficial ownership of such entity. 28 | 29 | "You" (or "Your") shall mean an individual or Legal Entity 30 | exercising permissions granted by this License. 31 | 32 | "Source" form shall mean the preferred form for making modifications, 33 | including but not limited to software source code, documentation 34 | source, and configuration files. 35 | 36 | "Object" form shall mean any form resulting from mechanical 37 | transformation or translation of a Source form, including but 38 | not limited to compiled object code, generated documentation, 39 | and conversions to other media types. 40 | 41 | "Work" shall mean the work of authorship, whether in Source or 42 | Object form, made available under the License, as indicated by a 43 | copyright notice that is included in or attached to the work 44 | (an example is provided in the Appendix below). 45 | 46 | "Derivative Works" shall mean any work, whether in Source or Object 47 | form, that is based on (or derived from) the Work and for which the 48 | editorial revisions, annotations, elaborations, or other modifications 49 | represent, as a whole, an original work of authorship. For the purposes 50 | of this License, Derivative Works shall not include works that remain 51 | separable from, or merely link (or bind by name) to the interfaces of, 52 | the Work and Derivative Works thereof. 53 | 54 | "Contribution" shall mean any work of authorship, including 55 | the original version of the Work and any modifications or additions 56 | to that Work or Derivative Works thereof, that is intentionally 57 | submitted to Licensor for inclusion in the Work by the copyright owner 58 | or by an individual or Legal Entity authorized to submit on behalf of 59 | the copyright owner. For the purposes of this definition, "submitted" 60 | means any form of electronic, verbal, or written communication sent 61 | to the Licensor or its representatives, including but not limited to 62 | communication on electronic mailing lists, source code control systems, 63 | and issue tracking systems that are managed by, or on behalf of, the 64 | Licensor for the purpose of discussing and improving the Work, but 65 | excluding communication that is conspicuously marked or otherwise 66 | designated in writing by the copyright owner as "Not a Contribution." 67 | 68 | "Contributor" shall mean Licensor and any individual or Legal Entity 69 | on behalf of whom a Contribution has been received by Licensor and 70 | subsequently incorporated within the Work. 71 | 72 | 2. Grant of Copyright License. Subject to the terms and conditions of 73 | this License, each Contributor hereby grants to You a perpetual, 74 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 75 | copyright license to reproduce, prepare Derivative Works of, 76 | publicly display, publicly perform, sublicense, and distribute the 77 | Work and such Derivative Works in Source or Object form. 78 | 79 | 3. Grant of Patent License. Subject to the terms and conditions of 80 | this License, each Contributor hereby grants to You a perpetual, 81 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 82 | (except as stated in this section) patent license to make, have made, 83 | use, offer to sell, sell, import, and otherwise transfer the Work, 84 | where such license applies only to those patent claims licensable 85 | by such Contributor that are necessarily infringed by their 86 | Contribution(s) alone or by combination of their Contribution(s) 87 | with the Work to which such Contribution(s) was submitted. If You 88 | institute patent litigation against any entity (including a 89 | cross-claim or counterclaim in a lawsuit) alleging that the Work 90 | or a Contribution incorporated within the Work constitutes direct 91 | or contributory patent infringement, then any patent licenses 92 | granted to You under this License for that Work shall terminate 93 | as of the date such litigation is filed. 94 | 95 | 4. Redistribution. You may reproduce and distribute copies of the 96 | Work or Derivative Works thereof in any medium, with or without 97 | modifications, and in Source or Object form, provided that You 98 | meet the following conditions: 99 | 100 | (a) You must give any other recipients of the Work or 101 | Derivative Works a copy of this License; and 102 | 103 | (b) You must cause any modified files to carry prominent notices 104 | stating that You changed the files; and 105 | 106 | (c) You must retain, in the Source form of any Derivative Works 107 | that You distribute, all copyright, patent, trademark, and 108 | attribution notices from the Source form of the Work, 109 | excluding those notices that do not pertain to any part of 110 | the Derivative Works; and 111 | 112 | (d) If the Work includes a "NOTICE" text file as part of its 113 | distribution, then any Derivative Works that You distribute must 114 | include a readable copy of the attribution notices contained 115 | within such NOTICE file, excluding those notices that do not 116 | pertain to any part of the Derivative Works, in at least one 117 | of the following places: within a NOTICE text file distributed 118 | as part of the Derivative Works; within the Source form or 119 | documentation, if provided along with the Derivative Works; or, 120 | within a display generated by the Derivative Works, if and 121 | wherever such third-party notices normally appear. The contents 122 | of the NOTICE file are for informational purposes only and 123 | do not modify the License. You may add Your own attribution 124 | notices within Derivative Works that You distribute, alongside 125 | or as an addendum to the NOTICE text from the Work, provided 126 | that such additional attribution notices cannot be construed 127 | as modifying the License. 128 | 129 | You may add Your own copyright statement to Your modifications and 130 | may provide additional or different license terms and conditions 131 | for use, reproduction, or distribution of Your modifications, or 132 | for any such Derivative Works as a whole, provided Your use, 133 | reproduction, and distribution of the Work otherwise complies with 134 | the conditions stated in this License. 135 | 136 | 5. Submission of Contributions. Unless You explicitly state otherwise, 137 | any Contribution intentionally submitted for inclusion in the Work 138 | by You to the Licensor shall be under the terms and conditions of 139 | this License, without any additional terms or conditions. 140 | Notwithstanding the above, nothing herein shall supersede or modify 141 | the terms of any separate license agreement you may have executed 142 | with Licensor regarding such Contributions. 143 | 144 | 6. Trademarks. This License does not grant permission to use the trade 145 | names, trademarks, service marks, or product names of the Licensor, 146 | except as required for reasonable and customary use in describing the 147 | origin of the Work and reproducing the content of the NOTICE file. 148 | 149 | 7. Disclaimer of Warranty. Unless required by applicable law or 150 | agreed to in writing, Licensor provides the Work (and each 151 | Contributor provides its Contributions) on an "AS IS" BASIS, 152 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 153 | implied, including, without limitation, any warranties or conditions 154 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 155 | PARTICULAR PURPOSE. You are solely responsible for determining the 156 | appropriateness of using or redistributing the Work and assume any 157 | risks associated with Your exercise of permissions under this License. 158 | 159 | 8. Limitation of Liability. In no event and under no legal theory, 160 | whether in tort (including negligence), contract, or otherwise, 161 | unless required by applicable law (such as deliberate and grossly 162 | negligent acts) or agreed to in writing, shall any Contributor be 163 | liable to You for damages, including any direct, indirect, special, 164 | incidental, or consequential damages of any character arising as a 165 | result of this License or out of the use or inability to use the 166 | Work (including but not limited to damages for loss of goodwill, 167 | work stoppage, computer failure or malfunction, or any and all 168 | other commercial damages or losses), even if such Contributor 169 | has been advised of the possibility of such damages. 170 | 171 | 9. Accepting Warranty or Additional Liability. While redistributing 172 | the Work or Derivative Works thereof, You may choose to offer, 173 | and charge a fee for, acceptance of support, warranty, indemnity, 174 | or other liability obligations and/or rights consistent with this 175 | License. However, in accepting such obligations, You may act only 176 | on Your own behalf and on Your sole responsibility, not on behalf 177 | of any other Contributor, and only if You agree to indemnify, 178 | defend, and hold each Contributor harmless for any liability 179 | incurred by, or claims asserted against, such Contributor by reason 180 | of your accepting any such warranty or additional liability. 181 | 182 | END OF TERMS AND CONDITIONS 183 | 184 | APPENDIX: How to apply the Apache License to your work. 185 | 186 | To apply the Apache License to your work, attach the following 187 | boilerplate notice, with the fields enclosed by brackets "[]" 188 | replaced with your own identifying information. (Don't include 189 | the brackets!) The text should be enclosed in the appropriate 190 | comment syntax for the file format. We also recommend that a 191 | file or class name and description of purpose be included on the 192 | same "printed page" as the copyright notice for easier 193 | identification within third-party archives. 194 | 195 | Copyright 2023 The KerasCV Authors 196 | 197 | Licensed under the Apache License, Version 2.0 (the "License"); 198 | you may not use this file except in compliance with the License. 199 | You may obtain a copy of the License at 200 | 201 | http://www.apache.org/licenses/LICENSE-2.0 202 | 203 | Unless required by applicable law or agreed to in writing, software 204 | distributed under the License is distributed on an "AS IS" BASIS, 205 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 206 | See the License for the specific language governing permissions and 207 | limitations under the License. 208 | 209 | # The following applies only to the code appearing in 210 | # keras_cv/layers/preprocessing_3d/waymo 211 | 212 | License: https://github.com/keras-team/keras-cv/blob/master/keras_cv/layers/preprocessing_3d/waymo/LICENSE 213 | -------------------------------------------------------------------------------- /third_party/keras_cv/diffusion_model.py: -------------------------------------------------------------------------------- 1 | 2 | # Copyright 2022 The KerasCV Authors 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # The following code is modifed from 18 | # https://github.com/keras-team/keras-cv/blob/master/keras_cv/models/stable_diffusion/diffusion_model.py 19 | 20 | from keras_cv.src.models.stable_diffusion.padded_conv2d import ( 21 | PaddedConv2D, 22 | ) 23 | import tensorflow as tf 24 | from tensorflow import keras 25 | 26 | class DiffusionModel(keras.Model): 27 | 28 | def __init__( 29 | self, 30 | img_height, 31 | img_width, 32 | max_text_length, 33 | name=None, 34 | download_weights=True, 35 | ): 36 | context = keras.layers.Input((max_text_length, 768)) 37 | t_embed_input = keras.layers.Input((320,)) 38 | latent = keras.layers.Input((img_height // 8, img_width // 8, 4)) 39 | 40 | t_emb = keras.layers.Dense(1280)(t_embed_input) 41 | t_emb = keras.layers.Activation("swish")(t_emb) 42 | t_emb = keras.layers.Dense(1280)(t_emb) 43 | 44 | # Downsampling flow 45 | outputs = [] 46 | x = PaddedConv2D(320, kernel_size=3, padding=1)(latent) 47 | outputs.append(x) 48 | 49 | weight_64 = 0.0 50 | x_weights_64 = 0.0 51 | for _ in range(2): 52 | x = ResBlock(320)([x, t_emb]) 53 | x,temp, x_temp = SpatialTransformer(8, 40, fully_connected=False)([x, context]) 54 | weight_64 = tf.math.add(weight_64, temp/5) 55 | x_weights_64 = tf.math.add(x_weights_64, x_temp/5) 56 | outputs.append(x) 57 | x = PaddedConv2D(320, 3, strides=2, padding=1)(x) # Downsample 2x 58 | outputs.append(x) 59 | 60 | weight_32 = 0.0 61 | x_weights_32 = 0.0 62 | for _ in range(2): 63 | x = ResBlock(640)([x, t_emb]) 64 | x,temp, x_temp = SpatialTransformer(8, 80, fully_connected=False)([x, context]) 65 | weight_32 = tf.math.add(weight_32, temp/5) 66 | x_weights_32 = tf.math.add(x_weights_32, x_temp/5) 67 | outputs.append(x) 68 | x = PaddedConv2D(640, 3, strides=2, padding=1)(x) # Downsample 2x 69 | outputs.append(x) 70 | 71 | weight_16 = 0.0 72 | x_weights_16 = 0.0 73 | for _ in range(2): 74 | x = ResBlock(1280)([x, t_emb]) 75 | x,temp, x_temp = SpatialTransformer(8, 160, fully_connected=False)([x, context]) 76 | weight_16 = tf.math.add(weight_16, temp/5) 77 | x_weights_16 = tf.math.add(x_weights_16, x_temp/5) 78 | outputs.append(x) 79 | x = PaddedConv2D(1280, 3, strides=2, padding=1)(x) # Downsample 2x 80 | outputs.append(x) 81 | 82 | for _ in range(2): 83 | x = ResBlock(1280)([x, t_emb]) 84 | outputs.append(x) 85 | 86 | # Middle flow 87 | 88 | x = ResBlock(1280)([x, t_emb]) 89 | x, weight_8, x_weights_8 = SpatialTransformer(8, 160, fully_connected=False)([x, context]) 90 | x = ResBlock(1280)([x, t_emb]) 91 | 92 | # Upsampling flow 93 | 94 | for _ in range(3): 95 | x = keras.layers.Concatenate()([x, outputs.pop()]) 96 | x = ResBlock(1280)([x, t_emb]) 97 | x = Upsample(1280)(x) 98 | 99 | for _ in range(3): 100 | x = keras.layers.Concatenate()([x, outputs.pop()]) 101 | x = ResBlock(1280)([x, t_emb]) 102 | x, temp, x_temp = SpatialTransformer(8, 160, fully_connected=False)([x, context]) 103 | weight_16 = tf.math.add(weight_16, temp/5) 104 | x_weights_16 = tf.math.add(x_weights_16, x_temp/5) 105 | x = Upsample(1280)(x) 106 | 107 | for _ in range(3): 108 | x = keras.layers.Concatenate()([x, outputs.pop()]) 109 | x = ResBlock(640)([x, t_emb]) 110 | x, temp, x_temp = SpatialTransformer(8, 80, fully_connected=False)([x, context]) 111 | weight_32 = tf.math.add(weight_32, temp/5) 112 | x_weights_32 = tf.math.add(x_weights_32, x_temp/5) 113 | x = Upsample(640)(x) 114 | 115 | for _ in range(3): 116 | x = keras.layers.Concatenate()([x, outputs.pop()]) 117 | x = ResBlock(320)([x, t_emb]) 118 | x, temp, x_temp = SpatialTransformer(8, 40, fully_connected=False)([x, context]) 119 | weight_64 = tf.math.add(weight_64, temp/5) 120 | x_weights_64 = tf.math.add(x_weights_64, x_temp/3) 121 | 122 | # Exit flow 123 | 124 | x = keras.layers.GroupNormalization(epsilon=1e-5)(x) 125 | x = keras.layers.Activation("swish")(x) 126 | output = PaddedConv2D(4, kernel_size=3, padding=1)(x) 127 | 128 | outputs = [output, weight_64, weight_32, weight_16, weight_8, x_weights_64, x_weights_32, x_weights_16, x_weights_8 129 | ] 130 | super().__init__([latent, t_embed_input, context], outputs, name=name) 131 | 132 | if download_weights: 133 | diffusion_model_weights_fpath = keras.utils.get_file( 134 | origin="https://huggingface.co/fchollet/stable-diffusion/resolve/main/kcv_diffusion_model.h5", # noqa: E501 135 | file_hash=( # noqa: E501 136 | "8799ff9763de13d7f30a683d653018e114ed24a6a819667da4f5ee10f9e805fe" 137 | ), 138 | ) 139 | self.load_weights(diffusion_model_weights_fpath) 140 | 141 | class ResBlock(keras.layers.Layer): 142 | 143 | def __init__(self, output_dim, **kwargs): 144 | super().__init__(**kwargs) 145 | self.output_dim = output_dim 146 | self.entry_flow = [ 147 | keras.layers.GroupNormalization(epsilon=1e-5), 148 | keras.layers.Activation("swish"), 149 | PaddedConv2D(output_dim, 3, padding=1), 150 | ] 151 | self.embedding_flow = [ 152 | keras.layers.Activation("swish"), 153 | keras.layers.Dense(output_dim), 154 | ] 155 | self.exit_flow = [ 156 | keras.layers.GroupNormalization(epsilon=1e-5), 157 | keras.layers.Activation("swish"), 158 | PaddedConv2D(output_dim, 3, padding=1), 159 | ] 160 | 161 | def build(self, input_shape): 162 | if input_shape[0][-1] != self.output_dim: 163 | self.residual_projection = PaddedConv2D(self.output_dim, 1) 164 | else: 165 | self.residual_projection = lambda x: x 166 | 167 | def call(self, inputs): 168 | inputs, embeddings = inputs 169 | x = inputs 170 | for layer in self.entry_flow: 171 | x = layer(x) 172 | for layer in self.embedding_flow: 173 | embeddings = layer(embeddings) 174 | x = x + embeddings[:, None, None] 175 | for layer in self.exit_flow: 176 | x = layer(x) 177 | return x + self.residual_projection(inputs) 178 | 179 | class SpatialTransformer(keras.layers.Layer): 180 | 181 | def __init__(self, num_heads, head_size, fully_connected=False, **kwargs): 182 | super().__init__(**kwargs) 183 | self.norm = keras.layers.GroupNormalization(epsilon=1e-5) 184 | channels = num_heads * head_size 185 | if fully_connected: 186 | self.proj1 = keras.layers.Dense(num_heads * head_size) 187 | else: 188 | self.proj1 = PaddedConv2D(num_heads * head_size, 1) 189 | self.transformer_block = BasicTransformerBlock( 190 | channels, num_heads, head_size 191 | ) 192 | if fully_connected: 193 | self.proj2 = keras.layers.Dense(channels) 194 | else: 195 | self.proj2 = PaddedConv2D(channels, 1) 196 | 197 | def call(self, inputs): 198 | inputs, context = inputs 199 | _, h, w, c = inputs.shape 200 | x = self.norm(inputs) 201 | x = self.proj1(x) 202 | x = tf.reshape(x, (-1, h * w, c)) 203 | x,self_weights,cross_weights = self.transformer_block([x, context]) 204 | x = tf.reshape(x, (-1, h, w, c)) 205 | return self.proj2(x) + inputs, self_weights, cross_weights 206 | 207 | 208 | class BasicTransformerBlock(keras.layers.Layer): 209 | 210 | def __init__(self, dim, num_heads, head_size, **kwargs): 211 | super().__init__(**kwargs) 212 | self.norm1 = keras.layers.LayerNormalization(epsilon=1e-5) 213 | self.attn1 = CrossAttention(num_heads, head_size) 214 | self.norm2 = keras.layers.LayerNormalization(epsilon=1e-5) 215 | self.attn2 = CrossAttention(num_heads, head_size) 216 | self.norm3 = keras.layers.LayerNormalization(epsilon=1e-5) 217 | self.geglu = GEGLU(dim * 4) 218 | self.dense = keras.layers.Dense(dim) 219 | 220 | def call(self, inputs): 221 | inputs, context = inputs 222 | x, self_weights = self.attn1([self.norm1(inputs), None]) 223 | x = x + inputs 224 | x_temp, cross_weights = self.attn2([self.norm2(x), context]) 225 | x = x_temp + x 226 | return self.dense(self.geglu(self.norm3(x))) + x, self_weights, cross_weights 227 | 228 | class CrossAttention(keras.layers.Layer): 229 | 230 | def __init__(self, num_heads, head_size, **kwargs): 231 | super().__init__(**kwargs) 232 | self.to_q = keras.layers.Dense(num_heads * head_size, use_bias=False) 233 | self.to_k = keras.layers.Dense(num_heads * head_size, use_bias=False) 234 | self.to_v = keras.layers.Dense(num_heads * head_size, use_bias=False) 235 | self.scale = head_size**-0.5 236 | self.num_heads = num_heads 237 | self.head_size = head_size 238 | self.out_proj = keras.layers.Dense(num_heads * head_size) 239 | 240 | def call(self, inputs): 241 | inputs, context = inputs 242 | context = inputs if context is None else context 243 | q, k, v = self.to_q(inputs), self.to_k(context), self.to_v(context) 244 | q = tf.reshape(q, (-1, inputs.shape[1], self.num_heads, self.head_size)) 245 | k = tf.reshape(k, (-1, context.shape[1], self.num_heads, self.head_size)) 246 | v = tf.reshape(v, (-1, context.shape[1], self.num_heads, self.head_size)) 247 | q = tf.transpose(q, (0, 2, 1, 3)) # (bs, num_heads, time, head_size) 248 | k = tf.transpose(k, (0, 2, 3, 1)) # (bs, num_heads, head_size, time) 249 | v = tf.transpose(v, (0, 2, 1, 3)) # (bs, num_heads, time, head_size) 250 | 251 | score = td_dot(q, k) * self.scale 252 | weights = keras.activations.softmax(score) # (bs, num_heads, time, time) 253 | attn = td_dot(weights, v) 254 | attn = tf.transpose(attn, (0, 2, 1, 3)) # (bs, time, num_heads, head_size) 255 | out = tf.reshape( 256 | attn, (-1, inputs.shape[1], self.num_heads * self.head_size) 257 | ) 258 | return self.out_proj(out), weights 259 | 260 | 261 | class Upsample(keras.layers.Layer): 262 | 263 | def __init__(self, channels, **kwargs): 264 | super().__init__(**kwargs) 265 | self.ups = keras.layers.UpSampling2D(2) 266 | self.conv = PaddedConv2D(channels, 3, padding=1) 267 | 268 | def call(self, inputs): 269 | return self.conv(self.ups(inputs)) 270 | 271 | 272 | class GEGLU(keras.layers.Layer): 273 | 274 | def __init__(self, output_dim, **kwargs): 275 | super().__init__(**kwargs) 276 | self.output_dim = output_dim 277 | self.dense = keras.layers.Dense(output_dim * 2) 278 | 279 | def call(self, inputs): 280 | x = self.dense(inputs) 281 | x, gate = x[..., : self.output_dim], x[..., self.output_dim :] 282 | tanh_res = keras.activations.tanh( 283 | gate * 0.7978845608 * (1 + 0.044715 * (gate**2)) 284 | ) 285 | return x * 0.5 * gate * (1 + tanh_res) 286 | 287 | 288 | def td_dot(a, b): 289 | aa = tf.reshape(a, (-1, a.shape[2], a.shape[3])) 290 | bb = tf.reshape(b, (-1, b.shape[2], b.shape[3])) 291 | cc = keras.backend.batch_dot(aa, bb) 292 | return tf.reshape(cc, (-1, a.shape[1], cc.shape[1], cc.shape[2])) 293 | -------------------------------------------------------------------------------- /third_party/keras_cv/stable_diffusion.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The KerasCV Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | # The following code is modifed from 17 | # https://github.com/keras-team/keras-cv/blob/master/keras_cv/models/stable_diffusion/stable_diffusion.py 18 | 19 | """Keras implementation of StableDiffusion. 20 | 21 | Credits: 22 | 23 | - Original implementation: 24 | https://github.com/CompVis/stable-diffusion 25 | - Initial TF/Keras port: 26 | https://github.com/divamgupta/stable-diffusion-tensorflow 27 | 28 | The current implementation is a rewrite of the initial TF/Keras port by 29 | Divam Gupta. 30 | """ 31 | 32 | import math 33 | 34 | from keras_cv.src.models.stable_diffusion.clip_tokenizer import SimpleTokenizer 35 | from keras_cv.src.models.stable_diffusion.constants import _ALPHAS_CUMPROD 36 | from keras_cv.src.models.stable_diffusion.constants import _UNCONDITIONAL_TOKENS 37 | from keras_cv.src.models.stable_diffusion.decoder import Decoder 38 | from .diffusion_model import DiffusionModel 39 | from keras_cv.src.models.stable_diffusion.image_encoder import ImageEncoder 40 | from keras_cv.src.models.stable_diffusion.text_encoder import TextEncoder 41 | from keras_cv.src.models.stable_diffusion.text_encoder import TextEncoderV2 42 | import numpy as np 43 | import tensorflow as tf 44 | from tensorflow import keras 45 | 46 | MAX_PROMPT_LENGTH = 77 47 | 48 | class StableDiffusionBase: 49 | """Base class for stable diffusion and stable diffusion v2 model.""" 50 | 51 | def __init__( 52 | self, 53 | img_height=512, 54 | img_width=512, 55 | jit_compile=False, 56 | ): 57 | # UNet requires multiples of 2**7 = 128 58 | img_height = round(img_height / 128) * 128 59 | img_width = round(img_width / 128) * 128 60 | self.img_height = img_height 61 | self.img_width = img_width 62 | 63 | # lazy initialize the component models and the tokenizer 64 | self._image_encoder = None 65 | self._text_encoder = None 66 | self._diffusion_model = None 67 | self._decoder = None 68 | self._tokenizer = None 69 | 70 | self.jit_compile = jit_compile 71 | 72 | def text_to_image( 73 | self, 74 | prompt, 75 | negative_prompt=None, 76 | batch_size=1, 77 | num_steps=50, 78 | unconditional_guidance_scale=7.5, 79 | seed=None, 80 | latent=None, 81 | timestep=None, 82 | ): 83 | if prompt is not None: 84 | encoded_text = self.encode_text(prompt) 85 | else: 86 | encoded_text = None 87 | 88 | return self.generate_image( 89 | encoded_text, 90 | negative_prompt=negative_prompt, 91 | batch_size=batch_size, 92 | num_steps=num_steps, 93 | unconditional_guidance_scale=unconditional_guidance_scale, 94 | seed=seed, 95 | latent=latent, 96 | timestep=timestep, 97 | ) 98 | 99 | def encode_text(self, prompt): 100 | """Encodes a prompt into a latent text encoding. 101 | 102 | The encoding produced by this method should be used as the 103 | `encoded_text` parameter of `StableDiffusion.generate_image`. Encoding 104 | text separately from generating an image can be used to arbitrarily 105 | modify the text encoding prior to image generation, e.g. for walking 106 | between two prompts. 107 | 108 | Args: 109 | prompt: a string to encode, must be 77 tokens or shorter. 110 | 111 | Example: 112 | 113 | ```python 114 | from keras_cv.models import StableDiffusion 115 | 116 | model = StableDiffusion(img_height=512, img_width=512, jit_compile=True) 117 | encoded_text = model.encode_text("Tacos at dawn") 118 | img = model.generate_image(encoded_text) 119 | ``` 120 | """ 121 | # Tokenize prompt (i.e. starting context) 122 | inputs = self.tokenizer.encode(prompt) 123 | if len(inputs) > MAX_PROMPT_LENGTH: 124 | raise ValueError( 125 | f"Prompt is too long (should be <= {MAX_PROMPT_LENGTH} tokens)" 126 | ) 127 | phrase = inputs + [49407] * (MAX_PROMPT_LENGTH - len(inputs)) 128 | phrase = tf.convert_to_tensor([phrase], dtype=tf.int32) 129 | 130 | context = self.text_encoder.predict_on_batch([phrase, self._get_pos_ids()]) 131 | 132 | return context 133 | 134 | def generate_image( 135 | self, 136 | encoded_text, 137 | negative_prompt=None, 138 | batch_size=1, 139 | num_steps=50, 140 | unconditional_guidance_scale=7.5, 141 | diffusion_noise=None, 142 | seed=None, 143 | latent=None, 144 | timestep=None, 145 | ): 146 | """Generates an image based on encoded text. 147 | 148 | The encoding passed to this method should be derived from 149 | `StableDiffusion.encode_text`. 150 | 151 | Args: 152 | encoded_text: Tensor of shape (`batch_size`, 77, 768), or a Tensor of 153 | shape (77, 768). When the batch axis is omitted, the same encoded text 154 | will be used to produce every generated image. 155 | batch_size: int, number of images to generate, defaults to 1. 156 | negative_prompt: a string containing information to negatively guide the 157 | image generation (e.g. by removing or altering certain aspects of the 158 | generated image), defaults to None. 159 | num_steps: int, number of diffusion steps (controls image quality), 160 | defaults to 50. 161 | unconditional_guidance_scale: float, controlling how closely the image 162 | should adhere to the prompt. Larger values result in more closely 163 | adhering to the prompt, but will make the image noisier. Defaults to 164 | 7.5. 165 | diffusion_noise: Tensor of shape (`batch_size`, img_height // 8, 166 | img_width // 8, 4), or a Tensor of shape (img_height // 8, img_width 167 | // 8, 4). Optional custom noise to seed the diffusion process. When 168 | the batch axis is omitted, the same noise will be used to seed 169 | diffusion for every generated image. 170 | seed: integer which is used to seed the random generation of diffusion 171 | noise, only to be specified if `diffusion_noise` is None. 172 | 173 | Example: 174 | 175 | ```python 176 | from keras_cv.models import StableDiffusion 177 | 178 | batch_size = 8 179 | model = StableDiffusion(img_height=512, img_width=512, jit_compile=True) 180 | e_tacos = model.encode_text("Tacos at dawn") 181 | e_watermelons = model.encode_text("Watermelons at dusk") 182 | 183 | e_interpolated = tf.linspace(e_tacos, e_watermelons, batch_size) 184 | images = model.generate_image(e_interpolated, batch_size=batch_size) 185 | ``` 186 | """ 187 | if diffusion_noise is not None and seed is not None: 188 | raise ValueError( 189 | "`diffusion_noise` and `seed` should not both be passed to " 190 | "`generate_image`. `seed` is only used to generate diffusion " 191 | "noise when it's not already user-specified." 192 | ) 193 | 194 | 195 | if negative_prompt is None: 196 | unconditional_context = tf.repeat( 197 | self._get_unconditional_context(), batch_size, axis=0 198 | ) 199 | else: 200 | unconditional_context = self.encode_text(negative_prompt) 201 | unconditional_context = self._expand_tensor( 202 | unconditional_context, batch_size 203 | ) 204 | # ====================== Extract attention maps ====================== 205 | if latent is not None and timestep is not None: 206 | t_emb = self._get_timestep_embedding(timestep, batch_size) 207 | if encoded_text is not None: 208 | context = self._expand_tensor(encoded_text, batch_size) 209 | conditional_latent, weight_64, weight_32, weight_16, weight_8, x_weights_64, x_weights_32, x_weights_16, x_weights_8 = self.diffusion_model.predict_on_batch( 210 | [latent, t_emb, context] 211 | ) 212 | else: 213 | unconditional_latent, weight_64, weight_32, weight_16, weight_8, x_weights_64, x_weights_32, x_weights_16, x_weights_8 = self.diffusion_model.predict_on_batch( 214 | [latent, t_emb, unconditional_context] 215 | ) 216 | # ====================== Extract attention maps ====================== 217 | else: 218 | context = self._expand_tensor(encoded_text, batch_size) 219 | if diffusion_noise is not None: 220 | diffusion_noise = tf.squeeze(diffusion_noise) 221 | if diffusion_noise.shape.rank == 3: 222 | diffusion_noise = tf.repeat( 223 | tf.expand_dims(diffusion_noise, axis=0), batch_size, axis=0 224 | ) 225 | latent = diffusion_noise 226 | else: 227 | latent = self._get_initial_diffusion_noise(batch_size, seed) 228 | 229 | # Iterative reverse diffusion stage 230 | timesteps = tf.range(1, 1000, 1000 // num_steps) 231 | alphas, alphas_prev = self._get_initial_alphas(timesteps) 232 | progbar = keras.utils.Progbar(len(timesteps)) 233 | iteration = 0 234 | for index, timestep in list(enumerate(timesteps))[::-1]: 235 | latent_prev = latent # Set aside the previous latent vector 236 | t_emb = self._get_timestep_embedding(timestep, batch_size) 237 | 238 | #,_,_,_,_,_,_,_ 239 | unconditional_latent,_,_,_,_,_,_,_,_ = self.diffusion_model.predict_on_batch( 240 | [latent, t_emb, unconditional_context] 241 | ) 242 | #, down1_weights, down2_weights, down3_weights, mid_weights, up1_weights, up2_weights, up3_weights 243 | latent, weight_64, weight_32, weight_16, weight_8, x_weights_64, x_weights_32, x_weights_16, x_weights_8\ 244 | = self.diffusion_model.predict_on_batch([latent, t_emb, context]) 245 | 246 | latent = unconditional_latent + unconditional_guidance_scale * ( 247 | latent - unconditional_latent 248 | ) 249 | a_t, a_prev = alphas[index], alphas_prev[index] 250 | pred_x0 = (latent_prev - math.sqrt(1 - a_t) * latent) / math.sqrt(a_t) 251 | latent = latent * math.sqrt(1.0 - a_prev) + math.sqrt(a_prev) * pred_x0 252 | iteration += 1 253 | progbar.update(iteration) 254 | 255 | # Decoding stage 256 | decoded = self.decoder.predict_on_batch(latent) 257 | decoded = ((decoded + 1) / 2) * 255 258 | return np.clip(decoded, 0, 255).astype("uint8"), weight_64, weight_32, weight_16, weight_8, x_weights_64, x_weights_32, x_weights_16, x_weights_8 259 | 260 | def _get_unconditional_context(self): 261 | unconditional_tokens = tf.convert_to_tensor( 262 | [_UNCONDITIONAL_TOKENS], dtype=tf.int32 263 | ) 264 | unconditional_context = self.text_encoder.predict_on_batch( 265 | [unconditional_tokens, self._get_pos_ids()] 266 | ) 267 | 268 | return unconditional_context 269 | 270 | def _expand_tensor(self, text_embedding, batch_size): 271 | """Extends a tensor by repeating it to fit the shape of the given batch 272 | 273 | size. 274 | """ 275 | text_embedding = tf.squeeze(text_embedding) 276 | if text_embedding.shape.rank == 2: 277 | text_embedding = tf.repeat( 278 | tf.expand_dims(text_embedding, axis=0), batch_size, axis=0 279 | ) 280 | return text_embedding 281 | 282 | @property 283 | def image_encoder(self): 284 | """image_encoder returns the VAE Encoder with pretrained weights. 285 | 286 | Usage: 287 | ```python 288 | sd = keras_cv.models.StableDiffusion() 289 | my_image = np.ones((512, 512, 3)) 290 | latent_representation = sd.image_encoder.predict(my_image) 291 | ``` 292 | """ 293 | if self._image_encoder is None: 294 | self._image_encoder = ImageEncoder() 295 | if self.jit_compile: 296 | self._image_encoder.compile(jit_compile=True) 297 | return self._image_encoder 298 | 299 | @property 300 | def text_encoder(self): 301 | pass 302 | 303 | @property 304 | def diffusion_model(self): 305 | pass 306 | 307 | @property 308 | def decoder(self): 309 | """decoder returns the diffusion image decoder model with pretrained 310 | 311 | weights. Can be overriden for tasks where the decoder needs to be 312 | modified. 313 | """ 314 | if self._decoder is None: 315 | self._decoder = Decoder(self.img_height, self.img_width) 316 | if self.jit_compile: 317 | self._decoder.compile(jit_compile=True) 318 | return self._decoder 319 | 320 | @property 321 | def tokenizer(self): 322 | """tokenizer returns the tokenizer used for text inputs. 323 | 324 | Can be overriden for tasks like textual inversion where the tokenizer needs 325 | to be modified. 326 | """ 327 | if self._tokenizer is None: 328 | self._tokenizer = SimpleTokenizer() 329 | return self._tokenizer 330 | 331 | def _get_timestep_embedding( 332 | self, timestep, batch_size, dim=320, max_period=10000 333 | ): 334 | half = dim // 2 335 | freqs = tf.math.exp( 336 | -math.log(max_period) * tf.range(0, half, dtype=tf.float32) / half 337 | ) 338 | args = tf.convert_to_tensor([timestep], dtype=tf.float32) * freqs 339 | embedding = tf.concat([tf.math.cos(args), tf.math.sin(args)], 0) 340 | embedding = tf.reshape(embedding, [1, -1]) 341 | return tf.repeat(embedding, batch_size, axis=0) 342 | 343 | def _get_initial_alphas(self, timesteps): 344 | alphas = [_ALPHAS_CUMPROD[t] for t in timesteps] 345 | alphas_prev = [1.0] + alphas[:-1] 346 | 347 | return alphas, alphas_prev 348 | 349 | def _get_initial_diffusion_noise(self, batch_size, seed): 350 | if seed is not None: 351 | return tf.random.stateless_normal( 352 | (batch_size, self.img_height // 8, self.img_width // 8, 4), 353 | seed=[seed, seed], 354 | ) 355 | else: 356 | return tf.random.normal( 357 | (batch_size, self.img_height // 8, self.img_width // 8, 4) 358 | ) 359 | 360 | @staticmethod 361 | def _get_pos_ids(): 362 | return tf.convert_to_tensor( 363 | [list(range(MAX_PROMPT_LENGTH))], dtype=tf.int32 364 | ) 365 | 366 | 367 | class StableDiffusion(StableDiffusionBase): 368 | """Keras implementation of Stable Diffusion. 369 | 370 | Note that the StableDiffusion API, as well as the APIs of the sub-components 371 | of StableDiffusion (e.g. ImageEncoder, DiffusionModel) should be considered 372 | unstable at this point. We do not guarantee backwards compatability for 373 | future changes to these APIs. 374 | 375 | Stable Diffusion is a powerful image generation model that can be used, 376 | among other things, to generate pictures according to a short text 377 | description (called a "prompt"). 378 | 379 | Arguments: 380 | img_height: int, height of the images to generate, in pixel. Note that 381 | only multiples of 128 are supported; the value provided will be rounded 382 | to the nearest valid value. Defaults to 512. 383 | img_width: int, width of the images to generate, in pixel. Note that only 384 | multiples of 128 are supported; the value provided will be rounded to 385 | the nearest valid value. Defaults to 512. 386 | jit_compile: bool, whether to compile the underlying models to XLA. This 387 | can lead to a significant speedup on some systems. Defaults to False. 388 | 389 | Example: 390 | 391 | ```python 392 | from keras_cv.models import StableDiffusion 393 | from PIL import Image 394 | 395 | model = StableDiffusion(img_height=512, img_width=512, jit_compile=True) 396 | img = model.text_to_image( 397 | prompt="A beautiful horse running through a field", 398 | batch_size=1, # How many images to generate at once 399 | num_steps=25, # Number of iterations (controls image quality) 400 | seed=123, # Set this to always get the same image from the same prompt 401 | ) 402 | Image.fromarray(img[0]).save("horse.png") 403 | print("saved at horse.png") 404 | ``` 405 | 406 | References: 407 | - [About Stable 408 | Diffusion](https://stability.ai/blog/stable-diffusion-announcement) 409 | - [Original implementation](https://github.com/CompVis/stable-diffusion) 410 | """ # noqa: E501 411 | 412 | def __init__( 413 | self, 414 | img_height=512, 415 | img_width=512, 416 | jit_compile=False, 417 | ): 418 | super().__init__(img_height, img_width, jit_compile) 419 | print( 420 | "By using this model checkpoint, you acknowledge that its usage is " 421 | "subject to the terms of the CreativeML Open RAIL-M license at " 422 | "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/LICENSE" # noqa: E501 423 | ) 424 | 425 | @property 426 | def text_encoder(self): 427 | """text_encoder returns the text encoder with pretrained weights. 428 | 429 | Can be overriden for tasks like textual inversion where the text encoder 430 | needs to be modified. 431 | """ 432 | if self._text_encoder is None: 433 | self._text_encoder = TextEncoder(MAX_PROMPT_LENGTH) 434 | if self.jit_compile: 435 | self._text_encoder.compile(jit_compile=True) 436 | return self._text_encoder 437 | 438 | @property 439 | def diffusion_model(self): 440 | """diffusion_model returns the diffusion model with pretrained weights. 441 | 442 | Can be overriden for tasks where the diffusion model needs to be modified. 443 | """ 444 | if self._diffusion_model is None: 445 | self._diffusion_model = DiffusionModel( 446 | self.img_height, self.img_width, MAX_PROMPT_LENGTH 447 | ) 448 | if self.jit_compile: 449 | self._diffusion_model.compile(jit_compile=True) 450 | return self._diffusion_model --------------------------------------------------------------------------------