├── .gitignore ├── 01-setup-clip.ipynb ├── 02-download-unsplash-dataset.ipynb ├── 03-process-unsplash-dataset.ipynb ├── 04-search-image-dataset.ipynb ├── 09-search-image-api.ipynb ├── LICENSE ├── README.md ├── colab ├── unsplash-image-search-api.ipynb └── unsplash-image-search.ipynb ├── images ├── example_dogs.png ├── example_feeling.png ├── example_love.png └── example_sydney.png ├── requirements.txt ├── unsplash-dataset ├── full │ ├── features │ │ └── .empty │ └── photos │ │ └── .empty └── lite │ ├── features │ └── .empty │ └── photos │ └── .empty └── unsplash-proxy ├── handler.py ├── package-lock.json ├── package.json └── serverless.yml /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # Other 132 | .vscode 133 | .DS_Store 134 | 135 | # CLIP files 136 | CLIP 137 | model.py 138 | clip.py 139 | simple_tokenizer.py 140 | bpe_simple_vocab_16e6.txt.gz 141 | 142 | # Unsplash dataset files 143 | *.tsv* 144 | unsplash-dataset/**/*.jpg 145 | unsplash-dataset/**/*.csv 146 | unsplash-dataset/**/*.npy 147 | 148 | # Serverless 149 | .serverless 150 | node_modules 151 | -------------------------------------------------------------------------------- /01-setup-clip.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "language_info": { 4 | "codemirror_mode": { 5 | "name": "ipython", 6 | "version": 3 7 | }, 8 | "file_extension": ".py", 9 | "mimetype": "text/x-python", 10 | "name": "python", 11 | "nbconvert_exporter": "python", 12 | "pygments_lexer": "ipython3", 13 | "version": "3.8.5-final" 14 | }, 15 | "orig_nbformat": 2, 16 | "kernelspec": { 17 | "name": "python38564bitvenvvenv9efd950bbdd2494f8072faf3b588558e", 18 | "display_name": "Python 3.8.5 64-bit ('.venv': venv)", 19 | "language": "python" 20 | } 21 | }, 22 | "nbformat": 4, 23 | "nbformat_minor": 2, 24 | "cells": [ 25 | { 26 | "source": [ 27 | "# Unsplash Image Search\n", 28 | "\n", 29 | "The project allows you to search images on Unsplash by using a natural words description. It is powered by OpenAI's [CLIP model](https://github.com/openai/CLIP).\n", 30 | "\n", 31 | "Use this notebook to setup your environment." 32 | ], 33 | "cell_type": "markdown", 34 | "metadata": {} 35 | }, 36 | { 37 | "source": [ 38 | "## Install the Dependencies\n", 39 | "\n", 40 | "To install the project's dependencies (including those for CLIP) run the following in the terminal. It is a good idea to create a virtual environment.\n", 41 | "\n", 42 | "```\n", 43 | "pip install -r requirements.txt\n", 44 | "```" 45 | ], 46 | "cell_type": "markdown", 47 | "metadata": {} 48 | }, 49 | { 50 | "source": [ 51 | "## Clone the CLIP repository and copy the code" 52 | ], 53 | "cell_type": "markdown", 54 | "metadata": {} 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 1, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "output_type": "stream", 63 | "name": "stdout", 64 | "text": [ 65 | "Cloning into 'CLIP'...\n", 66 | "remote: Enumerating objects: 24, done.\u001b[K\n", 67 | "remote: Total 24 (delta 0), reused 0 (delta 0), pack-reused 24\u001b[K\n", 68 | "Receiving objects: 100% (24/24), 6.20 MiB | 4.26 MiB/s, done.\n", 69 | "Resolving deltas: 100% (9/9), done.\n" 70 | ] 71 | } 72 | ], 73 | "source": [ 74 | "# Clone the CLIP repository\n", 75 | "!git clone https://github.com/openai/CLIP.git\n", 76 | "\n", 77 | "# Move the CLIP source files and the vocabulary in the root directory. \n", 78 | "# Unfortunately, the CLIP code is not organized as a module, so it cannot be imported easily\n", 79 | "!mv CLIP/*.py .\n", 80 | "!mv CLIP/*.gz .\n" 81 | ] 82 | } 83 | ] 84 | } -------------------------------------------------------------------------------- /02-download-unsplash-dataset.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "language_info": { 4 | "codemirror_mode": { 5 | "name": "ipython", 6 | "version": 3 7 | }, 8 | "file_extension": ".py", 9 | "mimetype": "text/x-python", 10 | "name": "python", 11 | "nbconvert_exporter": "python", 12 | "pygments_lexer": "ipython3", 13 | "version": "3.6.9-final" 14 | }, 15 | "orig_nbformat": 2, 16 | "kernelspec": { 17 | "name": "python3", 18 | "display_name": "Python 3", 19 | "language": "python" 20 | } 21 | }, 22 | "nbformat": 4, 23 | "nbformat_minor": 2, 24 | "cells": [ 25 | { 26 | "source": [ 27 | "\n", 28 | "# Download the Unsplash dataset\n", 29 | "\n", 30 | "This notebook can be used to download all images from the Unsplash dataset: https://github.com/unsplash/datasets. There are two versions Lite (25000 images) and Full (2M images). For the Full one you will need to apply for access (see [here](https://unsplash.com/data)). This will allow you to run CLIP on the whole dataset yourself. \n", 31 | "\n", 32 | "Put the .TSV files in the folder `unsplash-dataset/full` or `unsplash-dataset/lite` or adjust the path in the cell below. " 33 | ], 34 | "cell_type": "markdown", 35 | "metadata": {} 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 1, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "from pathlib import Path\n", 44 | "\n", 45 | "dataset_version = \"lite\" # either \"lite\" or \"full\"\n", 46 | "unsplash_dataset_path = Path(\"unsplash-dataset\") / dataset_version" 47 | ] 48 | }, 49 | { 50 | "source": [ 51 | "## Load the dataset\n", 52 | "\n", 53 | "The `photos.tsv000` contains metadata about the photos in the dataset, but not the photos themselves. We will use the URLs of the photos to download the actual images." 54 | ], 55 | "cell_type": "markdown", 56 | "metadata": {} 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 2, 61 | "metadata": {}, 62 | "outputs": [ 63 | { 64 | "output_type": "stream", 65 | "name": "stdout", 66 | "text": [ 67 | "Photos in the dataset: 25000\n" 68 | ] 69 | } 70 | ], 71 | "source": [ 72 | "import pandas as pd\n", 73 | "\n", 74 | "# Read the photos table\n", 75 | "photos = pd.read_csv(unsplash_dataset_path / \"photos.tsv000\", sep='\\t', header=0)\n", 76 | "\n", 77 | "# Extract the IDs and the URLs of the photos\n", 78 | "photo_urls = photos[['photo_id', 'photo_image_url']].values.tolist()\n", 79 | "\n", 80 | "# Print some statistics\n", 81 | "print(f'Photos in the dataset: {len(photo_urls)}')" 82 | ] 83 | }, 84 | { 85 | "source": [ 86 | "The file name of each photo corresponds to its unique ID from Unsplash. We will download the photos in a reduced resolution (640 pixels width), because they are downscaled by CLIP anyway." 87 | ], 88 | "cell_type": "markdown", 89 | "metadata": {} 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 3, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "import urllib.request\n", 98 | "\n", 99 | "# Path where the photos will be downloaded\n", 100 | "photos_donwload_path = unsplash_dataset_path / \"photos\"\n", 101 | "\n", 102 | "# Function that downloads a single photo\n", 103 | "def download_photo(photo):\n", 104 | " # Get the ID of the photo\n", 105 | " photo_id = photo[0]\n", 106 | "\n", 107 | " # Get the URL of the photo (setting the width to 640 pixels)\n", 108 | " photo_url = photo[1] + \"?w=640\"\n", 109 | "\n", 110 | " # Path where the photo will be stored\n", 111 | " photo_path = photos_donwload_path / (photo_id + \".jpg\")\n", 112 | "\n", 113 | " # Only download a photo if it doesn't exist\n", 114 | " if not photo_path.exists():\n", 115 | " try:\n", 116 | " urllib.request.urlretrieve(photo_url, photo_path)\n", 117 | " except:\n", 118 | " # Catch the exception if the download fails for some reason\n", 119 | " print(f\"Cannot download {photo_url}\")\n", 120 | " pass" 121 | ] 122 | }, 123 | { 124 | "source": [ 125 | "Now the actual download! The download can be parallelized very well, so we will use a thread pool. You may need to tune the `threads_count` parameter to achieve the optimzal performance based on your Internet connection. For me even 128 worked quite well." 126 | ], 127 | "cell_type": "markdown", 128 | "metadata": {} 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": null, 133 | "metadata": {}, 134 | "outputs": [], 135 | "source": [ 136 | "from multiprocessing.pool import ThreadPool\n", 137 | "\n", 138 | "# Create the thread pool\n", 139 | "threads_count = 16\n", 140 | "pool = ThreadPool(threads_count)\n", 141 | "\n", 142 | "# Start the download\n", 143 | "pool.map(download_photo, photo_urls)\n", 144 | "\n", 145 | "# Display some statistics\n", 146 | "display(f'Photos downloaded: {len(photos)}')" 147 | ] 148 | } 149 | ] 150 | } -------------------------------------------------------------------------------- /03-process-unsplash-dataset.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "language_info": { 4 | "codemirror_mode": { 5 | "name": "ipython", 6 | "version": 3 7 | }, 8 | "file_extension": ".py", 9 | "mimetype": "text/x-python", 10 | "name": "python", 11 | "nbconvert_exporter": "python", 12 | "pygments_lexer": "ipython3", 13 | "version": "3.8.5-final" 14 | }, 15 | "orig_nbformat": 2, 16 | "kernelspec": { 17 | "name": "python38564bitvenvvenv9efd950bbdd2494f8072faf3b588558e", 18 | "display_name": "Python 3.8.5 64-bit ('.venv': venv)", 19 | "language": "python" 20 | } 21 | }, 22 | "nbformat": 4, 23 | "nbformat_minor": 2, 24 | "cells": [ 25 | { 26 | "source": [ 27 | "# Process the Unsplash dataset with CLIP\n", 28 | "\n", 29 | "This notebook processes all the downloaded photos using OpenAI's [CLIP neural network](https://github.com/openai/CLIP). For each image we get a feature vector containing 512 float numbers, which we will store in a file. These feature vectors will be used later to compare them to the text feature vectors.\n", 30 | "\n", 31 | "This step will be significantly faster if you have a GPU, but it will also work on the CPU." 32 | ], 33 | "cell_type": "markdown", 34 | "metadata": {} 35 | }, 36 | { 37 | "source": [ 38 | "## Load the photos\n", 39 | "\n", 40 | "Load all photos from the folder they were stored." 41 | ], 42 | "cell_type": "markdown", 43 | "metadata": {} 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 1, 48 | "metadata": {}, 49 | "outputs": [ 50 | { 51 | "output_type": "stream", 52 | "name": "stdout", 53 | "text": [ 54 | "Photos found: 24996\n" 55 | ] 56 | } 57 | ], 58 | "source": [ 59 | "from pathlib import Path\n", 60 | "\n", 61 | "# Set the path to the photos\n", 62 | "dataset_version = \"lite\" # Use \"lite\" or \"full\"\n", 63 | "photos_path = Path(\"unsplash-dataset\") / dataset_version / \"photos\"\n", 64 | "\n", 65 | "# List all JPGs in the folder\n", 66 | "photos_files = list(photos_path.glob(\"*.jpg\"))\n", 67 | "\n", 68 | "# Print some statistics\n", 69 | "print(f\"Photos found: {len(photos_files)}\")" 70 | ] 71 | }, 72 | { 73 | "source": [ 74 | "## Load the CLIP net\n", 75 | "\n", 76 | "Load the CLIP net and define the function that computes the feature vectors" 77 | ], 78 | "cell_type": "markdown", 79 | "metadata": {} 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 2, 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "import clip\n", 88 | "import torch\n", 89 | "from PIL import Image\n", 90 | "\n", 91 | "# Load the open CLIP model\n", 92 | "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", 93 | "model, preprocess = clip.load(\"ViT-B/32\", device=device)\n", 94 | "\n", 95 | "# Function that computes the feature vectors for a batch of images\n", 96 | "def compute_clip_features(photos_batch):\n", 97 | " # Load all the photos from the files\n", 98 | " photos = [Image.open(photo_file) for photo_file in photos_batch]\n", 99 | " \n", 100 | " # Preprocess all photos\n", 101 | " photos_preprocessed = torch.stack([preprocess(photo) for photo in photos]).to(device)\n", 102 | "\n", 103 | " with torch.no_grad():\n", 104 | " # Encode the photos batch to compute the feature vectors and normalize them\n", 105 | " photos_features = model.encode_image(photos_preprocessed)\n", 106 | " photos_features /= photos_features.norm(dim=-1, keepdim=True)\n", 107 | "\n", 108 | " # Transfer the feature vectors back to the CPU and convert to numpy\n", 109 | " return photos_features.cpu().numpy()" 110 | ] 111 | }, 112 | { 113 | "source": [ 114 | "## Process all photos\n", 115 | "\n", 116 | "Now we need to compute the features for all photos. We will do that in batches, because it is much more efficient. You should tune the batch size so that it fits on your GPU. The processing on the GPU is fairly fast, so the bottleneck will probably be loading the photos from the disk.\n", 117 | "\n", 118 | "In this step the feature vectors and the photo IDs of each batch will be saved to a file separately. This makes the whole process more robust. We will merge the data later." 119 | ], 120 | "cell_type": "markdown", 121 | "metadata": {} 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 3, 126 | "metadata": { 127 | "tags": [] 128 | }, 129 | "outputs": [ 130 | { 131 | "output_type": "stream", 132 | "name": "stdout", 133 | "text": [ 134 | "atch 816/1563\n", 135 | "Processing batch 817/1563\n", 136 | "Processing batch 818/1563\n", 137 | "Processing batch 819/1563\n", 138 | "Processing batch 820/1563\n", 139 | "Processing batch 821/1563\n", 140 | "Processing batch 822/1563\n", 141 | "Processing batch 823/1563\n", 142 | "Processing batch 824/1563\n", 143 | "Processing batch 825/1563\n", 144 | "Processing batch 826/1563\n", 145 | "Processing batch 827/1563\n", 146 | "Processing batch 828/1563\n", 147 | "Processing batch 829/1563\n", 148 | "Processing batch 830/1563\n", 149 | "Processing batch 831/1563\n", 150 | "Processing batch 832/1563\n", 151 | "Processing batch 833/1563\n", 152 | "Processing batch 834/1563\n", 153 | "Processing batch 835/1563\n", 154 | "Processing batch 836/1563\n", 155 | "Processing batch 837/1563\n", 156 | "Processing batch 838/1563\n", 157 | "Processing batch 839/1563\n", 158 | "Processing batch 840/1563\n", 159 | "Processing batch 841/1563\n", 160 | "Processing batch 842/1563\n", 161 | "Processing batch 843/1563\n", 162 | "Processing batch 844/1563\n", 163 | "Processing batch 845/1563\n", 164 | "Processing batch 846/1563\n", 165 | "Processing batch 847/1563\n", 166 | "Processing batch 848/1563\n", 167 | "Processing batch 849/1563\n", 168 | "Processing batch 850/1563\n", 169 | "Processing batch 851/1563\n", 170 | "Processing batch 852/1563\n", 171 | "Processing batch 853/1563\n", 172 | "Processing batch 854/1563\n", 173 | "Processing batch 855/1563\n", 174 | "Processing batch 856/1563\n", 175 | "Processing batch 857/1563\n", 176 | "Processing batch 858/1563\n", 177 | "Processing batch 859/1563\n", 178 | "Processing batch 860/1563\n", 179 | "Processing batch 861/1563\n", 180 | "Processing batch 862/1563\n", 181 | "Processing batch 863/1563\n", 182 | "Processing batch 864/1563\n", 183 | "Processing batch 865/1563\n", 184 | "Processing batch 866/1563\n", 185 | "Processing batch 867/1563\n", 186 | "Processing batch 868/1563\n", 187 | "Processing batch 869/1563\n", 188 | "Processing batch 870/1563\n", 189 | "Processing batch 871/1563\n", 190 | "Processing batch 872/1563\n", 191 | "Processing batch 873/1563\n", 192 | "Processing batch 874/1563\n", 193 | "Processing batch 875/1563\n", 194 | "Processing batch 876/1563\n", 195 | "Processing batch 877/1563\n", 196 | "Processing batch 878/1563\n", 197 | "Processing batch 879/1563\n", 198 | "Processing batch 880/1563\n", 199 | "Processing batch 881/1563\n", 200 | "Processing batch 882/1563\n", 201 | "Processing batch 883/1563\n", 202 | "Processing batch 884/1563\n", 203 | "Processing batch 885/1563\n", 204 | "Processing batch 886/1563\n", 205 | "Processing batch 887/1563\n", 206 | "Processing batch 888/1563\n", 207 | "Processing batch 889/1563\n", 208 | "Processing batch 890/1563\n", 209 | "Processing batch 891/1563\n", 210 | "Processing batch 892/1563\n", 211 | "Processing batch 893/1563\n", 212 | "Processing batch 894/1563\n", 213 | "Processing batch 895/1563\n", 214 | "Processing batch 896/1563\n", 215 | "Processing batch 897/1563\n", 216 | "Processing batch 898/1563\n", 217 | "Processing batch 899/1563\n", 218 | "Processing batch 900/1563\n", 219 | "Processing batch 901/1563\n", 220 | "Processing batch 902/1563\n", 221 | "Processing batch 903/1563\n", 222 | "Processing batch 904/1563\n", 223 | "Processing batch 905/1563\n", 224 | "Processing batch 906/1563\n", 225 | "Processing batch 907/1563\n", 226 | "Processing batch 908/1563\n", 227 | "Processing batch 909/1563\n", 228 | "Processing batch 910/1563\n", 229 | "Processing batch 911/1563\n", 230 | "Processing batch 912/1563\n", 231 | "Processing batch 913/1563\n", 232 | "Processing batch 914/1563\n", 233 | "Processing batch 915/1563\n", 234 | "Processing batch 916/1563\n", 235 | "Processing batch 917/1563\n", 236 | "Processing batch 918/1563\n", 237 | "Processing batch 919/1563\n", 238 | "Processing batch 920/1563\n", 239 | "Processing batch 921/1563\n", 240 | "Processing batch 922/1563\n", 241 | "Processing batch 923/1563\n", 242 | "Processing batch 924/1563\n", 243 | "Processing batch 925/1563\n", 244 | "Processing batch 926/1563\n", 245 | "Processing batch 927/1563\n", 246 | "Processing batch 928/1563\n", 247 | "Processing batch 929/1563\n", 248 | "Processing batch 930/1563\n", 249 | "Processing batch 931/1563\n", 250 | "Processing batch 932/1563\n", 251 | "Processing batch 933/1563\n", 252 | "Processing batch 934/1563\n", 253 | "Processing batch 935/1563\n", 254 | "Processing batch 936/1563\n", 255 | "Processing batch 937/1563\n", 256 | "Processing batch 938/1563\n", 257 | "Processing batch 939/1563\n", 258 | "Processing batch 940/1563\n", 259 | "Processing batch 941/1563\n", 260 | "Processing batch 942/1563\n", 261 | "Processing batch 943/1563\n", 262 | "Processing batch 944/1563\n", 263 | "Processing batch 945/1563\n", 264 | "Processing batch 946/1563\n", 265 | "Processing batch 947/1563\n", 266 | "Processing batch 948/1563\n", 267 | "Processing batch 949/1563\n", 268 | "Processing batch 950/1563\n", 269 | "Processing batch 951/1563\n", 270 | "Processing batch 952/1563\n", 271 | "Processing batch 953/1563\n", 272 | "Processing batch 954/1563\n", 273 | "Processing batch 955/1563\n", 274 | "Processing batch 956/1563\n", 275 | "Processing batch 957/1563\n", 276 | "Processing batch 958/1563\n", 277 | "Processing batch 959/1563\n", 278 | "Processing batch 960/1563\n", 279 | "Processing batch 961/1563\n", 280 | "Processing batch 962/1563\n", 281 | "Processing batch 963/1563\n", 282 | "Processing batch 964/1563\n", 283 | "Processing batch 965/1563\n", 284 | "Processing batch 966/1563\n", 285 | "Processing batch 967/1563\n", 286 | "Processing batch 968/1563\n", 287 | "Processing batch 969/1563\n", 288 | "Processing batch 970/1563\n", 289 | "Processing batch 971/1563\n", 290 | "Processing batch 972/1563\n", 291 | "Processing batch 973/1563\n", 292 | "Processing batch 974/1563\n", 293 | "Processing batch 975/1563\n", 294 | "Processing batch 976/1563\n", 295 | "Processing batch 977/1563\n", 296 | "Processing batch 978/1563\n", 297 | "Processing batch 979/1563\n", 298 | "Processing batch 980/1563\n", 299 | "Processing batch 981/1563\n", 300 | "Processing batch 982/1563\n", 301 | "Processing batch 983/1563\n", 302 | "Processing batch 984/1563\n", 303 | "Processing batch 985/1563\n", 304 | "Processing batch 986/1563\n", 305 | "Processing batch 987/1563\n", 306 | "Processing batch 988/1563\n", 307 | "Processing batch 989/1563\n", 308 | "Processing batch 990/1563\n", 309 | "Processing batch 991/1563\n", 310 | "Processing batch 992/1563\n", 311 | "Processing batch 993/1563\n", 312 | "Processing batch 994/1563\n", 313 | "Processing batch 995/1563\n", 314 | "Processing batch 996/1563\n", 315 | "Processing batch 997/1563\n", 316 | "Processing batch 998/1563\n", 317 | "Processing batch 999/1563\n", 318 | "Processing batch 1000/1563\n", 319 | "Processing batch 1001/1563\n", 320 | "Processing batch 1002/1563\n", 321 | "Processing batch 1003/1563\n", 322 | "Processing batch 1004/1563\n", 323 | "Processing batch 1005/1563\n", 324 | "Processing batch 1006/1563\n", 325 | "Processing batch 1007/1563\n", 326 | "Processing batch 1008/1563\n", 327 | "Processing batch 1009/1563\n", 328 | "Processing batch 1010/1563\n", 329 | "Processing batch 1011/1563\n", 330 | "Processing batch 1012/1563\n", 331 | "Processing batch 1013/1563\n", 332 | "Processing batch 1014/1563\n", 333 | "Processing batch 1015/1563\n", 334 | "Processing batch 1016/1563\n", 335 | "Processing batch 1017/1563\n", 336 | "Processing batch 1018/1563\n", 337 | "Processing batch 1019/1563\n", 338 | "Processing batch 1020/1563\n", 339 | "Processing batch 1021/1563\n", 340 | "Processing batch 1022/1563\n", 341 | "Processing batch 1023/1563\n", 342 | "Processing batch 1024/1563\n", 343 | "Processing batch 1025/1563\n", 344 | "Processing batch 1026/1563\n", 345 | "Processing batch 1027/1563\n", 346 | "Processing batch 1028/1563\n", 347 | "Processing batch 1029/1563\n", 348 | "Processing batch 1030/1563\n", 349 | "Processing batch 1031/1563\n", 350 | "Processing batch 1032/1563\n", 351 | "Processing batch 1033/1563\n", 352 | "Processing batch 1034/1563\n", 353 | "Processing batch 1035/1563\n", 354 | "Processing batch 1036/1563\n", 355 | "Processing batch 1037/1563\n", 356 | "Processing batch 1038/1563\n", 357 | "Processing batch 1039/1563\n", 358 | "Processing batch 1040/1563\n", 359 | "Processing batch 1041/1563\n", 360 | "Processing batch 1042/1563\n", 361 | "Processing batch 1043/1563\n", 362 | "Processing batch 1044/1563\n", 363 | "Processing batch 1045/1563\n", 364 | "Processing batch 1046/1563\n", 365 | "Processing batch 1047/1563\n", 366 | "Processing batch 1048/1563\n", 367 | "Processing batch 1049/1563\n", 368 | "Processing batch 1050/1563\n", 369 | "Processing batch 1051/1563\n", 370 | "Processing batch 1052/1563\n", 371 | "Processing batch 1053/1563\n", 372 | "Processing batch 1054/1563\n", 373 | "Processing batch 1055/1563\n", 374 | "Processing batch 1056/1563\n", 375 | "Processing batch 1057/1563\n", 376 | "Processing batch 1058/1563\n", 377 | "Processing batch 1059/1563\n", 378 | "Processing batch 1060/1563\n", 379 | "Processing batch 1061/1563\n", 380 | "Processing batch 1062/1563\n", 381 | "Processing batch 1063/1563\n", 382 | "Processing batch 1064/1563\n", 383 | "Processing batch 1065/1563\n", 384 | "Processing batch 1066/1563\n", 385 | "Processing batch 1067/1563\n", 386 | "Processing batch 1068/1563\n", 387 | "Processing batch 1069/1563\n", 388 | "Processing batch 1070/1563\n", 389 | "Processing batch 1071/1563\n", 390 | "Processing batch 1072/1563\n", 391 | "Processing batch 1073/1563\n", 392 | "Processing batch 1074/1563\n", 393 | "Processing batch 1075/1563\n", 394 | "Processing batch 1076/1563\n", 395 | "Processing batch 1077/1563\n", 396 | "Processing batch 1078/1563\n", 397 | "Processing batch 1079/1563\n", 398 | "Processing batch 1080/1563\n", 399 | "Processing batch 1081/1563\n", 400 | "Processing batch 1082/1563\n", 401 | "Processing batch 1083/1563\n", 402 | "Processing batch 1084/1563\n", 403 | "Processing batch 1085/1563\n", 404 | "Processing batch 1086/1563\n", 405 | "Processing batch 1087/1563\n", 406 | "Processing batch 1088/1563\n", 407 | "Processing batch 1089/1563\n", 408 | "Processing batch 1090/1563\n", 409 | "Processing batch 1091/1563\n", 410 | "Processing batch 1092/1563\n", 411 | "Processing batch 1093/1563\n", 412 | "Processing batch 1094/1563\n", 413 | "Processing batch 1095/1563\n", 414 | "Processing batch 1096/1563\n", 415 | "Processing batch 1097/1563\n", 416 | "Processing batch 1098/1563\n", 417 | "Processing batch 1099/1563\n", 418 | "Processing batch 1100/1563\n", 419 | "Processing batch 1101/1563\n", 420 | "Processing batch 1102/1563\n", 421 | "Processing batch 1103/1563\n", 422 | "Processing batch 1104/1563\n", 423 | "Processing batch 1105/1563\n", 424 | "Processing batch 1106/1563\n", 425 | "Processing batch 1107/1563\n", 426 | "Processing batch 1108/1563\n", 427 | "Processing batch 1109/1563\n", 428 | "Processing batch 1110/1563\n", 429 | "Processing batch 1111/1563\n", 430 | "Processing batch 1112/1563\n", 431 | "Processing batch 1113/1563\n", 432 | "Processing batch 1114/1563\n", 433 | "Processing batch 1115/1563\n", 434 | "Processing batch 1116/1563\n", 435 | "Processing batch 1117/1563\n", 436 | "Processing batch 1118/1563\n", 437 | "Processing batch 1119/1563\n", 438 | "Processing batch 1120/1563\n", 439 | "Processing batch 1121/1563\n", 440 | "Processing batch 1122/1563\n", 441 | "Processing batch 1123/1563\n", 442 | "Processing batch 1124/1563\n", 443 | "Processing batch 1125/1563\n", 444 | "Processing batch 1126/1563\n", 445 | "Processing batch 1127/1563\n", 446 | "Processing batch 1128/1563\n", 447 | "Processing batch 1129/1563\n", 448 | "Processing batch 1130/1563\n", 449 | "Processing batch 1131/1563\n", 450 | "Processing batch 1132/1563\n", 451 | "Processing batch 1133/1563\n", 452 | "Processing batch 1134/1563\n", 453 | "Processing batch 1135/1563\n", 454 | "Processing batch 1136/1563\n", 455 | "Processing batch 1137/1563\n", 456 | "Processing batch 1138/1563\n", 457 | "Processing batch 1139/1563\n", 458 | "Processing batch 1140/1563\n", 459 | "Processing batch 1141/1563\n", 460 | "Processing batch 1142/1563\n", 461 | "Processing batch 1143/1563\n", 462 | "Processing batch 1144/1563\n", 463 | "Processing batch 1145/1563\n", 464 | "Processing batch 1146/1563\n", 465 | "Processing batch 1147/1563\n", 466 | "Processing batch 1148/1563\n", 467 | "Processing batch 1149/1563\n", 468 | "Processing batch 1150/1563\n", 469 | "Processing batch 1151/1563\n", 470 | "Processing batch 1152/1563\n", 471 | "Processing batch 1153/1563\n", 472 | "Processing batch 1154/1563\n", 473 | "Processing batch 1155/1563\n", 474 | "Processing batch 1156/1563\n", 475 | "Processing batch 1157/1563\n", 476 | "Processing batch 1158/1563\n", 477 | "Processing batch 1159/1563\n", 478 | "Processing batch 1160/1563\n", 479 | "Processing batch 1161/1563\n", 480 | "Processing batch 1162/1563\n", 481 | "Processing batch 1163/1563\n", 482 | "Processing batch 1164/1563\n", 483 | "Processing batch 1165/1563\n", 484 | "Processing batch 1166/1563\n", 485 | "Processing batch 1167/1563\n", 486 | "Processing batch 1168/1563\n", 487 | "Processing batch 1169/1563\n", 488 | "Processing batch 1170/1563\n", 489 | "Processing batch 1171/1563\n", 490 | "Processing batch 1172/1563\n", 491 | "Processing batch 1173/1563\n", 492 | "Processing batch 1174/1563\n", 493 | "Processing batch 1175/1563\n", 494 | "Processing batch 1176/1563\n", 495 | "Processing batch 1177/1563\n", 496 | "Processing batch 1178/1563\n", 497 | "Processing batch 1179/1563\n", 498 | "Processing batch 1180/1563\n", 499 | "Processing batch 1181/1563\n", 500 | "Processing batch 1182/1563\n", 501 | "Processing batch 1183/1563\n", 502 | "Processing batch 1184/1563\n", 503 | "Processing batch 1185/1563\n", 504 | "Processing batch 1186/1563\n", 505 | "Processing batch 1187/1563\n", 506 | "Processing batch 1188/1563\n", 507 | "Processing batch 1189/1563\n", 508 | "Processing batch 1190/1563\n", 509 | "Processing batch 1191/1563\n", 510 | "Processing batch 1192/1563\n", 511 | "Processing batch 1193/1563\n", 512 | "Processing batch 1194/1563\n", 513 | "Processing batch 1195/1563\n", 514 | "Processing batch 1196/1563\n", 515 | "Processing batch 1197/1563\n", 516 | "Processing batch 1198/1563\n", 517 | "Processing batch 1199/1563\n", 518 | "Processing batch 1200/1563\n", 519 | "Processing batch 1201/1563\n", 520 | "Processing batch 1202/1563\n", 521 | "Processing batch 1203/1563\n", 522 | "Processing batch 1204/1563\n", 523 | "Processing batch 1205/1563\n", 524 | "Processing batch 1206/1563\n", 525 | "Processing batch 1207/1563\n", 526 | "Processing batch 1208/1563\n", 527 | "Processing batch 1209/1563\n", 528 | "Processing batch 1210/1563\n", 529 | "Processing batch 1211/1563\n", 530 | "Processing batch 1212/1563\n", 531 | "Processing batch 1213/1563\n", 532 | "Processing batch 1214/1563\n", 533 | "Processing batch 1215/1563\n", 534 | "Processing batch 1216/1563\n", 535 | "Processing batch 1217/1563\n", 536 | "Processing batch 1218/1563\n", 537 | "Processing batch 1219/1563\n", 538 | "Processing batch 1220/1563\n", 539 | "Processing batch 1221/1563\n", 540 | "Processing batch 1222/1563\n", 541 | "Processing batch 1223/1563\n", 542 | "Processing batch 1224/1563\n", 543 | "Processing batch 1225/1563\n", 544 | "Processing batch 1226/1563\n", 545 | "Processing batch 1227/1563\n", 546 | "Processing batch 1228/1563\n", 547 | "Processing batch 1229/1563\n", 548 | "Processing batch 1230/1563\n", 549 | "Processing batch 1231/1563\n", 550 | "Processing batch 1232/1563\n", 551 | "Processing batch 1233/1563\n", 552 | "Processing batch 1234/1563\n", 553 | "Processing batch 1235/1563\n", 554 | "Processing batch 1236/1563\n", 555 | "Processing batch 1237/1563\n", 556 | "Processing batch 1238/1563\n", 557 | "Processing batch 1239/1563\n", 558 | "Processing batch 1240/1563\n", 559 | "Processing batch 1241/1563\n", 560 | "Processing batch 1242/1563\n", 561 | "Processing batch 1243/1563\n", 562 | "Processing batch 1244/1563\n", 563 | "Processing batch 1245/1563\n", 564 | "Processing batch 1246/1563\n", 565 | "Processing batch 1247/1563\n", 566 | "Processing batch 1248/1563\n", 567 | "Processing batch 1249/1563\n", 568 | "Processing batch 1250/1563\n", 569 | "Processing batch 1251/1563\n", 570 | "Processing batch 1252/1563\n", 571 | "Processing batch 1253/1563\n", 572 | "Processing batch 1254/1563\n", 573 | "Processing batch 1255/1563\n", 574 | "Processing batch 1256/1563\n", 575 | "Processing batch 1257/1563\n", 576 | "Processing batch 1258/1563\n", 577 | "Processing batch 1259/1563\n", 578 | "Processing batch 1260/1563\n", 579 | "Processing batch 1261/1563\n", 580 | "Processing batch 1262/1563\n", 581 | "Processing batch 1263/1563\n", 582 | "Processing batch 1264/1563\n", 583 | "Processing batch 1265/1563\n", 584 | "Processing batch 1266/1563\n", 585 | "Processing batch 1267/1563\n", 586 | "Processing batch 1268/1563\n", 587 | "Processing batch 1269/1563\n", 588 | "Processing batch 1270/1563\n", 589 | "Processing batch 1271/1563\n", 590 | "Processing batch 1272/1563\n", 591 | "Processing batch 1273/1563\n", 592 | "Processing batch 1274/1563\n", 593 | "Processing batch 1275/1563\n", 594 | "Processing batch 1276/1563\n", 595 | "Processing batch 1277/1563\n", 596 | "Processing batch 1278/1563\n", 597 | "Processing batch 1279/1563\n", 598 | "Processing batch 1280/1563\n", 599 | "Processing batch 1281/1563\n", 600 | "Processing batch 1282/1563\n", 601 | "Processing batch 1283/1563\n", 602 | "Processing batch 1284/1563\n", 603 | "Processing batch 1285/1563\n", 604 | "Processing batch 1286/1563\n", 605 | "Processing batch 1287/1563\n", 606 | "Processing batch 1288/1563\n", 607 | "Processing batch 1289/1563\n", 608 | "Processing batch 1290/1563\n", 609 | "Processing batch 1291/1563\n", 610 | "Processing batch 1292/1563\n", 611 | "Processing batch 1293/1563\n", 612 | "Processing batch 1294/1563\n", 613 | "Processing batch 1295/1563\n", 614 | "Processing batch 1296/1563\n", 615 | "Processing batch 1297/1563\n", 616 | "Processing batch 1298/1563\n", 617 | "Processing batch 1299/1563\n", 618 | "Processing batch 1300/1563\n", 619 | "Processing batch 1301/1563\n", 620 | "Processing batch 1302/1563\n", 621 | "Processing batch 1303/1563\n", 622 | "Processing batch 1304/1563\n", 623 | "Processing batch 1305/1563\n", 624 | "Processing batch 1306/1563\n", 625 | "Processing batch 1307/1563\n", 626 | "Processing batch 1308/1563\n", 627 | "Processing batch 1309/1563\n", 628 | "Processing batch 1310/1563\n", 629 | "Processing batch 1311/1563\n", 630 | "Processing batch 1312/1563\n", 631 | "Processing batch 1313/1563\n", 632 | "Processing batch 1314/1563\n", 633 | "Processing batch 1315/1563\n", 634 | "Processing batch 1316/1563\n", 635 | "Processing batch 1317/1563\n", 636 | "Processing batch 1318/1563\n", 637 | "Processing batch 1319/1563\n", 638 | "Processing batch 1320/1563\n", 639 | "Processing batch 1321/1563\n", 640 | "Processing batch 1322/1563\n", 641 | "Processing batch 1323/1563\n", 642 | "Processing batch 1324/1563\n", 643 | "Processing batch 1325/1563\n", 644 | "Processing batch 1326/1563\n", 645 | "Processing batch 1327/1563\n", 646 | "Processing batch 1328/1563\n", 647 | "Processing batch 1329/1563\n", 648 | "Processing batch 1330/1563\n", 649 | "Processing batch 1331/1563\n", 650 | "Processing batch 1332/1563\n", 651 | "Processing batch 1333/1563\n", 652 | "Processing batch 1334/1563\n", 653 | "Processing batch 1335/1563\n", 654 | "Processing batch 1336/1563\n", 655 | "Processing batch 1337/1563\n", 656 | "Processing batch 1338/1563\n", 657 | "Processing batch 1339/1563\n", 658 | "Processing batch 1340/1563\n", 659 | "Processing batch 1341/1563\n", 660 | "Processing batch 1342/1563\n", 661 | "Processing batch 1343/1563\n", 662 | "Processing batch 1344/1563\n", 663 | "Processing batch 1345/1563\n", 664 | "Processing batch 1346/1563\n", 665 | "Processing batch 1347/1563\n", 666 | "Processing batch 1348/1563\n", 667 | "Processing batch 1349/1563\n", 668 | "Processing batch 1350/1563\n", 669 | "Processing batch 1351/1563\n", 670 | "Processing batch 1352/1563\n", 671 | "Processing batch 1353/1563\n", 672 | "Processing batch 1354/1563\n", 673 | "Processing batch 1355/1563\n", 674 | "Processing batch 1356/1563\n", 675 | "Processing batch 1357/1563\n", 676 | "Processing batch 1358/1563\n", 677 | "Processing batch 1359/1563\n", 678 | "Processing batch 1360/1563\n", 679 | "Processing batch 1361/1563\n", 680 | "Processing batch 1362/1563\n", 681 | "Processing batch 1363/1563\n", 682 | "Processing batch 1364/1563\n", 683 | "Processing batch 1365/1563\n", 684 | "Processing batch 1366/1563\n", 685 | "Processing batch 1367/1563\n", 686 | "Processing batch 1368/1563\n", 687 | "Processing batch 1369/1563\n", 688 | "Processing batch 1370/1563\n", 689 | "Processing batch 1371/1563\n", 690 | "Processing batch 1372/1563\n", 691 | "Processing batch 1373/1563\n", 692 | "Processing batch 1374/1563\n", 693 | "Processing batch 1375/1563\n", 694 | "Processing batch 1376/1563\n", 695 | "Processing batch 1377/1563\n", 696 | "Processing batch 1378/1563\n", 697 | "Processing batch 1379/1563\n", 698 | "Processing batch 1380/1563\n", 699 | "Processing batch 1381/1563\n", 700 | "Processing batch 1382/1563\n", 701 | "Processing batch 1383/1563\n", 702 | "Processing batch 1384/1563\n", 703 | "Processing batch 1385/1563\n", 704 | "Processing batch 1386/1563\n", 705 | "Processing batch 1387/1563\n", 706 | "Processing batch 1388/1563\n", 707 | "Processing batch 1389/1563\n", 708 | "Processing batch 1390/1563\n", 709 | "Processing batch 1391/1563\n", 710 | "Processing batch 1392/1563\n", 711 | "Processing batch 1393/1563\n", 712 | "Processing batch 1394/1563\n", 713 | "Processing batch 1395/1563\n", 714 | "Processing batch 1396/1563\n", 715 | "Processing batch 1397/1563\n", 716 | "Processing batch 1398/1563\n", 717 | "Processing batch 1399/1563\n", 718 | "Processing batch 1400/1563\n", 719 | "Processing batch 1401/1563\n", 720 | "Processing batch 1402/1563\n", 721 | "Processing batch 1403/1563\n", 722 | "Processing batch 1404/1563\n", 723 | "Processing batch 1405/1563\n", 724 | "Processing batch 1406/1563\n", 725 | "Processing batch 1407/1563\n", 726 | "Processing batch 1408/1563\n", 727 | "Processing batch 1409/1563\n", 728 | "Processing batch 1410/1563\n", 729 | "Processing batch 1411/1563\n", 730 | "Processing batch 1412/1563\n", 731 | "Processing batch 1413/1563\n", 732 | "Processing batch 1414/1563\n", 733 | "Processing batch 1415/1563\n", 734 | "Processing batch 1416/1563\n", 735 | "Processing batch 1417/1563\n", 736 | "Processing batch 1418/1563\n", 737 | "Processing batch 1419/1563\n", 738 | "Processing batch 1420/1563\n", 739 | "Processing batch 1421/1563\n", 740 | "Processing batch 1422/1563\n", 741 | "Processing batch 1423/1563\n", 742 | "Processing batch 1424/1563\n", 743 | "Processing batch 1425/1563\n", 744 | "Processing batch 1426/1563\n", 745 | "Processing batch 1427/1563\n", 746 | "Processing batch 1428/1563\n", 747 | "Processing batch 1429/1563\n", 748 | "Processing batch 1430/1563\n", 749 | "Processing batch 1431/1563\n", 750 | "Processing batch 1432/1563\n", 751 | "Processing batch 1433/1563\n", 752 | "Processing batch 1434/1563\n", 753 | "Processing batch 1435/1563\n", 754 | "Processing batch 1436/1563\n", 755 | "Processing batch 1437/1563\n", 756 | "Processing batch 1438/1563\n", 757 | "Processing batch 1439/1563\n", 758 | "Processing batch 1440/1563\n", 759 | "Processing batch 1441/1563\n", 760 | "Processing batch 1442/1563\n", 761 | "Processing batch 1443/1563\n", 762 | "Processing batch 1444/1563\n", 763 | "Processing batch 1445/1563\n", 764 | "Processing batch 1446/1563\n", 765 | "Processing batch 1447/1563\n", 766 | "Processing batch 1448/1563\n", 767 | "Processing batch 1449/1563\n", 768 | "Processing batch 1450/1563\n", 769 | "Processing batch 1451/1563\n", 770 | "Processing batch 1452/1563\n", 771 | "Processing batch 1453/1563\n", 772 | "Processing batch 1454/1563\n", 773 | "Processing batch 1455/1563\n", 774 | "Processing batch 1456/1563\n", 775 | "Processing batch 1457/1563\n", 776 | "Processing batch 1458/1563\n", 777 | "Processing batch 1459/1563\n", 778 | "Processing batch 1460/1563\n", 779 | "Processing batch 1461/1563\n", 780 | "Processing batch 1462/1563\n", 781 | "Processing batch 1463/1563\n", 782 | "Processing batch 1464/1563\n", 783 | "Processing batch 1465/1563\n", 784 | "Processing batch 1466/1563\n", 785 | "Processing batch 1467/1563\n", 786 | "Processing batch 1468/1563\n", 787 | "Processing batch 1469/1563\n", 788 | "Processing batch 1470/1563\n", 789 | "Processing batch 1471/1563\n", 790 | "Processing batch 1472/1563\n", 791 | "Processing batch 1473/1563\n", 792 | "Processing batch 1474/1563\n", 793 | "Processing batch 1475/1563\n", 794 | "Processing batch 1476/1563\n", 795 | "Processing batch 1477/1563\n", 796 | "Processing batch 1478/1563\n", 797 | "Processing batch 1479/1563\n", 798 | "Processing batch 1480/1563\n", 799 | "Processing batch 1481/1563\n", 800 | "Processing batch 1482/1563\n", 801 | "Processing batch 1483/1563\n", 802 | "Processing batch 1484/1563\n", 803 | "Processing batch 1485/1563\n", 804 | "Processing batch 1486/1563\n", 805 | "Processing batch 1487/1563\n", 806 | "Processing batch 1488/1563\n", 807 | "Processing batch 1489/1563\n", 808 | "Processing batch 1490/1563\n", 809 | "Processing batch 1491/1563\n", 810 | "Processing batch 1492/1563\n", 811 | "Processing batch 1493/1563\n", 812 | "Processing batch 1494/1563\n", 813 | "Processing batch 1495/1563\n", 814 | "Processing batch 1496/1563\n", 815 | "Processing batch 1497/1563\n", 816 | "Processing batch 1498/1563\n", 817 | "Processing batch 1499/1563\n", 818 | "Processing batch 1500/1563\n", 819 | "Processing batch 1501/1563\n", 820 | "Processing batch 1502/1563\n", 821 | "Processing batch 1503/1563\n", 822 | "Processing batch 1504/1563\n", 823 | "Processing batch 1505/1563\n", 824 | "Processing batch 1506/1563\n", 825 | "Processing batch 1507/1563\n", 826 | "Processing batch 1508/1563\n", 827 | "Processing batch 1509/1563\n", 828 | "Processing batch 1510/1563\n", 829 | "Processing batch 1511/1563\n", 830 | "Processing batch 1512/1563\n", 831 | "Processing batch 1513/1563\n", 832 | "Processing batch 1514/1563\n", 833 | "Processing batch 1515/1563\n", 834 | "Processing batch 1516/1563\n", 835 | "Processing batch 1517/1563\n", 836 | "Processing batch 1518/1563\n", 837 | "Processing batch 1519/1563\n", 838 | "Processing batch 1520/1563\n", 839 | "Processing batch 1521/1563\n", 840 | "Processing batch 1522/1563\n", 841 | "Processing batch 1523/1563\n", 842 | "Processing batch 1524/1563\n", 843 | "Processing batch 1525/1563\n", 844 | "Processing batch 1526/1563\n", 845 | "Processing batch 1527/1563\n", 846 | "Processing batch 1528/1563\n", 847 | "Processing batch 1529/1563\n", 848 | "Processing batch 1530/1563\n", 849 | "Processing batch 1531/1563\n", 850 | "Processing batch 1532/1563\n", 851 | "Processing batch 1533/1563\n", 852 | "Processing batch 1534/1563\n", 853 | "Processing batch 1535/1563\n", 854 | "Processing batch 1536/1563\n", 855 | "Processing batch 1537/1563\n", 856 | "Processing batch 1538/1563\n", 857 | "Processing batch 1539/1563\n", 858 | "Processing batch 1540/1563\n", 859 | "Processing batch 1541/1563\n", 860 | "Processing batch 1542/1563\n", 861 | "Processing batch 1543/1563\n", 862 | "Processing batch 1544/1563\n", 863 | "Processing batch 1545/1563\n", 864 | "Processing batch 1546/1563\n", 865 | "Processing batch 1547/1563\n", 866 | "Processing batch 1548/1563\n", 867 | "Processing batch 1549/1563\n", 868 | "Processing batch 1550/1563\n", 869 | "Processing batch 1551/1563\n", 870 | "Processing batch 1552/1563\n", 871 | "Processing batch 1553/1563\n", 872 | "Processing batch 1554/1563\n", 873 | "Processing batch 1555/1563\n", 874 | "Processing batch 1556/1563\n", 875 | "Processing batch 1557/1563\n", 876 | "Processing batch 1558/1563\n", 877 | "Processing batch 1559/1563\n", 878 | "Processing batch 1560/1563\n", 879 | "Processing batch 1561/1563\n", 880 | "Processing batch 1562/1563\n", 881 | "Processing batch 1563/1563\n" 882 | ] 883 | } 884 | ], 885 | "source": [ 886 | "import math\n", 887 | "import numpy as np\n", 888 | "import pandas as pd\n", 889 | "\n", 890 | "# Define the batch size so that it fits on your GPU. You can also do the processing on the CPU, but it will be slower.\n", 891 | "batch_size = 16\n", 892 | "\n", 893 | "# Path where the feature vectors will be stored\n", 894 | "features_path = Path(\"unsplash-dataset\") / dataset_version / \"features\"\n", 895 | "\n", 896 | "# Compute how many batches are needed\n", 897 | "batches = math.ceil(len(photos_files) / batch_size)\n", 898 | "\n", 899 | "# Process each batch\n", 900 | "for i in range(batches):\n", 901 | " print(f\"Processing batch {i+1}/{batches}\")\n", 902 | "\n", 903 | " batch_ids_path = features_path / f\"{i:010d}.csv\"\n", 904 | " batch_features_path = features_path / f\"{i:010d}.npy\"\n", 905 | " \n", 906 | " # Only do the processing if the batch wasn't processed yet\n", 907 | " if not batch_features_path.exists():\n", 908 | " try:\n", 909 | " # Select the photos for the current batch\n", 910 | " batch_files = photos_files[i*batch_size : (i+1)*batch_size]\n", 911 | "\n", 912 | " # Compute the features and save to a numpy file\n", 913 | " batch_features = compute_clip_features(batch_files)\n", 914 | " np.save(batch_features_path, batch_features)\n", 915 | "\n", 916 | " # Save the photo IDs to a CSV file\n", 917 | " photo_ids = [photo_file.name.split(\".\")[0] for photo_file in batch_files]\n", 918 | " photo_ids_data = pd.DataFrame(photo_ids, columns=['photo_id'])\n", 919 | " photo_ids_data.to_csv(batch_ids_path, index=False)\n", 920 | " except:\n", 921 | " # Catch problems with the processing to make the process more robust\n", 922 | " print(f'Problem with batch {i}')" 923 | ] 924 | }, 925 | { 926 | "source": [ 927 | "Merge the features and the photo IDs. The resulting files are `features.npy` and `photo_ids.csv`. Feel free to delete the intermediate results." 928 | ], 929 | "cell_type": "markdown", 930 | "metadata": {} 931 | }, 932 | { 933 | "cell_type": "code", 934 | "execution_count": 4, 935 | "metadata": {}, 936 | "outputs": [], 937 | "source": [ 938 | "import numpy as np\n", 939 | "import pandas as pd\n", 940 | "\n", 941 | "# Load all numpy files\n", 942 | "features_list = [np.load(features_file) for features_file in sorted(features_path.glob(\"*.npy\"))]\n", 943 | "\n", 944 | "# Concatenate the features and store in a merged file\n", 945 | "features = np.concatenate(features_list)\n", 946 | "np.save(features_path / \"features.npy\", features)\n", 947 | "\n", 948 | "# Load all the photo IDs\n", 949 | "photo_ids = pd.concat([pd.read_csv(ids_file) for ids_file in sorted(features_path.glob(\"*.csv\"))])\n", 950 | "photo_ids.to_csv(features_path / \"photo_ids.csv\", index=False)" 951 | ] 952 | } 953 | ] 954 | } -------------------------------------------------------------------------------- /04-search-image-dataset.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "language_info": { 4 | "codemirror_mode": { 5 | "name": "ipython", 6 | "version": 3 7 | }, 8 | "file_extension": ".py", 9 | "mimetype": "text/x-python", 10 | "name": "python", 11 | "nbconvert_exporter": "python", 12 | "pygments_lexer": "ipython3", 13 | "version": "3.8.5-final" 14 | }, 15 | "orig_nbformat": 2, 16 | "kernelspec": { 17 | "name": "python38564bitvenvvenv9efd950bbdd2494f8072faf3b588558e", 18 | "display_name": "Python 3.8.5 64-bit ('.venv': venv)", 19 | "language": "python" 20 | } 21 | }, 22 | "nbformat": 4, 23 | "nbformat_minor": 2, 24 | "cells": [ 25 | { 26 | "source": [ 27 | "# Search image in the Dataset\n", 28 | "\n", 29 | "With this notebook you can search for photos using natural language." 30 | ], 31 | "cell_type": "markdown", 32 | "metadata": {} 33 | }, 34 | { 35 | "source": [ 36 | "## Load the dataset\n", 37 | "\n", 38 | "You will need the Unsplash Dataset and the precomputed feature vetors for this. You don't want to process the whole dataset yourself, you can download the preprocessed feature vectors from [here](https://drive.google.com/drive/folders/1ozUUr8UQ2YWDSJyYIIun9V8Qg1TjqXEs?usp=sharing)." 39 | ], 40 | "cell_type": "markdown", 41 | "metadata": {} 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 1, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "from pathlib import Path\n", 50 | "import numpy as np\n", 51 | "import pandas as pd\n", 52 | "\n", 53 | "# Set the paths\n", 54 | "dataset_version = \"lite\" # choose \"lite\" or \"full\"\n", 55 | "unsplash_dataset_path = Path(\"unsplash-dataset\") / dataset_version\n", 56 | "features_path = Path(\"unsplash-dataset\") / dataset_version / \"features\"\n", 57 | "\n", 58 | "# Read the photos table\n", 59 | "photos = pd.read_csv(unsplash_dataset_path / \"photos.tsv000\", sep='\\t', header=0)\n", 60 | "\n", 61 | "# Load the features and the corresponding IDs\n", 62 | "photo_features = np.load(features_path / \"features.npy\")\n", 63 | "photo_ids = pd.read_csv(features_path / \"photo_ids.csv\")\n", 64 | "photo_ids = list(photo_ids['photo_id'])" 65 | ] 66 | }, 67 | { 68 | "source": [ 69 | "Load the CLIP network." 70 | ], 71 | "cell_type": "markdown", 72 | "metadata": {} 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 2, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "import clip\n", 81 | "import torch\n", 82 | "\n", 83 | "# Load the open CLIP model\n", 84 | "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", 85 | "model, preprocess = clip.load(\"ViT-B/32\", device=device)" 86 | ] 87 | }, 88 | { 89 | "source": [ 90 | "## Search\n", 91 | "\n", 92 | "Specify your search query and encode it to a feature vector using CLIP." 93 | ], 94 | "cell_type": "markdown", 95 | "metadata": {} 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": 3, 100 | "metadata": {}, 101 | "outputs": [], 102 | "source": [ 103 | "search_query = \"Two dogs playing in the snow\"\n", 104 | "\n", 105 | "with torch.no_grad():\n", 106 | " # Encode and normalize the description using CLIP\n", 107 | " text_encoded = model.encode_text(clip.tokenize(search_query).to(device))\n", 108 | " text_encoded /= text_encoded.norm(dim=-1, keepdim=True)" 109 | ] 110 | }, 111 | { 112 | "source": [ 113 | "Compare the text features to the image features and find the best match." 114 | ], 115 | "cell_type": "markdown", 116 | "metadata": {} 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 4, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "# Retrieve the description vector and the photo vectors\n", 125 | "text_features = text_encoded.cpu().numpy()\n", 126 | "\n", 127 | "# Compute the similarity between the descrption and each photo using the Cosine similarity\n", 128 | "similarities = list((text_features @ photo_features.T).squeeze(0))\n", 129 | "\n", 130 | "# Sort the photos by their similarity score\n", 131 | "best_photos = sorted(zip(similarities, range(photo_features.shape[0])), key=lambda x: x[0], reverse=True)" 132 | ] 133 | }, 134 | { 135 | "source": [ 136 | "## Display the results\n", 137 | "\n", 138 | "Show the top 3 results." 139 | ], 140 | "cell_type": "markdown", 141 | "metadata": {} 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": 5, 146 | "metadata": {}, 147 | "outputs": [ 148 | { 149 | "output_type": "display_data", 150 | "data": { 151 | "text/html": "", 152 | "text/plain": "" 153 | }, 154 | "metadata": {} 155 | }, 156 | { 157 | "output_type": "display_data", 158 | "data": { 159 | "text/plain": "", 160 | "text/html": "Photo by Luka Vovk on Unsplash" 161 | }, 162 | "metadata": {} 163 | }, 164 | { 165 | "output_type": "stream", 166 | "name": "stdout", 167 | "text": [ 168 | "\n" 169 | ] 170 | }, 171 | { 172 | "output_type": "display_data", 173 | "data": { 174 | "text/html": "", 175 | "text/plain": "" 176 | }, 177 | "metadata": {} 178 | }, 179 | { 180 | "output_type": "display_data", 181 | "data": { 182 | "text/plain": "", 183 | "text/html": "Photo by Luka Vovk on Unsplash" 184 | }, 185 | "metadata": {} 186 | }, 187 | { 188 | "output_type": "stream", 189 | "name": "stdout", 190 | "text": [ 191 | "\n" 192 | ] 193 | }, 194 | { 195 | "output_type": "display_data", 196 | "data": { 197 | "text/html": "", 198 | "text/plain": "" 199 | }, 200 | "metadata": {} 201 | }, 202 | { 203 | "output_type": "display_data", 204 | "data": { 205 | "text/plain": "", 206 | "text/html": "Photo by Tadeusz Lakota on Unsplash" 207 | }, 208 | "metadata": {} 209 | }, 210 | { 211 | "output_type": "stream", 212 | "name": "stdout", 213 | "text": [ 214 | "\n" 215 | ] 216 | } 217 | ], 218 | "source": [ 219 | "from IPython.display import Image\n", 220 | "from IPython.core.display import HTML\n", 221 | "\n", 222 | "# Iterate over the top 3 results\n", 223 | "for i in range(3):\n", 224 | " # Retrieve the photo ID\n", 225 | " idx = best_photos[i][1]\n", 226 | " photo_id = photo_ids[idx]\n", 227 | "\n", 228 | " # Get all metadata for this photo\n", 229 | " photo_data = photos[photos[\"photo_id\"] == photo_id].iloc[0]\n", 230 | "\n", 231 | " # Display the photo\n", 232 | " display(Image(url=photo_data[\"photo_image_url\"] + \"?w=640\"))\n", 233 | "\n", 234 | " # Display the attribution text\n", 235 | " display(HTML(f'Photo by {photo_data[\"photographer_first_name\"]} {photo_data[\"photographer_last_name\"]} on Unsplash'))\n", 236 | " print()\n", 237 | " " 238 | ] 239 | } 240 | ] 241 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Vladimir Haltakov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Unsplash Image Search 2 | 3 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/haltakov/natural-language-image-search/blob/main/colab/unsplash-image-search.ipynb) 4 | 5 | Search photos on Unsplash using natural language descriptions. The search is powered by OpenAI's [CLIP model](https://github.com/openai/CLIP) and the [Unsplash Dataset](https://unsplash.com/data). 6 | 7 | ### "Two dogs playing in the snow" 8 | 9 | ![Search results for "Two dogs playing in the snow"](images/example_dogs.png) 10 | Photos by [Richard Burlton](https://unsplash.com/@richardworks?utm_source=NaturalLanguageImageSearch&utm_medium=referral), [Karl Anderson](https://unsplash.com/@karlkiwi90?utm_source=NaturalLanguageImageSearch&utm_medium=referral) and [Xuecheng Chen](https://unsplash.com/@samaritan_?utm_source=NaturalLanguageImageSearch&utm_medium=referral) on [Unsplash](https://unsplash.com/?utm_source=NaturalLanguageImageSearch&utm_medium=referral). 11 | 12 | ### "The word love written on the wall" 13 | 14 | ![Search results for "The word love written on the wall"](images/example_love.png) 15 | Photos by [Genton Damian](https://unsplash.com/@damiangenton96?utm_source=NaturalLanguageImageSearch&utm_medium=referral) , [Anna Rozwadowska](https://unsplash.com/@arozwadowska?utm_source=NaturalLanguageImageSearch&utm_medium=referral), [Jude Beck](https://unsplash.com/@judebeck?utm_source=NaturalLanguageImageSearch&utm_medium=referral) on [Unsplash](https://unsplash.com/?utm_source=NaturalLanguageImageSearch&utm_medium=referral). 16 | 17 | ### "The feeling when your program finally works" 18 | 19 | ![Search results for "The feeling when your program finally works"](images/example_feeling.png) 20 | Photos by [bruce mars](https://unsplash.com/@brucemars?utm_source=NaturalLanguageImageSearch&utm_medium=referral), [LOGAN WEAVER](https://unsplash.com/@lgnwvr?utm_source=NaturalLanguageImageSearch&utm_medium=referral), [Vasyl Skunziak](https://unsplash.com/@vskvsk1?utm_source=NaturalLanguageImageSearch&utm_medium=referral) on [Unsplash](https://unsplash.com/?utm_source=NaturalLanguageImageSearch&utm_medium=referral). 21 | 22 | ### "The Syndey Opera House and the Harbour Bridge at night" 23 | 24 | ![Search results for "The Syndey Opera House and the Harbour Bridge at night"](images/example_sydney.png) 25 | Photos by [Dalal Nizam](https://unsplash.com/@dilson?utm_source=NaturalLanguageImageSearch&utm_medium=referral) and [Anna Tremewan](https://unsplash.com/@annatre?utm_source=NaturalLanguageImageSearch&utm_medium=referral) on [Unsplash](https://unsplash.com/?utm_source=NaturalLanguageImageSearch&utm_medium=referral). 26 | 27 | ## How It Works? 28 | 29 | OpenAI's [CLIP](https://openai.com/blog/clip/) neural network is able to transform both images and text into the same latent space, where they can be compared using a similarity measure. 30 | 31 | For this project, all photos from the full [Unsplash Dataset](https://unsplash.com/data) (almost 2M photos) were downloaded and processed with CLIP. 32 | 33 | The pre-computed feature vectors for all images can then be used to find the best match to a natural language search query. 34 | 35 | ## How To Run The Code? 36 | 37 | ### On Google Colab 38 | 39 | If you just want to play around with different queries jump to the [Colab notebook](https://colab.research.google.com/github/haltakov/natural-language-image-search/blob/main/colab/unsplash-image-search.ipynb). 40 | 41 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/haltakov/natural-language-image-search/blob/main/colab/unsplash-image-search.ipynb) 42 | 43 | ### On your machine 44 | 45 | Before running any of the code, make sure to install all dependencies: 46 | 47 | ``` 48 | pip install -r requirements.txt 49 | ``` 50 | 51 | If you want to run all the code yourself open the Jupyter notebooks in the order they are numbered and follow the instructions there: 52 | 53 | - `01-setup-clip.ipynb` - setup the environment checking out and preparing the CLIP code. 54 | - `02-download-unsplash-dataset.ipynb` - download the photos from the Unsplash dataset 55 | - `03-process-unsplash-dataset.ipynb` - process all photos from the dataset with CLIP 56 | - `04-search-image-dataset.ipynb` - search for a photo in the dataset using natural language queries 57 | - `09-search-image-api.ipynb` - search for a photo using the Unsplash Search API and filter the results using CLIP. 58 | 59 | > NOTE: only the Lite version of the Unsplash Dataset is publicly available. If you want to use the Full version, you will need to [apply](https://unsplash.com/data) for (free) access. 60 | 61 | > NOTE: searching for images using the Unsplash Search API doesn't require access to the Unsplash Dataset, but will probably deliver worse results. 62 | 63 | ## Acknowledgements 64 | 65 | This project was inspired by these projects: 66 | 67 | - [Beyond tags and entering the semantic search era on images with OpenAI CLIP](https://towardsdatascience.com/beyond-tags-and-entering-the-semantic-search-era-on-images-with-openai-clip-1f7d629a9978) by [Ramsri Goutham Golla](https://twitter.com/ramsri_goutham) 68 | - [Alph, The Sacred River](https://github.com/thoppe/alph-the-sacred-river) by [Travis Hoppe](https://twitter.com/metasemantic) 69 | - [OpenAI's CLIP](https://github.com/openai/CLIP) 70 | - [Unsplash](https://unsplash.com/) 71 | -------------------------------------------------------------------------------- /colab/unsplash-image-search-api.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "unsplash-image-search.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [ 9 | "Ons94QRxCQR2", 10 | "lJVrkmy6DVj2", 11 | "ujCKerTnFBk4" 12 | ] 13 | }, 14 | "kernelspec": { 15 | "name": "python3", 16 | "display_name": "Python 3" 17 | }, 18 | "accelerator": "GPU" 19 | }, 20 | "cells": [ 21 | { 22 | "cell_type": "markdown", 23 | "metadata": { 24 | "id": "97qkK30wAzXb" 25 | }, 26 | "source": [ 27 | "# Unsplash Image Search\n", 28 | "\n", 29 | "Using this notebook you can search for images from the [Unsplash Dataset](https://unsplash.com/data) using natural language queries. The search is powered by OpenAI's [CLIP](https://github.com/openai/CLIP) neural network.\n", 30 | "\n", 31 | "This notebook uses the precomputed feature vectors for almost 2 million images from the full version of the [Unsplash Dataset](https://unsplash.com/data). If you want to compute the features yourself, see [here](https://github.com/haltakov/natural-language-image-search#on-your-machine).\n", 32 | "\n", 33 | "This project was created by [Vladimir Haltakov](https://twitter.com/haltakov) and the full code is open-sourced on [GitHub](https://github.com/haltakov/natural-language-image-search)." 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": { 39 | "id": "Ons94QRxCQR2" 40 | }, 41 | "source": [ 42 | "## Setup Environment\n", 43 | "\n", 44 | "In this section we will setup the environment." 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": { 50 | "id": "DYdIafWsOOUV" 51 | }, 52 | "source": [ 53 | "First we need to install CLIP and then upgrade the version of torch to 1.7.1 with CUDA support (by default CLIP installs torch 1.7.1 without CUDA). Google Colab currently has torch 1.7.0 which doesn't work well with CLIP." 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "metadata": { 59 | "id": "djgE7IjbV3sv" 60 | }, 61 | "source": [ 62 | "!pip install git+https://github.com/openai/CLIP.git\n", 63 | "!pip install torch==1.7.1+cu101 torchvision==0.8.2+cu101 -f https://download.pytorch.org/whl/torch_stable.html" 64 | ], 65 | "execution_count": null, 66 | "outputs": [] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": { 71 | "id": "B7_Sk-T7DBEm" 72 | }, 73 | "source": [ 74 | "We can now load the pretrained public CLIP model." 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "metadata": { 80 | "id": "_6FzbzS6W1R5" 81 | }, 82 | "source": [ 83 | "import clip\n", 84 | "import torch\n", 85 | "\n", 86 | "# Load the open CLIP model\n", 87 | "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", 88 | "model, preprocess = clip.load(\"ViT-B/32\", device=device)" 89 | ], 90 | "execution_count": null, 91 | "outputs": [] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "metadata": { 96 | "id": "lJVrkmy6DVj2" 97 | }, 98 | "source": [ 99 | "## Download the Precomputed Data\n", 100 | "\n", 101 | "In this section the precomputed feature vectors for all photos are downloaded." 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": { 107 | "id": "18alAEjEOdSC" 108 | }, 109 | "source": [ 110 | "In order to compare the photos from the Unsplash dataset to a text query, we need to compute the feature vector of each photo using CLIP. This is a time consuming task, so you can use the feature vectors that I precomputed and uploaded to Google Drive (with the permission from Unsplash). If you want to compute the features yourself, see [here](https://github.com/haltakov/natural-language-image-search#on-your-machine).\n", 111 | "\n", 112 | "We need to download two files:\n", 113 | "* `photo_ids.csv` - a list of the photo IDs for all images in the dataset. The photo ID can be used to get the actual photo from Unsplash.\n", 114 | "* `features.npy` - a matrix containing the precomputed 512 element feature vector for each photo in the dataset.\n", 115 | "\n", 116 | "The files are available on [Google Drive](https://drive.google.com/drive/folders/1WQmedVCDIQKA2R33dkS1f980YsJXRZ-q?usp=sharing)." 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "metadata": { 122 | "id": "BAb15OJQZRkt" 123 | }, 124 | "source": [ 125 | "from pathlib import Path\n", 126 | "\n", 127 | "# Create a folder for the precomputed features\n", 128 | "!mkdir unsplash-dataset\n", 129 | "\n", 130 | "# Download the photo IDs and the feature vectors\n", 131 | "!gdown --id 1FdmDEzBQCf3OxqY9SbU-jLfH_yZ6UPSj -O unsplash-dataset/photo_ids.csv\n", 132 | "!gdown --id 1L7ulhn4VeN-2aOM-fYmljza_TQok-j9F -O unsplash-dataset/features.npy\n", 133 | "\n", 134 | "# Download from alternative source, if the download doesn't work for some reason (for example download quota limit exceeded)\n", 135 | "if not Path('unsplash-dataset/photo_ids.csv').exists():\n", 136 | " !wget https://transfer.army/api/download/-4O3XMtkPKs/VZcRXj84 -O unsplash-dataset/photo_ids.csv\n", 137 | "\n", 138 | "if not Path('unsplash-dataset/features.npy').exists():\n", 139 | " !wget https://transfer.army/api/download/OZmCwAJNoY0/xrCL7niq -O unsplash-dataset/features.npy\n", 140 | " " 141 | ], 142 | "execution_count": null, 143 | "outputs": [] 144 | }, 145 | { 146 | "cell_type": "markdown", 147 | "metadata": { 148 | "id": "TVjuUh6oEtPt" 149 | }, 150 | "source": [ 151 | "After the files are downloaded we need to load them using `pandas` and `numpy`." 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "metadata": { 157 | "id": "GQHcmMo1Ztjz", 158 | "outputId": "87cc44a3-668b-48a3-81f0-954c0c2a759f", 159 | "colab": { 160 | "base_uri": "https://localhost:8080/" 161 | } 162 | }, 163 | "source": [ 164 | "import pandas as pd\n", 165 | "import numpy as np\n", 166 | "\n", 167 | "# Load the photo IDs\n", 168 | "photo_ids = pd.read_csv(\"unsplash-dataset/photo_ids.csv\")\n", 169 | "photo_ids = list(photo_ids['photo_id'])\n", 170 | "\n", 171 | "# Load the features vectors\n", 172 | "photo_features = np.load(\"unsplash-dataset/features.npy\")\n", 173 | "\n", 174 | "# Convert features to Tensors: Float32 on CPU and Float16 on GPU\n", 175 | "if device == \"cpu\":\n", 176 | " photo_features = torch.from_numpy(photo_features).float().to(device)\n", 177 | "else:\n", 178 | " photo_features = torch.from_numpy(photo_features).to(device)\n", 179 | "\n", 180 | "# Print some statistics\n", 181 | "print(f\"Photos loaded: {len(photo_ids)}\")" 182 | ], 183 | "execution_count": 4, 184 | "outputs": [ 185 | { 186 | "output_type": "stream", 187 | "text": [ 188 | "Photos loaded: 1981161\n" 189 | ], 190 | "name": "stdout" 191 | } 192 | ] 193 | }, 194 | { 195 | "cell_type": "markdown", 196 | "metadata": { 197 | "id": "ujCKerTnFBk4" 198 | }, 199 | "source": [ 200 | "## Define Functions\n", 201 | "\n", 202 | "Some important functions for processing the data are defined here.\n", 203 | "\n" 204 | ] 205 | }, 206 | { 207 | "cell_type": "markdown", 208 | "metadata": { 209 | "id": "pYVNtF-JFtfj" 210 | }, 211 | "source": [ 212 | "The `encode_search_query` function takes a text description and encodes it into a feature vector using the CLIP model." 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "metadata": { 218 | "id": "d0hmOh3qbcxK" 219 | }, 220 | "source": [ 221 | "def encode_search_query(search_query):\n", 222 | " with torch.no_grad():\n", 223 | " # Encode and normalize the search query using CLIP\n", 224 | " text_encoded = model.encode_text(clip.tokenize(search_query).to(device))\n", 225 | " text_encoded /= text_encoded.norm(dim=-1, keepdim=True)\n", 226 | "\n", 227 | " # Retrieve the feature vector\n", 228 | " return text_encoded" 229 | ], 230 | "execution_count": 5, 231 | "outputs": [] 232 | }, 233 | { 234 | "cell_type": "markdown", 235 | "metadata": { 236 | "id": "Vh1yyJtEGCAX" 237 | }, 238 | "source": [ 239 | "The `find_best_matches` function compares the text feature vector to the feature vectors of all images and finds the best matches. The function returns the IDs of the best matching photos." 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "metadata": { 245 | "id": "3TcmI5KIbe5F" 246 | }, 247 | "source": [ 248 | "def find_best_matches(text_features, photo_features, photo_ids, results_count=3):\n", 249 | " # Compute the similarity between the search query and each photo using the Cosine similarity\n", 250 | " similarities = (photo_features @ text_features.T).squeeze(1)\n", 251 | "\n", 252 | " # Sort the photos by their similarity score\n", 253 | " best_photo_idx = (-similarities).argsort()\n", 254 | "\n", 255 | " # Return the photo IDs of the best matches\n", 256 | " return [photo_ids[i] for i in best_photo_idx[:results_count]]" 257 | ], 258 | "execution_count": 6, 259 | "outputs": [] 260 | }, 261 | { 262 | "cell_type": "markdown", 263 | "metadata": { 264 | "id": "gEmt0F4iHbL0" 265 | }, 266 | "source": [ 267 | "The `display_photo` function displays a photo from Unsplash given its ID. \n", 268 | "\n", 269 | "This function needs to call the Unsplash API to get the URL of the photo and some metadata about the photographer. Since I'm [not allowed](https://help.unsplash.com/en/articles/2511245-unsplash-api-guidelines) to share my Unsplash API access key publicly, I created a small proxy that queries the Unsplash API and returns the data (see the code [here](https://github.com/haltakov/natural-language-image-search/tree/main/unsplash-proxy)). In this way you can play around without creating a developer account at Unsplash, while keeping my key private. I hope I don't hit the API rate limit.\n", 270 | "\n", 271 | "If you already have an Unsplash developer account, you can uncomment the relevant code and plugin your own access key." 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "metadata": { 277 | "id": "RC4HD8cBYOon" 278 | }, 279 | "source": [ 280 | "from IPython.display import Image\n", 281 | "from IPython.core.display import HTML\n", 282 | "from urllib.request import urlopen\n", 283 | "import json\n", 284 | "\n", 285 | "def display_photo(photo_id):\n", 286 | " # Proxy for the Unsplash API so that I don't expose my access key\n", 287 | " unsplash_api_url = f\"https://haltakov.net/unsplash-proxy/{photo_id}\"\n", 288 | " \n", 289 | " # Alternatively, you can use your own Unsplash developer account with this code\n", 290 | " # unsplash_api_url = f\"https://api.unsplash.com/photos/{photo_id}?client_id=YOUR_UNSPLASH_ACCESS_KEY\"\n", 291 | " \n", 292 | " # Fetch the photo metadata from the Unsplash API\n", 293 | " photo_data = json.loads(urlopen(unsplash_api_url).read().decode(\"utf-8\"))\n", 294 | "\n", 295 | " # Get the URL of the photo resized to have a width of 480px\n", 296 | " photo_image_url = photo_data[\"urls\"][\"raw\"] + \"&w=320\"\n", 297 | "\n", 298 | " # Display the photo\n", 299 | " display(Image(url=photo_image_url))\n", 300 | "\n", 301 | " # Display the attribution text\n", 302 | " display(HTML(f'Photo by {photo_data[\"user\"][\"name\"]} on Unsplash'))\n", 303 | " print()" 304 | ], 305 | "execution_count": 7, 306 | "outputs": [] 307 | }, 308 | { 309 | "cell_type": "markdown", 310 | "metadata": { 311 | "id": "_3ojinZ0JYBC" 312 | }, 313 | "source": [ 314 | "Putting it all together in one function." 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "metadata": { 320 | "id": "LvUcljF5JcRn" 321 | }, 322 | "source": [ 323 | "def search_unslash(search_query, photo_features, photo_ids, results_count=3):\n", 324 | " # Encode the search query\n", 325 | " text_features = encode_search_query(search_query)\n", 326 | "\n", 327 | " # Find the best matches\n", 328 | " best_photo_ids = find_best_matches(text_features, photo_features, photo_ids, results_count)\n", 329 | "\n", 330 | " # Display the best photos\n", 331 | " for photo_id in best_photo_ids:\n", 332 | " display_photo(photo_id)\n" 333 | ], 334 | "execution_count": 8, 335 | "outputs": [] 336 | }, 337 | { 338 | "cell_type": "markdown", 339 | "metadata": { 340 | "id": "xbym_cYJJH6v" 341 | }, 342 | "source": [ 343 | "## Search Unsplash\n", 344 | "\n", 345 | "Now we are ready to search the dataset using natural language. Check out the examples below and feel free to try out your own queries.\n", 346 | "\n", 347 | "> ⚠️ WARNING ⚠️ \n", 348 | "> Since many people are currently using the notebook, it seems that the Unsplash API limit is hit from time to time (even with caching in the proxy). I applied for production status which will solve the problem. In the meantime, you can just try when a new hour starts. Alternatively, you can use your own Unsplash API key." 349 | ] 350 | }, 351 | { 352 | "cell_type": "markdown", 353 | "metadata": { 354 | "id": "-RmOFAq5NtlI" 355 | }, 356 | "source": [ 357 | "### \"Two dogs playing in the snow\"" 358 | ] 359 | }, 360 | { 361 | "cell_type": "code", 362 | "metadata": { 363 | "id": "CF7HuxAlFNXT", 364 | "outputId": "a084af95-42ca-403c-e728-0e30adfc9754", 365 | "colab": { 366 | "base_uri": "https://localhost:8080/", 367 | "height": 776 368 | } 369 | }, 370 | "source": [ 371 | "search_query = \"Two dogs playing in the snow\"\n", 372 | "\n", 373 | "search_unslash(search_query, photo_features, photo_ids, 3)" 374 | ], 375 | "execution_count": 9, 376 | "outputs": [ 377 | { 378 | "output_type": "display_data", 379 | "data": { 380 | "text/html": [ 381 | "" 382 | ], 383 | "text/plain": [ 384 | "" 385 | ] 386 | }, 387 | "metadata": { 388 | "tags": [] 389 | } 390 | }, 391 | { 392 | "output_type": "display_data", 393 | "data": { 394 | "text/html": [ 395 | "Photo by Richard Burlton on Unsplash" 396 | ], 397 | "text/plain": [ 398 | "" 399 | ] 400 | }, 401 | "metadata": { 402 | "tags": [] 403 | } 404 | }, 405 | { 406 | "output_type": "stream", 407 | "text": [ 408 | "\n" 409 | ], 410 | "name": "stdout" 411 | }, 412 | { 413 | "output_type": "display_data", 414 | "data": { 415 | "text/html": [ 416 | "" 417 | ], 418 | "text/plain": [ 419 | "" 420 | ] 421 | }, 422 | "metadata": { 423 | "tags": [] 424 | } 425 | }, 426 | { 427 | "output_type": "display_data", 428 | "data": { 429 | "text/html": [ 430 | "Photo by Karl Anderson on Unsplash" 431 | ], 432 | "text/plain": [ 433 | "" 434 | ] 435 | }, 436 | "metadata": { 437 | "tags": [] 438 | } 439 | }, 440 | { 441 | "output_type": "stream", 442 | "text": [ 443 | "\n" 444 | ], 445 | "name": "stdout" 446 | }, 447 | { 448 | "output_type": "display_data", 449 | "data": { 450 | "text/html": [ 451 | "" 452 | ], 453 | "text/plain": [ 454 | "" 455 | ] 456 | }, 457 | "metadata": { 458 | "tags": [] 459 | } 460 | }, 461 | { 462 | "output_type": "display_data", 463 | "data": { 464 | "text/html": [ 465 | "Photo by Xuecheng Chen on Unsplash" 466 | ], 467 | "text/plain": [ 468 | "" 469 | ] 470 | }, 471 | "metadata": { 472 | "tags": [] 473 | } 474 | }, 475 | { 476 | "output_type": "stream", 477 | "text": [ 478 | "\n" 479 | ], 480 | "name": "stdout" 481 | } 482 | ] 483 | }, 484 | { 485 | "cell_type": "markdown", 486 | "metadata": { 487 | "id": "PtaYocbjN0VQ" 488 | }, 489 | "source": [ 490 | "### \"The word love written on the wall\"" 491 | ] 492 | }, 493 | { 494 | "cell_type": "code", 495 | "metadata": { 496 | "id": "OswqrzaeMy1J", 497 | "outputId": "aa541ae5-cec6-4b82-d484-1ba82e030843", 498 | "colab": { 499 | "base_uri": "https://localhost:8080/", 500 | "height": 743 501 | } 502 | }, 503 | "source": [ 504 | "search_query = \"The word love written on the wall\"\n", 505 | "\n", 506 | "search_unslash(search_query, photo_features, photo_ids, 3)" 507 | ], 508 | "execution_count": 10, 509 | "outputs": [ 510 | { 511 | "output_type": "display_data", 512 | "data": { 513 | "text/html": [ 514 | "" 515 | ], 516 | "text/plain": [ 517 | "" 518 | ] 519 | }, 520 | "metadata": { 521 | "tags": [] 522 | } 523 | }, 524 | { 525 | "output_type": "display_data", 526 | "data": { 527 | "text/html": [ 528 | "Photo by Genton Damian on Unsplash" 529 | ], 530 | "text/plain": [ 531 | "" 532 | ] 533 | }, 534 | "metadata": { 535 | "tags": [] 536 | } 537 | }, 538 | { 539 | "output_type": "stream", 540 | "text": [ 541 | "\n" 542 | ], 543 | "name": "stdout" 544 | }, 545 | { 546 | "output_type": "display_data", 547 | "data": { 548 | "text/html": [ 549 | "" 550 | ], 551 | "text/plain": [ 552 | "" 553 | ] 554 | }, 555 | "metadata": { 556 | "tags": [] 557 | } 558 | }, 559 | { 560 | "output_type": "display_data", 561 | "data": { 562 | "text/html": [ 563 | "Photo by Anna Rozwadowska on Unsplash" 564 | ], 565 | "text/plain": [ 566 | "" 567 | ] 568 | }, 569 | "metadata": { 570 | "tags": [] 571 | } 572 | }, 573 | { 574 | "output_type": "stream", 575 | "text": [ 576 | "\n" 577 | ], 578 | "name": "stdout" 579 | }, 580 | { 581 | "output_type": "display_data", 582 | "data": { 583 | "text/html": [ 584 | "" 585 | ], 586 | "text/plain": [ 587 | "" 588 | ] 589 | }, 590 | "metadata": { 591 | "tags": [] 592 | } 593 | }, 594 | { 595 | "output_type": "display_data", 596 | "data": { 597 | "text/html": [ 598 | "Photo by Jude Beck on Unsplash" 599 | ], 600 | "text/plain": [ 601 | "" 602 | ] 603 | }, 604 | "metadata": { 605 | "tags": [] 606 | } 607 | }, 608 | { 609 | "output_type": "stream", 610 | "text": [ 611 | "\n" 612 | ], 613 | "name": "stdout" 614 | } 615 | ] 616 | }, 617 | { 618 | "cell_type": "markdown", 619 | "metadata": { 620 | "id": "sUdySrczN4ZX" 621 | }, 622 | "source": [ 623 | "### \"The feeling when your program finally works\"" 624 | ] 625 | }, 626 | { 627 | "cell_type": "code", 628 | "metadata": { 629 | "id": "SRyCZMHQMzOP", 630 | "outputId": "72bdfad7-eca5-472a-c10c-90f232da54b9", 631 | "colab": { 632 | "base_uri": "https://localhost:8080/", 633 | "height": 839 634 | } 635 | }, 636 | "source": [ 637 | "search_query = \"The feeling when your program finally works\"\n", 638 | "\n", 639 | "search_unslash(search_query, photo_features, photo_ids, 3)" 640 | ], 641 | "execution_count": 11, 642 | "outputs": [ 643 | { 644 | "output_type": "display_data", 645 | "data": { 646 | "text/html": [ 647 | "" 648 | ], 649 | "text/plain": [ 650 | "" 651 | ] 652 | }, 653 | "metadata": { 654 | "tags": [] 655 | } 656 | }, 657 | { 658 | "output_type": "display_data", 659 | "data": { 660 | "text/html": [ 661 | "Photo by bruce mars on Unsplash" 662 | ], 663 | "text/plain": [ 664 | "" 665 | ] 666 | }, 667 | "metadata": { 668 | "tags": [] 669 | } 670 | }, 671 | { 672 | "output_type": "stream", 673 | "text": [ 674 | "\n" 675 | ], 676 | "name": "stdout" 677 | }, 678 | { 679 | "output_type": "display_data", 680 | "data": { 681 | "text/html": [ 682 | "" 683 | ], 684 | "text/plain": [ 685 | "" 686 | ] 687 | }, 688 | "metadata": { 689 | "tags": [] 690 | } 691 | }, 692 | { 693 | "output_type": "display_data", 694 | "data": { 695 | "text/html": [ 696 | "Photo by LOGAN WEAVER on Unsplash" 697 | ], 698 | "text/plain": [ 699 | "" 700 | ] 701 | }, 702 | "metadata": { 703 | "tags": [] 704 | } 705 | }, 706 | { 707 | "output_type": "stream", 708 | "text": [ 709 | "\n" 710 | ], 711 | "name": "stdout" 712 | }, 713 | { 714 | "output_type": "display_data", 715 | "data": { 716 | "text/html": [ 717 | "" 718 | ], 719 | "text/plain": [ 720 | "" 721 | ] 722 | }, 723 | "metadata": { 724 | "tags": [] 725 | } 726 | }, 727 | { 728 | "output_type": "display_data", 729 | "data": { 730 | "text/html": [ 731 | "Photo by Vasyl Skunziak on Unsplash" 732 | ], 733 | "text/plain": [ 734 | "" 735 | ] 736 | }, 737 | "metadata": { 738 | "tags": [] 739 | } 740 | }, 741 | { 742 | "output_type": "stream", 743 | "text": [ 744 | "\n" 745 | ], 746 | "name": "stdout" 747 | } 748 | ] 749 | }, 750 | { 751 | "cell_type": "markdown", 752 | "metadata": { 753 | "id": "aR4aDfQYN8J1" 754 | }, 755 | "source": [ 756 | "### \"The Syndey Opera House and the Harbour Bridge at night\"" 757 | ] 758 | }, 759 | { 760 | "cell_type": "code", 761 | "metadata": { 762 | "id": "wWkWfHhnMzZe", 763 | "outputId": "eabb87e8-6b6a-428d-cacd-e12c87d372a2", 764 | "colab": { 765 | "base_uri": "https://localhost:8080/", 766 | "height": 720 767 | } 768 | }, 769 | "source": [ 770 | "search_query = \"The Syndey Opera House and the Harbour Bridge at night\"\n", 771 | "\n", 772 | "search_unslash(search_query, photo_features, photo_ids, 3)" 773 | ], 774 | "execution_count": 12, 775 | "outputs": [ 776 | { 777 | "output_type": "display_data", 778 | "data": { 779 | "text/html": [ 780 | "" 781 | ], 782 | "text/plain": [ 783 | "" 784 | ] 785 | }, 786 | "metadata": { 787 | "tags": [] 788 | } 789 | }, 790 | { 791 | "output_type": "display_data", 792 | "data": { 793 | "text/html": [ 794 | "Photo by Dalal Nizam on Unsplash" 795 | ], 796 | "text/plain": [ 797 | "" 798 | ] 799 | }, 800 | "metadata": { 801 | "tags": [] 802 | } 803 | }, 804 | { 805 | "output_type": "stream", 806 | "text": [ 807 | "\n" 808 | ], 809 | "name": "stdout" 810 | }, 811 | { 812 | "output_type": "display_data", 813 | "data": { 814 | "text/html": [ 815 | "" 816 | ], 817 | "text/plain": [ 818 | "" 819 | ] 820 | }, 821 | "metadata": { 822 | "tags": [] 823 | } 824 | }, 825 | { 826 | "output_type": "display_data", 827 | "data": { 828 | "text/html": [ 829 | "Photo by Dalal Nizam on Unsplash" 830 | ], 831 | "text/plain": [ 832 | "" 833 | ] 834 | }, 835 | "metadata": { 836 | "tags": [] 837 | } 838 | }, 839 | { 840 | "output_type": "stream", 841 | "text": [ 842 | "\n" 843 | ], 844 | "name": "stdout" 845 | }, 846 | { 847 | "output_type": "display_data", 848 | "data": { 849 | "text/html": [ 850 | "" 851 | ], 852 | "text/plain": [ 853 | "" 854 | ] 855 | }, 856 | "metadata": { 857 | "tags": [] 858 | } 859 | }, 860 | { 861 | "output_type": "display_data", 862 | "data": { 863 | "text/html": [ 864 | "Photo by Anna Tremewan on Unsplash" 865 | ], 866 | "text/plain": [ 867 | "" 868 | ] 869 | }, 870 | "metadata": { 871 | "tags": [] 872 | } 873 | }, 874 | { 875 | "output_type": "stream", 876 | "text": [ 877 | "\n" 878 | ], 879 | "name": "stdout" 880 | } 881 | ] 882 | } 883 | ] 884 | } 885 | -------------------------------------------------------------------------------- /colab/unsplash-image-search.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "unsplash-image-search.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [ 9 | "Ons94QRxCQR2", 10 | "lJVrkmy6DVj2", 11 | "ujCKerTnFBk4", 12 | "lsdZUxzJK_af" 13 | ] 14 | }, 15 | "kernelspec": { 16 | "name": "python3", 17 | "display_name": "Python 3" 18 | }, 19 | "accelerator": "GPU" 20 | }, 21 | "cells": [ 22 | { 23 | "cell_type": "markdown", 24 | "metadata": { 25 | "id": "97qkK30wAzXb" 26 | }, 27 | "source": [ 28 | "# Unsplash Image Search\n", 29 | "\n", 30 | "Using this notebook you can search for images from the [Unsplash Dataset](https://unsplash.com/data) using natural language queries. The search is powered by OpenAI's [CLIP](https://github.com/openai/CLIP) neural network.\n", 31 | "\n", 32 | "This notebook uses the precomputed feature vectors for almost 2 million images from the full version of the [Unsplash Dataset](https://unsplash.com/data). If you want to compute the features yourself, see [here](https://github.com/haltakov/natural-language-image-search#on-your-machine).\n", 33 | "\n", 34 | "This project was created by [Vladimir Haltakov](https://twitter.com/haltakov) and the full code is open-sourced on [GitHub](https://github.com/haltakov/natural-language-image-search)." 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": { 40 | "id": "Ons94QRxCQR2" 41 | }, 42 | "source": [ 43 | "## Setup Environment\n", 44 | "\n", 45 | "In this section we will setup the environment." 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": { 51 | "id": "DYdIafWsOOUV" 52 | }, 53 | "source": [ 54 | "First we need to install CLIP and then make sure that we have torch 1.7.1 with CUDA support." 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "metadata": { 60 | "id": "djgE7IjbV3sv" 61 | }, 62 | "source": [ 63 | "!pip install git+https://github.com/openai/CLIP.git\n", 64 | "!pip install torch==1.7.1+cu101 torchvision==0.8.2+cu101 -f https://download.pytorch.org/whl/torch_stable.html" 65 | ], 66 | "execution_count": null, 67 | "outputs": [] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": { 72 | "id": "B7_Sk-T7DBEm" 73 | }, 74 | "source": [ 75 | "We can now load the pretrained public CLIP model." 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "metadata": { 81 | "id": "_6FzbzS6W1R5" 82 | }, 83 | "source": [ 84 | "import clip\n", 85 | "import torch\n", 86 | "\n", 87 | "# Load the open CLIP model\n", 88 | "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", 89 | "model, preprocess = clip.load(\"ViT-B/32\", device=device)" 90 | ], 91 | "execution_count": null, 92 | "outputs": [] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": { 97 | "id": "lJVrkmy6DVj2" 98 | }, 99 | "source": [ 100 | "## Download the Precomputed Data\n", 101 | "\n", 102 | "In this section the precomputed feature vectors for all photos are downloaded." 103 | ] 104 | }, 105 | { 106 | "cell_type": "markdown", 107 | "metadata": { 108 | "id": "18alAEjEOdSC" 109 | }, 110 | "source": [ 111 | "In order to compare the photos from the Unsplash dataset to a text query, we need to compute the feature vector of each photo using CLIP. This is a time consuming task, so you can use the feature vectors that I precomputed and uploaded to Google Drive (with the permission from Unsplash). If you want to compute the features yourself, see [here](https://github.com/haltakov/natural-language-image-search#on-your-machine).\n", 112 | "\n", 113 | "We need to download two files:\n", 114 | "* `photo_ids.csv` - a list of the photo IDs for all images in the dataset. The photo ID can be used to get the actual photo from Unsplash.\n", 115 | "* `features.npy` - a matrix containing the precomputed 512 element feature vector for each photo in the dataset.\n", 116 | "\n", 117 | "The files are available on [Google Drive](https://drive.google.com/drive/folders/1WQmedVCDIQKA2R33dkS1f980YsJXRZ-q?usp=sharing)." 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "metadata": { 123 | "id": "BAb15OJQZRkt" 124 | }, 125 | "source": [ 126 | "from pathlib import Path\n", 127 | "\n", 128 | "# Create a folder for the precomputed features\n", 129 | "!mkdir unsplash-dataset\n", 130 | "\n", 131 | "# Download from Github Releases\n", 132 | "if not Path('unsplash-dataset/photo_ids.csv').exists():\n", 133 | " !wget https://github.com/haltakov/natural-language-image-search/releases/download/1.0.0/photo_ids.csv -O unsplash-dataset/photo_ids.csv\n", 134 | "\n", 135 | "if not Path('unsplash-dataset/features.npy').exists():\n", 136 | " !wget https://github.com/haltakov/natural-language-image-search/releases/download/1.0.0/features.npy -O unsplash-dataset/features.npy\n", 137 | " " 138 | ], 139 | "execution_count": null, 140 | "outputs": [] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": { 145 | "id": "TVjuUh6oEtPt" 146 | }, 147 | "source": [ 148 | "After the files are downloaded we need to load them using `pandas` and `numpy`." 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "metadata": { 154 | "id": "GQHcmMo1Ztjz", 155 | "colab": { 156 | "base_uri": "https://localhost:8080/" 157 | }, 158 | "outputId": "8b77c4b9-a194-4c11-fb70-828e6a6cae6b" 159 | }, 160 | "source": [ 161 | "import pandas as pd\n", 162 | "import numpy as np\n", 163 | "\n", 164 | "# Load the photo IDs\n", 165 | "photo_ids = pd.read_csv(\"unsplash-dataset/photo_ids.csv\")\n", 166 | "photo_ids = list(photo_ids['photo_id'])\n", 167 | "\n", 168 | "# Load the features vectors\n", 169 | "photo_features = np.load(\"unsplash-dataset/features.npy\")\n", 170 | "\n", 171 | "# Convert features to Tensors: Float32 on CPU and Float16 on GPU\n", 172 | "if device == \"cpu\":\n", 173 | " photo_features = torch.from_numpy(photo_features).float().to(device)\n", 174 | "else:\n", 175 | " photo_features = torch.from_numpy(photo_features).to(device)\n", 176 | "\n", 177 | "# Print some statistics\n", 178 | "print(f\"Photos loaded: {len(photo_ids)}\")" 179 | ], 180 | "execution_count": 7, 181 | "outputs": [ 182 | { 183 | "output_type": "stream", 184 | "text": [ 185 | "Photos loaded: 1981161\n" 186 | ], 187 | "name": "stdout" 188 | } 189 | ] 190 | }, 191 | { 192 | "cell_type": "markdown", 193 | "metadata": { 194 | "id": "ujCKerTnFBk4" 195 | }, 196 | "source": [ 197 | "## Define Functions\n", 198 | "\n", 199 | "Some important functions for processing the data are defined here.\n", 200 | "\n" 201 | ] 202 | }, 203 | { 204 | "cell_type": "markdown", 205 | "metadata": { 206 | "id": "pYVNtF-JFtfj" 207 | }, 208 | "source": [ 209 | "The `encode_search_query` function takes a text description and encodes it into a feature vector using the CLIP model." 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "metadata": { 215 | "id": "d0hmOh3qbcxK" 216 | }, 217 | "source": [ 218 | "def encode_search_query(search_query):\n", 219 | " with torch.no_grad():\n", 220 | " # Encode and normalize the search query using CLIP\n", 221 | " text_encoded = model.encode_text(clip.tokenize(search_query).to(device))\n", 222 | " text_encoded /= text_encoded.norm(dim=-1, keepdim=True)\n", 223 | "\n", 224 | " # Retrieve the feature vector\n", 225 | " return text_encoded" 226 | ], 227 | "execution_count": 8, 228 | "outputs": [] 229 | }, 230 | { 231 | "cell_type": "markdown", 232 | "metadata": { 233 | "id": "Vh1yyJtEGCAX" 234 | }, 235 | "source": [ 236 | "The `find_best_matches` function compares the text feature vector to the feature vectors of all images and finds the best matches. The function returns the IDs of the best matching photos." 237 | ] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "metadata": { 242 | "id": "3TcmI5KIbe5F" 243 | }, 244 | "source": [ 245 | "def find_best_matches(text_features, photo_features, photo_ids, results_count=3):\n", 246 | " # Compute the similarity between the search query and each photo using the Cosine similarity\n", 247 | " similarities = (photo_features @ text_features.T).squeeze(1)\n", 248 | "\n", 249 | " # Sort the photos by their similarity score\n", 250 | " best_photo_idx = (-similarities).argsort()\n", 251 | "\n", 252 | " # Return the photo IDs of the best matches\n", 253 | " return [photo_ids[i] for i in best_photo_idx[:results_count]]" 254 | ], 255 | "execution_count": 9, 256 | "outputs": [] 257 | }, 258 | { 259 | "cell_type": "markdown", 260 | "metadata": { 261 | "id": "gEmt0F4iHbL0" 262 | }, 263 | "source": [ 264 | "The `display_photo` function displays a photo from Unsplash given its ID and link to the original photo on Unsplash. " 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "metadata": { 270 | "id": "RC4HD8cBYOon" 271 | }, 272 | "source": [ 273 | "from IPython.display import Image\n", 274 | "from IPython.core.display import HTML\n", 275 | "\n", 276 | "def display_photo(photo_id):\n", 277 | " # Get the URL of the photo resized to have a width of 320px\n", 278 | " photo_image_url = f\"https://unsplash.com/photos/{photo_id}/download?w=320\"\n", 279 | "\n", 280 | " # Display the photo\n", 281 | " display(Image(url=photo_image_url))\n", 282 | "\n", 283 | " # Display the attribution text\n", 284 | " display(HTML(f'Photo on Unsplash '))\n", 285 | " print()" 286 | ], 287 | "execution_count": 10, 288 | "outputs": [] 289 | }, 290 | { 291 | "cell_type": "markdown", 292 | "metadata": { 293 | "id": "_3ojinZ0JYBC" 294 | }, 295 | "source": [ 296 | "Putting it all together in one function." 297 | ] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "metadata": { 302 | "id": "LvUcljF5JcRn" 303 | }, 304 | "source": [ 305 | "def search_unslash(search_query, photo_features, photo_ids, results_count=3):\n", 306 | " # Encode the search query\n", 307 | " text_features = encode_search_query(search_query)\n", 308 | "\n", 309 | " # Find the best matches\n", 310 | " best_photo_ids = find_best_matches(text_features, photo_features, photo_ids, results_count)\n", 311 | "\n", 312 | " # Display the best photos\n", 313 | " for photo_id in best_photo_ids:\n", 314 | " display_photo(photo_id)\n" 315 | ], 316 | "execution_count": 11, 317 | "outputs": [] 318 | }, 319 | { 320 | "cell_type": "markdown", 321 | "metadata": { 322 | "id": "xbym_cYJJH6v" 323 | }, 324 | "source": [ 325 | "## Search Unsplash\n", 326 | "\n", 327 | "Now we are ready to search the dataset using natural language. Check out the examples below and feel free to try out your own queries." 328 | ] 329 | }, 330 | { 331 | "cell_type": "markdown", 332 | "metadata": { 333 | "id": "-RmOFAq5NtlI" 334 | }, 335 | "source": [ 336 | "### \"Two dogs playing in the snow\"" 337 | ] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "metadata": { 342 | "id": "CF7HuxAlFNXT", 343 | "colab": { 344 | "base_uri": "https://localhost:8080/", 345 | "height": 779 346 | }, 347 | "outputId": "fb778e0c-d85b-4146-9c7f-ef3cd7781e9a" 348 | }, 349 | "source": [ 350 | "search_query = \"Two dogs playing in the snow\"\n", 351 | "\n", 352 | "search_unslash(search_query, photo_features, photo_ids, 3)" 353 | ], 354 | "execution_count": 21, 355 | "outputs": [ 356 | { 357 | "output_type": "display_data", 358 | "data": { 359 | "text/html": [ 360 | "" 361 | ], 362 | "text/plain": [ 363 | "" 364 | ] 365 | }, 366 | "metadata": {} 367 | }, 368 | { 369 | "output_type": "display_data", 370 | "data": { 371 | "text/html": [ 372 | "Photo on Unsplash " 373 | ], 374 | "text/plain": [ 375 | "" 376 | ] 377 | }, 378 | "metadata": {} 379 | }, 380 | { 381 | "output_type": "stream", 382 | "text": [ 383 | "\n" 384 | ], 385 | "name": "stdout" 386 | }, 387 | { 388 | "output_type": "display_data", 389 | "data": { 390 | "text/html": [ 391 | "" 392 | ], 393 | "text/plain": [ 394 | "" 395 | ] 396 | }, 397 | "metadata": {} 398 | }, 399 | { 400 | "output_type": "display_data", 401 | "data": { 402 | "text/html": [ 403 | "Photo on Unsplash " 404 | ], 405 | "text/plain": [ 406 | "" 407 | ] 408 | }, 409 | "metadata": {} 410 | }, 411 | { 412 | "output_type": "stream", 413 | "text": [ 414 | "\n" 415 | ], 416 | "name": "stdout" 417 | }, 418 | { 419 | "output_type": "display_data", 420 | "data": { 421 | "text/html": [ 422 | "" 423 | ], 424 | "text/plain": [ 425 | "" 426 | ] 427 | }, 428 | "metadata": {} 429 | }, 430 | { 431 | "output_type": "display_data", 432 | "data": { 433 | "text/html": [ 434 | "Photo on Unsplash " 435 | ], 436 | "text/plain": [ 437 | "" 438 | ] 439 | }, 440 | "metadata": {} 441 | }, 442 | { 443 | "output_type": "stream", 444 | "text": [ 445 | "\n" 446 | ], 447 | "name": "stdout" 448 | } 449 | ] 450 | }, 451 | { 452 | "cell_type": "markdown", 453 | "metadata": { 454 | "id": "PtaYocbjN0VQ" 455 | }, 456 | "source": [ 457 | "### \"The word love written on the wall\"" 458 | ] 459 | }, 460 | { 461 | "cell_type": "code", 462 | "metadata": { 463 | "id": "OswqrzaeMy1J", 464 | "colab": { 465 | "base_uri": "https://localhost:8080/", 466 | "height": 557 467 | }, 468 | "outputId": "32abb6c3-5f98-4302-a6d2-5524efdb211f" 469 | }, 470 | "source": [ 471 | "search_query = \"The word love written on the wall\"\n", 472 | "\n", 473 | "search_unslash(search_query, photo_features, photo_ids, 3)" 474 | ], 475 | "execution_count": 13, 476 | "outputs": [ 477 | { 478 | "output_type": "display_data", 479 | "data": { 480 | "text/html": [ 481 | "" 482 | ], 483 | "text/plain": [ 484 | "" 485 | ] 486 | }, 487 | "metadata": {} 488 | }, 489 | { 490 | "output_type": "display_data", 491 | "data": { 492 | "text/html": [ 493 | "Photo on Unsplash " 494 | ], 495 | "text/plain": [ 496 | "" 497 | ] 498 | }, 499 | "metadata": {} 500 | }, 501 | { 502 | "output_type": "stream", 503 | "text": [ 504 | "\n" 505 | ], 506 | "name": "stdout" 507 | }, 508 | { 509 | "output_type": "display_data", 510 | "data": { 511 | "text/html": [ 512 | "" 513 | ], 514 | "text/plain": [ 515 | "" 516 | ] 517 | }, 518 | "metadata": {} 519 | }, 520 | { 521 | "output_type": "display_data", 522 | "data": { 523 | "text/html": [ 524 | "Photo on Unsplash " 525 | ], 526 | "text/plain": [ 527 | "" 528 | ] 529 | }, 530 | "metadata": {} 531 | }, 532 | { 533 | "output_type": "stream", 534 | "text": [ 535 | "\n" 536 | ], 537 | "name": "stdout" 538 | }, 539 | { 540 | "output_type": "display_data", 541 | "data": { 542 | "text/html": [ 543 | "" 544 | ], 545 | "text/plain": [ 546 | "" 547 | ] 548 | }, 549 | "metadata": {} 550 | }, 551 | { 552 | "output_type": "display_data", 553 | "data": { 554 | "text/html": [ 555 | "Photo on Unsplash " 556 | ], 557 | "text/plain": [ 558 | "" 559 | ] 560 | }, 561 | "metadata": {} 562 | }, 563 | { 564 | "output_type": "stream", 565 | "text": [ 566 | "\n" 567 | ], 568 | "name": "stdout" 569 | } 570 | ] 571 | }, 572 | { 573 | "cell_type": "markdown", 574 | "metadata": { 575 | "id": "sUdySrczN4ZX" 576 | }, 577 | "source": [ 578 | "### \"The feeling when your program finally works\"" 579 | ] 580 | }, 581 | { 582 | "cell_type": "code", 583 | "metadata": { 584 | "id": "SRyCZMHQMzOP", 585 | "colab": { 586 | "base_uri": "https://localhost:8080/", 587 | "height": 842 588 | }, 589 | "outputId": "ebab05a9-99f5-4759-a582-230403b50343" 590 | }, 591 | "source": [ 592 | "search_query = \"The feeling when your program finally works\"\n", 593 | "\n", 594 | "search_unslash(search_query, photo_features, photo_ids, 3)" 595 | ], 596 | "execution_count": 14, 597 | "outputs": [ 598 | { 599 | "output_type": "display_data", 600 | "data": { 601 | "text/html": [ 602 | "" 603 | ], 604 | "text/plain": [ 605 | "" 606 | ] 607 | }, 608 | "metadata": {} 609 | }, 610 | { 611 | "output_type": "display_data", 612 | "data": { 613 | "text/html": [ 614 | "Photo on Unsplash " 615 | ], 616 | "text/plain": [ 617 | "" 618 | ] 619 | }, 620 | "metadata": {} 621 | }, 622 | { 623 | "output_type": "stream", 624 | "text": [ 625 | "\n" 626 | ], 627 | "name": "stdout" 628 | }, 629 | { 630 | "output_type": "display_data", 631 | "data": { 632 | "text/html": [ 633 | "" 634 | ], 635 | "text/plain": [ 636 | "" 637 | ] 638 | }, 639 | "metadata": {} 640 | }, 641 | { 642 | "output_type": "display_data", 643 | "data": { 644 | "text/html": [ 645 | "Photo on Unsplash " 646 | ], 647 | "text/plain": [ 648 | "" 649 | ] 650 | }, 651 | "metadata": {} 652 | }, 653 | { 654 | "output_type": "stream", 655 | "text": [ 656 | "\n" 657 | ], 658 | "name": "stdout" 659 | }, 660 | { 661 | "output_type": "display_data", 662 | "data": { 663 | "text/html": [ 664 | "" 665 | ], 666 | "text/plain": [ 667 | "" 668 | ] 669 | }, 670 | "metadata": {} 671 | }, 672 | { 673 | "output_type": "display_data", 674 | "data": { 675 | "text/html": [ 676 | "Photo on Unsplash " 677 | ], 678 | "text/plain": [ 679 | "" 680 | ] 681 | }, 682 | "metadata": {} 683 | }, 684 | { 685 | "output_type": "stream", 686 | "text": [ 687 | "\n" 688 | ], 689 | "name": "stdout" 690 | } 691 | ] 692 | }, 693 | { 694 | "cell_type": "markdown", 695 | "metadata": { 696 | "id": "aR4aDfQYN8J1" 697 | }, 698 | "source": [ 699 | "### \"The Syndey Opera House and the Harbour Bridge at night\"" 700 | ] 701 | }, 702 | { 703 | "cell_type": "code", 704 | "metadata": { 705 | "id": "wWkWfHhnMzZe", 706 | "colab": { 707 | "base_uri": "https://localhost:8080/", 708 | "height": 395 709 | }, 710 | "outputId": "5d1958b6-2b7e-408f-fda8-83f2e2f2fdf6" 711 | }, 712 | "source": [ 713 | "search_query = \"The Syndey Opera House and the Harbour Bridge at night\"\n", 714 | "\n", 715 | "search_unslash(search_query, photo_features, photo_ids, 3)" 716 | ], 717 | "execution_count": 15, 718 | "outputs": [ 719 | { 720 | "output_type": "display_data", 721 | "data": { 722 | "text/html": [ 723 | "" 724 | ], 725 | "text/plain": [ 726 | "" 727 | ] 728 | }, 729 | "metadata": {} 730 | }, 731 | { 732 | "output_type": "display_data", 733 | "data": { 734 | "text/html": [ 735 | "Photo on Unsplash " 736 | ], 737 | "text/plain": [ 738 | "" 739 | ] 740 | }, 741 | "metadata": {} 742 | }, 743 | { 744 | "output_type": "stream", 745 | "text": [ 746 | "\n" 747 | ], 748 | "name": "stdout" 749 | }, 750 | { 751 | "output_type": "display_data", 752 | "data": { 753 | "text/html": [ 754 | "" 755 | ], 756 | "text/plain": [ 757 | "" 758 | ] 759 | }, 760 | "metadata": {} 761 | }, 762 | { 763 | "output_type": "display_data", 764 | "data": { 765 | "text/html": [ 766 | "Photo on Unsplash " 767 | ], 768 | "text/plain": [ 769 | "" 770 | ] 771 | }, 772 | "metadata": {} 773 | }, 774 | { 775 | "output_type": "stream", 776 | "text": [ 777 | "\n" 778 | ], 779 | "name": "stdout" 780 | }, 781 | { 782 | "output_type": "display_data", 783 | "data": { 784 | "text/html": [ 785 | "" 786 | ], 787 | "text/plain": [ 788 | "" 789 | ] 790 | }, 791 | "metadata": {} 792 | }, 793 | { 794 | "output_type": "display_data", 795 | "data": { 796 | "text/html": [ 797 | "Photo on Unsplash " 798 | ], 799 | "text/plain": [ 800 | "" 801 | ] 802 | }, 803 | "metadata": {} 804 | }, 805 | { 806 | "output_type": "stream", 807 | "text": [ 808 | "\n" 809 | ], 810 | "name": "stdout" 811 | } 812 | ] 813 | }, 814 | { 815 | "cell_type": "markdown", 816 | "metadata": { 817 | "id": "lsdZUxzJK_af" 818 | }, 819 | "source": [ 820 | "## Combine Text and Photo Seach Queries\n", 821 | "\n", 822 | "This is another experiment to combine a text query with another photo." 823 | ] 824 | }, 825 | { 826 | "cell_type": "markdown", 827 | "metadata": { 828 | "id": "_Yxox5LGfqEj" 829 | }, 830 | "source": [ 831 | "The idea here is to do a text search for a photo and then modify the search query by adding another photo to the search query in order to transfer some of the photo features to the search.\n", 832 | "\n", 833 | "This works by adding the features of the photo to the features of the text query. The photo features are multiplied with a weight in order to reduce the influence so that the text query is the main source.\n", 834 | "\n", 835 | "The results are somewhat sensitive to the prompt..." 836 | ] 837 | }, 838 | { 839 | "cell_type": "code", 840 | "metadata": { 841 | "id": "ZEBq8_TeUOFm" 842 | }, 843 | "source": [ 844 | "def search_by_text_and_photo(query_text, query_photo_id, photo_weight=0.5):\n", 845 | " # Encode the search query\n", 846 | " text_features = encode_search_query(query_text)\n", 847 | "\n", 848 | " # Find the feature vector for the specified photo ID\n", 849 | " query_photo_index = photo_ids.index(query_photo_id)\n", 850 | " query_photo_features = photo_features[query_photo_index]\n", 851 | "\n", 852 | " # Combine the test and photo queries and normalize again\n", 853 | " search_features = text_features + query_photo_features * photo_weight\n", 854 | " search_features /= search_features.norm(dim=-1, keepdim=True)\n", 855 | "\n", 856 | " # Find the best match\n", 857 | " best_photo_ids = find_best_matches(search_features, photo_features, photo_ids, 1)\n", 858 | "\n", 859 | " # Display the results\n", 860 | " print(\"Test search result\")\n", 861 | " search_unslash(query_text, photo_features, photo_ids, 1)\n", 862 | "\n", 863 | " print(\"Photo query\")\n", 864 | " display(Image(url=f\"https://unsplash.com/photos/{query_photo_id}/download?w=320\"))\n", 865 | "\n", 866 | " print(\"Result for text query + photo query\")\n", 867 | " display_photo(best_photo_ids[0])" 868 | ], 869 | "execution_count": 16, 870 | "outputs": [] 871 | }, 872 | { 873 | "cell_type": "markdown", 874 | "metadata": { 875 | "id": "h1foDoFYgRPm" 876 | }, 877 | "source": [ 878 | "## Results Combining Text and Photo Seach Queries\n", 879 | "\n", 880 | "Now some results for combining text and photo queries" 881 | ] 882 | }, 883 | { 884 | "cell_type": "markdown", 885 | "metadata": { 886 | "id": "g0uuA3aYguZe" 887 | }, 888 | "source": [ 889 | "### Sydney Opera House + night photo" 890 | ] 891 | }, 892 | { 893 | "cell_type": "code", 894 | "metadata": { 895 | "colab": { 896 | "base_uri": "https://localhost:8080/", 897 | "height": 634 898 | }, 899 | "id": "ruf7Tmy9LVor", 900 | "outputId": "9edbcaf3-e7f7-40ed-bbf5-997ede3179b4" 901 | }, 902 | "source": [ 903 | "search_by_text_and_photo(\"Sydney Opera house\", \"HSsOC5nqurA\")" 904 | ], 905 | "execution_count": 17, 906 | "outputs": [ 907 | { 908 | "output_type": "stream", 909 | "text": [ 910 | "Test search result\n" 911 | ], 912 | "name": "stdout" 913 | }, 914 | { 915 | "output_type": "display_data", 916 | "data": { 917 | "text/html": [ 918 | "" 919 | ], 920 | "text/plain": [ 921 | "" 922 | ] 923 | }, 924 | "metadata": {} 925 | }, 926 | { 927 | "output_type": "display_data", 928 | "data": { 929 | "text/html": [ 930 | "Photo on Unsplash " 931 | ], 932 | "text/plain": [ 933 | "" 934 | ] 935 | }, 936 | "metadata": {} 937 | }, 938 | { 939 | "output_type": "stream", 940 | "text": [ 941 | "\n", 942 | "Photo query\n" 943 | ], 944 | "name": "stdout" 945 | }, 946 | { 947 | "output_type": "display_data", 948 | "data": { 949 | "text/html": [ 950 | "" 951 | ], 952 | "text/plain": [ 953 | "" 954 | ] 955 | }, 956 | "metadata": {} 957 | }, 958 | { 959 | "output_type": "stream", 960 | "text": [ 961 | "Result for text query + photo query\n" 962 | ], 963 | "name": "stdout" 964 | }, 965 | { 966 | "output_type": "display_data", 967 | "data": { 968 | "text/html": [ 969 | "" 970 | ], 971 | "text/plain": [ 972 | "" 973 | ] 974 | }, 975 | "metadata": {} 976 | }, 977 | { 978 | "output_type": "display_data", 979 | "data": { 980 | "text/html": [ 981 | "Photo on Unsplash " 982 | ], 983 | "text/plain": [ 984 | "" 985 | ] 986 | }, 987 | "metadata": {} 988 | }, 989 | { 990 | "output_type": "stream", 991 | "text": [ 992 | "\n" 993 | ], 994 | "name": "stdout" 995 | } 996 | ] 997 | }, 998 | { 999 | "cell_type": "markdown", 1000 | "metadata": { 1001 | "id": "z_A9ui7ugyM8" 1002 | }, 1003 | "source": [ 1004 | "### Sydney Opera House + mist photo" 1005 | ] 1006 | }, 1007 | { 1008 | "cell_type": "code", 1009 | "metadata": { 1010 | "colab": { 1011 | "base_uri": "https://localhost:8080/", 1012 | "height": 662 1013 | }, 1014 | "id": "NXba1meDdfAt", 1015 | "outputId": "5748190d-0a2d-4908-fc97-de205fc0ff70" 1016 | }, 1017 | "source": [ 1018 | "search_by_text_and_photo(\"Sydney Opera house\", \"MaerUPAjPbs\")" 1019 | ], 1020 | "execution_count": 18, 1021 | "outputs": [ 1022 | { 1023 | "output_type": "stream", 1024 | "text": [ 1025 | "Test search result\n" 1026 | ], 1027 | "name": "stdout" 1028 | }, 1029 | { 1030 | "output_type": "display_data", 1031 | "data": { 1032 | "text/html": [ 1033 | "" 1034 | ], 1035 | "text/plain": [ 1036 | "" 1037 | ] 1038 | }, 1039 | "metadata": {} 1040 | }, 1041 | { 1042 | "output_type": "display_data", 1043 | "data": { 1044 | "text/html": [ 1045 | "Photo on Unsplash " 1046 | ], 1047 | "text/plain": [ 1048 | "" 1049 | ] 1050 | }, 1051 | "metadata": {} 1052 | }, 1053 | { 1054 | "output_type": "stream", 1055 | "text": [ 1056 | "\n", 1057 | "Photo query\n" 1058 | ], 1059 | "name": "stdout" 1060 | }, 1061 | { 1062 | "output_type": "display_data", 1063 | "data": { 1064 | "text/html": [ 1065 | "" 1066 | ], 1067 | "text/plain": [ 1068 | "" 1069 | ] 1070 | }, 1071 | "metadata": {} 1072 | }, 1073 | { 1074 | "output_type": "stream", 1075 | "text": [ 1076 | "Result for text query + photo query\n" 1077 | ], 1078 | "name": "stdout" 1079 | }, 1080 | { 1081 | "output_type": "display_data", 1082 | "data": { 1083 | "text/html": [ 1084 | "" 1085 | ], 1086 | "text/plain": [ 1087 | "" 1088 | ] 1089 | }, 1090 | "metadata": {} 1091 | }, 1092 | { 1093 | "output_type": "display_data", 1094 | "data": { 1095 | "text/html": [ 1096 | "Photo on Unsplash " 1097 | ], 1098 | "text/plain": [ 1099 | "" 1100 | ] 1101 | }, 1102 | "metadata": {} 1103 | }, 1104 | { 1105 | "output_type": "stream", 1106 | "text": [ 1107 | "\n" 1108 | ], 1109 | "name": "stdout" 1110 | } 1111 | ] 1112 | }, 1113 | { 1114 | "cell_type": "markdown", 1115 | "metadata": { 1116 | "id": "SQcwhrAOg0pg" 1117 | }, 1118 | "source": [ 1119 | "### Sydney Opera House + rain photo" 1120 | ] 1121 | }, 1122 | { 1123 | "cell_type": "code", 1124 | "metadata": { 1125 | "colab": { 1126 | "base_uri": "https://localhost:8080/", 1127 | "height": 1000 1128 | }, 1129 | "id": "rjq8GcBOfNDA", 1130 | "outputId": "4da7b557-8953-4b60-84cd-3551b2bf03d7" 1131 | }, 1132 | "source": [ 1133 | "search_by_text_and_photo(\"Sydney Opera house\", \"1pNBJ2zUfn4\", 0.4)" 1134 | ], 1135 | "execution_count": 19, 1136 | "outputs": [ 1137 | { 1138 | "output_type": "stream", 1139 | "text": [ 1140 | "Test search result\n" 1141 | ], 1142 | "name": "stdout" 1143 | }, 1144 | { 1145 | "output_type": "display_data", 1146 | "data": { 1147 | "text/html": [ 1148 | "" 1149 | ], 1150 | "text/plain": [ 1151 | "" 1152 | ] 1153 | }, 1154 | "metadata": {} 1155 | }, 1156 | { 1157 | "output_type": "display_data", 1158 | "data": { 1159 | "text/html": [ 1160 | "Photo on Unsplash " 1161 | ], 1162 | "text/plain": [ 1163 | "" 1164 | ] 1165 | }, 1166 | "metadata": {} 1167 | }, 1168 | { 1169 | "output_type": "stream", 1170 | "text": [ 1171 | "\n", 1172 | "Photo query\n" 1173 | ], 1174 | "name": "stdout" 1175 | }, 1176 | { 1177 | "output_type": "display_data", 1178 | "data": { 1179 | "text/html": [ 1180 | "" 1181 | ], 1182 | "text/plain": [ 1183 | "" 1184 | ] 1185 | }, 1186 | "metadata": {} 1187 | }, 1188 | { 1189 | "output_type": "stream", 1190 | "text": [ 1191 | "Result for text query + photo query\n" 1192 | ], 1193 | "name": "stdout" 1194 | }, 1195 | { 1196 | "output_type": "display_data", 1197 | "data": { 1198 | "text/html": [ 1199 | "" 1200 | ], 1201 | "text/plain": [ 1202 | "" 1203 | ] 1204 | }, 1205 | "metadata": {} 1206 | }, 1207 | { 1208 | "output_type": "display_data", 1209 | "data": { 1210 | "text/html": [ 1211 | "Photo on Unsplash " 1212 | ], 1213 | "text/plain": [ 1214 | "" 1215 | ] 1216 | }, 1217 | "metadata": {} 1218 | }, 1219 | { 1220 | "output_type": "stream", 1221 | "text": [ 1222 | "\n" 1223 | ], 1224 | "name": "stdout" 1225 | } 1226 | ] 1227 | }, 1228 | { 1229 | "cell_type": "markdown", 1230 | "metadata": { 1231 | "id": "dXXsOKhUg3nU" 1232 | }, 1233 | "source": [ 1234 | "### Sydney Opera House + sea photo" 1235 | ] 1236 | }, 1237 | { 1238 | "cell_type": "code", 1239 | "metadata": { 1240 | "colab": { 1241 | "base_uri": "https://localhost:8080/", 1242 | "height": 607 1243 | }, 1244 | "id": "kQiQ4w-HfeL6", 1245 | "outputId": "6e50498d-9383-470c-c7e0-c20d67a50892" 1246 | }, 1247 | "source": [ 1248 | "search_by_text_and_photo(\"Sydney Opera house\", \"jnBDclcdZ7A\", 0.4)" 1249 | ], 1250 | "execution_count": 20, 1251 | "outputs": [ 1252 | { 1253 | "output_type": "stream", 1254 | "text": [ 1255 | "Test search result\n" 1256 | ], 1257 | "name": "stdout" 1258 | }, 1259 | { 1260 | "output_type": "display_data", 1261 | "data": { 1262 | "text/html": [ 1263 | "" 1264 | ], 1265 | "text/plain": [ 1266 | "" 1267 | ] 1268 | }, 1269 | "metadata": {} 1270 | }, 1271 | { 1272 | "output_type": "display_data", 1273 | "data": { 1274 | "text/html": [ 1275 | "Photo on Unsplash " 1276 | ], 1277 | "text/plain": [ 1278 | "" 1279 | ] 1280 | }, 1281 | "metadata": {} 1282 | }, 1283 | { 1284 | "output_type": "stream", 1285 | "text": [ 1286 | "\n", 1287 | "Photo query\n" 1288 | ], 1289 | "name": "stdout" 1290 | }, 1291 | { 1292 | "output_type": "display_data", 1293 | "data": { 1294 | "text/html": [ 1295 | "" 1296 | ], 1297 | "text/plain": [ 1298 | "" 1299 | ] 1300 | }, 1301 | "metadata": {} 1302 | }, 1303 | { 1304 | "output_type": "stream", 1305 | "text": [ 1306 | "Result for text query + photo query\n" 1307 | ], 1308 | "name": "stdout" 1309 | }, 1310 | { 1311 | "output_type": "display_data", 1312 | "data": { 1313 | "text/html": [ 1314 | "" 1315 | ], 1316 | "text/plain": [ 1317 | "" 1318 | ] 1319 | }, 1320 | "metadata": {} 1321 | }, 1322 | { 1323 | "output_type": "display_data", 1324 | "data": { 1325 | "text/html": [ 1326 | "Photo on Unsplash " 1327 | ], 1328 | "text/plain": [ 1329 | "" 1330 | ] 1331 | }, 1332 | "metadata": {} 1333 | }, 1334 | { 1335 | "output_type": "stream", 1336 | "text": [ 1337 | "\n" 1338 | ], 1339 | "name": "stdout" 1340 | } 1341 | ] 1342 | } 1343 | ] 1344 | } -------------------------------------------------------------------------------- /images/example_dogs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/haltakov/natural-language-image-search/280f4736686b4436aa73c9219a7c4a9669ae78f0/images/example_dogs.png -------------------------------------------------------------------------------- /images/example_feeling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/haltakov/natural-language-image-search/280f4736686b4436aa73c9219a7c4a9669ae78f0/images/example_feeling.png -------------------------------------------------------------------------------- /images/example_love.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/haltakov/natural-language-image-search/280f4736686b4436aa73c9219a7c4a9669ae78f0/images/example_love.png -------------------------------------------------------------------------------- /images/example_sydney.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/haltakov/natural-language-image-search/280f4736686b4436aa73c9219a7c4a9669ae78f0/images/example_sydney.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | backcall==0.2.0 2 | decorator==4.4.2 3 | ipykernel==5.4.3 4 | ipython==7.19.0 5 | ipython-genutils==0.2.0 6 | jedi==0.18.0 7 | jupyter-client==6.1.11 8 | jupyter-core==4.7.0 9 | parso==0.8.1 10 | pexpect==4.8.0 11 | pickleshare==0.7.5 12 | prompt-toolkit==3.0.10 13 | ptyprocess==0.7.0 14 | Pygments==2.7.4 15 | python-dateutil==2.8.1 16 | pyzmq==21.0.1 17 | six==1.15.0 18 | tornado==6.1 19 | traitlets==5.0.5 20 | wcwidth==0.2.5 21 | python-dotenv==0.15.0 22 | ipyplot==1.1.0 23 | torch==1.7.1 24 | torchvision==0.8.2 25 | ftfy==5.8 26 | regex==2020.11.13 27 | tqdm==4.56.0 28 | pandas==1.2.0 29 | fastparquet==0.5.0 30 | pyarrow==2.0.0 31 | -------------------------------------------------------------------------------- /unsplash-dataset/full/features/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/haltakov/natural-language-image-search/280f4736686b4436aa73c9219a7c4a9669ae78f0/unsplash-dataset/full/features/.empty -------------------------------------------------------------------------------- /unsplash-dataset/full/photos/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/haltakov/natural-language-image-search/280f4736686b4436aa73c9219a7c4a9669ae78f0/unsplash-dataset/full/photos/.empty -------------------------------------------------------------------------------- /unsplash-dataset/lite/features/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/haltakov/natural-language-image-search/280f4736686b4436aa73c9219a7c4a9669ae78f0/unsplash-dataset/lite/features/.empty -------------------------------------------------------------------------------- /unsplash-dataset/lite/photos/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/haltakov/natural-language-image-search/280f4736686b4436aa73c9219a7c4a9669ae78f0/unsplash-dataset/lite/photos/.empty -------------------------------------------------------------------------------- /unsplash-proxy/handler.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import logging 3 | from urllib.request import urlopen 4 | 5 | 6 | # Setup the logger 7 | log = logging.getLogger() 8 | log.setLevel(logging.DEBUG) 9 | 10 | 11 | # Get the Unsplash Access Key from the Parameter Store 12 | client = boto3.client("ssm") 13 | access_key = client.get_parameter(Name="UNSPLAH_API_ACCESS_KEY")["Parameter"]["Value"] 14 | 15 | 16 | def get_photo(event, context): 17 | """ Function that proxies the call to the Unsplash API to retrieve the photo meta data """ 18 | 19 | # Get the photo ID from the parameters 20 | photo_id = event["pathParameters"]["id"] 21 | 22 | # Call the Unsplash API to get the photo metadata 23 | photo_data_url = ( 24 | f"https://api.unsplash.com/photos/{photo_id}?client_id={access_key}" 25 | ) 26 | log.debug(f"Fetching URL: {photo_data_url}") 27 | photo_data = urlopen(photo_data_url).read().decode("utf-8") 28 | 29 | # Create the repsonse and return 30 | headers = { 31 | "Access-Control-Allow-Headers": "Content-Type", 32 | "Access-Control-Allow-Origin": "*", 33 | "Access-Control-Allow-Methods": "GET", 34 | } 35 | 36 | return { 37 | "statusCode": 200, 38 | "headers": headers, 39 | "body": photo_data, 40 | } 41 | -------------------------------------------------------------------------------- /unsplash-proxy/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "unsplash-proxy", 3 | "version": "0.1.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "lodash.get": { 8 | "version": "4.4.2", 9 | "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", 10 | "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=", 11 | "dev": true 12 | }, 13 | "lodash.isempty": { 14 | "version": "4.4.0", 15 | "resolved": "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz", 16 | "integrity": "sha1-b4bL7di+TsmHvpqvM8loTbGzHn4=", 17 | "dev": true 18 | }, 19 | "lodash.split": { 20 | "version": "4.4.2", 21 | "resolved": "https://registry.npmjs.org/lodash.split/-/lodash.split-4.4.2.tgz", 22 | "integrity": "sha1-p/e9nzeWi5MSQo4vOsP7MXPEw8Y=", 23 | "dev": true 24 | }, 25 | "serverless-api-gateway-caching": { 26 | "version": "1.6.1", 27 | "resolved": "https://registry.npmjs.org/serverless-api-gateway-caching/-/serverless-api-gateway-caching-1.6.1.tgz", 28 | "integrity": "sha512-BRV8+yPK4o+hfywdRqkeoI94EGoiuES89DXS5UcnVMuTp9WJs1tQfMQL6JkiI871Q1xBctUaE/u3lNlNYKIJKw==", 29 | "dev": true, 30 | "requires": { 31 | "lodash.get": "^4.4.2", 32 | "lodash.isempty": "^4.4.0", 33 | "lodash.split": "^4.4.0" 34 | } 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /unsplash-proxy/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "unsplash-proxy", 3 | "description": "", 4 | "version": "0.1.0", 5 | "dependencies": {}, 6 | "devDependencies": { 7 | "serverless-api-gateway-caching": "^1.6.1" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /unsplash-proxy/serverless.yml: -------------------------------------------------------------------------------- 1 | service: unsplash-proxy 2 | app: unsplash-proxy 3 | org: haltakov 4 | 5 | frameworkVersion: "2" 6 | 7 | provider: 8 | name: aws 9 | runtime: python3.8 10 | iamRoleStatements: 11 | - Effect: Allow 12 | Action: 13 | - ssm:GetParameter 14 | Resource: "arn:aws:ssm:us-east-1:018469183656:parameter/UNSPLAH_API_ACCESS_KEY" 15 | 16 | plugins: 17 | - serverless-api-gateway-caching 18 | custom: 19 | apiGatewayCaching: 20 | enabled: true 21 | 22 | functions: 23 | get_photo: 24 | handler: handler.get_photo 25 | memorySize: 128 26 | events: 27 | - http: 28 | path: get_photo/{id} 29 | method: get 30 | cors: true 31 | caching: 32 | enabled: true 33 | cacheKeyParameters: 34 | - name: request.path.id 35 | --------------------------------------------------------------------------------