├── .gitignore ├── 2-multimodal ├── test │ ├── bird_audio.wav │ ├── dog_audio.wav │ ├── test-cat.jpg │ ├── test-cat.mp4 │ ├── test-dog.jpg │ ├── test-dog.mp4 │ ├── test-meerkat.jpg │ └── test-meerkat.mp4 ├── source │ ├── image │ │ ├── cat1.jpg │ │ ├── cat2.jpg │ │ ├── cat3.jpg │ │ ├── dog1.jpg │ │ ├── dog2.jpg │ │ ├── dog3.jpg │ │ ├── meerkat1.jpg │ │ ├── meerkat2.jpg │ │ └── meerkat3.jpg │ ├── video │ │ ├── cat-clean.mp4 │ │ ├── cat-play.mp4 │ │ ├── meerkat-dig.mp4 │ │ ├── dog-high-five.mp4 │ │ ├── dog-with-stick.mp4 │ │ └── meerkat-watch.mp4 │ └── audio │ │ ├── mixkit-cow-moo-1744.wav │ │ ├── mixkit-dog-barking-twice-1.wav │ │ ├── mixkit-jungle-ape-sound-2419.wav │ │ ├── mixkit-cartoon-kitty-begging-meow-92.wav │ │ ├── mixkit-rooster-crowing-in-the-morning-2462.wav │ │ └── mixkit-little-birds-singing-in-the-trees-17.wav ├── docker-compose.yml ├── 2-multimedia-rag.ipynb ├── 1-multimedia-search-complete.ipynb └── 1-multimedia-search.ipynb ├── 1-intro ├── docker-compose.yml ├── jeopardy_tiny.json ├── 2-query.ipynb ├── 2-query-complete.ipynb ├── 1-load-data.ipynb └── 1-load-data-complete.ipynb └── readme.md /.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | -------------------------------------------------------------------------------- /2-multimodal/test/bird_audio.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/test/bird_audio.wav -------------------------------------------------------------------------------- /2-multimodal/test/dog_audio.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/test/dog_audio.wav -------------------------------------------------------------------------------- /2-multimodal/test/test-cat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/test/test-cat.jpg -------------------------------------------------------------------------------- /2-multimodal/test/test-cat.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/test/test-cat.mp4 -------------------------------------------------------------------------------- /2-multimodal/test/test-dog.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/test/test-dog.jpg -------------------------------------------------------------------------------- /2-multimodal/test/test-dog.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/test/test-dog.mp4 -------------------------------------------------------------------------------- /2-multimodal/source/image/cat1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/image/cat1.jpg -------------------------------------------------------------------------------- /2-multimodal/source/image/cat2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/image/cat2.jpg -------------------------------------------------------------------------------- /2-multimodal/source/image/cat3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/image/cat3.jpg -------------------------------------------------------------------------------- /2-multimodal/source/image/dog1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/image/dog1.jpg -------------------------------------------------------------------------------- /2-multimodal/source/image/dog2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/image/dog2.jpg -------------------------------------------------------------------------------- /2-multimodal/source/image/dog3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/image/dog3.jpg -------------------------------------------------------------------------------- /2-multimodal/test/test-meerkat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/test/test-meerkat.jpg -------------------------------------------------------------------------------- /2-multimodal/test/test-meerkat.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/test/test-meerkat.mp4 -------------------------------------------------------------------------------- /2-multimodal/source/image/meerkat1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/image/meerkat1.jpg -------------------------------------------------------------------------------- /2-multimodal/source/image/meerkat2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/image/meerkat2.jpg -------------------------------------------------------------------------------- /2-multimodal/source/image/meerkat3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/image/meerkat3.jpg -------------------------------------------------------------------------------- /2-multimodal/source/video/cat-clean.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/video/cat-clean.mp4 -------------------------------------------------------------------------------- /2-multimodal/source/video/cat-play.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/video/cat-play.mp4 -------------------------------------------------------------------------------- /2-multimodal/source/video/meerkat-dig.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/video/meerkat-dig.mp4 -------------------------------------------------------------------------------- /2-multimodal/source/video/dog-high-five.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/video/dog-high-five.mp4 -------------------------------------------------------------------------------- /2-multimodal/source/video/dog-with-stick.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/video/dog-with-stick.mp4 -------------------------------------------------------------------------------- /2-multimodal/source/video/meerkat-watch.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/video/meerkat-watch.mp4 -------------------------------------------------------------------------------- /2-multimodal/source/audio/mixkit-cow-moo-1744.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/audio/mixkit-cow-moo-1744.wav -------------------------------------------------------------------------------- /2-multimodal/source/audio/mixkit-dog-barking-twice-1.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/audio/mixkit-dog-barking-twice-1.wav -------------------------------------------------------------------------------- /2-multimodal/source/audio/mixkit-jungle-ape-sound-2419.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/audio/mixkit-jungle-ape-sound-2419.wav -------------------------------------------------------------------------------- /2-multimodal/source/audio/mixkit-cartoon-kitty-begging-meow-92.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/audio/mixkit-cartoon-kitty-begging-meow-92.wav -------------------------------------------------------------------------------- /2-multimodal/source/audio/mixkit-rooster-crowing-in-the-morning-2462.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/audio/mixkit-rooster-crowing-in-the-morning-2462.wav -------------------------------------------------------------------------------- /2-multimodal/source/audio/mixkit-little-birds-singing-in-the-trees-17.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/HEAD/2-multimodal/source/audio/mixkit-little-birds-singing-in-the-trees-17.wav -------------------------------------------------------------------------------- /2-multimodal/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.4' 3 | services: 4 | weaviate: 5 | command: 6 | - --host 7 | - 0.0.0.0 8 | - --port 9 | - '8080' 10 | - --scheme 11 | - http 12 | image: semitechnologies/weaviate:1.23.7 13 | ports: 14 | - 8080:8080 15 | - 50051:50051 16 | restart: on-failure:0 17 | environment: 18 | QUERY_DEFAULTS_LIMIT: 25 19 | AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' 20 | PERSISTENCE_DATA_PATH: '/var/lib/weaviate' 21 | DEFAULT_VECTORIZER_MODULE: 'multi2vec-bind' 22 | ENABLE_MODULES: 'multi2vec-bind' 23 | BIND_INFERENCE_API: 'http://multi2vec-bind:8080' 24 | CLUSTER_HOSTNAME: 'node1' 25 | multi2vec-bind: 26 | mem_limit: 12G 27 | image: semitechnologies/multi2vec-bind:imagebind 28 | environment: 29 | ENABLE_CUDA: '0' 30 | ... -------------------------------------------------------------------------------- /1-intro/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.4' 3 | services: 4 | weaviate: 5 | command: 6 | - --host 7 | - 0.0.0.0 8 | - --port 9 | - '8080' 10 | - --scheme 11 | - http 12 | image: semitechnologies/weaviate:1.24.1 13 | ports: 14 | - 8080:8080 15 | - 50051:50051 16 | volumes: 17 | - weaviate_data:/var/lib/weaviate 18 | restart: on-failure:0 19 | environment: 20 | QUERY_DEFAULTS_LIMIT: 25 21 | AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' 22 | PERSISTENCE_DATA_PATH: '/var/lib/weaviate' 23 | DEFAULT_VECTORIZER_MODULE: 'none' 24 | ENABLE_MODULES: 'text2vec-cohere,text2vec-huggingface,text2vec-palm,text2vec-openai,generative-openai,generative-cohere,generative-palm,ref2vec-centroid,reranker-cohere,qna-openai' 25 | CLUSTER_HOSTNAME: 'node1' 26 | volumes: 27 | weaviate_data: 28 | ... -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | 2 | # Instructions on how to run the multimodal part of the workshop with GitHub codespaces 3 | 4 | 1. Login to your GitHub account. 5 | 1. Go to this repository: [https://github.com/weaviate-tutorials/multimodal-workshop](https://github.com/weaviate-tutorials/multimodal-workshop). 6 | 1. Run the project with Codespaces 7 | * Click on `"Code"` > `Codespaces`. 8 | * You will see a section labeled "Codespaces" with options "+" and "…" to the right. Click `"…"` and then select `"New with Options."` 9 | * Upgrade the Machine type to `4-core` 10 | * Click `"Create codespace."` 11 | * It may take a few minutes for your Codespaces container to start. Once it does, you will see the IDE, terminal, and repository similar to VSCode. 12 | 1. From the terminal in the project, navigate to `2-multimodal`: 13 | ``` 14 | cd 2-multimodal 15 | ``` 16 | and start the docker image defined in `docker-compose.yml` by calling: 17 | ``` 18 | docker compose up 19 | ``` 20 | 1. After a few minutes (depending on the download speed, it might take 5-10 minutes 😅), you should see a message like this: 21 | ``` 22 | multimodal-workshop-multi2vec-bind-1 | INFO: 172.19.0.3:48294 - "GET /meta HTTP/1.1" 200 OK. 23 | ``` 24 | -------------------------------------------------------------------------------- /1-intro/jeopardy_tiny.json: -------------------------------------------------------------------------------- 1 | [{"Category":"SCIENCE","Question":"This organ removes excess glucose from the blood & stores it as glycogen","Answer":"Liver"},{"Category":"ANIMALS","Question":"It's the only living mammal in the order Proboseidea","Answer":"Elephant"},{"Category":"ANIMALS","Question":"The gavial looks very much like a crocodile except for this bodily feature","Answer":"the nose or snout"},{"Category":"ANIMALS","Question":"Weighing around a ton, the eland is the largest species of this animal in Africa","Answer":"Antelope"},{"Category":"ANIMALS","Question":"Heaviest of all poisonous snakes is this North American rattlesnake","Answer":"the diamondback rattler"},{"Category":"SCIENCE","Question":"2000 news: the Gunnison sage grouse isn't just another northern sage grouse, but a new one of this classification","Answer":"species"},{"Category":"SCIENCE","Question":"A metal that is ductile can be pulled into this while cold & under pressure","Answer":"wire"},{"Category":"SCIENCE","Question":"In 1953 Watson & Crick built a model of the molecular structure of this, the gene-carrying substance","Answer":"DNA"},{"Category":"SCIENCE","Question":"Changes in the tropospheric layer of this are what gives us weather","Answer":"the atmosphere"},{"Category":"SCIENCE","Question":"In 70-degree air, a plane traveling at about 1,130 feet per second breaks it","Answer":"Sound barrier"}] -------------------------------------------------------------------------------- /1-intro/2-query.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Time for Search and RAG" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Connect to Weaviate" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import weaviate, os\n", 24 | "\n", 25 | "# Connect with Weaviate Embedded\n", 26 | "client = weaviate.connect_to_embedded(\n", 27 | " version=\"1.23.7\",\n", 28 | " headers={\n", 29 | " \"X-OpenAI-Api-Key\": os.getenv(\"OPENAI_API_KEY\"), # Replace with your inference API key\n", 30 | " })\n", 31 | "\n", 32 | "# Connect to the local instance deployed with Docker Compose\n", 33 | "# client = weaviate.connect_to_local(\n", 34 | "# headers={\n", 35 | "# \"X-OpenAI-Api-Key\": os.getenv(\"OPENAI_API_KEY\"), # Replace with your inference API key\n", 36 | "# }\n", 37 | "# )\n", 38 | "\n", 39 | "client.is_ready()" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "metadata": {}, 45 | "source": [ 46 | "## Vector search\n", 47 | "[Docs - near_text](https://weaviate.io/developers/weaviate/search/similarity#an-input-medium)" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "# client.query." 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": {}, 62 | "source": [ 63 | "## Search with filters\n", 64 | "[Docs - Filters](https://weaviate.io/developers/weaviate/search/filters)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "# client.query." 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "## Hybrid search\n", 81 | "[Docs - hybrid](https://weaviate.io/developers/weaviate/search/hybrid)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "# client.query." 91 | ] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "metadata": {}, 96 | "source": [ 97 | "## Retrieval Augmented Generation!!!\n", 98 | "[Docs - RAG](https://weaviate.io/developers/weaviate/search/generative)" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "# client.generate" 108 | ] 109 | } 110 | ], 111 | "metadata": { 112 | "kernelspec": { 113 | "display_name": "Python 3", 114 | "language": "python", 115 | "name": "python3" 116 | }, 117 | "language_info": { 118 | "name": "python", 119 | "version": "3.11.7" 120 | } 121 | }, 122 | "nbformat": 4, 123 | "nbformat_minor": 2 124 | } 125 | -------------------------------------------------------------------------------- /1-intro/2-query-complete.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Time to Build" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Connect to Weaviate" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import weaviate, os\n", 24 | "\n", 25 | "# Connect with Weaviate Embedded\n", 26 | "# client = weaviate.connect_to_embedded(\n", 27 | "# version=\"1.23.7\",\n", 28 | "# headers={\n", 29 | "# \"X-OpenAI-Api-Key\": os.getenv(\"OPENAI_API_KEY\"), # Replace with your inference API key\n", 30 | "# })\n", 31 | "\n", 32 | "# Connect to a cloud instance of Weaviate (with WCS)\n", 33 | "# client = weaviate.connect_to_wcs(\n", 34 | "# cluster_url=os.getenv(\"WCS_MM_DEMO_URL\"),\n", 35 | "# auth_credentials=weaviate.auth.AuthApiKey(os.getenv(\"WCS_MM_DEMO_KEY\")),\n", 36 | "# headers={\n", 37 | "# \"X-OpenAI-Api-Key\": os.getenv(\"OPENAI_API_KEY\"), # Replace with your inference API key\n", 38 | "# \"X-Cohere-Api-Key\": os.getenv(\"COHERE_API_KEY\"), # Replace with your inference API key\n", 39 | "# }\n", 40 | "# )\n", 41 | "\n", 42 | "# Connect to the local instance deployed with Docker Compose\n", 43 | "client = weaviate.connect_to_local(\n", 44 | " headers={\n", 45 | " \"X-OpenAI-Api-Key\": os.getenv(\"OPENAI_API_KEY\"), # Replace with your inference API key\n", 46 | " }\n", 47 | ")\n", 48 | "\n", 49 | "client.is_ready()" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": {}, 55 | "source": [ 56 | "## Vector search\n", 57 | "[Docs - near_text](https://weaviate.io/developers/weaviate/search/similarity#an-input-medium)" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "questions = client.collections.get(\"Questions\")\n", 67 | "\n", 68 | "response = questions.query.near_text(\n", 69 | " query=\"musical instruments\",\n", 70 | " limit=5\n", 71 | ")\n", 72 | "\n", 73 | "for item in response.objects:\n", 74 | " print(item.properties)" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "## Search with filters\n", 82 | "[Docs - Filters](https://weaviate.io/developers/weaviate/search/filters)" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "import weaviate.classes.query as wq\n", 92 | "questions = client.collections.get(\"Questions\")\n", 93 | "\n", 94 | "response = questions.query.near_text(\n", 95 | " query=\"musical instruments\",\n", 96 | " limit=5,\n", 97 | " filters=wq.Filter.by_property(\"value\").greater_than(500)\n", 98 | ")\n", 99 | "\n", 100 | "for item in response.objects:\n", 101 | " print(item.properties)" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": {}, 107 | "source": [ 108 | "## Hybrid search\n", 109 | "[Docs - hybrid](https://weaviate.io/developers/weaviate/search/hybrid)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": null, 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "questions = client.collections.get(\"Questions\")\n", 119 | "\n", 120 | "response = questions.query.hybrid(\n", 121 | " query=\"musical instruments\",\n", 122 | " alpha=0.7,\n", 123 | " limit=5,\n", 124 | ")\n", 125 | "\n", 126 | "for item in response.objects:\n", 127 | " print(item.properties)" 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "metadata": {}, 133 | "source": [ 134 | "## Retrieval Augmented Generation!!!\n", 135 | "[Docs - RAG](https://weaviate.io/developers/weaviate/search/generative)" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": null, 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [ 144 | "questions = client.collections.get(\"Questions\")\n", 145 | "\n", 146 | "response = questions.generate.near_text(\n", 147 | " query=\"musical instruments\",\n", 148 | " limit=4,\n", 149 | " single_prompt=\"Write a short tweet about: {question}\"\n", 150 | ")\n", 151 | "\n", 152 | "for item in response.objects:\n", 153 | " print(item.properties)\n", 154 | " print(item.generated)\n" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": null, 160 | "metadata": {}, 161 | "outputs": [], 162 | "source": [ 163 | "questions = client.collections.get(\"Questions\")\n", 164 | "\n", 165 | "response = questions.generate.near_text(\n", 166 | " query=\"musical instruments\",\n", 167 | " limit=4,\n", 168 | " grouped_task=\"Explain what this content is about.\"\n", 169 | ")\n", 170 | "\n", 171 | "print (response.generated)" 172 | ] 173 | } 174 | ], 175 | "metadata": { 176 | "kernelspec": { 177 | "display_name": "Python 3", 178 | "language": "python", 179 | "name": "python3" 180 | }, 181 | "language_info": { 182 | "codemirror_mode": { 183 | "name": "ipython", 184 | "version": 3 185 | }, 186 | "file_extension": ".py", 187 | "mimetype": "text/x-python", 188 | "name": "python", 189 | "nbconvert_exporter": "python", 190 | "pygments_lexer": "ipython3", 191 | "version": "3.11.7" 192 | } 193 | }, 194 | "nbformat": 4, 195 | "nbformat_minor": 2 196 | } 197 | -------------------------------------------------------------------------------- /2-multimodal/2-multimedia-rag.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### Dependencies\n", 8 | "\n", 9 | "1. OpenAI Python Library" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "! pip install --upgrade openai" 19 | ] 20 | }, 21 | { 22 | "attachments": {}, 23 | "cell_type": "markdown", 24 | "metadata": {}, 25 | "source": [ 26 | "### Connect to Weaviate" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [ 35 | "import weaviate, json\n", 36 | "\n", 37 | "client = weaviate.connect_to_local()\n", 38 | "\n", 39 | "client.is_ready()" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "metadata": {}, 45 | "source": [ 46 | "## Multimodal RAG\n", 47 | "\n", 48 | "> Retreive Image → Pass to LMM (Large Multimodal Model) → Get Text/Image Output" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "### Step 1 – Retrieve content from the database with a query" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "from IPython.display import Image\n", 65 | "import weaviate.classes.query as wq\n", 66 | "\n", 67 | "animals = client.collections.get(\"Animals\")\n", 68 | "\n", 69 | "def retrieve_image(query):\n", 70 | " response = animals.query.near_text(\n", 71 | " query=query,\n", 72 | " filters=wq.Filter.by_property(\"mediaType\").equal(\"image\"),\n", 73 | " return_properties=['name','path','mediaType','image'],\n", 74 | " limit = 1,\n", 75 | " )\n", 76 | " result = response.objects[0].properties\n", 77 | "\n", 78 | " print(\"Retrieved image object:\",json.dumps(result, indent=2))\n", 79 | "\n", 80 | " return result\n", 81 | "\n", 82 | "# response = retrieve_image(\"animal on a log\")\n", 83 | "response = retrieve_image(\"dog with a sign\")\n", 84 | "\n", 85 | "SOURCE_IMAGE = response['image']\n", 86 | "\n", 87 | "Image(response['path'])" 88 | ] 89 | }, 90 | { 91 | "cell_type": "markdown", 92 | "metadata": {}, 93 | "source": [ 94 | "### Step 2 - generate a description of the image" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": null, 100 | "metadata": {}, 101 | "outputs": [], 102 | "source": [ 103 | "import requests\n", 104 | "import openai, os\n", 105 | "\n", 106 | "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n", 107 | "\n", 108 | "def generate_description_from_image_gpt4(prompt, image64):\n", 109 | " headers = {\n", 110 | " \"Content-Type\": \"application/json\",\n", 111 | " \"Authorization\": f\"Bearer {openai.api_key}\"\n", 112 | " }\n", 113 | "\n", 114 | " payload = {\n", 115 | " \"model\": \"gpt-4-vision-preview\",\n", 116 | " \"messages\": [\n", 117 | " {\n", 118 | " \"role\": \"user\",\n", 119 | " \"content\": [\n", 120 | " {\n", 121 | " \"type\": \"text\",\n", 122 | " \"text\": prompt\n", 123 | " },\n", 124 | " {\n", 125 | " \"type\": \"image_url\",\n", 126 | " \"image_url\": {\n", 127 | " # \"url\": f\"data:image/jpeg;base64,{response.objects[0].properties['image']}\" #base64 encoded image from Weaviate\n", 128 | " \"url\": f\"data:image/jpeg;base64,{image64}\" #base64 encoded image from Weaviate\n", 129 | " }\n", 130 | " }\n", 131 | " ]\n", 132 | " }\n", 133 | " ],\n", 134 | " \"max_tokens\": 300\n", 135 | " }\n", 136 | "\n", 137 | " response_oai = requests.post(\"https://api.openai.com/v1/chat/completions\", headers=headers, json=payload)\n", 138 | "\n", 139 | " result = response_oai.json()['choices'][0]['message']['content']\n", 140 | " print(f\"Generated description: {result}\")\n", 141 | "\n", 142 | " return result\n", 143 | "\n", 144 | "\n", 145 | "GENERATED_DESCRIPTION = generate_description_from_image_gpt4(\n", 146 | " prompt=\"This is an image of my pet, please give me a cute and vivid description.\",\n", 147 | " image64=SOURCE_IMAGE\n", 148 | ")" 149 | ] 150 | }, 151 | { 152 | "cell_type": "markdown", 153 | "metadata": {}, 154 | "source": [ 155 | "### Step 3 - use the image description to generate a new image with DALL·E 3" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "metadata": {}, 162 | "outputs": [], 163 | "source": [ 164 | "from openai import OpenAI\n", 165 | "\n", 166 | "def generate_image_dalee3(prompt):\n", 167 | " openai_client = OpenAI()\n", 168 | "\n", 169 | " response_oai = openai_client.images.generate(\n", 170 | " model=\"dall-e-3\",\n", 171 | " prompt=str(prompt),\n", 172 | " size=\"1792x1024\",\n", 173 | " quality=\"standard\",\n", 174 | " n=1,\n", 175 | " )\n", 176 | "\n", 177 | " result = response_oai.data[0].url\n", 178 | " print (f\"Generated image url: {result}\")\n", 179 | "\n", 180 | " return result\n", 181 | "\n", 182 | "image_url = generate_image_dalee3(GENERATED_DESCRIPTION)\n", 183 | "Image(url=image_url)" 184 | ] 185 | }, 186 | { 187 | "cell_type": "markdown", 188 | "metadata": {}, 189 | "source": [ 190 | "## All together" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": null, 196 | "metadata": {}, 197 | "outputs": [], 198 | "source": [ 199 | "# Step 1 - retrieve an image – Weaviate\n", 200 | "retrieved_image = retrieve_image(\"animal on a log\")\n", 201 | "SOURCE_IMAGE = retrieved_image['image']\n", 202 | "\n", 203 | "# Step 2 - generate a description - GPT4\n", 204 | "GENERATED_DESCRIPTION = generate_description_from_image_gpt4(\n", 205 | " prompt=\"This is an image of my pet, please give me a cute and vivid description.\",\n", 206 | " image64=SOURCE_IMAGE\n", 207 | ")\n", 208 | "\n", 209 | "# Step 3 - use the description to generate a new image – DALE-E 3\n", 210 | "GENERATED_IMAGE_URL = generate_image_dalee3(GENERATED_DESCRIPTION)\n", 211 | "\n", 212 | "Image(url=str(GENERATED_IMAGE_URL))" 213 | ] 214 | } 215 | ], 216 | "metadata": { 217 | "kernelspec": { 218 | "display_name": "Python 3 (ipykernel)", 219 | "language": "python", 220 | "name": "python3" 221 | }, 222 | "language_info": { 223 | "codemirror_mode": { 224 | "name": "ipython", 225 | "version": 3 226 | }, 227 | "file_extension": ".py", 228 | "mimetype": "text/x-python", 229 | "name": "python", 230 | "nbconvert_exporter": "python", 231 | "pygments_lexer": "ipython3", 232 | "version": "3.11.7" 233 | } 234 | }, 235 | "nbformat": 4, 236 | "nbformat_minor": 2 237 | } 238 | -------------------------------------------------------------------------------- /1-intro/1-load-data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Initial Setup" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Install Weaviate Python Client v4\n", 15 | "> This notebook was created with Weaviate `1.23.7` and the Weaviate Client `4.4.1`\n", 16 | "\n", 17 | "Run the below command to install the latest version of the Weaviate Python Client v4." 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": null, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "!pip install --pre -I \"weaviate-client==4.4.1\"" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "## Deploy Weaviate\n", 34 | "\n", 35 | "Weaviate offers 3 deployment options:\n", 36 | "* Embedded\n", 37 | "* Self-hosted - with Docker Compose\n", 38 | "* Cloud deployment - [Weaviate Cloud Service](https://console.weaviate.cloud/)" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "# Time to Build" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "## Connect to Weaviate\n", 53 | "\n", 54 | "* If you are new to OpenAI, register at [https://platform.openai.com](https://platform.openai.com/) and head to [https://platform.openai.com/api-keys](https://platform.openai.com/api-keys) to create your API key.\n", 55 | "* If you are new to Cohere, register at [https://cohere.com](https://https://cohere.com) and head to [https://dashboard.cohere.com/api-keys](https://dashboard.cohere.com/api-keys) to create your API key." 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "import weaviate, os\n", 65 | "\n", 66 | "# Connect with Weaviate Embedded\n", 67 | "client = weaviate.connect_to_embedded(\n", 68 | " version=\"1.23.7\", # run Weaviate 1.23.7\n", 69 | " headers={\n", 70 | " \"X-OpenAI-Api-Key\": os.getenv(\"OPENAI_API_KEY\"), # Replace with your inference API key\n", 71 | " \"X-Cohere-Api-Key\": os.getenv(\"COHERE_API_KEY\"), # Replace with your inference API key\n", 72 | " })\n", 73 | "\n", 74 | "# Connect to the local instance deployed with Docker Compose\n", 75 | "# client = weaviate.connect_to_local(\n", 76 | "# headers={\n", 77 | "# \"X-OpenAI-Api-Key\": os.getenv(\"OPENAI_API_KEY\"), # Replace with your inference API key\n", 78 | "# \"X-Cohere-Api-Key\": os.getenv(\"COHERE_API_KEY\"), # Replace with your inference API key\n", 79 | "# }\n", 80 | "# )\n", 81 | "\n", 82 | "client.is_ready()" 83 | ] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "metadata": {}, 88 | "source": [ 89 | "## Create a collection\n", 90 | "[Weaviate Docs - collection creation and configuration](https://weaviate.io/developers/weaviate/configuration/schema-configuration)" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "# Create a collection here - with Cohere as a vectorizer" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "## Import data" 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "metadata": {}, 112 | "source": [ 113 | "### Sample Data" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": null, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "import requests, json\n", 123 | "\n", 124 | "def load_data(path):\n", 125 | " resp = requests.get(path)\n", 126 | " return json.loads(resp.text)\n", 127 | "\n", 128 | "data_10 = load_data(\"https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/main/1-intro/jeopardy_tiny.json\")\n", 129 | "\n", 130 | "print(json.dumps(data_10, indent=2))" 131 | ] 132 | }, 133 | { 134 | "cell_type": "markdown", 135 | "metadata": {}, 136 | "source": [ 137 | "### Insert Many\n", 138 | "[Weaviate Docs - insert many](https://weaviate.io/developers/weaviate/manage-data/import)" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": null, 144 | "metadata": {}, 145 | "outputs": [], 146 | "source": [ 147 | "# Insert data" 148 | ] 149 | }, 150 | { 151 | "cell_type": "markdown", 152 | "metadata": {}, 153 | "source": [ 154 | "### Data preview" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": null, 160 | "metadata": {}, 161 | "outputs": [], 162 | "source": [ 163 | "# Show data preview" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": null, 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "# Show data preview - with vectors" 173 | ] 174 | }, 175 | { 176 | "cell_type": "markdown", 177 | "metadata": {}, 178 | "source": [ 179 | "### A super quick query example" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": null, 185 | "metadata": {}, 186 | "outputs": [], 187 | "source": [ 188 | "questions = client.collections.get(\"Questions\")\n", 189 | "response = questions.query.near_text(\n", 190 | " \"Afrikan animals\",\n", 191 | " # \"Zwierzęta afrykańskie\", #African animals in Polish\n", 192 | " # \"アフリカの動物\", #African animals in Japanese\n", 193 | " limit=2\n", 194 | ")\n", 195 | "\n", 196 | "for item in response.objects:\n", 197 | " print(item.properties)" 198 | ] 199 | }, 200 | { 201 | "cell_type": "markdown", 202 | "metadata": {}, 203 | "source": [ 204 | "## Create a collection with OpenAI and Generative module" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": null, 210 | "metadata": {}, 211 | "outputs": [], 212 | "source": [ 213 | "# new collection with 1k objects and OpenAI vectorizer and generative model" 214 | ] 215 | }, 216 | { 217 | "cell_type": "markdown", 218 | "metadata": {}, 219 | "source": [ 220 | "### Import data - 1k objects" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": null, 226 | "metadata": {}, 227 | "outputs": [], 228 | "source": [ 229 | "data_1k = load_data(\"https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/main/1-intro/jeopardy_1k.json\")\n", 230 | "\n", 231 | "print(json.dumps(data_1k, indent=2))" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": null, 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [ 240 | "# Insert data\n", 241 | "questions = client.collections.get(\"Questions\")\n", 242 | "questions.data.insert_many(data_1k)" 243 | ] 244 | } 245 | ], 246 | "metadata": { 247 | "kernelspec": { 248 | "display_name": "Python 3", 249 | "language": "python", 250 | "name": "python3" 251 | }, 252 | "language_info": { 253 | "codemirror_mode": { 254 | "name": "ipython", 255 | "version": 3 256 | }, 257 | "file_extension": ".py", 258 | "mimetype": "text/x-python", 259 | "name": "python", 260 | "nbconvert_exporter": "python", 261 | "pygments_lexer": "ipython3", 262 | "version": "3.11.7" 263 | } 264 | }, 265 | "nbformat": 4, 266 | "nbformat_minor": 2 267 | } 268 | -------------------------------------------------------------------------------- /1-intro/1-load-data-complete.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Initial Setup" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Install Weaviate Python Client v4\n", 15 | "> This notebook was created with Weaviate `1.23.7` and the Weaviate Client `4.4.0`\n", 16 | "\n", 17 | "Run the below command to install the latest version of the Weaviate Python Client v4." 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": null, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "!pip install --pre -I \"weaviate-client==4.4.1\"" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "## Deploy Weaviate\n", 34 | "\n", 35 | "Weaviate offers 3 deployment options:\n", 36 | "* Embedded\n", 37 | "* Self-hosted - with Docker Compose\n", 38 | "* Cloud deployment - [Weaviate Cloud Service](https://console.weaviate.cloud/)" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "# Time to Build" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "## Connect to Weaviate\n", 53 | "\n", 54 | "* If you are new to OpenAI, register at [https://platform.openai.com](https://platform.openai.com/) and head to [https://platform.openai.com/api-keys](https://platform.openai.com/api-keys) to create your API key.\n", 55 | "* If you are new to Cohere, register at [https://cohere.com](https://https://cohere.com) and head to [https://dashboard.cohere.com/api-keys](https://dashboard.cohere.com/api-keys) to create your API key." 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "import weaviate, os\n", 65 | "\n", 66 | "# Connect with Weaviate Embedded\n", 67 | "# client = weaviate.connect_to_embedded(\n", 68 | "# version=\"1.23.7\",\n", 69 | "# headers={\n", 70 | "# \"X-OpenAI-Api-Key\": os.getenv(\"OPENAI_API_KEY\"), # Replace with your inference API key\n", 71 | "# # \"X-Cohere-Api-Key\": os.getenv(\"COHERE_API_KEY\"), # Replace with your inference API key\n", 72 | "# })\n", 73 | "\n", 74 | "# Connect to a cloud instance of Weaviate (with WCS)\n", 75 | "# client = weaviate.connect_to_wcs(\n", 76 | "# cluster_url=os.getenv(\"WCS_MM_DEMO_URL\"),\n", 77 | "# auth_credentials=weaviate.auth.AuthApiKey(os.getenv(\"WCS_MM_DEMO_KEY\")),\n", 78 | "# headers={\n", 79 | "# \"X-OpenAI-Api-Key\": os.getenv(\"OPENAI_API_KEY\"), # Replace with your inference API key\n", 80 | "# \"X-Cohere-Api-Key\": os.getenv(\"COHERE_API_KEY\"), # Replace with your inference API key\n", 81 | "# }\n", 82 | "# )\n", 83 | "\n", 84 | "# Connect to the local instance deployed with Docker Compose\n", 85 | "client = weaviate.connect_to_local(\n", 86 | " headers={\n", 87 | " \"X-OpenAI-Api-Key\": os.getenv(\"OPENAI_API_KEY\"), # Replace with your inference API key\n", 88 | " \"X-Cohere-Api-Key\": os.getenv(\"COHERE_API_KEY\"), # Replace with your inference API key\n", 89 | " }\n", 90 | ")\n", 91 | "\n", 92 | "client.is_ready()" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": {}, 98 | "source": [ 99 | "## Create a collection\n", 100 | "[Weaviate Docs - collection creation and configuration](https://weaviate.io/developers/weaviate/configuration/schema-configuration)" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "import weaviate.classes as wvc\n", 110 | "\n", 111 | "if client.collections.exists(\"Questions\"):\n", 112 | " client.collections.delete(\"Questions\")\n", 113 | "\n", 114 | "# Create a collection here - with Cohere as a vectorizer\n", 115 | "client.collections.create(\n", 116 | " name=\"Questions\",\n", 117 | " vectorizer_config=wvc.config.Configure.Vectorizer.text2vec_cohere()\n", 118 | ")" 119 | ] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "metadata": {}, 124 | "source": [ 125 | "## Import data" 126 | ] 127 | }, 128 | { 129 | "cell_type": "markdown", 130 | "metadata": {}, 131 | "source": [ 132 | "### Sample Data" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": null, 138 | "metadata": {}, 139 | "outputs": [], 140 | "source": [ 141 | "import requests, json\n", 142 | "\n", 143 | "def load_data(path):\n", 144 | " resp = requests.get(path)\n", 145 | " return json.loads(resp.text)\n", 146 | "\n", 147 | "data_10 = load_data(\"https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/main/1-intro/jeopardy_tiny.json\")\n", 148 | "\n", 149 | "print(json.dumps(data_10, indent=2))" 150 | ] 151 | }, 152 | { 153 | "cell_type": "markdown", 154 | "metadata": {}, 155 | "source": [ 156 | "### Insert Many\n", 157 | "[Weaviate Docs - insert many](https://weaviate.io/developers/weaviate/manage-data/import)" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": null, 163 | "metadata": {}, 164 | "outputs": [], 165 | "source": [ 166 | "# Insert data\n", 167 | "questions = client.collections.get(\"Questions\")\n", 168 | "questions.data.insert_many(data_10)" 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "metadata": {}, 174 | "source": [ 175 | "### Data preview" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": null, 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "# Show data preview\n", 185 | "questions = client.collections.get(\"Questions\")\n", 186 | "response = questions.query.fetch_objects(limit=4)\n", 187 | "\n", 188 | "for item in response.objects:\n", 189 | " print(item.uuid, item.properties)" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": null, 195 | "metadata": {}, 196 | "outputs": [], 197 | "source": [ 198 | "# Show data preview - with vectors\n", 199 | "questions = client.collections.get(\"Questions\")\n", 200 | "response = questions.query.fetch_objects(\n", 201 | " limit=4,\n", 202 | " include_vector=True\n", 203 | ")\n", 204 | "\n", 205 | "for item in response.objects:\n", 206 | " print(item.properties)\n", 207 | " print(item.vector, '\\n')" 208 | ] 209 | }, 210 | { 211 | "cell_type": "markdown", 212 | "metadata": {}, 213 | "source": [ 214 | "### Super quick query example" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": null, 220 | "metadata": {}, 221 | "outputs": [], 222 | "source": [ 223 | "response = questions.query.near_text(\n", 224 | " \"Afrikan animals\",\n", 225 | " # \"Zwierzęta afrykańskie\", #African animals in Polish\n", 226 | " # \"アフリカの動物\", #African animals in Japanese\n", 227 | " limit=2\n", 228 | ")\n", 229 | "\n", 230 | "for item in response.objects:\n", 231 | " print(item.properties)" 232 | ] 233 | }, 234 | { 235 | "cell_type": "markdown", 236 | "metadata": {}, 237 | "source": [ 238 | "## Create a collection with OpenAI and Generative module" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "metadata": {}, 245 | "outputs": [], 246 | "source": [ 247 | "# new collection with 1k objects and OpenAI vectorizer and generative model\n", 248 | "\n", 249 | "import weaviate.classes as wvc\n", 250 | "\n", 251 | "if client.collections.exists(\"Questions\"):\n", 252 | " client.collections.delete(\"Questions\")\n", 253 | "\n", 254 | "# Create a collection here - with Cohere as a vectorizer\n", 255 | "client.collections.create(\n", 256 | " name=\"Questions\",\n", 257 | " vectorizer_config=wvc.config.Configure.Vectorizer.text2vec_openai(),\n", 258 | " generative_config=wvc.config.Configure.Generative.openai(model=\"gpt-4\")\n", 259 | ")" 260 | ] 261 | }, 262 | { 263 | "cell_type": "markdown", 264 | "metadata": {}, 265 | "source": [ 266 | "### Import data - 1k objects" 267 | ] 268 | }, 269 | { 270 | "cell_type": "code", 271 | "execution_count": null, 272 | "metadata": {}, 273 | "outputs": [], 274 | "source": [ 275 | "data_1k = load_data(\"https://raw.githubusercontent.com/weaviate-tutorials/multimodal-workshop/main/1-intro/jeopardy_1k.json\")\n", 276 | "\n", 277 | "print(json.dumps(data_1k, indent=2))" 278 | ] 279 | }, 280 | { 281 | "cell_type": "code", 282 | "execution_count": null, 283 | "metadata": {}, 284 | "outputs": [], 285 | "source": [ 286 | "# Insert data\n", 287 | "questions = client.collections.get(\"Questions\")\n", 288 | "questions.data.insert_many(data_1k)" 289 | ] 290 | }, 291 | { 292 | "cell_type": "markdown", 293 | "metadata": {}, 294 | "source": [ 295 | "### RAG Examples\n", 296 | "\n", 297 | "* `single_prompt` - generate text per returned object\n", 298 | "* `group_task` - generate a single text for all returned objects" 299 | ] 300 | }, 301 | { 302 | "cell_type": "code", 303 | "execution_count": null, 304 | "metadata": {}, 305 | "outputs": [], 306 | "source": [ 307 | "questions = client.collections.get(\"Questions\")\n", 308 | "\n", 309 | "response = questions.generate.near_text(\n", 310 | " query=\"musical instruments\",\n", 311 | " limit=3,\n", 312 | " single_prompt=\"Write a short tweet about: {answer} that would match following description: {question}\"\n", 313 | ")\n", 314 | "\n", 315 | "for item in response.objects:\n", 316 | " print(item.properties)\n", 317 | " print(item.generated, '\\n')" 318 | ] 319 | }, 320 | { 321 | "cell_type": "code", 322 | "execution_count": null, 323 | "metadata": {}, 324 | "outputs": [], 325 | "source": [ 326 | "questions = client.collections.get(\"Questions\")\n", 327 | "\n", 328 | "response = questions.generate.near_text(\n", 329 | " query=\"african animals\",\n", 330 | " limit=4,\n", 331 | " grouped_task=\"Explain what this content is about.\"\n", 332 | ")\n", 333 | "\n", 334 | "print(response.generated)" 335 | ] 336 | } 337 | ], 338 | "metadata": { 339 | "kernelspec": { 340 | "display_name": "Python 3", 341 | "language": "python", 342 | "name": "python3" 343 | }, 344 | "language_info": { 345 | "codemirror_mode": { 346 | "name": "ipython", 347 | "version": 3 348 | }, 349 | "file_extension": ".py", 350 | "mimetype": "text/x-python", 351 | "name": "python", 352 | "nbconvert_exporter": "python", 353 | "pygments_lexer": "ipython3", 354 | "version": "3.11.7" 355 | } 356 | }, 357 | "nbformat": 4, 358 | "nbformat_minor": 2 359 | } 360 | -------------------------------------------------------------------------------- /2-multimodal/1-multimedia-search-complete.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Building MultiModal Search with Vector Databases " 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "This notebook demonstrates how build multi-modal search (image, audio, video) `Meta AI ImageBind` model ([multi2vec-bind](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/multi2vec-bind)).\n", 15 | "\n", 16 | "ImageBind allows us to search through text, images, audio and video files.\n", 17 | "\n", 18 | "This recipe will focus on searching through image, audio and video:\n", 19 | "\n", 20 | "* [text-to-media search](#text-to-media-search) - provide text as input to search through media\n", 21 | "* [image-to-media search](#image-to-media-search) - provide image as input to search through media\n", 22 | "* [audio-to-media search](#audio-to-media-search) - provide audio as input to search through media\n", 23 | "* [video-to-media search](#video-to-media-search) - provide video as input to search through media" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "### Weaviate Setup" 31 | ] 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "metadata": {}, 36 | "source": [ 37 | "The ImageBind model is only available with local Weaviate deployments with Docker or Kubernetes.\n", 38 | "\n", 39 | "ImageBind is not supported with Weaviate Cloud Services (WCS).\n", 40 | "\n", 41 | "### Steps to deploy Weaviate locally with ImageBind\n", 42 | "\n", 43 | "1. Locate a docker compose file.\n", 44 | " There is a prepared docker compose file at `/2-multimodal/docker-compose.yml`, which contains the necessary configuration to run Weaviate with `Meta's ImageBind` model.\n", 45 | "\n", 46 | " Navigate to the multimodal folder:\n", 47 | " ```\n", 48 | " cd 2-multimodal\n", 49 | " ```\n", 50 | "\n", 51 | "2. Run Weaviate & ImageBind with Docker Compose\n", 52 | "\n", 53 | " > If you are new to `Docker Compose`, [here are instructions on how to install it](https://docs.docker.com/compose/install/).\n", 54 | "\n", 55 | " To start the docker image defined in the `docker-compose.yml` file, call:\n", 56 | "\n", 57 | " ```bash\n", 58 | " docker compose up\n", 59 | " ```\n", 60 | " \n", 61 | " > Note #1 - the first time you run the command, Docker will download a ~6GB image.\n", 62 | " \n", 63 | " > Note #2 – after the image is downloaded (or when we restart the image), it usually takes 30-60 seconds for the image to be ready.\n", 64 | "\n", 65 | " > Note #3 – to shut down a running docker image, press CMD+C or CTRL+C.\n" 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": {}, 71 | "source": [ 72 | "### Dependencies\n", 73 | "\n", 74 | " 1. The Weaviate Python Client" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "! pip install --pre -I \"weaviate-client==4.4.1\"" 84 | ] 85 | }, 86 | { 87 | "attachments": {}, 88 | "cell_type": "markdown", 89 | "metadata": {}, 90 | "source": [ 91 | "### Connect to Weaviate" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "import weaviate, os\n", 101 | "\n", 102 | "client = weaviate.connect_to_local()\n", 103 | "\n", 104 | "client.is_ready()" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": null, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "client.get_meta()" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": {}, 119 | "source": [ 120 | "### Create the `Animals` Collection" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": null, 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "import weaviate.classes as wvc\n", 130 | "\n", 131 | "if(client.collections.exists(\"Animals\")):\n", 132 | " client.collections.delete(\"Animals\")\n", 133 | "\n", 134 | "client.collections.create(\n", 135 | " name=\"Animals\",\n", 136 | " vectorizer_config=wvc.config.Configure.Vectorizer.multi2vec_bind(\n", 137 | " audio_fields=[\"audio\"],\n", 138 | " image_fields=[\"image\"],\n", 139 | " video_fields=[\"video\"],\n", 140 | " )\n", 141 | ")" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": 4, 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "import base64\n", 151 | "\n", 152 | "# Helper function to convert a file to base64 representation\n", 153 | "def toBase64(path):\n", 154 | " with open(path, 'rb') as file:\n", 155 | " return base64.b64encode(file.read()).decode('utf-8')\n" 156 | ] 157 | }, 158 | { 159 | "cell_type": "markdown", 160 | "metadata": {}, 161 | "source": [ 162 | "### Insert Images into Weaviate" 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": null, 168 | "metadata": { 169 | "scrolled": true 170 | }, 171 | "outputs": [], 172 | "source": [ 173 | "animals = client.collections.get(\"Animals\")\n", 174 | "\n", 175 | "source = os.listdir(\"./source/image/\")\n", 176 | "items = list()\n", 177 | "\n", 178 | "for name in source:\n", 179 | " \n", 180 | " print(f\"Adding {name}\")\n", 181 | " \n", 182 | " path = \"./source/image/\" + name\n", 183 | " \n", 184 | " items.append({\n", 185 | " \"name\": name, # name of the file\n", 186 | " \"path\": path, # path to the file to display result\n", 187 | " \"image\": toBase64(path), # this gets vectorized - \"image\" was configured in vectorizer_config as the property holding images\n", 188 | " \"mediaType\": \"image\", # a label telling us how to display the resource \n", 189 | " })\n", 190 | "\n", 191 | " # import images in batches of 5\n", 192 | " if (len(items) > 5):\n", 193 | " print(f\"Inserting 5 new image objects.\")\n", 194 | " animals.data.insert_many(items)\n", 195 | " items.clear()\n", 196 | "\n", 197 | "# Insert any remaining items\n", 198 | "if (len(items) > 0):\n", 199 | " print(f\"Inserting remaining ({len(items)}) items.\")\n", 200 | " animals.data.insert_many(items)" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "metadata": {}, 207 | "outputs": [], 208 | "source": [ 209 | "#Object count\n", 210 | "animals = client.collections.get(\"Animals\")\n", 211 | "animals.aggregate.over_all()" 212 | ] 213 | }, 214 | { 215 | "cell_type": "markdown", 216 | "metadata": {}, 217 | "source": [ 218 | "### Insert Audio Files into Weaviate" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": null, 224 | "metadata": {}, 225 | "outputs": [], 226 | "source": [ 227 | "animals = client.collections.get(\"Animals\")\n", 228 | "\n", 229 | "source = os.listdir(\"./source/audio/\")\n", 230 | "items = list()\n", 231 | "\n", 232 | "for name in source:\n", 233 | " print(f\"Adding {name}\")\n", 234 | " \n", 235 | " path = \"./source/audio/\" + name\n", 236 | " items.append({\n", 237 | " \"name\": name,\n", 238 | " \"path\": path,\n", 239 | " \"audio\": toBase64(path),\n", 240 | " \"mediaType\": \"audio\"\n", 241 | " })\n", 242 | "\n", 243 | " # import images in batches of 3\n", 244 | " if(len(items) == 3):\n", 245 | " print(f\"Inserting 3 new audio objects.\")\n", 246 | " animals.data.insert_many(items)\n", 247 | " items.clear()\n", 248 | "\n", 249 | "# Insert any remaining items\n", 250 | "if (len(items) > 0):\n", 251 | " print(f\"Inserting remaining ({len(items)}) items.\")\n", 252 | " animals.data.insert_many(items)" 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": null, 258 | "metadata": {}, 259 | "outputs": [], 260 | "source": [ 261 | "animals.aggregate.over_all()" 262 | ] 263 | }, 264 | { 265 | "cell_type": "markdown", 266 | "metadata": {}, 267 | "source": [ 268 | "### Insert Video Files into Weaviate" 269 | ] 270 | }, 271 | { 272 | "cell_type": "code", 273 | "execution_count": null, 274 | "metadata": {}, 275 | "outputs": [], 276 | "source": [ 277 | "animals = client.collections.get(\"Animals\")\n", 278 | "\n", 279 | "source = os.listdir(\"./source/video/\")\n", 280 | "\n", 281 | "for name in source:\n", 282 | " print(f\"Adding {name}\")\n", 283 | " \n", 284 | " path = \"./source/video/\" + name\n", 285 | " item = {\n", 286 | " \"name\": name,\n", 287 | " \"path\": path,\n", 288 | " \"video\": toBase64(path),\n", 289 | " \"mediaType\": \"video\"\n", 290 | " }\n", 291 | " \n", 292 | " # insert videos one by one\n", 293 | " animals.data.insert(item)" 294 | ] 295 | }, 296 | { 297 | "cell_type": "code", 298 | "execution_count": null, 299 | "metadata": {}, 300 | "outputs": [], 301 | "source": [ 302 | "animals.aggregate.over_all()" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": null, 308 | "metadata": {}, 309 | "outputs": [], 310 | "source": [ 311 | "agg = animals.aggregate.over_all(\n", 312 | " group_by=\"mediaType\"\n", 313 | ")\n", 314 | "\n", 315 | "for group in agg.groups:\n", 316 | " print(group)\n" 317 | ] 318 | }, 319 | { 320 | "cell_type": "markdown", 321 | "metadata": {}, 322 | "source": [ 323 | "### Check all the media files added to the Vector Database" 324 | ] 325 | }, 326 | { 327 | "cell_type": "code", 328 | "execution_count": null, 329 | "metadata": {}, 330 | "outputs": [], 331 | "source": [ 332 | "itr = animals.iterator(\n", 333 | " return_properties=[\"name\", \"mediaType\"],\n", 334 | " # include_vector=True, # in case you want to see the vectors\n", 335 | ")\n", 336 | "\n", 337 | "for item in itr:\n", 338 | " print(item.properties)" 339 | ] 340 | }, 341 | { 342 | "cell_type": "markdown", 343 | "metadata": {}, 344 | "source": [ 345 | "# Multimodal Search\n", 346 | "## Helper functions" 347 | ] 348 | }, 349 | { 350 | "cell_type": "code", 351 | "execution_count": 15, 352 | "metadata": {}, 353 | "outputs": [], 354 | "source": [ 355 | "# Helper functions to display results\n", 356 | "import json\n", 357 | "from IPython.display import Image, Audio, Video\n", 358 | "\n", 359 | "def json_print(data):\n", 360 | " print(json.dumps(data, indent=2))\n", 361 | "\n", 362 | "def display_media(item):\n", 363 | " path = item[\"path\"]\n", 364 | "\n", 365 | " if(item[\"mediaType\"] == \"image\"):\n", 366 | " display(Image(path))\n", 367 | "\n", 368 | " elif(item[\"mediaType\"] == \"video\"):\n", 369 | " display(Video(path))\n", 370 | " \n", 371 | " elif(item[\"mediaType\"] == \"audio\"):\n", 372 | " display(Audio(path))" 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "execution_count": 26, 378 | "metadata": {}, 379 | "outputs": [], 380 | "source": [ 381 | "import base64, requests\n", 382 | "\n", 383 | "# Helper function – get base64 representation from an online image\n", 384 | "def url_to_base64(url):\n", 385 | " image_response = requests.get(url)\n", 386 | " content = image_response.content\n", 387 | " return base64.b64encode(content).decode('utf-8')\n", 388 | "\n", 389 | "# Helper function - get base64 representation from a local file\n", 390 | "def file_to_base64(path):\n", 391 | " with open(path, 'rb') as file:\n", 392 | " return base64.b64encode(file.read()).decode('utf-8')\n", 393 | "\n", 394 | "# Update the url and path to test\n", 395 | "#test_image_base64 = url_to_base64(\"https://path-to-some-online-image.jpg\")\n", 396 | "#test_file_base64 = file_to_base64(\"./test/meerkat.jpeg\")" 397 | ] 398 | }, 399 | { 400 | "cell_type": "markdown", 401 | "metadata": {}, 402 | "source": [ 403 | "\n", 404 | "### Text to Media Search" 405 | ] 406 | }, 407 | { 408 | "cell_type": "code", 409 | "execution_count": 13, 410 | "metadata": {}, 411 | "outputs": [], 412 | "source": [ 413 | "response = animals.query.near_text(\n", 414 | " query=\"dog with stick\",\n", 415 | " return_properties=['name','path','mediaType'],\n", 416 | " limit=3\n", 417 | ")" 418 | ] 419 | }, 420 | { 421 | "cell_type": "code", 422 | "execution_count": null, 423 | "metadata": {}, 424 | "outputs": [], 425 | "source": [ 426 | "for obj in response.objects:\n", 427 | " json_print(obj.properties)\n", 428 | " display_media(obj.properties)" 429 | ] 430 | }, 431 | { 432 | "cell_type": "markdown", 433 | "metadata": {}, 434 | "source": [ 435 | "\n", 436 | "### Image to Media Search" 437 | ] 438 | }, 439 | { 440 | "cell_type": "code", 441 | "execution_count": null, 442 | "metadata": {}, 443 | "outputs": [], 444 | "source": [ 445 | "Image(\"./test/test-cat.jpg\")" 446 | ] 447 | }, 448 | { 449 | "cell_type": "code", 450 | "execution_count": 18, 451 | "metadata": {}, 452 | "outputs": [], 453 | "source": [ 454 | "response = animals.query.near_image(\n", 455 | " near_image=toBase64(\"./test/test-cat.jpg\"),\n", 456 | " return_properties=['name','path','mediaType'],\n", 457 | " limit=3\n", 458 | ")" 459 | ] 460 | }, 461 | { 462 | "cell_type": "code", 463 | "execution_count": null, 464 | "metadata": {}, 465 | "outputs": [], 466 | "source": [ 467 | "for obj in response.objects:\n", 468 | " json_print(obj.properties)\n", 469 | " display_media(obj.properties)" 470 | ] 471 | }, 472 | { 473 | "cell_type": "markdown", 474 | "metadata": {}, 475 | "source": [ 476 | "\n", 477 | "### Audio to Media Search" 478 | ] 479 | }, 480 | { 481 | "cell_type": "code", 482 | "execution_count": null, 483 | "metadata": {}, 484 | "outputs": [], 485 | "source": [ 486 | "Audio(\"./test/dog_audio.wav\")" 487 | ] 488 | }, 489 | { 490 | "cell_type": "code", 491 | "execution_count": 20, 492 | "metadata": {}, 493 | "outputs": [], 494 | "source": [ 495 | "import weaviate.classes.query as wq\n", 496 | "\n", 497 | "response = animals.query.near_media(\n", 498 | " media=toBase64(\"./test/dog_audio.wav\"),\n", 499 | " media_type=wq.NearMediaType.AUDIO,\n", 500 | " return_properties=['name','path','mediaType'],\n", 501 | " limit=3\n", 502 | ")" 503 | ] 504 | }, 505 | { 506 | "cell_type": "code", 507 | "execution_count": null, 508 | "metadata": {}, 509 | "outputs": [], 510 | "source": [ 511 | "for obj in response.objects:\n", 512 | " json_print(obj.properties)\n", 513 | " display_media(obj.properties)" 514 | ] 515 | }, 516 | { 517 | "cell_type": "markdown", 518 | "metadata": {}, 519 | "source": [ 520 | "\n", 521 | "### Video to Media Search" 522 | ] 523 | }, 524 | { 525 | "cell_type": "code", 526 | "execution_count": null, 527 | "metadata": {}, 528 | "outputs": [], 529 | "source": [ 530 | "Video(\"./test/test-meerkat.mp4\")" 531 | ] 532 | }, 533 | { 534 | "cell_type": "code", 535 | "execution_count": 150, 536 | "metadata": {}, 537 | "outputs": [], 538 | "source": [ 539 | "response = animals.query.near_media(\n", 540 | " media=toBase64(\"./test/test-meerkat.mp4\"),\n", 541 | " media_type=wq.NearMediaType.VIDEO,\n", 542 | " return_properties=['name','path','mediaType'],\n", 543 | " limit=3\n", 544 | ")" 545 | ] 546 | }, 547 | { 548 | "cell_type": "code", 549 | "execution_count": null, 550 | "metadata": {}, 551 | "outputs": [], 552 | "source": [ 553 | "for obj in response.objects:\n", 554 | " json_print(obj.properties)\n", 555 | " display_media(obj.properties)" 556 | ] 557 | } 558 | ], 559 | "metadata": { 560 | "kernelspec": { 561 | "display_name": "Python 3 (ipykernel)", 562 | "language": "python", 563 | "name": "python3" 564 | }, 565 | "language_info": { 566 | "codemirror_mode": { 567 | "name": "ipython", 568 | "version": 3 569 | }, 570 | "file_extension": ".py", 571 | "mimetype": "text/x-python", 572 | "name": "python", 573 | "nbconvert_exporter": "python", 574 | "pygments_lexer": "ipython3", 575 | "version": "3.11.7" 576 | } 577 | }, 578 | "nbformat": 4, 579 | "nbformat_minor": 2 580 | } 581 | -------------------------------------------------------------------------------- /2-multimodal/1-multimedia-search.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Building MultiModal Search with Vector Databases " 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "This notebook demonstrates how build multi-modal search (image, audio, video) `Meta AI ImageBind` model ([multi2vec-bind](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/multi2vec-bind)).\n", 15 | "\n", 16 | "ImageBind allows us to search through text, images, audio and video files.\n", 17 | "\n", 18 | "This recipe will focus on searching through image, audio and video:\n", 19 | "\n", 20 | "* [text-to-media search](#text-to-media-search) - provide text as input to search through media\n", 21 | "* [image-to-media search](#image-to-media-search) - provide image as input to search through media\n", 22 | "* [audio-to-media search](#audio-to-media-search) - provide audio as input to search through media\n", 23 | "* [video-to-media search](#video-to-media-search) - provide video as input to search through media" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "### Weaviate Setup" 31 | ] 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "metadata": {}, 36 | "source": [ 37 | "The ImageBind model is only available with local Weaviate deployments with Docker or Kubernetes.\n", 38 | "\n", 39 | "ImageBind is not supported with Weaviate Cloud Services (WCS).\n", 40 | "\n", 41 | "### Steps to deploy Weaviate locally with ImageBind\n", 42 | "\n", 43 | "1. Locate a docker compose file.\n", 44 | " There is a prepared docker compose file at `/2-multimodal/docker-compose.yml`, which contains the necessary configuration to run Weaviate with `Meta's ImageBind` model.\n", 45 | "\n", 46 | " Navigate to the multimodal folder:\n", 47 | " ```\n", 48 | " cd 2-multimodal\n", 49 | " ```\n", 50 | "\n", 51 | "2. Run Weaviate & ImageBind with Docker Compose\n", 52 | "\n", 53 | " > If you are new to `Docker Compose`, [here are instructions on how to install it](https://docs.docker.com/compose/install/).\n", 54 | "\n", 55 | " To start the docker image defined in the `docker-compose.yml` file, call:\n", 56 | "\n", 57 | " ```bash\n", 58 | " docker compose up\n", 59 | " ```\n", 60 | " \n", 61 | " > Note #1 - the first time you run the command, Docker will download a ~6GB image.\n", 62 | " \n", 63 | " > Note #2 – after the image is downloaded (or when we restart the image), it usually takes 30-60 seconds for the image to be ready.\n", 64 | "\n", 65 | " > Note #3 – to shut down a running docker image, press CMD+C or CTRL+C.\n" 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": {}, 71 | "source": [ 72 | "### Dependencies\n", 73 | "\n", 74 | " 1. The Weaviate Python Client" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "! pip install --pre -I \"weaviate-client==4.4.b1\"" 84 | ] 85 | }, 86 | { 87 | "attachments": {}, 88 | "cell_type": "markdown", 89 | "metadata": {}, 90 | "source": [ 91 | "### Connect to Weaviate" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "import weaviate, os\n", 101 | "import weaviate.classes as wvc\n", 102 | "\n", 103 | "# Connect to your local Docker instance\n", 104 | "# TODO\n", 105 | "client = None\n", 106 | "\n", 107 | "# And check if the connection is ready\n", 108 | "client.is_ready()" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "client.get_meta()" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": {}, 123 | "source": [ 124 | "### Create the `Animals` Collection" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 15, 130 | "metadata": {}, 131 | "outputs": [ 132 | { 133 | "data": { 134 | "text/plain": [ 135 | "" 136 | ] 137 | }, 138 | "execution_count": 15, 139 | "metadata": {}, 140 | "output_type": "execute_result" 141 | } 142 | ], 143 | "source": [ 144 | "# Let's delete the collection in case you want to rerun this code\n", 145 | "if(client.collections.exists(\"Animals\")):\n", 146 | " client.collections.delete(\"Animals\")\n", 147 | "\n", 148 | "# Create a new collection with multi2vec bind\n", 149 | "# use audio, image and video as the media fields\n", 150 | "# TODO" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": 4, 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [ 159 | "import base64\n", 160 | "\n", 161 | "# Helper function to convert a file to base64 representation\n", 162 | "def toBase64(path):\n", 163 | " with open(path, 'rb') as file:\n", 164 | " return base64.b64encode(file.read()).decode('utf-8')\n" 165 | ] 166 | }, 167 | { 168 | "cell_type": "markdown", 169 | "metadata": {}, 170 | "source": [ 171 | "### Insert Images into Weaviate" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": null, 177 | "metadata": { 178 | "scrolled": true 179 | }, 180 | "outputs": [], 181 | "source": [ 182 | "source = os.listdir(\"./source/image/\")\n", 183 | "\n", 184 | "items = list()\n", 185 | "\n", 186 | "for name in source:\n", 187 | " \n", 188 | " print(f\"Adding {name}\")\n", 189 | " \n", 190 | " path = \"./source/image/\" + name\n", 191 | " \n", 192 | " items.append({\n", 193 | " \"name\": name,\n", 194 | " \"path\": path,\n", 195 | " \"image\": toBase64(path),\n", 196 | " \"mediaType\": \"image\"\n", 197 | " })\n", 198 | "\n", 199 | "# TODO - connect get the animals collection\n", 200 | "# and insert the images\n", 201 | "\n", 202 | "\n", 203 | "# it is fine to insert all 9 items in one go\n", 204 | "# but if we deal with bigger objects like videos or audio, then we should load them in smaller batches" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": null, 210 | "metadata": {}, 211 | "outputs": [], 212 | "source": [ 213 | "# Check object count\n", 214 | "animals = client.collections.get(\"Animals\")\n", 215 | "animals.aggregate.over_all()" 216 | ] 217 | }, 218 | { 219 | "cell_type": "markdown", 220 | "metadata": {}, 221 | "source": [ 222 | "### Insert Audio Files into Weaviate" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": null, 228 | "metadata": {}, 229 | "outputs": [], 230 | "source": [ 231 | "animals = client.collections.get(\"Animals\")\n", 232 | "\n", 233 | "source = os.listdir(\"./source/audio/\")\n", 234 | "\n", 235 | "items = []\n", 236 | "counter = 0\n", 237 | "\n", 238 | "for name in source:\n", 239 | " print(f\"Adding {name}\")\n", 240 | " \n", 241 | " path = \"./source/audio/\" + name\n", 242 | " items.append({\n", 243 | " \"name\": name,\n", 244 | " \"path\": path,\n", 245 | " \"audio\": toBase64(path),\n", 246 | " \"mediaType\": \"audio\"\n", 247 | " })\n", 248 | "\n", 249 | " # TODO - load items in batches of 3, and clear\n", 250 | "\n", 251 | "# TODO - Insert any remaining items\n", 252 | "if (len(items) > 0):\n", 253 | " print(f\"Inserting remaining ({len(items)}) items. Total counter: {counter}\")\n", 254 | " # TODO add code here" 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": null, 260 | "metadata": {}, 261 | "outputs": [], 262 | "source": [ 263 | "# Check object count again\n", 264 | "animals.aggregate.over_all()" 265 | ] 266 | }, 267 | { 268 | "cell_type": "markdown", 269 | "metadata": {}, 270 | "source": [ 271 | "### Insert Video Files into Weaviate" 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": null, 277 | "metadata": {}, 278 | "outputs": [], 279 | "source": [ 280 | "animals = client.collections.get(\"Animals\")\n", 281 | "\n", 282 | "source = os.listdir(\"./source/video/\")\n", 283 | "\n", 284 | "for name in source:\n", 285 | " print(f\"Adding {name}\")\n", 286 | " \n", 287 | " path = \"./source/video/\" + name\n", 288 | " item = {\n", 289 | " \"name\": name,\n", 290 | " \"path\": path,\n", 291 | " \"video\": toBase64(path),\n", 292 | " \"mediaType\": \"video\"\n", 293 | " }\n", 294 | " \n", 295 | " # Videos are big, so we should avoid inserting them in bulk\n", 296 | " # TODO: insert video objects one by one" 297 | ] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "execution_count": 137, 302 | "metadata": {}, 303 | "outputs": [ 304 | { 305 | "data": { 306 | "text/plain": [ 307 | "_AggregateReturn(properties={}, total_count=26)" 308 | ] 309 | }, 310 | "execution_count": 137, 311 | "metadata": {}, 312 | "output_type": "execute_result" 313 | } 314 | ], 315 | "source": [ 316 | "# Check the object count again\n", 317 | "animals.aggregate.over_all()" 318 | ] 319 | }, 320 | { 321 | "cell_type": "code", 322 | "execution_count": null, 323 | "metadata": {}, 324 | "outputs": [], 325 | "source": [ 326 | "# TODO - let's group by \"mediaType\" to see how many objects we have per each group\n" 327 | ] 328 | }, 329 | { 330 | "cell_type": "markdown", 331 | "metadata": {}, 332 | "source": [ 333 | "### Check all the media files added to the Vector Database" 334 | ] 335 | }, 336 | { 337 | "cell_type": "code", 338 | "execution_count": null, 339 | "metadata": {}, 340 | "outputs": [], 341 | "source": [ 342 | "itr = animals.iterator(\n", 343 | " return_properties=[\"name\", \"mediaType\"],\n", 344 | " # include_vector=True, # in case you want to see the vectors\n", 345 | ")\n", 346 | "\n", 347 | "for item in itr:\n", 348 | " print(item.properties)" 349 | ] 350 | }, 351 | { 352 | "cell_type": "markdown", 353 | "metadata": {}, 354 | "source": [ 355 | "# Multimodal Search\n", 356 | "\n", 357 | "> Note, this usually should exist in a separe file/flow, as we shouldn't keep collection creation (which is usually a one-off process) and data import – together with queries (this is usually what you use on a daily basis). As to avoid recreating the collection and importing the data.\n", 358 | "\n", 359 | "## Helper functions" 360 | ] 361 | }, 362 | { 363 | "cell_type": "code", 364 | "execution_count": 15, 365 | "metadata": {}, 366 | "outputs": [], 367 | "source": [ 368 | "# Helper functions to display results\n", 369 | "import json\n", 370 | "from IPython.display import Image, Audio, Video\n", 371 | "\n", 372 | "def json_print(data):\n", 373 | " print(json.dumps(data, indent=2))\n", 374 | "\n", 375 | "def display_media(item):\n", 376 | " path = item[\"path\"]\n", 377 | "\n", 378 | " if(item[\"mediaType\"] == \"image\"):\n", 379 | " display(Image(path))\n", 380 | "\n", 381 | " elif(item[\"mediaType\"] == \"video\"):\n", 382 | " display(Video(path))\n", 383 | " \n", 384 | " elif(item[\"mediaType\"] == \"audio\"):\n", 385 | " display(Audio(path))" 386 | ] 387 | }, 388 | { 389 | "cell_type": "code", 390 | "execution_count": 16, 391 | "metadata": {}, 392 | "outputs": [], 393 | "source": [ 394 | "import base64, requests\n", 395 | "\n", 396 | "# Helper function – get base64 representation from an online image\n", 397 | "def url_to_base64(url):\n", 398 | " image_response = requests.get(url)\n", 399 | " content = image_response.content\n", 400 | " return base64.b64encode(content).decode('utf-8')\n", 401 | "\n", 402 | "# Helper function - get base64 representation from a local file\n", 403 | "def file_to_base64(path):\n", 404 | " with open(path, 'rb') as file:\n", 405 | " return base64.b64encode(file.read()).decode('utf-8')\n", 406 | "\n", 407 | "# Update the url and path to test\n", 408 | "#test_image_base64 = url_to_base64(\"https://path-to-some-online-image.jpg\")\n", 409 | "#test_file_base64 = file_to_base64(\"./test/meerkat.jpeg\")" 410 | ] 411 | }, 412 | { 413 | "cell_type": "markdown", 414 | "metadata": {}, 415 | "source": [ 416 | "\n", 417 | "### Text to Media Search" 418 | ] 419 | }, 420 | { 421 | "cell_type": "code", 422 | "execution_count": 13, 423 | "metadata": {}, 424 | "outputs": [], 425 | "source": [ 426 | "# TODO: use near text to search for \"dog with stick\" and return 3 items\n", 427 | "# TODO: make sure we capture the result as \"response\" \n", 428 | "# response = animals.query." 429 | ] 430 | }, 431 | { 432 | "cell_type": "code", 433 | "execution_count": null, 434 | "metadata": {}, 435 | "outputs": [], 436 | "source": [ 437 | "# print results\n", 438 | "for obj in response.objects:\n", 439 | " json_print(obj.properties)\n", 440 | " display_media(obj.properties)" 441 | ] 442 | }, 443 | { 444 | "cell_type": "markdown", 445 | "metadata": {}, 446 | "source": [ 447 | "\n", 448 | "### Image to Media Search" 449 | ] 450 | }, 451 | { 452 | "cell_type": "code", 453 | "execution_count": null, 454 | "metadata": {}, 455 | "outputs": [], 456 | "source": [ 457 | "Image(\"./test/test-cat.jpg\")" 458 | ] 459 | }, 460 | { 461 | "cell_type": "code", 462 | "execution_count": 18, 463 | "metadata": {}, 464 | "outputs": [], 465 | "source": [ 466 | "# TODO: use near image to search with \"./test/test-cat.jpg\" and return 3 items\n", 467 | "# HINT: use toBase64 to convert the file to base64\n", 468 | "# or use Path(\"./your-file-path-here\"),\n", 469 | "# response = animals.query." 470 | ] 471 | }, 472 | { 473 | "cell_type": "code", 474 | "execution_count": null, 475 | "metadata": {}, 476 | "outputs": [], 477 | "source": [ 478 | "for obj in response.objects:\n", 479 | " json_print(obj.properties)\n", 480 | " display_media(obj.properties)" 481 | ] 482 | }, 483 | { 484 | "cell_type": "markdown", 485 | "metadata": {}, 486 | "source": [ 487 | "\n", 488 | "### Audio to Media Search" 489 | ] 490 | }, 491 | { 492 | "cell_type": "code", 493 | "execution_count": null, 494 | "metadata": {}, 495 | "outputs": [], 496 | "source": [ 497 | "Audio(\"./test/dog_audio.wav\")" 498 | ] 499 | }, 500 | { 501 | "cell_type": "code", 502 | "execution_count": 20, 503 | "metadata": {}, 504 | "outputs": [], 505 | "source": [ 506 | "# TODO: use near audio to search with \"./test/dog_audio.wav\" and return 3 items\n", 507 | "# HINT: use toBase64 to convert the file to base64\n", 508 | "\n", 509 | "# response = animals.query.near_audio(\n", 510 | "# )" 511 | ] 512 | }, 513 | { 514 | "cell_type": "code", 515 | "execution_count": null, 516 | "metadata": {}, 517 | "outputs": [], 518 | "source": [ 519 | "for obj in response.objects:\n", 520 | " json_print(obj.properties)\n", 521 | " display_media(obj.properties)" 522 | ] 523 | }, 524 | { 525 | "cell_type": "markdown", 526 | "metadata": {}, 527 | "source": [ 528 | "\n", 529 | "### Video to Media Search" 530 | ] 531 | }, 532 | { 533 | "cell_type": "code", 534 | "execution_count": null, 535 | "metadata": {}, 536 | "outputs": [], 537 | "source": [ 538 | "Video(\"./test/test-meerkat.mp4\")" 539 | ] 540 | }, 541 | { 542 | "cell_type": "code", 543 | "execution_count": 150, 544 | "metadata": {}, 545 | "outputs": [], 546 | "source": [ 547 | "# TODO: use near audio to search with \"./test/meerkat.mp4\" and return 3 items\n", 548 | "# HINT: use toBase64 to convert the file to base64\n", 549 | "\n", 550 | "# response = animals.query.near_video(\n", 551 | "# )" 552 | ] 553 | }, 554 | { 555 | "cell_type": "code", 556 | "execution_count": null, 557 | "metadata": {}, 558 | "outputs": [], 559 | "source": [ 560 | "for obj in response.objects:\n", 561 | " json_print(obj.properties)\n", 562 | " display_media(obj.properties)" 563 | ] 564 | } 565 | ], 566 | "metadata": { 567 | "kernelspec": { 568 | "display_name": "Python 3 (ipykernel)", 569 | "language": "python", 570 | "name": "python3" 571 | }, 572 | "language_info": { 573 | "codemirror_mode": { 574 | "name": "ipython", 575 | "version": 3 576 | }, 577 | "file_extension": ".py", 578 | "mimetype": "text/x-python", 579 | "name": "python", 580 | "nbconvert_exporter": "python", 581 | "pygments_lexer": "ipython3", 582 | "version": "3.11.5" 583 | } 584 | }, 585 | "nbformat": 4, 586 | "nbformat_minor": 2 587 | } 588 | --------------------------------------------------------------------------------