├── README.md ├── elk.ipynb ├── embedded.csv ├── indexMapping.py ├── myntra_products_catalog.csv └── requirements.txt /README.md: -------------------------------------------------------------------------------- 1 | # Advanced-Semantic-Search-intent-based 2 | 3 | Building Intent Based Advanced Semantic Search From Scratch 4 | 5 | Tech Stack: 6 | New json mode of GPT 3.5 Turbo Model 7 | Elasticsearch 8.11 ( which supports vector dimensions <= 4096) 8 | OpenAI’s text-embedding-ada-002 model 9 | 10 | ![image](https://github.com/abidsaudagar/Advanced-Semantic-Search-intent-based-/assets/20873579/4dcdc6e4-a04e-4918-a1c5-c82ab5a92cbc) 11 | 12 | In this video, we are going to build advanced semantic search from scratch by using new updated elastic search and the new json mode of chat GPT. If a user searches for: Blue T shirts for men under 2k. Normal semantic search results might give results which contain other color T-shirts as well. It can contain T-shirts for women as well and it will most probably not understand that we want T-shirts only under the price of 2000. We will see how we can fine-tune these results so that the user will get most accurate products according to his/her query 13 | -------------------------------------------------------------------------------- /elk.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "# pip install pandas, openai, elasticsearch\n", 10 | "import pandas as pd" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 2, 16 | "metadata": {}, 17 | "outputs": [ 18 | { 19 | "data": { 20 | "text/html": [ 21 | "
\n", 22 | "\n", 35 | "\n", 36 | " \n", 37 | " \n", 38 | " \n", 39 | " \n", 40 | " \n", 41 | " \n", 42 | " \n", 43 | " \n", 44 | " \n", 45 | " \n", 46 | " \n", 47 | " \n", 48 | " \n", 49 | " \n", 50 | " \n", 51 | " \n", 52 | " \n", 53 | " \n", 54 | " \n", 55 | " \n", 56 | " \n", 57 | " \n", 58 | " \n", 59 | " \n", 60 | " \n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | "
ProductIDProductNameProductBrandGenderPrice (INR)NumImagesDescriptionPrimaryColor
010017413DKNY Unisex Black & Grey Printed Medium Trolle...DKNYUnisex117457Black and grey printed medium trolley bag, sec...Black
110016283EthnoVogue Women Beige & Grey Made to Measure ...EthnoVogueWomen58107Beige & Grey made to measure kurta with churid...Beige
210009781SPYKAR Women Pink Alexa Super Skinny Fit High-...SPYKARWomen8997Pink coloured wash 5-pocket high-rise cropped ...Pink
310015921Raymond Men Blue Self-Design Single-Breasted B...RaymondMen55995Blue self-design bandhgala suitBlue self-desig...Blue
410017833Parx Men Brown & Off-White Slim Fit Printed Ca...ParxMen7595Brown and off-white printed casual shirt, has ...White
\n", 107 | "
" 108 | ], 109 | "text/plain": [ 110 | " ProductID ProductName ProductBrand \\\n", 111 | "0 10017413 DKNY Unisex Black & Grey Printed Medium Trolle... DKNY \n", 112 | "1 10016283 EthnoVogue Women Beige & Grey Made to Measure ... EthnoVogue \n", 113 | "2 10009781 SPYKAR Women Pink Alexa Super Skinny Fit High-... SPYKAR \n", 114 | "3 10015921 Raymond Men Blue Self-Design Single-Breasted B... Raymond \n", 115 | "4 10017833 Parx Men Brown & Off-White Slim Fit Printed Ca... Parx \n", 116 | "\n", 117 | " Gender Price (INR) NumImages \\\n", 118 | "0 Unisex 11745 7 \n", 119 | "1 Women 5810 7 \n", 120 | "2 Women 899 7 \n", 121 | "3 Men 5599 5 \n", 122 | "4 Men 759 5 \n", 123 | "\n", 124 | " Description PrimaryColor \n", 125 | "0 Black and grey printed medium trolley bag, sec... Black \n", 126 | "1 Beige & Grey made to measure kurta with churid... Beige \n", 127 | "2 Pink coloured wash 5-pocket high-rise cropped ... Pink \n", 128 | "3 Blue self-design bandhgala suitBlue self-desig... Blue \n", 129 | "4 Brown and off-white printed casual shirt, has ... White " 130 | ] 131 | }, 132 | "execution_count": 2, 133 | "metadata": {}, 134 | "output_type": "execute_result" 135 | } 136 | ], 137 | "source": [ 138 | "df = pd.read_csv(\"myntra_products_catalog.csv\")[:500]\n", 139 | "df.head()" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": 3, 145 | "metadata": {}, 146 | "outputs": [ 147 | { 148 | "data": { 149 | "text/plain": [ 150 | "(468, 8)" 151 | ] 152 | }, 153 | "execution_count": 3, 154 | "metadata": {}, 155 | "output_type": "execute_result" 156 | } 157 | ], 158 | "source": [ 159 | "df.dropna(inplace=True)\n", 160 | "df.shape" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 4, 166 | "metadata": {}, 167 | "outputs": [ 168 | { 169 | "data": { 170 | "text/html": [ 171 | "
\n", 172 | "\n", 185 | "\n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | "
ProductIDProductNameProductBrandGenderPrice (INR)NumImagesDescriptionPrimaryColorNameDescription
010017413DKNY Unisex Black & Grey Printed Medium Trolle...DKNYUnisex117457Black and grey printed medium trolley bag, sec...BlackDKNY Unisex Black & Grey Printed Medium Trolle...
110016283EthnoVogue Women Beige & Grey Made to Measure ...EthnoVogueWomen58107Beige & Grey made to measure kurta with churid...BeigeEthnoVogue Women Beige & Grey Made to Measure ...
\n", 227 | "
" 228 | ], 229 | "text/plain": [ 230 | " ProductID ProductName ProductBrand \\\n", 231 | "0 10017413 DKNY Unisex Black & Grey Printed Medium Trolle... DKNY \n", 232 | "1 10016283 EthnoVogue Women Beige & Grey Made to Measure ... EthnoVogue \n", 233 | "\n", 234 | " Gender Price (INR) NumImages \\\n", 235 | "0 Unisex 11745 7 \n", 236 | "1 Women 5810 7 \n", 237 | "\n", 238 | " Description PrimaryColor \\\n", 239 | "0 Black and grey printed medium trolley bag, sec... Black \n", 240 | "1 Beige & Grey made to measure kurta with churid... Beige \n", 241 | "\n", 242 | " NameDescription \n", 243 | "0 DKNY Unisex Black & Grey Printed Medium Trolle... \n", 244 | "1 EthnoVogue Women Beige & Grey Made to Measure ... " 245 | ] 246 | }, 247 | "execution_count": 4, 248 | "metadata": {}, 249 | "output_type": "execute_result" 250 | } 251 | ], 252 | "source": [ 253 | "df[\"NameDescription\"] = df[\"ProductName\"] + df[\"Description\"]\n", 254 | "df.head(2)" 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": 45, 260 | "metadata": {}, 261 | "outputs": [ 262 | { 263 | "name": "stdout", 264 | "output_type": "stream", 265 | "text": [ 266 | "DKNY Unisex Black & Grey Printed Medium Trolley Bag\n", 267 | "Unisex\n", 268 | "11745\n", 269 | " Black\n", 270 | "EthnoVogue Women Beige & Grey Made to Measure Custom Made Kurta Set with Jacket\n", 271 | "Women\n", 272 | "5810\n", 273 | " Beige\n", 274 | "SPYKAR Women Pink Alexa Super Skinny Fit High-Rise Clean Look Stretchable Cropped Jeans\n", 275 | "Women\n", 276 | "899\n", 277 | " Pink\n" 278 | ] 279 | } 280 | ], 281 | "source": [ 282 | "for i in range(3):\n", 283 | " print(df[\"ProductName\"].iloc[i])\n", 284 | " print(df[\"Gender\"].iloc[i])\n", 285 | " print(df[\"Price (INR)\"].iloc[i])\n", 286 | " print(df[\"PrimaryColor\"].iloc[i])" 287 | ] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "execution_count": null, 292 | "metadata": {}, 293 | "outputs": [], 294 | "source": [] 295 | }, 296 | { 297 | "cell_type": "markdown", 298 | "metadata": {}, 299 | "source": [ 300 | "### GET EMBEDDINGS OF TEXT USING OPENAI's text-embedding-ada-002 MODEL" 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "metadata": {}, 306 | "source": [] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": 5, 311 | "metadata": {}, 312 | "outputs": [], 313 | "source": [ 314 | "from openai import OpenAI\n", 315 | "client = OpenAI(api_key=\"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\")\n", 316 | "\n", 317 | "def get_embedding(text, model=\"text-embedding-ada-002\"):\n", 318 | " text = text.replace(\"\\n\", \" \")\n", 319 | " return client.embeddings.create(input = [text], model=model).data[0].embedding\n", 320 | "\n", 321 | "# df['ada_embedding'] = df.combined.apply(lambda x: get_embedding(x, model='text-embedding-ada-002'))\n", 322 | "# df.to_csv('output/embedded_1k_reviews.csv', index=False)\n" 323 | ] 324 | }, 325 | { 326 | "cell_type": "code", 327 | "execution_count": 6, 328 | "metadata": {}, 329 | "outputs": [], 330 | "source": [ 331 | "sample_embedding = get_embedding(\"magic\")" 332 | ] 333 | }, 334 | { 335 | "cell_type": "code", 336 | "execution_count": 7, 337 | "metadata": {}, 338 | "outputs": [ 339 | { 340 | "data": { 341 | "text/plain": [ 342 | "1536" 343 | ] 344 | }, 345 | "execution_count": 7, 346 | "metadata": {}, 347 | "output_type": "execute_result" 348 | } 349 | ], 350 | "source": [ 351 | "len(sample_embedding)" 352 | ] 353 | }, 354 | { 355 | "cell_type": "code", 356 | "execution_count": 52, 357 | "metadata": {}, 358 | "outputs": [], 359 | "source": [ 360 | "# df[\"NameDescriptionVector\"] = df[\"NameDescription\"].apply(lambda x: get_embedding(x, model='text-embedding-ada-002') )\n", 361 | "# df.to_csv(\"embedded.csv\")" 362 | ] 363 | }, 364 | { 365 | "cell_type": "markdown", 366 | "metadata": {}, 367 | "source": [] 368 | }, 369 | { 370 | "cell_type": "markdown", 371 | "metadata": {}, 372 | "source": [ 373 | "### Insert data in elasticsearch index" 374 | ] 375 | }, 376 | { 377 | "cell_type": "code", 378 | "execution_count": 11, 379 | "metadata": {}, 380 | "outputs": [ 381 | { 382 | "data": { 383 | "text/plain": [ 384 | "True" 385 | ] 386 | }, 387 | "execution_count": 11, 388 | "metadata": {}, 389 | "output_type": "execute_result" 390 | } 391 | ], 392 | "source": [ 393 | "# Connecing to Elasticsearch\n", 394 | "from elasticsearch import Elasticsearch\n", 395 | "\n", 396 | "es = Elasticsearch(\n", 397 | " \"https://localhost:9200\",\n", 398 | " basic_auth=(\"elastic\",\"nakVe=bPePShh=hIYXLI\"),\n", 399 | " ca_certs=\"/Users/abidsaudagar/Downloads/elasticsearch-8.11.1/config/certs/http_ca.crt\"\n", 400 | ")\n", 401 | "es.ping()" 402 | ] 403 | }, 404 | { 405 | "cell_type": "code", 406 | "execution_count": 12, 407 | "metadata": {}, 408 | "outputs": [], 409 | "source": [ 410 | "from indexMapping import indexMapping\n", 411 | "import numpy as np" 412 | ] 413 | }, 414 | { 415 | "cell_type": "code", 416 | "execution_count": 56, 417 | "metadata": {}, 418 | "outputs": [ 419 | { 420 | "data": { 421 | "text/plain": [ 422 | "ObjectApiResponse({'acknowledged': True, 'shards_acknowledged': True, 'index': 'my_products'})" 423 | ] 424 | }, 425 | "execution_count": 56, 426 | "metadata": {}, 427 | "output_type": "execute_result" 428 | } 429 | ], 430 | "source": [ 431 | "es.indices.create(index=\"my_products\", mappings=indexMapping)" 432 | ] 433 | }, 434 | { 435 | "cell_type": "code", 436 | "execution_count": 20, 437 | "metadata": {}, 438 | "outputs": [], 439 | "source": [ 440 | "embedding_df = pd.read_csv(\"embedded.csv\",index_col=0)\n", 441 | "# embedding_df['NameDescriptionVector'] = embedding_df.NameDescriptionVector.apply(eval).apply(np.array)" 442 | ] 443 | }, 444 | { 445 | "cell_type": "code", 446 | "execution_count": 71, 447 | "metadata": {}, 448 | "outputs": [ 449 | { 450 | "data": { 451 | "text/plain": [ 452 | "[{'ProductID': 10017413,\n", 453 | " 'ProductName': 'DKNY Unisex Black & Grey Printed Medium Trolley Bag',\n", 454 | " 'ProductBrand': 'DKNY',\n", 455 | " 'Gender': 'Unisex',\n", 456 | " 'Price (INR)': 11745,\n", 457 | " 'NumImages': 7,\n", 458 | " 'Description': 'Black and grey printed medium trolley bag, secured with a TSA lockOne handle on the top and one on the side, has a trolley with a retractable handle on the top and four corner mounted inline skate wheelsOne main zip compartment, zip lining, two compression straps with click clasps, one zip compartment on the flap with three zip pocketsWarranty: 5 yearsWarranty provided by Brand Owner / Manufacturer',\n", 459 | " 'PrimaryColor': ' Black',\n", 460 | " 'NameDescription': 'DKNY Unisex Black & Grey Printed Medium Trolley BagBlack and grey printed medium trolley bag, secured with a TSA lockOne handle on the top and one on the side, has a trolley with a retractable handle on the top and four corner mounted inline skate wheelsOne main zip compartment, zip lining, two compression straps with click clasps, one zip compartment on the flap with three zip pocketsWarranty: 5 yearsWarranty provided by Brand Owner / Manufacturer',\n", 461 | " 'NameDescriptionVector': array([ 0.00081929, 0.00621943, 0.01336323, ..., 0.00440417,\n", 462 | " -0.00277312, -0.02118021])},\n", 463 | " {'ProductID': 10016283,\n", 464 | " 'ProductName': 'EthnoVogue Women Beige & Grey Made to Measure Custom Made Kurta Set with Jacket',\n", 465 | " 'ProductBrand': 'EthnoVogue',\n", 466 | " 'Gender': 'Women',\n", 467 | " 'Price (INR)': 5810,\n", 468 | " 'NumImages': 7,\n", 469 | " 'Description': 'Beige & Grey made to measure kurta with churidar and\\xa0dupattaBeige made to measure calf\\xa0length kurta, has a V-neck, three-quarter sleeves, lightly padded on bust, flared hem, concealed zip closureGrey solid made to measure churidar, drawstring closureGrey net sequined dupatta, has printed tapingWhat is Made to Measure?Customised Kurta Set according to your Bust and Length. So please refer to the Size Chart to pick your perfect size.How to measure bust?Measure under your arms and around your chest to find your bust size in inchesHow to measure Kurta length?Measure from shoulder till barefoot to find kurta length in',\n", 470 | " 'PrimaryColor': ' Beige',\n", 471 | " 'NameDescription': 'EthnoVogue Women Beige & Grey Made to Measure Custom Made Kurta Set with JacketBeige & Grey made to measure kurta with churidar and\\xa0dupattaBeige made to measure calf\\xa0length kurta, has a V-neck, three-quarter sleeves, lightly padded on bust, flared hem, concealed zip closureGrey solid made to measure churidar, drawstring closureGrey net sequined dupatta, has printed tapingWhat is Made to Measure?Customised Kurta Set according to your Bust and Length. So please refer to the Size Chart to pick your perfect size.How to measure bust?Measure under your arms and around your chest to find your bust size in inchesHow to measure Kurta length?Measure from shoulder till barefoot to find kurta length in',\n", 472 | " 'NameDescriptionVector': array([-0.0036404 , -0.01336824, 0.01255703, ..., 0.00684838,\n", 473 | " -0.0009872 , -0.01688126])},\n", 474 | " {'ProductID': 10009781,\n", 475 | " 'ProductName': 'SPYKAR Women Pink Alexa Super Skinny Fit High-Rise Clean Look Stretchable Cropped Jeans',\n", 476 | " 'ProductBrand': 'SPYKAR',\n", 477 | " 'Gender': 'Women',\n", 478 | " 'Price (INR)': 899,\n", 479 | " 'NumImages': 7,\n", 480 | " 'Description': 'Pink coloured wash 5-pocket high-rise cropped jeans, clean look, no fade, has a button and zip closure, and waistband with belt loops',\n", 481 | " 'PrimaryColor': ' Pink',\n", 482 | " 'NameDescription': 'SPYKAR Women Pink Alexa Super Skinny Fit High-Rise Clean Look Stretchable Cropped JeansPink coloured wash 5-pocket high-rise cropped jeans, clean look, no fade, has a button and zip closure, and waistband with belt loops',\n", 483 | " 'NameDescriptionVector': array([-0.02304309, -0.01207571, 0.00012525, ..., 0.01022634,\n", 484 | " -0.00602819, -0.00835119])}]" 485 | ] 486 | }, 487 | "execution_count": 71, 488 | "metadata": {}, 489 | "output_type": "execute_result" 490 | } 491 | ], 492 | "source": [ 493 | "# docs = embedding_df.to_dict(\"records\")\n", 494 | "# docs[:3]" 495 | ] 496 | }, 497 | { 498 | "cell_type": "code", 499 | "execution_count": 72, 500 | "metadata": {}, 501 | "outputs": [], 502 | "source": [ 503 | "for doc in docs:\n", 504 | " try:\n", 505 | " es.index(index=\"my_products\", document=doc, id=doc[\"ProductID\"])\n", 506 | " except Exception as e:\n", 507 | " print(e)" 508 | ] 509 | }, 510 | { 511 | "cell_type": "code", 512 | "execution_count": 13, 513 | "metadata": {}, 514 | "outputs": [ 515 | { 516 | "data": { 517 | "text/plain": [ 518 | "ObjectApiResponse({'count': 468, '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0}})" 519 | ] 520 | }, 521 | "execution_count": 13, 522 | "metadata": {}, 523 | "output_type": "execute_result" 524 | } 525 | ], 526 | "source": [ 527 | "es.count(index=\"my_products\")" 528 | ] 529 | }, 530 | { 531 | "cell_type": "code", 532 | "execution_count": null, 533 | "metadata": {}, 534 | "outputs": [], 535 | "source": [] 536 | }, 537 | { 538 | "cell_type": "markdown", 539 | "metadata": {}, 540 | "source": [ 541 | "### Search data from index" 542 | ] 543 | }, 544 | { 545 | "cell_type": "code", 546 | "execution_count": 14, 547 | "metadata": {}, 548 | "outputs": [], 549 | "source": [ 550 | "input_keyword = \"Blue Shoes for men under 2k\"\n", 551 | "vector_of_input_keyword = get_embedding(input_keyword)" 552 | ] 553 | }, 554 | { 555 | "cell_type": "code", 556 | "execution_count": 15, 557 | "metadata": {}, 558 | "outputs": [ 559 | { 560 | "name": "stderr", 561 | "output_type": "stream", 562 | "text": [ 563 | "/var/folders/3k/901cf0jd1lngqxfl_y527qp00000gn/T/ipykernel_96516/1881674628.py:8: ElasticsearchWarning: The kNN search API has been replaced by the `knn` option in the search API.\n", 564 | " res = es.knn_search(index=\"my_products\", knn=query , source=[\"ProductName\",\"Description\",\"PrimaryColor\",\"Price (INR)\",\"Gender\"])\n" 565 | ] 566 | }, 567 | { 568 | "data": { 569 | "text/plain": [ 570 | "[{'_index': 'my_products',\n", 571 | " '_id': '10005997',\n", 572 | " '_score': 0.7492948,\n", 573 | " '_source': {'ProductName': 'ID Men Navy Blue Solid Leather Mid-Top Sneakers',\n", 574 | " 'Gender': 'Men',\n", 575 | " 'Price (INR)': 1286,\n", 576 | " 'Description': 'A pair of round-toe navy blue sneakers, has mid-top styling, lace-up detailLeather upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 577 | " 'PrimaryColor': 'Blue'}},\n", 578 | " {'_index': 'my_products',\n", 579 | " '_id': '10018075',\n", 580 | " '_score': 0.7472657,\n", 581 | " '_source': {'ProductName': 'Puma Men Blue Sneakers',\n", 582 | " 'Gender': 'Men',\n", 583 | " 'Price (INR)': 1749,\n", 584 | " 'Description': 'A pair of round-toe blue sneakers, has regular styling, lace-up detailTextile upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 585 | " 'PrimaryColor': 'Blue'}},\n", 586 | " {'_index': 'my_products',\n", 587 | " '_id': '10018013',\n", 588 | " '_score': 0.7472377,\n", 589 | " '_source': {'ProductName': 'Puma Men Blue Sneakers',\n", 590 | " 'Gender': 'Men',\n", 591 | " 'Price (INR)': 1799,\n", 592 | " 'Description': 'A pair of round-toe blue sneakers, has regular styling, lace-up detailTextile upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 593 | " 'PrimaryColor': 'Blue'}},\n", 594 | " {'_index': 'my_products',\n", 595 | " '_id': '10006087',\n", 596 | " '_score': 0.74446684,\n", 597 | " '_source': {'ProductName': 'ID Men Navy Blue Leather Derbys',\n", 598 | " 'Gender': 'Men',\n", 599 | " 'Price (INR)': 2685,\n", 600 | " 'Description': 'A pair of round-toe navy blue derbys, has regular styling, lace-up detailLeather upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 601 | " 'PrimaryColor': 'Blue'}},\n", 602 | " {'_index': 'my_products',\n", 603 | " '_id': '10006031',\n", 604 | " '_score': 0.73941416,\n", 605 | " '_source': {'ProductName': 'ID Men Brown Solid Leather Mid-Top Sneakers',\n", 606 | " 'Gender': 'Men',\n", 607 | " 'Price (INR)': 1286,\n", 608 | " 'Description': 'A pair of round-toe brown sneakers, has mid-top styling, lace-up detailLeather upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 609 | " 'PrimaryColor': ' Brown'}},\n", 610 | " {'_index': 'my_products',\n", 611 | " '_id': '10018011',\n", 612 | " '_score': 0.73855746,\n", 613 | " '_source': {'ProductName': 'Puma Men Navy Blue Sneakers',\n", 614 | " 'Gender': 'Men',\n", 615 | " 'Price (INR)': 1959,\n", 616 | " 'Description': 'A pair of round-toe navy blue & white sneakers, has regular styling, lace-up detailTextile upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 617 | " 'PrimaryColor': 'Blue'}},\n", 618 | " {'_index': 'my_products',\n", 619 | " '_id': '10015929',\n", 620 | " '_score': 0.7336286,\n", 621 | " '_source': {'ProductName': 'ColorPlus Men Blue Regular Fit Checked Formal Trousers',\n", 622 | " 'Gender': 'Men',\n", 623 | " 'Price (INR)': 1199,\n", 624 | " 'Description': 'Blue checked mid-rise formal trousers, has a button closure, four pockets',\n", 625 | " 'PrimaryColor': 'Blue'}},\n", 626 | " {'_index': 'my_products',\n", 627 | " '_id': '10006041',\n", 628 | " '_score': 0.7334776,\n", 629 | " '_source': {'ProductName': 'ID Men Beige Leather Derbys',\n", 630 | " 'Gender': 'Men',\n", 631 | " 'Price (INR)': 2685,\n", 632 | " 'Description': 'A pair of round-toe beige derbys, has regular styling, lace-up detailLeather upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 633 | " 'PrimaryColor': ' Beige'}},\n", 634 | " {'_index': 'my_products',\n", 635 | " '_id': '10006049',\n", 636 | " '_score': 0.7334776,\n", 637 | " '_source': {'ProductName': 'ID Men Beige Leather Derbys',\n", 638 | " 'Gender': 'Men',\n", 639 | " 'Price (INR)': 2985,\n", 640 | " 'Description': 'A pair of round-toe beige derbys, has regular styling, lace-up detailLeather upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 641 | " 'PrimaryColor': ' Beige'}},\n", 642 | " {'_index': 'my_products',\n", 643 | " '_id': '10006045',\n", 644 | " '_score': 0.73244864,\n", 645 | " '_source': {'ProductName': 'ID Men Tan Brown Leather Sneakers',\n", 646 | " 'Gender': 'Men',\n", 647 | " 'Price (INR)': 2685,\n", 648 | " 'Description': 'A pair of round-toe tan brown sneakers, has regular styling, lace-up detailLeather upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 649 | " 'PrimaryColor': ' Brown'}}]" 650 | ] 651 | }, 652 | "execution_count": 15, 653 | "metadata": {}, 654 | "output_type": "execute_result" 655 | } 656 | ], 657 | "source": [ 658 | "query = {\n", 659 | " \"field\" : \"NameDescriptionVector\",\n", 660 | " \"query_vector\" : vector_of_input_keyword,\n", 661 | " \"k\" : 10,\n", 662 | " \"num_candidates\" : 500, \n", 663 | "}\n", 664 | "\n", 665 | "res = es.knn_search(index=\"my_products\", knn=query , source=[\"ProductName\",\"Description\",\"PrimaryColor\",\"Price (INR)\",\"Gender\"])\n", 666 | "res[\"hits\"][\"hits\"]" 667 | ] 668 | }, 669 | { 670 | "cell_type": "code", 671 | "execution_count": 16, 672 | "metadata": {}, 673 | "outputs": [ 674 | { 675 | "name": "stderr", 676 | "output_type": "stream", 677 | "text": [ 678 | "/var/folders/3k/901cf0jd1lngqxfl_y527qp00000gn/T/ipykernel_96516/2612073556.py:42: DeprecationWarning: Passing transport options in the API method is deprecated. Use 'Elasticsearch.options()' instead.\n", 679 | " res = es.knn_search(index=\"my_products\", # change index name here.\n", 680 | "/var/folders/3k/901cf0jd1lngqxfl_y527qp00000gn/T/ipykernel_96516/2612073556.py:42: ElasticsearchWarning: The kNN search API has been replaced by the `knn` option in the search API.\n", 681 | " res = es.knn_search(index=\"my_products\", # change index name here.\n" 682 | ] 683 | }, 684 | { 685 | "data": { 686 | "text/plain": [ 687 | "[{'_index': 'my_products',\n", 688 | " '_id': '10005997',\n", 689 | " '_score': 0.7492948,\n", 690 | " '_source': {'ProductName': 'ID Men Navy Blue Solid Leather Mid-Top Sneakers',\n", 691 | " 'ProductBrand': 'ID',\n", 692 | " 'Gender': 'Men',\n", 693 | " 'Price (INR)': 1286,\n", 694 | " 'Description': 'A pair of round-toe navy blue sneakers, has mid-top styling, lace-up detailLeather upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 695 | " 'PrimaryColor': 'Blue'}},\n", 696 | " {'_index': 'my_products',\n", 697 | " '_id': '10018075',\n", 698 | " '_score': 0.7472657,\n", 699 | " '_source': {'ProductName': 'Puma Men Blue Sneakers',\n", 700 | " 'ProductBrand': 'Puma',\n", 701 | " 'Gender': 'Men',\n", 702 | " 'Price (INR)': 1749,\n", 703 | " 'Description': 'A pair of round-toe blue sneakers, has regular styling, lace-up detailTextile upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 704 | " 'PrimaryColor': 'Blue'}},\n", 705 | " {'_index': 'my_products',\n", 706 | " '_id': '10018013',\n", 707 | " '_score': 0.7472377,\n", 708 | " '_source': {'ProductName': 'Puma Men Blue Sneakers',\n", 709 | " 'ProductBrand': 'Puma',\n", 710 | " 'Gender': 'Men',\n", 711 | " 'Price (INR)': 1799,\n", 712 | " 'Description': 'A pair of round-toe blue sneakers, has regular styling, lace-up detailTextile upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 713 | " 'PrimaryColor': 'Blue'}},\n", 714 | " {'_index': 'my_products',\n", 715 | " '_id': '10006031',\n", 716 | " '_score': 0.73941416,\n", 717 | " '_source': {'ProductName': 'ID Men Brown Solid Leather Mid-Top Sneakers',\n", 718 | " 'ProductBrand': 'ID',\n", 719 | " 'Gender': 'Men',\n", 720 | " 'Price (INR)': 1286,\n", 721 | " 'Description': 'A pair of round-toe brown sneakers, has mid-top styling, lace-up detailLeather upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 722 | " 'PrimaryColor': ' Brown'}},\n", 723 | " {'_index': 'my_products',\n", 724 | " '_id': '10018011',\n", 725 | " '_score': 0.73855746,\n", 726 | " '_source': {'ProductName': 'Puma Men Navy Blue Sneakers',\n", 727 | " 'ProductBrand': 'Puma',\n", 728 | " 'Gender': 'Men',\n", 729 | " 'Price (INR)': 1959,\n", 730 | " 'Description': 'A pair of round-toe navy blue & white sneakers, has regular styling, lace-up detailTextile upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 731 | " 'PrimaryColor': 'Blue'}},\n", 732 | " {'_index': 'my_products',\n", 733 | " '_id': '10015929',\n", 734 | " '_score': 0.7336286,\n", 735 | " '_source': {'ProductName': 'ColorPlus Men Blue Regular Fit Checked Formal Trousers',\n", 736 | " 'ProductBrand': 'ColorPlus',\n", 737 | " 'Gender': 'Men',\n", 738 | " 'Price (INR)': 1199,\n", 739 | " 'Description': 'Blue checked mid-rise formal trousers, has a button closure, four pockets',\n", 740 | " 'PrimaryColor': 'Blue'}},\n", 741 | " {'_index': 'my_products',\n", 742 | " '_id': '10006029',\n", 743 | " '_score': 0.73195255,\n", 744 | " '_source': {'ProductName': 'ID Men Brown Leather Formal Slip-Ons',\n", 745 | " 'ProductBrand': 'ID',\n", 746 | " 'Gender': 'Men',\n", 747 | " 'Price (INR)': 956,\n", 748 | " 'Description': 'A pair of brown square-toed formal slip-on shoes, has elasticated gussets on both sidesLeather upper, has a textured detailCushioned footbedTextured TPR outsoleWarranty: 3 months against manufacturing defect onlyWarranty provided by the brand owner / manufacturer',\n", 749 | " 'PrimaryColor': ' Brown'}},\n", 750 | " {'_index': 'my_products',\n", 751 | " '_id': '10006073',\n", 752 | " '_score': 0.7309584,\n", 753 | " '_source': {'ProductName': 'ID Men Brown Leather Loafers',\n", 754 | " 'ProductBrand': 'ID',\n", 755 | " 'Gender': 'Men',\n", 756 | " 'Price (INR)': 1218,\n", 757 | " 'Description': 'A pair of square toe brown loafers, has regular styling, slip-on detailLeather upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 758 | " 'PrimaryColor': ' Brown'}},\n", 759 | " {'_index': 'my_products',\n", 760 | " '_id': '10006069',\n", 761 | " '_score': 0.7294342,\n", 762 | " '_source': {'ProductName': 'ID Men Black Leather Formal Derbys',\n", 763 | " 'ProductBrand': 'ID',\n", 764 | " 'Gender': 'Men',\n", 765 | " 'Price (INR)': 1627,\n", 766 | " 'Description': 'A pair of black round-toed formal derbys, has central lace-upsLeather upperCushioned footbedTextured PU outsole, has a stacked heelWarranty: 3 months against manufacturing defect onlyWarranty provided by the brand owner / manufacturer',\n", 767 | " 'PrimaryColor': ' Black'}},\n", 768 | " {'_index': 'my_products',\n", 769 | " '_id': '10015161',\n", 770 | " '_score': 0.72923595,\n", 771 | " '_source': {'ProductName': 'Urban Dog Men Navy Blue Printed Regular Fit Sports Shorts',\n", 772 | " 'ProductBrand': 'Urban Dog',\n", 773 | " 'Gender': 'Men',\n", 774 | " 'Price (INR)': 399,\n", 775 | " 'Description': 'Navy Blue printed mid-rise sports shorts, has two pockets, a drawstring closure',\n", 776 | " 'PrimaryColor': 'Blue'}}]" 777 | ] 778 | }, 779 | "execution_count": 16, 780 | "metadata": {}, 781 | "output_type": "execute_result" 782 | } 783 | ], 784 | "source": [ 785 | "# Apply filter on semantic search results\n", 786 | "\n", 787 | "q1 = {\n", 788 | " \"knn\": {\n", 789 | " \"field\": \"NameDescriptionVector\",\n", 790 | " \"query_vector\": vector_of_input_keyword,\n", 791 | " \"k\": 10,\n", 792 | " \"num_candidates\": 10000\n", 793 | " },\n", 794 | " \"_source\": [\"ProductName\",\"Description\",\"PrimaryColor\",\"Price (INR)\",\"ProductBrand\",\"Gender\"]\n", 795 | "}\n", 796 | "\n", 797 | "# Add Price greater than and less than filters\n", 798 | "min_price = 0\n", 799 | "max_price = 2000\n", 800 | "Gender = \"Men\"\n", 801 | "\n", 802 | "filter_query = {\n", 803 | " \"bool\": {\n", 804 | " \"must\": [\n", 805 | " {\n", 806 | " \"match\": {\n", 807 | " \"Gender\": {\n", 808 | " \"query\": Gender,\n", 809 | " \"fuzzy_transpositions\": \"false\",\n", 810 | " \"fuzziness\": 0\n", 811 | " }\n", 812 | " }\n", 813 | " },\n", 814 | " {\n", 815 | " \"range\": {\n", 816 | " \"Price (INR)\": {\n", 817 | " \"gte\": min_price,\n", 818 | " \"lte\": max_price\n", 819 | " }\n", 820 | " }\n", 821 | " }\n", 822 | " ]\n", 823 | " }\n", 824 | "}\n", 825 | "\n", 826 | "res = es.knn_search(index=\"my_products\", # change index name here.\n", 827 | " body=q1,\n", 828 | " request_timeout=5000,\n", 829 | " filter=filter_query)\n", 830 | "\n", 831 | "res[\"hits\"][\"hits\"]" 832 | ] 833 | }, 834 | { 835 | "cell_type": "code", 836 | "execution_count": null, 837 | "metadata": {}, 838 | "outputs": [], 839 | "source": [] 840 | }, 841 | { 842 | "cell_type": "markdown", 843 | "metadata": {}, 844 | "source": [ 845 | "### Understanding intent of user\n", 846 | "\n", 847 | "##### And getting filters from the search query" 848 | ] 849 | }, 850 | { 851 | "cell_type": "code", 852 | "execution_count": 21, 853 | "metadata": {}, 854 | "outputs": [ 855 | { 856 | "name": "stdout", 857 | "output_type": "stream", 858 | "text": [ 859 | "[' Black', ' Beige', ' Pink', 'Blue', ' White', ' Brown', ' Burgundy', ' Red', ' Green', ' Maroon', ' Navy', ' Gold', ' Yellow', ' Grey', ' Platinum', ' Silver', ' Khaki', ' Mustard', ' Lavender', ' Matte', ' Rose', ' Charcoal', ' Purple']\n", 860 | "['Unisex', 'Women', 'Men', 'Boys', 'Girls']\n" 861 | ] 862 | } 863 | ], 864 | "source": [ 865 | "color_list = embedding_df[\"PrimaryColor\"].drop_duplicates().to_list()\n", 866 | "print(color_list)\n", 867 | "gender_list = embedding_df[\"Gender\"].drop_duplicates().to_list()\n", 868 | "print(gender_list)" 869 | ] 870 | }, 871 | { 872 | "cell_type": "code", 873 | "execution_count": 35, 874 | "metadata": {}, 875 | "outputs": [], 876 | "source": [ 877 | "input_keyword = \"brown Shoes for men under 1500\"\n", 878 | "vector_of_input_keyword = get_embedding(input_keyword)\n" 879 | ] 880 | }, 881 | { 882 | "cell_type": "code", 883 | "execution_count": 36, 884 | "metadata": {}, 885 | "outputs": [ 886 | { 887 | "data": { 888 | "text/plain": [ 889 | "'I have data in elastic search of all clothing products with their description, color, price and the gender they belongs to.\\ngenders are [\\'Unisex\\', \\'Women\\', \\'Men\\', \\'Boys\\', \\'Girls\\']\\ncolors are [\\' Black\\', \\' Beige\\', \\' Pink\\', \\'Blue\\', \\' White\\', \\' Brown\\', \\' Burgundy\\', \\' Red\\', \\' Green\\', \\' Maroon\\', \\' Navy\\', \\' Gold\\', \\' Yellow\\', \\' Grey\\', \\' Platinum\\', \\' Silver\\', \\' Khaki\\', \\' Mustard\\', \\' Lavender\\', \\' Matte\\', \\' Rose\\', \\' Charcoal\\', \\' Purple\\']\\nprice can be anything from 0 to 100k\\nbased on user\\'s search query. give me json output as follows\\n{\\n\"color\": \"it should be what users want. give Not-Mentioned if user did not explicitly mentioned the color in query. If the color mentioned by user is not present in above color list, give Not-Found\",\\n\"gender\": \"gender should be from above list only. if not specified give Not-Mentioned.\"\\n\"max_price\":\\n\"min_price\":\\n}\\n\\nusers query : brown Shoes for men under 1500\\n'" 890 | ] 891 | }, 892 | "execution_count": 36, 893 | "metadata": {}, 894 | "output_type": "execute_result" 895 | } 896 | ], 897 | "source": [ 898 | "my_prompt = f\"\"\"I have data in elastic search of all clothing products with their description, color, price and the gender they belongs to.\n", 899 | "genders are {gender_list}\n", 900 | "colors are {color_list}\n", 901 | "price can be anything from 0 to 100k\n", 902 | "based on user's search query. give me json output as follows\n", 903 | "{{\n", 904 | "\"color\": \"it should be what users want. give Not-Mentioned if user did not explicitly mentioned the color in query. If the color mentioned by user is not present in above color list, give Not-Found\",\n", 905 | "\"gender\": \"gender should be from above list only. if not specified give Not-Mentioned.\"\n", 906 | "\"max_price\":\n", 907 | "\"min_price\":\n", 908 | "}}\n", 909 | "\n", 910 | "users query : {input_keyword}\n", 911 | "\"\"\"\n", 912 | "my_prompt" 913 | ] 914 | }, 915 | { 916 | "cell_type": "code", 917 | "execution_count": 37, 918 | "metadata": {}, 919 | "outputs": [ 920 | { 921 | "data": { 922 | "text/plain": [ 923 | "ChatCompletion(id='chatcmpl-8TqrOBIjWZcGYty2SaU4EBLxUwGdo', choices=[Choice(finish_reason='stop', index=0, message=ChatCompletionMessage(content='{\\n \"color\": \"Brown\",\\n \"gender\": \"Men\",\\n \"max_price\": 1500,\\n \"min_price\": 0\\n}', role='assistant', function_call=None, tool_calls=None))], created=1702125398, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint='fp_eeff13170a', usage=CompletionUsage(completion_tokens=33, prompt_tokens=268, total_tokens=301))" 924 | ] 925 | }, 926 | "execution_count": 37, 927 | "metadata": {}, 928 | "output_type": "execute_result" 929 | } 930 | ], 931 | "source": [ 932 | "\n", 933 | "response = client.chat.completions.create(\n", 934 | " model=\"gpt-3.5-turbo-1106\",\n", 935 | " response_format={ \"type\": \"json_object\" },\n", 936 | " messages=[\n", 937 | " {\"role\": \"system\", \"content\": \"You are a helpful assistant designed to output only in JSON format. No other text or explaination.\"},\n", 938 | " {\"role\": \"user\", \"content\": my_prompt}\n", 939 | " ]\n", 940 | ")\n", 941 | "\n", 942 | "response" 943 | ] 944 | }, 945 | { 946 | "cell_type": "code", 947 | "execution_count": 38, 948 | "metadata": {}, 949 | "outputs": [ 950 | { 951 | "data": { 952 | "text/plain": [ 953 | "'{\\n \"color\": \"Brown\",\\n \"gender\": \"Men\",\\n \"max_price\": 1500,\\n \"min_price\": 0\\n}'" 954 | ] 955 | }, 956 | "execution_count": 38, 957 | "metadata": {}, 958 | "output_type": "execute_result" 959 | } 960 | ], 961 | "source": [ 962 | "response.choices[0].message.content" 963 | ] 964 | }, 965 | { 966 | "cell_type": "code", 967 | "execution_count": 39, 968 | "metadata": {}, 969 | "outputs": [ 970 | { 971 | "data": { 972 | "text/plain": [ 973 | "{'color': 'Brown', 'gender': 'Men', 'max_price': 1500, 'min_price': 0}" 974 | ] 975 | }, 976 | "execution_count": 39, 977 | "metadata": {}, 978 | "output_type": "execute_result" 979 | } 980 | ], 981 | "source": [ 982 | "import json\n", 983 | "filter_map = json.loads(response.choices[0].message.content)\n", 984 | "filter_map" 985 | ] 986 | }, 987 | { 988 | "cell_type": "code", 989 | "execution_count": 40, 990 | "metadata": {}, 991 | "outputs": [ 992 | { 993 | "name": "stderr", 994 | "output_type": "stream", 995 | "text": [ 996 | "/var/folders/3k/901cf0jd1lngqxfl_y527qp00000gn/T/ipykernel_96516/3685903562.py:36: DeprecationWarning: Passing transport options in the API method is deprecated. Use 'Elasticsearch.options()' instead.\n", 997 | " res = es.knn_search(index=\"my_products\",\n", 998 | "/var/folders/3k/901cf0jd1lngqxfl_y527qp00000gn/T/ipykernel_96516/3685903562.py:36: ElasticsearchWarning: The kNN search API has been replaced by the `knn` option in the search API.\n", 999 | " res = es.knn_search(index=\"my_products\",\n" 1000 | ] 1001 | }, 1002 | { 1003 | "data": { 1004 | "text/plain": [ 1005 | "[{'_index': 'my_products',\n", 1006 | " '_id': '10006031',\n", 1007 | " '_score': 0.75574964,\n", 1008 | " '_source': {'ProductName': 'ID Men Brown Solid Leather Mid-Top Sneakers',\n", 1009 | " 'ProductBrand': 'ID',\n", 1010 | " 'Gender': 'Men',\n", 1011 | " 'Price (INR)': 1286,\n", 1012 | " 'Description': 'A pair of round-toe brown sneakers, has mid-top styling, lace-up detailLeather upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 1013 | " 'PrimaryColor': ' Brown'}},\n", 1014 | " {'_index': 'my_products',\n", 1015 | " '_id': '10006073',\n", 1016 | " '_score': 0.7465898,\n", 1017 | " '_source': {'ProductName': 'ID Men Brown Leather Loafers',\n", 1018 | " 'ProductBrand': 'ID',\n", 1019 | " 'Gender': 'Men',\n", 1020 | " 'Price (INR)': 1218,\n", 1021 | " 'Description': 'A pair of square toe brown loafers, has regular styling, slip-on detailLeather upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 1022 | " 'PrimaryColor': ' Brown'}},\n", 1023 | " {'_index': 'my_products',\n", 1024 | " '_id': '10006083',\n", 1025 | " '_score': 0.74521184,\n", 1026 | " '_source': {'ProductName': 'ID Men Tan Brown Leather Loafers',\n", 1027 | " 'ProductBrand': 'ID',\n", 1028 | " 'Gender': 'Men',\n", 1029 | " 'Price (INR)': 1218,\n", 1030 | " 'Description': 'A pair of square toe tan loafers, has regular styling, slip-on detailLeather upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 1031 | " 'PrimaryColor': ' Brown'}},\n", 1032 | " {'_index': 'my_products',\n", 1033 | " '_id': '10006029',\n", 1034 | " '_score': 0.7421721,\n", 1035 | " '_source': {'ProductName': 'ID Men Brown Leather Formal Slip-Ons',\n", 1036 | " 'ProductBrand': 'ID',\n", 1037 | " 'Gender': 'Men',\n", 1038 | " 'Price (INR)': 956,\n", 1039 | " 'Description': 'A pair of brown square-toed formal slip-on shoes, has elasticated gussets on both sidesLeather upper, has a textured detailCushioned footbedTextured TPR outsoleWarranty: 3 months against manufacturing defect onlyWarranty provided by the brand owner / manufacturer',\n", 1040 | " 'PrimaryColor': ' Brown'}},\n", 1041 | " {'_index': 'my_products',\n", 1042 | " '_id': '10006019',\n", 1043 | " '_score': 0.7388859,\n", 1044 | " '_source': {'ProductName': 'ID Men Tan Brown Formal Leather Oxfords',\n", 1045 | " 'ProductBrand': 'ID',\n", 1046 | " 'Gender': 'Men',\n", 1047 | " 'Price (INR)': 1432,\n", 1048 | " 'Description': 'A pair of tan brown square-toed formal oxfords, has central lace-upsLeather upperCushioned footbedTextured TPU outsole, has a stacked heelWarranty: 3 months against manufacturing defect onlyWarranty provided by the brand owner / manufacturer',\n", 1049 | " 'PrimaryColor': ' Brown'}},\n", 1050 | " {'_index': 'my_products',\n", 1051 | " '_id': '10006059',\n", 1052 | " '_score': 0.7294855,\n", 1053 | " '_source': {'ProductName': 'ID Men Brown Fisherman Leather Sandals',\n", 1054 | " 'ProductBrand': 'ID',\n", 1055 | " 'Gender': 'Men',\n", 1056 | " 'Price (INR)': 1197,\n", 1057 | " 'Description': 'A pair of brown fisherman sandalsLeather upper with velcro closureCushioned footbedPatterned synthetic outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer',\n", 1058 | " 'PrimaryColor': ' Brown'}},\n", 1059 | " {'_index': 'my_products',\n", 1060 | " '_id': '10014361',\n", 1061 | " '_score': 0.7212268,\n", 1062 | " '_source': {'ProductName': 'SHOWOFF Men Brown Solid Slim Fit Regular Shorts',\n", 1063 | " 'ProductBrand': 'SHOWOFF',\n", 1064 | " 'Gender': 'Men',\n", 1065 | " 'Price (INR)': 791,\n", 1066 | " 'Description': 'Brown solid low-rise regular shorts, has four pockets, a button closure',\n", 1067 | " 'PrimaryColor': ' Brown'}},\n", 1068 | " {'_index': 'my_products',\n", 1069 | " '_id': '10000595',\n", 1070 | " '_score': 0.7157342,\n", 1071 | " '_source': {'ProductName': 'Parx Men Brown Original Slim Fit Solid Regular Trousers',\n", 1072 | " 'ProductBrand': 'Parx',\n", 1073 | " 'Gender': 'Men',\n", 1074 | " 'Price (INR)': 874,\n", 1075 | " 'Description': 'Brown solid mid-rise regular trousers, has a button closure, four pockets',\n", 1076 | " 'PrimaryColor': ' Brown'}},\n", 1077 | " {'_index': 'my_products',\n", 1078 | " '_id': '10007811',\n", 1079 | " '_score': 0.7087305,\n", 1080 | " '_source': {'ProductName': 'her by invictus Women Tan Brown Cushioned Slip-on Shoes',\n", 1081 | " 'ProductBrand': 'her by invictus',\n", 1082 | " 'Gender': 'Women',\n", 1083 | " 'Price (INR)': 949,\n", 1084 | " 'Description': 'A pair of pointed closed toe tan brown slip-on shoes with cushioned footbed and comfortable lining, has regular styling, and closed backSynthetic upperFoam Cushioned footbedTextured and patterned outsole',\n", 1085 | " 'PrimaryColor': ' Brown'}},\n", 1086 | " {'_index': 'my_products',\n", 1087 | " '_id': '10002785',\n", 1088 | " '_score': 0.70150924,\n", 1089 | " '_source': {'ProductName': 'JBN Creation Boys Maroon & Brown Sherwani Set',\n", 1090 | " 'ProductBrand': 'JBN Creation',\n", 1091 | " 'Gender': 'Boys',\n", 1092 | " 'Price (INR)': 1154,\n", 1093 | " 'Description': 'Maroon and brown sherwaniMaroon woven design sherwani, has a mandarin collar, a full button placket, a welt pocket, multipple slitsComes with a pocket squareA pair of brown mid-rise pyjama with gathers and pleats, has an elasticated waistband',\n", 1094 | " 'PrimaryColor': ' Brown'}}]" 1095 | ] 1096 | }, 1097 | "execution_count": 40, 1098 | "metadata": {}, 1099 | "output_type": "execute_result" 1100 | } 1101 | ], 1102 | "source": [ 1103 | "\n", 1104 | "q1 = {\n", 1105 | " \"knn\": {\n", 1106 | " \"field\": \"NameDescriptionVector\",\n", 1107 | " \"query_vector\": vector_of_input_keyword,\n", 1108 | " \"k\": 10,\n", 1109 | " \"num_candidates\": 10000\n", 1110 | " },\n", 1111 | " \"_source\": [\"ProductName\",\"Description\",\"PrimaryColor\",\"Price (INR)\",\"ProductBrand\",\"Gender\"]\n", 1112 | "}\n", 1113 | "\n", 1114 | "\n", 1115 | "filter_query = {\n", 1116 | " \"bool\": {\n", 1117 | " \"must\": [\n", 1118 | " {\n", 1119 | " \"match\": {\n", 1120 | " \"PrimaryColor\": {\n", 1121 | " \"query\": filter_map[\"color\"],\n", 1122 | " \"fuzzy_transpositions\": \"false\",\n", 1123 | " \"fuzziness\": 0\n", 1124 | " }\n", 1125 | " }\n", 1126 | " },\n", 1127 | " {\n", 1128 | " \"range\": {\n", 1129 | " \"Price (INR)\": {\n", 1130 | " \"gte\": filter_map[\"min_price\"],\n", 1131 | " \"lte\": filter_map[\"max_price\"]\n", 1132 | " }\n", 1133 | " }\n", 1134 | " }\n", 1135 | " ]\n", 1136 | " }\n", 1137 | "}\n", 1138 | "\n", 1139 | "res = es.knn_search(index=\"my_products\",\n", 1140 | " body=q1,\n", 1141 | " request_timeout=5000,\n", 1142 | " filter=filter_query)\n", 1143 | "\n", 1144 | "res[\"hits\"][\"hits\"]" 1145 | ] 1146 | }, 1147 | { 1148 | "cell_type": "code", 1149 | "execution_count": null, 1150 | "metadata": {}, 1151 | "outputs": [], 1152 | "source": [] 1153 | } 1154 | ], 1155 | "metadata": { 1156 | "kernelspec": { 1157 | "display_name": "Python 3", 1158 | "language": "python", 1159 | "name": "python3" 1160 | }, 1161 | "language_info": { 1162 | "codemirror_mode": { 1163 | "name": "ipython", 1164 | "version": 3 1165 | }, 1166 | "file_extension": ".py", 1167 | "mimetype": "text/x-python", 1168 | "name": "python", 1169 | "nbconvert_exporter": "python", 1170 | "pygments_lexer": "ipython3", 1171 | "version": "3.11.5" 1172 | } 1173 | }, 1174 | "nbformat": 4, 1175 | "nbformat_minor": 2 1176 | } 1177 | -------------------------------------------------------------------------------- /indexMapping.py: -------------------------------------------------------------------------------- 1 | indexMapping = { 2 | "properties":{ 3 | "ProductID":{ 4 | "type":"long" 5 | }, 6 | "ProductName":{ 7 | "type":"text" 8 | }, 9 | "ProductBrand":{ 10 | "type":"text" 11 | }, 12 | "Gender":{ 13 | "type":"text" 14 | }, 15 | "Price (INR)":{ 16 | "type":"long" 17 | }, 18 | "NumImages":{ 19 | "type":"long" 20 | }, 21 | "Description":{ 22 | "type":"text" 23 | }, 24 | "PrimaryColor":{ 25 | "type":"text" 26 | }, 27 | "NameDescription":{ 28 | "type":"text" 29 | }, 30 | "NameDescriptionVector":{ 31 | "type":"dense_vector", 32 | "dims": 1536, 33 | "index":True, 34 | "similarity": "l2_norm" 35 | } 36 | 37 | } 38 | } -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | annotated-types==0.6.0 2 | anyio==3.7.1 3 | appnope==0.1.3 4 | asttokens==2.4.1 5 | certifi==2023.11.17 6 | comm==0.2.0 7 | debugpy==1.8.0 8 | decorator==5.1.1 9 | distro==1.8.0 10 | elastic-transport==8.10.0 11 | elasticsearch==8.11.1 12 | executing==2.0.1 13 | h11==0.14.0 14 | httpcore==1.0.2 15 | httpx==0.25.2 16 | idna==3.6 17 | ipykernel==6.27.1 18 | ipython==8.18.1 19 | jedi==0.19.1 20 | jupyter_client==8.6.0 21 | jupyter_core==5.5.0 22 | matplotlib-inline==0.1.6 23 | nest-asyncio==1.5.8 24 | numpy==1.26.2 25 | openai==1.3.7 26 | packaging==23.2 27 | pandas==2.1.3 28 | parso==0.8.3 29 | pexpect==4.9.0 30 | platformdirs==4.1.0 31 | prompt-toolkit==3.0.41 32 | psutil==5.9.6 33 | ptyprocess==0.7.0 34 | pure-eval==0.2.2 35 | pydantic==2.5.2 36 | pydantic_core==2.14.5 37 | Pygments==2.17.2 38 | python-dateutil==2.8.2 39 | pytz==2023.3.post1 40 | pyzmq==25.1.2 41 | six==1.16.0 42 | sniffio==1.3.0 43 | stack-data==0.6.3 44 | tornado==6.4 45 | tqdm==4.66.1 46 | traitlets==5.14.0 47 | typing_extensions==4.8.0 48 | tzdata==2023.3 49 | urllib3==2.1.0 50 | wcwidth==0.2.12 51 | --------------------------------------------------------------------------------