├── .example_env ├── .gitignore ├── README.md ├── package-lock.json ├── package.json ├── vector_triggers.js ├── vectorize_collection.js └── vectorize_queries.js /.example_env: -------------------------------------------------------------------------------- 1 | # MongoDB credentials 2 | MONGODB_USER=the_username 3 | MONGODB_PASSWORD=the_password 4 | MONGODB_HOSTNAME=someProjectName.someHostname -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .env 3 | .DS_Store -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hello-vector-search 2 | 3 | — Note in 2024 for archiving: I previously wrote a tutorial in 2022 that was embargoed by a Product Marketing Manager that was a introduction to vector search using MongoDB. Here lived the referenced code examples. Unfortunately, there were no good solutions for vectorization at scale, so I built one. The unerlying infra struggled with the nature of the workload so I left my full-time role at MongoDB and started an infra company. I hope this provides you ideas on how you can wholly manage a vector workload on teh Atlas platform. I've moved on from this wave to a new one, but if you stumbled upon it ask any questions you might have. — 4 | 5 | This project is meant to be an introduction to vector search using mongodb and precedes a blog which walks through all the utliity files described below and is not intended for production deployments. 6 | 7 | 1. A simple JavaScript program to run from your computer to vectorize the `sample_mflix.movies` collection: [vectorize_collection.js](vectorize_collection.js). 8 | 2. A simple JavaScript program to run as a function in Atlas with an HTTPS Endpoint that will vectorize queries, unless you want to call a vector API directly: [vectorize_queries.js](vectorize_queries.js) 9 | 3. A simple JavaScript program to work as a database trigger in Atlas to ensure any new documents added to the collection get the vector treatment: [vector_triggers.js](vector_triggers.js) 10 | 11 | ### That's it. 12 | 13 | - I hope it's all super easy. We will release a blog soon, but in the meantime feel free to step through this almost an all Atlas workflow so you can prepare to talk about it. Very important. Most of our customers know JavaScript. Feel free to open a PR or any issues so we can iron them out before the rush comes. 14 | 15 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "easy-vector-search", 3 | "lockfileVersion": 2, 4 | "requires": true, 5 | "packages": { 6 | "": { 7 | "dependencies": { 8 | "axios": "^0.27.2", 9 | "cors": "^2.8.5", 10 | "dotenv": "^16.0.2", 11 | "mongodb": "^4.10.0" 12 | } 13 | }, 14 | "node_modules/@types/node": { 15 | "version": "18.7.18", 16 | "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.18.tgz", 17 | "integrity": "sha512-m+6nTEOadJZuTPkKR/SYK3A2d7FZrgElol9UP1Kae90VVU4a6mxnPuLiIW1m4Cq4gZ/nWb9GrdVXJCoCazDAbg==" 18 | }, 19 | "node_modules/@types/webidl-conversions": { 20 | "version": "7.0.0", 21 | "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz", 22 | "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==" 23 | }, 24 | "node_modules/@types/whatwg-url": { 25 | "version": "8.2.2", 26 | "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.2.tgz", 27 | "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==", 28 | "dependencies": { 29 | "@types/node": "*", 30 | "@types/webidl-conversions": "*" 31 | } 32 | }, 33 | "node_modules/asynckit": { 34 | "version": "0.4.0", 35 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 36 | "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" 37 | }, 38 | "node_modules/axios": { 39 | "version": "0.27.2", 40 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", 41 | "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", 42 | "dependencies": { 43 | "follow-redirects": "^1.14.9", 44 | "form-data": "^4.0.0" 45 | } 46 | }, 47 | "node_modules/base64-js": { 48 | "version": "1.5.1", 49 | "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", 50 | "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", 51 | "funding": [ 52 | { 53 | "type": "github", 54 | "url": "https://github.com/sponsors/feross" 55 | }, 56 | { 57 | "type": "patreon", 58 | "url": "https://www.patreon.com/feross" 59 | }, 60 | { 61 | "type": "consulting", 62 | "url": "https://feross.org/support" 63 | } 64 | ] 65 | }, 66 | "node_modules/bson": { 67 | "version": "4.7.0", 68 | "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz", 69 | "integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==", 70 | "dependencies": { 71 | "buffer": "^5.6.0" 72 | }, 73 | "engines": { 74 | "node": ">=6.9.0" 75 | } 76 | }, 77 | "node_modules/buffer": { 78 | "version": "5.7.1", 79 | "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", 80 | "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", 81 | "funding": [ 82 | { 83 | "type": "github", 84 | "url": "https://github.com/sponsors/feross" 85 | }, 86 | { 87 | "type": "patreon", 88 | "url": "https://www.patreon.com/feross" 89 | }, 90 | { 91 | "type": "consulting", 92 | "url": "https://feross.org/support" 93 | } 94 | ], 95 | "dependencies": { 96 | "base64-js": "^1.3.1", 97 | "ieee754": "^1.1.13" 98 | } 99 | }, 100 | "node_modules/combined-stream": { 101 | "version": "1.0.8", 102 | "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", 103 | "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", 104 | "dependencies": { 105 | "delayed-stream": "~1.0.0" 106 | }, 107 | "engines": { 108 | "node": ">= 0.8" 109 | } 110 | }, 111 | "node_modules/cors": { 112 | "version": "2.8.5", 113 | "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", 114 | "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", 115 | "dependencies": { 116 | "object-assign": "^4", 117 | "vary": "^1" 118 | }, 119 | "engines": { 120 | "node": ">= 0.10" 121 | } 122 | }, 123 | "node_modules/delayed-stream": { 124 | "version": "1.0.0", 125 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 126 | "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", 127 | "engines": { 128 | "node": ">=0.4.0" 129 | } 130 | }, 131 | "node_modules/denque": { 132 | "version": "2.1.0", 133 | "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", 134 | "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", 135 | "engines": { 136 | "node": ">=0.10" 137 | } 138 | }, 139 | "node_modules/dotenv": { 140 | "version": "16.0.2", 141 | "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.2.tgz", 142 | "integrity": "sha512-JvpYKUmzQhYoIFgK2MOnF3bciIZoItIIoryihy0rIA+H4Jy0FmgyKYAHCTN98P5ybGSJcIFbh6QKeJdtZd1qhA==", 143 | "engines": { 144 | "node": ">=12" 145 | } 146 | }, 147 | "node_modules/follow-redirects": { 148 | "version": "1.15.2", 149 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", 150 | "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", 151 | "funding": [ 152 | { 153 | "type": "individual", 154 | "url": "https://github.com/sponsors/RubenVerborgh" 155 | } 156 | ], 157 | "engines": { 158 | "node": ">=4.0" 159 | }, 160 | "peerDependenciesMeta": { 161 | "debug": { 162 | "optional": true 163 | } 164 | } 165 | }, 166 | "node_modules/form-data": { 167 | "version": "4.0.0", 168 | "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", 169 | "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", 170 | "dependencies": { 171 | "asynckit": "^0.4.0", 172 | "combined-stream": "^1.0.8", 173 | "mime-types": "^2.1.12" 174 | }, 175 | "engines": { 176 | "node": ">= 6" 177 | } 178 | }, 179 | "node_modules/ieee754": { 180 | "version": "1.2.1", 181 | "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", 182 | "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", 183 | "funding": [ 184 | { 185 | "type": "github", 186 | "url": "https://github.com/sponsors/feross" 187 | }, 188 | { 189 | "type": "patreon", 190 | "url": "https://www.patreon.com/feross" 191 | }, 192 | { 193 | "type": "consulting", 194 | "url": "https://feross.org/support" 195 | } 196 | ] 197 | }, 198 | "node_modules/ip": { 199 | "version": "2.0.0", 200 | "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", 201 | "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==" 202 | }, 203 | "node_modules/memory-pager": { 204 | "version": "1.5.0", 205 | "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", 206 | "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", 207 | "optional": true 208 | }, 209 | "node_modules/mime-db": { 210 | "version": "1.52.0", 211 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", 212 | "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", 213 | "engines": { 214 | "node": ">= 0.6" 215 | } 216 | }, 217 | "node_modules/mime-types": { 218 | "version": "2.1.35", 219 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", 220 | "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", 221 | "dependencies": { 222 | "mime-db": "1.52.0" 223 | }, 224 | "engines": { 225 | "node": ">= 0.6" 226 | } 227 | }, 228 | "node_modules/mongodb": { 229 | "version": "4.10.0", 230 | "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.10.0.tgz", 231 | "integrity": "sha512-My2QxLTw0Cc1O9gih0mz4mqo145Jq4rLAQx0Glk/Ha9iYBzYpt4I2QFNRIh35uNFNfe8KFQcdwY1/HKxXBkinw==", 232 | "dependencies": { 233 | "bson": "^4.7.0", 234 | "denque": "^2.1.0", 235 | "mongodb-connection-string-url": "^2.5.3", 236 | "socks": "^2.7.0" 237 | }, 238 | "engines": { 239 | "node": ">=12.9.0" 240 | }, 241 | "optionalDependencies": { 242 | "saslprep": "^1.0.3" 243 | } 244 | }, 245 | "node_modules/mongodb-connection-string-url": { 246 | "version": "2.5.3", 247 | "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.5.3.tgz", 248 | "integrity": "sha512-f+/WsED+xF4B74l3k9V/XkTVj5/fxFH2o5ToKXd8Iyi5UhM+sO9u0Ape17Mvl/GkZaFtM0HQnzAG5OTmhKw+tQ==", 249 | "dependencies": { 250 | "@types/whatwg-url": "^8.2.1", 251 | "whatwg-url": "^11.0.0" 252 | } 253 | }, 254 | "node_modules/object-assign": { 255 | "version": "4.1.1", 256 | "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", 257 | "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", 258 | "engines": { 259 | "node": ">=0.10.0" 260 | } 261 | }, 262 | "node_modules/punycode": { 263 | "version": "2.1.1", 264 | "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", 265 | "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", 266 | "engines": { 267 | "node": ">=6" 268 | } 269 | }, 270 | "node_modules/saslprep": { 271 | "version": "1.0.3", 272 | "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", 273 | "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", 274 | "optional": true, 275 | "dependencies": { 276 | "sparse-bitfield": "^3.0.3" 277 | }, 278 | "engines": { 279 | "node": ">=6" 280 | } 281 | }, 282 | "node_modules/smart-buffer": { 283 | "version": "4.2.0", 284 | "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", 285 | "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", 286 | "engines": { 287 | "node": ">= 6.0.0", 288 | "npm": ">= 3.0.0" 289 | } 290 | }, 291 | "node_modules/socks": { 292 | "version": "2.7.0", 293 | "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.0.tgz", 294 | "integrity": "sha512-scnOe9y4VuiNUULJN72GrM26BNOjVsfPXI+j+98PkyEfsIXroa5ofyjT+FzGvn/xHs73U2JtoBYAVx9Hl4quSA==", 295 | "dependencies": { 296 | "ip": "^2.0.0", 297 | "smart-buffer": "^4.2.0" 298 | }, 299 | "engines": { 300 | "node": ">= 10.13.0", 301 | "npm": ">= 3.0.0" 302 | } 303 | }, 304 | "node_modules/sparse-bitfield": { 305 | "version": "3.0.3", 306 | "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", 307 | "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", 308 | "optional": true, 309 | "dependencies": { 310 | "memory-pager": "^1.0.2" 311 | } 312 | }, 313 | "node_modules/tr46": { 314 | "version": "3.0.0", 315 | "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", 316 | "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", 317 | "dependencies": { 318 | "punycode": "^2.1.1" 319 | }, 320 | "engines": { 321 | "node": ">=12" 322 | } 323 | }, 324 | "node_modules/vary": { 325 | "version": "1.1.2", 326 | "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", 327 | "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", 328 | "engines": { 329 | "node": ">= 0.8" 330 | } 331 | }, 332 | "node_modules/webidl-conversions": { 333 | "version": "7.0.0", 334 | "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", 335 | "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", 336 | "engines": { 337 | "node": ">=12" 338 | } 339 | }, 340 | "node_modules/whatwg-url": { 341 | "version": "11.0.0", 342 | "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", 343 | "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", 344 | "dependencies": { 345 | "tr46": "^3.0.0", 346 | "webidl-conversions": "^7.0.0" 347 | }, 348 | "engines": { 349 | "node": ">=12" 350 | } 351 | } 352 | }, 353 | "dependencies": { 354 | "@types/node": { 355 | "version": "18.7.18", 356 | "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.18.tgz", 357 | "integrity": "sha512-m+6nTEOadJZuTPkKR/SYK3A2d7FZrgElol9UP1Kae90VVU4a6mxnPuLiIW1m4Cq4gZ/nWb9GrdVXJCoCazDAbg==" 358 | }, 359 | "@types/webidl-conversions": { 360 | "version": "7.0.0", 361 | "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz", 362 | "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==" 363 | }, 364 | "@types/whatwg-url": { 365 | "version": "8.2.2", 366 | "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.2.tgz", 367 | "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==", 368 | "requires": { 369 | "@types/node": "*", 370 | "@types/webidl-conversions": "*" 371 | } 372 | }, 373 | "asynckit": { 374 | "version": "0.4.0", 375 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 376 | "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" 377 | }, 378 | "axios": { 379 | "version": "0.27.2", 380 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", 381 | "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", 382 | "requires": { 383 | "follow-redirects": "^1.14.9", 384 | "form-data": "^4.0.0" 385 | } 386 | }, 387 | "base64-js": { 388 | "version": "1.5.1", 389 | "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", 390 | "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" 391 | }, 392 | "bson": { 393 | "version": "4.7.0", 394 | "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz", 395 | "integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==", 396 | "requires": { 397 | "buffer": "^5.6.0" 398 | } 399 | }, 400 | "buffer": { 401 | "version": "5.7.1", 402 | "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", 403 | "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", 404 | "requires": { 405 | "base64-js": "^1.3.1", 406 | "ieee754": "^1.1.13" 407 | } 408 | }, 409 | "combined-stream": { 410 | "version": "1.0.8", 411 | "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", 412 | "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", 413 | "requires": { 414 | "delayed-stream": "~1.0.0" 415 | } 416 | }, 417 | "cors": { 418 | "version": "2.8.5", 419 | "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", 420 | "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", 421 | "requires": { 422 | "object-assign": "^4", 423 | "vary": "^1" 424 | } 425 | }, 426 | "delayed-stream": { 427 | "version": "1.0.0", 428 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 429 | "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" 430 | }, 431 | "denque": { 432 | "version": "2.1.0", 433 | "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", 434 | "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" 435 | }, 436 | "dotenv": { 437 | "version": "16.0.2", 438 | "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.2.tgz", 439 | "integrity": "sha512-JvpYKUmzQhYoIFgK2MOnF3bciIZoItIIoryihy0rIA+H4Jy0FmgyKYAHCTN98P5ybGSJcIFbh6QKeJdtZd1qhA==" 440 | }, 441 | "follow-redirects": { 442 | "version": "1.15.2", 443 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", 444 | "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" 445 | }, 446 | "form-data": { 447 | "version": "4.0.0", 448 | "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", 449 | "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", 450 | "requires": { 451 | "asynckit": "^0.4.0", 452 | "combined-stream": "^1.0.8", 453 | "mime-types": "^2.1.12" 454 | } 455 | }, 456 | "ieee754": { 457 | "version": "1.2.1", 458 | "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", 459 | "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" 460 | }, 461 | "ip": { 462 | "version": "2.0.0", 463 | "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", 464 | "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==" 465 | }, 466 | "memory-pager": { 467 | "version": "1.5.0", 468 | "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", 469 | "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", 470 | "optional": true 471 | }, 472 | "mime-db": { 473 | "version": "1.52.0", 474 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", 475 | "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" 476 | }, 477 | "mime-types": { 478 | "version": "2.1.35", 479 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", 480 | "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", 481 | "requires": { 482 | "mime-db": "1.52.0" 483 | } 484 | }, 485 | "mongodb": { 486 | "version": "4.10.0", 487 | "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.10.0.tgz", 488 | "integrity": "sha512-My2QxLTw0Cc1O9gih0mz4mqo145Jq4rLAQx0Glk/Ha9iYBzYpt4I2QFNRIh35uNFNfe8KFQcdwY1/HKxXBkinw==", 489 | "requires": { 490 | "bson": "^4.7.0", 491 | "denque": "^2.1.0", 492 | "mongodb-connection-string-url": "^2.5.3", 493 | "saslprep": "^1.0.3", 494 | "socks": "^2.7.0" 495 | } 496 | }, 497 | "mongodb-connection-string-url": { 498 | "version": "2.5.3", 499 | "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.5.3.tgz", 500 | "integrity": "sha512-f+/WsED+xF4B74l3k9V/XkTVj5/fxFH2o5ToKXd8Iyi5UhM+sO9u0Ape17Mvl/GkZaFtM0HQnzAG5OTmhKw+tQ==", 501 | "requires": { 502 | "@types/whatwg-url": "^8.2.1", 503 | "whatwg-url": "^11.0.0" 504 | } 505 | }, 506 | "object-assign": { 507 | "version": "4.1.1", 508 | "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", 509 | "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" 510 | }, 511 | "punycode": { 512 | "version": "2.1.1", 513 | "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", 514 | "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" 515 | }, 516 | "saslprep": { 517 | "version": "1.0.3", 518 | "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", 519 | "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", 520 | "optional": true, 521 | "requires": { 522 | "sparse-bitfield": "^3.0.3" 523 | } 524 | }, 525 | "smart-buffer": { 526 | "version": "4.2.0", 527 | "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", 528 | "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==" 529 | }, 530 | "socks": { 531 | "version": "2.7.0", 532 | "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.0.tgz", 533 | "integrity": "sha512-scnOe9y4VuiNUULJN72GrM26BNOjVsfPXI+j+98PkyEfsIXroa5ofyjT+FzGvn/xHs73U2JtoBYAVx9Hl4quSA==", 534 | "requires": { 535 | "ip": "^2.0.0", 536 | "smart-buffer": "^4.2.0" 537 | } 538 | }, 539 | "sparse-bitfield": { 540 | "version": "3.0.3", 541 | "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", 542 | "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", 543 | "optional": true, 544 | "requires": { 545 | "memory-pager": "^1.0.2" 546 | } 547 | }, 548 | "tr46": { 549 | "version": "3.0.0", 550 | "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", 551 | "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", 552 | "requires": { 553 | "punycode": "^2.1.1" 554 | } 555 | }, 556 | "vary": { 557 | "version": "1.1.2", 558 | "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", 559 | "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" 560 | }, 561 | "webidl-conversions": { 562 | "version": "7.0.0", 563 | "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", 564 | "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==" 565 | }, 566 | "whatwg-url": { 567 | "version": "11.0.0", 568 | "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", 569 | "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", 570 | "requires": { 571 | "tr46": "^3.0.0", 572 | "webidl-conversions": "^7.0.0" 573 | } 574 | } 575 | } 576 | } 577 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "axios": "^0.27.2", 4 | "cors": "^2.8.5", 5 | "dotenv": "^16.0.2", 6 | "mongodb": "^4.10.0" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /vector_triggers.js: -------------------------------------------------------------------------------- 1 | /* 2 | Stubbed change event so you know the structure 3 | changeEvent = { 4 | _id: { _data: '62548f79e7f11292792497cc' }, 5 | operationType: 'insert', 6 | clusterTime: { 7 | "$timestamp": { 8 | t: 1649712420, 9 | i:6 10 | } 11 | }, 12 | ns: { 13 | db: 'sample_mflix', 14 | coll: 'movies' 15 | }, 16 | documentKey: { 17 | plot: 'wtf', 18 | _id: { 19 | "$oid": "599af247bb69cd89961c986d" 20 | } 21 | }, 22 | fullDocument: { 23 | _id: { 24 | "$oid": "599af247bb69cd89961c986d" 25 | }, 26 | plot: 'wtf' 27 | } 28 | } 29 | End stubbed changeEvent to understand 30 | */ 31 | exports = async function (changeEvent) { 32 | 33 | 34 | let isNewPlot = async function () { 35 | if (changeEvent.operationType.type === "insert") { 36 | return true; 37 | } else { 38 | console.log("was not an insert"); 39 | } 40 | }; 41 | 42 | /* 43 | if insert happens add the vector 44 | TODO: Add some logic to support replace somewhere 45 | */ 46 | if (isNewPlot) { 47 | const collection = context.services 48 | .get("Cluster0") 49 | .db("sample_mflix") 50 | .collection("movies"); 51 | const docId = changeEvent.fullDocument._id; 52 | const field_to_vectorize = "plot"; 53 | const vector_field_name = field_to_vectorize + "_vector"; 54 | /* create a try/catch block to handle errors on requests */ 55 | try { 56 | /* create a variable to hold the response from the API */ 57 | 58 | const response = await context.http.post({ 59 | url: "https://scalethebrain.com/rest_vector", 60 | body: { 61 | field_to_vectorize: changeEvent.fullDocument[field_to_vectorize], 62 | }, 63 | encodeBodyAsJSON: true, 64 | }); 65 | /* store and parse the EJSON response */ 66 | let ejson_body = EJSON.parse(response.body.text()); 67 | /* update the document with the new value of vectors */ 68 | let update = collection.updateOne( 69 | { _id: docId }, 70 | { $set: { vector_field_name: ejson_body.vector } } 71 | ); 72 | 73 | /* return the response */ 74 | } catch (err) { 75 | /* if there is an error, log it */ 76 | console.log(err); 77 | } 78 | } 79 | return { message: "success" }; 80 | }; 81 | 82 | /* end of function to vectorize data added the DB */ 83 | -------------------------------------------------------------------------------- /vectorize_collection.js: -------------------------------------------------------------------------------- 1 | /* 2 | a function to initially vectorize data in the DB 3 | in the sample_mflix.movies namespace. You can run 4 | this from your workstation or a VM with network 5 | access to your Atlas cluster. I forgot a lot of 6 | what I learned from Crockford back in the day, 7 | */ 8 | const { MongoClient, ServerApiVersion } = require("mongodb"); 9 | const axios = require('axios'); 10 | /* see .example_env and create a .env file with your own credentials */ 11 | const path = require('path') 12 | require("dotenv").config({ path: path.resolve(__dirname, './.env') }); 13 | 14 | 15 | /* function to vectorize data in the sample_mflix.users namespace */ 16 | 17 | const username = process.env.MONGODB_USER 18 | const password = process.env.MONGODB_PASSWORD 19 | const clusterHostname = process.env.MONGODB_HOSTNAME; 20 | /* 21 | define a function to iterate though an entire collection 22 | and make updates to existing documents based on the returned 23 | value of an API. 24 | */ 25 | async function iterate_and_update(targetCollection,field) { 26 | try { 27 | /* find all docs where the field exists */ 28 | const queryObj = {}; 29 | const fieldName = field; 30 | queryObj[fieldName] = {$exists: true}; 31 | const docs = targetCollection.find(queryObj); 32 | /* iterate through all the documents in the collection where the field exists */ 33 | for await (const doc of docs) { 34 | 35 | await axios.post("https://scalethebrain.com/rest_vector", {"field_to_vectorize": doc[field]}) 36 | .then((response) => { 37 | const vectors_returned = response.data.vector; 38 | /* update the document with the new value of vectors and 39 | return the update to the collection */ 40 | return targetCollection.updateOne( 41 | { _id: doc._id }, 42 | { $set: {plot_vectors: vectors_returned } } 43 | ) 44 | } 45 | ) 46 | } 47 | } catch(err) { 48 | console.log(err); 49 | } 50 | } 51 | 52 | /* 53 | name the function to vectorize an entire collection 54 | and export it to be called from the command line 55 | with 56 | */ 57 | 58 | module.exports.vectorize_collection = async function (field = "plot") { 59 | const uri = `mongodb+srv://${username}:${password}@${clusterHostname}.mongodb.net/admin?retryWrites=true&w=majority`; 60 | 61 | /* Create a new MongoClient */ 62 | const client = new MongoClient(uri, { 63 | useNewUrlParser: true, 64 | useUnifiedTopology: true, 65 | serverApi: ServerApiVersion.v1 66 | }); 67 | /* connect to the cluster and collection */ 68 | await client.connect(); 69 | 70 | const collection = client.db("sample_mflix").collection("movies"); 71 | 72 | /* invoke the function to iterate through the collection and update the documents */ 73 | iterate_and_update(collection, field); 74 | 75 | /* close the connection to the cluster */ 76 | return { message: "success" }; 77 | client.close(); 78 | }; 79 | 80 | /* end of function to vectorize data in the sample_mflix.users namespace */ 81 | 82 | /* command to run vectorize_collection fuunction from the command line: */ 83 | module.exports.vectorize_collection() -------------------------------------------------------------------------------- /vectorize_queries.js: -------------------------------------------------------------------------------- 1 | // This function is the endpoint's request handler. 2 | // exports = async function({ query, headers, body}, response) { 3 | exports = async function ({ headers, body}, response) { 4 | const conn = context.services 5 | .get("mongodb-atlas") 6 | .db("sample_mflix") 7 | .collection("movies"); 8 | var vectorAPIResponse = await context.http 9 | .post({ 10 | url: "https://scalethebrain.com/rest_vector", 11 | body: { 12 | field_to_vectorize: body, 13 | }, 14 | encodeBodyAsJSON: true, 15 | }) 16 | .then((response) => { 17 | const ejson_body = EJSON.parse(response.body.text()); 18 | return ejson_body; 19 | }) 20 | .catch((err) => { 21 | console.log(err); 22 | }); 23 | 24 | /* store query vector and build the aggregation */ 25 | 26 | query_vector = vectorAPIResponse.vector; 27 | const searchAggregation = [ 28 | { 29 | $search: { 30 | knnBeta: { 31 | path: "plot_vectors", 32 | vector: query_vector, 33 | k: 5 34 | }, 35 | }, 36 | } 37 | ]; 38 | const searchResults = await conn.aggregate(searchAggregation).toArray(); 39 | 40 | /* return the search results */ 41 | return searchResults; 42 | }; --------------------------------------------------------------------------------