├── .env.example ├── .gitignore ├── README.md ├── package-lock.json ├── package.json ├── prisma └── schema.prisma ├── src ├── declare.t.ts ├── index.ts ├── obsidian │ ├── index.ts │ └── utils.ts ├── server.ts ├── utils.ts └── youtube │ └── index.ts ├── test ├── batch-obsidian-job.ts ├── data-processing.ts ├── index.ts ├── obsidian.ts ├── test.json └── youtube.ts └── tsconfig.json /.env.example: -------------------------------------------------------------------------------- 1 | API_KEY= 2 | OBSIDIAN_ROOT_PATH= 3 | DATABASE_URL= -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .env 3 | lib 4 | obsidian-docs 5 | .DS_Store -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # openai-embeddings-ts 2 | 3 | In its current state, the project sits alongside the Obsidian folder. Meaning, the project uses node.js to go to the Obsidian root folder and read all the markdown files. From there it chunks and embeds them as vectors with OpenAI embeddings (https://beta.openai.com/docs/guides/embeddings/what-are-embeddings). Then it stores those vectors in a local database (I went with Postgres https://www.postgresql.org/ but you could do anything that holds data like Mongo, Firebase, etc). To query, there is an endpoint (`/query`) that is like a Google Search Bar. You type what you want, and it gets converted into a vector itself (using OpenAI for this as well) and then it uses cosine similarity (https://en.wikipedia.org/wiki/Cosine_similarity) to find the best matches 4 | 5 | **very much WIP state!** 6 | 7 | ## installation 8 | 9 | git clone this repo -> `npm install` -> copy values from `.env.example` into `.env` 10 | 11 | ## usage 12 | 13 | - `POST /obsidian` - add a new Obsidian Markdown file to the database 14 | - `POST /query` - search all vectors in the database 15 | - `GET /files` - list all files in database 16 | 17 | ## what is semantic search 18 | 19 | semantic similarity is kind of like searching with a thesaurus, but instead of looking for synonyms you are looking for similar items. An example: the word "apple" can refer to the fruit or the technology, it depends on the context you feed it. If I said "pass me an apple" 9 times out of 10 I'd expect the fruit instead of a computer. If I said "Can I borrow your charger for my apple device?" you'd know I'm referring to the technology, even though the word "apple" is the same. 20 | 21 | The critical thing here is context. Semantic search understands how things are related (CS PHDs call it clustering). This search opens the door to searching for items contextually, meaning people no longer need to know an exact term in a database just something like "laptop" or "fruit" to get all instances of "apple" in the database 22 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "openai-embeddings", 3 | "version": "1.0.0", 4 | "lockfileVersion": 2, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "openai-embeddings", 9 | "version": "1.0.0", 10 | "license": "ISC", 11 | "dependencies": { 12 | "@prisma/client": "^3.7.0", 13 | "axios": "^0.24.0", 14 | "compute-cosine-similarity": "^1.0.0", 15 | "csv-writer": "^1.6.0", 16 | "dotenv": "^10.0.0", 17 | "express": "^4.17.2", 18 | "gpt-3-encoder": "^1.1.3", 19 | "lodash": "^4.17.21", 20 | "moment": "^2.29.1", 21 | "prisma-offset-pagination": "^0.0.4" 22 | }, 23 | "devDependencies": { 24 | "@types/express": "^4.17.13", 25 | "@types/lodash": "^4.14.178", 26 | "@types/node": "^16.11.12", 27 | "prisma": "^3.7.0" 28 | } 29 | }, 30 | "node_modules/@prisma/client": { 31 | "version": "3.7.0", 32 | "resolved": "https://registry.npmjs.org/@prisma/client/-/client-3.7.0.tgz", 33 | "integrity": "sha512-fUJMvBOX5C7JPc0e3CJD6Gbelbu4dMJB4ScYpiht8HMUnRShw20ULOipTopjNtl6ekHQJ4muI7pXlQxWS9nMbw==", 34 | "hasInstallScript": true, 35 | "dependencies": { 36 | "@prisma/engines-version": "3.7.0-31.8746e055198f517658c08a0c426c7eec87f5a85f" 37 | }, 38 | "engines": { 39 | "node": ">=12.6" 40 | }, 41 | "peerDependencies": { 42 | "prisma": "*" 43 | }, 44 | "peerDependenciesMeta": { 45 | "prisma": { 46 | "optional": true 47 | } 48 | } 49 | }, 50 | "node_modules/@prisma/engines": { 51 | "version": "3.7.0-31.8746e055198f517658c08a0c426c7eec87f5a85f", 52 | "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-3.7.0-31.8746e055198f517658c08a0c426c7eec87f5a85f.tgz", 53 | "integrity": "sha512-W549ub5NlgexNhR8EFstA/UwAWq3Zq0w9aNkraqsozVCt2CsX+lK4TK7IW5OZVSnxHwRjrgEAt3r9yPy8nZQRg==", 54 | "devOptional": true, 55 | "hasInstallScript": true 56 | }, 57 | "node_modules/@prisma/engines-version": { 58 | "version": "3.7.0-31.8746e055198f517658c08a0c426c7eec87f5a85f", 59 | "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-3.7.0-31.8746e055198f517658c08a0c426c7eec87f5a85f.tgz", 60 | "integrity": "sha512-+qx2b+HK7BKF4VCa0LZ/t1QCXsu6SmvhUQyJkOD2aPpmOzket4fEnSKQZSB0i5tl7rwCDsvAiSeK8o7rf+yvwg==" 61 | }, 62 | "node_modules/@types/body-parser": { 63 | "version": "1.19.2", 64 | "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", 65 | "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", 66 | "dev": true, 67 | "dependencies": { 68 | "@types/connect": "*", 69 | "@types/node": "*" 70 | } 71 | }, 72 | "node_modules/@types/connect": { 73 | "version": "3.4.35", 74 | "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", 75 | "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", 76 | "dev": true, 77 | "dependencies": { 78 | "@types/node": "*" 79 | } 80 | }, 81 | "node_modules/@types/express": { 82 | "version": "4.17.13", 83 | "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", 84 | "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", 85 | "dev": true, 86 | "dependencies": { 87 | "@types/body-parser": "*", 88 | "@types/express-serve-static-core": "^4.17.18", 89 | "@types/qs": "*", 90 | "@types/serve-static": "*" 91 | } 92 | }, 93 | "node_modules/@types/express-serve-static-core": { 94 | "version": "4.17.27", 95 | "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.27.tgz", 96 | "integrity": "sha512-e/sVallzUTPdyOTiqi8O8pMdBBphscvI6E4JYaKlja4Lm+zh7UFSSdW5VMkRbhDtmrONqOUHOXRguPsDckzxNA==", 97 | "dev": true, 98 | "dependencies": { 99 | "@types/node": "*", 100 | "@types/qs": "*", 101 | "@types/range-parser": "*" 102 | } 103 | }, 104 | "node_modules/@types/lodash": { 105 | "version": "4.14.178", 106 | "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.178.tgz", 107 | "integrity": "sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw==", 108 | "dev": true 109 | }, 110 | "node_modules/@types/mime": { 111 | "version": "1.3.2", 112 | "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", 113 | "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==", 114 | "dev": true 115 | }, 116 | "node_modules/@types/node": { 117 | "version": "16.11.12", 118 | "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.12.tgz", 119 | "integrity": "sha512-+2Iggwg7PxoO5Kyhvsq9VarmPbIelXP070HMImEpbtGCoyWNINQj4wzjbQCXzdHTRXnqufutJb5KAURZANNBAw==", 120 | "dev": true 121 | }, 122 | "node_modules/@types/qs": { 123 | "version": "6.9.7", 124 | "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", 125 | "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", 126 | "dev": true 127 | }, 128 | "node_modules/@types/range-parser": { 129 | "version": "1.2.4", 130 | "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", 131 | "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", 132 | "dev": true 133 | }, 134 | "node_modules/@types/serve-static": { 135 | "version": "1.13.10", 136 | "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", 137 | "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", 138 | "dev": true, 139 | "dependencies": { 140 | "@types/mime": "^1", 141 | "@types/node": "*" 142 | } 143 | }, 144 | "node_modules/accepts": { 145 | "version": "1.3.7", 146 | "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", 147 | "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", 148 | "dependencies": { 149 | "mime-types": "~2.1.24", 150 | "negotiator": "0.6.2" 151 | }, 152 | "engines": { 153 | "node": ">= 0.6" 154 | } 155 | }, 156 | "node_modules/array-flatten": { 157 | "version": "1.1.1", 158 | "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", 159 | "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" 160 | }, 161 | "node_modules/axios": { 162 | "version": "0.24.0", 163 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz", 164 | "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==", 165 | "dependencies": { 166 | "follow-redirects": "^1.14.4" 167 | } 168 | }, 169 | "node_modules/body-parser": { 170 | "version": "1.19.1", 171 | "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.1.tgz", 172 | "integrity": "sha512-8ljfQi5eBk8EJfECMrgqNGWPEY5jWP+1IzkzkGdFFEwFQZZyaZ21UqdaHktgiMlH0xLHqIFtE/u2OYE5dOtViA==", 173 | "dependencies": { 174 | "bytes": "3.1.1", 175 | "content-type": "~1.0.4", 176 | "debug": "2.6.9", 177 | "depd": "~1.1.2", 178 | "http-errors": "1.8.1", 179 | "iconv-lite": "0.4.24", 180 | "on-finished": "~2.3.0", 181 | "qs": "6.9.6", 182 | "raw-body": "2.4.2", 183 | "type-is": "~1.6.18" 184 | }, 185 | "engines": { 186 | "node": ">= 0.8" 187 | } 188 | }, 189 | "node_modules/bytes": { 190 | "version": "3.1.1", 191 | "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.1.tgz", 192 | "integrity": "sha512-dWe4nWO/ruEOY7HkUJ5gFt1DCFV9zPRoJr8pV0/ASQermOZjtq8jMjOprC0Kd10GLN+l7xaUPvxzJFWtxGu8Fg==", 193 | "engines": { 194 | "node": ">= 0.8" 195 | } 196 | }, 197 | "node_modules/compute-cosine-similarity": { 198 | "version": "1.0.0", 199 | "resolved": "https://registry.npmjs.org/compute-cosine-similarity/-/compute-cosine-similarity-1.0.0.tgz", 200 | "integrity": "sha1-KfK/Lnuu+iMcq6QFkaMcF7GaTpU=", 201 | "dependencies": { 202 | "compute-dot": "^1.1.0", 203 | "compute-l2norm": "^1.1.0", 204 | "validate.io-array": "^1.0.5", 205 | "validate.io-function": "^1.0.2" 206 | } 207 | }, 208 | "node_modules/compute-dot": { 209 | "version": "1.1.0", 210 | "resolved": "https://registry.npmjs.org/compute-dot/-/compute-dot-1.1.0.tgz", 211 | "integrity": "sha1-AaW6LHr3O5kAKsslhFnJV2qCMtw=", 212 | "dependencies": { 213 | "validate.io-array": "^1.0.3", 214 | "validate.io-function": "^1.0.2" 215 | } 216 | }, 217 | "node_modules/compute-l2norm": { 218 | "version": "1.1.0", 219 | "resolved": "https://registry.npmjs.org/compute-l2norm/-/compute-l2norm-1.1.0.tgz", 220 | "integrity": "sha1-vQkTHGs2yNcMaDNOF2AJpOCpiaw=", 221 | "dependencies": { 222 | "validate.io-array": "^1.0.3", 223 | "validate.io-function": "^1.0.2" 224 | } 225 | }, 226 | "node_modules/content-disposition": { 227 | "version": "0.5.4", 228 | "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", 229 | "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", 230 | "dependencies": { 231 | "safe-buffer": "5.2.1" 232 | }, 233 | "engines": { 234 | "node": ">= 0.6" 235 | } 236 | }, 237 | "node_modules/content-type": { 238 | "version": "1.0.4", 239 | "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", 240 | "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", 241 | "engines": { 242 | "node": ">= 0.6" 243 | } 244 | }, 245 | "node_modules/cookie": { 246 | "version": "0.4.1", 247 | "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.1.tgz", 248 | "integrity": "sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA==", 249 | "engines": { 250 | "node": ">= 0.6" 251 | } 252 | }, 253 | "node_modules/cookie-signature": { 254 | "version": "1.0.6", 255 | "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", 256 | "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" 257 | }, 258 | "node_modules/csv-writer": { 259 | "version": "1.6.0", 260 | "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", 261 | "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==" 262 | }, 263 | "node_modules/debug": { 264 | "version": "2.6.9", 265 | "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", 266 | "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", 267 | "dependencies": { 268 | "ms": "2.0.0" 269 | } 270 | }, 271 | "node_modules/depd": { 272 | "version": "1.1.2", 273 | "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", 274 | "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", 275 | "engines": { 276 | "node": ">= 0.6" 277 | } 278 | }, 279 | "node_modules/destroy": { 280 | "version": "1.0.4", 281 | "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", 282 | "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" 283 | }, 284 | "node_modules/dotenv": { 285 | "version": "10.0.0", 286 | "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz", 287 | "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==", 288 | "engines": { 289 | "node": ">=10" 290 | } 291 | }, 292 | "node_modules/ee-first": { 293 | "version": "1.1.1", 294 | "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", 295 | "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" 296 | }, 297 | "node_modules/encodeurl": { 298 | "version": "1.0.2", 299 | "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", 300 | "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", 301 | "engines": { 302 | "node": ">= 0.8" 303 | } 304 | }, 305 | "node_modules/escape-html": { 306 | "version": "1.0.3", 307 | "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", 308 | "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" 309 | }, 310 | "node_modules/etag": { 311 | "version": "1.8.1", 312 | "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", 313 | "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", 314 | "engines": { 315 | "node": ">= 0.6" 316 | } 317 | }, 318 | "node_modules/express": { 319 | "version": "4.17.2", 320 | "resolved": "https://registry.npmjs.org/express/-/express-4.17.2.tgz", 321 | "integrity": "sha512-oxlxJxcQlYwqPWKVJJtvQiwHgosH/LrLSPA+H4UxpyvSS6jC5aH+5MoHFM+KABgTOt0APue4w66Ha8jCUo9QGg==", 322 | "dependencies": { 323 | "accepts": "~1.3.7", 324 | "array-flatten": "1.1.1", 325 | "body-parser": "1.19.1", 326 | "content-disposition": "0.5.4", 327 | "content-type": "~1.0.4", 328 | "cookie": "0.4.1", 329 | "cookie-signature": "1.0.6", 330 | "debug": "2.6.9", 331 | "depd": "~1.1.2", 332 | "encodeurl": "~1.0.2", 333 | "escape-html": "~1.0.3", 334 | "etag": "~1.8.1", 335 | "finalhandler": "~1.1.2", 336 | "fresh": "0.5.2", 337 | "merge-descriptors": "1.0.1", 338 | "methods": "~1.1.2", 339 | "on-finished": "~2.3.0", 340 | "parseurl": "~1.3.3", 341 | "path-to-regexp": "0.1.7", 342 | "proxy-addr": "~2.0.7", 343 | "qs": "6.9.6", 344 | "range-parser": "~1.2.1", 345 | "safe-buffer": "5.2.1", 346 | "send": "0.17.2", 347 | "serve-static": "1.14.2", 348 | "setprototypeof": "1.2.0", 349 | "statuses": "~1.5.0", 350 | "type-is": "~1.6.18", 351 | "utils-merge": "1.0.1", 352 | "vary": "~1.1.2" 353 | }, 354 | "engines": { 355 | "node": ">= 0.10.0" 356 | } 357 | }, 358 | "node_modules/finalhandler": { 359 | "version": "1.1.2", 360 | "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", 361 | "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", 362 | "dependencies": { 363 | "debug": "2.6.9", 364 | "encodeurl": "~1.0.2", 365 | "escape-html": "~1.0.3", 366 | "on-finished": "~2.3.0", 367 | "parseurl": "~1.3.3", 368 | "statuses": "~1.5.0", 369 | "unpipe": "~1.0.0" 370 | }, 371 | "engines": { 372 | "node": ">= 0.8" 373 | } 374 | }, 375 | "node_modules/follow-redirects": { 376 | "version": "1.14.5", 377 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz", 378 | "integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==", 379 | "funding": [ 380 | { 381 | "type": "individual", 382 | "url": "https://github.com/sponsors/RubenVerborgh" 383 | } 384 | ], 385 | "engines": { 386 | "node": ">=4.0" 387 | }, 388 | "peerDependenciesMeta": { 389 | "debug": { 390 | "optional": true 391 | } 392 | } 393 | }, 394 | "node_modules/forwarded": { 395 | "version": "0.2.0", 396 | "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", 397 | "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", 398 | "engines": { 399 | "node": ">= 0.6" 400 | } 401 | }, 402 | "node_modules/fresh": { 403 | "version": "0.5.2", 404 | "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", 405 | "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", 406 | "engines": { 407 | "node": ">= 0.6" 408 | } 409 | }, 410 | "node_modules/gpt-3-encoder": { 411 | "version": "1.1.3", 412 | "resolved": "https://registry.npmjs.org/gpt-3-encoder/-/gpt-3-encoder-1.1.3.tgz", 413 | "integrity": "sha512-Ep1+I5hQ5VcLF5qKc2FyqvSj2T0Al9LNC0uXNq6Cx0Twl6Gq2RWG7Q5Mo/9Of30xcAegsgemNtuulPEiwanUww==" 414 | }, 415 | "node_modules/http-errors": { 416 | "version": "1.8.1", 417 | "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", 418 | "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", 419 | "dependencies": { 420 | "depd": "~1.1.2", 421 | "inherits": "2.0.4", 422 | "setprototypeof": "1.2.0", 423 | "statuses": ">= 1.5.0 < 2", 424 | "toidentifier": "1.0.1" 425 | }, 426 | "engines": { 427 | "node": ">= 0.6" 428 | } 429 | }, 430 | "node_modules/iconv-lite": { 431 | "version": "0.4.24", 432 | "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", 433 | "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", 434 | "dependencies": { 435 | "safer-buffer": ">= 2.1.2 < 3" 436 | }, 437 | "engines": { 438 | "node": ">=0.10.0" 439 | } 440 | }, 441 | "node_modules/inherits": { 442 | "version": "2.0.4", 443 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 444 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" 445 | }, 446 | "node_modules/ipaddr.js": { 447 | "version": "1.9.1", 448 | "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", 449 | "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", 450 | "engines": { 451 | "node": ">= 0.10" 452 | } 453 | }, 454 | "node_modules/lodash": { 455 | "version": "4.17.21", 456 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", 457 | "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" 458 | }, 459 | "node_modules/media-typer": { 460 | "version": "0.3.0", 461 | "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", 462 | "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", 463 | "engines": { 464 | "node": ">= 0.6" 465 | } 466 | }, 467 | "node_modules/merge-descriptors": { 468 | "version": "1.0.1", 469 | "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", 470 | "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" 471 | }, 472 | "node_modules/methods": { 473 | "version": "1.1.2", 474 | "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", 475 | "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=", 476 | "engines": { 477 | "node": ">= 0.6" 478 | } 479 | }, 480 | "node_modules/mime": { 481 | "version": "1.6.0", 482 | "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", 483 | "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", 484 | "bin": { 485 | "mime": "cli.js" 486 | }, 487 | "engines": { 488 | "node": ">=4" 489 | } 490 | }, 491 | "node_modules/mime-db": { 492 | "version": "1.51.0", 493 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", 494 | "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==", 495 | "engines": { 496 | "node": ">= 0.6" 497 | } 498 | }, 499 | "node_modules/mime-types": { 500 | "version": "2.1.34", 501 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", 502 | "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", 503 | "dependencies": { 504 | "mime-db": "1.51.0" 505 | }, 506 | "engines": { 507 | "node": ">= 0.6" 508 | } 509 | }, 510 | "node_modules/moment": { 511 | "version": "2.29.1", 512 | "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", 513 | "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==", 514 | "engines": { 515 | "node": "*" 516 | } 517 | }, 518 | "node_modules/ms": { 519 | "version": "2.0.0", 520 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", 521 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" 522 | }, 523 | "node_modules/negotiator": { 524 | "version": "0.6.2", 525 | "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", 526 | "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==", 527 | "engines": { 528 | "node": ">= 0.6" 529 | } 530 | }, 531 | "node_modules/on-finished": { 532 | "version": "2.3.0", 533 | "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", 534 | "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", 535 | "dependencies": { 536 | "ee-first": "1.1.1" 537 | }, 538 | "engines": { 539 | "node": ">= 0.8" 540 | } 541 | }, 542 | "node_modules/parseurl": { 543 | "version": "1.3.3", 544 | "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", 545 | "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", 546 | "engines": { 547 | "node": ">= 0.8" 548 | } 549 | }, 550 | "node_modules/path-to-regexp": { 551 | "version": "0.1.7", 552 | "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", 553 | "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" 554 | }, 555 | "node_modules/prisma": { 556 | "version": "3.7.0", 557 | "resolved": "https://registry.npmjs.org/prisma/-/prisma-3.7.0.tgz", 558 | "integrity": "sha512-pzgc95msPLcCHqOli7Hnabu/GRfSGSUWl5s2P6N13T/rgMB+NNeKbxCmzQiZT2yLOeLEPivV6YrW1oeQIwJxcg==", 559 | "devOptional": true, 560 | "hasInstallScript": true, 561 | "dependencies": { 562 | "@prisma/engines": "3.7.0-31.8746e055198f517658c08a0c426c7eec87f5a85f" 563 | }, 564 | "bin": { 565 | "prisma": "build/index.js", 566 | "prisma2": "build/index.js" 567 | }, 568 | "engines": { 569 | "node": ">=12.6" 570 | } 571 | }, 572 | "node_modules/prisma-offset-pagination": { 573 | "version": "0.0.4", 574 | "resolved": "https://registry.npmjs.org/prisma-offset-pagination/-/prisma-offset-pagination-0.0.4.tgz", 575 | "integrity": "sha512-I/ITNhDa6gc4qqcVIwfr/CV/3+UsXSoAvYbMdl5UmGF9rUv6MNYaai+dirGhpHIOstzEZqWa4yHgAMlDxlSa7w==" 576 | }, 577 | "node_modules/proxy-addr": { 578 | "version": "2.0.7", 579 | "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", 580 | "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", 581 | "dependencies": { 582 | "forwarded": "0.2.0", 583 | "ipaddr.js": "1.9.1" 584 | }, 585 | "engines": { 586 | "node": ">= 0.10" 587 | } 588 | }, 589 | "node_modules/qs": { 590 | "version": "6.9.6", 591 | "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.6.tgz", 592 | "integrity": "sha512-TIRk4aqYLNoJUbd+g2lEdz5kLWIuTMRagAXxl78Q0RiVjAOugHmeKNGdd3cwo/ktpf9aL9epCfFqWDEKysUlLQ==", 593 | "engines": { 594 | "node": ">=0.6" 595 | }, 596 | "funding": { 597 | "url": "https://github.com/sponsors/ljharb" 598 | } 599 | }, 600 | "node_modules/range-parser": { 601 | "version": "1.2.1", 602 | "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", 603 | "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", 604 | "engines": { 605 | "node": ">= 0.6" 606 | } 607 | }, 608 | "node_modules/raw-body": { 609 | "version": "2.4.2", 610 | "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.2.tgz", 611 | "integrity": "sha512-RPMAFUJP19WIet/99ngh6Iv8fzAbqum4Li7AD6DtGaW2RpMB/11xDoalPiJMTbu6I3hkbMVkATvZrqb9EEqeeQ==", 612 | "dependencies": { 613 | "bytes": "3.1.1", 614 | "http-errors": "1.8.1", 615 | "iconv-lite": "0.4.24", 616 | "unpipe": "1.0.0" 617 | }, 618 | "engines": { 619 | "node": ">= 0.8" 620 | } 621 | }, 622 | "node_modules/safe-buffer": { 623 | "version": "5.2.1", 624 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", 625 | "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", 626 | "funding": [ 627 | { 628 | "type": "github", 629 | "url": "https://github.com/sponsors/feross" 630 | }, 631 | { 632 | "type": "patreon", 633 | "url": "https://www.patreon.com/feross" 634 | }, 635 | { 636 | "type": "consulting", 637 | "url": "https://feross.org/support" 638 | } 639 | ] 640 | }, 641 | "node_modules/safer-buffer": { 642 | "version": "2.1.2", 643 | "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", 644 | "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" 645 | }, 646 | "node_modules/send": { 647 | "version": "0.17.2", 648 | "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", 649 | "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", 650 | "dependencies": { 651 | "debug": "2.6.9", 652 | "depd": "~1.1.2", 653 | "destroy": "~1.0.4", 654 | "encodeurl": "~1.0.2", 655 | "escape-html": "~1.0.3", 656 | "etag": "~1.8.1", 657 | "fresh": "0.5.2", 658 | "http-errors": "1.8.1", 659 | "mime": "1.6.0", 660 | "ms": "2.1.3", 661 | "on-finished": "~2.3.0", 662 | "range-parser": "~1.2.1", 663 | "statuses": "~1.5.0" 664 | }, 665 | "engines": { 666 | "node": ">= 0.8.0" 667 | } 668 | }, 669 | "node_modules/send/node_modules/ms": { 670 | "version": "2.1.3", 671 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", 672 | "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" 673 | }, 674 | "node_modules/serve-static": { 675 | "version": "1.14.2", 676 | "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", 677 | "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", 678 | "dependencies": { 679 | "encodeurl": "~1.0.2", 680 | "escape-html": "~1.0.3", 681 | "parseurl": "~1.3.3", 682 | "send": "0.17.2" 683 | }, 684 | "engines": { 685 | "node": ">= 0.8.0" 686 | } 687 | }, 688 | "node_modules/setprototypeof": { 689 | "version": "1.2.0", 690 | "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", 691 | "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" 692 | }, 693 | "node_modules/statuses": { 694 | "version": "1.5.0", 695 | "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", 696 | "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", 697 | "engines": { 698 | "node": ">= 0.6" 699 | } 700 | }, 701 | "node_modules/toidentifier": { 702 | "version": "1.0.1", 703 | "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", 704 | "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", 705 | "engines": { 706 | "node": ">=0.6" 707 | } 708 | }, 709 | "node_modules/type-is": { 710 | "version": "1.6.18", 711 | "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", 712 | "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", 713 | "dependencies": { 714 | "media-typer": "0.3.0", 715 | "mime-types": "~2.1.24" 716 | }, 717 | "engines": { 718 | "node": ">= 0.6" 719 | } 720 | }, 721 | "node_modules/unpipe": { 722 | "version": "1.0.0", 723 | "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", 724 | "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", 725 | "engines": { 726 | "node": ">= 0.8" 727 | } 728 | }, 729 | "node_modules/utils-merge": { 730 | "version": "1.0.1", 731 | "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", 732 | "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", 733 | "engines": { 734 | "node": ">= 0.4.0" 735 | } 736 | }, 737 | "node_modules/validate.io-array": { 738 | "version": "1.0.6", 739 | "resolved": "https://registry.npmjs.org/validate.io-array/-/validate.io-array-1.0.6.tgz", 740 | "integrity": "sha1-W1osr9j4uFq7L4hroVPy2Tond00=" 741 | }, 742 | "node_modules/validate.io-function": { 743 | "version": "1.0.2", 744 | "resolved": "https://registry.npmjs.org/validate.io-function/-/validate.io-function-1.0.2.tgz", 745 | "integrity": "sha1-NDoZgC7TsZaCaceA5VjpNBHAutc=" 746 | }, 747 | "node_modules/vary": { 748 | "version": "1.1.2", 749 | "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", 750 | "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=", 751 | "engines": { 752 | "node": ">= 0.8" 753 | } 754 | } 755 | }, 756 | "dependencies": { 757 | "@prisma/client": { 758 | "version": "3.7.0", 759 | "resolved": "https://registry.npmjs.org/@prisma/client/-/client-3.7.0.tgz", 760 | "integrity": "sha512-fUJMvBOX5C7JPc0e3CJD6Gbelbu4dMJB4ScYpiht8HMUnRShw20ULOipTopjNtl6ekHQJ4muI7pXlQxWS9nMbw==", 761 | "requires": { 762 | "@prisma/engines-version": "3.7.0-31.8746e055198f517658c08a0c426c7eec87f5a85f" 763 | } 764 | }, 765 | "@prisma/engines": { 766 | "version": "3.7.0-31.8746e055198f517658c08a0c426c7eec87f5a85f", 767 | "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-3.7.0-31.8746e055198f517658c08a0c426c7eec87f5a85f.tgz", 768 | "integrity": "sha512-W549ub5NlgexNhR8EFstA/UwAWq3Zq0w9aNkraqsozVCt2CsX+lK4TK7IW5OZVSnxHwRjrgEAt3r9yPy8nZQRg==", 769 | "devOptional": true 770 | }, 771 | "@prisma/engines-version": { 772 | "version": "3.7.0-31.8746e055198f517658c08a0c426c7eec87f5a85f", 773 | "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-3.7.0-31.8746e055198f517658c08a0c426c7eec87f5a85f.tgz", 774 | "integrity": "sha512-+qx2b+HK7BKF4VCa0LZ/t1QCXsu6SmvhUQyJkOD2aPpmOzket4fEnSKQZSB0i5tl7rwCDsvAiSeK8o7rf+yvwg==" 775 | }, 776 | "@types/body-parser": { 777 | "version": "1.19.2", 778 | "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", 779 | "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", 780 | "dev": true, 781 | "requires": { 782 | "@types/connect": "*", 783 | "@types/node": "*" 784 | } 785 | }, 786 | "@types/connect": { 787 | "version": "3.4.35", 788 | "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", 789 | "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", 790 | "dev": true, 791 | "requires": { 792 | "@types/node": "*" 793 | } 794 | }, 795 | "@types/express": { 796 | "version": "4.17.13", 797 | "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", 798 | "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", 799 | "dev": true, 800 | "requires": { 801 | "@types/body-parser": "*", 802 | "@types/express-serve-static-core": "^4.17.18", 803 | "@types/qs": "*", 804 | "@types/serve-static": "*" 805 | } 806 | }, 807 | "@types/express-serve-static-core": { 808 | "version": "4.17.27", 809 | "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.27.tgz", 810 | "integrity": "sha512-e/sVallzUTPdyOTiqi8O8pMdBBphscvI6E4JYaKlja4Lm+zh7UFSSdW5VMkRbhDtmrONqOUHOXRguPsDckzxNA==", 811 | "dev": true, 812 | "requires": { 813 | "@types/node": "*", 814 | "@types/qs": "*", 815 | "@types/range-parser": "*" 816 | } 817 | }, 818 | "@types/lodash": { 819 | "version": "4.14.178", 820 | "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.178.tgz", 821 | "integrity": "sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw==", 822 | "dev": true 823 | }, 824 | "@types/mime": { 825 | "version": "1.3.2", 826 | "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", 827 | "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==", 828 | "dev": true 829 | }, 830 | "@types/node": { 831 | "version": "16.11.12", 832 | "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.12.tgz", 833 | "integrity": "sha512-+2Iggwg7PxoO5Kyhvsq9VarmPbIelXP070HMImEpbtGCoyWNINQj4wzjbQCXzdHTRXnqufutJb5KAURZANNBAw==", 834 | "dev": true 835 | }, 836 | "@types/qs": { 837 | "version": "6.9.7", 838 | "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", 839 | "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", 840 | "dev": true 841 | }, 842 | "@types/range-parser": { 843 | "version": "1.2.4", 844 | "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", 845 | "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", 846 | "dev": true 847 | }, 848 | "@types/serve-static": { 849 | "version": "1.13.10", 850 | "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", 851 | "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", 852 | "dev": true, 853 | "requires": { 854 | "@types/mime": "^1", 855 | "@types/node": "*" 856 | } 857 | }, 858 | "accepts": { 859 | "version": "1.3.7", 860 | "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", 861 | "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", 862 | "requires": { 863 | "mime-types": "~2.1.24", 864 | "negotiator": "0.6.2" 865 | } 866 | }, 867 | "array-flatten": { 868 | "version": "1.1.1", 869 | "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", 870 | "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" 871 | }, 872 | "axios": { 873 | "version": "0.24.0", 874 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz", 875 | "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==", 876 | "requires": { 877 | "follow-redirects": "^1.14.4" 878 | } 879 | }, 880 | "body-parser": { 881 | "version": "1.19.1", 882 | "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.1.tgz", 883 | "integrity": "sha512-8ljfQi5eBk8EJfECMrgqNGWPEY5jWP+1IzkzkGdFFEwFQZZyaZ21UqdaHktgiMlH0xLHqIFtE/u2OYE5dOtViA==", 884 | "requires": { 885 | "bytes": "3.1.1", 886 | "content-type": "~1.0.4", 887 | "debug": "2.6.9", 888 | "depd": "~1.1.2", 889 | "http-errors": "1.8.1", 890 | "iconv-lite": "0.4.24", 891 | "on-finished": "~2.3.0", 892 | "qs": "6.9.6", 893 | "raw-body": "2.4.2", 894 | "type-is": "~1.6.18" 895 | } 896 | }, 897 | "bytes": { 898 | "version": "3.1.1", 899 | "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.1.tgz", 900 | "integrity": "sha512-dWe4nWO/ruEOY7HkUJ5gFt1DCFV9zPRoJr8pV0/ASQermOZjtq8jMjOprC0Kd10GLN+l7xaUPvxzJFWtxGu8Fg==" 901 | }, 902 | "compute-cosine-similarity": { 903 | "version": "1.0.0", 904 | "resolved": "https://registry.npmjs.org/compute-cosine-similarity/-/compute-cosine-similarity-1.0.0.tgz", 905 | "integrity": "sha1-KfK/Lnuu+iMcq6QFkaMcF7GaTpU=", 906 | "requires": { 907 | "compute-dot": "^1.1.0", 908 | "compute-l2norm": "^1.1.0", 909 | "validate.io-array": "^1.0.5", 910 | "validate.io-function": "^1.0.2" 911 | } 912 | }, 913 | "compute-dot": { 914 | "version": "1.1.0", 915 | "resolved": "https://registry.npmjs.org/compute-dot/-/compute-dot-1.1.0.tgz", 916 | "integrity": "sha1-AaW6LHr3O5kAKsslhFnJV2qCMtw=", 917 | "requires": { 918 | "validate.io-array": "^1.0.3", 919 | "validate.io-function": "^1.0.2" 920 | } 921 | }, 922 | "compute-l2norm": { 923 | "version": "1.1.0", 924 | "resolved": "https://registry.npmjs.org/compute-l2norm/-/compute-l2norm-1.1.0.tgz", 925 | "integrity": "sha1-vQkTHGs2yNcMaDNOF2AJpOCpiaw=", 926 | "requires": { 927 | "validate.io-array": "^1.0.3", 928 | "validate.io-function": "^1.0.2" 929 | } 930 | }, 931 | "content-disposition": { 932 | "version": "0.5.4", 933 | "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", 934 | "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", 935 | "requires": { 936 | "safe-buffer": "5.2.1" 937 | } 938 | }, 939 | "content-type": { 940 | "version": "1.0.4", 941 | "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", 942 | "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" 943 | }, 944 | "cookie": { 945 | "version": "0.4.1", 946 | "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.1.tgz", 947 | "integrity": "sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA==" 948 | }, 949 | "cookie-signature": { 950 | "version": "1.0.6", 951 | "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", 952 | "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" 953 | }, 954 | "csv-writer": { 955 | "version": "1.6.0", 956 | "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", 957 | "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==" 958 | }, 959 | "debug": { 960 | "version": "2.6.9", 961 | "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", 962 | "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", 963 | "requires": { 964 | "ms": "2.0.0" 965 | } 966 | }, 967 | "depd": { 968 | "version": "1.1.2", 969 | "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", 970 | "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" 971 | }, 972 | "destroy": { 973 | "version": "1.0.4", 974 | "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", 975 | "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" 976 | }, 977 | "dotenv": { 978 | "version": "10.0.0", 979 | "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz", 980 | "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==" 981 | }, 982 | "ee-first": { 983 | "version": "1.1.1", 984 | "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", 985 | "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" 986 | }, 987 | "encodeurl": { 988 | "version": "1.0.2", 989 | "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", 990 | "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" 991 | }, 992 | "escape-html": { 993 | "version": "1.0.3", 994 | "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", 995 | "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" 996 | }, 997 | "etag": { 998 | "version": "1.8.1", 999 | "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", 1000 | "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" 1001 | }, 1002 | "express": { 1003 | "version": "4.17.2", 1004 | "resolved": "https://registry.npmjs.org/express/-/express-4.17.2.tgz", 1005 | "integrity": "sha512-oxlxJxcQlYwqPWKVJJtvQiwHgosH/LrLSPA+H4UxpyvSS6jC5aH+5MoHFM+KABgTOt0APue4w66Ha8jCUo9QGg==", 1006 | "requires": { 1007 | "accepts": "~1.3.7", 1008 | "array-flatten": "1.1.1", 1009 | "body-parser": "1.19.1", 1010 | "content-disposition": "0.5.4", 1011 | "content-type": "~1.0.4", 1012 | "cookie": "0.4.1", 1013 | "cookie-signature": "1.0.6", 1014 | "debug": "2.6.9", 1015 | "depd": "~1.1.2", 1016 | "encodeurl": "~1.0.2", 1017 | "escape-html": "~1.0.3", 1018 | "etag": "~1.8.1", 1019 | "finalhandler": "~1.1.2", 1020 | "fresh": "0.5.2", 1021 | "merge-descriptors": "1.0.1", 1022 | "methods": "~1.1.2", 1023 | "on-finished": "~2.3.0", 1024 | "parseurl": "~1.3.3", 1025 | "path-to-regexp": "0.1.7", 1026 | "proxy-addr": "~2.0.7", 1027 | "qs": "6.9.6", 1028 | "range-parser": "~1.2.1", 1029 | "safe-buffer": "5.2.1", 1030 | "send": "0.17.2", 1031 | "serve-static": "1.14.2", 1032 | "setprototypeof": "1.2.0", 1033 | "statuses": "~1.5.0", 1034 | "type-is": "~1.6.18", 1035 | "utils-merge": "1.0.1", 1036 | "vary": "~1.1.2" 1037 | } 1038 | }, 1039 | "finalhandler": { 1040 | "version": "1.1.2", 1041 | "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", 1042 | "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", 1043 | "requires": { 1044 | "debug": "2.6.9", 1045 | "encodeurl": "~1.0.2", 1046 | "escape-html": "~1.0.3", 1047 | "on-finished": "~2.3.0", 1048 | "parseurl": "~1.3.3", 1049 | "statuses": "~1.5.0", 1050 | "unpipe": "~1.0.0" 1051 | } 1052 | }, 1053 | "follow-redirects": { 1054 | "version": "1.14.5", 1055 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz", 1056 | "integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==" 1057 | }, 1058 | "forwarded": { 1059 | "version": "0.2.0", 1060 | "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", 1061 | "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" 1062 | }, 1063 | "fresh": { 1064 | "version": "0.5.2", 1065 | "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", 1066 | "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" 1067 | }, 1068 | "gpt-3-encoder": { 1069 | "version": "1.1.3", 1070 | "resolved": "https://registry.npmjs.org/gpt-3-encoder/-/gpt-3-encoder-1.1.3.tgz", 1071 | "integrity": "sha512-Ep1+I5hQ5VcLF5qKc2FyqvSj2T0Al9LNC0uXNq6Cx0Twl6Gq2RWG7Q5Mo/9Of30xcAegsgemNtuulPEiwanUww==" 1072 | }, 1073 | "http-errors": { 1074 | "version": "1.8.1", 1075 | "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", 1076 | "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", 1077 | "requires": { 1078 | "depd": "~1.1.2", 1079 | "inherits": "2.0.4", 1080 | "setprototypeof": "1.2.0", 1081 | "statuses": ">= 1.5.0 < 2", 1082 | "toidentifier": "1.0.1" 1083 | } 1084 | }, 1085 | "iconv-lite": { 1086 | "version": "0.4.24", 1087 | "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", 1088 | "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", 1089 | "requires": { 1090 | "safer-buffer": ">= 2.1.2 < 3" 1091 | } 1092 | }, 1093 | "inherits": { 1094 | "version": "2.0.4", 1095 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 1096 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" 1097 | }, 1098 | "ipaddr.js": { 1099 | "version": "1.9.1", 1100 | "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", 1101 | "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" 1102 | }, 1103 | "lodash": { 1104 | "version": "4.17.21", 1105 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", 1106 | "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" 1107 | }, 1108 | "media-typer": { 1109 | "version": "0.3.0", 1110 | "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", 1111 | "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" 1112 | }, 1113 | "merge-descriptors": { 1114 | "version": "1.0.1", 1115 | "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", 1116 | "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" 1117 | }, 1118 | "methods": { 1119 | "version": "1.1.2", 1120 | "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", 1121 | "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" 1122 | }, 1123 | "mime": { 1124 | "version": "1.6.0", 1125 | "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", 1126 | "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" 1127 | }, 1128 | "mime-db": { 1129 | "version": "1.51.0", 1130 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", 1131 | "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==" 1132 | }, 1133 | "mime-types": { 1134 | "version": "2.1.34", 1135 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", 1136 | "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", 1137 | "requires": { 1138 | "mime-db": "1.51.0" 1139 | } 1140 | }, 1141 | "moment": { 1142 | "version": "2.29.1", 1143 | "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", 1144 | "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" 1145 | }, 1146 | "ms": { 1147 | "version": "2.0.0", 1148 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", 1149 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" 1150 | }, 1151 | "negotiator": { 1152 | "version": "0.6.2", 1153 | "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", 1154 | "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" 1155 | }, 1156 | "on-finished": { 1157 | "version": "2.3.0", 1158 | "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", 1159 | "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", 1160 | "requires": { 1161 | "ee-first": "1.1.1" 1162 | } 1163 | }, 1164 | "parseurl": { 1165 | "version": "1.3.3", 1166 | "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", 1167 | "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" 1168 | }, 1169 | "path-to-regexp": { 1170 | "version": "0.1.7", 1171 | "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", 1172 | "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" 1173 | }, 1174 | "prisma": { 1175 | "version": "3.7.0", 1176 | "resolved": "https://registry.npmjs.org/prisma/-/prisma-3.7.0.tgz", 1177 | "integrity": "sha512-pzgc95msPLcCHqOli7Hnabu/GRfSGSUWl5s2P6N13T/rgMB+NNeKbxCmzQiZT2yLOeLEPivV6YrW1oeQIwJxcg==", 1178 | "devOptional": true, 1179 | "requires": { 1180 | "@prisma/engines": "3.7.0-31.8746e055198f517658c08a0c426c7eec87f5a85f" 1181 | } 1182 | }, 1183 | "prisma-offset-pagination": { 1184 | "version": "0.0.4", 1185 | "resolved": "https://registry.npmjs.org/prisma-offset-pagination/-/prisma-offset-pagination-0.0.4.tgz", 1186 | "integrity": "sha512-I/ITNhDa6gc4qqcVIwfr/CV/3+UsXSoAvYbMdl5UmGF9rUv6MNYaai+dirGhpHIOstzEZqWa4yHgAMlDxlSa7w==" 1187 | }, 1188 | "proxy-addr": { 1189 | "version": "2.0.7", 1190 | "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", 1191 | "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", 1192 | "requires": { 1193 | "forwarded": "0.2.0", 1194 | "ipaddr.js": "1.9.1" 1195 | } 1196 | }, 1197 | "qs": { 1198 | "version": "6.9.6", 1199 | "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.6.tgz", 1200 | "integrity": "sha512-TIRk4aqYLNoJUbd+g2lEdz5kLWIuTMRagAXxl78Q0RiVjAOugHmeKNGdd3cwo/ktpf9aL9epCfFqWDEKysUlLQ==" 1201 | }, 1202 | "range-parser": { 1203 | "version": "1.2.1", 1204 | "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", 1205 | "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" 1206 | }, 1207 | "raw-body": { 1208 | "version": "2.4.2", 1209 | "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.2.tgz", 1210 | "integrity": "sha512-RPMAFUJP19WIet/99ngh6Iv8fzAbqum4Li7AD6DtGaW2RpMB/11xDoalPiJMTbu6I3hkbMVkATvZrqb9EEqeeQ==", 1211 | "requires": { 1212 | "bytes": "3.1.1", 1213 | "http-errors": "1.8.1", 1214 | "iconv-lite": "0.4.24", 1215 | "unpipe": "1.0.0" 1216 | } 1217 | }, 1218 | "safe-buffer": { 1219 | "version": "5.2.1", 1220 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", 1221 | "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" 1222 | }, 1223 | "safer-buffer": { 1224 | "version": "2.1.2", 1225 | "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", 1226 | "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" 1227 | }, 1228 | "send": { 1229 | "version": "0.17.2", 1230 | "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", 1231 | "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", 1232 | "requires": { 1233 | "debug": "2.6.9", 1234 | "depd": "~1.1.2", 1235 | "destroy": "~1.0.4", 1236 | "encodeurl": "~1.0.2", 1237 | "escape-html": "~1.0.3", 1238 | "etag": "~1.8.1", 1239 | "fresh": "0.5.2", 1240 | "http-errors": "1.8.1", 1241 | "mime": "1.6.0", 1242 | "ms": "2.1.3", 1243 | "on-finished": "~2.3.0", 1244 | "range-parser": "~1.2.1", 1245 | "statuses": "~1.5.0" 1246 | }, 1247 | "dependencies": { 1248 | "ms": { 1249 | "version": "2.1.3", 1250 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", 1251 | "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" 1252 | } 1253 | } 1254 | }, 1255 | "serve-static": { 1256 | "version": "1.14.2", 1257 | "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", 1258 | "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", 1259 | "requires": { 1260 | "encodeurl": "~1.0.2", 1261 | "escape-html": "~1.0.3", 1262 | "parseurl": "~1.3.3", 1263 | "send": "0.17.2" 1264 | } 1265 | }, 1266 | "setprototypeof": { 1267 | "version": "1.2.0", 1268 | "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", 1269 | "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" 1270 | }, 1271 | "statuses": { 1272 | "version": "1.5.0", 1273 | "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", 1274 | "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" 1275 | }, 1276 | "toidentifier": { 1277 | "version": "1.0.1", 1278 | "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", 1279 | "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" 1280 | }, 1281 | "type-is": { 1282 | "version": "1.6.18", 1283 | "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", 1284 | "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", 1285 | "requires": { 1286 | "media-typer": "0.3.0", 1287 | "mime-types": "~2.1.24" 1288 | } 1289 | }, 1290 | "unpipe": { 1291 | "version": "1.0.0", 1292 | "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", 1293 | "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" 1294 | }, 1295 | "utils-merge": { 1296 | "version": "1.0.1", 1297 | "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", 1298 | "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" 1299 | }, 1300 | "validate.io-array": { 1301 | "version": "1.0.6", 1302 | "resolved": "https://registry.npmjs.org/validate.io-array/-/validate.io-array-1.0.6.tgz", 1303 | "integrity": "sha1-W1osr9j4uFq7L4hroVPy2Tond00=" 1304 | }, 1305 | "validate.io-function": { 1306 | "version": "1.0.2", 1307 | "resolved": "https://registry.npmjs.org/validate.io-function/-/validate.io-function-1.0.2.tgz", 1308 | "integrity": "sha1-NDoZgC7TsZaCaceA5VjpNBHAutc=" 1309 | }, 1310 | "vary": { 1311 | "version": "1.1.2", 1312 | "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", 1313 | "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" 1314 | } 1315 | } 1316 | } 1317 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "openai-embeddings", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "src/index.ts", 6 | "scripts": { 7 | "start": "tsc --build && node lib/src/index.js", 8 | "test": "tsc --build && node lib/test/index.js", 9 | "server": "tsc --build && node lib/src/server.js", 10 | "batch-obsidian-job": "tsc --build && node lib/test/batch-obsidian-job.js", 11 | "test-obsidian": "tsc --build && node lib/test/obsidian.js", 12 | "test-youtube": "tsc --build && node lib/test/youtube.js", 13 | "build": "tsc --build", 14 | "clean": "tsc --build --clean" 15 | }, 16 | "keywords": [], 17 | "author": "", 18 | "license": "ISC", 19 | "dependencies": { 20 | "@prisma/client": "^3.7.0", 21 | "axios": "^0.24.0", 22 | "compute-cosine-similarity": "^1.0.0", 23 | "csv-writer": "^1.6.0", 24 | "dotenv": "^10.0.0", 25 | "express": "^4.17.2", 26 | "gpt-3-encoder": "^1.1.3", 27 | "lodash": "^4.17.21", 28 | "moment": "^2.29.1", 29 | "prisma-offset-pagination": "^0.0.4" 30 | }, 31 | "devDependencies": { 32 | "@types/express": "^4.17.13", 33 | "@types/lodash": "^4.14.178", 34 | "@types/node": "^16.11.12", 35 | "prisma": "^3.7.0" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /prisma/schema.prisma: -------------------------------------------------------------------------------- 1 | // This is your Prisma schema file, 2 | // learn more about it in the docs: https://pris.ly/d/prisma-schema 3 | 4 | generator client { 5 | provider = "prisma-client-js" 6 | } 7 | 8 | datasource db { 9 | provider = "postgresql" 10 | url = env("DATABASE_URL") 11 | } 12 | 13 | model obsidian { 14 | id Int @id @default(autoincrement()) 15 | doc Json 16 | createdAt DateTime @default(now()) 17 | updatedAt DateTime @default(now()) 18 | filename String @unique 19 | } 20 | -------------------------------------------------------------------------------- /src/declare.t.ts: -------------------------------------------------------------------------------- 1 | declare module 'compute-cosine-similarity'; 2 | declare module 'prisma-offset-pagination'; -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { createEndpoint, OpenAIResponse, EmbeddingsResponse } from './utils' 2 | import axios from 'axios'; 3 | import * as fs from 'fs/promises'; 4 | 5 | export default class Embeddings { 6 | 7 | apiKey: string; 8 | endpoint: string; 9 | embeddings: EmbeddingsResponse; 10 | 11 | constructor(apiKey: string) { 12 | this.apiKey = apiKey; 13 | this.endpoint = this.setEngine('ada-similarity'); 14 | this.embeddings = { 15 | embeddings: [], 16 | text: [] // should be the same length as embeddings 17 | }; 18 | } 19 | 20 | setEngine = (engine: string) => { 21 | this.endpoint = createEndpoint(engine) 22 | return this.endpoint 23 | } 24 | 25 | writeEmbeddings = async (embeddings: EmbeddingsResponse, filename: string) => { 26 | const writer = await fs.writeFile(filename, JSON.stringify(embeddings, null, 4)) 27 | return writer 28 | } 29 | 30 | createEmbeddings = async ( 31 | input: string[], 32 | ): Promise => { 33 | try { 34 | input = input.map(inp => inp.replace(/\n/g, ' ')) 35 | const body = { input }; 36 | 37 | const response = await axios.post(this.endpoint, JSON.stringify(body), 38 | { 39 | headers: { 40 | 'Content-Type': 'application/json', 41 | 'Authorization': `Bearer ${this.apiKey}` 42 | } 43 | } 44 | ); 45 | const data: OpenAIResponse = await response.data; 46 | const embeddings = data.data.map(d => d.embedding); 47 | 48 | this.embeddings = { embeddings, text: input }; 49 | return { embeddings, text: input }; 50 | } catch (err) { 51 | console.error(err); 52 | return null; 53 | } 54 | } 55 | } -------------------------------------------------------------------------------- /src/obsidian/index.ts: -------------------------------------------------------------------------------- 1 | import * as dotenv from 'dotenv'; 2 | import { chunkDocument, readFile, EmbeddingsResponse, search } from '../utils' 3 | import Embeddings from '../index'; 4 | import _ from 'lodash' 5 | import { PrismaClient } from '@prisma/client' 6 | 7 | dotenv.config(); 8 | 9 | export default class Obsidian { 10 | private apiKey: string; 11 | tag: string; 12 | embeddingsObj: Embeddings; 13 | documentWithEmbeddings: { 14 | filename: any; 15 | chunks: string[][]; 16 | embeddingsResponse: any; 17 | } 18 | prismaClient: PrismaClient; 19 | 20 | constructor(apiKey: string, engine: string = 'babbage-search-document') { 21 | this.apiKey = apiKey; 22 | this.tag = 'obsidian'; 23 | this.embeddingsObj = new Embeddings(this.apiKey); 24 | this.prismaClient = new PrismaClient(); 25 | this.embeddingsObj.setEngine(engine); 26 | this.documentWithEmbeddings = { 27 | filename: '', 28 | chunks: [], 29 | embeddingsResponse: {} 30 | } 31 | } 32 | 33 | async embedObsidianDocument(filename: string) { 34 | const txt = await readFile(filename, true) 35 | const chunks = txt 36 | .split('\n') 37 | .filter(line => line.trim().length > 0) 38 | .map(line => JSON.stringify(line.trim())) 39 | .map(doc => chunkDocument(doc)) 40 | 41 | const flattenedChunks = _.flatten(chunks) 42 | 43 | if (flattenedChunks.length === 0) { 44 | console.log('No text found in document') 45 | return { 46 | filename: filename, 47 | chunks: [], 48 | embeddingsResponse: { 49 | embeddings: [], 50 | text: [] 51 | } 52 | } 53 | } 54 | 55 | const docEmbeddings = await this.embeddingsObj.createEmbeddings(flattenedChunks!) 56 | 57 | const doc = { 58 | 'filename': filename, 59 | 'chunks': chunks, 60 | 'embeddingsResponse': docEmbeddings as { 61 | embeddings: number[][] 62 | text: string[] 63 | } 64 | } 65 | 66 | this.documentWithEmbeddings = doc // TODO: for classes do I need to set this or just return as normal? 67 | return doc 68 | } 69 | } -------------------------------------------------------------------------------- /src/obsidian/utils.ts: -------------------------------------------------------------------------------- 1 | import { PrismaClient } from '@prisma/client' 2 | import { EmbeddingsResponse, search } from '../utils' 3 | import Obsidian from './index'; 4 | import _ from 'lodash' 5 | import { prismaOffsetPagination } from 'prisma-offset-pagination'; 6 | 7 | 8 | export async function findObsidianDocumentByFilename(prismaClient: PrismaClient, filename: string) { 9 | return prismaClient.obsidian.findUnique({ 10 | where: { 11 | filename 12 | } 13 | }).then((result) => { 14 | return result; 15 | }) 16 | .catch((error) => { 17 | console.log(error); 18 | return null 19 | }); 20 | } 21 | 22 | export async function writeObsidianDocumentToPostgres(prismaClient: PrismaClient, obsidianDocument: { 23 | filename: string; 24 | chunks: string[][]; 25 | embeddingsResponse: { 26 | embeddings: number[][]; 27 | text: string[]; 28 | }; 29 | }) { 30 | return prismaClient.obsidian.create({ 31 | data: { 32 | doc: obsidianDocument, 33 | filename: obsidianDocument.filename, 34 | } 35 | }).then((result) => { 36 | return result; 37 | }) 38 | .catch((error) => { 39 | console.log(error); 40 | return error; 41 | }); 42 | } 43 | 44 | export async function updateObsidianDocument(prismaClient: PrismaClient, filename: string, obsidianDocument: { 45 | filename: string; 46 | chunks: string[][]; 47 | embeddingsResponse: { 48 | embeddings: number[][]; 49 | text: string[]; 50 | }; 51 | }, oldFilename?: string) { 52 | return prismaClient.obsidian.update({ 53 | where: { 54 | filename: oldFilename ? oldFilename : filename 55 | }, 56 | data: { 57 | doc: obsidianDocument, 58 | filename: obsidianDocument.filename, 59 | updatedAt: new Date() 60 | } 61 | }).then((result) => { 62 | return result; 63 | }).catch((error) => { 64 | console.log(error); 65 | return error; 66 | }); 67 | } 68 | 69 | export async function findAllObsidianDocuments(prismaClient: PrismaClient) { 70 | 71 | try { 72 | // offset method ~ 13 seconds for 1000 documents 73 | const count = await prismaClient.obsidian.count() 74 | const pages = Math.ceil(count / 100) 75 | const allDocsPromises = [] 76 | for (let i = 0; i < pages; i++) { 77 | const page = prismaClient.obsidian.findMany({ 78 | skip: i * 100, 79 | take: 100 80 | }) 81 | allDocsPromises.push(page) 82 | } 83 | const allDocs = await Promise.all(allDocsPromises) 84 | return _.flatten(allDocs) 85 | 86 | // cursor method ~25 seconds for 1000 documents 87 | // const allDocs = [] 88 | // let queryResults = await prismaClient.obsidian.findMany({ 89 | // take: 100 90 | // }) 91 | // while (queryResults.length > 0) { 92 | // allDocs.push(...queryResults) 93 | // console.log(queryResults[queryResults.length - 1]!.id) 94 | // queryResults = await prismaClient.obsidian.findMany({ 95 | // take: 100, 96 | // skip: 1, 97 | // cursor: { 98 | // id: queryResults[queryResults.length - 1]!.id 99 | // } 100 | // }) 101 | // } 102 | // console.log(allDocs.length) 103 | // return allDocs 104 | 105 | // prisma-offset-pagination method fails with: Invalid `prismaModel.findMany()` invocation 106 | // const result = await prismaOffsetPagination({ 107 | // model: { name: 'obsidian' }, 108 | // size: 100, 109 | // orderBy: 'id', 110 | // orderDirection: 'desc', 111 | // buttonNum: 11, 112 | // prisma: prismaClient 113 | // }); 114 | // console.log(result) 115 | // return [{ 116 | // doc: { filename: 'string', chunks: [['q']], embeddingsResponse: { embeddings: [[0]], text: ['hi'] } }, 117 | // id: 1, 118 | // filename: 'blah', 119 | // updatedAt: new Date(), 120 | // createdAt: new Date() 121 | // }]; 122 | } catch (err) { 123 | console.error(err) 124 | return [{ 125 | doc: { filename: 'string', chunks: [['q']], embeddingsResponse: { embeddings: [[0]], text: ['hi'] } }, 126 | id: 1, 127 | filename: 'blah', 128 | updatedAt: new Date(), 129 | createdAt: new Date() 130 | }]; 131 | } 132 | } 133 | 134 | export function returnTopResult(obsidianDocuments: ObsidianDocumentWithEmbeddings[], queryEmbeddings: EmbeddingsResponse, queries: string[], numTopResults: number = 3) { 135 | try { 136 | const searchResults = [] 137 | let offset = 0 138 | for (let i = 0; i < obsidianDocuments.length; i++) { 139 | const doc = obsidianDocuments[i]; 140 | 141 | if (doc && doc.embeddingsResponse) searchResults.push(search(doc!.embeddingsResponse!.embeddings, queryEmbeddings.embeddings, 1)) 142 | else { 143 | console.log(`No embeddings for ${doc!.filename}`) 144 | offset++ // used for empty documents, not sure how to handle them yet 145 | } 146 | } 147 | 148 | const similarityPerDocument = [] 149 | for (let i = 0; i < queries.length; i++) { 150 | const forQuery = [] 151 | for (let j = 0; j < searchResults.length - offset; j++) { 152 | const searchResult = searchResults[j] 153 | forQuery.push({ 154 | searchResult: searchResult![i]![0], 155 | docNum: j 156 | }) 157 | } 158 | similarityPerDocument.push(forQuery) 159 | } 160 | 161 | const topResults = [] 162 | for (let i = 0; i < similarityPerDocument.length; i++) { 163 | const groupedByQuery = similarityPerDocument[i] 164 | const sortedBySimilarity = _.reverse(_.sortBy(groupedByQuery, 'searchResult.similarity')) 165 | 166 | const results = [] 167 | for (let j = 0; j < Math.min(sortedBySimilarity.length, numTopResults); j++) { 168 | results.push({ 169 | query: queries[i], 170 | result: obsidianDocuments[sortedBySimilarity[j]!.docNum - offset]?.embeddingsResponse?.text[sortedBySimilarity[j]!.searchResult!.index], 171 | filename: obsidianDocuments[sortedBySimilarity[j]!.docNum - offset]?.filename, 172 | tag: 'obsidian' 173 | }) 174 | } 175 | 176 | topResults.push(results) 177 | } 178 | 179 | return topResults 180 | } catch (err) { 181 | console.log(err) 182 | return [] 183 | } 184 | } 185 | 186 | export async function ObsidianFactory(apiKey: string, documentFilePath: string, engine: string = 'babbage-search-document') { 187 | const obsidian = new Obsidian(apiKey, engine) 188 | const doc = await obsidian.embedObsidianDocument(documentFilePath) 189 | return { 190 | obsidian, 191 | doc 192 | } 193 | } 194 | 195 | // INTERFACES 196 | 197 | export interface ObsidianDocumentWithEmbeddings { 198 | filename: string; 199 | chunks: string[][]; 200 | embeddingsResponse: { 201 | embeddings: number[][] 202 | text: string[] 203 | } | null; 204 | } -------------------------------------------------------------------------------- /src/server.ts: -------------------------------------------------------------------------------- 1 | import express from "express"; 2 | const app = express(); 3 | const port = 8080; // default port to listen 4 | import { writeObsidianDocumentToPostgres, findAllObsidianDocuments, ObsidianFactory, returnTopResult, findObsidianDocumentByFilename, updateObsidianDocument } from '../src/obsidian/utils' 5 | import { embedQuery } from '../src/utils' 6 | import moment from "moment"; 7 | 8 | import * as dotenv from 'dotenv'; 9 | dotenv.config(); 10 | 11 | import { PrismaClient } from '@prisma/client' 12 | const prismaClient = new PrismaClient({ 13 | log: [ 14 | { 15 | emit: 'event', 16 | level: 'query', 17 | }, 18 | { 19 | emit: 'stdout', 20 | level: 'error', 21 | }, 22 | { 23 | emit: 'stdout', 24 | level: 'info', 25 | }, 26 | { 27 | emit: 'stdout', 28 | level: 'warn', 29 | }, 30 | ], 31 | }); 32 | 33 | prismaClient.$on('query', (e:any) => { 34 | console.log('Query: ' + e.query) 35 | console.log('Duration: ' + e.duration + 'ms') 36 | }) 37 | 38 | const obsidianRootPath = process.env.OBSIDIAN_ROOT_PATH; 39 | 40 | app.use(express.json()); 41 | 42 | // define a route handler for the default home page 43 | app.get( "/", ( req, res ) => { 44 | // render the index template 45 | res.send( "Hello world!" ); 46 | } ); 47 | 48 | app.post('/obsidian', async (req, res) => { 49 | try { 50 | const filename = req.body.filename; 51 | const oldFilename = req.body.oldFilename; 52 | const dontUpdate = req.body.dontUpdate; // a flag to not update existing files 53 | let document 54 | if (oldFilename) { 55 | document = await findObsidianDocumentByFilename(prismaClient, obsidianRootPath + oldFilename) 56 | } else { 57 | document = await findObsidianDocumentByFilename(prismaClient, obsidianRootPath + filename) 58 | } 59 | if (document) { 60 | if (dontUpdate) res.status(201).send(document); 61 | const obsidianInstance = await ObsidianFactory(process.env.API_KEY!, obsidianRootPath + filename); 62 | let saved 63 | if (oldFilename) { 64 | saved = await updateObsidianDocument(prismaClient, obsidianRootPath + filename, obsidianInstance.doc, obsidianRootPath + oldFilename); 65 | } else { 66 | saved = await updateObsidianDocument(prismaClient, obsidianRootPath + filename, obsidianInstance.doc); 67 | } 68 | 69 | res.send(saved); 70 | } else { 71 | const obsidianInstance = await ObsidianFactory(process.env.API_KEY!, obsidianRootPath + filename); 72 | const saved = await writeObsidianDocumentToPostgres(prismaClient, obsidianInstance.doc); 73 | res.send(saved); 74 | } 75 | } catch (err) { 76 | console.error(err) 77 | res.status(500).send(err); 78 | } 79 | }) 80 | 81 | app.get('/files', async (req, res) => { 82 | try { 83 | const files = await findAllObsidianDocuments(prismaClient); 84 | res.send(files.map((file) => { 85 | return { 86 | filename: file.filename, 87 | updatedAt: moment(file.updatedAt).format('LLLL') 88 | } 89 | })); 90 | } catch (err) { 91 | res.status(500).send(err); 92 | } 93 | }) 94 | 95 | app.post('/query', async (req, res) => { 96 | try { 97 | const queries = [req.body.query]; 98 | const queryEmbedding = await embedQuery(queries, 'babbage-search-query', process.env.API_KEY!) 99 | const allDocsDB = await findAllObsidianDocuments(prismaClient); 100 | console.log(allDocsDB.length) 101 | const docs = allDocsDB!.map((doc) => { 102 | const remappedTypeForTS = doc.doc as { 103 | filename: string; 104 | chunks: string[][]; 105 | embeddingsResponse: { 106 | embeddings: number[][]; 107 | text: string[]; 108 | }; 109 | } 110 | return remappedTypeForTS 111 | }).filter((doc) => { 112 | return doc.embeddingsResponse && doc.embeddingsResponse.embeddings.length > 0 113 | }) 114 | const top_result = returnTopResult(docs, queryEmbedding!, queries) 115 | res.status(200).send(top_result); 116 | } catch (err) { 117 | console.error(err); 118 | res.status(500).send(err); 119 | } 120 | }) 121 | 122 | // start the express server 123 | app.listen( port, () => { 124 | // tslint:disable-next-line:no-console 125 | console.log( `server started at http://localhost:${ port }` ); 126 | } ); -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | import similarity from 'compute-cosine-similarity'; 2 | import Embeddings from './index'; 3 | const { encode } = require('gpt-3-encoder') 4 | import fs from 'fs' 5 | import path from 'path'; 6 | 7 | 8 | export function readFile(filePath: string, isFullPath: boolean = false): Promise { 9 | let fullPath = path.resolve(filePath) 10 | if (!isFullPath) fullPath = path.join(__dirname, filePath) 11 | return new Promise((resolve, reject) => { 12 | fs.readFile(fullPath, 'utf-8', (err, data) => { 13 | if (err) { 14 | reject(err) 15 | return 16 | } 17 | if (data === undefined) { 18 | console.log(`File ${fullPath} not found`) 19 | reject(new Error(`File ${fullPath} not found`)) 20 | return 21 | } 22 | console.log(data) 23 | resolve(data.toString()) 24 | }) 25 | }) 26 | } 27 | 28 | function checkLength(text: string, maxChunkLength: number = 2000) { 29 | const encoded = encode(text) 30 | if (encoded.length > maxChunkLength) { 31 | return true 32 | } 33 | return false 34 | } 35 | 36 | /** 37 | * a chunking algorithm that splits a string into chunks of a maximum length as dictated by max tokens allowed by the API 38 | * Does not respect spaces, so it is not suitable for splitting text into sentences 39 | * @param document document to be chunked 40 | * @returns chunks of the document 41 | */ 42 | export function chunkDocument (document: string): string[] { 43 | const chunks: string[] = [] 44 | let chunksAboveTokenLimit = [] // [true, false, etc] want all to be false 45 | let numOfSubdivisions = 0 46 | chunksAboveTokenLimit.push(checkLength(document)) 47 | while (chunksAboveTokenLimit.includes(true)) { 48 | numOfSubdivisions++ 49 | chunksAboveTokenLimit = [] 50 | let maxChunkLength = Math.floor(document.length / numOfSubdivisions) 51 | for (let i = 0; i < numOfSubdivisions; i++) { 52 | const chunk = document.slice(i * maxChunkLength, (i + 1) * maxChunkLength) 53 | chunksAboveTokenLimit.push(checkLength(chunk)) 54 | } 55 | } 56 | 57 | if(numOfSubdivisions > 1) { 58 | let maxChunkLength = Math.floor(document.length / numOfSubdivisions) 59 | for (let i = 0; i < numOfSubdivisions; i++) { 60 | const chunk = document.slice(i * maxChunkLength, (i + 1) * maxChunkLength) 61 | chunks.push(chunk) 62 | } 63 | } else { 64 | chunks.push(document) 65 | } 66 | 67 | return chunks 68 | } 69 | 70 | /** 71 | * 72 | * @param engine engine to be used for embedding 73 | * @returns http endpoint for embedding 74 | */ 75 | export function createEndpoint(engine: string): string { 76 | switch (engine) { 77 | case 'ada-similarity': 78 | engine = 'ada-similarity'; 79 | break; 80 | case 'babbage-similarity': 81 | engine = 'babbage-similarity'; 82 | break; 83 | case 'curie-similarity': 84 | engine = 'curie-similarity'; 85 | break; 86 | case 'davinci-similarity': 87 | engine = 'davinci-similarity'; 88 | break; 89 | case 'ada-search-document': 90 | engine = 'ada-search-document'; 91 | break; 92 | case 'ada-search-query': 93 | engine = 'ada-search-query'; 94 | break; 95 | case 'babbage-search-document': 96 | engine = 'babbage-search-document'; 97 | break; 98 | case 'babbage-search-query': 99 | engine = 'babbage-search-query'; 100 | break; 101 | case 'curie-search-document': 102 | engine = 'curie-search-document'; 103 | break; 104 | case 'curie-search-query': 105 | engine = 'curie-search-query'; 106 | break; 107 | case 'ada-code-search-code': 108 | engine = 'ada-code-search-code'; 109 | break; 110 | case 'ada-code-search-text': 111 | engine = 'ada-code-search-text'; 112 | break; 113 | case 'babbage-code-search-code': 114 | engine = 'babbage-code-search-code'; 115 | break; 116 | case 'babbage-code-search-text': 117 | engine = 'babbage-code-search-text'; 118 | break; 119 | default: 120 | engine = 'ada-similarity'; 121 | break; 122 | } 123 | return `https://api.openai.com/v1/engines/${engine}/embeddings`; 124 | } 125 | 126 | export function cleanString(str: string): string { 127 | return str.replace(/\n/g, " ").trim() // new lines should be replaced with spaces as per https://beta.openai.com/docs/guides/embeddings/use-cases 128 | } 129 | 130 | /** 131 | * example https://github.com/openai/openai-python/blob/main/examples/embeddings/Obtain_dataset.ipynb 132 | * @param data any json object 133 | * @param indexedField field to be indexed from json object, like title, text, etc. 134 | * @returns an object with text and original data for linking 135 | */ 136 | export function processJSONData(data: any[], indexedField: string): ProcessedData { 137 | const result: string[] = []; 138 | data.forEach((item: any) => { 139 | result.push(item[indexedField]); 140 | }); 141 | 142 | return { 143 | text: result, 144 | original: data 145 | }; 146 | } 147 | 148 | /** 149 | * fetch index from ProcessedData 150 | * @param data ProcessedData object 151 | * @param idx number of index to be fetched from original data 152 | * @returns single item from original data 153 | */ 154 | export function fetchDataFromOriginal(data: ProcessedData, idx: number) { 155 | return { 156 | original: data.original[idx], 157 | text: data.text[idx] 158 | }; 159 | } 160 | 161 | /** 162 | * use the openai api to embed a query 163 | * @param query query|queries to be searched 164 | * @param engine what engine to encode query with 165 | * @param apiKey api key for openai api 166 | * @returns Embeddings object 167 | */ 168 | export async function embedQuery(query: string | string[], engine: string, apiKey: string): Promise { 169 | try { 170 | const queryEmbedding = new Embeddings(apiKey!) 171 | queryEmbedding.setEngine(engine) 172 | if (!Array.isArray(query)) { 173 | query = [query]; 174 | } 175 | return queryEmbedding.createEmbeddings(query) 176 | } catch (err) { 177 | console.error(err) 178 | return null 179 | } 180 | } 181 | 182 | /** 183 | * Run cosine similarity search on all documents in the database against a list of queries, and return the top {numResults} results 184 | * @param documentEmbeddings number[][] - embeddings of source document 185 | * @param queryEmbeddings number[][] - embeddings of queries (queries are a list) 186 | * @param numResults number - number of results to be returned 187 | * @returns SearchResult[][] - list of results 188 | */ 189 | export function search(documentEmbeddings: number[][], queryEmbeddings: number[][], numResults: number = 3): SearchResults[][] { 190 | const results: SearchResults[][] = []; 191 | queryEmbeddings.map((queryEmbedding) => { 192 | const similarityRankings = documentEmbeddings.map((vector, i) => { 193 | const similarity = cosineSimilarity(vector, queryEmbedding); 194 | return { 195 | index: i, 196 | similarity, 197 | vector 198 | } 199 | }) 200 | .sort((a, b) => b.similarity - a.similarity); 201 | 202 | const slicedRankings = similarityRankings.slice(0, numResults); 203 | results.push(slicedRankings) 204 | }); 205 | return results; 206 | } 207 | 208 | /** 209 | * run cosine similarity on two vectors 210 | * @param vector1 number[] 211 | * @param vector2 number[] 212 | * @returns number 213 | */ 214 | export function cosineSimilarity(vector1: number[], vector2: number[]): number { 215 | return similarity(vector1, vector2) 216 | } 217 | 218 | /* INTERFACES */ 219 | 220 | interface SearchResults { 221 | index: number, 222 | similarity: number, 223 | vector: number[] 224 | } 225 | 226 | export interface OpenAIResponse { 227 | data: { 228 | embedding: number[] 229 | }[] 230 | } 231 | 232 | export interface EmbeddingsResponse { 233 | embeddings: number[][] 234 | text: string[] 235 | } 236 | 237 | interface ProcessedData { 238 | text: string[] 239 | original: any 240 | } -------------------------------------------------------------------------------- /src/youtube/index.ts: -------------------------------------------------------------------------------- 1 | // youtube transcript search 2 | import Embeddings from '../../src/index'; 3 | import { processJSONData, EmbeddingsResponse } from '../../src/utils' 4 | 5 | 6 | export default class Youtube { 7 | private apiKey: string; 8 | tag: string; 9 | videoId: string; 10 | transcript:{ text: string, start: number, duration: number }[] = []; 11 | transcriptText: string[]; 12 | embeddingsObj: Embeddings; 13 | embeddings: EmbeddingsResponse; 14 | 15 | constructor(apiKey: string) { 16 | this.apiKey = apiKey; 17 | this.tag = 'youtube'; 18 | 19 | this.embeddingsObj = new Embeddings(this.apiKey); 20 | this.embeddingsObj.setEngine('babbage-search-document'); 21 | } 22 | 23 | 24 | setVideoId(videoId: string) { 25 | this.videoId = videoId; 26 | } 27 | 28 | setTranscript(transcript: { text: string, start: number, duration: number }[]) { 29 | this.transcript = transcript; 30 | this.setTranscriptText(); 31 | } 32 | 33 | private setTranscriptText () { 34 | const processed = processJSONData(this.transcript, 'text'); 35 | this.transcriptText = processed.text; 36 | } 37 | 38 | async setTranscriptEmbeddings () { 39 | const res = await this.embeddingsObj.createEmbeddings(this.transcriptText); 40 | if (res) { 41 | this.embeddings = res; 42 | } 43 | } 44 | } 45 | 46 | -------------------------------------------------------------------------------- /test/batch-obsidian-job.ts: -------------------------------------------------------------------------------- 1 | /* DO NOT RUN UNLESS YOU WANT ALL FILES INDEXED */ 2 | import fs, { PathLike } from 'fs' 3 | import path from 'path' 4 | import * as dotenv from 'dotenv'; 5 | import axios from 'axios' 6 | const { encode } = require('gpt-3-encoder') 7 | dotenv.config(); 8 | 9 | // https://coderrocketfuel.com/article/recursively-list-all-the-files-in-a-directory-using-node-js 10 | const getAllFiles = function (dirPath: string, arrayOfFiles: string[] = []) { 11 | if (dirPath.includes('.obsidian')) return [] 12 | 13 | const files = fs.readdirSync(dirPath) 14 | 15 | arrayOfFiles = arrayOfFiles || [] 16 | 17 | files.forEach(function (file) { 18 | if (fs.statSync(dirPath + "/" + file).isDirectory()) { 19 | arrayOfFiles = getAllFiles(dirPath + "/" + file, arrayOfFiles) 20 | } else { 21 | if (file.includes('.md')) { 22 | const removeRootPath = dirPath.replace(process.env.OBSIDIAN_ROOT_PATH!, '') 23 | const cleanedDirPath = removeRootPath.startsWith('/') ? removeRootPath.slice(1) : removeRootPath 24 | arrayOfFiles.push(path.join(cleanedDirPath, file)) 25 | } 26 | } 27 | }) 28 | 29 | return arrayOfFiles 30 | } 31 | 32 | const main = async () => { 33 | try { 34 | 35 | const arrayOfFiles = getAllFiles(process.env.OBSIDIAN_ROOT_PATH!) 36 | 37 | for (let i = 1093; i < arrayOfFiles.length; i++) { 38 | const filename = arrayOfFiles[i] 39 | console.log(`inserting ${filename}`) 40 | await axios.post(`http://localhost:8080/obsidian`, {filename, dontUpdate: true}) 41 | console.log(`inserted ${filename}`) 42 | } 43 | 44 | } catch (e) { 45 | console.log(e) 46 | } 47 | } 48 | 49 | // main() 50 | 51 | 52 | // if faucet gets halted this will be the last file indexed 53 | // const arrayOfFiles = getAllFiles(process.env.OBSIDIAN_ROOT_PATH!) 54 | // console.log(arrayOfFiles.length) 55 | // for (let i = 0; i < arrayOfFiles.length; i++) { // faucet got halted here 56 | // if (arrayOfFiles[i]!.includes('turndown.md')) console.log(i) 57 | // } 58 | -------------------------------------------------------------------------------- /test/data-processing.ts: -------------------------------------------------------------------------------- 1 | import { processJSONData, fetchDataFromOriginal } from '../src/utils' 2 | 3 | const transcript = [{'text': 'Everybody buonasera, good evening,\nit is a pleasure, honor to host and', 'start': 6.54, 'duration': 9.15}, {'text': 'present this event for Algorand tonight.\nThank you very much for being here. My', 'start': 15.69, 'duration': 5.13}, {'text': "name is Nicola Santoni, I'm principal\nof Lemniscap is a blockchain", 'start': 20.82, 'duration': 4.64}, {'text': "investments and advisory firm out of Hong\nKong. I'm so excited to listen what the", 'start': 25.46, 'duration': 7.15}, {'text': "speaker of tonight is gonna tell us so\nI'm gonna be very, very brief. If we, we", 'start': 32.61, 'duration': 5.58}, {'text': 'are here for sure we share the same\nvision. We believe the in the social and', 'start': 38.19, 'duration': 10.619}, {'text': 'societal impact that this technology\nwill enable. I think we we all believe', 'start': 48.809, 'duration': 7.371}, {'text': 'this is the second business model for\nInternet, where the value, the wealth is', 'start': 56.18, 'duration': 7.29}, {'text': 'gonna be distributed to the network.\nOwners to the network actors, we', 'start': 63.47, 'duration': 9.61}, {'text': 'believe in expanding the design space\nallowing is us to accrue from this', 'start': 73.08, 'duration': 7.65}] 4 | 5 | const data = processJSONData(transcript, 'text') 6 | const second = fetchDataFromOriginal(data, 2) 7 | console.log(second) -------------------------------------------------------------------------------- /test/index.ts: -------------------------------------------------------------------------------- 1 | import Embeddings from '../src/index'; 2 | import * as dotenv from 'dotenv'; 3 | import { embedQuery, processJSONData, search, fetchDataFromOriginal } from '../src/utils' 4 | dotenv.config(); 5 | 6 | const apiKey = process.env.API_KEY; 7 | const transcript = [{'text': 'Everybody buonasera, good evening,\nit is a pleasure, honor to host and', 'start': 6.54, 'duration': 9.15}, {'text': 'present this event for Algorand tonight.\nThank you very much for being here. My', 'start': 15.69, 'duration': 5.13}, {'text': "name is Nicola Santoni, I'm principal\nof Lemniscap is a blockchain", 'start': 20.82, 'duration': 4.64}, {'text': "investments and advisory firm out of Hong\nKong. I'm so excited to listen what the", 'start': 25.46, 'duration': 7.15}, {'text': "speaker of tonight is gonna tell us so\nI'm gonna be very, very brief. If we, we", 'start': 32.61, 'duration': 5.58}, {'text': 'are here for sure we share the same\nvision. We believe the in the social and', 'start': 38.19, 'duration': 10.619}, {'text': 'societal impact that this technology\nwill enable. I think we we all believe', 'start': 48.809, 'duration': 7.371}, {'text': 'this is the second business model for\nInternet, where the value, the wealth is', 'start': 56.18, 'duration': 7.29}, {'text': 'gonna be distributed to the network.\nOwners to the network actors, we', 'start': 63.47, 'duration': 9.61}, {'text': 'believe in expanding the design space\nallowing is us to accrue from this', 'start': 73.08, 'duration': 7.65}] 8 | const queries = ['algorand', 'italy'] 9 | 10 | const main = async () => { 11 | const embeddings = new Embeddings(apiKey!) 12 | embeddings.setEngine('babbage-search-document') 13 | const transcriptData = processJSONData(transcript, 'text'); 14 | const docEmbeddings = await embeddings.createEmbeddings(transcriptData.text) 15 | embeddings.writeEmbeddings(docEmbeddings!, './test/test.json') 16 | const queryEmbedding = await embedQuery(queries, 'babbage-search-query', apiKey!) 17 | const results = await search(docEmbeddings!.embeddings, queryEmbedding!.embeddings, 3) 18 | 19 | results.forEach((result, idx: number) => { 20 | console.log(`qry: ${queries[idx]}`) 21 | const highest = fetchDataFromOriginal(transcriptData, result![0]!['index']) 22 | console.log(highest) 23 | }) 24 | } 25 | 26 | main() -------------------------------------------------------------------------------- /test/obsidian.ts: -------------------------------------------------------------------------------- 1 | import * as dotenv from 'dotenv'; 2 | import { embedQuery } from '../src/utils' 3 | import _ from 'lodash' 4 | import { PrismaClient } from '@prisma/client' 5 | import { findAllObsidianDocuments, findObsidianDocumentByFilename, writeObsidianDocumentToPostgres, returnTopResult, ObsidianFactory } from '../src/obsidian/utils' 6 | 7 | dotenv.config(); 8 | 9 | const apiKey = process.env.API_KEY; 10 | const obsidianRootPath = process.env.OBSIDIAN_ROOT_PATH; 11 | const queries = ['henry ford industry', 'samuel morse', 'water bottle', 'apple watch', 'new experiences', 'habits', 'dwayne johnson'] 12 | 13 | const main = async () => { 14 | const prismaClient = new PrismaClient(); 15 | const res = await findAllObsidianDocuments(prismaClient); 16 | console.log(res); 17 | 18 | const queryEmbedding = await embedQuery(queries, 'babbage-search-query', apiKey!) 19 | 20 | const docs = res!.map((doc) => { 21 | const d2 = doc.doc as { 22 | filename: string; 23 | chunks: string[][]; 24 | embeddingsResponse: { 25 | embeddings: number[][]; 26 | text: string[]; 27 | }; 28 | } 29 | return d2 30 | }) 31 | 32 | const top_results = returnTopResult(docs, queryEmbedding!, queries) 33 | console.log(top_results); 34 | } 35 | 36 | main() -------------------------------------------------------------------------------- /test/youtube.ts: -------------------------------------------------------------------------------- 1 | import * as dotenv from 'dotenv'; 2 | import Youtube from '../src/youtube'; 3 | 4 | dotenv.config(); 5 | 6 | const apiKey = process.env.API_KEY; 7 | 8 | 9 | const transcript = [{'text': 'Everybody buonasera, good evening,\nit is a pleasure, honor to host and', 'start': 6.54, 'duration': 9.15}, {'text': 'present this event for Algorand tonight.\nThank you very much for being here. My', 'start': 15.69, 'duration': 5.13}, {'text': "name is Nicola Santoni, I'm principal\nof Lemniscap is a blockchain", 'start': 20.82, 'duration': 4.64}, {'text': "investments and advisory firm out of Hong\nKong. I'm so excited to listen what the", 'start': 25.46, 'duration': 7.15}, {'text': "speaker of tonight is gonna tell us so\nI'm gonna be very, very brief. If we, we", 'start': 32.61, 'duration': 5.58}, {'text': 'are here for sure we share the same\nvision. We believe the in the social and', 'start': 38.19, 'duration': 10.619}, {'text': 'societal impact that this technology\nwill enable. I think we we all believe', 'start': 48.809, 'duration': 7.371}, {'text': 'this is the second business model for\nInternet, where the value, the wealth is', 'start': 56.18, 'duration': 7.29}, {'text': 'gonna be distributed to the network.\nOwners to the network actors, we', 'start': 63.47, 'duration': 9.61}, {'text': 'believe in expanding the design space\nallowing is us to accrue from this', 'start': 73.08, 'duration': 7.65}] 10 | 11 | 12 | const main2 = async () => { 13 | const yt = new Youtube(apiKey!) 14 | yt.setTranscript(transcript) 15 | yt.setVideoId('_Y8dQOcYBhM') 16 | await yt.setTranscriptEmbeddings() 17 | console.log(yt.embeddings.embeddings.length) 18 | console.log(yt.videoId) 19 | } 20 | 21 | main2() -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | // project options 4 | "lib": [ 5 | "ESNext", 6 | "ES2017", 7 | "dom" 8 | ], // specifies which default set of type definitions to use ("DOM", "ES6", etc) 9 | "outDir": "lib", // .js (as well as .d.ts, .js.map, etc.) files will be emitted into this directory., 10 | "removeComments": true, // Strips all comments from TypeScript files when converting into JavaScript- you rarely read compiled code so this saves space 11 | "target": "ES6", // Target environment. Most modern browsers support ES6, but you may want to set it to newer or older. (defaults to ES3) 12 | 13 | // Module resolution 14 | "baseUrl": "./", // Lets you set a base directory to resolve non-absolute module names. 15 | "esModuleInterop": true, // fixes some issues TS originally had with the ES6 spec where TypeScript treats CommonJS/AMD/UMD modules similar to ES6 module 16 | "moduleResolution": "node", // Pretty much always node for modern JS. Other option is "classic" 17 | "paths": {}, // A series of entries which re-map imports to lookup locations relative to the baseUrl 18 | 19 | // Source Map 20 | "sourceMap": true, // enables the use of source maps for debuggers and error reporting etc 21 | "sourceRoot": "/", // Specify the location where a debugger should locate TypeScript files instead of relative source locations. 22 | 23 | "module": "commonjs", // Specifies the target module code should be generated for. 24 | // Strict Checks 25 | "alwaysStrict": true, // Ensures that your files are parsed in the ECMAScript strict mode, and emit “use strict” for each source file. 26 | "allowUnreachableCode": false, // pick up dead code paths 27 | "noImplicitAny": true, // In some cases where no type annotations are present, TypeScript will fall back to a type of any for a variable when it cannot infer the type. 28 | "strictNullChecks": true, // When strictNullChecks is true, null and undefined have their own distinct types and you’ll get a type error if you try to use them where a concrete value is expected. 29 | 30 | // Linter Checks 31 | "noImplicitReturns": true, 32 | "noUncheckedIndexedAccess": true, // accessing index must always check for undefined 33 | "noUnusedLocals": false, // Report errors on unused local variables. 34 | "noUnusedParameters": false // Report errors on unused parameters in functions 35 | }, 36 | "include": ["./**/*.ts"], 37 | "exclude": [ 38 | "node_modules/**/*" 39 | ] 40 | } --------------------------------------------------------------------------------