├── .env.copy ├── .gitignore ├── .nvmrc ├── README.md ├── docs └── data.txt ├── index.js ├── package-lock.json └── package.json /.env.copy: -------------------------------------------------------------------------------- 1 | OPENAI_API_KEY="" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | Documents.index 3 | node_modules/ -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | v18.16.0 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Process local files with Langchain 2 | 3 | Sample Node.js app that uses OpenAI to process local data using Langchain 4 | 5 | Medium Article: https://medium.com/@allenchun/feed-local-data-to-llm-using-langchain-node-js-fd7ac44093e9 6 | 7 | 8 | ### How it works 9 | - Langchain processes it by loading documents inside docs/ (In this case, we have a sample data.txt) 10 | - It works by taking big source of data, take for example a 50-page PDF and breaking it down into chunks 11 | - These chunks are then embedded into a Vector Store which serves as a local database and can be used for data processing 12 | 13 | ### Prerequisites 14 | - Node.js <= 18 15 | 16 | ### Setting Up 17 | - npm install 18 | - mv .env.copy .env 19 | - Replace `OPEN-API-KEY` in .env with your actual API Key from OpenAPI 20 | 21 | 22 | ### Running prompts 23 | - Asking questions related to the document 24 | ``` 25 | $ node index.js "Describe this applicant's employment history" 26 | { 27 | text: ' This applicant has 5+ years of experience in IT, with experience in System Administration, Network Configuration, Software Installation, Troubleshooting, Windows Environment, Customer Service, and Technical Support. They worked as a Senior IT Specialist at XYZ Global from 2018-Present, an IT Support Specialist at Zero Web from 2015-2017, and a Junior Desktop Support Engineer at Calumcoro Medical from 2014-2015.' 28 | } 29 | ``` 30 | - Asking questions not related to the document 31 | ``` 32 | $ node index.js "What is 1+1?" 33 | { text: " I don't know." } 34 | ``` 35 | 36 | ### What if we want to reference Langchain using our local data and OpenAI LLM 37 | ``` 38 | const chain = new RetrievalQAChain({ 39 | combineDocumentsChain: loadQARefineChain(model), 40 | retriever: vectorStore.asRetriever(), 41 | }); 42 | ``` -------------------------------------------------------------------------------- /docs/data.txt: -------------------------------------------------------------------------------- 1 | Jannet Morgan 2 | 3 | IT Specialist 4 | 5 | 202-555-0109 6 | 7 | jannet.j.morgan@me.com 8 | 9 | linkedin.com/in/jjmorgan81 10 | 11 | github.com/jannetjoannsix 12 | 13 | 14 | 15 | Professional Summary 16 | 17 | 18 | 19 | Dependable and goal-oriented IT Specialist with 5+ years of experience maintaining in-house IT systems and providing comprehensive customer support. At XYZ Global, saved 4 workhours a week for a team of 15 specialists through creating scripts to automate scheduled system patching. Seeking to join ABC Corp to optimize your IT processes while effectively cutting costs. 20 | 21 | 22 | 23 | Work History 24 | 25 | 26 | 27 | Senior IT Specialist 28 | 29 | XYZ Global, Manhattan, NY 30 | 31 | Jan 2018–Present 32 | 33 | Maintained 250+ Windows computers and peripherals, including all configuring and monitoring. Worked with vendors to cut equipment costs by 20%. 34 | Installed 200+ desktop computers during a company-wide upgrade. 35 | Improved the overall network capabilities by 18% through designing and implementing new connectivity network configurations. 36 | Spearheaded hardware and software upgrade rollouts. 37 | Key achievement: Wrote scripts to automate scheduled system patching. Saved 4 hours a week. 38 | 39 | 40 | 41 | IT Support Specialist 42 | 43 | Zero Web, Newark, NJ 44 | Dec 2015–Dec 2017 45 | 46 | Provided Help Desk-based IT phone support to end-users for a fast-paced web hosting firm, including troubleshooting, server support, and customer service. 47 | Maintained 15% above average customer satisfaction in post-call surveys. Used deep compassion and listening skills for the best customer experience. 48 | Became a trusted resource through high-level problem-solving skills. Solved customer issues with 12% more success than the company average. 49 | Kept 250 employees up and running on Windows 10. 50 | 51 | 52 | Junior Desktop Support Engineer 53 | 54 | Calumcoro Medical, Queens, NY 55 | 56 | Jan 2014–Dec 2015 57 | 58 | Handled all desktop support issues in a high-volume manufacturing firm. 59 | Handled trouble tickets 25% faster than other desktop support engineers. 60 | Commended by management for exemplary troubleshooting skills. 61 | 62 | 63 | Education 64 | 65 | 66 | 67 | BSc, Computer Science 68 | The State University of New York, Queens, NY 69 | 70 | 2014 71 | 72 | 73 | 74 | Key Skills 75 | 76 | 77 | 78 | System Administration 79 | Network Configuration 80 | Software Installation 81 | Troubleshooting 82 | Windows Environment 83 | Customer Service 84 | Technical Support 85 | 86 | 87 | Certifications 88 | 89 | 90 | 91 | 2016, CompTIA A+ 92 | 2019, MS Server -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | // 1. Import document loaders for different file formats 2 | import { DirectoryLoader } from "langchain/document_loaders/fs/directory"; 3 | import { TextLoader } from "langchain/document_loaders/fs/text"; 4 | import { JSONLoader } from "langchain/document_loaders/fs/json"; 5 | 6 | // 2. Import OpenAI langugage model and other related modules 7 | import { OpenAI } from "langchain/llms/openai"; 8 | import { RetrievalQAChain, loadQARefineChain } from "langchain/chains"; 9 | import { HNSWLib } from "langchain/vectorstores/hnswlib"; 10 | import { OpenAIEmbeddings } from "langchain/embeddings/openai"; 11 | import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; 12 | 13 | // 3. Import dotenv for loading environment variables and fs for file system operations 14 | import * as dotenv from 'dotenv'; 15 | 16 | // 4. Load Environment Variables 17 | dotenv.config() 18 | 19 | // 5. Load local files such as .json and .txt from ./docs 20 | const loader = new DirectoryLoader("./docs", { 21 | ".json": (path) => new JSONLoader(path), 22 | ".txt": (path) => new TextLoader(path) 23 | }) 24 | 25 | 26 | // 6. Define a function to normalize the content of the documents 27 | const normalizeDocuments = (docs) => { 28 | return docs.map((doc) => { 29 | if (typeof doc.pageContent === "string") { 30 | return doc.pageContent; 31 | } else if (Array.isArray(doc.pageContent)) { 32 | return doc.pageContent.join("\n"); 33 | } 34 | }); 35 | } 36 | 37 | const VECTOR_STORE_PATH = "Documents.index"; 38 | 39 | // 7. Define the main function to run the entire process 40 | export const run = async (params) => { 41 | const prompt = params[0] 42 | console.log('Prompt:', prompt) 43 | 44 | console.log("Loading docs...") 45 | const docs = await loader.load(); 46 | 47 | console.log('Processing...') 48 | const model = new OpenAI({ openAIApiKey: process.env.OPENAI_API_KEY }); 49 | 50 | let vectorStore; 51 | 52 | console.log('Creating new vector store...') 53 | const textSplitter = new RecursiveCharacterTextSplitter({ 54 | chunkSize: 1000, 55 | }); 56 | const normalizedDocs = normalizeDocuments(docs); 57 | const splitDocs = await textSplitter.createDocuments(normalizedDocs); 58 | 59 | // 8. Generate the vector store from the documents 60 | vectorStore = await HNSWLib.fromDocuments( 61 | splitDocs, 62 | new OpenAIEmbeddings() 63 | ); 64 | 65 | await vectorStore.save(VECTOR_STORE_PATH); 66 | console.log("Vector store created.") 67 | 68 | console.log("Creating retrieval chain...") 69 | // 9. Query the retrieval chain with the specified question 70 | // const chain = RetrievalQAChain.fromLLM(model, vectorStore.asRetriever()) 71 | 72 | const chain = new RetrievalQAChain({ 73 | combineDocumentsChain: loadQARefineChain(model), 74 | retriever: vectorStore.asRetriever(), 75 | }); 76 | 77 | console.log("Querying chain...") 78 | const res = await chain.call({ query: prompt }) 79 | 80 | console.log({ res }) 81 | } 82 | 83 | run(process.argv.slice(2)) -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "langchain", 3 | "version": "1.0.0", 4 | "lockfileVersion": 3, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "langchain", 9 | "version": "1.0.0", 10 | "license": "ISC", 11 | "dependencies": { 12 | "dotenv": "^16.3.1", 13 | "hnswlib-node": "^1.4.2", 14 | "langchain": "^0.0.103" 15 | } 16 | }, 17 | "node_modules/@anthropic-ai/sdk": { 18 | "version": "0.4.4", 19 | "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.4.4.tgz", 20 | "integrity": "sha512-Z/39nQi1sSUCeLII3lsAbL1u+0JF6cR2XmUEX9sLH0VtxmIjY6cjOUYjCkYh4oapTxOkhAFnVSAFJ6cxml2qXg==", 21 | "dependencies": { 22 | "@fortaine/fetch-event-source": "^3.0.6", 23 | "cross-fetch": "^3.1.5" 24 | } 25 | }, 26 | "node_modules/@fortaine/fetch-event-source": { 27 | "version": "3.0.6", 28 | "resolved": "https://registry.npmjs.org/@fortaine/fetch-event-source/-/fetch-event-source-3.0.6.tgz", 29 | "integrity": "sha512-621GAuLMvKtyZQ3IA6nlDWhV1V/7PGOTNIGLUifxt0KzM+dZIweJ6F3XvQF3QnqeNfS1N7WQ0Kil1Di/lhChEw==", 30 | "engines": { 31 | "node": ">=16.15" 32 | } 33 | }, 34 | "node_modules/@types/retry": { 35 | "version": "0.12.0", 36 | "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", 37 | "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==" 38 | }, 39 | "node_modules/@types/uuid": { 40 | "version": "9.0.2", 41 | "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.2.tgz", 42 | "integrity": "sha512-kNnC1GFBLuhImSnV7w4njQkUiJi0ZXUycu1rUaouPqiKlXkh77JKgdRnTAp1x5eBwcIwbtI+3otwzuIDEuDoxQ==" 43 | }, 44 | "node_modules/ansi-styles": { 45 | "version": "5.2.0", 46 | "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", 47 | "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", 48 | "engines": { 49 | "node": ">=10" 50 | }, 51 | "funding": { 52 | "url": "https://github.com/chalk/ansi-styles?sponsor=1" 53 | } 54 | }, 55 | "node_modules/argparse": { 56 | "version": "2.0.1", 57 | "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", 58 | "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" 59 | }, 60 | "node_modules/asynckit": { 61 | "version": "0.4.0", 62 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 63 | "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" 64 | }, 65 | "node_modules/axios": { 66 | "version": "0.26.1", 67 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", 68 | "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", 69 | "dependencies": { 70 | "follow-redirects": "^1.14.8" 71 | } 72 | }, 73 | "node_modules/base64-js": { 74 | "version": "1.5.1", 75 | "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", 76 | "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", 77 | "funding": [ 78 | { 79 | "type": "github", 80 | "url": "https://github.com/sponsors/feross" 81 | }, 82 | { 83 | "type": "patreon", 84 | "url": "https://www.patreon.com/feross" 85 | }, 86 | { 87 | "type": "consulting", 88 | "url": "https://feross.org/support" 89 | } 90 | ] 91 | }, 92 | "node_modules/binary-extensions": { 93 | "version": "2.2.0", 94 | "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", 95 | "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", 96 | "engines": { 97 | "node": ">=8" 98 | } 99 | }, 100 | "node_modules/binary-search": { 101 | "version": "1.3.6", 102 | "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz", 103 | "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==" 104 | }, 105 | "node_modules/bindings": { 106 | "version": "1.5.0", 107 | "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", 108 | "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", 109 | "dependencies": { 110 | "file-uri-to-path": "1.0.0" 111 | } 112 | }, 113 | "node_modules/camelcase": { 114 | "version": "6.3.0", 115 | "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", 116 | "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", 117 | "engines": { 118 | "node": ">=10" 119 | }, 120 | "funding": { 121 | "url": "https://github.com/sponsors/sindresorhus" 122 | } 123 | }, 124 | "node_modules/combined-stream": { 125 | "version": "1.0.8", 126 | "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", 127 | "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", 128 | "dependencies": { 129 | "delayed-stream": "~1.0.0" 130 | }, 131 | "engines": { 132 | "node": ">= 0.8" 133 | } 134 | }, 135 | "node_modules/commander": { 136 | "version": "10.0.1", 137 | "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", 138 | "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", 139 | "engines": { 140 | "node": ">=14" 141 | } 142 | }, 143 | "node_modules/cross-fetch": { 144 | "version": "3.1.8", 145 | "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", 146 | "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==", 147 | "dependencies": { 148 | "node-fetch": "^2.6.12" 149 | } 150 | }, 151 | "node_modules/decamelize": { 152 | "version": "1.2.0", 153 | "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", 154 | "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", 155 | "engines": { 156 | "node": ">=0.10.0" 157 | } 158 | }, 159 | "node_modules/delayed-stream": { 160 | "version": "1.0.0", 161 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 162 | "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", 163 | "engines": { 164 | "node": ">=0.4.0" 165 | } 166 | }, 167 | "node_modules/dotenv": { 168 | "version": "16.3.1", 169 | "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", 170 | "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", 171 | "engines": { 172 | "node": ">=12" 173 | }, 174 | "funding": { 175 | "url": "https://github.com/motdotla/dotenv?sponsor=1" 176 | } 177 | }, 178 | "node_modules/eventemitter3": { 179 | "version": "4.0.7", 180 | "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", 181 | "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" 182 | }, 183 | "node_modules/expr-eval": { 184 | "version": "2.0.2", 185 | "resolved": "https://registry.npmjs.org/expr-eval/-/expr-eval-2.0.2.tgz", 186 | "integrity": "sha512-4EMSHGOPSwAfBiibw3ndnP0AvjDWLsMvGOvWEZ2F96IGk0bIVdjQisOHxReSkE13mHcfbuCiXw+G4y0zv6N8Eg==" 187 | }, 188 | "node_modules/file-uri-to-path": { 189 | "version": "1.0.0", 190 | "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", 191 | "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" 192 | }, 193 | "node_modules/flat": { 194 | "version": "5.0.2", 195 | "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", 196 | "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", 197 | "bin": { 198 | "flat": "cli.js" 199 | } 200 | }, 201 | "node_modules/follow-redirects": { 202 | "version": "1.15.2", 203 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", 204 | "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", 205 | "funding": [ 206 | { 207 | "type": "individual", 208 | "url": "https://github.com/sponsors/RubenVerborgh" 209 | } 210 | ], 211 | "engines": { 212 | "node": ">=4.0" 213 | }, 214 | "peerDependenciesMeta": { 215 | "debug": { 216 | "optional": true 217 | } 218 | } 219 | }, 220 | "node_modules/form-data": { 221 | "version": "4.0.0", 222 | "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", 223 | "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", 224 | "dependencies": { 225 | "asynckit": "^0.4.0", 226 | "combined-stream": "^1.0.8", 227 | "mime-types": "^2.1.12" 228 | }, 229 | "engines": { 230 | "node": ">= 6" 231 | } 232 | }, 233 | "node_modules/hnswlib-node": { 234 | "version": "1.4.2", 235 | "resolved": "https://registry.npmjs.org/hnswlib-node/-/hnswlib-node-1.4.2.tgz", 236 | "integrity": "sha512-76PIzOaNcX8kOpKwlFPl07uelpctqDMzbiC+Qsk2JWNVkzeU/6iXRk4tfE9z3DoK1RCBrOaFXmQ6RFb1BVF9LA==", 237 | "hasInstallScript": true, 238 | "dependencies": { 239 | "bindings": "^1.5.0", 240 | "node-addon-api": "^6.0.0" 241 | } 242 | }, 243 | "node_modules/is-any-array": { 244 | "version": "2.0.1", 245 | "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-2.0.1.tgz", 246 | "integrity": "sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ==" 247 | }, 248 | "node_modules/js-tiktoken": { 249 | "version": "1.0.7", 250 | "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.7.tgz", 251 | "integrity": "sha512-biba8u/clw7iesNEWLOLwrNGoBP2lA+hTaBLs/D45pJdUPFXyxD6nhcDVtADChghv4GgyAiMKYMiRx7x6h7Biw==", 252 | "dependencies": { 253 | "base64-js": "^1.5.1" 254 | } 255 | }, 256 | "node_modules/js-yaml": { 257 | "version": "4.1.0", 258 | "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", 259 | "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", 260 | "dependencies": { 261 | "argparse": "^2.0.1" 262 | }, 263 | "bin": { 264 | "js-yaml": "bin/js-yaml.js" 265 | } 266 | }, 267 | "node_modules/jsonpointer": { 268 | "version": "5.0.1", 269 | "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", 270 | "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", 271 | "engines": { 272 | "node": ">=0.10.0" 273 | } 274 | }, 275 | "node_modules/langchain": { 276 | "version": "0.0.103", 277 | "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.0.103.tgz", 278 | "integrity": "sha512-vRkDSDPA76dpM23WrSeZr4F616E1EW08Ec+2HX15WFKoiUCd+fmkdfbjeJLXWYwFxKG5iy2jVE3d00nJ/8tqAw==", 279 | "dependencies": { 280 | "@anthropic-ai/sdk": "^0.4.3", 281 | "ansi-styles": "^5.0.0", 282 | "binary-extensions": "^2.2.0", 283 | "camelcase": "6", 284 | "decamelize": "^1.2.0", 285 | "expr-eval": "^2.0.2", 286 | "flat": "^5.0.2", 287 | "js-tiktoken": "^1.0.7", 288 | "js-yaml": "^4.1.0", 289 | "jsonpointer": "^5.0.1", 290 | "langchainplus-sdk": "^0.0.19", 291 | "ml-distance": "^4.0.0", 292 | "object-hash": "^3.0.0", 293 | "openai": "^3.3.0", 294 | "openapi-types": "^12.1.3", 295 | "p-queue": "^6.6.2", 296 | "p-retry": "4", 297 | "uuid": "^9.0.0", 298 | "yaml": "^2.2.1", 299 | "zod": "^3.21.4", 300 | "zod-to-json-schema": "^3.20.4" 301 | }, 302 | "engines": { 303 | "node": ">=18" 304 | }, 305 | "peerDependencies": { 306 | "@aws-sdk/client-dynamodb": "^3.310.0", 307 | "@aws-sdk/client-lambda": "^3.310.0", 308 | "@aws-sdk/client-s3": "^3.310.0", 309 | "@aws-sdk/client-sagemaker-runtime": "^3.310.0", 310 | "@aws-sdk/client-sfn": "^3.310.0", 311 | "@clickhouse/client": "^0.0.14", 312 | "@elastic/elasticsearch": "^8.4.0", 313 | "@getmetal/metal-sdk": "*", 314 | "@getzep/zep-js": "^0.4.1", 315 | "@gomomento/sdk": "^1.23.0", 316 | "@google-cloud/storage": "^6.10.1", 317 | "@huggingface/inference": "^1.5.1", 318 | "@notionhq/client": "^2.2.5", 319 | "@opensearch-project/opensearch": "*", 320 | "@pinecone-database/pinecone": "*", 321 | "@qdrant/js-client-rest": "^1.2.0", 322 | "@supabase/postgrest-js": "^1.1.1", 323 | "@supabase/supabase-js": "^2.10.0", 324 | "@tensorflow-models/universal-sentence-encoder": "*", 325 | "@tensorflow/tfjs-converter": "*", 326 | "@tensorflow/tfjs-core": "*", 327 | "@tigrisdata/vector": "^1.1.0", 328 | "@upstash/redis": "^1.20.6", 329 | "@zilliz/milvus2-sdk-node": ">=2.2.7", 330 | "apify-client": "^2.7.1", 331 | "axios": "*", 332 | "cheerio": "^1.0.0-rc.12", 333 | "chromadb": "^1.5.2", 334 | "cohere-ai": "^5.0.2", 335 | "d3-dsv": "^2.0.0", 336 | "epub2": "^3.0.1", 337 | "faiss-node": "^0.2.1", 338 | "google-auth-library": "^8.8.0", 339 | "hnswlib-node": "^1.4.2", 340 | "html-to-text": "^9.0.5", 341 | "ignore": "^5.2.0", 342 | "mammoth": "*", 343 | "mongodb": "^5.2.0", 344 | "mysql2": "^3.3.3", 345 | "notion-to-md": "^3.1.0", 346 | "pdf-parse": "1.1.1", 347 | "peggy": "^3.0.2", 348 | "pg": "^8.11.0", 349 | "pickleparser": "^0.1.0", 350 | "playwright": "^1.32.1", 351 | "puppeteer": "^19.7.2", 352 | "redis": "^4.6.4", 353 | "replicate": "^0.9.0", 354 | "srt-parser-2": "^1.2.2", 355 | "typeorm": "^0.3.12", 356 | "typesense": "^1.5.3", 357 | "vectordb": "^0.1.4", 358 | "weaviate-ts-client": "^1.0.0" 359 | }, 360 | "peerDependenciesMeta": { 361 | "@aws-sdk/client-dynamodb": { 362 | "optional": true 363 | }, 364 | "@aws-sdk/client-lambda": { 365 | "optional": true 366 | }, 367 | "@aws-sdk/client-s3": { 368 | "optional": true 369 | }, 370 | "@aws-sdk/client-sagemaker-runtime": { 371 | "optional": true 372 | }, 373 | "@aws-sdk/client-sfn": { 374 | "optional": true 375 | }, 376 | "@clickhouse/client": { 377 | "optional": true 378 | }, 379 | "@elastic/elasticsearch": { 380 | "optional": true 381 | }, 382 | "@getmetal/metal-sdk": { 383 | "optional": true 384 | }, 385 | "@getzep/zep-js": { 386 | "optional": true 387 | }, 388 | "@gomomento/sdk": { 389 | "optional": true 390 | }, 391 | "@google-cloud/storage": { 392 | "optional": true 393 | }, 394 | "@huggingface/inference": { 395 | "optional": true 396 | }, 397 | "@notionhq/client": { 398 | "optional": true 399 | }, 400 | "@opensearch-project/opensearch": { 401 | "optional": true 402 | }, 403 | "@pinecone-database/pinecone": { 404 | "optional": true 405 | }, 406 | "@qdrant/js-client-rest": { 407 | "optional": true 408 | }, 409 | "@supabase/postgrest-js": { 410 | "optional": true 411 | }, 412 | "@supabase/supabase-js": { 413 | "optional": true 414 | }, 415 | "@tensorflow-models/universal-sentence-encoder": { 416 | "optional": true 417 | }, 418 | "@tensorflow/tfjs-converter": { 419 | "optional": true 420 | }, 421 | "@tensorflow/tfjs-core": { 422 | "optional": true 423 | }, 424 | "@tigrisdata/vector": { 425 | "optional": true 426 | }, 427 | "@upstash/redis": { 428 | "optional": true 429 | }, 430 | "@zilliz/milvus2-sdk-node": { 431 | "optional": true 432 | }, 433 | "apify-client": { 434 | "optional": true 435 | }, 436 | "axios": { 437 | "optional": true 438 | }, 439 | "cheerio": { 440 | "optional": true 441 | }, 442 | "chromadb": { 443 | "optional": true 444 | }, 445 | "cohere-ai": { 446 | "optional": true 447 | }, 448 | "d3-dsv": { 449 | "optional": true 450 | }, 451 | "epub2": { 452 | "optional": true 453 | }, 454 | "faiss-node": { 455 | "optional": true 456 | }, 457 | "google-auth-library": { 458 | "optional": true 459 | }, 460 | "hnswlib-node": { 461 | "optional": true 462 | }, 463 | "html-to-text": { 464 | "optional": true 465 | }, 466 | "ignore": { 467 | "optional": true 468 | }, 469 | "mammoth": { 470 | "optional": true 471 | }, 472 | "mongodb": { 473 | "optional": true 474 | }, 475 | "mysql2": { 476 | "optional": true 477 | }, 478 | "notion-to-md": { 479 | "optional": true 480 | }, 481 | "pdf-parse": { 482 | "optional": true 483 | }, 484 | "peggy": { 485 | "optional": true 486 | }, 487 | "pg": { 488 | "optional": true 489 | }, 490 | "pickleparser": { 491 | "optional": true 492 | }, 493 | "playwright": { 494 | "optional": true 495 | }, 496 | "puppeteer": { 497 | "optional": true 498 | }, 499 | "redis": { 500 | "optional": true 501 | }, 502 | "replicate": { 503 | "optional": true 504 | }, 505 | "srt-parser-2": { 506 | "optional": true 507 | }, 508 | "typeorm": { 509 | "optional": true 510 | }, 511 | "typesense": { 512 | "optional": true 513 | }, 514 | "vectordb": { 515 | "optional": true 516 | }, 517 | "weaviate-ts-client": { 518 | "optional": true 519 | } 520 | } 521 | }, 522 | "node_modules/langchainplus-sdk": { 523 | "version": "0.0.19", 524 | "resolved": "https://registry.npmjs.org/langchainplus-sdk/-/langchainplus-sdk-0.0.19.tgz", 525 | "integrity": "sha512-WKMN90E8M/JWO9lajbw8sv32xVHUZM+fN9I4LUveffBoX4qoJFi6uzYeDH0loW2SjnOSh3Tnzlz+Qp6ZenCu6g==", 526 | "dependencies": { 527 | "@types/uuid": "^9.0.1", 528 | "commander": "^10.0.1", 529 | "p-queue": "^6.6.2", 530 | "p-retry": "4", 531 | "uuid": "^9.0.0" 532 | }, 533 | "bin": { 534 | "langchain": "dist/cli/main.cjs" 535 | } 536 | }, 537 | "node_modules/mime-db": { 538 | "version": "1.52.0", 539 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", 540 | "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", 541 | "engines": { 542 | "node": ">= 0.6" 543 | } 544 | }, 545 | "node_modules/mime-types": { 546 | "version": "2.1.35", 547 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", 548 | "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", 549 | "dependencies": { 550 | "mime-db": "1.52.0" 551 | }, 552 | "engines": { 553 | "node": ">= 0.6" 554 | } 555 | }, 556 | "node_modules/ml-array-mean": { 557 | "version": "1.1.6", 558 | "resolved": "https://registry.npmjs.org/ml-array-mean/-/ml-array-mean-1.1.6.tgz", 559 | "integrity": "sha512-MIdf7Zc8HznwIisyiJGRH9tRigg3Yf4FldW8DxKxpCCv/g5CafTw0RRu51nojVEOXuCQC7DRVVu5c7XXO/5joQ==", 560 | "dependencies": { 561 | "ml-array-sum": "^1.1.6" 562 | } 563 | }, 564 | "node_modules/ml-array-sum": { 565 | "version": "1.1.6", 566 | "resolved": "https://registry.npmjs.org/ml-array-sum/-/ml-array-sum-1.1.6.tgz", 567 | "integrity": "sha512-29mAh2GwH7ZmiRnup4UyibQZB9+ZLyMShvt4cH4eTK+cL2oEMIZFnSyB3SS8MlsTh6q/w/yh48KmqLxmovN4Dw==", 568 | "dependencies": { 569 | "is-any-array": "^2.0.0" 570 | } 571 | }, 572 | "node_modules/ml-distance": { 573 | "version": "4.0.1", 574 | "resolved": "https://registry.npmjs.org/ml-distance/-/ml-distance-4.0.1.tgz", 575 | "integrity": "sha512-feZ5ziXs01zhyFUUUeZV5hwc0f5JW0Sh0ckU1koZe/wdVkJdGxcP06KNQuF0WBTj8FttQUzcvQcpcrOp/XrlEw==", 576 | "dependencies": { 577 | "ml-array-mean": "^1.1.6", 578 | "ml-distance-euclidean": "^2.0.0", 579 | "ml-tree-similarity": "^1.0.0" 580 | } 581 | }, 582 | "node_modules/ml-distance-euclidean": { 583 | "version": "2.0.0", 584 | "resolved": "https://registry.npmjs.org/ml-distance-euclidean/-/ml-distance-euclidean-2.0.0.tgz", 585 | "integrity": "sha512-yC9/2o8QF0A3m/0IXqCTXCzz2pNEzvmcE/9HFKOZGnTjatvBbsn4lWYJkxENkA4Ug2fnYl7PXQxnPi21sgMy/Q==" 586 | }, 587 | "node_modules/ml-tree-similarity": { 588 | "version": "1.0.0", 589 | "resolved": "https://registry.npmjs.org/ml-tree-similarity/-/ml-tree-similarity-1.0.0.tgz", 590 | "integrity": "sha512-XJUyYqjSuUQkNQHMscr6tcjldsOoAekxADTplt40QKfwW6nd++1wHWV9AArl0Zvw/TIHgNaZZNvr8QGvE8wLRg==", 591 | "dependencies": { 592 | "binary-search": "^1.3.5", 593 | "num-sort": "^2.0.0" 594 | } 595 | }, 596 | "node_modules/node-addon-api": { 597 | "version": "6.1.0", 598 | "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", 599 | "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==" 600 | }, 601 | "node_modules/node-fetch": { 602 | "version": "2.6.12", 603 | "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", 604 | "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", 605 | "dependencies": { 606 | "whatwg-url": "^5.0.0" 607 | }, 608 | "engines": { 609 | "node": "4.x || >=6.0.0" 610 | }, 611 | "peerDependencies": { 612 | "encoding": "^0.1.0" 613 | }, 614 | "peerDependenciesMeta": { 615 | "encoding": { 616 | "optional": true 617 | } 618 | } 619 | }, 620 | "node_modules/num-sort": { 621 | "version": "2.1.0", 622 | "resolved": "https://registry.npmjs.org/num-sort/-/num-sort-2.1.0.tgz", 623 | "integrity": "sha512-1MQz1Ed8z2yckoBeSfkQHHO9K1yDRxxtotKSJ9yvcTUUxSvfvzEq5GwBrjjHEpMlq/k5gvXdmJ1SbYxWtpNoVg==", 624 | "engines": { 625 | "node": ">=8" 626 | }, 627 | "funding": { 628 | "url": "https://github.com/sponsors/sindresorhus" 629 | } 630 | }, 631 | "node_modules/object-hash": { 632 | "version": "3.0.0", 633 | "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", 634 | "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", 635 | "engines": { 636 | "node": ">= 6" 637 | } 638 | }, 639 | "node_modules/openai": { 640 | "version": "3.3.0", 641 | "resolved": "https://registry.npmjs.org/openai/-/openai-3.3.0.tgz", 642 | "integrity": "sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==", 643 | "dependencies": { 644 | "axios": "^0.26.0", 645 | "form-data": "^4.0.0" 646 | } 647 | }, 648 | "node_modules/openapi-types": { 649 | "version": "12.1.3", 650 | "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", 651 | "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==" 652 | }, 653 | "node_modules/p-finally": { 654 | "version": "1.0.0", 655 | "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", 656 | "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", 657 | "engines": { 658 | "node": ">=4" 659 | } 660 | }, 661 | "node_modules/p-queue": { 662 | "version": "6.6.2", 663 | "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", 664 | "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", 665 | "dependencies": { 666 | "eventemitter3": "^4.0.4", 667 | "p-timeout": "^3.2.0" 668 | }, 669 | "engines": { 670 | "node": ">=8" 671 | }, 672 | "funding": { 673 | "url": "https://github.com/sponsors/sindresorhus" 674 | } 675 | }, 676 | "node_modules/p-retry": { 677 | "version": "4.6.2", 678 | "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", 679 | "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", 680 | "dependencies": { 681 | "@types/retry": "0.12.0", 682 | "retry": "^0.13.1" 683 | }, 684 | "engines": { 685 | "node": ">=8" 686 | } 687 | }, 688 | "node_modules/p-timeout": { 689 | "version": "3.2.0", 690 | "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", 691 | "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", 692 | "dependencies": { 693 | "p-finally": "^1.0.0" 694 | }, 695 | "engines": { 696 | "node": ">=8" 697 | } 698 | }, 699 | "node_modules/retry": { 700 | "version": "0.13.1", 701 | "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", 702 | "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", 703 | "engines": { 704 | "node": ">= 4" 705 | } 706 | }, 707 | "node_modules/tr46": { 708 | "version": "0.0.3", 709 | "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", 710 | "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" 711 | }, 712 | "node_modules/uuid": { 713 | "version": "9.0.0", 714 | "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", 715 | "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", 716 | "bin": { 717 | "uuid": "dist/bin/uuid" 718 | } 719 | }, 720 | "node_modules/webidl-conversions": { 721 | "version": "3.0.1", 722 | "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", 723 | "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" 724 | }, 725 | "node_modules/whatwg-url": { 726 | "version": "5.0.0", 727 | "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", 728 | "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", 729 | "dependencies": { 730 | "tr46": "~0.0.3", 731 | "webidl-conversions": "^3.0.0" 732 | } 733 | }, 734 | "node_modules/yaml": { 735 | "version": "2.3.1", 736 | "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.1.tgz", 737 | "integrity": "sha512-2eHWfjaoXgTBC2jNM1LRef62VQa0umtvRiDSk6HSzW7RvS5YtkabJrwYLLEKWBc8a5U2PTSCs+dJjUTJdlHsWQ==", 738 | "engines": { 739 | "node": ">= 14" 740 | } 741 | }, 742 | "node_modules/zod": { 743 | "version": "3.21.4", 744 | "resolved": "https://registry.npmjs.org/zod/-/zod-3.21.4.tgz", 745 | "integrity": "sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==", 746 | "funding": { 747 | "url": "https://github.com/sponsors/colinhacks" 748 | } 749 | }, 750 | "node_modules/zod-to-json-schema": { 751 | "version": "3.21.3", 752 | "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.21.3.tgz", 753 | "integrity": "sha512-09W/9oyxeF1/wWnzCb6MursW+lOzgKi91QwE7eTBbC+t/qgfuLsUVDai3lHemSQnQu/UONAcT/fv3ZnDvbTeKg==", 754 | "peerDependencies": { 755 | "zod": "^3.21.4" 756 | } 757 | } 758 | } 759 | } 760 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "langchain", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "type": "module", 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "dotenv": "^16.3.1", 14 | "hnswlib-node": "^1.4.2", 15 | "langchain": "^0.0.103" 16 | } 17 | } 18 | --------------------------------------------------------------------------------