├── Documents.index ├── args.json ├── docstore.json └── hnswlib.index ├── README.md ├── documents ├── langchain.csv ├── langchain.json └── langchain.txt ├── index.js ├── package-lock.json └── package.json /Documents.index/args.json: -------------------------------------------------------------------------------- 1 | {"space":"cosine","numDimensions":1536} -------------------------------------------------------------------------------- /Documents.index/docstore.json: -------------------------------------------------------------------------------- 1 | [["0",{"pageContent":"© 2019 Apple Inc. All rights reserved. Apple and the Apple logo are trademarks of Apple Inc., registered in the U.S. and \n \nother countries. ENERGY STAR® is a U.S. registered trademark. 036-00019-A\nENERGY STAR® Compliance\nAs an ENERGY STAR partner, Apple has determined that standard configurations of this product meet the ENERGY STAR \nguidelines for energy efficiency. The ENERGY STAR program is a partnership with electronic equipment manufacturers to \npromote energy-efficient products. Reducing energy consumption of products saves money and helps conserve valuable \nresources.\nThis computer is shipped with power management enabled with the computer set to sleep after 10 minutes of user inactivity. \nTo wake your computer, click the mouse or press any key on the keyboard.\nFor more information about ENERGY STAR, visit: www.energystar.gov\nApple and the Environment\nAt Apple, we recognize our responsibility to minimize the environmental impacts of our operations and products.","metadata":{"loc":{"lines":{"from":1,"to":13}}}}],["1",{"pageContent":"Apple and the Environment\nAt Apple, we recognize our responsibility to minimize the environmental impacts of our operations and products. \n \nFor more information, go to: \nwww.apple.com/environment/\nConformité ENERGY STAR®\nEn tant que partenaire ENERGY\n \nSTAR, Apple a établi que les configurations standard de ce produit sont conformes aux \ndirectives d’ENERGY\n \nSTAR en matière d’efficacité énergétique. Le programme ENERGY\n \nSTAR est un partenariat avec les \nfabricants d’équipements électroniques dont l’objectif est de promouvoir des produits écoénergétiques. La réduction de la \nconsommation d’énergie des produits permet de faire des économies et de préserver de précieuses ressources.\nCet ordinateur est livré avec l’économiseur d’énergie activé de sorte que l’ordinateur se mette en veille après 10\n \nminutes \nd’inactivité. Pour réactiver votre ordinateur, cliquez sur le bouton de la souris ou du pavé tactile, ou appuyez sur n’importe \nquelle touche du clavier.\nPour en savoir plus sur ENERGY","metadata":{"loc":{"lines":{"from":13,"to":34}}}}],["2",{"pageContent":"minutes \nd’inactivité. Pour réactiver votre ordinateur, cliquez sur le bouton de la souris ou du pavé tactile, ou appuyez sur n’importe \nquelle touche du clavier.\nPour en savoir plus sur ENERGY\n \nSTAR, rendez-vous sur\n \n: www.energystar.gov\nApple et l’environnement\nChez Apple, nous reconnaissons notre responsabilité de minimiser les impacts environnementaux de nos activités et de nos \nproduits. Pour en savoir plus, rendez-vous sur\n \n: \nwww.apple.com/ca/fr/environment/\n© 2019 Apple Inc. Tous droits réservés. Apple et le logo Apple sont des marques de commerce d’Apple Inc., déposées aux \nÉtats-Unis et dans d’autres pays. ENERGY STAR® est une marque déposée aux États-Unis.","metadata":{"loc":{"lines":{"from":34,"to":49}}}}],["3",{"pageContent":"Application: Chatbots\nDescription: Langchain can be used to build chatbots that understand natural language and provide intelligent responses.","metadata":{"loc":{"lines":{"from":1,"to":2}}}}],["4",{"pageContent":"Application: Content Generation\nDescription: Langchain can generate coherent and contextually relevant text, making it suitable for content generation tasks.","metadata":{"loc":{"lines":{"from":1,"to":2}}}}],["5",{"pageContent":"Application: Sentiment Analysis\nDescription: Langchain can analyze the sentiment of text, allowing developers to build applications that understand the emotions expressed in text.","metadata":{"loc":{"lines":{"from":1,"to":2}}}}],["6",{"pageContent":"Application: Language Translation\nDescription: Langchain can be used for language translation tasks, enabling the translation of text from one language to another.","metadata":{"loc":{"lines":{"from":1,"to":2}}}}],["7",{"pageContent":"Application: Information Retrieval\nDescription: Langchain's document retrieval capabilities make it ideal for building information retrieval systems and search engines.","metadata":{"loc":{"lines":{"from":1,"to":2}}}}],["8",{"pageContent":"Langchain is a library for working with language models. It provides an easy-to-use interface for tasks such as document retrieval, text embeddings, and question-answering. Langchain is designed to help developers integrate language models into their applications and leverage the power of modern AI models.","metadata":{"loc":{"lines":{"from":1,"to":1}}}}],["9",{"pageContent":"Chatbots","metadata":{"loc":{"lines":{"from":1,"to":1}}}}],["10",{"pageContent":"Langchain can be used to build chatbots that understand natural language queries and provide intelligent and contextually relevant responses to users.","metadata":{"loc":{"lines":{"from":1,"to":1}}}}],["11",{"pageContent":"Content Generation","metadata":{"loc":{"lines":{"from":1,"to":1}}}}],["12",{"pageContent":"Langchain can generate coherent and contextually relevant text, making it suitable for content generation tasks such as article writing, social media posts, and creative writing.","metadata":{"loc":{"lines":{"from":1,"to":1}}}}],["13",{"pageContent":"Sentiment Analysis","metadata":{"loc":{"lines":{"from":1,"to":1}}}}],["14",{"pageContent":"Langchain can analyze the sentiment of text, allowing developers to build applications that understand the emotions and opinions expressed in text, such as customer reviews and social media comments.","metadata":{"loc":{"lines":{"from":1,"to":1}}}}],["15",{"pageContent":"Language Translation","metadata":{"loc":{"lines":{"from":1,"to":1}}}}],["16",{"pageContent":"Langchain can be used for language translation tasks, enabling the translation of text from one language to another, which is useful for multilingual applications and global communication.","metadata":{"loc":{"lines":{"from":1,"to":1}}}}],["17",{"pageContent":"Information Retrieval","metadata":{"loc":{"lines":{"from":1,"to":1}}}}],["18",{"pageContent":"Langchain's document retrieval capabilities make it ideal for building information retrieval systems and search engines that can quickly find relevant documents based on user queries.","metadata":{"loc":{"lines":{"from":1,"to":1}}}}],["19",{"pageContent":"Langchain is a powerful library for working with language models. Some of its key features include:\n\n- Document Retrieval: Langchain allows you to retrieve relevant documents based on a given query. This feature is useful for building search engines and information retrieval systems.\n\n- Text Embeddings: Langchain can generate vector representations (embeddings) of text. These embeddings capture the semantic meaning of the text and can be used for tasks such as text classification and clustering.\n\n- Question-Answering: Langchain provides a question-answering functionality that can answer questions based on a set of documents. This feature is valuable for building intelligent chatbots and virtual assistants.\n\n- Support for Multiple File Formats: Langchain supports loading documents from various file formats, including JSON, CSV, TXT, and PDF. This flexibility makes it easy to work with different types of data.","metadata":{"loc":{"lines":{"from":1,"to":9}}}}],["20",{"pageContent":"- Vector Store: Langchain allows you to create and manage vector stores, which are collections of vector embeddings. Vector stores can be used for efficient similarity search and retrieval.","metadata":{"loc":{"lines":{"from":11,"to":11}}}}]] -------------------------------------------------------------------------------- /Documents.index/hnswlib.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/developersdigest/langchain-document-loaders-in-node-js/daa236337842661de9942b4f10202f1fd9ebe0f0/Documents.index/hnswlib.index -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | From my Youtube Video 2 | Talk to Your Files: Conversational AI for Any Folder of Documents with Langchain in Node.js 3 | https://www.youtube.com/watch?v=EFM-xutgAvY 4 | -------------------------------------------------------------------------------- /documents/langchain.csv: -------------------------------------------------------------------------------- 1 | Application,Description 2 | "Chatbots","Langchain can be used to build chatbots that understand natural language and provide intelligent responses." 3 | "Content Generation","Langchain can generate coherent and contextually relevant text, making it suitable for content generation tasks." 4 | "Sentiment Analysis","Langchain can analyze the sentiment of text, allowing developers to build applications that understand the emotions expressed in text." 5 | "Language Translation","Langchain can be used for language translation tasks, enabling the translation of text from one language to another." 6 | "Information Retrieval","Langchain's document retrieval capabilities make it ideal for building information retrieval systems and search engines." 7 | -------------------------------------------------------------------------------- /documents/langchain.json: -------------------------------------------------------------------------------- 1 | { 2 | "langchain": { 3 | "overview": "Langchain is a library for working with language models. It provides an easy-to-use interface for tasks such as document retrieval, text embeddings, and question-answering. Langchain is designed to help developers integrate language models into their applications and leverage the power of modern AI models.", 4 | "use_cases": [ 5 | { 6 | "title": "Chatbots", 7 | "description": "Langchain can be used to build chatbots that understand natural language queries and provide intelligent and contextually relevant responses to users." 8 | }, 9 | { 10 | "title": "Content Generation", 11 | "description": "Langchain can generate coherent and contextually relevant text, making it suitable for content generation tasks such as article writing, social media posts, and creative writing." 12 | }, 13 | { 14 | "title": "Sentiment Analysis", 15 | "description": "Langchain can analyze the sentiment of text, allowing developers to build applications that understand the emotions and opinions expressed in text, such as customer reviews and social media comments." 16 | }, 17 | { 18 | "title": "Language Translation", 19 | "description": "Langchain can be used for language translation tasks, enabling the translation of text from one language to another, which is useful for multilingual applications and global communication." 20 | }, 21 | { 22 | "title": "Information Retrieval", 23 | "description": "Langchain's document retrieval capabilities make it ideal for building information retrieval systems and search engines that can quickly find relevant documents based on user queries." 24 | } 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /documents/langchain.txt: -------------------------------------------------------------------------------- 1 | Langchain is a powerful library for working with language models. Some of its key features include: 2 | 3 | - Document Retrieval: Langchain allows you to retrieve relevant documents based on a given query. This feature is useful for building search engines and information retrieval systems. 4 | 5 | - Text Embeddings: Langchain can generate vector representations (embeddings) of text. These embeddings capture the semantic meaning of the text and can be used for tasks such as text classification and clustering. 6 | 7 | - Question-Answering: Langchain provides a question-answering functionality that can answer questions based on a set of documents. This feature is valuable for building intelligent chatbots and virtual assistants. 8 | 9 | - Support for Multiple File Formats: Langchain supports loading documents from various file formats, including JSON, CSV, TXT, and PDF. This flexibility makes it easy to work with different types of data. 10 | 11 | - Vector Store: Langchain allows you to create and manage vector stores, which are collections of vector embeddings. Vector stores can be used for efficient similarity search and retrieval. 12 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | // 1. Import document loaders for different file formats 2 | import { DirectoryLoader } from "langchain/document_loaders/fs/directory"; 3 | import { JSONLoader } from "langchain/document_loaders/fs/json"; 4 | import { TextLoader } from "langchain/document_loaders/fs/text"; 5 | import { CSVLoader } from "langchain/document_loaders/fs/csv"; 6 | import { PDFLoader } from "langchain/document_loaders/fs/pdf"; 7 | 8 | // 2. Import OpenAI language model and other related modules 9 | import { OpenAI } from "langchain/llms/openai"; 10 | import { RetrievalQAChain } from "langchain/chains"; 11 | import { HNSWLib } from "langchain/vectorstores/hnswlib"; 12 | import { OpenAIEmbeddings } from "langchain/embeddings/openai"; 13 | import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; 14 | 15 | // 3. Import Tiktoken for token counting 16 | import { Tiktoken } from "@dqbd/tiktoken/lite"; 17 | import { load } from "@dqbd/tiktoken/load"; 18 | import registry from "@dqbd/tiktoken/registry.json" assert { type: "json" }; 19 | import models from "@dqbd/tiktoken/model_to_encoding.json" assert { type: "json" }; 20 | 21 | // 4. Import dotenv for loading environment variables and fs for file system operations 22 | import dotenv from "dotenv"; 23 | import fs from "fs"; 24 | dotenv.config(); 25 | 26 | // 5. Initialize the document loader with supported file formats 27 | const loader = new DirectoryLoader("./documents", { 28 | ".json": (path) => new JSONLoader(path), 29 | ".txt": (path) => new TextLoader(path), 30 | ".csv": (path) => new CSVLoader(path), 31 | ".pdf": (path) => new PDFLoader(path), 32 | }); 33 | 34 | // 6. Load documents from the specified directory 35 | console.log("Loading docs..."); 36 | const docs = await loader.load(); 37 | console.log("Docs loaded."); 38 | 39 | // 7. Define a function to calculate the cost of tokenizing the documents 40 | async function calculateCost() { 41 | const modelName = "text-embedding-ada-002"; 42 | const modelKey = models[modelName]; 43 | const model = await load(registry[modelKey]); 44 | const encoder = new Tiktoken( 45 | model.bpe_ranks, 46 | model.special_tokens, 47 | model.pat_str 48 | ); 49 | const tokens = encoder.encode(JSON.stringify(docs)); 50 | const tokenCount = tokens.length; 51 | const ratePerThousandTokens = 0.0004; 52 | const cost = (tokenCount / 1000) * ratePerThousandTokens; 53 | encoder.free(); 54 | return cost; 55 | } 56 | 57 | const VECTOR_STORE_PATH = "Documents.index"; 58 | const question = "Tell me about these docs"; 59 | 60 | // 8. Define a function to normalize the content of the documents 61 | function normalizeDocuments(docs) { 62 | return docs.map((doc) => { 63 | if (typeof doc.pageContent === "string") { 64 | return doc.pageContent; 65 | } else if (Array.isArray(doc.pageContent)) { 66 | return doc.pageContent.join("\n"); 67 | } 68 | }); 69 | } 70 | 71 | // 9. Define the main function to run the entire process 72 | export const run = async () => { 73 | // 10. Calculate the cost of tokenizing the documents 74 | console.log("Calculating cost..."); 75 | const cost = await calculateCost(); 76 | console.log("Cost calculated:", cost); 77 | 78 | // 11. Check if the cost is within the acceptable limit 79 | if (cost <= 1) { 80 | // 12. Initialize the OpenAI language model 81 | const model = new OpenAI({}); 82 | 83 | let vectorStore; 84 | 85 | // 13. Check if an existing vector store is available 86 | console.log("Checking for existing vector store..."); 87 | if (fs.existsSync(VECTOR_STORE_PATH)) { 88 | // 14. Load the existing vector store 89 | console.log("Loading existing vector store..."); 90 | vectorStore = await HNSWLib.load( 91 | VECTOR_STORE_PATH, 92 | new OpenAIEmbeddings() 93 | ); 94 | console.log("Vector store loaded."); 95 | } else { 96 | // 15. Create a new vector store if one does not exist 97 | console.log("Creating new vector store..."); 98 | const textSplitter = new RecursiveCharacterTextSplitter({ 99 | chunkSize: 1000, 100 | }); 101 | const normalizedDocs = normalizeDocuments(docs); 102 | const splitDocs = await textSplitter.createDocuments(normalizedDocs); 103 | 104 | // 16. Generate the vector store from the documents 105 | vectorStore = await HNSWLib.fromDocuments( 106 | splitDocs, 107 | new OpenAIEmbeddings() 108 | ); 109 | // 17. Save the vector store to the specified path 110 | await vectorStore.save(VECTOR_STORE_PATH); 111 | 112 | console.log("Vector store created."); 113 | } 114 | 115 | // 18. Create a retrieval chain using the language model and vector store 116 | console.log("Creating retrieval chain..."); 117 | const chain = RetrievalQAChain.fromLLM(model, vectorStore.asRetriever()); 118 | 119 | // 19. Query the retrieval chain with the specified question 120 | console.log("Querying chain..."); 121 | const res = await chain.call({ query: question }); 122 | console.log({ res }); 123 | } else { 124 | // 20. If the cost exceeds the limit, skip the embedding process 125 | console.log("The cost of embedding exceeds $1. Skipping embeddings."); 126 | } 127 | }; 128 | 129 | // 21. Run the main function 130 | run(); -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "langchain-document-loader", 3 | "version": "1.0.0", 4 | "lockfileVersion": 3, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "langchain-document-loader", 9 | "version": "1.0.0", 10 | "license": "ISC", 11 | "dependencies": { 12 | "@dqbd/tiktoken": "^1.0.7", 13 | "dotenv": "^16.0.3", 14 | "hnswlib-node": "^1.4.2", 15 | "langchain": "^0.0.67", 16 | "pdf-parse": "^1.1.1" 17 | } 18 | }, 19 | "node_modules/@anthropic-ai/sdk": { 20 | "version": "0.4.3", 21 | "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.4.3.tgz", 22 | "integrity": "sha512-SZrlXvjUUYT9rPmSzlTtmVk1OjVNpkCzILRluhiYwNcxXfQyvPJDi0CI6PyymygcgtqEF5EVqhKmC/PtPsNEIw==", 23 | "dependencies": { 24 | "@fortaine/fetch-event-source": "^3.0.6", 25 | "cross-fetch": "^3.1.5" 26 | } 27 | }, 28 | "node_modules/@dqbd/tiktoken": { 29 | "version": "1.0.7", 30 | "resolved": "https://registry.npmjs.org/@dqbd/tiktoken/-/tiktoken-1.0.7.tgz", 31 | "integrity": "sha512-bhR5k5W+8GLzysjk8zTMVygQZsgvf7W1F0IlL4ZQ5ugjo5rCyiwGM5d8DYriXspytfu98tv59niang3/T+FoDw==" 32 | }, 33 | "node_modules/@fortaine/fetch-event-source": { 34 | "version": "3.0.6", 35 | "resolved": "https://registry.npmjs.org/@fortaine/fetch-event-source/-/fetch-event-source-3.0.6.tgz", 36 | "integrity": "sha512-621GAuLMvKtyZQ3IA6nlDWhV1V/7PGOTNIGLUifxt0KzM+dZIweJ6F3XvQF3QnqeNfS1N7WQ0Kil1Di/lhChEw==", 37 | "engines": { 38 | "node": ">=16.15" 39 | } 40 | }, 41 | "node_modules/@selderee/plugin-htmlparser2": { 42 | "version": "0.11.0", 43 | "resolved": "https://registry.npmjs.org/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz", 44 | "integrity": "sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==", 45 | "optional": true, 46 | "peer": true, 47 | "dependencies": { 48 | "domhandler": "^5.0.3", 49 | "selderee": "^0.11.0" 50 | }, 51 | "funding": { 52 | "url": "https://ko-fi.com/killymxi" 53 | } 54 | }, 55 | "node_modules/@types/node": { 56 | "version": "18.16.3", 57 | "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.3.tgz", 58 | "integrity": "sha512-OPs5WnnT1xkCBiuQrZA4+YAV4HEJejmHneyraIaxsbev5yCEr6KMwINNFP9wQeFIw8FWcoTqF3vQsa5CDaI+8Q==", 59 | "optional": true, 60 | "peer": true 61 | }, 62 | "node_modules/@types/retry": { 63 | "version": "0.12.0", 64 | "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", 65 | "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==" 66 | }, 67 | "node_modules/adm-zip": { 68 | "version": "0.5.10", 69 | "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.10.tgz", 70 | "integrity": "sha512-x0HvcHqVJNTPk/Bw8JbLWlWoo6Wwnsug0fnYYro1HBrjxZ3G7/AZk7Ahv8JwDe1uIcz8eBqvu86FuF1POiG7vQ==", 71 | "optional": true, 72 | "peer": true, 73 | "engines": { 74 | "node": ">=6.0" 75 | } 76 | }, 77 | "node_modules/ansi-styles": { 78 | "version": "5.2.0", 79 | "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", 80 | "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", 81 | "engines": { 82 | "node": ">=10" 83 | }, 84 | "funding": { 85 | "url": "https://github.com/chalk/ansi-styles?sponsor=1" 86 | } 87 | }, 88 | "node_modules/argparse": { 89 | "version": "1.0.10", 90 | "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", 91 | "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", 92 | "optional": true, 93 | "peer": true, 94 | "dependencies": { 95 | "sprintf-js": "~1.0.2" 96 | } 97 | }, 98 | "node_modules/array-hyper-unique": { 99 | "version": "2.1.4", 100 | "resolved": "https://registry.npmjs.org/array-hyper-unique/-/array-hyper-unique-2.1.4.tgz", 101 | "integrity": "sha512-RVsGx2YpFGhGpgdkK7A0VjFQecVUCowpkQerGCsyXVRXHxccAlPPTDt9ueF/X7Zq/6z6duZ49i9WzTCzcnQygQ==", 102 | "optional": true, 103 | "peer": true, 104 | "dependencies": { 105 | "deep-eql": "= 4.0.0", 106 | "lodash": "^4.17.21" 107 | } 108 | }, 109 | "node_modules/asynckit": { 110 | "version": "0.4.0", 111 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 112 | "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" 113 | }, 114 | "node_modules/axios": { 115 | "version": "0.26.1", 116 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", 117 | "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", 118 | "dependencies": { 119 | "follow-redirects": "^1.14.8" 120 | } 121 | }, 122 | "node_modules/binary-extensions": { 123 | "version": "2.2.0", 124 | "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", 125 | "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", 126 | "engines": { 127 | "node": ">=8" 128 | } 129 | }, 130 | "node_modules/binary-search": { 131 | "version": "1.3.6", 132 | "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz", 133 | "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==" 134 | }, 135 | "node_modules/bindings": { 136 | "version": "1.5.0", 137 | "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", 138 | "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", 139 | "dependencies": { 140 | "file-uri-to-path": "1.0.0" 141 | } 142 | }, 143 | "node_modules/bluebird": { 144 | "version": "3.4.7", 145 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", 146 | "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==", 147 | "optional": true, 148 | "peer": true 149 | }, 150 | "node_modules/browser-or-node": { 151 | "version": "2.1.1", 152 | "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-2.1.1.tgz", 153 | "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==" 154 | }, 155 | "node_modules/combined-stream": { 156 | "version": "1.0.8", 157 | "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", 158 | "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", 159 | "dependencies": { 160 | "delayed-stream": "~1.0.0" 161 | }, 162 | "engines": { 163 | "node": ">= 0.8" 164 | } 165 | }, 166 | "node_modules/commander": { 167 | "version": "2.20.3", 168 | "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", 169 | "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", 170 | "optional": true, 171 | "peer": true 172 | }, 173 | "node_modules/core-util-is": { 174 | "version": "1.0.3", 175 | "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", 176 | "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", 177 | "optional": true, 178 | "peer": true 179 | }, 180 | "node_modules/crlf-normalize": { 181 | "version": "1.0.19", 182 | "resolved": "https://registry.npmjs.org/crlf-normalize/-/crlf-normalize-1.0.19.tgz", 183 | "integrity": "sha512-cpV1h7YwFtIA36NHtyWuMMMPGxUp6zrzxjRnFEDLh1ZH0SPNUqCWmM8RlKVycxvKHgZOxWXs3XxX/DAlBAjFzA==", 184 | "optional": true, 185 | "peer": true, 186 | "dependencies": { 187 | "ts-type": ">=2" 188 | } 189 | }, 190 | "node_modules/cross-fetch": { 191 | "version": "3.1.5", 192 | "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", 193 | "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", 194 | "dependencies": { 195 | "node-fetch": "2.6.7" 196 | } 197 | }, 198 | "node_modules/d3-dsv": { 199 | "version": "2.0.0", 200 | "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-2.0.0.tgz", 201 | "integrity": "sha512-E+Pn8UJYx9mViuIUkoc93gJGGYut6mSDKy2+XaPwccwkRGlR+LO97L2VCCRjQivTwLHkSnAJG7yo00BWY6QM+w==", 202 | "optional": true, 203 | "peer": true, 204 | "dependencies": { 205 | "commander": "2", 206 | "iconv-lite": "0.4", 207 | "rw": "1" 208 | }, 209 | "bin": { 210 | "csv2json": "bin/dsv2json", 211 | "csv2tsv": "bin/dsv2dsv", 212 | "dsv2dsv": "bin/dsv2dsv", 213 | "dsv2json": "bin/dsv2json", 214 | "json2csv": "bin/json2dsv", 215 | "json2dsv": "bin/json2dsv", 216 | "json2tsv": "bin/json2dsv", 217 | "tsv2csv": "bin/dsv2dsv", 218 | "tsv2json": "bin/dsv2json" 219 | } 220 | }, 221 | "node_modules/debug": { 222 | "version": "3.2.7", 223 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", 224 | "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", 225 | "dependencies": { 226 | "ms": "^2.1.1" 227 | } 228 | }, 229 | "node_modules/deep-eql": { 230 | "version": "4.0.0", 231 | "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.0.0.tgz", 232 | "integrity": "sha512-GxJC5MOg2KyQlv6WiUF/VAnMj4MWnYiXo4oLgeptOELVoknyErb4Z8+5F/IM/K4g9/80YzzatxmWcyRwUseH0A==", 233 | "optional": true, 234 | "peer": true, 235 | "dependencies": { 236 | "type-detect": "^4.0.0" 237 | }, 238 | "engines": { 239 | "node": ">=6" 240 | } 241 | }, 242 | "node_modules/deepmerge": { 243 | "version": "4.3.1", 244 | "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", 245 | "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", 246 | "optional": true, 247 | "peer": true, 248 | "engines": { 249 | "node": ">=0.10.0" 250 | } 251 | }, 252 | "node_modules/delayed-stream": { 253 | "version": "1.0.0", 254 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 255 | "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", 256 | "engines": { 257 | "node": ">=0.4.0" 258 | } 259 | }, 260 | "node_modules/dingbat-to-unicode": { 261 | "version": "1.0.1", 262 | "resolved": "https://registry.npmjs.org/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz", 263 | "integrity": "sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==", 264 | "optional": true, 265 | "peer": true 266 | }, 267 | "node_modules/dom-serializer": { 268 | "version": "2.0.0", 269 | "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", 270 | "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", 271 | "optional": true, 272 | "peer": true, 273 | "dependencies": { 274 | "domelementtype": "^2.3.0", 275 | "domhandler": "^5.0.2", 276 | "entities": "^4.2.0" 277 | }, 278 | "funding": { 279 | "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" 280 | } 281 | }, 282 | "node_modules/domelementtype": { 283 | "version": "2.3.0", 284 | "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", 285 | "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", 286 | "funding": [ 287 | { 288 | "type": "github", 289 | "url": "https://github.com/sponsors/fb55" 290 | } 291 | ], 292 | "optional": true, 293 | "peer": true 294 | }, 295 | "node_modules/domhandler": { 296 | "version": "5.0.3", 297 | "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", 298 | "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", 299 | "optional": true, 300 | "peer": true, 301 | "dependencies": { 302 | "domelementtype": "^2.3.0" 303 | }, 304 | "engines": { 305 | "node": ">= 4" 306 | }, 307 | "funding": { 308 | "url": "https://github.com/fb55/domhandler?sponsor=1" 309 | } 310 | }, 311 | "node_modules/domutils": { 312 | "version": "3.1.0", 313 | "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", 314 | "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", 315 | "optional": true, 316 | "peer": true, 317 | "dependencies": { 318 | "dom-serializer": "^2.0.0", 319 | "domelementtype": "^2.3.0", 320 | "domhandler": "^5.0.3" 321 | }, 322 | "funding": { 323 | "url": "https://github.com/fb55/domutils?sponsor=1" 324 | } 325 | }, 326 | "node_modules/dotenv": { 327 | "version": "16.0.3", 328 | "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz", 329 | "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==", 330 | "engines": { 331 | "node": ">=12" 332 | } 333 | }, 334 | "node_modules/duck": { 335 | "version": "0.1.12", 336 | "resolved": "https://registry.npmjs.org/duck/-/duck-0.1.12.tgz", 337 | "integrity": "sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==", 338 | "optional": true, 339 | "peer": true, 340 | "dependencies": { 341 | "underscore": "^1.13.1" 342 | } 343 | }, 344 | "node_modules/entities": { 345 | "version": "4.5.0", 346 | "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", 347 | "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", 348 | "optional": true, 349 | "peer": true, 350 | "engines": { 351 | "node": ">=0.12" 352 | }, 353 | "funding": { 354 | "url": "https://github.com/fb55/entities?sponsor=1" 355 | } 356 | }, 357 | "node_modules/epub2": { 358 | "version": "3.0.1", 359 | "resolved": "https://registry.npmjs.org/epub2/-/epub2-3.0.1.tgz", 360 | "integrity": "sha512-qwAPzDvcfKOZza1FU38BHo7qyiHMZXpJKVkdLK7I+wHi4nUhIhh+6qQbOGxPdiPaqAuI985d/nIUuJmbZ1ACrA==", 361 | "optional": true, 362 | "peer": true, 363 | "dependencies": { 364 | "adm-zip": "^0.5.9", 365 | "array-hyper-unique": "^2.1.4", 366 | "bluebird": "^3.7.2", 367 | "crlf-normalize": "^1.0.18", 368 | "tslib": "^2.4.1", 369 | "xml2js": "^0.4.23" 370 | } 371 | }, 372 | "node_modules/epub2/node_modules/bluebird": { 373 | "version": "3.7.2", 374 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", 375 | "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", 376 | "optional": true, 377 | "peer": true 378 | }, 379 | "node_modules/eventemitter3": { 380 | "version": "4.0.7", 381 | "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", 382 | "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" 383 | }, 384 | "node_modules/expr-eval": { 385 | "version": "2.0.2", 386 | "resolved": "https://registry.npmjs.org/expr-eval/-/expr-eval-2.0.2.tgz", 387 | "integrity": "sha512-4EMSHGOPSwAfBiibw3ndnP0AvjDWLsMvGOvWEZ2F96IGk0bIVdjQisOHxReSkE13mHcfbuCiXw+G4y0zv6N8Eg==" 388 | }, 389 | "node_modules/file-uri-to-path": { 390 | "version": "1.0.0", 391 | "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", 392 | "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" 393 | }, 394 | "node_modules/flat": { 395 | "version": "5.0.2", 396 | "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", 397 | "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", 398 | "bin": { 399 | "flat": "cli.js" 400 | } 401 | }, 402 | "node_modules/follow-redirects": { 403 | "version": "1.15.2", 404 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", 405 | "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", 406 | "funding": [ 407 | { 408 | "type": "individual", 409 | "url": "https://github.com/sponsors/RubenVerborgh" 410 | } 411 | ], 412 | "engines": { 413 | "node": ">=4.0" 414 | }, 415 | "peerDependenciesMeta": { 416 | "debug": { 417 | "optional": true 418 | } 419 | } 420 | }, 421 | "node_modules/form-data": { 422 | "version": "4.0.0", 423 | "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", 424 | "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", 425 | "dependencies": { 426 | "asynckit": "^0.4.0", 427 | "combined-stream": "^1.0.8", 428 | "mime-types": "^2.1.12" 429 | }, 430 | "engines": { 431 | "node": ">= 6" 432 | } 433 | }, 434 | "node_modules/hnswlib-node": { 435 | "version": "1.4.2", 436 | "resolved": "https://registry.npmjs.org/hnswlib-node/-/hnswlib-node-1.4.2.tgz", 437 | "integrity": "sha512-76PIzOaNcX8kOpKwlFPl07uelpctqDMzbiC+Qsk2JWNVkzeU/6iXRk4tfE9z3DoK1RCBrOaFXmQ6RFb1BVF9LA==", 438 | "hasInstallScript": true, 439 | "dependencies": { 440 | "bindings": "^1.5.0", 441 | "node-addon-api": "^6.0.0" 442 | } 443 | }, 444 | "node_modules/html-to-text": { 445 | "version": "9.0.5", 446 | "resolved": "https://registry.npmjs.org/html-to-text/-/html-to-text-9.0.5.tgz", 447 | "integrity": "sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg==", 448 | "optional": true, 449 | "peer": true, 450 | "dependencies": { 451 | "@selderee/plugin-htmlparser2": "^0.11.0", 452 | "deepmerge": "^4.3.1", 453 | "dom-serializer": "^2.0.0", 454 | "htmlparser2": "^8.0.2", 455 | "selderee": "^0.11.0" 456 | }, 457 | "engines": { 458 | "node": ">=14" 459 | } 460 | }, 461 | "node_modules/htmlparser2": { 462 | "version": "8.0.2", 463 | "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", 464 | "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", 465 | "funding": [ 466 | "https://github.com/fb55/htmlparser2?sponsor=1", 467 | { 468 | "type": "github", 469 | "url": "https://github.com/sponsors/fb55" 470 | } 471 | ], 472 | "optional": true, 473 | "peer": true, 474 | "dependencies": { 475 | "domelementtype": "^2.3.0", 476 | "domhandler": "^5.0.3", 477 | "domutils": "^3.0.1", 478 | "entities": "^4.4.0" 479 | } 480 | }, 481 | "node_modules/iconv-lite": { 482 | "version": "0.4.24", 483 | "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", 484 | "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", 485 | "optional": true, 486 | "peer": true, 487 | "dependencies": { 488 | "safer-buffer": ">= 2.1.2 < 3" 489 | }, 490 | "engines": { 491 | "node": ">=0.10.0" 492 | } 493 | }, 494 | "node_modules/immediate": { 495 | "version": "3.0.6", 496 | "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", 497 | "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", 498 | "optional": true, 499 | "peer": true 500 | }, 501 | "node_modules/inherits": { 502 | "version": "2.0.4", 503 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 504 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", 505 | "optional": true, 506 | "peer": true 507 | }, 508 | "node_modules/is-any-array": { 509 | "version": "2.0.1", 510 | "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-2.0.1.tgz", 511 | "integrity": "sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ==" 512 | }, 513 | "node_modules/isarray": { 514 | "version": "1.0.0", 515 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", 516 | "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", 517 | "optional": true, 518 | "peer": true 519 | }, 520 | "node_modules/jsonpointer": { 521 | "version": "5.0.1", 522 | "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", 523 | "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", 524 | "engines": { 525 | "node": ">=0.10.0" 526 | } 527 | }, 528 | "node_modules/jszip": { 529 | "version": "3.10.1", 530 | "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", 531 | "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", 532 | "optional": true, 533 | "peer": true, 534 | "dependencies": { 535 | "lie": "~3.3.0", 536 | "pako": "~1.0.2", 537 | "readable-stream": "~2.3.6", 538 | "setimmediate": "^1.0.5" 539 | } 540 | }, 541 | "node_modules/langchain": { 542 | "version": "0.0.67", 543 | "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.0.67.tgz", 544 | "integrity": "sha512-OO9NEoVYJyNTmrA76rgisA48LkA6Si7qVAS+1hakzKwf/Hj7GhvDe/NpVaWmOFtkAHusJHSbCplbeJKWIgFR2g==", 545 | "dependencies": { 546 | "@anthropic-ai/sdk": "^0.4.3", 547 | "@dqbd/tiktoken": "^1.0.7", 548 | "ansi-styles": "^5.0.0", 549 | "binary-extensions": "^2.2.0", 550 | "browser-or-node": "^2.1.1", 551 | "expr-eval": "^2.0.2", 552 | "flat": "^5.0.2", 553 | "jsonpointer": "^5.0.1", 554 | "ml-distance": "^4.0.0", 555 | "object-hash": "^3.0.0", 556 | "openai": "^3.2.0", 557 | "p-queue": "^6.6.2", 558 | "p-retry": "4", 559 | "uuid": "^9.0.0", 560 | "yaml": "^2.2.1", 561 | "zod": "^3.21.4", 562 | "zod-to-json-schema": "^3.20.4" 563 | }, 564 | "engines": { 565 | "node": ">=18" 566 | }, 567 | "peerDependencies": { 568 | "@aws-sdk/client-lambda": "^3.310.0", 569 | "@aws-sdk/client-s3": "^3.310.0", 570 | "@getmetal/metal-sdk": "*", 571 | "@huggingface/inference": "^1.5.1", 572 | "@opensearch-project/opensearch": "*", 573 | "@pinecone-database/pinecone": "*", 574 | "@supabase/supabase-js": "^2.10.0", 575 | "@tensorflow-models/universal-sentence-encoder": "*", 576 | "@tensorflow/tfjs-converter": "*", 577 | "@tensorflow/tfjs-core": "*", 578 | "@zilliz/milvus2-sdk-node": "^2.2.0", 579 | "axios": "*", 580 | "cheerio": "^1.0.0-rc.12", 581 | "chromadb": "^1.4.0", 582 | "cohere-ai": "^5.0.2", 583 | "d3-dsv": "^2.0.0", 584 | "epub2": "^3.0.1", 585 | "hnswlib-node": "^1.4.2", 586 | "html-to-text": "^9.0.5", 587 | "mammoth": "*", 588 | "mongodb": "^5.2.0", 589 | "pdf-parse": "1.1.1", 590 | "playwright": "^1.32.1", 591 | "puppeteer": "^19.7.2", 592 | "redis": "^4.6.4", 593 | "replicate": "^0.9.0", 594 | "srt-parser-2": "^1.2.2", 595 | "typeorm": "^0.3.12", 596 | "weaviate-ts-client": "^1.0.0" 597 | }, 598 | "peerDependenciesMeta": { 599 | "@aws-sdk/client-lambda": { 600 | "optional": true 601 | }, 602 | "@aws-sdk/client-s3": { 603 | "optional": true 604 | }, 605 | "@getmetal/metal-sdk": { 606 | "optional": true 607 | }, 608 | "@huggingface/inference": { 609 | "optional": true 610 | }, 611 | "@opensearch-project/opensearch": { 612 | "optional": true 613 | }, 614 | "@pinecone-database/pinecone": { 615 | "optional": true 616 | }, 617 | "@supabase/supabase-js": { 618 | "optional": true 619 | }, 620 | "@tensorflow-models/universal-sentence-encoder": { 621 | "optional": true 622 | }, 623 | "@tensorflow/tfjs-converter": { 624 | "optional": true 625 | }, 626 | "@tensorflow/tfjs-core": { 627 | "optional": true 628 | }, 629 | "@zilliz/milvus2-sdk-node": { 630 | "optional": true 631 | }, 632 | "axios": { 633 | "optional": true 634 | }, 635 | "cheerio": { 636 | "optional": true 637 | }, 638 | "chromadb": { 639 | "optional": true 640 | }, 641 | "cohere-ai": { 642 | "optional": true 643 | }, 644 | "d3-dsv": { 645 | "optional": true 646 | }, 647 | "epub2": { 648 | "optional": true 649 | }, 650 | "hnswlib-node": { 651 | "optional": true 652 | }, 653 | "html-to-text": { 654 | "optional": true 655 | }, 656 | "mammoth": { 657 | "optional": true 658 | }, 659 | "mongodb": { 660 | "optional": true 661 | }, 662 | "pdf-parse": { 663 | "optional": true 664 | }, 665 | "playwright": { 666 | "optional": true 667 | }, 668 | "puppeteer": { 669 | "optional": true 670 | }, 671 | "redis": { 672 | "optional": true 673 | }, 674 | "replicate": { 675 | "optional": true 676 | }, 677 | "srt-parser-2": { 678 | "optional": true 679 | }, 680 | "typeorm": { 681 | "optional": true 682 | }, 683 | "weaviate-ts-client": { 684 | "optional": true 685 | } 686 | } 687 | }, 688 | "node_modules/leac": { 689 | "version": "0.6.0", 690 | "resolved": "https://registry.npmjs.org/leac/-/leac-0.6.0.tgz", 691 | "integrity": "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==", 692 | "optional": true, 693 | "peer": true, 694 | "funding": { 695 | "url": "https://ko-fi.com/killymxi" 696 | } 697 | }, 698 | "node_modules/lie": { 699 | "version": "3.3.0", 700 | "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", 701 | "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", 702 | "optional": true, 703 | "peer": true, 704 | "dependencies": { 705 | "immediate": "~3.0.5" 706 | } 707 | }, 708 | "node_modules/lodash": { 709 | "version": "4.17.21", 710 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", 711 | "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", 712 | "optional": true, 713 | "peer": true 714 | }, 715 | "node_modules/lop": { 716 | "version": "0.4.1", 717 | "resolved": "https://registry.npmjs.org/lop/-/lop-0.4.1.tgz", 718 | "integrity": "sha512-9xyho9why2A2tzm5aIcMWKvzqKsnxrf9B5I+8O30olh6lQU8PH978LqZoI4++37RBgS1Em5i54v1TFs/3wnmXQ==", 719 | "optional": true, 720 | "peer": true, 721 | "dependencies": { 722 | "duck": "^0.1.12", 723 | "option": "~0.2.1", 724 | "underscore": "^1.13.1" 725 | } 726 | }, 727 | "node_modules/mammoth": { 728 | "version": "1.5.1", 729 | "resolved": "https://registry.npmjs.org/mammoth/-/mammoth-1.5.1.tgz", 730 | "integrity": "sha512-7ZioZBf/1HjYrm1qZJOO+DD+rYxLvwrHS+HVOwW89hwIp+r6ZqJ/Eq2rXSS+8ezZ3/DuW6FUUp2Dfz6e7B2pBQ==", 731 | "optional": true, 732 | "peer": true, 733 | "dependencies": { 734 | "argparse": "~1.0.3", 735 | "bluebird": "~3.4.0", 736 | "dingbat-to-unicode": "^1.0.1", 737 | "jszip": "^3.7.1", 738 | "lop": "^0.4.1", 739 | "path-is-absolute": "^1.0.0", 740 | "sax": "~1.1.1", 741 | "underscore": "^1.13.1", 742 | "xmlbuilder": "^10.0.0" 743 | }, 744 | "bin": { 745 | "mammoth": "bin/mammoth" 746 | } 747 | }, 748 | "node_modules/mime-db": { 749 | "version": "1.52.0", 750 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", 751 | "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", 752 | "engines": { 753 | "node": ">= 0.6" 754 | } 755 | }, 756 | "node_modules/mime-types": { 757 | "version": "2.1.35", 758 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", 759 | "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", 760 | "dependencies": { 761 | "mime-db": "1.52.0" 762 | }, 763 | "engines": { 764 | "node": ">= 0.6" 765 | } 766 | }, 767 | "node_modules/ml-array-mean": { 768 | "version": "1.1.6", 769 | "resolved": "https://registry.npmjs.org/ml-array-mean/-/ml-array-mean-1.1.6.tgz", 770 | "integrity": "sha512-MIdf7Zc8HznwIisyiJGRH9tRigg3Yf4FldW8DxKxpCCv/g5CafTw0RRu51nojVEOXuCQC7DRVVu5c7XXO/5joQ==", 771 | "dependencies": { 772 | "ml-array-sum": "^1.1.6" 773 | } 774 | }, 775 | "node_modules/ml-array-sum": { 776 | "version": "1.1.6", 777 | "resolved": "https://registry.npmjs.org/ml-array-sum/-/ml-array-sum-1.1.6.tgz", 778 | "integrity": "sha512-29mAh2GwH7ZmiRnup4UyibQZB9+ZLyMShvt4cH4eTK+cL2oEMIZFnSyB3SS8MlsTh6q/w/yh48KmqLxmovN4Dw==", 779 | "dependencies": { 780 | "is-any-array": "^2.0.0" 781 | } 782 | }, 783 | "node_modules/ml-distance": { 784 | "version": "4.0.0", 785 | "resolved": "https://registry.npmjs.org/ml-distance/-/ml-distance-4.0.0.tgz", 786 | "integrity": "sha512-zj7+UGZpHk3uL7n79XTfGNUjIGnhLn8xVvrxYvBHvXFxo3jq1q+/UjP311hZxnLVhbxbXCjUniThX8gozjacYA==", 787 | "dependencies": { 788 | "ml-array-mean": "^1.1.6", 789 | "ml-distance-euclidean": "^2.0.0", 790 | "ml-tree-similarity": "^1.0.0" 791 | } 792 | }, 793 | "node_modules/ml-distance-euclidean": { 794 | "version": "2.0.0", 795 | "resolved": "https://registry.npmjs.org/ml-distance-euclidean/-/ml-distance-euclidean-2.0.0.tgz", 796 | "integrity": "sha512-yC9/2o8QF0A3m/0IXqCTXCzz2pNEzvmcE/9HFKOZGnTjatvBbsn4lWYJkxENkA4Ug2fnYl7PXQxnPi21sgMy/Q==" 797 | }, 798 | "node_modules/ml-tree-similarity": { 799 | "version": "1.0.0", 800 | "resolved": "https://registry.npmjs.org/ml-tree-similarity/-/ml-tree-similarity-1.0.0.tgz", 801 | "integrity": "sha512-XJUyYqjSuUQkNQHMscr6tcjldsOoAekxADTplt40QKfwW6nd++1wHWV9AArl0Zvw/TIHgNaZZNvr8QGvE8wLRg==", 802 | "dependencies": { 803 | "binary-search": "^1.3.5", 804 | "num-sort": "^2.0.0" 805 | } 806 | }, 807 | "node_modules/ms": { 808 | "version": "2.1.3", 809 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", 810 | "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" 811 | }, 812 | "node_modules/node-addon-api": { 813 | "version": "6.1.0", 814 | "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", 815 | "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==" 816 | }, 817 | "node_modules/node-ensure": { 818 | "version": "0.0.0", 819 | "resolved": "https://registry.npmjs.org/node-ensure/-/node-ensure-0.0.0.tgz", 820 | "integrity": "sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==" 821 | }, 822 | "node_modules/node-fetch": { 823 | "version": "2.6.7", 824 | "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", 825 | "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", 826 | "dependencies": { 827 | "whatwg-url": "^5.0.0" 828 | }, 829 | "engines": { 830 | "node": "4.x || >=6.0.0" 831 | }, 832 | "peerDependencies": { 833 | "encoding": "^0.1.0" 834 | }, 835 | "peerDependenciesMeta": { 836 | "encoding": { 837 | "optional": true 838 | } 839 | } 840 | }, 841 | "node_modules/num-sort": { 842 | "version": "2.1.0", 843 | "resolved": "https://registry.npmjs.org/num-sort/-/num-sort-2.1.0.tgz", 844 | "integrity": "sha512-1MQz1Ed8z2yckoBeSfkQHHO9K1yDRxxtotKSJ9yvcTUUxSvfvzEq5GwBrjjHEpMlq/k5gvXdmJ1SbYxWtpNoVg==", 845 | "engines": { 846 | "node": ">=8" 847 | }, 848 | "funding": { 849 | "url": "https://github.com/sponsors/sindresorhus" 850 | } 851 | }, 852 | "node_modules/object-hash": { 853 | "version": "3.0.0", 854 | "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", 855 | "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", 856 | "engines": { 857 | "node": ">= 6" 858 | } 859 | }, 860 | "node_modules/openai": { 861 | "version": "3.2.1", 862 | "resolved": "https://registry.npmjs.org/openai/-/openai-3.2.1.tgz", 863 | "integrity": "sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==", 864 | "dependencies": { 865 | "axios": "^0.26.0", 866 | "form-data": "^4.0.0" 867 | } 868 | }, 869 | "node_modules/option": { 870 | "version": "0.2.4", 871 | "resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz", 872 | "integrity": "sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==", 873 | "optional": true, 874 | "peer": true 875 | }, 876 | "node_modules/p-finally": { 877 | "version": "1.0.0", 878 | "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", 879 | "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", 880 | "engines": { 881 | "node": ">=4" 882 | } 883 | }, 884 | "node_modules/p-queue": { 885 | "version": "6.6.2", 886 | "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", 887 | "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", 888 | "dependencies": { 889 | "eventemitter3": "^4.0.4", 890 | "p-timeout": "^3.2.0" 891 | }, 892 | "engines": { 893 | "node": ">=8" 894 | }, 895 | "funding": { 896 | "url": "https://github.com/sponsors/sindresorhus" 897 | } 898 | }, 899 | "node_modules/p-retry": { 900 | "version": "4.6.2", 901 | "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", 902 | "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", 903 | "dependencies": { 904 | "@types/retry": "0.12.0", 905 | "retry": "^0.13.1" 906 | }, 907 | "engines": { 908 | "node": ">=8" 909 | } 910 | }, 911 | "node_modules/p-timeout": { 912 | "version": "3.2.0", 913 | "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", 914 | "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", 915 | "dependencies": { 916 | "p-finally": "^1.0.0" 917 | }, 918 | "engines": { 919 | "node": ">=8" 920 | } 921 | }, 922 | "node_modules/pako": { 923 | "version": "1.0.11", 924 | "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", 925 | "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", 926 | "optional": true, 927 | "peer": true 928 | }, 929 | "node_modules/parseley": { 930 | "version": "0.12.1", 931 | "resolved": "https://registry.npmjs.org/parseley/-/parseley-0.12.1.tgz", 932 | "integrity": "sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==", 933 | "optional": true, 934 | "peer": true, 935 | "dependencies": { 936 | "leac": "^0.6.0", 937 | "peberminta": "^0.9.0" 938 | }, 939 | "funding": { 940 | "url": "https://ko-fi.com/killymxi" 941 | } 942 | }, 943 | "node_modules/path-is-absolute": { 944 | "version": "1.0.1", 945 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 946 | "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", 947 | "optional": true, 948 | "peer": true, 949 | "engines": { 950 | "node": ">=0.10.0" 951 | } 952 | }, 953 | "node_modules/pdf-parse": { 954 | "version": "1.1.1", 955 | "resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-1.1.1.tgz", 956 | "integrity": "sha512-v6ZJ/efsBpGrGGknjtq9J/oC8tZWq0KWL5vQrk2GlzLEQPUDB1ex+13Rmidl1neNN358Jn9EHZw5y07FFtaC7A==", 957 | "dependencies": { 958 | "debug": "^3.1.0", 959 | "node-ensure": "^0.0.0" 960 | }, 961 | "engines": { 962 | "node": ">=6.8.1" 963 | } 964 | }, 965 | "node_modules/peberminta": { 966 | "version": "0.9.0", 967 | "resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.9.0.tgz", 968 | "integrity": "sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==", 969 | "optional": true, 970 | "peer": true, 971 | "funding": { 972 | "url": "https://ko-fi.com/killymxi" 973 | } 974 | }, 975 | "node_modules/process-nextick-args": { 976 | "version": "2.0.1", 977 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", 978 | "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", 979 | "optional": true, 980 | "peer": true 981 | }, 982 | "node_modules/readable-stream": { 983 | "version": "2.3.8", 984 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", 985 | "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", 986 | "optional": true, 987 | "peer": true, 988 | "dependencies": { 989 | "core-util-is": "~1.0.0", 990 | "inherits": "~2.0.3", 991 | "isarray": "~1.0.0", 992 | "process-nextick-args": "~2.0.0", 993 | "safe-buffer": "~5.1.1", 994 | "string_decoder": "~1.1.1", 995 | "util-deprecate": "~1.0.1" 996 | } 997 | }, 998 | "node_modules/retry": { 999 | "version": "0.13.1", 1000 | "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", 1001 | "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", 1002 | "engines": { 1003 | "node": ">= 4" 1004 | } 1005 | }, 1006 | "node_modules/rw": { 1007 | "version": "1.3.3", 1008 | "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", 1009 | "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", 1010 | "optional": true, 1011 | "peer": true 1012 | }, 1013 | "node_modules/safe-buffer": { 1014 | "version": "5.1.2", 1015 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", 1016 | "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", 1017 | "optional": true, 1018 | "peer": true 1019 | }, 1020 | "node_modules/safer-buffer": { 1021 | "version": "2.1.2", 1022 | "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", 1023 | "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", 1024 | "optional": true, 1025 | "peer": true 1026 | }, 1027 | "node_modules/sax": { 1028 | "version": "1.1.6", 1029 | "resolved": "https://registry.npmjs.org/sax/-/sax-1.1.6.tgz", 1030 | "integrity": "sha512-8zci48uUQyfqynGDSkUMD7FCJB96hwLnlZOXlgs1l3TX+LW27t3psSWKUxC0fxVgA86i8tL4NwGcY1h/6t3ESg==", 1031 | "optional": true, 1032 | "peer": true 1033 | }, 1034 | "node_modules/selderee": { 1035 | "version": "0.11.0", 1036 | "resolved": "https://registry.npmjs.org/selderee/-/selderee-0.11.0.tgz", 1037 | "integrity": "sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==", 1038 | "optional": true, 1039 | "peer": true, 1040 | "dependencies": { 1041 | "parseley": "^0.12.0" 1042 | }, 1043 | "funding": { 1044 | "url": "https://ko-fi.com/killymxi" 1045 | } 1046 | }, 1047 | "node_modules/setimmediate": { 1048 | "version": "1.0.5", 1049 | "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", 1050 | "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", 1051 | "optional": true, 1052 | "peer": true 1053 | }, 1054 | "node_modules/sprintf-js": { 1055 | "version": "1.0.3", 1056 | "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", 1057 | "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", 1058 | "optional": true, 1059 | "peer": true 1060 | }, 1061 | "node_modules/string_decoder": { 1062 | "version": "1.1.1", 1063 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", 1064 | "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", 1065 | "optional": true, 1066 | "peer": true, 1067 | "dependencies": { 1068 | "safe-buffer": "~5.1.0" 1069 | } 1070 | }, 1071 | "node_modules/tr46": { 1072 | "version": "0.0.3", 1073 | "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", 1074 | "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" 1075 | }, 1076 | "node_modules/ts-toolbelt": { 1077 | "version": "9.6.0", 1078 | "resolved": "https://registry.npmjs.org/ts-toolbelt/-/ts-toolbelt-9.6.0.tgz", 1079 | "integrity": "sha512-nsZd8ZeNUzukXPlJmTBwUAuABDe/9qtVDelJeT/qW0ow3ZS3BsQJtNkan1802aM9Uf68/Y8ljw86Hu0h5IUW3w==", 1080 | "optional": true, 1081 | "peer": true 1082 | }, 1083 | "node_modules/ts-type": { 1084 | "version": "3.0.1", 1085 | "resolved": "https://registry.npmjs.org/ts-type/-/ts-type-3.0.1.tgz", 1086 | "integrity": "sha512-cleRydCkBGBFQ4KAvLH0ARIkciduS745prkGVVxPGvcRGhMMoSJUB7gNR1ByKhFTEYrYRg2CsMRGYnqp+6op+g==", 1087 | "optional": true, 1088 | "peer": true, 1089 | "dependencies": { 1090 | "@types/node": "*", 1091 | "tslib": ">=2", 1092 | "typedarray-dts": "^1.0.0" 1093 | }, 1094 | "peerDependencies": { 1095 | "ts-toolbelt": "^9.6.0" 1096 | } 1097 | }, 1098 | "node_modules/tslib": { 1099 | "version": "2.5.0", 1100 | "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", 1101 | "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==", 1102 | "optional": true, 1103 | "peer": true 1104 | }, 1105 | "node_modules/type-detect": { 1106 | "version": "4.0.8", 1107 | "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", 1108 | "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", 1109 | "optional": true, 1110 | "peer": true, 1111 | "engines": { 1112 | "node": ">=4" 1113 | } 1114 | }, 1115 | "node_modules/typedarray-dts": { 1116 | "version": "1.0.0", 1117 | "resolved": "https://registry.npmjs.org/typedarray-dts/-/typedarray-dts-1.0.0.tgz", 1118 | "integrity": "sha512-Ka0DBegjuV9IPYFT1h0Qqk5U4pccebNIJCGl8C5uU7xtOs+jpJvKGAY4fHGK25hTmXZOEUl9Cnsg5cS6K/b5DA==", 1119 | "optional": true, 1120 | "peer": true 1121 | }, 1122 | "node_modules/underscore": { 1123 | "version": "1.13.6", 1124 | "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", 1125 | "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==", 1126 | "optional": true, 1127 | "peer": true 1128 | }, 1129 | "node_modules/util-deprecate": { 1130 | "version": "1.0.2", 1131 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 1132 | "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", 1133 | "optional": true, 1134 | "peer": true 1135 | }, 1136 | "node_modules/uuid": { 1137 | "version": "9.0.0", 1138 | "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", 1139 | "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", 1140 | "bin": { 1141 | "uuid": "dist/bin/uuid" 1142 | } 1143 | }, 1144 | "node_modules/webidl-conversions": { 1145 | "version": "3.0.1", 1146 | "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", 1147 | "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" 1148 | }, 1149 | "node_modules/whatwg-url": { 1150 | "version": "5.0.0", 1151 | "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", 1152 | "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", 1153 | "dependencies": { 1154 | "tr46": "~0.0.3", 1155 | "webidl-conversions": "^3.0.0" 1156 | } 1157 | }, 1158 | "node_modules/xml2js": { 1159 | "version": "0.4.23", 1160 | "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", 1161 | "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", 1162 | "optional": true, 1163 | "peer": true, 1164 | "dependencies": { 1165 | "sax": ">=0.6.0", 1166 | "xmlbuilder": "~11.0.0" 1167 | }, 1168 | "engines": { 1169 | "node": ">=4.0.0" 1170 | } 1171 | }, 1172 | "node_modules/xml2js/node_modules/xmlbuilder": { 1173 | "version": "11.0.1", 1174 | "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", 1175 | "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", 1176 | "optional": true, 1177 | "peer": true, 1178 | "engines": { 1179 | "node": ">=4.0" 1180 | } 1181 | }, 1182 | "node_modules/xmlbuilder": { 1183 | "version": "10.1.1", 1184 | "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-10.1.1.tgz", 1185 | "integrity": "sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==", 1186 | "optional": true, 1187 | "peer": true, 1188 | "engines": { 1189 | "node": ">=4.0" 1190 | } 1191 | }, 1192 | "node_modules/yaml": { 1193 | "version": "2.2.2", 1194 | "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.2.tgz", 1195 | "integrity": "sha512-CBKFWExMn46Foo4cldiChEzn7S7SRV+wqiluAb6xmueD/fGyRHIhX8m14vVGgeFWjN540nKCNVj6P21eQjgTuA==", 1196 | "engines": { 1197 | "node": ">= 14" 1198 | } 1199 | }, 1200 | "node_modules/zod": { 1201 | "version": "3.21.4", 1202 | "resolved": "https://registry.npmjs.org/zod/-/zod-3.21.4.tgz", 1203 | "integrity": "sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==", 1204 | "funding": { 1205 | "url": "https://github.com/sponsors/colinhacks" 1206 | } 1207 | }, 1208 | "node_modules/zod-to-json-schema": { 1209 | "version": "3.21.0", 1210 | "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.21.0.tgz", 1211 | "integrity": "sha512-+KyFCzqKwE6CxMSZxEUBaGmdXzB09BoFebO+xef/ISE4cTfReQlyThYbS8aqd3uWkdt9fz5BGHsY0CbY+Ra9oA==", 1212 | "peerDependencies": { 1213 | "zod": "^3.21.4" 1214 | } 1215 | } 1216 | } 1217 | } 1218 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "langchain-document-loader", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [], 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "@dqbd/tiktoken": "^1.0.7", 14 | "dotenv": "^16.0.3", 15 | "hnswlib-node": "^1.4.2", 16 | "langchain": "^0.0.67" 17 | }, 18 | "type": "module" 19 | } 20 | --------------------------------------------------------------------------------