├── .env.example ├── .eslintignore ├── .eslintrc.cjs ├── .github └── FUNDING.yml ├── .gitignore ├── .vscode └── launch.json ├── LICENSE ├── README.md ├── chat_logs └── .gitignore ├── docs └── example.md ├── package.json ├── prettier.config.cjs ├── src ├── chatLogger.ts ├── commands.ts ├── commands │ ├── addDocumentCommand.ts │ ├── addURLCommand.ts │ ├── addYouTubeCommand.ts │ ├── command.ts │ ├── helpCommand.ts │ ├── listContextStoresCommand.ts │ ├── quitCommand.ts │ ├── resetChatCommand.ts │ ├── setContextConfigCommand.ts │ ├── setMemoryConfigCommand.ts │ ├── switchContextStoreCommand.ts │ └── toggleWindowBufferMemoryCommand.ts ├── config │ └── index.ts ├── global.d.ts ├── index.ts ├── lib │ ├── contextManager.ts │ ├── crawler.ts │ ├── memoryManager.ts │ └── vectorStoreUtils.ts ├── prompt.txt ├── updateReadme.ts └── utils │ ├── createDirectory.ts │ ├── getDirectoryFiles.ts │ ├── getDirectoryListWithDetails.ts │ ├── resolveURL.ts │ └── sanitizeInput.ts └── tsconfig.json /.env.example: -------------------------------------------------------------------------------- 1 | OPENAI_API_KEY=sk-xxxxxxxxxx # your OpenAI API key 2 | VECTOR_STORE_BASE_DIR=db # relative to the project root, used to list available vector stores 3 | VECTOR_STORE_DIR=db/default # relative to the project root, used to set the default vector store 4 | DOCS_DIR=docs # relative to the project root, used to store documents that will be loaded into the vector store on first run 5 | MEMORY_VECTOR_STORE_DIR=memory # relative to the project root, used to store the memory vector store 6 | MODEL=gpt-4 # the OpenAI model to use, one of gpt-3.5-turbo or gpt-4 7 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | .eslintrc.cjs 2 | src/agentTest.ts 3 | -------------------------------------------------------------------------------- /.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: [ 3 | 'airbnb-base', 4 | 'airbnb-typescript/base', 5 | 'eslint:recommended', 6 | 'prettier', 7 | 'plugin:@typescript-eslint/recommended' 8 | ], 9 | parserOptions: { 10 | project: './tsconfig.json', 11 | }, 12 | settings: { 13 | 'import/parsers': { 14 | '@typescript-eslint/parser': ['.ts', '.tsx'] 15 | }, 16 | "import/resolver": { 17 | "typescript": { 18 | "alwaysTryTypes": true, // always try to resolve types under `@types` directory even it doesn't contain any source code, like `@types/unist` 19 | project: './tsconfig.json' 20 | } 21 | } 22 | }, 23 | parserOptions: { 24 | project: './tsconfig.json', 25 | ecmaVersion: 'latest', 26 | parser: '@typescript-eslint/parser', 27 | sourceType: 'module' 28 | }, 29 | plugins: ['@typescript-eslint', 'import'], 30 | rules: { 31 | 'import/no-unresolved': 'error' 32 | } 33 | }; 34 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [gmickel] 4 | ko_fi: gmickel 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | pnpm-lock.yaml 4 | .env 5 | logs 6 | dist 7 | docs/* 8 | !docs/example.md 9 | db/* 10 | memory/* 11 | chat_logs/* 12 | tmp/* 13 | prompttest.txt 14 | agentTest.ts 15 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Launch Program", 11 | "sourceMaps": true, 12 | "skipFiles": [ 13 | "/**" 14 | ], 15 | "program": "${workspaceFolder}/src/index.ts", 16 | "preLaunchTask": "tsc: build - tsconfig.json", 17 | "outFiles": [ 18 | "${workspaceFolder}/dist/**/*.js" 19 | ] 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Gordon Mickel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Memory Bot 2 | 3 | Memory Bot is designed to support context-aware requests to the OpenAI API, providing unlimited context and chat history for more cost-efficient and accurate content generation. 4 | 5 | Memory Bot can be used for a variety of purposes, including context-aware content generation, such as marketing materials, website copy, and social media content. It can also be used for document-based question answering (DBQA), allowing you to add an unlimited amount of documents to the chatbot and ask questions such as “What was the xxx's revenue in 2021?”. This feature enables more efficient and accurate retrieval of information from your own content. 6 | 7 | Memory Bot supports adding various types of context, such as documents, web pages and youtube videos. 8 | 9 | This project was originally featured on my blog [ByteSizedBrainwaves - Building a GPT-4 Powered Chatbot with Node.js: Unlimited Context and Chat History in Under 100 Lines of Code 10 | ](https://medium.com/byte-sized-brainwaves/unlimited-chatbot-context-and-chat-history-in-under-100-lines-of-code-with-langchain-and-node-js-1190fcc20708). 11 | 12 | ## Features 13 | 14 | ### Configurable Context Retrieval 15 | 16 | Memory Bot enables users to save costs by sending only relevant context in the prompt. It allows loading of various types of documents, such as .txt, .md, .json, .pdf, .epub, .csv into a vector store index. This is useful for context-aware content generation or content retrieval for context-aware Q&A chatbots. Users can configure the number of relevant documents to retrieve. 17 | 18 | ### Configurable Long-Term Memory Retrieval 19 | 20 | Memory Bot saves an unlimited amount of conversation history to a separate memory vector store index. This feature allows users to save costs by sending only relevant context in the prompt. Users can configure the number of relevant conversation parts to retrieve. 21 | 22 | ### Configurable Short-Term Memory 23 | 24 | Memory Bot uses a short-term converstaion buffer window memory so you can refine its most recent outputs. This feature can be activated or deactivated. 25 | 26 | ### Seamlessly switch between Contexts 27 | 28 | Memory Bot facilitates the management of concurrent projects by allowing you to create distinct Context Vector Stores for each project. This functionality enables you to segregate and organize your work efficiently. Switching between different project contexts is as simple as executing a single command. For detailed information on how to change your context store, please refer to the "/change-context-store" command under the [Commands](#commands) section. 29 | 30 | ### Daily Chat Logs 31 | 32 | Memory Bot automatically saves all chats in daily log files in the `chat_logs` directory. This feature allows users to keep track of all conversations and review them later if needed. The log files are saved in a human-readable format and can be easily accessed and analyzed. 33 | 34 | ## Prerequisites 35 | 36 | You will need an OpenAI Account and API key: 37 | 38 | - Sign up for an OpenAI account here if you don't already have one: [OpenAI signup page](https://platform.openai.com/signup) 39 | - After registering and logging in, create an API key here: [API keys](https://platform.openai.com/account/api-keys) 40 | 41 | ## Installation 42 | 43 | 1. Clone the repository or download the source code: 44 | 45 | ``` 46 | git clone git@github.com:gmickel/memorybot.git 47 | ``` 48 | 49 | 2. Navigate to the project directory: 50 | 51 | ``` 52 | cd memorybot 53 | ``` 54 | 55 | 3. Install the required dependencies: 56 | 57 | ``` 58 | npm install 59 | ``` 60 | 61 | Please note: On Windows, you might need to install Visual Studio first in order to properly build the hnswlib-node package. Alternatively you can use [Windows Subsystem for Linux](https://learn.microsoft.com/en-us/windows/wsl/). 62 | 63 | 4. Set up environment variables by creating a `.env` file in the project root directory with the necessary API keys and configuration options. You can use the provided `.env.example` file as a template. 64 | 65 | - **Important:** If you do not have access to GPT-4 yet, set the MODEL env variable to `gpt-3.5-turbo`. You can request access to GPT-4 [here](https://openai.com/waitlist/gpt-4-api). 66 | 67 | 5. Add some context to the chatbot: 68 | 69 | **Make sure you understand that any content you add will be sent to the OpenAI API** - see [Considerations](#considerations) 70 | 71 | ### Adding documents as context 72 | 73 | To populate the context vector index on startup, replace [example.md](docs/example.md) in the _docs_ folder with the context you want to add before starting the bot. 74 | 75 | You can also add new context at runtime, see [Commands](#commands) for more details. 76 | 77 | #### Supported document file types 78 | 79 | **.md**, **.txt**, **.json**, **.pdf**, **.docx**, **.epub**, **.csv** 80 | 81 | ### Crawling Webpages as context 82 | 83 | Content from Webpages can be added to the bot's context at runtime. see [Commands](#commands) for more details. 84 | 85 | ### Adding Youtube Video transcripts as context 86 | 87 | Content from Youtube Videos can be added to the bot's context at runtime. see [Commands](#commands) for more details. 88 | 89 | 6. Run the chatbot: 90 | 91 | ``` 92 | npm start 93 | ``` 94 | 95 | ## Usage 96 | 97 | ### Tweaking's memorybot's output by tuning memorybot's context and memory at runtime 98 | 99 | By tuning the number of relevant documents returned from memorybot's vector store indexes we can tweak its behaviour. 100 | 101 | The default values of 6 relevant context documents and 4 relevant memory documents offer a balance between searching for context in the provided context and our conversation history. 102 | 103 | Be aware that higher values equate to sending larger (and more expensive) requests to the OpenAI API. 104 | 105 | #### Examples 106 | 107 | You are generating content or asking questions concerning a lengthy E-Book. It might make sense to set the number of context documents to retrieve to a high value such as 10 using `/cc 10` 108 | 109 | You can completely deactivate either of the vector stores by setting `/cc 0` and `/mc 0` respectively. 110 | 111 | You can also completely disable the bot's window buffer memory (its short-term transient conversation memory) by using the `/wm` command. 112 | 113 | See the [Commands](#commands) section for more information. 114 | 115 | ### Changing the prompt 116 | 117 | - Change the [system prompt](src/prompt.txt) to whatever you need and restart the bot. 118 | 119 | ### Resetting the chat history/context 120 | 121 | Both the context and chat history are currently persisted and reused on every run. To reset the context simply delete the contents of the _db_ folder. To start a new conversation, resetting both the bots short-term transient and long-term vector store index memory, type the command `/reset`, see the [Commands](#commands) section for more information. 122 | 123 | Alternatively, change the variables in .env to point to different folders. 124 | 125 | Restart the bot after these steps. 126 | 127 | ### Running 128 | 129 | After starting the chatbot, simply type your questions or messages and press Enter. The chatbot will respond with context-aware answers based on the conversation history and any provided context. 130 | 131 | ### Commands 132 | 133 | 134 | - `/add-docs` (/docs) - Adds new documents from your configured docs directory to the context vector store. 135 | 136 | Usage: /add-docs example.txt example.md 137 | 138 | Supports the following file types: .txt, .md, .pdf, .docx, .csv, .epub 139 | - `/add-url` (/url) - Scrapes the content from a url and adds it to the context vector store. 140 | 141 | Arguments: `url`, `selector to extract` (Default: body), `Maximum number of links to follow` (Default: 20), `Ignore pages with less than n characters` (Default: 200) 142 | 143 | Example: /add-url https://dociq.io main 10 500 144 | 145 | This operation may try to generate a large number of embeddings depending on the structure of the web pages and may lead to rate-limiting. 146 | 147 | To avoid this, you can try to target a specific selector such as `.main` 148 | - `/add-youtube` (/yt) - Adds the transcript from a youtube video and adds it to the context vector store. 149 | 150 | Arguments: `youtube url` or `youtube videoid` 151 | 152 | Example: /add-url https://www.youtube.com/watch?v=VMj-3S1tku0 153 | - `/help` (/h, /?) - Show the list of available commands 154 | - `/list-context-stores` (/lcs) - Lists all available context vector stores and their details. 155 | 156 | - `/quit` (/q) - Terminates the script 157 | - `/reset` - Resets the chat and starts a new conversation - This clears the memory vector store and the buffer window memory. 158 | - `/context-config` (/cc) - Sets the number of relevant documents to return from the context vector store. 159 | 160 | Arguments: `number of documents` (Default: 4) 161 | 162 | Example: `/context-config 10` 163 | - `/memory-config` (/mc) - Sets the number of relevant documents to return from the memory vector store. 164 | 165 | Arguments: `number of documents` (Default: 4) 166 | 167 | Example: /memory-config 10 168 | - `/change-context-store` (/ccs) - Loads an existing or creates a new empty context vector store as a subdirectory of the db directory. 169 | 170 | Arguments: `subdirectory` 171 | 172 | Example: /change-context-store newcontext 173 | - `/toggle-window-memory` (/wm) - Toggles the window buffer memory (MemoryBot's short-term transient memory) on or off. 174 | 175 | 176 | ## Documentation 177 | 178 | For a detailed guide on building and customizing the Bot, please refer to the blog post on [ByteSizedBrainwaves](https://bytesizedbrainwaves.hashnode.dev/series/ai-conversations). 179 | 180 | ## Considerations 181 | 182 | Consider the implications of sharing any sensitive content with third parties. Any context or history that is generated is sent to OpenAI to create the necessary embeddings. Always ensure that you're adhering to your organization's security policies and best practices to protect your valuable assets. 183 | 184 | ## License 185 | 186 | This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details. 187 | 188 | ## Acknowledgements 189 | 190 | - OpenAI for their powerful language models 191 | - Langchain for seamless integration with language models 192 | - HNSWLib for efficient vector search and storage 193 | - Node.js and the open-source community for providing useful libraries and tools 194 | -------------------------------------------------------------------------------- /chat_logs/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /docs/example.md: -------------------------------------------------------------------------------- 1 | # ACME Ltd 2 | 3 | ACME Ltd was founded in 2023 for the sole purpose of producing MacGuffins. Since its inception, the company has become a global leader in crafting the most obscure and whimsical gadgets known to humankind. Their flagship products include: 4 | 5 | ## 1. The Whatchamacallit 3000 6 | A device so versatile, it does everything and nothing at the same time! With the Whatchamacallit 3000, you'll never need another gadget again... or will you? 7 | 8 | ## 2. The Thingamabob Deluxe 9 | The Thingamabob Deluxe is the ultimate tool for all your doohickey needs. It can tighten, loosen, twist, untwist, and even unexist screws, bolts, and other fasteners. It's the only tool you'll ever need, except when you don't. 10 | 11 | ## 3. The Gizmotron 12 | Introducing the Gizmotron, the world's first self-replicating gizmo. It's like a Swiss Army knife that can clone itself, making it the perfect companion for any situation. Just don't leave it unattended, or you might find your house filled with an army of Gizmotrons! 13 | 14 | ## 4. The Contraptionizer 15 | Ever wanted to transform your ordinary household items into extraordinary contraptions? With the Contraptionizer, you can turn your toaster into a time machine, or your vacuum cleaner into a teleportation device! The possibilities are endless, and often result in hilarious consequences. 16 | 17 | At ACME Ltd, we pride ourselves on our ability to create the most nonsensical and amusing products on the market. Our mission is to bring laughter and confusion to the world, one MacGuffin at a time. 18 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "memory-chatterbox", 3 | "version": "0.0.1", 4 | "description": "", 5 | "type": "module", 6 | "main": "./dist/index.js", 7 | "private": "true", 8 | "engines": { 9 | "node": ">=18" 10 | }, 11 | "scripts": { 12 | "build": "tsc --declaration --outDir dist/", 13 | "dev": "tsx -r dotenv/config src/index.ts --inspect", 14 | "start": "npm run build && node -r dotenv/config dist/index.js", 15 | "lint": "eslint src", 16 | "lint:fix": "npm run lint --fix", 17 | "format": "prettier --write \"**/*.ts\"", 18 | "format:check": "prettier --list-different \"**/*.ts\"", 19 | "update-readme": "tsx -r dotenv/config src/updateReadme.ts" 20 | }, 21 | "repository": { 22 | "type": "git", 23 | "url": "git+https://" 24 | }, 25 | "keywords": [ 26 | "ai-chatbot", 27 | "nodejs", 28 | "context-aware", 29 | "chatgpt", 30 | "gpt-4" 31 | ], 32 | "author": "Gordon Mickel (https://bytesizedbrainwaves.substack.com/)", 33 | "license": "MIT", 34 | "homepage": "https://bytesizedbrainwaves.substack.com/", 35 | "devDependencies": { 36 | "@tsconfig/recommended": "^1.0.3", 37 | "@types/common-tags": "^1.8.2", 38 | "@types/crawler": "^1.2.3", 39 | "@types/fs-extra": "^11.0.2", 40 | "@types/node": "^20.8.3", 41 | "@types/turndown": "^5.0.2", 42 | "@typescript-eslint/eslint-plugin": "^6.7.4", 43 | "@typescript-eslint/parser": "^6.7.4", 44 | "eslint": "^8.51.0", 45 | "eslint-config-airbnb-base": "^15.0.0", 46 | "eslint-config-airbnb-typescript": "^17.1.0", 47 | "eslint-config-prettier": "^9.0.0", 48 | "eslint-import-resolver-typescript": "^3.6.1", 49 | "eslint-plugin-import": "^2.28.1", 50 | "eslint-plugin-prettier": "^5.0.0", 51 | "prettier": "^3.0.3", 52 | "tsx": "^3.13.0", 53 | "typescript": "^5.2.2" 54 | }, 55 | "dependencies": { 56 | "chalk": "^5.3.0", 57 | "cheerio": "^1.0.0-rc.12", 58 | "common-tags": "^1.8.2", 59 | "crawler": "^1.4.0", 60 | "d3-dsv": "2.0.0", 61 | "dotenv": "^16.3.1", 62 | "epub2": "^3.0.2", 63 | "fs-extra": "^11.1.1", 64 | "hnswlib-node": "1.4.2", 65 | "html-to-text": "^9.0.5", 66 | "langchain": "^0.0.163", 67 | "mammoth": "^1.6.0", 68 | "ora": "^7.0.1", 69 | "pdf-parse": "^1.1.1", 70 | "turndown": "^7.1.2", 71 | "youtube-transcript": "^1.0.6" 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /prettier.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | printWidth: 120, 3 | tabWidth: 2, 4 | useTabs: false, 5 | semi: true, 6 | singleQuote: true, 7 | trailingComma: 'es5', 8 | bracketSpacing: true, 9 | bracketSameLine: false, 10 | arrowParens: 'always', 11 | proseWrap: 'preserve', 12 | htmlWhitespaceSensitivity: 'css', 13 | endOfLine: 'lf', 14 | vueIndentScriptAndStyle: false, 15 | jsxSingleQuote: true, 16 | }; 17 | -------------------------------------------------------------------------------- /src/chatLogger.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs-extra'; 2 | import path from 'path'; 3 | 4 | interface ChatHistory { 5 | timestamp: string; 6 | question: string; 7 | answer: string; 8 | } 9 | 10 | const ensureLogDirectory = (logDirectory: string): void => { 11 | fs.ensureDirSync(logDirectory); 12 | }; 13 | 14 | const getLogFilename = (): string => { 15 | const currentDate = new Date(); 16 | const year = currentDate.getFullYear(); 17 | const month = String(currentDate.getMonth() + 1).padStart(2, '0'); 18 | const day = String(currentDate.getDate()).padStart(2, '0'); 19 | 20 | return `${year}-${month}-${day}.json`; 21 | }; 22 | 23 | const logChat = async (logDirectory: string, question: string, answer: string): Promise => { 24 | const timestamp = new Date().toISOString(); 25 | const chatHistory: ChatHistory = { timestamp, question, answer }; 26 | const logFilename = getLogFilename(); 27 | const logFilePath = path.join(logDirectory, logFilename); 28 | 29 | ensureLogDirectory(logDirectory); 30 | 31 | if (!fs.existsSync(logFilePath)) { 32 | await fs.writeJson(logFilePath, [chatHistory]); 33 | } else { 34 | const chatHistoryArray = await fs.readJson(logFilePath); 35 | chatHistoryArray.push(chatHistory); 36 | await fs.writeJson(logFilePath, chatHistoryArray); 37 | } 38 | }; 39 | 40 | export default logChat; 41 | -------------------------------------------------------------------------------- /src/commands.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import changeContextStoreCommand from './commands/switchContextStoreCommand.js'; 3 | import helpCommand from './commands/helpCommand.js'; 4 | import quitCommand from './commands/quitCommand.js'; 5 | import resetChatCommand from './commands/resetChatCommand.js'; 6 | import addDocumentCommand from './commands/addDocumentCommand.js'; 7 | import addURLCommand from './commands/addURLCommand.js'; 8 | import addYouTubeCommand from './commands/addYouTubeCommand.js'; 9 | import setContextConfigCommand from './commands/setContextConfigCommand.js'; 10 | import setMemoryConfigCommand from './commands/setMemoryConfigCommand.js'; 11 | import toggleWindowBufferMemoryCommand from './commands/toggleWindowBufferMemoryCommand.js'; 12 | import listContextStoresCommand from './commands/listContextStoresCommand.js'; 13 | 14 | function createCommandHandler(): CommandHandler { 15 | const commands: Command[] = [ 16 | helpCommand, 17 | quitCommand, 18 | resetChatCommand, 19 | addDocumentCommand, 20 | addURLCommand, 21 | addYouTubeCommand, 22 | setContextConfigCommand, 23 | setMemoryConfigCommand, 24 | toggleWindowBufferMemoryCommand, 25 | listContextStoresCommand, 26 | changeContextStoreCommand, 27 | ]; 28 | 29 | function getCommands() { 30 | return commands; 31 | } 32 | 33 | const commandHandler: CommandHandler = { 34 | getCommands, 35 | async execute(commandName: string, args: string[], output: NodeJS.WriteStream) { 36 | const command = commands.find((cmd) => cmd.name === commandName || cmd.aliases.includes(commandName)); 37 | if (command) { 38 | await command.execute(args, output, commandHandler); 39 | } else { 40 | output.write(chalk.red('Unknown command. Type /help to see the list of available commands.\n')); 41 | } 42 | }, 43 | }; 44 | return commandHandler; 45 | } 46 | 47 | export default createCommandHandler; 48 | -------------------------------------------------------------------------------- /src/commands/addDocumentCommand.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import createCommand from './command.js'; 3 | import { addDocument } from '../lib/contextManager.js'; 4 | 5 | const addDocumentCommand = createCommand( 6 | 'add-docs', 7 | ['docs'], 8 | `Adds new documents from your configured docs directory to the context vector store.\n 9 | Usage: /add-docs example.txt example.md\n 10 | Supports the following file types: .txt, .md, .pdf, .docx, .csv, .epub`, 11 | async (args: string[], output: NodeJS.WriteStream) => { 12 | if (!args) { 13 | output.write(chalk.red('Invalid number of arguments. Usage: /add-docs example.txt example.md\n')); 14 | return; 15 | } 16 | await addDocument(args); 17 | } 18 | ); 19 | export default addDocumentCommand; 20 | -------------------------------------------------------------------------------- /src/commands/addURLCommand.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import createCommand from './command.js'; 3 | import { addURL } from '../lib/contextManager.js'; 4 | 5 | const addURLCommand = createCommand( 6 | 'add-url', 7 | ['url'], 8 | `Scrapes the content from a url and adds it to the context vector store.\n 9 | Arguments: \`url\`, \`selector to extract\` (Default: body), \`Maximum number of links to follow\` (Default: 20), \`Ignore pages with less than n characters\` (Default: 200)\n 10 | Example: /add-url https://dociq.io main 10 500\n 11 | This operation may try to generate a large number of embeddings depending on the structure of the web pages and may lead to rate-limiting.\n 12 | To avoid this, you can try to target a specific selector such as \`.main\``, 13 | async (args, output) => { 14 | if (!args || args.length > 4) { 15 | output.write( 16 | chalk.red( 17 | 'Invalid number of arguments. Usage: /add-url `url` `selector to extract` `Maximum number of links to follow` `Ignore pages with less than n characters`\n' 18 | ) 19 | ); 20 | return; 21 | } 22 | const url = args[0]; 23 | const selector = args[1]; 24 | const maxLinks = parseInt(args[2], 10) || 20; 25 | const minChars = parseInt(args[3], 10) || 200; 26 | await addURL(url, selector, maxLinks, minChars); 27 | } 28 | ); 29 | export default addURLCommand; 30 | -------------------------------------------------------------------------------- /src/commands/addYouTubeCommand.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import createCommand from './command.js'; 3 | import { addYouTube } from '../lib/contextManager.js'; 4 | 5 | const addYouTubeCommand = createCommand( 6 | 'add-youtube', 7 | ['yt'], 8 | `Adds the transcript from a youtube video and adds it to the context vector store.\n 9 | Arguments: \`youtube url\` or \`youtube videoid\`\n 10 | Example: /add-url https://www.youtube.com/watch?v=VMj-3S1tku0`, 11 | async (args, output) => { 12 | if (!args || args.length !== 1) { 13 | output.write(chalk.red('Invalid number of arguments. Usage: /add-url `youtube url` or `youtube videoid`\n')); 14 | return; 15 | } 16 | const URLOrVideoID = args[0]; 17 | await addYouTube(URLOrVideoID); 18 | } 19 | ); 20 | export default addYouTubeCommand; 21 | -------------------------------------------------------------------------------- /src/commands/command.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * The function creates a command object with a name, aliases, description, and an execute function 3 | * that returns a Promise. 4 | * @param {string} name - A string representing the name of the command. 5 | * @param {string[]} aliases - An array of alternative names that can be used to call the command. For 6 | * example, if the command is named "help", aliases could include "h" or "info". 7 | * @param {string} description - A brief description of what the command does. 8 | * @param execute - The `execute` parameter is a function that takes in three arguments: 9 | * @returns A `Command` object is being returned. 10 | */ 11 | function createCommand( 12 | name: string, 13 | aliases: string[], 14 | description: string, 15 | execute: (args: string[], output: NodeJS.WriteStream, commandHandler: CommandHandler) => Promise 16 | ): Command { 17 | return { name, aliases, description, execute }; 18 | } 19 | export default createCommand; 20 | -------------------------------------------------------------------------------- /src/commands/helpCommand.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import createCommand from './command.js'; 3 | 4 | const helpCommand = createCommand( 5 | 'help', 6 | ['h', '?'], 7 | 'Show the list of available commands', 8 | (_args, output, commandHandler) => 9 | new Promise((resolve) => { 10 | output.write(chalk.blue('Usage:\n')); 11 | output.write('Ask memorybot to write some marketing materials and press enter.\n'); 12 | output.write(chalk.blue('\nAvailable commands:\n')); 13 | commandHandler.getCommands().forEach((command) => { 14 | const aliases = command.aliases.length > 0 ? ` (/${command.aliases.join(', /')})` : ''; 15 | output.write(chalk.yellow(`/${command.name}${aliases}`)); 16 | output.write(` - ${command.description}`); 17 | output.write('\n'); 18 | }); 19 | resolve(); 20 | }) 21 | ); 22 | 23 | export default helpCommand; 24 | -------------------------------------------------------------------------------- /src/commands/listContextStoresCommand.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import createCommand from './command.js'; 3 | import { listContextStores } from '../lib/contextManager.js'; 4 | 5 | const listContextStoresCommand = createCommand( 6 | 'list-context-stores', 7 | ['lcs'], 8 | `Lists all available context vector stores and their details.\n`, 9 | async (args, output) => { 10 | if (!args || args.length > 0) { 11 | output.write(chalk.red('Invalid number of arguments. Usage: /list-context-stores\n')); 12 | return; 13 | } 14 | await listContextStores(); 15 | } 16 | ); 17 | export default listContextStoresCommand; 18 | -------------------------------------------------------------------------------- /src/commands/quitCommand.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import createCommand from './command.js'; 3 | 4 | const exitCommand = createCommand('quit', ['q'], 'Terminates the script', (_args, output) => { 5 | output.write(chalk.yellow('\nThanks for talking, bye!\n')); 6 | process.exit(0); 7 | }); 8 | 9 | export default exitCommand; 10 | -------------------------------------------------------------------------------- /src/commands/resetChatCommand.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import createCommand from './command.js'; 3 | import { resetBufferWindowMemory, resetMemoryVectorStore, setMemoryVectorStore } from '../lib/memoryManager.js'; 4 | 5 | const resetChatCommand = createCommand( 6 | 'reset', 7 | [], 8 | 'Resets the chat and starts a new conversation - This clears the memory vector store and the buffer window memory.', 9 | async (_args, output) => { 10 | output.write(chalk.yellow('\nResetting the chat!\n')); 11 | await resetMemoryVectorStore((newMemoryVectorStore) => { 12 | setMemoryVectorStore(newMemoryVectorStore); 13 | }); 14 | resetBufferWindowMemory(); 15 | } 16 | ); 17 | export default resetChatCommand; 18 | -------------------------------------------------------------------------------- /src/commands/setContextConfigCommand.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import createCommand from './command.js'; 3 | import { setNumContextDocumentsToRetrieve, getConfig } from '../config/index.js'; 4 | 5 | const setContextConfigCommand = createCommand( 6 | 'context-config', 7 | ['cc'], 8 | `Sets the number of relevant documents to return from the context vector store.\n 9 | Arguments: \`number of documents\` (Default: 4)\n 10 | Example: \`/context-config 10\``, 11 | async (args, output) => { 12 | if (!args || args.length !== 1) { 13 | output.write(chalk.red('Invalid number of arguments. Usage: /context-config `number of documents`\n')); 14 | return; 15 | } 16 | const numContextDocumentsToRetrieve = parseInt(args[0], 10); 17 | setNumContextDocumentsToRetrieve(numContextDocumentsToRetrieve); 18 | const config = getConfig(); 19 | output.write(chalk.blue(`Number of context documents to retrieve set to ${config.numContextDocumentsToRetrieve}`)); 20 | } 21 | ); 22 | export default setContextConfigCommand; 23 | -------------------------------------------------------------------------------- /src/commands/setMemoryConfigCommand.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import createCommand from './command.js'; 3 | import { setNumMemoryDocumentsToRetrieve, getConfig } from '../config/index.js'; 4 | 5 | const setMemoryConfigCommand = createCommand( 6 | 'memory-config', 7 | ['mc'], 8 | `Sets the number of relevant documents to return from the memory vector store.\n 9 | Arguments: \`number of documents\` (Default: 4)\n 10 | Example: /memory-config 10`, 11 | async (args, output) => { 12 | if (!args || args.length !== 1) { 13 | output.write(chalk.red('Invalid number of arguments. Usage: /memory-config `number of documents`\n')); 14 | return; 15 | } 16 | const numMemoryDocumentsToRetrieve = parseInt(args[0], 10); 17 | setNumMemoryDocumentsToRetrieve(numMemoryDocumentsToRetrieve); 18 | const config = getConfig(); 19 | output.write(chalk.blue(`Number of memory documents to retrieve set to ${config.numMemoryDocumentsToRetrieve}`)); 20 | } 21 | ); 22 | export default setMemoryConfigCommand; 23 | -------------------------------------------------------------------------------- /src/commands/switchContextStoreCommand.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import createCommand from './command.js'; 3 | import { loadOrCreateEmptyVectorStore } from '../lib/contextManager.js'; 4 | 5 | const changeContextStoreCommand = createCommand( 6 | 'change-context-store', 7 | ['ccs'], 8 | `Loads an existing or creates a new empty context vector store as a subdirectory of the db directory.\n 9 | Arguments: \`subdirectory\`\n 10 | Example: /change-context-store newcontext`, 11 | async (args, output) => { 12 | if (!args || args.length !== 1) { 13 | output.write(chalk.red('Invalid number of arguments. Usage: /change-context-store `subdirectory`\n')); 14 | return; 15 | } 16 | const subDirectory = args[0]; 17 | await loadOrCreateEmptyVectorStore(subDirectory); 18 | } 19 | ); 20 | export default changeContextStoreCommand; 21 | -------------------------------------------------------------------------------- /src/commands/toggleWindowBufferMemoryCommand.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import createCommand from './command.js'; 3 | import { setUseWindowMemory, getConfig } from '../config/index.js'; 4 | 5 | const toggleWindowBufferMemoryCommand = createCommand( 6 | 'toggle-window-memory', 7 | ['wm'], 8 | `Toggles the window buffer memory (MemoryBot's short-term transient memory) on or off.`, 9 | async (_args, output) => { 10 | setUseWindowMemory(!getConfig().useWindowMemory); 11 | const config = getConfig(); 12 | output.write(chalk.blue(`Use Window Buffer Memory set to ${config.useWindowMemory}`)); 13 | } 14 | ); 15 | export default toggleWindowBufferMemoryCommand; 16 | -------------------------------------------------------------------------------- /src/config/index.ts: -------------------------------------------------------------------------------- 1 | import type { Options } from 'ora'; 2 | import type { Writable } from 'stream'; 3 | import { fileURLToPath } from 'url'; 4 | import path from 'path'; 5 | 6 | export function getProjectRoot() { 7 | const currentModulePath = fileURLToPath(import.meta.url); 8 | const projectRoot = path.resolve(path.dirname(currentModulePath), '..', '..'); 9 | return projectRoot; 10 | } 11 | 12 | export function getDefaultOraOptions(output: Writable): Options { 13 | return { 14 | text: 'Loading', 15 | stream: output, 16 | discardStdin: false, 17 | }; 18 | } 19 | 20 | const defaultConfig: Config = { 21 | currentVectorStoreDatabasePath: path.join(getProjectRoot(), process.env.VECTOR_STORE_DIR || 'db/default'), 22 | numContextDocumentsToRetrieve: 4, 23 | numMemoryDocumentsToRetrieve: 4, 24 | useWindowMemory: true, 25 | chunkSize: 700, 26 | chunkOverlap: 50, 27 | }; 28 | 29 | let config: Config = { ...defaultConfig }; 30 | 31 | export function getConfig(): Config { 32 | return config; 33 | } 34 | 35 | export function setCurrentVectorStoreDatabasePath(currentVectorStoreDatabasePath: string) { 36 | config = { ...config, currentVectorStoreDatabasePath }; 37 | } 38 | 39 | export function setNumContextDocumentsToRetrieve(numContextDocumentsToRetrieve: number) { 40 | config = { ...config, numContextDocumentsToRetrieve }; 41 | } 42 | 43 | export function setNumMemoryDocumentsToRetrieve(numMemoryDocumentsToRetrieve: number) { 44 | config = { ...config, numMemoryDocumentsToRetrieve }; 45 | } 46 | 47 | export function setUseWindowMemory(useWindowMemory: boolean) { 48 | config = { ...config, useWindowMemory }; 49 | } 50 | -------------------------------------------------------------------------------- /src/global.d.ts: -------------------------------------------------------------------------------- 1 | type Command = { 2 | name: string; 3 | aliases: string[]; 4 | description: string; 5 | execute: (args: string[], output: NodeJS.WriteStream, commandHandler: CommandHandler) => Promise; 6 | }; 7 | 8 | type CommandHandler = { 9 | getCommands: () => Command[]; 10 | execute: (commandName: string, args: string[], output: NodeJS.WriteStream) => Promise; 11 | }; 12 | 13 | type Page = { 14 | url: string; 15 | text: string; 16 | title: string; 17 | }; 18 | 19 | interface Config { 20 | currentVectorStoreDatabasePath: string; 21 | numContextDocumentsToRetrieve: number; 22 | numMemoryDocumentsToRetrieve: number; 23 | useWindowMemory: boolean; 24 | chunkSize: number; 25 | chunkOverlap: number; 26 | } 27 | 28 | interface FileInfo { 29 | name: string; 30 | size: number; 31 | } 32 | 33 | interface DirectoryContent { 34 | [directory: string]: FileInfo[]; 35 | } 36 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-await-in-loop */ 2 | import dotenv from 'dotenv'; 3 | import { OpenAIChat } from 'langchain/llms/openai'; 4 | // eslint-disable-next-line import/no-unresolved 5 | import * as readline from 'node:readline/promises'; 6 | import path from 'path'; 7 | import fs from 'fs'; 8 | /* This line of code is importing the `stdin` and `stdout` streams from the `process` module in 9 | Node.js. These streams are used for reading input from the user and writing output to the console, 10 | respectively. */ 11 | import { stdin as input, stdout as output } from 'node:process'; 12 | import { CallbackManager } from 'langchain/callbacks'; 13 | import { ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate } from 'langchain/prompts'; 14 | import { LLMChain } from 'langchain/chains'; 15 | import { oneLine } from 'common-tags'; 16 | import chalk from 'chalk'; 17 | import logChat from './chatLogger.js'; 18 | import createCommandHandler from './commands.js'; 19 | import { getMemoryVectorStore, addDocumentsToMemoryVectorStore, getBufferWindowMemory } from './lib/memoryManager.js'; 20 | import { getContextVectorStore } from './lib/contextManager.js'; 21 | import { getRelevantContext } from './lib/vectorStoreUtils.js'; 22 | import sanitizeInput from './utils/sanitizeInput.js'; 23 | import { getConfig, getProjectRoot } from './config/index.js'; 24 | 25 | const projectRootDir = getProjectRoot(); 26 | 27 | dotenv.config(); 28 | 29 | // Set up the chat log directory 30 | const chatLogDirectory = path.join(projectRootDir, 'chat_logs'); 31 | 32 | // Get the prompt template 33 | const systemPromptTemplate = fs.readFileSync(path.join(projectRootDir, 'src/prompt.txt'), 'utf8'); 34 | 35 | // Set up the readline interface to read input from the user and write output to the console 36 | const rl = readline.createInterface({ input, output }); 37 | 38 | // Set up CLI commands 39 | const commandHandler: CommandHandler = createCommandHandler(); 40 | 41 | const callbackManager = CallbackManager.fromHandlers({ 42 | // This function is called when the LLM generates a new token (i.e., a prediction for the next word) 43 | async handleLLMNewToken(token: string) { 44 | // Write the token to the output stream (i.e., the console) 45 | output.write(token); 46 | }, 47 | }); 48 | 49 | const llm = new OpenAIChat({ 50 | streaming: true, 51 | callbackManager, 52 | modelName: process.env.MODEL || 'gpt-3.5-turbo', 53 | }); 54 | 55 | const systemPrompt = SystemMessagePromptTemplate.fromTemplate(oneLine` 56 | ${systemPromptTemplate} 57 | `); 58 | 59 | const chatPrompt = ChatPromptTemplate.fromPromptMessages([ 60 | systemPrompt, 61 | HumanMessagePromptTemplate.fromTemplate('QUESTION: """{input}"""'), 62 | ]); 63 | 64 | const windowMemory = getBufferWindowMemory(); 65 | 66 | const chain = new LLMChain({ 67 | prompt: chatPrompt, 68 | memory: windowMemory, 69 | llm, 70 | }); 71 | 72 | // eslint-disable-next-line no-constant-condition 73 | while (true) { 74 | output.write(chalk.green('\nStart chatting or type /help for a list of commands\n')); 75 | const userInput = await rl.question('> '); 76 | let response; 77 | if (userInput.startsWith('/')) { 78 | const [command, ...args] = userInput.slice(1).split(' '); 79 | await commandHandler.execute(command, args, output); 80 | } else { 81 | const memoryVectorStore = await getMemoryVectorStore(); 82 | const contextVectorStore = await getContextVectorStore(); 83 | const question = sanitizeInput(userInput); 84 | const config = getConfig(); 85 | const context = await getRelevantContext(contextVectorStore, question, config.numContextDocumentsToRetrieve); 86 | const history = await getRelevantContext(memoryVectorStore, question, config.numMemoryDocumentsToRetrieve); 87 | try { 88 | response = await chain.call({ 89 | input: question, 90 | context, 91 | history, 92 | immediate_history: config.useWindowMemory ? windowMemory : '', 93 | }); 94 | if (response) { 95 | await addDocumentsToMemoryVectorStore([ 96 | { content: question, metadataType: 'question' }, 97 | { content: response.text, metadataType: 'answer' }, 98 | ]); 99 | await logChat(chatLogDirectory, question, response.response); 100 | } 101 | } catch (error) { 102 | if (error instanceof Error && error.message.includes('Cancel:')) { 103 | // TODO: Handle cancel 104 | } else if (error instanceof Error) { 105 | output.write(chalk.red(error.message)); 106 | } else { 107 | output.write(chalk.red(error)); 108 | } 109 | } 110 | } 111 | output.write('\n'); 112 | } 113 | -------------------------------------------------------------------------------- /src/lib/contextManager.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import { stdout as output } from 'node:process'; 3 | import { OpenAIEmbeddings } from 'langchain/embeddings/openai'; 4 | import { HNSWLib } from 'langchain/vectorstores/hnswlib'; 5 | import { JSONLoader } from 'langchain/document_loaders/fs/json'; 6 | import { TextLoader } from 'langchain/document_loaders/fs/text'; 7 | import { PDFLoader } from 'langchain/document_loaders/fs/pdf'; 8 | import { DocxLoader } from 'langchain/document_loaders/fs/docx'; 9 | import { EPubLoader } from 'langchain/document_loaders/fs/epub'; 10 | import { CSVLoader } from 'langchain/document_loaders/fs/csv'; 11 | import ora from 'ora'; 12 | import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'; 13 | import { Document } from 'langchain/document'; 14 | import path from 'path'; 15 | import { YoutubeTranscript } from 'youtube-transcript'; 16 | import getDirectoryListWithDetails from '../utils/getDirectoryListWithDetails.js'; 17 | import createDirectory from '../utils/createDirectory.js'; 18 | import { getConfig, getDefaultOraOptions, getProjectRoot, setCurrentVectorStoreDatabasePath } from '../config/index.js'; 19 | import getDirectoryFiles from '../utils/getDirectoryFiles.js'; 20 | import WebCrawler from './crawler.js'; 21 | 22 | const projectRootDir = getProjectRoot(); 23 | 24 | const defaultOraOptions = getDefaultOraOptions(output); 25 | 26 | const defaultRecursiveCharacterTextSplitter = new RecursiveCharacterTextSplitter({ 27 | chunkSize: getConfig().chunkSize, 28 | chunkOverlap: getConfig().chunkOverlap, 29 | }); 30 | 31 | const markdownRecursiveCharacterTextSplitter = RecursiveCharacterTextSplitter.fromLanguage('markdown', { 32 | chunkSize: getConfig().chunkSize, 33 | chunkOverlap: getConfig().chunkOverlap, 34 | }); 35 | 36 | /** 37 | * This function loads and splits a file based on its extension using different loaders and text 38 | * splitters. 39 | * @param {string} filePath - A string representing the path to the file that needs to be loaded and 40 | * split into documents. 41 | * @returns The function `loadAndSplitFile` returns a Promise that resolves to an array of `Document` 42 | * objects, where each `Document` represents a split portion of the input file. The type of the 43 | * `Document` object is `Document>`, which means it has a generic type 44 | * parameter that is an object with string keys and unknown values. 45 | */ 46 | async function loadAndSplitFile(filePath: string): Promise>[]> { 47 | const fileExtension = path.extname(filePath); 48 | let loader; 49 | let documents: Document>[]; 50 | switch (fileExtension) { 51 | case '.json': 52 | loader = new JSONLoader(filePath); 53 | documents = await loader.loadAndSplit(defaultRecursiveCharacterTextSplitter); 54 | break; 55 | case '.txt': 56 | loader = new TextLoader(filePath); 57 | documents = await loader.loadAndSplit(defaultRecursiveCharacterTextSplitter); 58 | break; 59 | case '.md': 60 | loader = new TextLoader(filePath); 61 | documents = await loader.loadAndSplit(markdownRecursiveCharacterTextSplitter); 62 | break; 63 | case '.pdf': 64 | loader = new PDFLoader(filePath, { splitPages: false }); 65 | documents = await loader.loadAndSplit(defaultRecursiveCharacterTextSplitter); 66 | break; 67 | case '.docx': 68 | loader = new DocxLoader(filePath); 69 | documents = await loader.loadAndSplit(defaultRecursiveCharacterTextSplitter); 70 | break; 71 | case '.csv': 72 | loader = new CSVLoader(filePath); 73 | documents = await loader.loadAndSplit(defaultRecursiveCharacterTextSplitter); 74 | break; 75 | case '.epub': 76 | loader = new EPubLoader(filePath, { splitChapters: false }); 77 | documents = await loader.loadAndSplit(defaultRecursiveCharacterTextSplitter); 78 | break; 79 | default: 80 | throw new Error(`Unsupported file extension: ${fileExtension}`); 81 | } 82 | return documents; 83 | } 84 | 85 | /** 86 | * This function loads or creates a vector store using HNSWLib and OpenAIEmbeddings. 87 | * @returns The function `loadOrCreateVectorStore` returns a Promise that resolves to an instance of 88 | * the `HNSWLib` class, which is a vector store used for storing and searching high-dimensional 89 | * vectors. 90 | */ 91 | async function loadOrCreateVectorStore(): Promise { 92 | let vectorStore: HNSWLib; 93 | let spinner; 94 | await createDirectory(getConfig().currentVectorStoreDatabasePath); 95 | const dbDirectory = getConfig().currentVectorStoreDatabasePath; 96 | try { 97 | vectorStore = await HNSWLib.load(dbDirectory, new OpenAIEmbeddings({ maxConcurrency: 5 })); 98 | } catch { 99 | spinner = ora({ 100 | ...defaultOraOptions, 101 | text: chalk.blue(`Creating new Context Vector Store in the ${dbDirectory} directory`), 102 | }).start(); 103 | const docsDirectory = path.join(projectRootDir, process.env.DOCS_DIR || 'docs'); 104 | const filesToAdd = await getDirectoryFiles(docsDirectory); 105 | const documents = await Promise.all(filesToAdd.map((filePath) => loadAndSplitFile(filePath))); 106 | const flattenedDocuments = documents.reduce((acc, val) => acc.concat(val), []); 107 | vectorStore = await HNSWLib.fromDocuments(flattenedDocuments, new OpenAIEmbeddings({ maxConcurrency: 5 })); 108 | await vectorStore.save(dbDirectory); 109 | spinner.succeed(); 110 | } 111 | return vectorStore; 112 | } 113 | 114 | const contextVectorStore = await loadOrCreateVectorStore(); 115 | 116 | const contextWrapper = { 117 | contextInstance: contextVectorStore, 118 | }; 119 | 120 | /** 121 | * This function loads or creates a new empty Context Vector Store using HNSWLib and OpenAIEmbeddings. 122 | * @returns a Promise that resolves to an instance of the HNSWLib class, which represents a 123 | * hierarchical navigable small world graph used for nearest neighbor search. The instance is either 124 | * loaded from an existing directory or created as a new empty Context Vector Store with specified 125 | * parameters. 126 | */ 127 | async function loadOrCreateEmptyVectorStore(subDirectory: string): Promise { 128 | let vectorStore: HNSWLib; 129 | let spinner; 130 | const newContextVectorStorePath = path.join(projectRootDir, process.env.VECTOR_STORE_BASE_DIR || 'db', subDirectory); 131 | await createDirectory(newContextVectorStorePath); 132 | setCurrentVectorStoreDatabasePath(newContextVectorStorePath); 133 | const dbDirectory = getConfig().currentVectorStoreDatabasePath; 134 | try { 135 | vectorStore = await HNSWLib.load(dbDirectory, new OpenAIEmbeddings({ maxConcurrency: 5 })); 136 | output.write(chalk.blue(`Using Context Vector Store in the ${dbDirectory} directory\n`)); 137 | } catch { 138 | spinner = ora({ 139 | ...defaultOraOptions, 140 | text: chalk.blue(`Creating new empty Context Vector Store in the ${dbDirectory} directory`), 141 | }).start(); 142 | vectorStore = new HNSWLib(new OpenAIEmbeddings({ maxConcurrency: 5 }), { 143 | space: 'cosine', 144 | numDimensions: 1536, 145 | }); 146 | spinner.succeed(); 147 | output.write( 148 | chalk.red.bold( 149 | `\nThe Context Vector Store is currently empty and unsaved, add context to is using \`/add-docs\`, \`/add-url\` or \`/add-youtube\`` 150 | ) 151 | ); 152 | } 153 | contextWrapper.contextInstance = vectorStore; 154 | return vectorStore; 155 | } 156 | 157 | async function getContextVectorStore() { 158 | return contextWrapper.contextInstance; 159 | } 160 | 161 | /** 162 | * This function adds documents to a context vector store and saves them. 163 | * @param {string[]} filePaths - The `filePaths` parameter is an array of strings representing the file 164 | * paths of the documents that need to be added to the Context Vector Store. 165 | * @returns nothing (`undefined`). 166 | */ 167 | async function addDocument(filePaths: string[]) { 168 | let spinner; 169 | const dbDirectory = getConfig().currentVectorStoreDatabasePath; 170 | try { 171 | spinner = ora({ ...defaultOraOptions, text: `Adding files to the Context Vector Store` }).start(); 172 | const docsDirectory = path.join(projectRootDir, process.env.DOCS_DIR || 'docs'); 173 | const documents = await Promise.all( 174 | filePaths.map((filePath) => loadAndSplitFile(path.join(docsDirectory, filePath))) 175 | ); 176 | const flattenedDocuments = documents.reduce((acc, val) => acc.concat(val), []); 177 | const vectorStore = await getContextVectorStore(); 178 | await vectorStore.addDocuments(flattenedDocuments); 179 | await vectorStore.save(dbDirectory); 180 | spinner.succeed(); 181 | return; 182 | } catch (error) { 183 | if (spinner) { 184 | spinner.fail(chalk.red(error)); 185 | } else { 186 | output.write(chalk.red(error)); 187 | } 188 | } 189 | } 190 | 191 | /** 192 | * The function adds a YouTube video transcript to a Context Vector Store. 193 | * @param {string} URLOrVideoID - The URLOrVideoID parameter is a string that represents either the URL 194 | * or the video ID of a YouTube video. 195 | * @returns Nothing is being returned explicitly in the code, but the function is expected to return 196 | * undefined after completing its execution. 197 | */ 198 | async function addYouTube(URLOrVideoID: string) { 199 | let spinner; 200 | const dbDirectory = getConfig().currentVectorStoreDatabasePath; 201 | try { 202 | spinner = ora({ 203 | ...defaultOraOptions, 204 | text: `Adding Video transcript from ${URLOrVideoID} to the Context Vector Store`, 205 | }).start(); 206 | const transcript = await YoutubeTranscript.fetchTranscript(URLOrVideoID); 207 | const text = transcript.map((part) => part.text).join(' '); 208 | const videoDocs = await defaultRecursiveCharacterTextSplitter.splitDocuments([ 209 | new Document({ 210 | pageContent: text, 211 | }), 212 | ]); 213 | const vectorStore = await getContextVectorStore(); 214 | await vectorStore.addDocuments(videoDocs); 215 | await vectorStore.save(dbDirectory); 216 | spinner.succeed(); 217 | return; 218 | } catch (error) { 219 | if (spinner) { 220 | spinner.fail(chalk.red(error)); 221 | } else { 222 | output.write(chalk.red(error)); 223 | } 224 | } 225 | } 226 | 227 | /** 228 | * The function crawls a given URL, extracts text from the pages, splits the text into documents, 229 | * generates embeddings for the documents, and saves them to a vector store. 230 | * @param {string} URL - The URL of the website to crawl and extract text from. 231 | * @param {string} selector - The selector parameter is a string that represents a CSS selector used to 232 | * identify the HTML elements to be crawled on the web page. The WebCrawler will only crawl the 233 | * elements that match the selector. 234 | * @param {number} maxPages - The maximum number of pages to crawl for the given URL. 235 | * @param {number} numberOfCharactersRequired - `numberOfCharactersRequired` is a number that specifies 236 | * the minimum number of characters required for a document to be considered valid and used for 237 | * generating embeddings. Any document with less than this number of characters will be discarded. 238 | * @returns Nothing is being returned explicitly in the function, but it is implied that the function 239 | * will return undefined if there are no errors. 240 | */ 241 | async function addURL(URL: string, selector: string, maxPages: number, numberOfCharactersRequired: number) { 242 | const dbDirectory = getConfig().currentVectorStoreDatabasePath; 243 | const addUrlSpinner = ora({ ...defaultOraOptions, text: `Crawling ${URL}` }); 244 | let documents; 245 | try { 246 | addUrlSpinner.start(); 247 | const progressCallback = (linksFound: number, linksCrawled: number, currentUrl: string) => { 248 | addUrlSpinner.text = `Links found: ${linksFound} - Links crawled: ${linksCrawled} - Crawling ${currentUrl}`; 249 | }; 250 | 251 | const crawler = new WebCrawler([URL], progressCallback, selector, maxPages, numberOfCharactersRequired); 252 | const pages = (await crawler.start()) as Page[]; 253 | 254 | documents = await Promise.all( 255 | pages.map((row) => { 256 | const splitter = defaultRecursiveCharacterTextSplitter; 257 | 258 | const webDocs = splitter.splitDocuments([ 259 | new Document({ 260 | pageContent: row.text, 261 | }), 262 | ]); 263 | return webDocs; 264 | }) 265 | ); 266 | addUrlSpinner.succeed(); 267 | } catch (error) { 268 | addUrlSpinner.fail(chalk.red(error)); 269 | } 270 | if (documents) { 271 | const generateEmbeddingsSpinner = ora({ ...defaultOraOptions, text: `Generating Embeddings` }); 272 | try { 273 | const flattenedDocuments = documents.flat(); 274 | generateEmbeddingsSpinner.text = `Generating Embeddings for ${flattenedDocuments.length} documents`; 275 | generateEmbeddingsSpinner.start(); 276 | const vectorStore = await getContextVectorStore(); 277 | await vectorStore.addDocuments(flattenedDocuments); 278 | await vectorStore.save(dbDirectory); 279 | generateEmbeddingsSpinner.succeed(); 280 | return; 281 | } catch (error) { 282 | generateEmbeddingsSpinner.fail(chalk.red(error)); 283 | } 284 | } 285 | } 286 | 287 | async function listContextStores() { 288 | const projectRoot = getProjectRoot(); // Please replace this with your actual function to get the project root 289 | const vectorStoreDir = process.env.VECTOR_STORE_BASE_DIR || 'db'; 290 | const targetDir = path.join(projectRoot, vectorStoreDir); 291 | const contextVectorStoresList = await getDirectoryListWithDetails(targetDir); 292 | output.write(chalk.blue(`Context Vector Stores in ${targetDir}:\n\n`)); 293 | Object.entries(contextVectorStoresList).forEach(([dir, files]) => { 294 | output.write(chalk.yellow(`Directory: ${dir}`)); 295 | if (dir === getConfig().currentVectorStoreDatabasePath) { 296 | output.write(chalk.green(` (Currently selected)`)); 297 | } 298 | output.write('\n'); 299 | files.forEach((file) => { 300 | output.write(chalk.yellow(` File: ${file.name}, Size: ${file.size} KB\n`)); 301 | }); 302 | }); 303 | } 304 | 305 | export { getContextVectorStore, addDocument, addURL, addYouTube, listContextStores, loadOrCreateEmptyVectorStore }; 306 | -------------------------------------------------------------------------------- /src/lib/crawler.ts: -------------------------------------------------------------------------------- 1 | import * as cheerio from 'cheerio'; 2 | import Crawler, { CrawlerRequestResponse } from 'crawler'; 3 | import { stderr } from 'node:process'; 4 | import resolveURL from '../utils/resolveURL.js'; 5 | 6 | // import TurndownService from 'turndown'; 7 | 8 | // const turndownService = new TurndownService(); 9 | 10 | type ProgressCallback = (linksFound: number, linksCrawled: number, currentUrl: string) => void; 11 | 12 | interface Page { 13 | url: string; 14 | text: string; 15 | title: string; 16 | } 17 | 18 | /* The WebCrawler class is a TypeScript implementation of a web crawler that can extract text from web 19 | pages and follow links to crawl more pages. */ 20 | class WebCrawler { 21 | pages: Page[]; 22 | 23 | limit: number; 24 | 25 | urls: string[]; 26 | 27 | count: number; 28 | 29 | textLengthMinimum: number; 30 | 31 | selector: string; 32 | 33 | progressCallback: ProgressCallback; 34 | 35 | crawler: Crawler; 36 | 37 | constructor( 38 | urls: string[], 39 | progressCallback: ProgressCallback, 40 | selector = 'body', 41 | limit = 20, 42 | textLengthMinimum = 200 43 | ) { 44 | this.urls = urls; 45 | this.selector = selector; 46 | this.limit = limit; 47 | this.textLengthMinimum = textLengthMinimum; 48 | this.progressCallback = progressCallback; 49 | this.count = 0; 50 | this.pages = []; 51 | this.crawler = new Crawler({ 52 | maxConnections: 10, 53 | callback: this.handleRequest, 54 | userAgent: 'node-crawler', 55 | }); 56 | } 57 | 58 | /* `handleRequest` is a method that handles the response of a web page request made by the `crawler` 59 | object. It takes in three parameters: `error`, `res`, and `done`. */ 60 | handleRequest = (error: Error | null, res: CrawlerRequestResponse, done: () => void) => { 61 | if (error) { 62 | stderr.write(error.message); 63 | done(); 64 | return; 65 | } 66 | 67 | const $ = cheerio.load(res.body); 68 | // Remove obviously superfluous elements 69 | $('script').remove(); 70 | $('header').remove(); 71 | $('nav').remove(); 72 | $('style').remove(); 73 | $('img').remove(); 74 | $('svg').remove(); 75 | const title = $('title').text() || ''; 76 | const text = $(this.selector).text(); 77 | // const text = turndownService.turndown(html || ''); 78 | 79 | const page: Page = { 80 | url: res.request.uri.href, 81 | text, 82 | title, 83 | }; 84 | if (text.length > this.textLengthMinimum) { 85 | this.pages.push(page); 86 | this.progressCallback(this.count + 1, this.pages.length, res.request.uri.href); 87 | } 88 | 89 | $('a').each((_i: number, elem: cheerio.Element) => { 90 | if (this.count >= this.limit) { 91 | return false; // Stop iterating once the limit is reached 92 | } 93 | 94 | const href = $(elem).attr('href')?.split('#')[0]; 95 | const uri = res.request.uri.href; 96 | const url = href && resolveURL(uri, href); 97 | // crawl more 98 | if (url && this.urls.some((u) => url.includes(u))) { 99 | this.crawler.queue(url); 100 | this.count += 1; 101 | } 102 | return true; // Continue iterating when the limit is not reached 103 | }); 104 | 105 | done(); 106 | }; 107 | 108 | start = async () => { 109 | this.pages = []; 110 | return new Promise((resolve) => { 111 | this.crawler.on('drain', () => { 112 | resolve(this.pages); 113 | }); 114 | this.urls.forEach((url) => { 115 | this.crawler.queue(url); 116 | }); 117 | }); 118 | }; 119 | } 120 | 121 | export default WebCrawler; 122 | -------------------------------------------------------------------------------- /src/lib/memoryManager.ts: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import { HNSWLib } from 'langchain/vectorstores/hnswlib'; 3 | import fs from 'fs/promises'; 4 | import path from 'path'; 5 | import { stdout as output } from 'node:process'; 6 | import { OpenAIEmbeddings } from 'langchain/embeddings/openai'; 7 | import { Document } from 'langchain/document'; 8 | import { BufferWindowMemory } from 'langchain/memory'; 9 | import { getProjectRoot } from '../config/index.js'; 10 | 11 | const projectRootDir = getProjectRoot(); 12 | 13 | const memoryDirectory = path.join(projectRootDir, process.env.MEMORY_VECTOR_STORE_DIR || 'memory'); 14 | 15 | let memoryVectorStore: HNSWLib; 16 | try { 17 | memoryVectorStore = await HNSWLib.load(memoryDirectory, new OpenAIEmbeddings()); 18 | } catch { 19 | output.write(`${chalk.blue(`Creating a new memory vector store index in the ${memoryDirectory} directory`)}\n`); 20 | memoryVectorStore = new HNSWLib(new OpenAIEmbeddings(), { 21 | space: 'cosine', 22 | numDimensions: 1536, 23 | }); 24 | } 25 | 26 | const bufferWindowMemory = new BufferWindowMemory({ 27 | returnMessages: false, 28 | memoryKey: 'immediate_history', 29 | inputKey: 'input', 30 | k: 2, 31 | }); 32 | 33 | const memoryWrapper = { 34 | vectorStoreInstance: memoryVectorStore, 35 | }; 36 | 37 | async function getMemoryVectorStore() { 38 | return memoryWrapper.vectorStoreInstance; 39 | } 40 | 41 | function getBufferWindowMemory() { 42 | return bufferWindowMemory; 43 | } 44 | 45 | async function saveMemoryVectorStore() { 46 | await memoryWrapper.vectorStoreInstance.save(memoryDirectory); 47 | } 48 | 49 | async function addDocumentsToMemoryVectorStore( 50 | documents: Array<{ content: string; metadataType: string }> 51 | ): Promise { 52 | const formattedDocuments = documents.map( 53 | (doc) => new Document({ pageContent: doc.content, metadata: { type: doc.metadataType } }) 54 | ); 55 | await memoryWrapper.vectorStoreInstance.addDocuments(formattedDocuments); 56 | await saveMemoryVectorStore(); 57 | } 58 | 59 | function resetBufferWindowMemory() { 60 | bufferWindowMemory.clear(); 61 | } 62 | 63 | async function deleteMemoryDirectory() { 64 | try { 65 | const files = await fs.readdir(memoryDirectory); 66 | const deletePromises = files.map((file) => fs.unlink(path.join(memoryDirectory, file))); 67 | await Promise.all(deletePromises); 68 | return `All files in the memory directory have been deleted.`; 69 | } catch (error) { 70 | if (error instanceof Error) { 71 | return chalk.red(`All files in the memory directory have been deleted: ${error.message}`); 72 | } 73 | return chalk.red(`All files in the memory directory have been deleted: ${error}`); 74 | } 75 | } 76 | 77 | async function resetMemoryVectorStore(onReset: (newMemoryVectorStore: HNSWLib) => void) { 78 | const newMemoryVectorStore = new HNSWLib(new OpenAIEmbeddings(), { 79 | space: 'cosine', 80 | numDimensions: 1536, 81 | }); 82 | await deleteMemoryDirectory(); 83 | onReset(newMemoryVectorStore); 84 | } 85 | 86 | function setMemoryVectorStore(newMemoryVectorStore: HNSWLib) { 87 | memoryWrapper.vectorStoreInstance = newMemoryVectorStore; 88 | } 89 | 90 | export { 91 | getMemoryVectorStore, 92 | setMemoryVectorStore, 93 | addDocumentsToMemoryVectorStore, 94 | resetMemoryVectorStore, 95 | getBufferWindowMemory, 96 | resetBufferWindowMemory, 97 | }; 98 | -------------------------------------------------------------------------------- /src/lib/vectorStoreUtils.ts: -------------------------------------------------------------------------------- 1 | import { HNSWLib } from 'langchain/vectorstores/hnswlib'; 2 | 3 | /** 4 | * Retrieves relevant context for the given question by performing a similarity search on the provided vector store. 5 | * @param {HNSWLib} vectorStore - HNSWLib is a library for approximate nearest neighbor search, used to 6 | * search for similar vectors in a high-dimensional space. 7 | * @param {string} sanitizedQuestion - The sanitized version of the question that needs to be answered. 8 | * It is a string input. 9 | * @param {number} numDocuments - The `numDocuments` parameter is the number of documents that the 10 | * `getRelevantContext` function should retrieve from the `vectorStore` based on their similarity to 11 | * the `sanitizedQuestion`. 12 | * @returns The function `getRelevantContext` is returning a Promise that resolves to a string. The 13 | * string is the concatenation of the `pageContent` property of the top `numDocuments` documents 14 | * returned by a similarity search performed on a `vectorStore` using the `sanitizedQuestion` as the 15 | * query. The resulting string is trimmed and all newline characters are replaced with spaces. 16 | */ 17 | async function getRelevantContext( 18 | vectorStore: HNSWLib, 19 | sanitizedQuestion: string, 20 | numDocuments: number 21 | ): Promise { 22 | const documents = await vectorStore.similaritySearch(sanitizedQuestion, numDocuments); 23 | return documents 24 | .map((doc) => doc.pageContent) 25 | .join(', ') 26 | .trim() 27 | .replaceAll('\n', ' '); 28 | } 29 | 30 | // eslint-disable-next-line import/prefer-default-export 31 | export { getRelevantContext }; 32 | -------------------------------------------------------------------------------- /src/prompt.txt: -------------------------------------------------------------------------------- 1 | You are an AI assistant called MemoryBot and world class copywriter and marketing professional, 2 | you will craft high-quality, professional, relevant and engaging marketing materials and copy, such as website copy, slogans, keywords, newsletters and social 3 | media posts tailored to the user's target audience. 4 | Some of the chat history is provided as JSON, don't output this JSON. Don't preface your answer with AI: or "As an AI assistant" 5 | You have access to the chat history with the user (CHATHISTORY/MEMORY) and to context (RELEVANTDOCS) provided by the user. 6 | When answering think about whether the question refers to something in the MEMORY or CHATHISTORY before checking the RELEVANTDOCS. 7 | Don’t justify your answers. Don't refer to yourself in any of the created content. 8 | 9 | RELEVANTDOCS: {context} 10 | 11 | CHATHISTORY: {history} 12 | 13 | MEMORY: {immediate_history} 14 | -------------------------------------------------------------------------------- /src/updateReadme.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | import path from 'path'; 3 | import { getProjectRoot } from './config/index.js'; 4 | 5 | const projectRootDir = getProjectRoot(); 6 | 7 | const commandsDir = path.join(projectRootDir, 'src', 'commands'); 8 | const readmePath = path.join(projectRootDir, 'README.md'); 9 | 10 | const commandFiles = fs.readdirSync(commandsDir).filter((file) => file !== 'command.ts'); 11 | 12 | async function getCommandsMarkdown() { 13 | const commandsPromises = commandFiles.map(async (file) => { 14 | const commandModule = await import(path.join(commandsDir, file)); 15 | const command = commandModule.default; 16 | const aliases = 17 | command.aliases.length > 0 ? ` (${command.aliases.map((alias: string) => `/${alias}`).join(', ')})` : ''; 18 | return `- \`/${command.name}\`${aliases} - ${command.description}`; 19 | }); 20 | 21 | const commands = await Promise.all(commandsPromises); 22 | return commands.join('\n'); 23 | } 24 | 25 | (async () => { 26 | const commandsMarkdown = await getCommandsMarkdown(); 27 | const readmeContent = fs.readFileSync(readmePath, 'utf8'); 28 | const updatedReadmeContent = readmeContent.replace( 29 | /([\s\S]*?)/, 30 | `\n${commandsMarkdown}\n` 31 | ); 32 | 33 | fs.writeFileSync(readmePath, updatedReadmeContent, 'utf8'); 34 | })(); 35 | -------------------------------------------------------------------------------- /src/utils/createDirectory.ts: -------------------------------------------------------------------------------- 1 | import fs from 'node:fs/promises'; 2 | 3 | export default async function createDirectory(directoryPath: string): Promise { 4 | if (await fs.stat(directoryPath).catch(() => false)) { 5 | return; 6 | } 7 | await fs.mkdir(directoryPath, { recursive: true }); 8 | } 9 | -------------------------------------------------------------------------------- /src/utils/getDirectoryFiles.ts: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import fs from 'node:fs/promises'; 3 | 4 | export default async function getDirectoryFiles(directoryPath: string): Promise { 5 | const fileNames = await fs.readdir(directoryPath); 6 | 7 | const filePathsPromises = fileNames.map(async (fileName) => { 8 | const filePath = path.join(directoryPath, fileName); 9 | const stat = await fs.stat(filePath); 10 | 11 | if (stat.isDirectory()) { 12 | const subDirectoryFiles = await getDirectoryFiles(filePath); 13 | return subDirectoryFiles; 14 | } 15 | return filePath; 16 | }); 17 | 18 | const filePathsArray = await Promise.all(filePathsPromises); 19 | const filePaths = filePathsArray.flat(); 20 | return filePaths; 21 | } 22 | -------------------------------------------------------------------------------- /src/utils/getDirectoryListWithDetails.ts: -------------------------------------------------------------------------------- 1 | import fs from 'node:fs/promises'; 2 | import path from 'path'; 3 | 4 | export default async function getDirectoryListWithDetails( 5 | directory: string, 6 | contents: DirectoryContent = {} 7 | ): Promise { 8 | const dirents = await fs.readdir(directory, { withFileTypes: true }); 9 | const newContents: DirectoryContent = { ...contents }; 10 | const files: FileInfo[] = []; 11 | 12 | const actions = dirents.map(async (dirent) => { 13 | const res = path.resolve(directory, dirent.name); 14 | if (dirent.isDirectory()) { 15 | const subdirContents = await getDirectoryListWithDetails(res, newContents); 16 | Object.assign(newContents, subdirContents); 17 | } else if (dirent.isFile() && dirent.name !== '.gitignore') { 18 | const stats = await fs.stat(res); 19 | files.push({ name: dirent.name, size: Math.ceil(stats.size / 1024) }); 20 | } 21 | }); 22 | 23 | await Promise.all(actions); 24 | 25 | if (files.length) { 26 | newContents[directory] = files; 27 | } 28 | 29 | return newContents; 30 | } 31 | -------------------------------------------------------------------------------- /src/utils/resolveURL.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * The function resolves a URL from a given base URL and returns the resolved URL as a string. 3 | * @param {string} from - The `from` parameter is a string representing the base URL that the `to` 4 | * parameter will be resolved against. It can be an absolute or relative URL. 5 | * @param {string} to - The `to` parameter is a string representing the URL that needs to be resolved. 6 | * It can be an absolute URL or a relative URL. 7 | * @returns The function `resolve` returns a string that represents the resolved URL. If the `to` 8 | * parameter is a relative URL, the function returns a string that represents the resolved URL relative 9 | * to the `from` parameter. If the `to` parameter is an absolute URL, the function returns a string 10 | * that represents the resolved URL. 11 | */ 12 | export default function resolve(from: string, to: string) { 13 | const resolvedUrl = new URL(to, new URL(from, 'resolve://')); 14 | if (resolvedUrl.protocol === 'resolve:') { 15 | // `from` is a relative URL. 16 | const { pathname, search, hash } = resolvedUrl; 17 | return pathname + search + hash; 18 | } 19 | return resolvedUrl.toString(); 20 | } 21 | -------------------------------------------------------------------------------- /src/utils/sanitizeInput.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * The function sanitizes a string input by removing leading/trailing white spaces and replacing new 3 | * lines with spaces. 4 | * @param {string} input - The input parameter is a string that needs to be sanitized. 5 | * @returns The function `sanitizeInput` is returning a string. The string is the input string with 6 | * leading and trailing whitespace removed, and all newline characters replaced with a space character. 7 | */ 8 | export default function sanitizeInput(input: string): string { 9 | return input.trim().replaceAll('\n', ' '); 10 | } 11 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@tsconfig/recommended", 3 | "compilerOptions": { 4 | "outDir": "dist", 5 | "rootDir": "src", 6 | "lib": ["ES2021", "ES2022.Object", "DOM", "ES2021.String"], 7 | "target": "es2021", 8 | "module": "NodeNext", 9 | "moduleResolution": "NodeNext", 10 | "sourceMap": true, 11 | "allowSyntheticDefaultImports": true, 12 | "baseUrl": "./src", 13 | "declaration": true, 14 | "experimentalDecorators": true, 15 | "noImplicitReturns": true, 16 | "noFallthroughCasesInSwitch": true, 17 | "noUnusedLocals": true, 18 | "noUnusedParameters": true, 19 | "useDefineForClassFields": true, 20 | "strictPropertyInitialization": false 21 | }, 22 | "exclude": ["node_modules/", "dist/", "tests/", "docs/", "db/", "memory/", "tmp/"], 23 | "include": ["./src", "./src/lib", "./src/utils", "./src/global.d.ts", "src/updateReadme.ts"] 24 | } 25 | --------------------------------------------------------------------------------