├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── src ├── api-client.ts ├── handler-registry.ts ├── handlers │ ├── add-documentation.ts │ ├── base-handler.ts │ ├── clear-queue.ts │ ├── extract-urls.ts │ ├── index.ts │ ├── list-queue.ts │ ├── list-sources.ts │ ├── remove-documentation.ts │ ├── run-queue.ts │ └── search-documentation.ts ├── index.ts ├── tools │ ├── base-tool.ts │ ├── clear-queue.ts │ ├── extract-urls.ts │ ├── index.ts │ ├── list-queue.ts │ ├── list-sources.ts │ ├── remove-documentation.ts │ ├── run-queue.ts │ └── search-documentation.ts └── types.ts └── tsconfig.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | node_modules/ 3 | .pnp/ 4 | .pnp.js 5 | 6 | # Build output 7 | build/ 8 | dist/ 9 | *.tsbuildinfo 10 | 11 | # Environment variables 12 | .env 13 | .env.local 14 | .env.development.local 15 | .env.test.local 16 | .env.production.local 17 | 18 | # Logs 19 | logs/ 20 | *.log 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | 25 | # Editor directories and files 26 | .idea/ 27 | .vscode/ 28 | *.swp 29 | *.swo 30 | .DS_Store 31 | 32 | # Test coverage 33 | coverage/ 34 | 35 | # Local documentation files 36 | INTERNAL.TXT 37 | queue.txt 38 | MCPguide.txt 39 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [1.1.0] - 2024-03-14 4 | 5 | ### Initial Feature Addition 6 | - Implemented new clear_queue tool for queue management 7 | - Created src/tools/clear-queue.ts with core functionality 8 | - Added handler in src/handlers/clear-queue.ts 9 | - Integrated with existing queue management system 10 | - Added tool exports and registration 11 | 12 | ### Code Organization 13 | - Improved tool ordering in handler-registry.ts 14 | - Moved remove_documentation before extract_urls 15 | - Enhanced logical grouping of related tools 16 | - Updated imports to match new ordering 17 | 18 | ### Documentation Enhancement Phase 1 19 | - Enhanced tool descriptions in handler-registry.ts: 20 | 1. search_documentation 21 | - Added natural language query support details 22 | - Clarified result ranking and context 23 | - Improved limit parameter documentation 24 | 2. list_sources 25 | - Added details about indexed documentation 26 | - Clarified source information returned 27 | 3. extract_urls 28 | - Enhanced URL crawling explanation 29 | - Added queue integration details 30 | - Clarified URL validation requirements 31 | 4. remove_documentation 32 | - Added permanence warning 33 | - Clarified URL matching requirements 34 | 5. list_queue 35 | - Added queue monitoring details 36 | - Clarified status checking capabilities 37 | 6. run_queue 38 | - Added processing behavior details 39 | - Documented error handling 40 | 7. clear_queue 41 | - Detailed queue clearing behavior 42 | - Added permanence warnings 43 | - Documented URL re-adding requirements 44 | 45 | ### Documentation Enhancement Phase 2 46 | - Updated README.md 47 | - Removed add_documentation and queue_documentation tools 48 | - Updated tool descriptions to match handler-registry.ts 49 | - Added parameter format requirements 50 | - Enhanced usage guidance -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Hannes Rudolph 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RAG Documentation MCP Server 2 | 3 | An MCP server implementation that provides tools for retrieving and processing documentation through vector search, enabling AI assistants to augment their responses with relevant documentation context. 4 | 5 | ## Features 6 | 7 | - Vector-based documentation search and retrieval 8 | - Support for multiple documentation sources 9 | - Semantic search capabilities 10 | - Automated documentation processing 11 | - Real-time context augmentation for LLMs 12 | 13 | ## Tools 14 | 15 | ### search_documentation 16 | Search through stored documentation using natural language queries. Returns matching excerpts with context, ranked by relevance. 17 | 18 | **Inputs:** 19 | - `query` (string): The text to search for in the documentation. Can be a natural language query, specific terms, or code snippets. 20 | - `limit` (number, optional): Maximum number of results to return (1-20, default: 5). Higher limits provide more comprehensive results but may take longer to process. 21 | 22 | ### list_sources 23 | List all documentation sources currently stored in the system. Returns a comprehensive list of all indexed documentation including source URLs, titles, and last update times. Use this to understand what documentation is available for searching or to verify if specific sources have been indexed. 24 | 25 | ### extract_urls 26 | Extract and analyze all URLs from a given web page. This tool crawls the specified webpage, identifies all hyperlinks, and optionally adds them to the processing queue. 27 | 28 | **Inputs:** 29 | - `url` (string): The complete URL of the webpage to analyze (must include protocol, e.g., https://). The page must be publicly accessible. 30 | - `add_to_queue` (boolean, optional): If true, automatically add extracted URLs to the processing queue for later indexing. Use with caution on large sites to avoid excessive queuing. 31 | 32 | ### remove_documentation 33 | Remove specific documentation sources from the system by their URLs. The removal is permanent and will affect future search results. 34 | 35 | **Inputs:** 36 | - `urls` (string[]): Array of URLs to remove from the database. Each URL must exactly match the URL used when the documentation was added. 37 | 38 | ### list_queue 39 | List all URLs currently waiting in the documentation processing queue. Shows pending documentation sources that will be processed when run_queue is called. Use this to monitor queue status, verify URLs were added correctly, or check processing backlog. 40 | 41 | ### run_queue 42 | Process and index all URLs currently in the documentation queue. Each URL is processed sequentially, with proper error handling and retry logic. Progress updates are provided as processing occurs. Long-running operations will process until the queue is empty or an unrecoverable error occurs. 43 | 44 | ### clear_queue 45 | Remove all pending URLs from the documentation processing queue. Use this to reset the queue when you want to start fresh, remove unwanted URLs, or cancel pending processing. This operation is immediate and permanent - URLs will need to be re-added if you want to process them later. 46 | 47 | ## Usage 48 | 49 | The RAG Documentation tool is designed for: 50 | 51 | - Enhancing AI responses with relevant documentation 52 | - Building documentation-aware AI assistants 53 | - Creating context-aware tooling for developers 54 | - Implementing semantic documentation search 55 | - Augmenting existing knowledge bases 56 | 57 | ## Configuration 58 | 59 | ### Usage with Claude Desktop 60 | 61 | Add this to your `claude_desktop_config.json`: 62 | 63 | ```json 64 | { 65 | "mcpServers": { 66 | "rag-docs": { 67 | "command": "npx", 68 | "args": [ 69 | "-y", 70 | "@hannesrudolph/mcp-ragdocs" 71 | ], 72 | "env": { 73 | "OPENAI_API_KEY": "", 74 | "QDRANT_URL": "", 75 | "QDRANT_API_KEY": "" 76 | } 77 | } 78 | } 79 | } 80 | ``` 81 | 82 | You'll need to provide values for the following environment variables: 83 | - `OPENAI_API_KEY`: Your OpenAI API key for embeddings generation 84 | - `QDRANT_URL`: URL of your Qdrant vector database instance 85 | - `QDRANT_API_KEY`: API key for authenticating with Qdrant 86 | 87 | ## License 88 | 89 | This MCP server is licensed under the MIT License. This means you are free to use, modify, and distribute the software, subject to the terms and conditions of the MIT License. For more details, please see the LICENSE file in the project repository. 90 | 91 | ## Acknowledgments 92 | 93 | This project is a fork of [qpd-v/mcp-ragdocs](https://github.com/qpd-v/mcp-ragdocs), originally developed by qpd-v. The original project provided the foundation for this implementation. -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@hannesrudolph/mcp-ragdocs", 3 | "version": "1.0.0", 4 | "lockfileVersion": 3, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "@hannesrudolph/mcp-ragdocs", 9 | "version": "1.0.0", 10 | "license": "MIT", 11 | "dependencies": { 12 | "@azure/openai": "^2.0.0", 13 | "@modelcontextprotocol/sdk": "^1.0.3", 14 | "@qdrant/js-client-rest": "^1.12.0", 15 | "axios": "^1.7.9", 16 | "cheerio": "^1.0.0", 17 | "openai": "^4.76.2", 18 | "playwright": "^1.49.1" 19 | }, 20 | "bin": { 21 | "mcp-ragdocs": "build/index.js" 22 | }, 23 | "devDependencies": { 24 | "@types/node": "^20.17.10", 25 | "ts-node": "^10.9.2", 26 | "typescript": "^5.7.2" 27 | } 28 | }, 29 | "node_modules/@azure-rest/core-client": { 30 | "version": "2.3.1", 31 | "resolved": "https://registry.npmjs.org/@azure-rest/core-client/-/core-client-2.3.1.tgz", 32 | "integrity": "sha512-sGTdh2Ln95F/Jqikr9OybQvx00EVvljwgxjfcxTqjID0PBVGDuNR0ie9e9HsTA1vJT23BlVRd/dCIGzJriYw9g==", 33 | "license": "MIT", 34 | "dependencies": { 35 | "@azure/abort-controller": "^2.0.0", 36 | "@azure/core-auth": "^1.3.0", 37 | "@azure/core-rest-pipeline": "^1.5.0", 38 | "@azure/core-tracing": "^1.0.1", 39 | "@azure/core-util": "^1.0.0", 40 | "tslib": "^2.6.2" 41 | }, 42 | "engines": { 43 | "node": ">=18.0.0" 44 | } 45 | }, 46 | "node_modules/@azure/abort-controller": { 47 | "version": "2.1.2", 48 | "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", 49 | "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", 50 | "license": "MIT", 51 | "dependencies": { 52 | "tslib": "^2.6.2" 53 | }, 54 | "engines": { 55 | "node": ">=18.0.0" 56 | } 57 | }, 58 | "node_modules/@azure/core-auth": { 59 | "version": "1.9.0", 60 | "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.9.0.tgz", 61 | "integrity": "sha512-FPwHpZywuyasDSLMqJ6fhbOK3TqUdviZNF8OqRGA4W5Ewib2lEEZ+pBsYcBa88B2NGO/SEnYPGhyBqNlE8ilSw==", 62 | "license": "MIT", 63 | "dependencies": { 64 | "@azure/abort-controller": "^2.0.0", 65 | "@azure/core-util": "^1.11.0", 66 | "tslib": "^2.6.2" 67 | }, 68 | "engines": { 69 | "node": ">=18.0.0" 70 | } 71 | }, 72 | "node_modules/@azure/core-rest-pipeline": { 73 | "version": "1.18.1", 74 | "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.18.1.tgz", 75 | "integrity": "sha512-/wS73UEDrxroUEVywEm7J0p2c+IIiVxyfigCGfsKvCxxCET4V/Hef2aURqltrXMRjNmdmt5IuOgIpl8f6xdO5A==", 76 | "license": "MIT", 77 | "dependencies": { 78 | "@azure/abort-controller": "^2.0.0", 79 | "@azure/core-auth": "^1.8.0", 80 | "@azure/core-tracing": "^1.0.1", 81 | "@azure/core-util": "^1.11.0", 82 | "@azure/logger": "^1.0.0", 83 | "http-proxy-agent": "^7.0.0", 84 | "https-proxy-agent": "^7.0.0", 85 | "tslib": "^2.6.2" 86 | }, 87 | "engines": { 88 | "node": ">=18.0.0" 89 | } 90 | }, 91 | "node_modules/@azure/core-tracing": { 92 | "version": "1.2.0", 93 | "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.2.0.tgz", 94 | "integrity": "sha512-UKTiEJPkWcESPYJz3X5uKRYyOcJD+4nYph+KpfdPRnQJVrZfk0KJgdnaAWKfhsBBtAf/D58Az4AvCJEmWgIBAg==", 95 | "license": "MIT", 96 | "dependencies": { 97 | "tslib": "^2.6.2" 98 | }, 99 | "engines": { 100 | "node": ">=18.0.0" 101 | } 102 | }, 103 | "node_modules/@azure/core-util": { 104 | "version": "1.11.0", 105 | "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.11.0.tgz", 106 | "integrity": "sha512-DxOSLua+NdpWoSqULhjDyAZTXFdP/LKkqtYuxxz1SCN289zk3OG8UOpnCQAz/tygyACBtWp/BoO72ptK7msY8g==", 107 | "license": "MIT", 108 | "dependencies": { 109 | "@azure/abort-controller": "^2.0.0", 110 | "tslib": "^2.6.2" 111 | }, 112 | "engines": { 113 | "node": ">=18.0.0" 114 | } 115 | }, 116 | "node_modules/@azure/logger": { 117 | "version": "1.1.4", 118 | "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.1.4.tgz", 119 | "integrity": "sha512-4IXXzcCdLdlXuCG+8UKEwLA1T1NHqUfanhXYHiQTn+6sfWCZXduqbtXDGceg3Ce5QxTGo7EqmbV6Bi+aqKuClQ==", 120 | "license": "MIT", 121 | "dependencies": { 122 | "tslib": "^2.6.2" 123 | }, 124 | "engines": { 125 | "node": ">=18.0.0" 126 | } 127 | }, 128 | "node_modules/@azure/openai": { 129 | "version": "2.0.0", 130 | "resolved": "https://registry.npmjs.org/@azure/openai/-/openai-2.0.0.tgz", 131 | "integrity": "sha512-zSNhwarYbqg3P048uKMjEjbge41OnAgmiiE1elCHVsuCCXRyz2BXnHMJkW6WR6ZKQy5NHswJNUNSWsuqancqFA==", 132 | "license": "MIT", 133 | "dependencies": { 134 | "@azure-rest/core-client": "^2.2.0", 135 | "tslib": "^2.6.3" 136 | }, 137 | "engines": { 138 | "node": ">=18.0.0" 139 | } 140 | }, 141 | "node_modules/@cspotcode/source-map-support": { 142 | "version": "0.8.1", 143 | "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", 144 | "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", 145 | "dev": true, 146 | "license": "MIT", 147 | "dependencies": { 148 | "@jridgewell/trace-mapping": "0.3.9" 149 | }, 150 | "engines": { 151 | "node": ">=12" 152 | } 153 | }, 154 | "node_modules/@fastify/busboy": { 155 | "version": "2.1.1", 156 | "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", 157 | "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", 158 | "license": "MIT", 159 | "engines": { 160 | "node": ">=14" 161 | } 162 | }, 163 | "node_modules/@jridgewell/resolve-uri": { 164 | "version": "3.1.2", 165 | "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", 166 | "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", 167 | "dev": true, 168 | "license": "MIT", 169 | "engines": { 170 | "node": ">=6.0.0" 171 | } 172 | }, 173 | "node_modules/@jridgewell/sourcemap-codec": { 174 | "version": "1.5.0", 175 | "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", 176 | "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", 177 | "dev": true, 178 | "license": "MIT" 179 | }, 180 | "node_modules/@jridgewell/trace-mapping": { 181 | "version": "0.3.9", 182 | "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", 183 | "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", 184 | "dev": true, 185 | "license": "MIT", 186 | "dependencies": { 187 | "@jridgewell/resolve-uri": "^3.0.3", 188 | "@jridgewell/sourcemap-codec": "^1.4.10" 189 | } 190 | }, 191 | "node_modules/@modelcontextprotocol/sdk": { 192 | "version": "1.0.3", 193 | "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.0.3.tgz", 194 | "integrity": "sha512-2as3cX/VJ0YBHGmdv3GFyTpoM8q2gqE98zh3Vf1NwnsSY0h3mvoO07MUzfygCKkWsFjcZm4otIiqD6Xh7kiSBQ==", 195 | "license": "MIT", 196 | "dependencies": { 197 | "content-type": "^1.0.5", 198 | "raw-body": "^3.0.0", 199 | "zod": "^3.23.8" 200 | } 201 | }, 202 | "node_modules/@qdrant/js-client-rest": { 203 | "version": "1.12.0", 204 | "resolved": "https://registry.npmjs.org/@qdrant/js-client-rest/-/js-client-rest-1.12.0.tgz", 205 | "integrity": "sha512-H8VokZq2DYe9yfKG3c7xPNR+Oc5ZvwMUtPEr1wUO4xVi9w5P89MScJaCc9UW8mS5AR+/Y1h2t1YjSxBFPIYT2Q==", 206 | "license": "Apache-2.0", 207 | "dependencies": { 208 | "@qdrant/openapi-typescript-fetch": "1.2.6", 209 | "@sevinf/maybe": "0.5.0", 210 | "undici": "~5.28.4" 211 | }, 212 | "engines": { 213 | "node": ">=18.0.0", 214 | "pnpm": ">=8" 215 | }, 216 | "peerDependencies": { 217 | "typescript": ">=4.7" 218 | } 219 | }, 220 | "node_modules/@qdrant/openapi-typescript-fetch": { 221 | "version": "1.2.6", 222 | "resolved": "https://registry.npmjs.org/@qdrant/openapi-typescript-fetch/-/openapi-typescript-fetch-1.2.6.tgz", 223 | "integrity": "sha512-oQG/FejNpItrxRHoyctYvT3rwGZOnK4jr3JdppO/c78ktDvkWiPXPHNsrDf33K9sZdRb6PR7gi4noIapu5q4HA==", 224 | "license": "MIT", 225 | "engines": { 226 | "node": ">=18.0.0", 227 | "pnpm": ">=8" 228 | } 229 | }, 230 | "node_modules/@sevinf/maybe": { 231 | "version": "0.5.0", 232 | "resolved": "https://registry.npmjs.org/@sevinf/maybe/-/maybe-0.5.0.tgz", 233 | "integrity": "sha512-ARhyoYDnY1LES3vYI0fiG6e9esWfTNcXcO6+MPJJXcnyMV3bim4lnFt45VXouV7y82F4x3YH8nOQ6VztuvUiWg==", 234 | "license": "MIT" 235 | }, 236 | "node_modules/@tsconfig/node10": { 237 | "version": "1.0.11", 238 | "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", 239 | "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==", 240 | "dev": true, 241 | "license": "MIT" 242 | }, 243 | "node_modules/@tsconfig/node12": { 244 | "version": "1.0.11", 245 | "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", 246 | "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", 247 | "dev": true, 248 | "license": "MIT" 249 | }, 250 | "node_modules/@tsconfig/node14": { 251 | "version": "1.0.3", 252 | "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", 253 | "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", 254 | "dev": true, 255 | "license": "MIT" 256 | }, 257 | "node_modules/@tsconfig/node16": { 258 | "version": "1.0.4", 259 | "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", 260 | "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", 261 | "dev": true, 262 | "license": "MIT" 263 | }, 264 | "node_modules/@types/node": { 265 | "version": "20.17.10", 266 | "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.10.tgz", 267 | "integrity": "sha512-/jrvh5h6NXhEauFFexRin69nA0uHJ5gwk4iDivp/DeoEua3uwCUto6PC86IpRITBOs4+6i2I56K5x5b6WYGXHA==", 268 | "license": "MIT", 269 | "dependencies": { 270 | "undici-types": "~6.19.2" 271 | } 272 | }, 273 | "node_modules/@types/node-fetch": { 274 | "version": "2.6.12", 275 | "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", 276 | "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", 277 | "license": "MIT", 278 | "dependencies": { 279 | "@types/node": "*", 280 | "form-data": "^4.0.0" 281 | } 282 | }, 283 | "node_modules/abort-controller": { 284 | "version": "3.0.0", 285 | "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", 286 | "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", 287 | "license": "MIT", 288 | "dependencies": { 289 | "event-target-shim": "^5.0.0" 290 | }, 291 | "engines": { 292 | "node": ">=6.5" 293 | } 294 | }, 295 | "node_modules/acorn": { 296 | "version": "8.14.0", 297 | "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", 298 | "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", 299 | "dev": true, 300 | "license": "MIT", 301 | "bin": { 302 | "acorn": "bin/acorn" 303 | }, 304 | "engines": { 305 | "node": ">=0.4.0" 306 | } 307 | }, 308 | "node_modules/acorn-walk": { 309 | "version": "8.3.4", 310 | "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", 311 | "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", 312 | "dev": true, 313 | "license": "MIT", 314 | "dependencies": { 315 | "acorn": "^8.11.0" 316 | }, 317 | "engines": { 318 | "node": ">=0.4.0" 319 | } 320 | }, 321 | "node_modules/agent-base": { 322 | "version": "7.1.3", 323 | "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", 324 | "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", 325 | "license": "MIT", 326 | "engines": { 327 | "node": ">= 14" 328 | } 329 | }, 330 | "node_modules/agentkeepalive": { 331 | "version": "4.5.0", 332 | "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", 333 | "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", 334 | "license": "MIT", 335 | "dependencies": { 336 | "humanize-ms": "^1.2.1" 337 | }, 338 | "engines": { 339 | "node": ">= 8.0.0" 340 | } 341 | }, 342 | "node_modules/arg": { 343 | "version": "4.1.3", 344 | "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", 345 | "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", 346 | "dev": true, 347 | "license": "MIT" 348 | }, 349 | "node_modules/asynckit": { 350 | "version": "0.4.0", 351 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 352 | "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", 353 | "license": "MIT" 354 | }, 355 | "node_modules/axios": { 356 | "version": "1.7.9", 357 | "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.9.tgz", 358 | "integrity": "sha512-LhLcE7Hbiryz8oMDdDptSrWowmB4Bl6RCt6sIJKpRB4XtVf0iEgewX3au/pJqm+Py1kCASkb/FFKjxQaLtxJvw==", 359 | "license": "MIT", 360 | "dependencies": { 361 | "follow-redirects": "^1.15.6", 362 | "form-data": "^4.0.0", 363 | "proxy-from-env": "^1.1.0" 364 | } 365 | }, 366 | "node_modules/boolbase": { 367 | "version": "1.0.0", 368 | "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", 369 | "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", 370 | "license": "ISC" 371 | }, 372 | "node_modules/bytes": { 373 | "version": "3.1.2", 374 | "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", 375 | "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", 376 | "license": "MIT", 377 | "engines": { 378 | "node": ">= 0.8" 379 | } 380 | }, 381 | "node_modules/cheerio": { 382 | "version": "1.0.0", 383 | "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", 384 | "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==", 385 | "license": "MIT", 386 | "dependencies": { 387 | "cheerio-select": "^2.1.0", 388 | "dom-serializer": "^2.0.0", 389 | "domhandler": "^5.0.3", 390 | "domutils": "^3.1.0", 391 | "encoding-sniffer": "^0.2.0", 392 | "htmlparser2": "^9.1.0", 393 | "parse5": "^7.1.2", 394 | "parse5-htmlparser2-tree-adapter": "^7.0.0", 395 | "parse5-parser-stream": "^7.1.2", 396 | "undici": "^6.19.5", 397 | "whatwg-mimetype": "^4.0.0" 398 | }, 399 | "engines": { 400 | "node": ">=18.17" 401 | }, 402 | "funding": { 403 | "url": "https://github.com/cheeriojs/cheerio?sponsor=1" 404 | } 405 | }, 406 | "node_modules/cheerio-select": { 407 | "version": "2.1.0", 408 | "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", 409 | "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", 410 | "license": "BSD-2-Clause", 411 | "dependencies": { 412 | "boolbase": "^1.0.0", 413 | "css-select": "^5.1.0", 414 | "css-what": "^6.1.0", 415 | "domelementtype": "^2.3.0", 416 | "domhandler": "^5.0.3", 417 | "domutils": "^3.0.1" 418 | }, 419 | "funding": { 420 | "url": "https://github.com/sponsors/fb55" 421 | } 422 | }, 423 | "node_modules/cheerio/node_modules/undici": { 424 | "version": "6.21.0", 425 | "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.0.tgz", 426 | "integrity": "sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw==", 427 | "license": "MIT", 428 | "engines": { 429 | "node": ">=18.17" 430 | } 431 | }, 432 | "node_modules/combined-stream": { 433 | "version": "1.0.8", 434 | "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", 435 | "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", 436 | "license": "MIT", 437 | "dependencies": { 438 | "delayed-stream": "~1.0.0" 439 | }, 440 | "engines": { 441 | "node": ">= 0.8" 442 | } 443 | }, 444 | "node_modules/content-type": { 445 | "version": "1.0.5", 446 | "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", 447 | "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", 448 | "license": "MIT", 449 | "engines": { 450 | "node": ">= 0.6" 451 | } 452 | }, 453 | "node_modules/create-require": { 454 | "version": "1.1.1", 455 | "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", 456 | "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", 457 | "dev": true, 458 | "license": "MIT" 459 | }, 460 | "node_modules/css-select": { 461 | "version": "5.1.0", 462 | "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", 463 | "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", 464 | "license": "BSD-2-Clause", 465 | "dependencies": { 466 | "boolbase": "^1.0.0", 467 | "css-what": "^6.1.0", 468 | "domhandler": "^5.0.2", 469 | "domutils": "^3.0.1", 470 | "nth-check": "^2.0.1" 471 | }, 472 | "funding": { 473 | "url": "https://github.com/sponsors/fb55" 474 | } 475 | }, 476 | "node_modules/css-what": { 477 | "version": "6.1.0", 478 | "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", 479 | "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", 480 | "license": "BSD-2-Clause", 481 | "engines": { 482 | "node": ">= 6" 483 | }, 484 | "funding": { 485 | "url": "https://github.com/sponsors/fb55" 486 | } 487 | }, 488 | "node_modules/debug": { 489 | "version": "4.4.0", 490 | "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", 491 | "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", 492 | "license": "MIT", 493 | "dependencies": { 494 | "ms": "^2.1.3" 495 | }, 496 | "engines": { 497 | "node": ">=6.0" 498 | }, 499 | "peerDependenciesMeta": { 500 | "supports-color": { 501 | "optional": true 502 | } 503 | } 504 | }, 505 | "node_modules/delayed-stream": { 506 | "version": "1.0.0", 507 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 508 | "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", 509 | "license": "MIT", 510 | "engines": { 511 | "node": ">=0.4.0" 512 | } 513 | }, 514 | "node_modules/depd": { 515 | "version": "2.0.0", 516 | "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", 517 | "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", 518 | "license": "MIT", 519 | "engines": { 520 | "node": ">= 0.8" 521 | } 522 | }, 523 | "node_modules/diff": { 524 | "version": "4.0.2", 525 | "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", 526 | "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", 527 | "dev": true, 528 | "license": "BSD-3-Clause", 529 | "engines": { 530 | "node": ">=0.3.1" 531 | } 532 | }, 533 | "node_modules/dom-serializer": { 534 | "version": "2.0.0", 535 | "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", 536 | "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", 537 | "license": "MIT", 538 | "dependencies": { 539 | "domelementtype": "^2.3.0", 540 | "domhandler": "^5.0.2", 541 | "entities": "^4.2.0" 542 | }, 543 | "funding": { 544 | "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" 545 | } 546 | }, 547 | "node_modules/domelementtype": { 548 | "version": "2.3.0", 549 | "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", 550 | "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", 551 | "funding": [ 552 | { 553 | "type": "github", 554 | "url": "https://github.com/sponsors/fb55" 555 | } 556 | ], 557 | "license": "BSD-2-Clause" 558 | }, 559 | "node_modules/domhandler": { 560 | "version": "5.0.3", 561 | "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", 562 | "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", 563 | "license": "BSD-2-Clause", 564 | "dependencies": { 565 | "domelementtype": "^2.3.0" 566 | }, 567 | "engines": { 568 | "node": ">= 4" 569 | }, 570 | "funding": { 571 | "url": "https://github.com/fb55/domhandler?sponsor=1" 572 | } 573 | }, 574 | "node_modules/domutils": { 575 | "version": "3.1.0", 576 | "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", 577 | "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", 578 | "license": "BSD-2-Clause", 579 | "dependencies": { 580 | "dom-serializer": "^2.0.0", 581 | "domelementtype": "^2.3.0", 582 | "domhandler": "^5.0.3" 583 | }, 584 | "funding": { 585 | "url": "https://github.com/fb55/domutils?sponsor=1" 586 | } 587 | }, 588 | "node_modules/encoding-sniffer": { 589 | "version": "0.2.0", 590 | "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.0.tgz", 591 | "integrity": "sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==", 592 | "license": "MIT", 593 | "dependencies": { 594 | "iconv-lite": "^0.6.3", 595 | "whatwg-encoding": "^3.1.1" 596 | }, 597 | "funding": { 598 | "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" 599 | } 600 | }, 601 | "node_modules/entities": { 602 | "version": "4.5.0", 603 | "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", 604 | "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", 605 | "license": "BSD-2-Clause", 606 | "engines": { 607 | "node": ">=0.12" 608 | }, 609 | "funding": { 610 | "url": "https://github.com/fb55/entities?sponsor=1" 611 | } 612 | }, 613 | "node_modules/event-target-shim": { 614 | "version": "5.0.1", 615 | "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", 616 | "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", 617 | "license": "MIT", 618 | "engines": { 619 | "node": ">=6" 620 | } 621 | }, 622 | "node_modules/follow-redirects": { 623 | "version": "1.15.9", 624 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", 625 | "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", 626 | "funding": [ 627 | { 628 | "type": "individual", 629 | "url": "https://github.com/sponsors/RubenVerborgh" 630 | } 631 | ], 632 | "license": "MIT", 633 | "engines": { 634 | "node": ">=4.0" 635 | }, 636 | "peerDependenciesMeta": { 637 | "debug": { 638 | "optional": true 639 | } 640 | } 641 | }, 642 | "node_modules/form-data": { 643 | "version": "4.0.1", 644 | "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", 645 | "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", 646 | "license": "MIT", 647 | "dependencies": { 648 | "asynckit": "^0.4.0", 649 | "combined-stream": "^1.0.8", 650 | "mime-types": "^2.1.12" 651 | }, 652 | "engines": { 653 | "node": ">= 6" 654 | } 655 | }, 656 | "node_modules/form-data-encoder": { 657 | "version": "1.7.2", 658 | "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", 659 | "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", 660 | "license": "MIT" 661 | }, 662 | "node_modules/formdata-node": { 663 | "version": "4.4.1", 664 | "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", 665 | "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", 666 | "license": "MIT", 667 | "dependencies": { 668 | "node-domexception": "1.0.0", 669 | "web-streams-polyfill": "4.0.0-beta.3" 670 | }, 671 | "engines": { 672 | "node": ">= 12.20" 673 | } 674 | }, 675 | "node_modules/fsevents": { 676 | "version": "2.3.2", 677 | "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", 678 | "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", 679 | "hasInstallScript": true, 680 | "license": "MIT", 681 | "optional": true, 682 | "os": [ 683 | "darwin" 684 | ], 685 | "engines": { 686 | "node": "^8.16.0 || ^10.6.0 || >=11.0.0" 687 | } 688 | }, 689 | "node_modules/htmlparser2": { 690 | "version": "9.1.0", 691 | "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", 692 | "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", 693 | "funding": [ 694 | "https://github.com/fb55/htmlparser2?sponsor=1", 695 | { 696 | "type": "github", 697 | "url": "https://github.com/sponsors/fb55" 698 | } 699 | ], 700 | "license": "MIT", 701 | "dependencies": { 702 | "domelementtype": "^2.3.0", 703 | "domhandler": "^5.0.3", 704 | "domutils": "^3.1.0", 705 | "entities": "^4.5.0" 706 | } 707 | }, 708 | "node_modules/http-errors": { 709 | "version": "2.0.0", 710 | "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", 711 | "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", 712 | "license": "MIT", 713 | "dependencies": { 714 | "depd": "2.0.0", 715 | "inherits": "2.0.4", 716 | "setprototypeof": "1.2.0", 717 | "statuses": "2.0.1", 718 | "toidentifier": "1.0.1" 719 | }, 720 | "engines": { 721 | "node": ">= 0.8" 722 | } 723 | }, 724 | "node_modules/http-proxy-agent": { 725 | "version": "7.0.2", 726 | "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", 727 | "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", 728 | "license": "MIT", 729 | "dependencies": { 730 | "agent-base": "^7.1.0", 731 | "debug": "^4.3.4" 732 | }, 733 | "engines": { 734 | "node": ">= 14" 735 | } 736 | }, 737 | "node_modules/https-proxy-agent": { 738 | "version": "7.0.6", 739 | "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", 740 | "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", 741 | "license": "MIT", 742 | "dependencies": { 743 | "agent-base": "^7.1.2", 744 | "debug": "4" 745 | }, 746 | "engines": { 747 | "node": ">= 14" 748 | } 749 | }, 750 | "node_modules/humanize-ms": { 751 | "version": "1.2.1", 752 | "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", 753 | "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", 754 | "license": "MIT", 755 | "dependencies": { 756 | "ms": "^2.0.0" 757 | } 758 | }, 759 | "node_modules/iconv-lite": { 760 | "version": "0.6.3", 761 | "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", 762 | "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", 763 | "license": "MIT", 764 | "dependencies": { 765 | "safer-buffer": ">= 2.1.2 < 3.0.0" 766 | }, 767 | "engines": { 768 | "node": ">=0.10.0" 769 | } 770 | }, 771 | "node_modules/inherits": { 772 | "version": "2.0.4", 773 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 774 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", 775 | "license": "ISC" 776 | }, 777 | "node_modules/make-error": { 778 | "version": "1.3.6", 779 | "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", 780 | "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", 781 | "dev": true, 782 | "license": "ISC" 783 | }, 784 | "node_modules/mime-db": { 785 | "version": "1.52.0", 786 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", 787 | "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", 788 | "license": "MIT", 789 | "engines": { 790 | "node": ">= 0.6" 791 | } 792 | }, 793 | "node_modules/mime-types": { 794 | "version": "2.1.35", 795 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", 796 | "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", 797 | "license": "MIT", 798 | "dependencies": { 799 | "mime-db": "1.52.0" 800 | }, 801 | "engines": { 802 | "node": ">= 0.6" 803 | } 804 | }, 805 | "node_modules/ms": { 806 | "version": "2.1.3", 807 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", 808 | "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", 809 | "license": "MIT" 810 | }, 811 | "node_modules/node-domexception": { 812 | "version": "1.0.0", 813 | "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", 814 | "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", 815 | "funding": [ 816 | { 817 | "type": "github", 818 | "url": "https://github.com/sponsors/jimmywarting" 819 | }, 820 | { 821 | "type": "github", 822 | "url": "https://paypal.me/jimmywarting" 823 | } 824 | ], 825 | "license": "MIT", 826 | "engines": { 827 | "node": ">=10.5.0" 828 | } 829 | }, 830 | "node_modules/node-fetch": { 831 | "version": "2.7.0", 832 | "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", 833 | "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", 834 | "license": "MIT", 835 | "dependencies": { 836 | "whatwg-url": "^5.0.0" 837 | }, 838 | "engines": { 839 | "node": "4.x || >=6.0.0" 840 | }, 841 | "peerDependencies": { 842 | "encoding": "^0.1.0" 843 | }, 844 | "peerDependenciesMeta": { 845 | "encoding": { 846 | "optional": true 847 | } 848 | } 849 | }, 850 | "node_modules/nth-check": { 851 | "version": "2.1.1", 852 | "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", 853 | "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", 854 | "license": "BSD-2-Clause", 855 | "dependencies": { 856 | "boolbase": "^1.0.0" 857 | }, 858 | "funding": { 859 | "url": "https://github.com/fb55/nth-check?sponsor=1" 860 | } 861 | }, 862 | "node_modules/openai": { 863 | "version": "4.76.2", 864 | "resolved": "https://registry.npmjs.org/openai/-/openai-4.76.2.tgz", 865 | "integrity": "sha512-T9ZyxAFwLNZz3onC+SFvSR0POF18egIsY8lLze9e2YBe1wzQNf8IHcIgFPWizGPpoCGv/9i3IdTAx3EnLmTL4A==", 866 | "license": "Apache-2.0", 867 | "dependencies": { 868 | "@types/node": "^18.11.18", 869 | "@types/node-fetch": "^2.6.4", 870 | "abort-controller": "^3.0.0", 871 | "agentkeepalive": "^4.2.1", 872 | "form-data-encoder": "1.7.2", 873 | "formdata-node": "^4.3.2", 874 | "node-fetch": "^2.6.7" 875 | }, 876 | "bin": { 877 | "openai": "bin/cli" 878 | }, 879 | "peerDependencies": { 880 | "zod": "^3.23.8" 881 | }, 882 | "peerDependenciesMeta": { 883 | "zod": { 884 | "optional": true 885 | } 886 | } 887 | }, 888 | "node_modules/openai/node_modules/@types/node": { 889 | "version": "18.19.68", 890 | "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.68.tgz", 891 | "integrity": "sha512-QGtpFH1vB99ZmTa63K4/FU8twThj4fuVSBkGddTp7uIL/cuoLWIUSL2RcOaigBhfR+hg5pgGkBnkoOxrTVBMKw==", 892 | "license": "MIT", 893 | "dependencies": { 894 | "undici-types": "~5.26.4" 895 | } 896 | }, 897 | "node_modules/openai/node_modules/undici-types": { 898 | "version": "5.26.5", 899 | "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", 900 | "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", 901 | "license": "MIT" 902 | }, 903 | "node_modules/parse5": { 904 | "version": "7.2.1", 905 | "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz", 906 | "integrity": "sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==", 907 | "license": "MIT", 908 | "dependencies": { 909 | "entities": "^4.5.0" 910 | }, 911 | "funding": { 912 | "url": "https://github.com/inikulin/parse5?sponsor=1" 913 | } 914 | }, 915 | "node_modules/parse5-htmlparser2-tree-adapter": { 916 | "version": "7.1.0", 917 | "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", 918 | "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", 919 | "license": "MIT", 920 | "dependencies": { 921 | "domhandler": "^5.0.3", 922 | "parse5": "^7.0.0" 923 | }, 924 | "funding": { 925 | "url": "https://github.com/inikulin/parse5?sponsor=1" 926 | } 927 | }, 928 | "node_modules/parse5-parser-stream": { 929 | "version": "7.1.2", 930 | "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", 931 | "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", 932 | "license": "MIT", 933 | "dependencies": { 934 | "parse5": "^7.0.0" 935 | }, 936 | "funding": { 937 | "url": "https://github.com/inikulin/parse5?sponsor=1" 938 | } 939 | }, 940 | "node_modules/playwright": { 941 | "version": "1.49.1", 942 | "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.49.1.tgz", 943 | "integrity": "sha512-VYL8zLoNTBxVOrJBbDuRgDWa3i+mfQgDTrL8Ah9QXZ7ax4Dsj0MSq5bYgytRnDVVe+njoKnfsYkH3HzqVj5UZA==", 944 | "license": "Apache-2.0", 945 | "dependencies": { 946 | "playwright-core": "1.49.1" 947 | }, 948 | "bin": { 949 | "playwright": "cli.js" 950 | }, 951 | "engines": { 952 | "node": ">=18" 953 | }, 954 | "optionalDependencies": { 955 | "fsevents": "2.3.2" 956 | } 957 | }, 958 | "node_modules/playwright-core": { 959 | "version": "1.49.1", 960 | "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.49.1.tgz", 961 | "integrity": "sha512-BzmpVcs4kE2CH15rWfzpjzVGhWERJfmnXmniSyKeRZUs9Ws65m+RGIi7mjJK/euCegfn3i7jvqWeWyHe9y3Vgg==", 962 | "license": "Apache-2.0", 963 | "bin": { 964 | "playwright-core": "cli.js" 965 | }, 966 | "engines": { 967 | "node": ">=18" 968 | } 969 | }, 970 | "node_modules/proxy-from-env": { 971 | "version": "1.1.0", 972 | "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", 973 | "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", 974 | "license": "MIT" 975 | }, 976 | "node_modules/raw-body": { 977 | "version": "3.0.0", 978 | "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.0.tgz", 979 | "integrity": "sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g==", 980 | "license": "MIT", 981 | "dependencies": { 982 | "bytes": "3.1.2", 983 | "http-errors": "2.0.0", 984 | "iconv-lite": "0.6.3", 985 | "unpipe": "1.0.0" 986 | }, 987 | "engines": { 988 | "node": ">= 0.8" 989 | } 990 | }, 991 | "node_modules/safer-buffer": { 992 | "version": "2.1.2", 993 | "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", 994 | "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", 995 | "license": "MIT" 996 | }, 997 | "node_modules/setprototypeof": { 998 | "version": "1.2.0", 999 | "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", 1000 | "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", 1001 | "license": "ISC" 1002 | }, 1003 | "node_modules/statuses": { 1004 | "version": "2.0.1", 1005 | "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", 1006 | "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", 1007 | "license": "MIT", 1008 | "engines": { 1009 | "node": ">= 0.8" 1010 | } 1011 | }, 1012 | "node_modules/toidentifier": { 1013 | "version": "1.0.1", 1014 | "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", 1015 | "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", 1016 | "license": "MIT", 1017 | "engines": { 1018 | "node": ">=0.6" 1019 | } 1020 | }, 1021 | "node_modules/tr46": { 1022 | "version": "0.0.3", 1023 | "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", 1024 | "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", 1025 | "license": "MIT" 1026 | }, 1027 | "node_modules/ts-node": { 1028 | "version": "10.9.2", 1029 | "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", 1030 | "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", 1031 | "dev": true, 1032 | "license": "MIT", 1033 | "dependencies": { 1034 | "@cspotcode/source-map-support": "^0.8.0", 1035 | "@tsconfig/node10": "^1.0.7", 1036 | "@tsconfig/node12": "^1.0.7", 1037 | "@tsconfig/node14": "^1.0.0", 1038 | "@tsconfig/node16": "^1.0.2", 1039 | "acorn": "^8.4.1", 1040 | "acorn-walk": "^8.1.1", 1041 | "arg": "^4.1.0", 1042 | "create-require": "^1.1.0", 1043 | "diff": "^4.0.1", 1044 | "make-error": "^1.1.1", 1045 | "v8-compile-cache-lib": "^3.0.1", 1046 | "yn": "3.1.1" 1047 | }, 1048 | "bin": { 1049 | "ts-node": "dist/bin.js", 1050 | "ts-node-cwd": "dist/bin-cwd.js", 1051 | "ts-node-esm": "dist/bin-esm.js", 1052 | "ts-node-script": "dist/bin-script.js", 1053 | "ts-node-transpile-only": "dist/bin-transpile.js", 1054 | "ts-script": "dist/bin-script-deprecated.js" 1055 | }, 1056 | "peerDependencies": { 1057 | "@swc/core": ">=1.2.50", 1058 | "@swc/wasm": ">=1.2.50", 1059 | "@types/node": "*", 1060 | "typescript": ">=2.7" 1061 | }, 1062 | "peerDependenciesMeta": { 1063 | "@swc/core": { 1064 | "optional": true 1065 | }, 1066 | "@swc/wasm": { 1067 | "optional": true 1068 | } 1069 | } 1070 | }, 1071 | "node_modules/tslib": { 1072 | "version": "2.8.1", 1073 | "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", 1074 | "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", 1075 | "license": "0BSD" 1076 | }, 1077 | "node_modules/typescript": { 1078 | "version": "5.7.2", 1079 | "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.2.tgz", 1080 | "integrity": "sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg==", 1081 | "license": "Apache-2.0", 1082 | "bin": { 1083 | "tsc": "bin/tsc", 1084 | "tsserver": "bin/tsserver" 1085 | }, 1086 | "engines": { 1087 | "node": ">=14.17" 1088 | } 1089 | }, 1090 | "node_modules/undici": { 1091 | "version": "5.28.4", 1092 | "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz", 1093 | "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==", 1094 | "license": "MIT", 1095 | "dependencies": { 1096 | "@fastify/busboy": "^2.0.0" 1097 | }, 1098 | "engines": { 1099 | "node": ">=14.0" 1100 | } 1101 | }, 1102 | "node_modules/undici-types": { 1103 | "version": "6.19.8", 1104 | "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", 1105 | "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", 1106 | "license": "MIT" 1107 | }, 1108 | "node_modules/unpipe": { 1109 | "version": "1.0.0", 1110 | "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", 1111 | "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", 1112 | "license": "MIT", 1113 | "engines": { 1114 | "node": ">= 0.8" 1115 | } 1116 | }, 1117 | "node_modules/v8-compile-cache-lib": { 1118 | "version": "3.0.1", 1119 | "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", 1120 | "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", 1121 | "dev": true, 1122 | "license": "MIT" 1123 | }, 1124 | "node_modules/web-streams-polyfill": { 1125 | "version": "4.0.0-beta.3", 1126 | "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", 1127 | "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", 1128 | "license": "MIT", 1129 | "engines": { 1130 | "node": ">= 14" 1131 | } 1132 | }, 1133 | "node_modules/webidl-conversions": { 1134 | "version": "3.0.1", 1135 | "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", 1136 | "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", 1137 | "license": "BSD-2-Clause" 1138 | }, 1139 | "node_modules/whatwg-encoding": { 1140 | "version": "3.1.1", 1141 | "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", 1142 | "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", 1143 | "license": "MIT", 1144 | "dependencies": { 1145 | "iconv-lite": "0.6.3" 1146 | }, 1147 | "engines": { 1148 | "node": ">=18" 1149 | } 1150 | }, 1151 | "node_modules/whatwg-mimetype": { 1152 | "version": "4.0.0", 1153 | "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", 1154 | "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", 1155 | "license": "MIT", 1156 | "engines": { 1157 | "node": ">=18" 1158 | } 1159 | }, 1160 | "node_modules/whatwg-url": { 1161 | "version": "5.0.0", 1162 | "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", 1163 | "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", 1164 | "license": "MIT", 1165 | "dependencies": { 1166 | "tr46": "~0.0.3", 1167 | "webidl-conversions": "^3.0.0" 1168 | } 1169 | }, 1170 | "node_modules/yn": { 1171 | "version": "3.1.1", 1172 | "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", 1173 | "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", 1174 | "dev": true, 1175 | "license": "MIT", 1176 | "engines": { 1177 | "node": ">=6" 1178 | } 1179 | }, 1180 | "node_modules/zod": { 1181 | "version": "3.24.1", 1182 | "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.1.tgz", 1183 | "integrity": "sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A==", 1184 | "license": "MIT", 1185 | "funding": { 1186 | "url": "https://github.com/sponsors/colinhacks" 1187 | } 1188 | } 1189 | } 1190 | } 1191 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@hannesrudolph/mcp-ragdocs", 3 | "version": "1.1.0", 4 | "description": "An MCP server for semantic documentation search and retrieval using vector databases to augment LLM capabilities.", 5 | "private": false, 6 | "type": "module", 7 | "bin": { 8 | "@hannesrudolph/mcp-ragdocs": "./build/index.js" 9 | }, 10 | "files": [ 11 | "build", 12 | "README.md", 13 | "LICENSE" 14 | ], 15 | "scripts": { 16 | "build": "tsc && node -e \"require('fs').chmodSync('build/index.js', '755')\"", 17 | "prepare": "npm run build", 18 | "watch": "tsc --watch", 19 | "inspector": "npx @modelcontextprotocol/inspector build/index.js", 20 | "start": "node build/index.js" 21 | }, 22 | "keywords": [ 23 | "mcp", 24 | "model-context-protocol", 25 | "rag", 26 | "documentation", 27 | "vector-database", 28 | "qdrant", 29 | "claude", 30 | "llm" 31 | ], 32 | "author": "hannesrudolph", 33 | "license": "MIT", 34 | "repository": { 35 | "type": "git", 36 | "url": "git+https://github.com/hannesrudolph/mcp-ragdocs.git" 37 | }, 38 | "bugs": { 39 | "url": "https://github.com/hannesrudolph/mcp-ragdocs/issues" 40 | }, 41 | "homepage": "https://github.com/hannesrudolph/mcp-ragdocs#readme", 42 | "dependencies": { 43 | "@azure/openai": "2.0.0", 44 | "@modelcontextprotocol/sdk": "1.0.3", 45 | "@qdrant/js-client-rest": "1.12.0", 46 | "axios": "1.7.9", 47 | "cheerio": "1.0.0", 48 | "openai": "4.76.2", 49 | "playwright": "1.49.1" 50 | }, 51 | "devDependencies": { 52 | "@types/node": "^20.17.10", 53 | "ts-node": "^10.9.2", 54 | "typescript": "^5.7.2" 55 | }, 56 | "publishConfig": { 57 | "access": "public" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/api-client.ts: -------------------------------------------------------------------------------- 1 | import { QdrantClient } from '@qdrant/js-client-rest'; 2 | import OpenAI from 'openai'; 3 | import { chromium } from 'playwright'; 4 | import { McpError, ErrorCode } from '@modelcontextprotocol/sdk/types.js'; 5 | 6 | // Environment variables for configuration 7 | const OPENAI_API_KEY = process.env.OPENAI_API_KEY; 8 | const QDRANT_URL = process.env.QDRANT_URL; 9 | const QDRANT_API_KEY = process.env.QDRANT_API_KEY; 10 | 11 | if (!QDRANT_URL) { 12 | throw new Error('QDRANT_URL environment variable is required for cloud storage'); 13 | } 14 | 15 | if (!QDRANT_API_KEY) { 16 | throw new Error('QDRANT_API_KEY environment variable is required for cloud storage'); 17 | } 18 | 19 | export class ApiClient { 20 | qdrantClient: QdrantClient; 21 | openaiClient?: OpenAI; 22 | browser: any; 23 | 24 | constructor() { 25 | // Initialize Qdrant client with cloud configuration 26 | this.qdrantClient = new QdrantClient({ 27 | url: QDRANT_URL, 28 | apiKey: QDRANT_API_KEY, 29 | }); 30 | 31 | // Initialize OpenAI client if API key is provided 32 | if (OPENAI_API_KEY) { 33 | this.openaiClient = new OpenAI({ 34 | apiKey: OPENAI_API_KEY, 35 | }); 36 | } 37 | } 38 | 39 | async initBrowser() { 40 | if (!this.browser) { 41 | this.browser = await chromium.launch(); 42 | } 43 | } 44 | 45 | async cleanup() { 46 | if (this.browser) { 47 | await this.browser.close(); 48 | } 49 | } 50 | 51 | async getEmbeddings(text: string): Promise { 52 | if (!this.openaiClient) { 53 | throw new McpError( 54 | ErrorCode.InvalidRequest, 55 | 'OpenAI API key not configured' 56 | ); 57 | } 58 | 59 | try { 60 | const response = await this.openaiClient.embeddings.create({ 61 | model: 'text-embedding-ada-002', 62 | input: text, 63 | }); 64 | return response.data[0].embedding; 65 | } catch (error) { 66 | throw new McpError( 67 | ErrorCode.InternalError, 68 | `Failed to generate embeddings: ${error}` 69 | ); 70 | } 71 | } 72 | 73 | async initCollection(COLLECTION_NAME: string) { 74 | try { 75 | const collections = await this.qdrantClient.getCollections(); 76 | const exists = collections.collections.some(c => c.name === COLLECTION_NAME); 77 | 78 | if (!exists) { 79 | await this.qdrantClient.createCollection(COLLECTION_NAME, { 80 | vectors: { 81 | size: 1536, // OpenAI ada-002 embedding size 82 | distance: 'Cosine', 83 | }, 84 | // Add optimized settings for cloud deployment 85 | optimizers_config: { 86 | default_segment_number: 2, 87 | memmap_threshold: 20000, 88 | }, 89 | replication_factor: 2, 90 | }); 91 | } 92 | } catch (error) { 93 | if (error instanceof Error) { 94 | if (error.message.includes('unauthorized')) { 95 | throw new McpError( 96 | ErrorCode.InvalidRequest, 97 | 'Failed to authenticate with Qdrant cloud. Please check your API key.' 98 | ); 99 | } else if (error.message.includes('ECONNREFUSED') || error.message.includes('ETIMEDOUT')) { 100 | throw new McpError( 101 | ErrorCode.InternalError, 102 | 'Failed to connect to Qdrant cloud. Please check your QDRANT_URL.' 103 | ); 104 | } 105 | } 106 | throw new McpError( 107 | ErrorCode.InternalError, 108 | `Failed to initialize Qdrant cloud collection: ${error}` 109 | ); 110 | } 111 | } 112 | } -------------------------------------------------------------------------------- /src/handler-registry.ts: -------------------------------------------------------------------------------- 1 | import { 2 | CallToolRequestSchema, 3 | ErrorCode, 4 | ListToolsRequestSchema, 5 | McpError, 6 | } from '@modelcontextprotocol/sdk/types.js'; 7 | import { Server } from '@modelcontextprotocol/sdk/server/index.js'; 8 | import { ApiClient } from './api-client.js'; 9 | import { ToolDefinition } from './types.js'; 10 | import { 11 | AddDocumentationHandler, 12 | SearchDocumentationHandler, 13 | ListSourcesHandler, 14 | RemoveDocumentationHandler, 15 | ExtractUrlsHandler, 16 | ListQueueHandler, 17 | RunQueueHandler, 18 | ClearQueueHandler, 19 | } from './handlers/index.js'; 20 | 21 | const COLLECTION_NAME = 'documentation'; 22 | 23 | export class HandlerRegistry { 24 | private server: Server; 25 | private apiClient: ApiClient; 26 | private handlers: Map; 27 | 28 | constructor(server: Server, apiClient: ApiClient) { 29 | this.server = server; 30 | this.apiClient = apiClient; 31 | this.handlers = new Map(); 32 | this.setupHandlers(); 33 | this.registerHandlers(); 34 | } 35 | 36 | private setupHandlers() { 37 | this.handlers.set('add_documentation', new AddDocumentationHandler(this.server, this.apiClient)); 38 | this.handlers.set('search_documentation', new SearchDocumentationHandler(this.server, this.apiClient)); 39 | this.handlers.set('list_sources', new ListSourcesHandler(this.server, this.apiClient)); 40 | this.handlers.set('remove_documentation', new RemoveDocumentationHandler(this.server, this.apiClient)); 41 | this.handlers.set('extract_urls', new ExtractUrlsHandler(this.server, this.apiClient)); 42 | this.handlers.set('list_queue', new ListQueueHandler(this.server, this.apiClient)); 43 | this.handlers.set('run_queue', new RunQueueHandler(this.server, this.apiClient)); 44 | this.handlers.set('clear_queue', new ClearQueueHandler(this.server, this.apiClient)); 45 | } 46 | 47 | private registerHandlers() { 48 | this.server.setRequestHandler(ListToolsRequestSchema, async () => ({ 49 | tools: [ 50 | { 51 | name: 'search_documentation', 52 | description: 'Search through stored documentation using natural language queries. Use this tool to find relevant information across all stored documentation sources. Returns matching excerpts with context, ranked by relevance. Useful for finding specific information, code examples, or related documentation.', 53 | inputSchema: { 54 | type: 'object', 55 | properties: { 56 | query: { 57 | type: 'string', 58 | description: 'The text to search for in the documentation. Can be a natural language query, specific terms, or code snippets.', 59 | }, 60 | limit: { 61 | type: 'number', 62 | description: 'Maximum number of results to return (1-20). Higher limits provide more comprehensive results but may take longer to process. Default is 5.', 63 | default: 5, 64 | }, 65 | }, 66 | required: ['query'], 67 | }, 68 | } as ToolDefinition, 69 | { 70 | name: 'list_sources', 71 | description: 'List all documentation sources currently stored in the system. Returns a comprehensive list of all indexed documentation including source URLs, titles, and last update times. Use this to understand what documentation is available for searching or to verify if specific sources have been indexed.', 72 | inputSchema: { 73 | type: 'object', 74 | properties: {}, 75 | }, 76 | } as ToolDefinition, 77 | { 78 | name: 'extract_urls', 79 | description: 'Extract and analyze all URLs from a given web page. This tool crawls the specified webpage, identifies all hyperlinks, and optionally adds them to the processing queue. Useful for discovering related documentation pages, API references, or building a documentation graph. Handles various URL formats and validates links before extraction.', 80 | inputSchema: { 81 | type: 'object', 82 | properties: { 83 | url: { 84 | type: 'string', 85 | description: 'The complete URL of the webpage to analyze (must include protocol, e.g., https://). The page must be publicly accessible.', 86 | }, 87 | add_to_queue: { 88 | type: 'boolean', 89 | description: 'If true, automatically add extracted URLs to the processing queue for later indexing. This enables recursive documentation discovery. Use with caution on large sites to avoid excessive queuing.', 90 | default: false, 91 | }, 92 | }, 93 | required: ['url'], 94 | }, 95 | } as ToolDefinition, 96 | { 97 | name: 'remove_documentation', 98 | description: 'Remove specific documentation sources from the system by their URLs. Use this tool to clean up outdated documentation, remove incorrect sources, or manage the documentation collection. The removal is permanent and will affect future search results. Supports removing multiple URLs in a single operation.', 99 | inputSchema: { 100 | type: 'object', 101 | properties: { 102 | urls: { 103 | type: 'array', 104 | items: { 105 | type: 'string', 106 | description: 'The complete URL of the documentation source to remove. Must exactly match the URL used when the documentation was added.', 107 | }, 108 | description: 'Array of URLs to remove from the database', 109 | }, 110 | }, 111 | required: ['urls'], 112 | }, 113 | } as ToolDefinition, 114 | { 115 | name: 'list_queue', 116 | description: 'List all URLs currently waiting in the documentation processing queue. Shows pending documentation sources that will be processed when run_queue is called. Use this to monitor queue status, verify URLs were added correctly, or check processing backlog. Returns URLs in the order they will be processed.', 117 | inputSchema: { 118 | type: 'object', 119 | properties: {}, 120 | }, 121 | } as ToolDefinition, 122 | { 123 | name: 'run_queue', 124 | description: 'Process and index all URLs currently in the documentation queue. Each URL is processed sequentially, with proper error handling and retry logic. Progress updates are provided as processing occurs. Use this after adding new URLs to ensure all documentation is indexed and searchable. Long-running operations will process until the queue is empty or an unrecoverable error occurs.', 125 | inputSchema: { 126 | type: 'object', 127 | properties: {}, 128 | }, 129 | } as ToolDefinition, 130 | { 131 | name: 'clear_queue', 132 | description: 'Remove all pending URLs from the documentation processing queue. Use this to reset the queue when you want to start fresh, remove unwanted URLs, or cancel pending processing. This operation is immediate and permanent - URLs will need to be re-added if you want to process them later. Returns the number of URLs that were cleared from the queue.', 133 | inputSchema: { 134 | type: 'object', 135 | properties: {}, 136 | }, 137 | } as ToolDefinition, 138 | ], 139 | })); 140 | 141 | this.server.setRequestHandler(CallToolRequestSchema, async (request) => { 142 | await this.apiClient.initCollection(COLLECTION_NAME); 143 | 144 | const handler = this.handlers.get(request.params.name); 145 | if (!handler) { 146 | throw new McpError( 147 | ErrorCode.MethodNotFound, 148 | `Unknown tool: ${request.params.name}` 149 | ); 150 | } 151 | 152 | const response = await handler.handle(request.params.arguments); 153 | return { 154 | _meta: {}, 155 | ...response 156 | }; 157 | }); 158 | } 159 | } -------------------------------------------------------------------------------- /src/handlers/add-documentation.ts: -------------------------------------------------------------------------------- 1 | import { McpError, ErrorCode } from '@modelcontextprotocol/sdk/types.js'; 2 | import { BaseHandler } from './base-handler.js'; 3 | import { DocumentChunk, McpToolResponse } from '../types.js'; 4 | import * as cheerio from 'cheerio'; 5 | import crypto from 'crypto'; 6 | 7 | const COLLECTION_NAME = 'documentation'; 8 | 9 | export class AddDocumentationHandler extends BaseHandler { 10 | async handle(args: any): Promise { 11 | if (!args.url || typeof args.url !== 'string') { 12 | throw new McpError(ErrorCode.InvalidParams, 'URL is required'); 13 | } 14 | 15 | try { 16 | const chunks = await this.fetchAndProcessUrl(args.url); 17 | 18 | // Batch process chunks for better performance 19 | const batchSize = 100; 20 | for (let i = 0; i < chunks.length; i += batchSize) { 21 | const batch = chunks.slice(i, i + batchSize); 22 | const points = await Promise.all( 23 | batch.map(async (chunk) => { 24 | const embedding = await this.apiClient.getEmbeddings(chunk.text); 25 | return { 26 | id: this.generatePointId(), 27 | vector: embedding, 28 | payload: { 29 | ...chunk, 30 | _type: 'DocumentChunk' as const, 31 | } as Record, 32 | }; 33 | }) 34 | ); 35 | 36 | try { 37 | await this.apiClient.qdrantClient.upsert(COLLECTION_NAME, { 38 | wait: true, 39 | points, 40 | }); 41 | } catch (error) { 42 | if (error instanceof Error) { 43 | if (error.message.includes('unauthorized')) { 44 | throw new McpError( 45 | ErrorCode.InvalidRequest, 46 | 'Failed to authenticate with Qdrant cloud while adding documents' 47 | ); 48 | } else if (error.message.includes('ECONNREFUSED') || error.message.includes('ETIMEDOUT')) { 49 | throw new McpError( 50 | ErrorCode.InternalError, 51 | 'Connection to Qdrant cloud failed while adding documents' 52 | ); 53 | } 54 | } 55 | throw error; 56 | } 57 | } 58 | 59 | return { 60 | content: [ 61 | { 62 | type: 'text', 63 | text: `Successfully added documentation from ${args.url} (${chunks.length} chunks processed in ${Math.ceil(chunks.length / batchSize)} batches)`, 64 | }, 65 | ], 66 | }; 67 | } catch (error) { 68 | if (error instanceof McpError) { 69 | throw error; 70 | } 71 | return { 72 | content: [ 73 | { 74 | type: 'text', 75 | text: `Failed to add documentation: ${error}`, 76 | }, 77 | ], 78 | isError: true, 79 | }; 80 | } 81 | } 82 | 83 | private async fetchAndProcessUrl(url: string): Promise { 84 | await this.apiClient.initBrowser(); 85 | const page = await this.apiClient.browser.newPage(); 86 | 87 | try { 88 | await page.goto(url, { waitUntil: 'networkidle' }); 89 | const content = await page.content(); 90 | const $ = cheerio.load(content); 91 | 92 | // Remove script tags, style tags, and comments 93 | $('script').remove(); 94 | $('style').remove(); 95 | $('noscript').remove(); 96 | 97 | // Extract main content 98 | const title = $('title').text() || url; 99 | const mainContent = $('main, article, .content, .documentation, body').text(); 100 | 101 | // Split content into chunks 102 | const chunks = this.chunkText(mainContent, 1000); 103 | 104 | return chunks.map(chunk => ({ 105 | text: chunk, 106 | url, 107 | title, 108 | timestamp: new Date().toISOString(), 109 | })); 110 | } catch (error) { 111 | throw new McpError( 112 | ErrorCode.InternalError, 113 | `Failed to fetch URL ${url}: ${error}` 114 | ); 115 | } finally { 116 | await page.close(); 117 | } 118 | } 119 | 120 | private chunkText(text: string, maxChunkSize: number): string[] { 121 | const words = text.split(/\s+/); 122 | const chunks: string[] = []; 123 | let currentChunk: string[] = []; 124 | 125 | for (const word of words) { 126 | currentChunk.push(word); 127 | const currentLength = currentChunk.join(' ').length; 128 | 129 | if (currentLength >= maxChunkSize) { 130 | chunks.push(currentChunk.join(' ')); 131 | currentChunk = []; 132 | } 133 | } 134 | 135 | if (currentChunk.length > 0) { 136 | chunks.push(currentChunk.join(' ')); 137 | } 138 | 139 | return chunks; 140 | } 141 | 142 | private generatePointId(): string { 143 | return crypto.randomBytes(16).toString('hex'); 144 | } 145 | } -------------------------------------------------------------------------------- /src/handlers/base-handler.ts: -------------------------------------------------------------------------------- 1 | import { Server } from '@modelcontextprotocol/sdk/server/index.js'; 2 | import { ApiClient } from '../api-client.js'; 3 | import { McpToolResponse } from '../types.js'; 4 | 5 | export abstract class BaseHandler { 6 | protected server: Server; 7 | protected apiClient: ApiClient; 8 | 9 | constructor(server: Server, apiClient: ApiClient) { 10 | this.server = server; 11 | this.apiClient = apiClient; 12 | } 13 | 14 | protected abstract handle(args: any): Promise; 15 | } -------------------------------------------------------------------------------- /src/handlers/clear-queue.ts: -------------------------------------------------------------------------------- 1 | import { Server } from '@modelcontextprotocol/sdk/server/index.js'; 2 | import { ApiClient } from '../api-client.js'; 3 | import { ClearQueueTool } from '../tools/clear-queue.js'; 4 | 5 | export class ClearQueueHandler extends ClearQueueTool { 6 | constructor(server: Server, apiClient: ApiClient) { 7 | super(); 8 | } 9 | 10 | async handle(args: any) { 11 | return this.execute(args); 12 | } 13 | } -------------------------------------------------------------------------------- /src/handlers/extract-urls.ts: -------------------------------------------------------------------------------- 1 | import { McpError, ErrorCode } from '@modelcontextprotocol/sdk/types.js'; 2 | import { BaseHandler } from './base-handler.js'; 3 | import { McpToolResponse } from '../types.js'; 4 | import * as cheerio from 'cheerio'; 5 | import fs from 'fs/promises'; 6 | import path from 'path'; 7 | import { fileURLToPath } from 'url'; 8 | 9 | // Get current directory in ES modules 10 | const __filename = fileURLToPath(import.meta.url); 11 | const __dirname = path.dirname(__filename); 12 | const QUEUE_FILE = path.join(__dirname, '..', '..', 'queue.txt'); 13 | 14 | export class ExtractUrlsHandler extends BaseHandler { 15 | async handle(args: any): Promise { 16 | if (!args.url || typeof args.url !== 'string') { 17 | throw new McpError(ErrorCode.InvalidParams, 'URL is required'); 18 | } 19 | 20 | await this.apiClient.initBrowser(); 21 | const page = await this.apiClient.browser.newPage(); 22 | 23 | try { 24 | const baseUrl = new URL(args.url); 25 | const basePath = baseUrl.pathname.split('/').slice(0, 3).join('/'); // Get the base path (e.g., /3/ for Python docs) 26 | 27 | await page.goto(args.url, { waitUntil: 'networkidle' }); 28 | const content = await page.content(); 29 | const $ = cheerio.load(content); 30 | const urls = new Set(); 31 | 32 | $('a[href]').each((_, element) => { 33 | const href = $(element).attr('href'); 34 | if (href) { 35 | try { 36 | const url = new URL(href, args.url); 37 | // Only include URLs from the same documentation section 38 | if (url.hostname === baseUrl.hostname && 39 | url.pathname.startsWith(basePath) && 40 | !url.hash && 41 | !url.href.endsWith('#')) { 42 | urls.add(url.href); 43 | } 44 | } catch (e) { 45 | // Ignore invalid URLs 46 | } 47 | } 48 | }); 49 | 50 | const urlArray = Array.from(urls); 51 | 52 | if (args.add_to_queue) { 53 | try { 54 | // Ensure queue file exists 55 | try { 56 | await fs.access(QUEUE_FILE); 57 | } catch { 58 | await fs.writeFile(QUEUE_FILE, ''); 59 | } 60 | 61 | // Append URLs to queue 62 | const urlsToAdd = urlArray.join('\n') + (urlArray.length > 0 ? '\n' : ''); 63 | await fs.appendFile(QUEUE_FILE, urlsToAdd); 64 | 65 | return { 66 | content: [ 67 | { 68 | type: 'text', 69 | text: `Successfully added ${urlArray.length} URLs to the queue`, 70 | }, 71 | ], 72 | }; 73 | } catch (error) { 74 | return { 75 | content: [ 76 | { 77 | type: 'text', 78 | text: `Failed to add URLs to queue: ${error}`, 79 | }, 80 | ], 81 | isError: true, 82 | }; 83 | } 84 | } 85 | 86 | return { 87 | content: [ 88 | { 89 | type: 'text', 90 | text: urlArray.join('\n') || 'No URLs found on this page.', 91 | }, 92 | ], 93 | }; 94 | } catch (error) { 95 | return { 96 | content: [ 97 | { 98 | type: 'text', 99 | text: `Failed to extract URLs: ${error}`, 100 | }, 101 | ], 102 | isError: true, 103 | }; 104 | } finally { 105 | await page.close(); 106 | } 107 | } 108 | } -------------------------------------------------------------------------------- /src/handlers/index.ts: -------------------------------------------------------------------------------- 1 | export * from './base-handler.js'; 2 | export * from './add-documentation.js'; 3 | export * from './search-documentation.js'; 4 | export * from './list-sources.js'; 5 | export * from './extract-urls.js'; 6 | export * from './remove-documentation.js'; 7 | export * from './list-queue.js'; 8 | export * from './run-queue.js'; 9 | export * from './clear-queue.js'; -------------------------------------------------------------------------------- /src/handlers/list-queue.ts: -------------------------------------------------------------------------------- 1 | import { Server } from '@modelcontextprotocol/sdk/server/index.js'; 2 | import { ApiClient } from '../api-client.js'; 3 | import { BaseHandler } from './base-handler.js'; 4 | import fs from 'fs/promises'; 5 | import path from 'path'; 6 | import { fileURLToPath } from 'url'; 7 | 8 | // Get current directory in ES modules 9 | const __filename = fileURLToPath(import.meta.url); 10 | const __dirname = path.dirname(__filename); 11 | const QUEUE_FILE = path.join(__dirname, '..', '..', 'queue.txt'); 12 | 13 | export class ListQueueHandler extends BaseHandler { 14 | constructor(server: Server, apiClient: ApiClient) { 15 | super(server, apiClient); 16 | } 17 | 18 | async handle(_args: any) { 19 | try { 20 | // Check if queue file exists 21 | try { 22 | await fs.access(QUEUE_FILE); 23 | } catch { 24 | return { 25 | content: [ 26 | { 27 | type: 'text', 28 | text: 'Queue is empty (queue file does not exist)', 29 | }, 30 | ], 31 | }; 32 | } 33 | 34 | // Read queue file 35 | const content = await fs.readFile(QUEUE_FILE, 'utf-8'); 36 | const urls = content.split('\n').filter(url => url.trim() !== ''); 37 | 38 | if (urls.length === 0) { 39 | return { 40 | content: [ 41 | { 42 | type: 'text', 43 | text: 'Queue is empty', 44 | }, 45 | ], 46 | }; 47 | } 48 | 49 | return { 50 | content: [ 51 | { 52 | type: 'text', 53 | text: `Queue contains ${urls.length} URLs:\n${urls.join('\n')}`, 54 | }, 55 | ], 56 | }; 57 | } catch (error) { 58 | return { 59 | content: [ 60 | { 61 | type: 'text', 62 | text: `Failed to read queue: ${error}`, 63 | }, 64 | ], 65 | isError: true, 66 | }; 67 | } 68 | } 69 | } -------------------------------------------------------------------------------- /src/handlers/list-sources.ts: -------------------------------------------------------------------------------- 1 | import { McpError, ErrorCode } from '@modelcontextprotocol/sdk/types.js'; 2 | import { BaseHandler } from './base-handler.js'; 3 | import { McpToolResponse, isDocumentPayload } from '../types.js'; 4 | 5 | const COLLECTION_NAME = 'documentation'; 6 | 7 | interface Source { 8 | title: string; 9 | url: string; 10 | } 11 | 12 | interface GroupedSources { 13 | [domain: string]: { 14 | [subdomain: string]: Source[]; 15 | }; 16 | } 17 | 18 | export class ListSourcesHandler extends BaseHandler { 19 | private groupSourcesByDomainAndSubdomain(sources: Source[]): GroupedSources { 20 | const grouped: GroupedSources = {}; 21 | 22 | for (const source of sources) { 23 | try { 24 | const url = new URL(source.url); 25 | const domain = url.hostname; 26 | const pathParts = url.pathname.split('/').filter(p => p); 27 | const subdomain = pathParts[0] || '/'; 28 | 29 | if (!grouped[domain]) { 30 | grouped[domain] = {}; 31 | } 32 | if (!grouped[domain][subdomain]) { 33 | grouped[domain][subdomain] = []; 34 | } 35 | grouped[domain][subdomain].push(source); 36 | } catch (error) { 37 | console.error(`Invalid URL: ${source.url}`); 38 | } 39 | } 40 | 41 | return grouped; 42 | } 43 | 44 | private formatGroupedSources(grouped: GroupedSources): string { 45 | const output: string[] = []; 46 | let domainCounter = 1; 47 | 48 | for (const [domain, subdomains] of Object.entries(grouped)) { 49 | output.push(`${domainCounter}. ${domain}`); 50 | 51 | // Create a Set of unique URL+title combinations 52 | const uniqueSources = new Map(); 53 | for (const sources of Object.values(subdomains)) { 54 | for (const source of sources) { 55 | uniqueSources.set(source.url, source); 56 | } 57 | } 58 | 59 | // Convert to array and sort 60 | const sortedSources = Array.from(uniqueSources.values()) 61 | .sort((a, b) => a.title.localeCompare(b.title)); 62 | 63 | // Use letters for subdomain entries 64 | sortedSources.forEach((source, index) => { 65 | output.push(`${domainCounter}.${index + 1}. ${source.title} (${source.url})`); 66 | }); 67 | 68 | output.push(''); // Add blank line between domains 69 | domainCounter++; 70 | } 71 | 72 | return output.join('\n'); 73 | } 74 | 75 | async handle(): Promise { 76 | try { 77 | await this.apiClient.initCollection(COLLECTION_NAME); 78 | 79 | const pageSize = 100; 80 | let offset = null; 81 | const sources: Source[] = []; 82 | 83 | while (true) { 84 | const scroll = await this.apiClient.qdrantClient.scroll(COLLECTION_NAME, { 85 | with_payload: true, 86 | with_vector: false, 87 | limit: pageSize, 88 | offset, 89 | }); 90 | 91 | if (scroll.points.length === 0) break; 92 | 93 | for (const point of scroll.points) { 94 | if (point.payload && typeof point.payload === 'object' && 'url' in point.payload && 'title' in point.payload) { 95 | const payload = point.payload as any; 96 | sources.push({ 97 | title: payload.title, 98 | url: payload.url 99 | }); 100 | } 101 | } 102 | 103 | if (scroll.points.length < pageSize) break; 104 | offset = scroll.points[scroll.points.length - 1].id; 105 | } 106 | 107 | if (sources.length === 0) { 108 | return { 109 | content: [ 110 | { 111 | type: 'text', 112 | text: 'No documentation sources found.', 113 | }, 114 | ], 115 | }; 116 | } 117 | 118 | const grouped = this.groupSourcesByDomainAndSubdomain(sources); 119 | const formattedOutput = this.formatGroupedSources(grouped); 120 | 121 | return { 122 | content: [ 123 | { 124 | type: 'text', 125 | text: formattedOutput, 126 | }, 127 | ], 128 | }; 129 | } catch (error) { 130 | if (error instanceof Error) { 131 | if (error.message.includes('unauthorized')) { 132 | throw new McpError( 133 | ErrorCode.InvalidRequest, 134 | 'Failed to authenticate with Qdrant cloud while listing sources' 135 | ); 136 | } else if (error.message.includes('ECONNREFUSED') || error.message.includes('ETIMEDOUT')) { 137 | throw new McpError( 138 | ErrorCode.InternalError, 139 | 'Connection to Qdrant cloud failed while listing sources' 140 | ); 141 | } 142 | } 143 | return { 144 | content: [ 145 | { 146 | type: 'text', 147 | text: `Failed to list sources: ${error}`, 148 | }, 149 | ], 150 | isError: true, 151 | }; 152 | } 153 | } 154 | } -------------------------------------------------------------------------------- /src/handlers/remove-documentation.ts: -------------------------------------------------------------------------------- 1 | import { McpError, ErrorCode } from '@modelcontextprotocol/sdk/types.js'; 2 | import { BaseHandler } from './base-handler.js'; 3 | import { McpToolResponse } from '../types.js'; 4 | 5 | const COLLECTION_NAME = 'documentation'; 6 | 7 | export class RemoveDocumentationHandler extends BaseHandler { 8 | async handle(args: any): Promise { 9 | if (!args.urls || !Array.isArray(args.urls) || args.urls.length === 0) { 10 | throw new McpError(ErrorCode.InvalidParams, 'urls must be a non-empty array'); 11 | } 12 | 13 | if (!args.urls.every((url: string) => typeof url === 'string')) { 14 | throw new McpError(ErrorCode.InvalidParams, 'All URLs must be strings'); 15 | } 16 | 17 | try { 18 | // Delete using filter to match any of the provided URLs 19 | const result = await this.apiClient.qdrantClient.delete(COLLECTION_NAME, { 20 | filter: { 21 | should: args.urls.map((url: string) => ({ 22 | key: 'url', 23 | match: { value: url } 24 | })) 25 | }, 26 | wait: true // Ensure deletion is complete before responding 27 | }); 28 | 29 | if (!['acknowledged', 'completed'].includes(result.status)) { 30 | throw new Error('Delete operation failed'); 31 | } 32 | 33 | return { 34 | content: [ 35 | { 36 | type: 'text', 37 | text: `Successfully removed documentation from ${args.urls.length} source${args.urls.length > 1 ? 's' : ''}: ${args.urls.join(', ')}`, 38 | }, 39 | ], 40 | }; 41 | } catch (error) { 42 | if (error instanceof Error) { 43 | if (error.message.includes('unauthorized')) { 44 | throw new McpError( 45 | ErrorCode.InvalidRequest, 46 | 'Failed to authenticate with Qdrant cloud while removing documentation' 47 | ); 48 | } else if (error.message.includes('ECONNREFUSED') || error.message.includes('ETIMEDOUT')) { 49 | throw new McpError( 50 | ErrorCode.InternalError, 51 | 'Connection to Qdrant cloud failed while removing documentation' 52 | ); 53 | } 54 | } 55 | return { 56 | content: [ 57 | { 58 | type: 'text', 59 | text: `Failed to remove documentation: ${error}`, 60 | }, 61 | ], 62 | isError: true, 63 | }; 64 | } 65 | } 66 | } -------------------------------------------------------------------------------- /src/handlers/run-queue.ts: -------------------------------------------------------------------------------- 1 | import { Server } from '@modelcontextprotocol/sdk/server/index.js'; 2 | import { ApiClient } from '../api-client.js'; 3 | import { BaseHandler } from './base-handler.js'; 4 | import { McpToolResponse } from '../types.js'; 5 | import { AddDocumentationHandler } from './add-documentation.js'; 6 | import fs from 'fs/promises'; 7 | import path from 'path'; 8 | import { fileURLToPath } from 'url'; 9 | 10 | // Get current directory in ES modules 11 | const __filename = fileURLToPath(import.meta.url); 12 | const __dirname = path.dirname(__filename); 13 | const QUEUE_FILE = path.join(__dirname, '..', '..', 'queue.txt'); 14 | 15 | export class RunQueueHandler extends BaseHandler { 16 | private addDocHandler: AddDocumentationHandler; 17 | 18 | constructor(server: Server, apiClient: ApiClient) { 19 | super(server, apiClient); 20 | this.addDocHandler = new AddDocumentationHandler(server, apiClient); 21 | } 22 | 23 | async handle(_args: any): Promise { 24 | try { 25 | // Check if queue file exists 26 | try { 27 | await fs.access(QUEUE_FILE); 28 | } catch { 29 | return { 30 | content: [ 31 | { 32 | type: 'text', 33 | text: 'Queue is empty (queue file does not exist)', 34 | }, 35 | ], 36 | }; 37 | } 38 | 39 | let processedCount = 0; 40 | let failedCount = 0; 41 | const failedUrls: string[] = []; 42 | 43 | while (true) { 44 | // Read current queue 45 | const content = await fs.readFile(QUEUE_FILE, 'utf-8'); 46 | const urls = content.split('\n').filter(url => url.trim() !== ''); 47 | 48 | if (urls.length === 0) { 49 | break; // Queue is empty 50 | } 51 | 52 | const currentUrl = urls[0]; // Get first URL 53 | 54 | try { 55 | // Process the URL using add_documentation handler 56 | await this.addDocHandler.handle({ url: currentUrl }); 57 | processedCount++; 58 | } catch (error) { 59 | failedCount++; 60 | failedUrls.push(currentUrl); 61 | console.error(`Failed to process URL ${currentUrl}:`, error); 62 | } 63 | 64 | // Remove the processed URL from queue 65 | const remainingUrls = urls.slice(1); 66 | await fs.writeFile(QUEUE_FILE, remainingUrls.join('\n') + (remainingUrls.length > 0 ? '\n' : '')); 67 | } 68 | 69 | let resultText = `Queue processing complete.\nProcessed: ${processedCount} URLs\nFailed: ${failedCount} URLs`; 70 | if (failedUrls.length > 0) { 71 | resultText += `\n\nFailed URLs:\n${failedUrls.join('\n')}`; 72 | } 73 | 74 | return { 75 | content: [ 76 | { 77 | type: 'text', 78 | text: resultText, 79 | }, 80 | ], 81 | }; 82 | } catch (error) { 83 | return { 84 | content: [ 85 | { 86 | type: 'text', 87 | text: `Failed to process queue: ${error}`, 88 | }, 89 | ], 90 | isError: true, 91 | }; 92 | } 93 | } 94 | } -------------------------------------------------------------------------------- /src/handlers/search-documentation.ts: -------------------------------------------------------------------------------- 1 | import { McpError, ErrorCode } from '@modelcontextprotocol/sdk/types.js'; 2 | import { BaseHandler } from './base-handler.js'; 3 | import { McpToolResponse, isDocumentPayload } from '../types.js'; 4 | 5 | const COLLECTION_NAME = 'documentation'; 6 | 7 | export class SearchDocumentationHandler extends BaseHandler { 8 | async handle(args: any): Promise { 9 | if (!args.query || typeof args.query !== 'string') { 10 | throw new McpError(ErrorCode.InvalidParams, 'Query is required'); 11 | } 12 | 13 | const limit = args.limit || 5; 14 | 15 | try { 16 | const queryEmbedding = await this.apiClient.getEmbeddings(args.query); 17 | 18 | const searchResults = await this.apiClient.qdrantClient.search(COLLECTION_NAME, { 19 | vector: queryEmbedding, 20 | limit, 21 | with_payload: true, 22 | with_vector: false, // Optimize network transfer by not retrieving vectors 23 | score_threshold: 0.7, // Only return relevant results 24 | }); 25 | 26 | const formattedResults = searchResults.map(result => { 27 | if (!isDocumentPayload(result.payload)) { 28 | throw new Error('Invalid payload type'); 29 | } 30 | return `[${result.payload.title}](${result.payload.url})\nScore: ${result.score.toFixed(3)}\nContent: ${result.payload.text}\n`; 31 | }).join('\n---\n'); 32 | 33 | return { 34 | content: [ 35 | { 36 | type: 'text', 37 | text: formattedResults || 'No results found matching the query.', 38 | }, 39 | ], 40 | }; 41 | } catch (error) { 42 | if (error instanceof Error) { 43 | if (error.message.includes('unauthorized')) { 44 | throw new McpError( 45 | ErrorCode.InvalidRequest, 46 | 'Failed to authenticate with Qdrant cloud while searching' 47 | ); 48 | } else if (error.message.includes('ECONNREFUSED') || error.message.includes('ETIMEDOUT')) { 49 | throw new McpError( 50 | ErrorCode.InternalError, 51 | 'Connection to Qdrant cloud failed while searching' 52 | ); 53 | } 54 | } 55 | return { 56 | content: [ 57 | { 58 | type: 'text', 59 | text: `Search failed: ${error}`, 60 | }, 61 | ], 62 | isError: true, 63 | }; 64 | } 65 | } 66 | } -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import { Server } from '@modelcontextprotocol/sdk/server/index.js'; 3 | import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; 4 | import { ApiClient } from './api-client.js'; 5 | import { HandlerRegistry } from './handler-registry.js'; 6 | 7 | class RagDocsServer { 8 | private server: Server; 9 | private apiClient: ApiClient; 10 | private handlerRegistry: HandlerRegistry; 11 | 12 | constructor() { 13 | this.server = new Server( 14 | { 15 | name: 'mcp-ragdocs', 16 | version: '0.1.0', 17 | }, 18 | { 19 | capabilities: { 20 | tools: {}, 21 | }, 22 | } 23 | ); 24 | 25 | this.apiClient = new ApiClient(); 26 | this.handlerRegistry = new HandlerRegistry(this.server, this.apiClient); 27 | 28 | // Error handling 29 | this.server.onerror = (error) => console.error('[MCP Error]', error); 30 | process.on('SIGINT', async () => { 31 | await this.cleanup(); 32 | process.exit(0); 33 | }); 34 | } 35 | 36 | private async cleanup() { 37 | await this.apiClient.cleanup(); 38 | await this.server.close(); 39 | } 40 | 41 | async run() { 42 | const transport = new StdioServerTransport(); 43 | await this.server.connect(transport); 44 | console.error('RAG Docs MCP server running on stdio'); 45 | } 46 | } 47 | 48 | const server = new RagDocsServer(); 49 | server.run().catch(console.error); -------------------------------------------------------------------------------- /src/tools/base-tool.ts: -------------------------------------------------------------------------------- 1 | import { ToolDefinition, McpToolResponse } from '../types.js'; 2 | 3 | export abstract class BaseTool { 4 | abstract get definition(): ToolDefinition; 5 | abstract execute(args: unknown): Promise; 6 | 7 | protected formatResponse(data: unknown): McpToolResponse { 8 | return { 9 | content: [ 10 | { 11 | type: 'text', 12 | text: JSON.stringify(data, null, 2), 13 | }, 14 | ], 15 | }; 16 | } 17 | 18 | protected handleError(error: any): McpToolResponse { 19 | return { 20 | content: [ 21 | { 22 | type: 'text', 23 | text: `Error: ${error}`, 24 | }, 25 | ], 26 | isError: true, 27 | }; 28 | } 29 | } -------------------------------------------------------------------------------- /src/tools/clear-queue.ts: -------------------------------------------------------------------------------- 1 | import { BaseTool } from './base-tool.js'; 2 | import { ToolDefinition, McpToolResponse } from '../types.js'; 3 | import fs from 'fs/promises'; 4 | import path from 'path'; 5 | import { fileURLToPath } from 'url'; 6 | 7 | // Get current directory in ES modules 8 | const __filename = fileURLToPath(import.meta.url); 9 | const __dirname = path.dirname(__filename); 10 | const QUEUE_FILE = path.join(__dirname, '..', '..', 'queue.txt'); 11 | 12 | export class ClearQueueTool extends BaseTool { 13 | get definition(): ToolDefinition { 14 | return { 15 | name: 'clear_queue', 16 | description: 'Clear all URLs from the queue', 17 | inputSchema: { 18 | type: 'object', 19 | properties: {}, 20 | required: [], 21 | }, 22 | }; 23 | } 24 | 25 | async execute(_args: any): Promise { 26 | try { 27 | // Check if queue file exists 28 | try { 29 | await fs.access(QUEUE_FILE); 30 | } catch { 31 | return { 32 | content: [ 33 | { 34 | type: 'text', 35 | text: 'Queue is already empty (queue file does not exist)', 36 | }, 37 | ], 38 | }; 39 | } 40 | 41 | // Read current queue to get count of URLs being cleared 42 | const content = await fs.readFile(QUEUE_FILE, 'utf-8'); 43 | const urlCount = content.split('\n').filter(url => url.trim() !== '').length; 44 | 45 | // Clear the queue by emptying the file 46 | await fs.writeFile(QUEUE_FILE, ''); 47 | 48 | return { 49 | content: [ 50 | { 51 | type: 'text', 52 | text: `Queue cleared successfully. Removed ${urlCount} URL${urlCount === 1 ? '' : 's'} from the queue.`, 53 | }, 54 | ], 55 | }; 56 | } catch (error) { 57 | return { 58 | content: [ 59 | { 60 | type: 'text', 61 | text: `Failed to clear queue: ${error}`, 62 | }, 63 | ], 64 | isError: true, 65 | }; 66 | } 67 | } 68 | } -------------------------------------------------------------------------------- /src/tools/extract-urls.ts: -------------------------------------------------------------------------------- 1 | import { BaseTool } from './base-tool.js'; 2 | import { ToolDefinition, McpToolResponse } from '../types.js'; 3 | import { ApiClient } from '../api-client.js'; 4 | import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'; 5 | import * as cheerio from 'cheerio'; 6 | import fs from 'fs/promises'; 7 | import path from 'path'; 8 | import { fileURLToPath } from 'url'; 9 | 10 | // Get current directory in ES modules 11 | const __filename = fileURLToPath(import.meta.url); 12 | const __dirname = path.dirname(__filename); 13 | const QUEUE_FILE = path.join(__dirname, '..', '..', 'queue.txt'); 14 | 15 | export class ExtractUrlsTool extends BaseTool { 16 | private apiClient: ApiClient; 17 | 18 | constructor(apiClient: ApiClient) { 19 | super(); 20 | this.apiClient = apiClient; 21 | } 22 | 23 | get definition(): ToolDefinition { 24 | return { 25 | name: 'extract_urls', 26 | description: 'Extract all URLs from a given web page', 27 | inputSchema: { 28 | type: 'object', 29 | properties: { 30 | url: { 31 | type: 'string', 32 | description: 'URL of the page to extract URLs from', 33 | }, 34 | add_to_queue: { 35 | type: 'boolean', 36 | description: 'If true, automatically add extracted URLs to the queue', 37 | default: false, 38 | }, 39 | }, 40 | required: ['url'], 41 | }, 42 | }; 43 | } 44 | 45 | async execute(args: any): Promise { 46 | if (!args.url || typeof args.url !== 'string') { 47 | throw new McpError(ErrorCode.InvalidParams, 'URL is required'); 48 | } 49 | 50 | await this.apiClient.initBrowser(); 51 | const page = await this.apiClient.browser.newPage(); 52 | 53 | try { 54 | await page.goto(args.url, { waitUntil: 'networkidle' }); 55 | const content = await page.content(); 56 | const $ = cheerio.load(content); 57 | const urls = new Set(); 58 | 59 | $('a[href]').each((_, element) => { 60 | const href = $(element).attr('href'); 61 | if (href) { 62 | try { 63 | const url = new URL(href, args.url); 64 | // Only include URLs from the same domain to avoid external links 65 | if (url.origin === new URL(args.url).origin && !url.hash && !url.href.endsWith('#')) { 66 | urls.add(url.href); 67 | } 68 | } catch (e) { 69 | // Ignore invalid URLs 70 | } 71 | } 72 | }); 73 | 74 | const urlArray = Array.from(urls); 75 | 76 | if (args.add_to_queue) { 77 | try { 78 | // Ensure queue file exists 79 | try { 80 | await fs.access(QUEUE_FILE); 81 | } catch { 82 | await fs.writeFile(QUEUE_FILE, ''); 83 | } 84 | 85 | // Append URLs to queue 86 | const urlsToAdd = urlArray.join('\n') + (urlArray.length > 0 ? '\n' : ''); 87 | await fs.appendFile(QUEUE_FILE, urlsToAdd); 88 | 89 | return { 90 | content: [ 91 | { 92 | type: 'text', 93 | text: `Successfully added ${urlArray.length} URLs to the queue`, 94 | }, 95 | ], 96 | }; 97 | } catch (error) { 98 | return { 99 | content: [ 100 | { 101 | type: 'text', 102 | text: `Failed to add URLs to queue: ${error}`, 103 | }, 104 | ], 105 | isError: true, 106 | }; 107 | } 108 | } 109 | 110 | return { 111 | content: [ 112 | { 113 | type: 'text', 114 | text: urlArray.join('\n') || 'No URLs found on this page.', 115 | }, 116 | ], 117 | }; 118 | } catch (error) { 119 | return { 120 | content: [ 121 | { 122 | type: 'text', 123 | text: `Failed to extract URLs: ${error}`, 124 | }, 125 | ], 126 | isError: true, 127 | }; 128 | } finally { 129 | await page.close(); 130 | } 131 | } 132 | } -------------------------------------------------------------------------------- /src/tools/index.ts: -------------------------------------------------------------------------------- 1 | export * from './search-documentation.js'; 2 | export * from './list-sources.js'; 3 | export * from './extract-urls.js'; 4 | export * from './remove-documentation.js'; 5 | export * from './list-queue.js'; 6 | export * from './run-queue.js'; 7 | export * from './clear-queue.js'; -------------------------------------------------------------------------------- /src/tools/list-queue.ts: -------------------------------------------------------------------------------- 1 | import { BaseTool } from './base-tool.js'; 2 | import { ToolDefinition, McpToolResponse } from '../types.js'; 3 | import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'; 4 | import fs from 'fs/promises'; 5 | import path from 'path'; 6 | 7 | const QUEUE_FILE = path.join(process.cwd(), 'queue.txt'); 8 | 9 | export class ListQueueTool extends BaseTool { 10 | constructor() { 11 | super(); 12 | } 13 | 14 | get definition(): ToolDefinition { 15 | return { 16 | name: 'list_queue', 17 | description: 'List all URLs currently in the documentation processing queue', 18 | inputSchema: { 19 | type: 'object', 20 | properties: {}, 21 | required: [], 22 | }, 23 | }; 24 | } 25 | 26 | async execute(_args: any): Promise { 27 | try { 28 | // Check if queue file exists 29 | try { 30 | await fs.access(QUEUE_FILE); 31 | } catch { 32 | return { 33 | content: [ 34 | { 35 | type: 'text', 36 | text: 'Queue is empty (queue file does not exist)', 37 | }, 38 | ], 39 | }; 40 | } 41 | 42 | // Read queue file 43 | const content = await fs.readFile(QUEUE_FILE, 'utf-8'); 44 | const urls = content.split('\n').filter(url => url.trim() !== ''); 45 | 46 | if (urls.length === 0) { 47 | return { 48 | content: [ 49 | { 50 | type: 'text', 51 | text: 'Queue is empty', 52 | }, 53 | ], 54 | }; 55 | } 56 | 57 | return { 58 | content: [ 59 | { 60 | type: 'text', 61 | text: `Queue contains ${urls.length} URLs:\n${urls.join('\n')}`, 62 | }, 63 | ], 64 | }; 65 | } catch (error) { 66 | return { 67 | content: [ 68 | { 69 | type: 'text', 70 | text: `Failed to read queue: ${error}`, 71 | }, 72 | ], 73 | isError: true, 74 | }; 75 | } 76 | } 77 | } -------------------------------------------------------------------------------- /src/tools/list-sources.ts: -------------------------------------------------------------------------------- 1 | import { BaseTool } from './base-tool.js'; 2 | import { ToolDefinition, McpToolResponse, isDocumentPayload } from '../types.js'; 3 | import { ApiClient } from '../api-client.js'; 4 | import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'; 5 | 6 | const COLLECTION_NAME = 'documentation'; 7 | 8 | export class ListSourcesTool extends BaseTool { 9 | private apiClient: ApiClient; 10 | 11 | constructor(apiClient: ApiClient) { 12 | super(); 13 | this.apiClient = apiClient; 14 | } 15 | 16 | get definition(): ToolDefinition { 17 | return { 18 | name: 'list_sources', 19 | description: 'List all documentation sources currently stored', 20 | inputSchema: { 21 | type: 'object', 22 | properties: {}, 23 | required: [], 24 | }, 25 | }; 26 | } 27 | 28 | async execute(args: any): Promise { 29 | try { 30 | // Use pagination for better performance with large datasets 31 | const pageSize = 100; 32 | let offset: string | null = null; 33 | const sources = new Set(); 34 | 35 | while (true) { 36 | const scroll = await this.apiClient.qdrantClient.scroll(COLLECTION_NAME, { 37 | with_payload: true, 38 | with_vector: false, // Optimize network transfer 39 | limit: pageSize, 40 | offset, 41 | }); 42 | 43 | if (scroll.points.length === 0) break; 44 | 45 | for (const point of scroll.points) { 46 | if (isDocumentPayload(point.payload)) { 47 | sources.add(`${point.payload.title} (${point.payload.url})`); 48 | } 49 | } 50 | 51 | if (scroll.points.length < pageSize) break; 52 | offset = scroll.points[scroll.points.length - 1].id as string; 53 | } 54 | 55 | return { 56 | content: [ 57 | { 58 | type: 'text', 59 | text: Array.from(sources).join('\n') || 'No documentation sources found in the cloud collection.', 60 | }, 61 | ], 62 | }; 63 | } catch (error) { 64 | if (error instanceof Error) { 65 | if (error.message.includes('unauthorized')) { 66 | throw new McpError( 67 | ErrorCode.InvalidRequest, 68 | 'Failed to authenticate with Qdrant cloud while listing sources' 69 | ); 70 | } else if (error.message.includes('ECONNREFUSED') || error.message.includes('ETIMEDOUT')) { 71 | throw new McpError( 72 | ErrorCode.InternalError, 73 | 'Connection to Qdrant cloud failed while listing sources' 74 | ); 75 | } 76 | } 77 | return { 78 | content: [ 79 | { 80 | type: 'text', 81 | text: `Failed to list sources: ${error}`, 82 | }, 83 | ], 84 | isError: true, 85 | }; 86 | } 87 | } 88 | } -------------------------------------------------------------------------------- /src/tools/remove-documentation.ts: -------------------------------------------------------------------------------- 1 | import { BaseTool } from './base-tool.js'; 2 | import { ToolDefinition, McpToolResponse } from '../types.js'; 3 | import { ApiClient } from '../api-client.js'; 4 | import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'; 5 | 6 | const COLLECTION_NAME = 'documentation'; 7 | 8 | export class RemoveDocumentationTool extends BaseTool { 9 | private apiClient: ApiClient; 10 | 11 | constructor(apiClient: ApiClient) { 12 | super(); 13 | this.apiClient = apiClient; 14 | } 15 | 16 | get definition(): ToolDefinition { 17 | return { 18 | name: 'remove_documentation', 19 | description: 'Remove one or more documentation sources by their URLs', 20 | inputSchema: { 21 | type: 'object', 22 | properties: { 23 | urls: { 24 | type: 'array', 25 | items: { 26 | type: 'string', 27 | description: 'URL of a documentation source to remove' 28 | }, 29 | description: 'Array of URLs to remove. Can be a single URL or multiple URLs.', 30 | minItems: 1 31 | } 32 | }, 33 | required: ['urls'], 34 | }, 35 | }; 36 | } 37 | 38 | async execute(args: { urls: string[] }): Promise { 39 | if (!Array.isArray(args.urls) || args.urls.length === 0) { 40 | throw new McpError(ErrorCode.InvalidParams, 'At least one URL is required'); 41 | } 42 | 43 | if (!args.urls.every(url => typeof url === 'string')) { 44 | throw new McpError(ErrorCode.InvalidParams, 'All URLs must be strings'); 45 | } 46 | 47 | try { 48 | // Delete using filter to match any of the provided URLs 49 | const result = await this.apiClient.qdrantClient.delete(COLLECTION_NAME, { 50 | filter: { 51 | should: args.urls.map(url => ({ 52 | key: 'url', 53 | match: { value: url } 54 | })) 55 | }, 56 | wait: true 57 | }); 58 | 59 | if (!['acknowledged', 'completed'].includes(result.status)) { 60 | throw new Error('Delete operation failed'); 61 | } 62 | 63 | return { 64 | content: [ 65 | { 66 | type: 'text', 67 | text: `Successfully removed documentation from ${args.urls.length} source${args.urls.length > 1 ? 's' : ''}: ${args.urls.join(', ')}`, 68 | }, 69 | ], 70 | }; 71 | } catch (error) { 72 | if (error instanceof Error) { 73 | if (error.message.includes('unauthorized')) { 74 | throw new McpError( 75 | ErrorCode.InvalidRequest, 76 | 'Failed to authenticate with Qdrant cloud while removing documentation' 77 | ); 78 | } else if (error.message.includes('ECONNREFUSED') || error.message.includes('ETIMEDOUT')) { 79 | throw new McpError( 80 | ErrorCode.InternalError, 81 | 'Connection to Qdrant cloud failed while removing documentation' 82 | ); 83 | } 84 | } 85 | return { 86 | content: [ 87 | { 88 | type: 'text', 89 | text: `Failed to remove documentation: ${error}`, 90 | }, 91 | ], 92 | isError: true, 93 | }; 94 | } 95 | } 96 | } -------------------------------------------------------------------------------- /src/tools/run-queue.ts: -------------------------------------------------------------------------------- 1 | import { BaseTool } from './base-tool.js'; 2 | import { ToolDefinition, McpToolResponse } from '../types.js'; 3 | import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'; 4 | import fs from 'fs/promises'; 5 | import path from 'path'; 6 | import { fileURLToPath } from 'url'; 7 | import { ApiClient } from '../api-client.js'; 8 | import { AddDocumentationHandler } from '../handlers/add-documentation.js'; 9 | import { Server } from '@modelcontextprotocol/sdk/server/index.js'; 10 | 11 | // Get current directory in ES modules 12 | const __filename = fileURLToPath(import.meta.url); 13 | const __dirname = path.dirname(__filename); 14 | const QUEUE_FILE = path.join(__dirname, '..', '..', 'queue.txt'); 15 | 16 | export class RunQueueTool extends BaseTool { 17 | private apiClient: ApiClient; 18 | private addDocHandler: AddDocumentationHandler; 19 | 20 | constructor(apiClient: ApiClient) { 21 | super(); 22 | this.apiClient = apiClient; 23 | // Create a temporary server instance just for the handler 24 | const tempServer = new Server( 25 | { name: 'temp', version: '0.0.0' }, 26 | { capabilities: { tools: {} } } 27 | ); 28 | this.addDocHandler = new AddDocumentationHandler(tempServer, apiClient); 29 | } 30 | 31 | get definition(): ToolDefinition { 32 | return { 33 | name: 'run_queue', 34 | description: 'Process URLs from the queue one at a time until complete', 35 | inputSchema: { 36 | type: 'object', 37 | properties: {}, 38 | required: [], 39 | }, 40 | }; 41 | } 42 | 43 | async execute(_args: any): Promise { 44 | try { 45 | // Check if queue file exists 46 | try { 47 | await fs.access(QUEUE_FILE); 48 | } catch { 49 | return { 50 | content: [ 51 | { 52 | type: 'text', 53 | text: 'Queue is empty (queue file does not exist)', 54 | }, 55 | ], 56 | }; 57 | } 58 | 59 | let processedCount = 0; 60 | let failedCount = 0; 61 | const failedUrls: string[] = []; 62 | 63 | while (true) { 64 | // Read current queue 65 | const content = await fs.readFile(QUEUE_FILE, 'utf-8'); 66 | const urls = content.split('\n').filter(url => url.trim() !== ''); 67 | 68 | if (urls.length === 0) { 69 | break; // Queue is empty 70 | } 71 | 72 | const currentUrl = urls[0]; // Get first URL 73 | 74 | try { 75 | // Process the URL using the handler 76 | await this.addDocHandler.handle({ url: currentUrl }); 77 | processedCount++; 78 | } catch (error) { 79 | failedCount++; 80 | failedUrls.push(currentUrl); 81 | console.error(`Failed to process URL ${currentUrl}:`, error); 82 | } 83 | 84 | // Remove the processed URL from queue 85 | const remainingUrls = urls.slice(1); 86 | await fs.writeFile(QUEUE_FILE, remainingUrls.join('\n') + (remainingUrls.length > 0 ? '\n' : '')); 87 | } 88 | 89 | let resultText = `Queue processing complete.\nProcessed: ${processedCount} URLs\nFailed: ${failedCount} URLs`; 90 | if (failedUrls.length > 0) { 91 | resultText += `\n\nFailed URLs:\n${failedUrls.join('\n')}`; 92 | } 93 | 94 | return { 95 | content: [ 96 | { 97 | type: 'text', 98 | text: resultText, 99 | }, 100 | ], 101 | }; 102 | } catch (error) { 103 | return { 104 | content: [ 105 | { 106 | type: 'text', 107 | text: `Failed to process queue: ${error}`, 108 | }, 109 | ], 110 | isError: true, 111 | }; 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/tools/search-documentation.ts: -------------------------------------------------------------------------------- 1 | import { BaseTool } from './base-tool.js'; 2 | import { ToolDefinition, McpToolResponse, isDocumentPayload } from '../types.js'; 3 | import { ApiClient } from '../api-client.js'; 4 | import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'; 5 | 6 | const COLLECTION_NAME = 'documentation'; 7 | 8 | export class SearchDocumentationTool extends BaseTool { 9 | private apiClient: ApiClient; 10 | 11 | constructor(apiClient: ApiClient) { 12 | super(); 13 | this.apiClient = apiClient; 14 | } 15 | 16 | get definition(): ToolDefinition { 17 | return { 18 | name: 'search_documentation', 19 | description: 'Search through stored documentation', 20 | inputSchema: { 21 | type: 'object', 22 | properties: { 23 | query: { 24 | type: 'string', 25 | description: 'Search query', 26 | }, 27 | limit: { 28 | type: 'number', 29 | description: 'Maximum number of results to return', 30 | default: 5, 31 | }, 32 | }, 33 | required: ['query'], 34 | }, 35 | }; 36 | } 37 | 38 | async execute(args: any): Promise { 39 | if (!args.query || typeof args.query !== 'string') { 40 | throw new McpError(ErrorCode.InvalidParams, 'Query is required'); 41 | } 42 | 43 | const limit = args.limit || 5; 44 | 45 | try { 46 | const queryEmbedding = await this.apiClient.getEmbeddings(args.query); 47 | 48 | const searchResults = await this.apiClient.qdrantClient.search(COLLECTION_NAME, { 49 | vector: queryEmbedding, 50 | limit, 51 | with_payload: true, 52 | with_vector: false, // Optimize network transfer by not retrieving vectors 53 | score_threshold: 0.7, // Only return relevant results 54 | }); 55 | 56 | const formattedResults = searchResults.map(result => { 57 | if (!isDocumentPayload(result.payload)) { 58 | throw new Error('Invalid payload type'); 59 | } 60 | return `[${result.payload.title}](${result.payload.url})\nScore: ${result.score.toFixed(3)}\nContent: ${result.payload.text}\n`; 61 | }).join('\n---\n'); 62 | 63 | return { 64 | content: [ 65 | { 66 | type: 'text', 67 | text: formattedResults || 'No results found matching the query.', 68 | }, 69 | ], 70 | }; 71 | } catch (error) { 72 | if (error instanceof Error) { 73 | if (error.message.includes('unauthorized')) { 74 | throw new McpError( 75 | ErrorCode.InvalidRequest, 76 | 'Failed to authenticate with Qdrant cloud while searching' 77 | ); 78 | } else if (error.message.includes('ECONNREFUSED') || error.message.includes('ETIMEDOUT')) { 79 | throw new McpError( 80 | ErrorCode.InternalError, 81 | 'Connection to Qdrant cloud failed while searching' 82 | ); 83 | } 84 | } 85 | return { 86 | content: [ 87 | { 88 | type: 'text', 89 | text: `Search failed: ${error}`, 90 | }, 91 | ], 92 | isError: true, 93 | }; 94 | } 95 | } 96 | } -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | export interface DocumentChunk { 2 | text: string; 3 | url: string; 4 | title: string; 5 | timestamp: string; 6 | } 7 | 8 | export interface DocumentPayload extends DocumentChunk { 9 | _type: 'DocumentChunk'; 10 | [key: string]: unknown; 11 | } 12 | 13 | export function isDocumentPayload(payload: unknown): payload is DocumentPayload { 14 | if (!payload || typeof payload !== 'object') return false; 15 | const p = payload as Partial; 16 | return ( 17 | p._type === 'DocumentChunk' && 18 | typeof p.text === 'string' && 19 | typeof p.url === 'string' && 20 | typeof p.title === 'string' && 21 | typeof p.timestamp === 'string' 22 | ); 23 | } 24 | 25 | export interface ToolDefinition { 26 | name: string; 27 | description: string; 28 | inputSchema: { 29 | type: string; 30 | properties: Record; 31 | required: string[]; 32 | }; 33 | } 34 | 35 | export interface McpToolResponse { 36 | content: Array<{ 37 | type: string; 38 | text: string; 39 | }>; 40 | isError?: boolean; 41 | } -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2022", 4 | "module": "Node16", 5 | "moduleResolution": "Node16", 6 | "outDir": "./build", 7 | "rootDir": "./src", 8 | "strict": true, 9 | "esModuleInterop": true, 10 | "skipLibCheck": true, 11 | "forceConsistentCasingInFileNames": true 12 | }, 13 | "include": ["src/**/*"], 14 | "exclude": ["node_modules"] 15 | } 16 | --------------------------------------------------------------------------------