├── .env.example ├── .eslintrc.js ├── .gitignore ├── .npmrc ├── .vscode └── launch.json ├── LICENSE ├── README.md ├── agent-mimir-cli ├── package.json ├── src │ ├── chat.ts │ ├── default-config.ts │ ├── index.ts │ └── utils.ts └── tsconfig.json ├── agent-mimir-discord ├── package.json ├── src │ ├── default-config.ts │ ├── index.ts │ └── utils.ts └── tsconfig.json ├── agent-mimir ├── jest.config.js ├── package.json ├── src │ ├── agent-manager │ │ ├── code-agent │ │ │ ├── agent.ts │ │ │ ├── executors │ │ │ │ ├── local-executor.ts │ │ │ │ └── python-code.ts │ │ │ ├── factory.ts │ │ │ ├── index.ts │ │ │ ├── prompt.ts │ │ │ ├── tool-node.ts │ │ │ ├── utils.ts │ │ │ └── wrapper.ts │ │ ├── constants.ts │ │ ├── factory.ts │ │ ├── index.ts │ │ ├── langgraph-agent.ts │ │ ├── message-utils.ts │ │ └── tool-agent │ │ │ ├── agent.ts │ │ │ ├── factory.ts │ │ │ ├── index.ts │ │ │ ├── tool-node.ts │ │ │ ├── utils.ts │ │ │ └── wrapper.ts │ ├── communication │ │ ├── helpers.ts │ │ └── multi-agent.ts │ ├── nodejs │ │ ├── filesystem-work-directory.ts │ │ └── index.ts │ ├── plugins │ │ ├── context-provider.test.ts │ │ ├── context-provider.ts │ │ ├── default-plugins.ts │ │ ├── index.ts │ │ └── workspace.ts │ ├── schema.ts │ ├── tools │ │ ├── image_view.ts │ │ ├── index.ts │ │ └── langchain.ts │ └── utils │ │ ├── format.ts │ │ └── instruction-mapper.ts ├── tsconfig.cjs.json └── tsconfig.json ├── assets ├── demo-discord.png ├── demo-screenshot.png ├── discord │ ├── botIntents.png │ ├── newApp.png │ └── tokenGenerate.png └── mimir_logo.svg ├── implementation-plan.md ├── lg-server ├── .gitignore ├── langgraph.json ├── package.json └── src │ └── index.ts ├── mimir-config.example ├── mimir-cfg.js └── package.json ├── package-lock.json ├── package.json ├── scripts ├── config-setup.js └── release-branch.sh ├── tool-examples ├── javascript-repl │ ├── mimir-cfg.js │ └── package.json ├── powershell │ ├── mimir-cfg.js │ └── package.json └── shell │ ├── mimir-cfg.js │ └── package.json ├── tools ├── code-interpreter │ ├── package.json │ ├── src │ │ └── index.ts │ └── tsconfig.json ├── desktop-control │ ├── molmo │ │ └── docker-compose.yml │ ├── package.json │ ├── python │ │ ├── mobile_sam.pt │ │ ├── requirementsCpu.txt │ │ ├── requirementsCuda11.txt │ │ ├── requirementsCuda12.txt │ │ └── server.py │ ├── scripts │ │ └── install.js │ ├── src │ │ ├── index.ts │ │ ├── molmo.ts │ │ └── sam.ts │ └── tsconfig.json ├── gameboy-play │ ├── package.json │ ├── src │ │ ├── controller.ts │ │ └── index.ts │ └── tsconfig.json ├── javascript-code-runner │ ├── README.md │ ├── package.json │ ├── src │ │ └── index.ts │ └── tsconfig.json ├── mcp-client │ ├── package.json │ ├── src │ │ ├── index.ts │ │ └── json-schema-to-zod │ │ │ ├── index.ts │ │ │ ├── json-schema-to-zod.ts │ │ │ ├── parsers │ │ │ ├── parse-all-of.ts │ │ │ ├── parse-any-of.ts │ │ │ ├── parse-array.ts │ │ │ ├── parse-boolean.ts │ │ │ ├── parse-const.ts │ │ │ ├── parse-default.ts │ │ │ ├── parse-enum.ts │ │ │ ├── parse-if-then-else.ts │ │ │ ├── parse-multiple-type.ts │ │ │ ├── parse-not.ts │ │ │ ├── parse-null.ts │ │ │ ├── parse-nullable.ts │ │ │ ├── parse-number.ts │ │ │ ├── parse-object.ts │ │ │ ├── parse-one-of.ts │ │ │ ├── parse-schema.ts │ │ │ └── parse-string.ts │ │ │ ├── types.ts │ │ │ └── utils │ │ │ ├── extend-schema.ts │ │ │ ├── half.ts │ │ │ ├── its.ts │ │ │ └── omit.ts │ └── tsconfig.json ├── playwright-browser │ ├── README.md │ ├── package.json │ ├── src │ │ ├── driver-manager.ts │ │ ├── html-processor.ts │ │ ├── index.ts │ │ ├── prompt │ │ │ ├── combiner-prompt.ts │ │ │ └── relevance-prompt.ts │ │ ├── to-markdown.ts │ │ ├── tools.ts │ │ └── xpath.ts │ └── tsconfig.json ├── selenium-browser │ ├── README.md │ ├── package.json │ ├── src │ │ ├── driver-manager.ts │ │ ├── html-processor.ts │ │ ├── index.ts │ │ ├── prompt │ │ │ ├── combiner-prompt.ts │ │ │ └── relevance-prompt.ts │ │ ├── to-markdown.ts │ │ ├── tools.ts │ │ └── xpath.ts │ └── tsconfig.json ├── serper-search │ ├── package.json │ ├── src │ │ └── index.ts │ └── tsconfig.json └── sqlite-tool │ ├── package.json │ ├── src │ └── index.ts │ └── tsconfig.json └── turbo.json /.env.example: -------------------------------------------------------------------------------- 1 | 2 | 3 | # OPENAI API KEY 4 | OPENAI_API_KEY= 5 | 6 | # Discord bot token 7 | #DISCORD_TOKEN= 8 | 9 | # Enable the code interpreter plugin. 10 | CODE_INTERPRETER_PLUGIN=true 11 | 12 | # Enable the web browser plugin. 13 | WEB_BROWSER_PLUGIN=true 14 | 15 | # Set the model to use for the AI agent. 16 | #AGENT_OPENAI_MODEL=gpt-4-turbo 17 | 18 | # Log tool response 19 | #MIMIR_LOG_TOOL_RESPONSE=true 20 | 21 | # Log AI response 22 | #MIMIR_LOG_AI_RESPONSE=true 23 | 24 | # Define a persistent working directory for the AI agents to persist there state. 25 | #WORK_DIRECTORY= 26 | 27 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | // This tells ESLint to load the config from the package `eslint-config-custom` 4 | rules: { 5 | // "@typescript-eslint/no-floating-promises": ["error"] 6 | }, 7 | extends: ["custom"], 8 | settings: { 9 | next: { 10 | rootDir: ["apps/*/"], 11 | }, 12 | }, 13 | }; 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | node_modules 5 | .pnp 6 | .pnp.js 7 | 8 | # testing 9 | coverage 10 | 11 | # next.js 12 | .next/ 13 | out/ 14 | build 15 | dist-cjs/ 16 | dist/ 17 | # misc 18 | .DS_Store 19 | *.pem 20 | 21 | # debug 22 | npm-debug.log* 23 | yarn-debug.log* 24 | yarn-error.log* 25 | 26 | # local env files 27 | .env 28 | .env.local 29 | .env.development.local 30 | .env.test.local 31 | .env.production.local 32 | *.traineddata 33 | # turbo 34 | .turbo 35 | 36 | # vercel 37 | .vercel 38 | 39 | .temp_custom_deps/ 40 | 41 | mimir-config/ 42 | # LangGraph API 43 | .langgraph_api 44 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | auto-install-peers = true 2 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "type": "node-terminal", 5 | "name": "Start Agent Mimir", 6 | "request": "launch", 7 | "command": "npm run start", 8 | "cwd": "${workspaceFolder}" 9 | } 10 | ] 11 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) Harrison Chase 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /agent-mimir-cli/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "agent-mimir-cli", 3 | "version": "0.0.1", 4 | "description": "", 5 | "main": "index.js", 6 | "type": "module", 7 | "scripts": { 8 | "start:dev": "tsx watch src/index.ts", 9 | "build": "tsc", 10 | "start": "tsx -C development src/index.ts", 11 | "test": "echo \"Error: no test specified\" && exit 1" 12 | }, 13 | "files": [ 14 | "dist/" 15 | ], 16 | "keywords": [], 17 | "author": "", 18 | "license": "ISC", 19 | "devDependencies": { 20 | "@types/debug": "^4.1.7", 21 | "@types/inquirer": "^9.0.3", 22 | "@types/node": "^18.15.10", 23 | "nodemon": "^2.0.22", 24 | "rimraf": "^4.4.1", 25 | "ts-node": "^10.9.1", 26 | "typescript": "^5.3.3" 27 | }, 28 | "dependencies": { 29 | "@langchain/openai": "0.3.16", 30 | "@agent-mimir/code-interpreter": "*", 31 | "@agent-mimir/selenium-browser": "*", 32 | "chalk": "^5.2.0", 33 | "readline": "^1.3.0" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /agent-mimir-cli/src/default-config.ts: -------------------------------------------------------------------------------- 1 | import { ChatOpenAI } from '@langchain/openai'; 2 | import { OpenAIEmbeddings } from '@langchain/openai'; 3 | 4 | const embeddings = new OpenAIEmbeddings({ 5 | openAIApiKey: process.env.OPENAI_API_KEY, 6 | }); 7 | 8 | const chatModel = new ChatOpenAI({ 9 | openAIApiKey: process.env.OPENAI_API_KEY, 10 | temperature: !isNaN(Number(process.env.AGENT_OPENAI_CHAT_TEMPERATURE)) ? Number(process.env.AGENT_OPENAI_CHAT_TEMPERATURE) : 0.0, 11 | modelName: process.env.AGENT_OPENAI_MODEL ?? 'gpt-4-turbo' 12 | }); 13 | 14 | const summaryModel = new ChatOpenAI({ 15 | openAIApiKey: process.env.OPENAI_API_KEY, 16 | temperature: 0.0, 17 | modelName: 'gpt-3.5-turbo-16k-0613', 18 | }); 19 | 20 | const workingDirectory = process.env.WORK_DIRECTORY ?? undefined; 21 | 22 | // eslint-disable-next-line import/no-anonymous-default-export 23 | export default async function () { 24 | const plugins = []; 25 | if (process.env.CODE_INTERPRETER_PLUGIN === 'true') { 26 | const CodeInterpreterPluginFactory = (await import('@agent-mimir/code-interpreter')).CodeInterpreterPluginFactory; 27 | plugins.push(new CodeInterpreterPluginFactory()); 28 | } 29 | 30 | if (process.env.WEB_BROWSER_PLUGIN === 'true') { 31 | const WebBrowserPluginFactory = (await import('@agent-mimir/selenium-browser')).WebBrowserPluginFactory; 32 | plugins.push(new WebBrowserPluginFactory({ browserConfig: { browserName: "chrome", disableHeadless: true }, maximumChunkSize: 6000, numberOfRelevantDocuments: 3 }, summaryModel, embeddings)); 33 | } 34 | 35 | return { 36 | continuousMode: false, 37 | workingDirectory: workingDirectory, 38 | embeddings: embeddings, 39 | agents: { 40 | 'Assistant': { 41 | mainAgent: true, 42 | description: 'An assistant', 43 | definition: { 44 | agentType: 'openai-function-agent' as const, 45 | visionSupport: 'openai' as const, 46 | chatModel: chatModel, 47 | profession: 'an Assistant', 48 | chatHistory: { 49 | summaryModel: summaryModel, 50 | tokenLimit: 4000, 51 | conversationTokenThreshold: 75, 52 | }, 53 | tools: [ 54 | ], 55 | plugins: [ 56 | ...plugins, 57 | ] 58 | } 59 | } 60 | } 61 | } 62 | } -------------------------------------------------------------------------------- /agent-mimir-cli/src/index.ts: -------------------------------------------------------------------------------- 1 | 2 | import { OrchestratorBuilder } from "agent-mimir/communication/multi-agent"; 3 | import { PluginFactory } from "agent-mimir/plugins"; 4 | import chalk from "chalk"; 5 | import { StructuredTool } from "@langchain/core/tools"; 6 | import { chatWithAgent } from "./chat.js"; 7 | import { promises as fs } from 'fs'; 8 | import os from 'os'; 9 | import path from "path"; 10 | import { FileSystemAgentWorkspace } from "agent-mimir/nodejs"; 11 | import { BaseLanguageModel } from "@langchain/core/language_models/base"; 12 | import { Embeddings } from "@langchain/core/embeddings"; 13 | import { BaseChatModel } from "@langchain/core/language_models/chat_models"; 14 | import { LangchainToolWrapperPluginFactory } from "agent-mimir/tools/langchain"; 15 | import { CodeAgentFactory, LocalPythonExecutor } from "agent-mimir/agent/code-agent"; 16 | import { BaseCheckpointSaver } from "@langchain/langgraph"; 17 | export type AgentDefinition = { 18 | mainAgent?: boolean; 19 | description: string; 20 | definition?: { 21 | profession: string; 22 | chatModel: BaseChatModel; 23 | taskModel?: BaseLanguageModel; 24 | constitution?: string; 25 | checkpointer?: BaseCheckpointSaver, 26 | plugins?: PluginFactory[]; 27 | visionSupport?: 'openai'; 28 | chatHistory?: { 29 | summaryModel?: BaseChatModel; 30 | tokenLimit?: number; 31 | conversationTokenThreshold?: number; 32 | } 33 | langChainTools?: StructuredTool[]; 34 | communicationWhitelist?: string[] | boolean; 35 | }, 36 | 37 | } 38 | type AgentMimirConfig = { 39 | agents: Record; 40 | embeddings: Embeddings, 41 | workingDirectory?: string; 42 | continuousMode?: boolean; 43 | } 44 | const getConfig = async () => { 45 | if (process.env.MIMIR_CFG_PATH) { 46 | let cfgFile = path.join(process.env.MIMIR_CFG_PATH, 'mimir-cfg.js'); 47 | const configFileExists = await fs.access(cfgFile, fs.constants.F_OK).then(() => true).catch(() => false); 48 | if (configFileExists) { 49 | console.log(chalk.green(`Loading configuration file`)); 50 | const configFunction: Promise | AgentMimirConfig = (await import(`file://${cfgFile}`)).default() 51 | return await Promise.resolve(configFunction); 52 | } 53 | } 54 | console.log(chalk.yellow("No config file found, using default ApenAI config")); 55 | return (await import("./default-config.js")).default(); 56 | }; 57 | 58 | export const run = async () => { 59 | 60 | const agentConfig: AgentMimirConfig = await getConfig(); 61 | const workingDirectory = agentConfig.workingDirectory ?? await fs.mkdtemp(path.join(os.tmpdir(), 'mimir-cli-')); 62 | console.log(`Using working directory ${workingDirectory}`); 63 | await fs.mkdir(workingDirectory, { recursive: true }); 64 | 65 | 66 | const workspaceFactory = async (agentName: string) => { 67 | const tempDir = path.join(workingDirectory, agentName); 68 | await fs.mkdir(tempDir, { recursive: true }); 69 | const workspace = new FileSystemAgentWorkspace(tempDir); 70 | await fs.mkdir(workspace.workingDirectory, { recursive: true }); 71 | return workspace; 72 | } 73 | 74 | const orchestratorBuilder = new OrchestratorBuilder(); 75 | const continousMode = agentConfig.continuousMode ?? false; 76 | const agents = await Promise.all(Object.entries(agentConfig.agents).map(async ([agentName, agentDefinition]) => { 77 | if (agentDefinition.definition) { 78 | const newAgent = { 79 | mainAgent: agentDefinition.mainAgent, 80 | name: agentName, 81 | agent: await orchestratorBuilder.initializeAgent( new CodeAgentFactory({ 82 | description: agentDefinition.description, 83 | profession: agentDefinition.definition.profession, 84 | model: agentDefinition.definition.chatModel, 85 | visionSupport: agentDefinition.definition.visionSupport, 86 | constitution: agentDefinition.definition.constitution, 87 | checkpointer: agentDefinition.definition.checkpointer, 88 | plugins: [...agentDefinition.definition.plugins ?? [], ...(agentDefinition.definition.langChainTools ?? []).map(t => new LangchainToolWrapperPluginFactory(t))], 89 | workspaceFactory: workspaceFactory, 90 | codeExecutor: (workspace) => new LocalPythonExecutor({workspace}), 91 | }), agentName, agentDefinition.definition.communicationWhitelist) 92 | } 93 | console.log(chalk.green(`Created agent "${agentName}" with profession "${agentDefinition.definition.profession}" and description "${agentDefinition.description}"`)); 94 | return newAgent; 95 | } else { 96 | throw new Error(`Agent "${agentName}" has no definition`); 97 | } 98 | 99 | })); 100 | 101 | 102 | const mainAgent = agents.length === 1 ? agents[0].agent : agents.find(a => a.mainAgent)?.agent; 103 | if (!mainAgent) { 104 | throw new Error("No main agent found"); 105 | } 106 | const chatAgentHandle = orchestratorBuilder.build(mainAgent); 107 | console.log(chalk.green(`Using "${mainAgent.name}" as main agent`)); 108 | await chatWithAgent(chatAgentHandle, continousMode); 109 | }; 110 | 111 | run(); 112 | 113 | -------------------------------------------------------------------------------- /agent-mimir-cli/src/utils.ts: -------------------------------------------------------------------------------- 1 | 2 | 3 | export async function Retry(action: () => Promise, retryInterval = 2000, maxAttemptCount = 1) { 4 | const exceptions = []; 5 | for (let attempted = 0; attempted < maxAttemptCount; attempted++) { 6 | try { 7 | if (attempted > 0) 8 | await sleep(retryInterval); 9 | return await action(); 10 | } 11 | catch (e) { 12 | console.log(`Attempt ${attempted + 1} of ${maxAttemptCount} failed.`, e); 13 | exceptions.push(e); 14 | } 15 | } 16 | 17 | throw new Error(`All ${maxAttemptCount} attempts failed. ${JSON.stringify(exceptions)}`); 18 | } 19 | 20 | function sleep(ms: number) { return new Promise(resolve => setTimeout(resolve, ms)); } -------------------------------------------------------------------------------- /agent-mimir-cli/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "ts-node": { 3 | "esm": true 4 | }, 5 | "include": [ 6 | "src/**/*" 7 | ], 8 | "exclude": [ 9 | "node_modules", 10 | "dist", 11 | "docs" 12 | ], 13 | "compilerOptions": { 14 | "target": "ES2021", 15 | "module": "NodeNext", 16 | "lib": [ 17 | "ES2021", 18 | "ES2022.Object", 19 | "DOM" 20 | ], 21 | "allowJs": true, 22 | "outDir": "dist", 23 | "rootDir": "src", 24 | "strict": true, 25 | "noImplicitAny": true, 26 | "esModuleInterop": true, 27 | "sourceMap": true, 28 | "moduleResolution": "nodenext", 29 | "skipLibCheck": true, 30 | "types": [ 31 | "node", 32 | ], 33 | } 34 | } -------------------------------------------------------------------------------- /agent-mimir-discord/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "agent-mimir-discord", 3 | "version": "0.0.1", 4 | "description": "", 5 | "main": "index.js", 6 | "type": "module", 7 | "scripts": { 8 | "start:dev": "tsx watch src/index.ts", 9 | "build": "tsc", 10 | "start": "tsx -C development src/index.ts", 11 | "test": "echo \"Error: no test specified\" && exit 1" 12 | }, 13 | "files": [ 14 | "dist/" 15 | ], 16 | "keywords": [], 17 | "author": "", 18 | "license": "ISC", 19 | "devDependencies": { 20 | "@types/debug": "^4.1.7", 21 | "@types/inquirer": "^9.0.3", 22 | "@types/node": "^18.15.10", 23 | "nodemon": "^2.0.22", 24 | "rimraf": "^4.4.1", 25 | "ts-node": "^10.9.1", 26 | "typescript": "^5.3.3" 27 | }, 28 | "dependencies": { 29 | "agent-mimir": "*", 30 | "@langchain/openai": "0.3.16", 31 | "@agent-mimir/code-interpreter": "*", 32 | "@agent-mimir/selenium-browser": "*", 33 | "chalk": "^5.2.0", 34 | "readline": "^1.3.0", 35 | "discord.js": "^14.13.0", 36 | "tmp-promise": "3.0.3", 37 | "lowdb": "7.0.1" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /agent-mimir-discord/src/default-config.ts: -------------------------------------------------------------------------------- 1 | import { ChatOpenAI } from '@langchain/openai'; 2 | import { OpenAIEmbeddings } from '@langchain/openai'; 3 | 4 | 5 | const embeddings = new OpenAIEmbeddings({ 6 | openAIApiKey: process.env.OPENAI_API_KEY, 7 | }); 8 | 9 | const chatModel = new ChatOpenAI({ 10 | openAIApiKey: process.env.OPENAI_API_KEY, 11 | temperature: !isNaN(Number(process.env.AGENT_OPENAI_CHAT_TEMPERATURE)) ? Number(process.env.AGENT_OPENAI_CHAT_TEMPERATURE) : 0.0, 12 | modelName: process.env.AGENT_OPENAI_MODEL ?? 'gpt-4-turbo' 13 | }); 14 | 15 | const summaryModel = new ChatOpenAI({ 16 | openAIApiKey: process.env.OPENAI_API_KEY, 17 | temperature: 0.0, 18 | modelName: 'gpt-3.5-turbo-16k-0613', 19 | }); 20 | const workingDirectory = process.env.WORK_DIRECTORY ?? undefined; 21 | 22 | // eslint-disable-next-line import/no-anonymous-default-export 23 | export default async function () { 24 | const plugins = []; 25 | if (process.env.CODE_INTERPRETER_PLUGIN === 'true') { 26 | const CodeInterpreterPluginFactory = (await import('@agent-mimir/code-interpreter')).CodeInterpreterPluginFactory; 27 | plugins.push(new CodeInterpreterPluginFactory()); 28 | } 29 | 30 | if (process.env.WEB_BROWSER_PLUGIN === 'true') { 31 | const WebBrowserPluginFactory = (await import('@agent-mimir/selenium-browser')).WebBrowserPluginFactory; 32 | plugins.push(new WebBrowserPluginFactory({ browserConfig: { browserName: "chrome", disableHeadless: true }, maximumChunkSize: 6000, numberOfRelevantDocuments: 3 }, summaryModel, embeddings)); 33 | } 34 | 35 | 36 | return { 37 | continuousMode: false, 38 | workingDirectory: workingDirectory, 39 | embeddings: embeddings, 40 | agents: { 41 | 'Assistant': { 42 | mainAgent: true, 43 | description: 'An assistant', 44 | definition: { 45 | agentType: 'openai-function-agent' as const, 46 | visionSupport: 'openai' as const, 47 | chatModel: chatModel, 48 | profession: 'an Assistant', 49 | chatHistory: { 50 | summaryModel: summaryModel, 51 | tokenLimit: 4000, 52 | conversationTokenThreshold: 75, 53 | }, 54 | tools: [ 55 | ], 56 | plugins: [ 57 | ...plugins, 58 | ] 59 | } 60 | } 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /agent-mimir-discord/src/utils.ts: -------------------------------------------------------------------------------- 1 | 2 | 3 | export async function Retry( action: () => Promise, retryInterval = 2000, maxAttemptCount = 3 ) 4 | { 5 | const exceptions = []; 6 | for ( let attempted = 0 ; attempted < maxAttemptCount ; attempted++ ) 7 | { 8 | try 9 | { 10 | if ( attempted > 0 ) 11 | await sleep( retryInterval ); 12 | return await action( ); 13 | } 14 | catch ( e ) 15 | { 16 | 17 | console.warn( `Attempt ${attempted + 1} of ${maxAttemptCount} failed. ${JSON.stringify(e)}`, e ); 18 | 19 | exceptions.push( e ); 20 | } 21 | } 22 | 23 | throw JSON.stringify(exceptions); 24 | } 25 | 26 | function sleep( ms: number ) { return new Promise( resolve => setTimeout( resolve, ms ) ); } -------------------------------------------------------------------------------- /agent-mimir-discord/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "ts-node": { 3 | "esm": true 4 | }, 5 | "include": [ 6 | "src/**/*" 7 | ], 8 | "exclude": [ 9 | "node_modules", 10 | "dist", 11 | "docs" 12 | ], 13 | "compilerOptions": { 14 | "target": "ES2021", 15 | "module": "NodeNext", 16 | "lib": [ 17 | "ES2021", 18 | "ES2022.Object", 19 | "DOM" 20 | ], 21 | "allowJs": true, 22 | "outDir": "dist", 23 | "rootDir": "src", 24 | "strict": true, 25 | "noImplicitAny": true, 26 | "esModuleInterop": true, 27 | "sourceMap": true, 28 | "moduleResolution": "nodenext", 29 | "skipLibCheck": true, 30 | "types": [ 31 | "node", 32 | ], 33 | } 34 | } -------------------------------------------------------------------------------- /agent-mimir/jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest').JestConfigWithTsJest} **/ 2 | export default { 3 | testEnvironment: "node", 4 | preset: 'ts-jest/presets/default-esm', 5 | roots: ['/src'], 6 | // transform: { 7 | // "^.+\.tsx?$": ["ts-jest",{}], 8 | // }, 9 | moduleNameMapper: { 10 | '^(\\.{1,2}/.*)\\.js$': '$1', // Keep if needed 11 | }, 12 | extensionsToTreatAsEsm: ['.ts'], 13 | }; -------------------------------------------------------------------------------- /agent-mimir/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "agent-mimir", 3 | "version": "0.4.0", 4 | "description": "", 5 | "type": "module", 6 | "files": [ 7 | "dist" 8 | ], 9 | "exports": { 10 | "./schema": { 11 | "types": "./dist/schema.d.ts", 12 | "import": { 13 | "development": "./src/agent-manager/schema.ts", 14 | "default": "./dist/schema.js" 15 | } 16 | }, 17 | "./agent/tool-agent": { 18 | "types": "./dist/agent-manager/tool-agent/index.d.ts", 19 | "import": { 20 | "development": "./src/agent-manager/tool-agent/index.ts", 21 | "default": "./dist/agent-manager/tool-agent/index.js" 22 | } 23 | }, 24 | "./agent/code-agent": { 25 | "types": "./dist/agent-manager/code-agent/index.d.ts", 26 | "import": { 27 | "development": "./src/agent-manager/code-agent/index.ts", 28 | "default": "./dist/agent-manager/code-agent/index.js" 29 | } 30 | }, 31 | "./agent": { 32 | "types": "./dist/agent-manager/index.d.ts", 33 | "import": { 34 | "development": "./src/agent-manager/index.ts", 35 | "default": "./dist/agent-manager/index.js" 36 | } 37 | }, 38 | "./plugins": { 39 | "types": "./dist/plugins/index.d.ts", 40 | "import": { 41 | "development": "./src/plugins/index.ts", 42 | "default": "./dist/plugins/index.js" 43 | } 44 | }, 45 | "./plugins/helpers": { 46 | "types": "./dist/plugins/helpers.d.ts", 47 | "import": { 48 | 49 | "development": "./src/plugins/helpers.ts", 50 | "default": "./dist/plugins/helpers.js" 51 | } 52 | }, 53 | "./communication/multi-agent": { 54 | "types": "./dist/communication/multi-agent.d.ts", 55 | "import": { 56 | "development": "./src/communication/multi-agent.ts", 57 | "default": "./dist/communication/multi-agent.js" 58 | } 59 | }, 60 | "./utils/format": { 61 | "types": "./dist/utils/format.d.ts", 62 | "import": { 63 | "development": "./src/utils/format.ts", 64 | "default": "./dist/utils/format.js" 65 | } 66 | }, 67 | "./nodejs": { 68 | "types": "./dist/nodejs/index.d.ts", 69 | "import": { 70 | "development": "./src/nodejs/index.ts", 71 | "default": "./dist/nodejs/index.js" 72 | } 73 | }, 74 | "./tools": { 75 | "types": "./dist/tools/index.d.ts", 76 | "import": { 77 | "development": "./src/tools/index.ts", 78 | "default": "./dist/tools/index.js" 79 | } 80 | }, 81 | "./tools/langchain": { 82 | "types": "./dist/tools/langchain.d.ts", 83 | "import": { 84 | "development": "./src/tools/langchain.ts", 85 | "default": "./dist/tools/langchain.js" 86 | } 87 | } 88 | }, 89 | "scripts": { 90 | "build": "npm run build:esm ", 91 | "build:esm": "tsc --outDir dist/ ", 92 | "build:cjs": "tsc --outDir dist-cjs/ -p tsconfig.cjs.json", 93 | "dev": "tsx watch src/index.ts", 94 | "release": "npm run build && release-it --only-version ", 95 | "test": "jest" 96 | }, 97 | "keywords": [], 98 | "author": "", 99 | "license": "ISC", 100 | "devDependencies": { 101 | "@types/debug": "^4.1.7", 102 | "@types/inquirer": "^9.0.3", 103 | "@types/node": "^18.15.10", 104 | "@types/xml2js": "0.4.14", 105 | "nodemon": "^2.0.22", 106 | "rimraf": "^4.4.1", 107 | "typescript": "^5.3.3", 108 | "ts-jest": "29.3.2", 109 | "@jest/globals": "29.7.0" 110 | }, 111 | "dependencies": { 112 | "jsonrepair": "^3.0.2", 113 | "ring-buffer-ts": "^1.2.0", 114 | "gpt-3-encoder": "v1.1.4", 115 | "xml2js": "0.6.2" 116 | } 117 | } -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/code-agent/executors/python-code.ts: -------------------------------------------------------------------------------- 1 | import { toPythonFunctionName } from "../utils.js"; 2 | 3 | 4 | function getPythonFunction(functionName: string): string { 5 | return ` 6 | def ${toPythonFunctionName(functionName)}(args:dict): 7 | result = asyncio.run(ws_channel.call("${functionName}", args=args)) 8 | call_value = result.result["value"] 9 | if result.result["error"]: 10 | raise Exception(f"Error in function call ${toPythonFunctionName(functionName)}: {call_value}") 11 | return call_value 12 | \n` 13 | } 14 | 15 | 16 | export function getPythonScript(port: number, tools: string[], code: string): string { 17 | return ` 18 | import uvicorn 19 | from fastapi import FastAPI 20 | from fastapi_websocket_rpc import RpcMethodsBase, WebsocketRPCEndpoint, RpcChannel 21 | import asyncio 22 | import logging 23 | import nest_asyncio 24 | 25 | nest_asyncio.apply() 26 | 27 | logger = logging.getLogger(__name__) 28 | 29 | logging.basicConfig(level=logging.ERROR) 30 | 31 | server: uvicorn.Server | None = None 32 | # An event to signal that the work is done and shutdown can commence 33 | shutdown_event = asyncio.Event() 34 | 35 | ws_channel: RpcChannel | None = None 36 | 37 | 38 | ${tools.map(tool => getPythonFunction(tool)).join('\n')} 39 | 40 | 41 | async def do_work(): 42 | """Performs the main task and signals for shutdown.""" 43 | try: 44 | print("Executing code...") 45 | ${indentText(code, ' ')} 46 | print("Code execution completed.") 47 | pass 48 | except Exception as e: 49 | print(f"Error in executing code: {e}") 50 | finally: 51 | if ws_channel: 52 | try: 53 | await ws_channel.close() 54 | except Exception as e: 55 | print(f"Error closing WebSocket channel: {e}") 56 | # Signal that the work is complete and server should shut down 57 | shutdown_event.set() 58 | 59 | # --- WebSocket Connection Handling --- 60 | async def on_connect(channel: RpcChannel): 61 | """Handles new WebSocket connections.""" 62 | global ws_channel 63 | # Ensure only the first connection triggers the work 64 | if ws_channel is None: 65 | ws_channel = channel 66 | # Start the main task without blocking the connection handler 67 | asyncio.create_task(do_work()) 68 | else: 69 | await channel.close() 70 | 71 | # --- FastAPI App Setup --- 72 | app = FastAPI() 73 | # Define the RPC endpoint (can be simplified if no server-side methods are needed) 74 | class EmptyMethods(RpcMethodsBase): 75 | pass 76 | 77 | endpoint = WebsocketRPCEndpoint(EmptyMethods(), on_connect=[on_connect]) 78 | endpoint.register_route(app, "/ws") 79 | 80 | async def main(): 81 | logging.getLogger("fastapi_ws_rpc").setLevel(logging.WARNING) 82 | """Sets up and runs the Uvicorn server.""" 83 | global server 84 | config = uvicorn.Config( 85 | app=app, 86 | host="0.0.0.0", 87 | port=${port}, 88 | log_level="warning", 89 | ws_max_size=16 * 1024 * 1024 # Example: Set max websocket message size if needed 90 | # Add other Uvicorn config options here if necessary 91 | ) 92 | server = uvicorn.Server(config) 93 | 94 | # Start the server in the background 95 | serve_task = asyncio.create_task(server.serve()) 96 | while not server.started: 97 | await asyncio.sleep(0.01) 98 | logging.getLogger().setLevel(logging.INFO) 99 | logger.info("INITIALIZED SERVER") 100 | 101 | 102 | # Wait until the shutdown event is set by do_work() 103 | await shutdown_event.wait() 104 | 105 | # Signal Uvicorn to gracefully shutdown 106 | server.should_exit = True 107 | 108 | # Give the server a moment to process the signal and shutdown tasks 109 | # You might need to adjust this delay or use server.shutdown() with awaits 110 | # if more complex cleanup is needed within Uvicorn/FastAPI's lifespan. 111 | await asyncio.sleep(0.1) 112 | 113 | # Wait for the server task to complete its shutdown 114 | await serve_task 115 | 116 | 117 | if __name__ == "__main__": 118 | try: 119 | asyncio.run(main()) 120 | except KeyboardInterrupt: 121 | print("Shutdown requested by user (Ctrl+C).") 122 | 123 | ` 124 | } 125 | 126 | 127 | function indentText(text: string, prefix: string) { 128 | let inLiteral = false; 129 | 130 | return text 131 | .split('\n') 132 | .map(line => { 133 | // count how many times """ appears on this line 134 | const matches = line.match(/"""/g) || []; 135 | const tripleCount = matches.length; 136 | 137 | // if there's at least one """ on this line, we must prefix it 138 | // (so the quotes stay indented under your async def) 139 | if (tripleCount > 0) { 140 | // prefix the line… 141 | const out = prefix + line; 142 | // …and toggle our “inLiteral” state once for each odd occurrence 143 | if (tripleCount % 2 === 1) inLiteral = !inLiteral; 144 | return out; 145 | } 146 | 147 | // no quotes here: 148 | // - if we’re inside a literal, emit the line _as is_ 149 | // - otherwise, prefix it 150 | return inLiteral ? line : prefix + line; 151 | }) 152 | .join('\n'); 153 | } -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/code-agent/factory.ts: -------------------------------------------------------------------------------- 1 | import { Agent, AgentWorkspace } from "../index.js"; 2 | import { AgentFactory, AgentConfig } from "../factory.js"; 3 | import { createAgent as createCodeAgent } from "./agent.js"; 4 | import { PluginFactory } from "../../plugins/index.js"; 5 | import { CodeToolExecutor } from "./index.js"; 6 | 7 | 8 | export type CodeAgentConfig = { 9 | codeExecutor: (workspace: AgentWorkspace) => CodeToolExecutor; 10 | } & AgentConfig; 11 | /** 12 | * Factory for creating code agents. 13 | * Implements the AgentFactory interface for code agents. 14 | */ 15 | export class CodeAgentFactory implements AgentFactory { 16 | private config: CodeAgentConfig; 17 | 18 | /** 19 | * Creates a new CodeAgentFactory with the specified configuration. 20 | * @param config Configuration for the code agent 21 | */ 22 | constructor(config: CodeAgentConfig) { 23 | this.config = config; 24 | } 25 | 26 | /** 27 | * Creates a code agent with the specified plugins. 28 | * @param plugins Array of plugin factories to extend agent functionality 29 | * @returns A Promise that resolves to the created code agent 30 | */ 31 | async create(name: string, plugins: PluginFactory[]): Promise { 32 | return createCodeAgent({ 33 | ...this.config, 34 | name: name, 35 | plugins: [...plugins, ...this.config.plugins] 36 | }); 37 | } 38 | } -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/code-agent/index.ts: -------------------------------------------------------------------------------- 1 | 2 | import { AgentTool } from "../../tools/index.js"; 3 | export { LocalPythonExecutor } from "./executors/local-executor.js"; 4 | export { CodeAgentFactory } from "./factory.js"; 5 | export { createAgent, createLgAgent } from "./agent.js"; 6 | 7 | export interface CodeToolExecutor { 8 | 9 | availableDependencies: string[]; 10 | 11 | execute(tools: AgentTool[], code: string, toolInitCallback: (wsUrl: string, tools: AgentTool[]) => void): Promise; 12 | } 13 | 14 | -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/code-agent/prompt.ts: -------------------------------------------------------------------------------- 1 | 2 | import { zodToJsonSchema } from "zod-to-json-schema"; 3 | import { AgentTool } from "../../tools/index.js"; 4 | import { toPythonFunctionName } from "./utils.js"; 5 | import { z } from "zod"; 6 | 7 | export const FUNCTION_PROMPT = ` 8 | 9 | You have the ability to execute code in a Python environment. To execute code, you can respond with a python code block wrapped in an "" xml tag. 10 | The code will be executed inside the Python environment, and whatever you print into the console will be returned to you. 11 | 12 | Python Environment Rules: 13 | - You can only include ONE block per response, do not include more than one block in your response. 14 | - Use this python environment to accomplish the task you are given, be proactive and use the function available to you but ask for help if you feel stuck on a task. 15 | - You must not ask permission or notify the user you plan on executing code, just do it. 16 | - You have been given access to a list of tools: these tools are Python functions which you can call with code. 17 | - The user cannot see the the result of the code being executed, any information you want to share with the user must responded back to them in a normal message. 18 | 19 | Example: 20 | 21 | import time 22 | import random 23 | 24 | print("Hello, world!") 25 | time.sleep(2) 26 | print("Random number:", random.randint(1, 100)) 27 | 28 | result = some_function({"field": "value"}); 29 | print(result) 30 | 31 | 32 | You are not allowed to execute any code that is not wrapped in the tag, it will be ignored. 33 | ONLY use the tag to execute code when needed, do not use it for any other purpose. 34 | 35 | ` 36 | 37 | 38 | function getFunctions(tool: AgentTool) { 39 | let outParameter = tool.outSchema ? JSON.stringify(zodToJsonSchema(tool.outSchema)) : "ToolResponse"; 40 | const toolDefinition = ` 41 | - FunctionName: ${toPythonFunctionName(tool.name)} 42 | - Description: ${tool.description} 43 | - Input Parameter: ${JSON.stringify(zodToJsonSchema(tool.schema))} 44 | - Function Output: ${outParameter} 45 | ` 46 | return toolDefinition; 47 | } 48 | 49 | 50 | export const getFunctionsPrompt = (dependencies: string[], tool: AgentTool[]) => { 51 | if (tool.length === 0) { 52 | return ""; 53 | } 54 | let functionList = [...tool.map((tool) => getFunctions(tool))] 55 | let functions = functionList.join("\n------\n"); 56 | return `\nThe python environment has the following functions available to it, use them to accomplish the requested goal from the user. 57 | The result of functions with an output parameter of "ToolResponse" can be printed with the "print" function, and the result will be returned to you. 58 | If the function has a different defined output type then its output can be used in other functions as an normal Python type. 59 | The parameters of this functions is a single Dictionary parameter, not a list of parameters. Example: functionName({"param1": "value1", "param2": "value2"}). 60 | FUNCTION LIST:\n${functions}\n\n---------------------------------\n 61 | ${dependencies.length > 0 ? `The following libraries are available in the Python environment: ${dependencies.join(", ")}` : ""} 62 | \n---------------------------------\n 63 | ` 64 | ; 65 | 66 | } 67 | 68 | export const PYTHON_SCRIPT_SCHEMA = ` 69 | 70 | 71 | 72 | Python code to be executed in the Python environment. The code must be wrapped in this tag. 73 | You can only include ONE block per response, do not include more than one block in your response. 74 | Only use the tag to execute code when needed, do not use it for any other purpose. 75 | 76 | 77 | 78 | 79 | ` -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/code-agent/utils.ts: -------------------------------------------------------------------------------- 1 | import { AIMessage, BaseMessage, HumanMessage, ToolMessage } from "@langchain/core/messages"; 2 | import { ToolResponseInfo } from "../index.js"; 3 | import { lCmessageContentToContent } from "../message-utils.js"; 4 | import { AiResponseMessage, NextMessageToolResponse } from "../../plugins/index.js"; 5 | import { extractTextContent } from "../../utils/format.js"; 6 | import { ResponseFieldMapper } from "../../utils/instruction-mapper.js"; 7 | import { v4 } from "uuid"; 8 | 9 | 10 | //TODO: remove this when we have a better way to handle this 11 | export function getExecutionCodeContentRegex(xmlString: string): string | null { 12 | if (typeof xmlString !== 'string') { 13 | console.error("Input must be a string."); 14 | return null; 15 | } 16 | 17 | // Regex explanation: 18 | // : Matches the literal opening tag. 19 | // ( : Start capturing group 1 (this is what we want to extract). 20 | // .*? : Matches any character (.), zero or more times (*), non-greedily (?). 21 | // Non-greedy is important to stop at the *first* closing tag. 22 | // ) : End capturing group 1. 23 | // <\/execution-code> : Matches the literal closing tag (the '/' needs escaping). 24 | // s : Flag to make '.' match newline characters as well (dotall). 25 | const regex = /(.*?)<\/execution-code>/s; 26 | 27 | const match = xmlString.match(regex); 28 | 29 | // If a match is found, match will be an array. 30 | // match[0] is the full matched string (e.g., "content") 31 | // match[1] is the content of the first capturing group (e.g., "content") 32 | if (match && match[1] !== undefined) { 33 | let scriptCode: string | null = match[1]; 34 | scriptCode = scriptCode.trim().length === 0 ? null : scriptCode; // Trim whitespace from the captured content 35 | return scriptCode; // Return the captured content 36 | } else { 37 | return null; // Tag not found or content is missing somehow 38 | } 39 | } 40 | 41 | export function getTextAfterLastExecutionCode(inputString: string) { 42 | // Ensure the input is a string 43 | if (typeof inputString !== 'string') { 44 | console.error("Input must be a string."); 45 | return ""; // Return empty string for non-string input 46 | } 47 | 48 | const endTag = ""; 49 | 50 | // Find the index of the *last* occurrence of the closing tag 51 | const lastIndex = inputString.lastIndexOf(endTag); 52 | 53 | // If the tag wasn't found, there's nothing after it 54 | if (lastIndex === -1) { 55 | return ""; 56 | } 57 | 58 | // Calculate the starting position of the text *after* the tag 59 | // This is the index of the end tag + the length of the tag itself 60 | const startIndex = lastIndex + endTag.length; 61 | 62 | // Extract the substring from the calculated start index to the end of the string 63 | // If startIndex is equal to the string length (tag is at the very end), 64 | // slice() correctly returns an empty string. 65 | return inputString.slice(startIndex); 66 | } 67 | 68 | export function getTextBeforeFirstExecutionCode(inputString: string) { 69 | // Ensure the input is a string 70 | if (typeof inputString !== 'string') { 71 | console.error("Input must be a string."); 72 | return ""; // Return empty string for non-string input 73 | } 74 | 75 | const startTag = ""; 76 | 77 | // Find the index of the *first* occurrence of the opening tag 78 | const firstIndex = inputString.indexOf(startTag); 79 | 80 | // If the tag wasn't found, return the entire string 81 | if (firstIndex === -1) { 82 | return inputString; 83 | } 84 | 85 | // If the tag was found, extract the portion of the string 86 | // from the beginning (index 0) up to the index where the tag starts. 87 | // If firstIndex is 0 (tag is at the beginning), slice(0, 0) correctly returns "". 88 | return inputString.slice(0, firstIndex); 89 | } 90 | 91 | 92 | export function langChainToolMessageToMimirHumanMessage(message: HumanMessage): NextMessageToolResponse { 93 | return { 94 | type: "TOOL_RESPONSE", 95 | toolName: "PYTHON_EXECUTION", 96 | toolCallId: "N/A", 97 | content: lCmessageContentToContent(message.content) 98 | }; 99 | } 100 | 101 | 102 | export function aiMessageToMimirAiMessage(aiMessage: AIMessage, files: AiResponseMessage["sharedFiles"], mapper: ResponseFieldMapper): AiResponseMessage { 103 | const textContent = extractTextContent(aiMessage.content); 104 | const scriptCode = getExecutionCodeContentRegex(textContent); 105 | const userContent = mapper.getUserMessage(lCmessageContentToContent(aiMessage.content)); 106 | 107 | const mimirMessage: AiResponseMessage = { 108 | id: aiMessage.id ?? v4(), 109 | content: userContent.tagFound ? userContent.result : scriptCode ? [] : [{ type: "text", text: textContent }], 110 | toolCalls: [], 111 | sharedFiles: files 112 | }; 113 | 114 | if (scriptCode) { 115 | mimirMessage.toolCalls = [ 116 | { 117 | id: "N/A", 118 | toolName: "PYTHON_EXECUTION", 119 | input: scriptCode 120 | }, 121 | ] 122 | } 123 | return mimirMessage; 124 | } 125 | 126 | 127 | 128 | export function toPythonFunctionName(input: string): string { 129 | // 1. Replace invalid characters with underscore 130 | let sanitized = input.replace(/[^A-Za-z0-9_]/g, '_'); 131 | 132 | // 2. If it starts with a digit, prefix an underscore 133 | if (/^[0-9]/.test(sanitized)) { 134 | sanitized = `_${sanitized}`; 135 | } 136 | 137 | // 3. Ensure it’s not empty (optional—returns '_' if input was all invalid) 138 | return sanitized.length > 0 ? sanitized : '_'; 139 | } -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/code-agent/wrapper.ts: -------------------------------------------------------------------------------- 1 | import { AgentTool } from "../../tools/index.js"; 2 | import { z } from "zod"; 3 | 4 | import { ComplexMessageContent } from "../../schema.js"; 5 | import { StructuredTool, ToolRunnableConfig } from "@langchain/core/tools"; 6 | import { complexResponseToLangchainMessageContent } from "./../../utils/format.js"; 7 | import { CallbackManagerForToolRun } from "@langchain/core/callbacks/manager"; 8 | 9 | export class MimirToolToLangchainTool extends StructuredTool { 10 | 11 | schema = this.tool.schema; 12 | name: string = this.tool.name; 13 | description: string = this.tool.description; 14 | 15 | constructor(private tool: AgentTool) { 16 | super(); 17 | } 18 | 19 | protected async _call(arg: z.input, runManager?: CallbackManagerForToolRun, parentConfig?: ToolRunnableConfig): Promise { 20 | const response = await this.tool.call(arg); 21 | return complexResponseToLangchainMessageContent(response as ComplexMessageContent[]); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/constants.ts: -------------------------------------------------------------------------------- 1 | export const CONSTANTS = { 2 | MESSAGE_DIVIDER: "\n--------------------------------------------------\n\n", 3 | CONTENT_SPACING: "\n-----------------------------------------------\n\n", 4 | MAX_TOOL_RESPONSE_LENGTH: 400, 5 | DEFAULT_THREAD_ID: "1", 6 | DB_FILENAME: "agent-chat.db" 7 | } as const; 8 | 9 | export const ERROR_MESSAGES = { 10 | UNREACHABLE: "Unreachable", 11 | UNSUPPORTED_CONTENT_TYPE: (type: string) => `Unsupported content type: ${type}`, 12 | } as const; 13 | 14 | 15 | 16 | export const DEFAULT_CONSTITUTION = `You are an expert assistant who can solve any task using code blobs. You will be given a task to solve as best you can.` -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/factory.ts: -------------------------------------------------------------------------------- 1 | import { Agent } from "./index.js"; 2 | import { PluginFactory } from "../plugins/index.js"; 3 | import { BaseChatModel } from "@langchain/core/language_models/chat_models"; 4 | import { WorkspaceFactory } from "./index.js"; 5 | import { BaseCheckpointSaver } from "@langchain/langgraph"; 6 | 7 | /** 8 | * Configuration for creating an agent. 9 | * Contains all necessary parameters except plugins, which are provided separately. 10 | */ 11 | export interface AgentConfig { 12 | /** The professional role or expertise of the agent */ 13 | profession: string; 14 | /** A description of the agent's purpose and capabilities */ 15 | description: string; 16 | /** The unique name identifier for the agent */ 17 | // name: string; 18 | /** The language model to be used by the agent */ 19 | model: BaseChatModel; 20 | /** Optional constitution defining agent behavior guidelines */ 21 | constitution?: string; 22 | /** Optional vision support type (currently only supports 'openai') */ 23 | visionSupport?: 'openai'; 24 | /** Factory function to create the agent's workspace */ 25 | workspaceFactory: WorkspaceFactory; 26 | 27 | plugins: PluginFactory[], 28 | 29 | checkpointer?: BaseCheckpointSaver 30 | } 31 | 32 | /** 33 | * Interface for agent factories that create different types of agents. 34 | * Provides a common interface for creating agents regardless of their implementation. 35 | */ 36 | export interface AgentFactory { 37 | /** 38 | * Creates an agent with the specified plugins. 39 | * @param plugins Array of plugin factories to extend agent functionality 40 | * @returns A Promise that resolves to the created agent 41 | */ 42 | create(name: string, plugins: PluginFactory[]): Promise; 43 | } -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/index.ts: -------------------------------------------------------------------------------- 1 | import { AgentCommand } from "../plugins/index.js"; 2 | import { ComplexMessageContent } from "../schema.js"; 3 | export { AgentFactory, AgentConfig } from "./factory.js"; 4 | /** 5 | * Represents a tool use request from an agent. 6 | * Contains information about which tool to use and its input parameters. 7 | */ 8 | export type MessageContentToolUse = { 9 | /** The name of the tool to be executed */ 10 | toolName: string; 11 | /** Input parameters for the tool as key-value pairs */ 12 | input: string, 13 | /** Optional unique identifier for this tool use request */ 14 | id?: string, 15 | } 16 | /** 17 | * Represents a message from an agent that includes tool use requests. 18 | * Extends InputAgentMessage to include an array of tool calls. 19 | */ 20 | export type AgentMessageToolRequest = { toolCalls: MessageContentToolUse[] } & OutputAgentMessage; 21 | 22 | export type OutputAgentMessage = {id: string} & InputAgentMessage 23 | /** 24 | * Represents an input message to an agent. 25 | * Contains the message content and optional shared files. 26 | */ 27 | export type InputAgentMessage = { 28 | /** Array of complex message content (can include text, images, etc.) */ 29 | content: ComplexMessageContent[], 30 | /** Optional array of shared files with their URLs and names */ 31 | sharedFiles?: SharedFile[] 32 | }; 33 | 34 | export type SharedFile = { 35 | /** URL where the file can be accessed */ 36 | url: string, 37 | /** Name of the file */ 38 | fileName: string, 39 | } 40 | 41 | /** 42 | * Factory function type for creating agent workspaces. 43 | * Takes a working directory path and returns a Promise that resolves to an AgentWorkspace. 44 | */ 45 | export type WorkspaceFactory = (workDirectory: string) => Promise; 46 | 47 | 48 | 49 | /** 50 | * Core interface defining an agent's structure and capabilities. 51 | * Represents a single agent instance with its properties and methods. 52 | */ 53 | export interface Agent { 54 | /** Unique name identifier for the agent */ 55 | name: string, 56 | /** Description of the agent's purpose and capabilities */ 57 | description: string, 58 | /** 59 | * Primary method for interacting with the agent. 60 | * Processes input messages and generates responses or tool requests. 61 | * @param args.message - The input message to process, can be null 62 | * @param args.noMessagesInTool - Optional flag to prevent message processing in tools 63 | * @returns AsyncGenerator yielding tool responses and final agent response 64 | */ 65 | call: (args: { 66 | message: InputAgentMessage | null, 67 | threadId: string, 68 | noMessagesInTool?: boolean 69 | }) => AsyncGenerator, 73 | /** 74 | * Processes specific commands sent to the agent. 75 | * @param args.command - The command request to handle 76 | * @returns AsyncGenerator yielding tool responses and final agent response 77 | */ 78 | handleCommand: (args: { 79 | command: CommandRequest, 80 | threadId: string 81 | }) => AsyncGenerator, 85 | /** The agent's workspace for file operations */ 86 | workspace: AgentWorkspace, 87 | /** Array of commands available to this agent */ 88 | commands: AgentCommand[], 89 | /** Resets the agent's state */ 90 | reset: (args: { 91 | threadId: string, 92 | checkpointId?: string 93 | }) => Promise, 94 | }; 95 | 96 | /** 97 | * Represents a response from an agent requesting to use a tool. 98 | */ 99 | export type AgentToolRequestResponse = { 100 | /** Identifies this as a tool request response */ 101 | type: "toolRequest", 102 | /** The tool request message */ 103 | output: AgentMessageToolRequest, 104 | /** Additional attributes for the response */ 105 | responseAttributes: Record 106 | } 107 | 108 | /** 109 | * Represents a response from an agent directed to a user. 110 | */ 111 | export type AgentUserMessageResponse = { 112 | // id: string, 113 | /** Identifies this as an agent response */ 114 | type: "agentResponse", 115 | /** The message content */ 116 | output: OutputAgentMessage, //TODO THIS IS MISSING AND ID 117 | /** Additional attributes for the response */ 118 | responseAttributes: Record 119 | } 120 | 121 | /** 122 | * Union type representing all possible agent response types. 123 | */ 124 | export type AgentResponse = AgentToolRequestResponse | AgentUserMessageResponse; 125 | 126 | /** 127 | * Represents a command request that can be sent to an agent. 128 | */ 129 | export type CommandRequest = { 130 | /** Name of the command to execute */ 131 | name: string, 132 | /** Optional arguments for the command */ 133 | arguments?: Record 134 | } 135 | 136 | export type IntermediateAgentMessage = { 137 | type: "toolResponse", 138 | toolResponse: ToolResponseInfo 139 | } | { 140 | type: "messageChunk", 141 | chunk: { 142 | id: string, 143 | content: ComplexMessageContent[] 144 | } 145 | } 146 | 147 | /** 148 | * Information about a tool's response after execution. 149 | */ 150 | export type ToolResponseInfo = { 151 | /** Optional unique identifier for the tool response */ 152 | id?: string, 153 | /** Name of the tool that was executed */ 154 | name: string, 155 | /** Array of complex message content representing the tool's response */ 156 | response: ComplexMessageContent[] 157 | } 158 | 159 | 160 | /** 161 | * Defines the workspace interface for an agent, providing file system operations 162 | * and directory management capabilities. 163 | */ 164 | export type AgentWorkspace = { 165 | /** Lists all files in the workspace */ 166 | listFiles(): Promise, 167 | /** Loads a file into the workspace from a URL */ 168 | loadFileToWorkspace(fileName: string, url: string): Promise, 169 | /** Resets the workspace to its initial state */ 170 | reset(): Promise, 171 | /** Gets the URL for a file in the workspace */ 172 | getUrlForFile(fileName: string): Promise, 173 | /** Gets a file's contents as a Buffer */ 174 | fileAsBuffer(fileName: string): Promise, 175 | /** The current working directory path */ 176 | workingDirectory: string, 177 | /** The root directory path of the workspace */ 178 | rootDirectory: string, 179 | } 180 | -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/message-utils.ts: -------------------------------------------------------------------------------- 1 | import { v4 } from "uuid"; 2 | import { AIMessage, BaseMessage, HumanMessage, MessageContent, MessageContentComplex, MessageContentImageUrl, MessageContentText, SystemMessage } from "@langchain/core/messages"; 3 | import { ComplexMessageContent, ImageMessageContent, TextMessageContent, SupportedImageTypes, } from "../schema.js"; 4 | import { CONSTANTS, ERROR_MESSAGES } from "./constants.js"; 5 | import { complexResponseToLangchainMessageContent } from "../utils/format.js"; 6 | import { InputAgentMessage } from "./index.js"; 7 | 8 | 9 | export function commandContentToBaseMessage(commandContent: { type: string, content: ComplexMessageContent[] }): BaseMessage { 10 | const id = v4(); 11 | const content = complexResponseToLangchainMessageContent(commandContent.content); 12 | 13 | if (commandContent.type === "assistant") { 14 | return new AIMessage({ id, content }); 15 | } else if (commandContent.type === "user") { 16 | return new HumanMessage({ id, content }); 17 | } 18 | throw new Error(ERROR_MESSAGES.UNREACHABLE); 19 | } 20 | 21 | export function lCmessageContentToContent(content: MessageContent): ComplexMessageContent[] { 22 | if (typeof content === 'string') { 23 | return [{ 24 | type: "text", 25 | text: content 26 | }]; 27 | } 28 | 29 | return (content as MessageContentComplex[]).map(c => { 30 | if (c.type === "text") { 31 | return { 32 | type: "text" as const, 33 | text: (c as MessageContentText).text 34 | } satisfies TextMessageContent; 35 | } 36 | 37 | if (c.type === "image_url") { 38 | const imgContent = c as MessageContentImageUrl; 39 | const imageUrl = typeof imgContent.image_url === 'string' ? 40 | imgContent.image_url : 41 | imgContent.image_url.url; 42 | 43 | return { 44 | type: "image_url" as const, 45 | image_url: { 46 | //TODO: We need to handle images per LLM provider, no LLM currently supports responding image types. 47 | //TODO THIS IS WRONG, WE DONT KNOW THE TYPE OF IMAGE RETURNED FROM THE LLM 48 | type: "jpeg" satisfies SupportedImageTypes, 49 | url: imageUrl, 50 | } 51 | } satisfies ImageMessageContent; 52 | } 53 | 54 | return null 55 | }).filter(e => e !== null).map(e => e!); 56 | } 57 | 58 | export function mergeSystemMessages(messages: SystemMessage[]): SystemMessage { 59 | return messages.reduce((prev, next) => { 60 | const prevContent = typeof prev.content === 'string' ? 61 | [{ type: "text", text: prev.content }] satisfies MessageContentText[] : 62 | prev.content satisfies MessageContentComplex[]; 63 | 64 | const nextContent = typeof next.content === 'string' ? 65 | [{ type: "text", text: next.content }] satisfies MessageContentText[] : 66 | next.content satisfies MessageContentComplex[]; 67 | 68 | return new SystemMessage({ content: [...prevContent, ...nextContent] }); 69 | }, new SystemMessage({ content: [] })); 70 | } 71 | 72 | export const dividerSystemMessage = { 73 | type: "text", 74 | text: CONSTANTS.MESSAGE_DIVIDER 75 | } satisfies ComplexMessageContent; 76 | 77 | 78 | export function humanMessageToInputAgentMessage(message: HumanMessage) : InputAgentMessage { 79 | return { 80 | content: lCmessageContentToContent(message.content), 81 | sharedFiles: [ 82 | ...(message.response_metadata?.["sharedFiles"] ?? []) 83 | ] 84 | } 85 | } -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/tool-agent/factory.ts: -------------------------------------------------------------------------------- 1 | import { Agent } from "../index.js"; 2 | import { AgentFactory, AgentConfig } from "../factory.js"; 3 | import { createAgent as createFunctionAgent } from "./agent.js"; 4 | import { PluginFactory } from "../../plugins/index.js"; 5 | 6 | /** 7 | * Factory for creating function agents. 8 | * Implements the AgentFactory interface for function agents. 9 | */ 10 | export class FunctionAgentFactory implements AgentFactory { 11 | private config: AgentConfig; 12 | 13 | /** 14 | * Creates a new FunctionAgentFactory with the specified configuration. 15 | * @param config Configuration for the function agent 16 | */ 17 | constructor(config: AgentConfig) { 18 | this.config = config; 19 | } 20 | 21 | /** 22 | * Creates a function agent with the specified plugins. 23 | * @param plugins Array of plugin factories to extend agent functionality 24 | * @returns A Promise that resolves to the created function agent 25 | */ 26 | async create(name: string, plugins: PluginFactory[]): Promise { 27 | return createFunctionAgent({ 28 | ...this.config, 29 | name: name, 30 | plugins: [...plugins, ...this.config.plugins] 31 | }); 32 | } 33 | } -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/tool-agent/index.ts: -------------------------------------------------------------------------------- 1 | export { FunctionAgentFactory } from "./factory.js"; 2 | export { createAgent, createLgAgent } from "./agent.js"; 3 | -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/tool-agent/tool-node.ts: -------------------------------------------------------------------------------- 1 | import { 2 | BaseMessage, 3 | ToolMessage, 4 | AIMessage, 5 | isBaseMessage, 6 | } from "@langchain/core/messages"; 7 | import { RunnableConfig, RunnableToolLike } from "@langchain/core/runnables"; 8 | import { StructuredToolInterface } from "@langchain/core/tools"; 9 | import { END, isCommand, isGraphInterrupt, MessagesAnnotation } from "@langchain/langgraph"; 10 | 11 | export type ToolNodeOptions = { 12 | name?: string; 13 | tags?: string[]; 14 | handleToolErrors?: boolean; 15 | }; 16 | 17 | export const toolNodeFunction = ( 18 | tools: (StructuredToolInterface | RunnableToolLike)[], 19 | options?: ToolNodeOptions) => { 20 | 21 | return async (input: typeof MessagesAnnotation.State, config: RunnableConfig) => { 22 | const message = Array.isArray(input) 23 | ? input[input.length - 1] 24 | : input.messages[input.messages.length - 1]; 25 | 26 | if (message?._getType() !== "ai") { 27 | throw new Error("ToolNode only accepts AIMessages as input."); 28 | } 29 | 30 | const outputs = []; 31 | for (const call of (message as AIMessage).tool_calls ?? []) { 32 | const tool = tools.find((tool) => tool.name === call.name); 33 | try { 34 | if (tool === undefined) { 35 | throw new Error(`Tool "${call.name}" not found.`); 36 | } 37 | const output = await tool.invoke( 38 | { ...call, type: "tool_call" }, 39 | config 40 | ); 41 | if ( 42 | (isBaseMessage(output) && output._getType() === "tool") || 43 | isCommand(output) 44 | ) { 45 | outputs.push(output); 46 | } else { 47 | outputs.push(new ToolMessage({ 48 | name: tool.name, 49 | content: 50 | typeof output === "string" ? output : JSON.stringify(output), 51 | tool_call_id: call.id!, 52 | })); 53 | } 54 | 55 | } catch (e: any) { 56 | if (!options?.handleToolErrors) { 57 | throw e; 58 | } 59 | if (isGraphInterrupt(e.name)) { 60 | // `NodeInterrupt` errors are a breakpoint to bring a human into the loop. 61 | // As such, they are not recoverable by the agent and shouldn't be fed 62 | // back. Instead, re-throw these errors even when `handleToolErrors = true`. 63 | throw e; 64 | } 65 | outputs.push(new ToolMessage({ 66 | content: `Error: ${e.message}\n Please fix your mistakes.`, 67 | name: call.name, 68 | tool_call_id: call.id ?? "", 69 | })); 70 | } 71 | } 72 | 73 | // Preserve existing behavior for non-command tool outputs for backwards compatibility 74 | if (!outputs.some(isCommand)) { 75 | return (Array.isArray(input) ? outputs : { messages: outputs }) as any; 76 | } 77 | 78 | // Handle mixed Command and non-Command outputs 79 | const combinedOutputs = outputs.map((output) => { 80 | if (isCommand(output)) { 81 | return output; 82 | } 83 | return Array.isArray(input) ? [output] : { messages: [output] }; 84 | }); 85 | return combinedOutputs as any; 86 | } 87 | }; 88 | 89 | 90 | export function toolsCondition( 91 | state: BaseMessage[] | typeof MessagesAnnotation.State 92 | ): "tools" | typeof END { 93 | const message = Array.isArray(state) 94 | ? state[state.length - 1] 95 | : state.messages[state.messages.length - 1]; 96 | 97 | if ( 98 | "tool_calls" in message && 99 | ((message as AIMessage).tool_calls?.length ?? 0) > 0 100 | ) { 101 | return "tools"; 102 | } else { 103 | return END; 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/tool-agent/utils.ts: -------------------------------------------------------------------------------- 1 | import { AIMessage, HumanMessage, ToolMessage } from "@langchain/core/messages"; 2 | import { AiResponseMessage, NextMessageToolResponse, NextMessageUser } from "../../plugins/index.js"; 3 | import { lCmessageContentToContent } from "../message-utils.js"; 4 | import { MessageContentToolUse, ToolResponseInfo } from "../index.js"; 5 | import { ResponseFieldMapper } from "../../utils/instruction-mapper.js"; 6 | import { v4 } from "uuid"; 7 | 8 | export function langChainToolMessageToMimirToolMessage(message: ToolMessage): NextMessageToolResponse { 9 | return { 10 | type: "TOOL_RESPONSE", 11 | toolName: message.name ?? "Unknown", 12 | toolCallId: message.tool_call_id, 13 | content: lCmessageContentToContent(message.content) 14 | }; 15 | } 16 | 17 | export function langChainHumanMessageToMimirHumanMessage(message: HumanMessage): NextMessageUser { 18 | return { 19 | type: "USER_MESSAGE", 20 | sharedFiles: [ 21 | ...(message.response_metadata?.["sharedFiles"] ?? []) 22 | ], 23 | content: lCmessageContentToContent(message.content) 24 | }; 25 | } 26 | 27 | export function aiMessageToMimirAiMessage(aiMessage: AIMessage, files: AiResponseMessage["sharedFiles"], mapper: ResponseFieldMapper): AiResponseMessage { 28 | const userContent = mapper.getUserMessage(lCmessageContentToContent(aiMessage.content)); 29 | const mimirMessage: AiResponseMessage = { 30 | id: aiMessage.id ?? v4(), 31 | content: userContent.result, 32 | toolCalls: [], 33 | sharedFiles: files 34 | }; 35 | 36 | if (aiMessage.tool_calls) { 37 | const tool_calls = aiMessage.tool_calls.map(t => { 38 | return { 39 | toolName: t.name, 40 | input: JSON.stringify(t.args), 41 | id: t.id 42 | } satisfies MessageContentToolUse 43 | }); 44 | mimirMessage.toolCalls = tool_calls; 45 | } 46 | 47 | return mimirMessage; 48 | } 49 | 50 | -------------------------------------------------------------------------------- /agent-mimir/src/agent-manager/tool-agent/wrapper.ts: -------------------------------------------------------------------------------- 1 | import { AgentTool } from "../../tools/index.js"; 2 | import { z } from "zod"; 3 | import { ComplexMessageContent } from "../../schema.js"; 4 | import { StructuredTool, ToolRunnableConfig } from "@langchain/core/tools"; 5 | import { complexResponseToLangchainMessageContent } from "../../utils/format.js"; 6 | import { CallbackManagerForToolRun } from "@langchain/core/callbacks/manager"; 7 | 8 | export class MimirToolToLangchainTool extends StructuredTool { 9 | schema; 10 | name; 11 | description; 12 | 13 | constructor(private tool: AgentTool) { 14 | super(); 15 | this.schema = tool.schema 16 | this.name = tool.name; 17 | this.description = tool.description; 18 | 19 | } 20 | 21 | protected async _call(arg: z.input, runManager?: CallbackManagerForToolRun, parentConfig?: ToolRunnableConfig): Promise { 22 | const response = await this.tool.call(arg); 23 | return complexResponseToLangchainMessageContent(response as ComplexMessageContent[]); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /agent-mimir/src/communication/helpers.ts: -------------------------------------------------------------------------------- 1 | 2 | import { AgentSystemMessage, AgentPlugin, PluginFactory, PluginContext, NextMessage, AttributeDescriptor } from "../plugins/index.js"; 3 | import { Agent } from "../agent-manager/index.js"; 4 | 5 | export type HelperPluginConfig = { 6 | name: string, 7 | helperSingleton: ReadonlyMap, 8 | destinationAgentFieldName: string, 9 | communicationWhitelist: string[] | null, 10 | } 11 | 12 | export class HelpersPluginFactory implements PluginFactory { 13 | 14 | name: string = "helpers"; 15 | 16 | constructor(private config: HelperPluginConfig) { 17 | } 18 | 19 | async create(context: PluginContext): Promise { 20 | 21 | return new HelpersPlugin(this.config); 22 | } 23 | } 24 | 25 | 26 | export class HelpersPlugin extends AgentPlugin { 27 | 28 | private helperSingleton: ReadonlyMap; 29 | private communicationWhitelist: string[] | null; 30 | private agentName: string; 31 | private destinationAgentFieldName: string; 32 | 33 | name: string = "Multi Agent Communication Plugin"; 34 | 35 | constructor(config: HelperPluginConfig) { 36 | super(); 37 | this.helperSingleton = config.helperSingleton; 38 | this.communicationWhitelist = config.communicationWhitelist; 39 | this.agentName = config.name; 40 | this.destinationAgentFieldName = config.destinationAgentFieldName; 41 | } 42 | 43 | async attributes(nextMessage: NextMessage): Promise { 44 | return [ 45 | { 46 | attributeType: "string", 47 | name: "agentNameToWhichSendTheMessage", 48 | required: false, 49 | variableName: this.destinationAgentFieldName, 50 | description: "Set this attribute to the name of the Agents you want to send a message. Only set it if you want to send a message to an Agents, else do not set it. When set, the message you send will be sent to that agent instead of the user. If not set you will be responding to the user.", 51 | } 52 | ]; 53 | } 54 | 55 | async getSystemMessages(): Promise { 56 | 57 | const helpers = [...this.helperSingleton.values()]; 58 | const whiteList = this.communicationWhitelist ?? helpers.map((helper) => helper.name) ?? []; 59 | const helperList = helpers.filter((helper) => helper.name !== this.agentName) 60 | .filter(element => whiteList.includes(element.name)) 61 | .map((helper) => `${helper.name}: ${helper.description}`) 62 | .join("\n") ?? ""; 63 | const helpersMessage = helperList !== "" ? `You have access to the following Agents that you can talk to in order to assist you in your tasks. To talk to them you must set their name on the XML element or else the message will be incorrectly sent to the user:\n${helperList}` : ``; 64 | 65 | return { 66 | content: [ 67 | { 68 | type: "text", 69 | text: helpersMessage 70 | } 71 | ] 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /agent-mimir/src/nodejs/filesystem-work-directory.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | 3 | import { promises as fs } from 'fs'; 4 | import { AgentWorkspace } from "../agent-manager/index.js"; 5 | 6 | export class FileSystemAgentWorkspace implements AgentWorkspace { 7 | 8 | get workingDirectory() { 9 | return path.join(this.agentRootDirectory, "workspace"); 10 | } 11 | 12 | get rootDirectory() { 13 | return this.agentRootDirectory; 14 | } 15 | 16 | constructor(private agentRootDirectory: string) { 17 | } 18 | 19 | async fileAsBuffer(fileName: string): Promise { 20 | if ((await this.listFiles()).includes(fileName)) { 21 | const fileData = await fs.readFile(path.join(this.workingDirectory, fileName)); 22 | return fileData; 23 | } 24 | return undefined; 25 | } 26 | 27 | 28 | async reset(): Promise { 29 | const files = await fs.readdir(this.workingDirectory); 30 | for (const file of files) { 31 | await fs.unlink(path.join(this.workingDirectory, file)); 32 | } 33 | } 34 | 35 | 36 | async listFiles(): Promise { 37 | const files = await fs.readdir(this.workingDirectory); 38 | return files; 39 | } 40 | async loadFileToWorkspace(fileName: string, url: string): Promise { 41 | const destination = path.join(this.workingDirectory, fileName); 42 | await fs.copyFile(url, destination); 43 | console.debug(`Copied file ${url} to ${destination}`); 44 | } 45 | 46 | async getUrlForFile(fileName: string): Promise { 47 | const file = path.join(this.workingDirectory, fileName); 48 | return file; 49 | } 50 | } -------------------------------------------------------------------------------- /agent-mimir/src/nodejs/index.ts: -------------------------------------------------------------------------------- 1 | 2 | 3 | export { FileSystemAgentWorkspace } from './filesystem-work-directory.js' -------------------------------------------------------------------------------- /agent-mimir/src/plugins/default-plugins.ts: -------------------------------------------------------------------------------- 1 | import { AgentSystemMessage, AgentPlugin, PluginFactory, PluginContext, NextMessage, AttributeDescriptor } from "./index.js"; 2 | 3 | export class DefaultPluginFactory implements PluginFactory { 4 | 5 | name: string = "time"; 6 | 7 | async create(context: PluginContext): Promise { 8 | return new DefaultPlugin(); 9 | } 10 | 11 | } 12 | 13 | class DefaultPlugin extends AgentPlugin { 14 | 15 | async getSystemMessages(): Promise { 16 | return { 17 | content: [ 18 | { 19 | type: "text", 20 | text: `The current time is: ${new Date().toISOString()}` 21 | } 22 | ] 23 | }; 24 | } 25 | 26 | async attributes(nextMessage: NextMessage): Promise { 27 | const attributes: AttributeDescriptor[] = []; 28 | if (nextMessage.type === "TOOL_RESPONSE") { 29 | attributes.push({ 30 | name: "stepResult", 31 | attributeType: "string", 32 | required: true, 33 | variableName: "taskDesc", 34 | description: "Description of results of your previous action as well as a description of the state of the lastest element you interacted with.", 35 | example: "Example 1: I can see that the file was modified correctly and now contains the edited text. Example 2: I can see that the file was not modified correctly the text was not added.", 36 | }); 37 | } 38 | return attributes; 39 | } 40 | } 41 | 42 | 43 | -------------------------------------------------------------------------------- /agent-mimir/src/plugins/index.ts: -------------------------------------------------------------------------------- 1 | import { AgentMessageToolRequest, AgentWorkspace, InputAgentMessage } from "../agent-manager/index.js"; 2 | import { ComplexMessageContent } from "../schema.js"; 3 | import { AgentTool } from "../tools/index.js"; 4 | 5 | /** 6 | * Represents a message response from the AI agent containing a tool request. 7 | */ 8 | export type AiResponseMessage = AgentMessageToolRequest; 9 | 10 | /** 11 | * Describes an attribute that a plugin can provide for the agent to populate. 12 | * Used to generate additional data useful for the plugin. 13 | */ 14 | export type AttributeDescriptor = { 15 | name: string, 16 | attributeType: string, 17 | variableName: string, 18 | description: string, 19 | required: boolean, 20 | example?: string, 21 | } 22 | 23 | /** 24 | * Represents a command that can be executed by the agent. 25 | * Similar to Discord commands, these are used to interact with the agent. 26 | */ 27 | export type AgentCommand = { 28 | name: string, 29 | description?: string, 30 | commandHandler: (args: Record) => Promise, 31 | arguments?: { 32 | name: string, 33 | description?: string, 34 | required: boolean 35 | }[] 36 | } 37 | 38 | 39 | /** 40 | * Context provided to plugins during initialization. 41 | * Contains workspace, persistence directory, and agent information. 42 | */ 43 | export type PluginContext = { 44 | workspace: AgentWorkspace, 45 | } 46 | 47 | 48 | 49 | /** 50 | * Represents the next message in the conversation flow. 51 | * Can be either a user message or a tool response. 52 | */ 53 | export type NextMessage = NextMessageUser | NextMessageToolResponse; 54 | 55 | /** 56 | * Represents a user message in the conversation flow. 57 | */ 58 | export type NextMessageUser = InputAgentMessage & { type: "USER_MESSAGE" } 59 | 60 | /** 61 | * Represents a tool's response message in the conversation flow. 62 | */ 63 | export type NextMessageToolResponse = { 64 | type: "TOOL_RESPONSE", 65 | toolCallId: string, 66 | toolName: string, 67 | content: ComplexMessageContent[] 68 | } 69 | 70 | 71 | /** 72 | * Represents additional content that can be added to messages. 73 | * Controls whether content should be saved to history and/or displayed. 74 | */ 75 | export type AdditionalContent = { 76 | saveToChatHistory: boolean | number, 77 | displayOnCurrentMessage: boolean, 78 | content: ComplexMessageContent[] 79 | } 80 | 81 | 82 | /** 83 | * Factory interface for creating Mimir agent plugins. 84 | * Provides a standardized way to instantiate plugins with context. 85 | */ 86 | export interface PluginFactory { 87 | name: string; 88 | create(context: PluginContext): Promise 89 | } 90 | 91 | /** 92 | * Abstract base class for Mimir agent plugins. 93 | * Provides the core functionality and lifecycle hooks that plugins can implement. 94 | */ 95 | export abstract class AgentPlugin { 96 | 97 | /** 98 | * Name of the plugin. 99 | */ 100 | name?: string; 101 | 102 | 103 | /** 104 | * Initializes the plugin. 105 | * Called when the plugin is first loaded. 106 | */ 107 | init(): Promise { 108 | return Promise.resolve(); 109 | } 110 | 111 | /** 112 | * Called when the agent is ready to proceed to the next message. 113 | * Allows plugins to prepare for the next interaction. 114 | * @param nextMessage - The next message in the conversation 115 | */ 116 | async readyToProceed(nextMessage: NextMessage): Promise { 117 | } 118 | 119 | /** 120 | * Adds additional content to the user's message. 121 | * @returns Array of additional content to be added 122 | */ 123 | async additionalMessageContent(message: InputAgentMessage): Promise { 124 | return []; 125 | } 126 | 127 | /** 128 | * Adds additional content to the system message. 129 | * @returns System message content to be added 130 | */ 131 | async getSystemMessages(): Promise { 132 | return { 133 | content: [] 134 | }; 135 | } 136 | 137 | /** 138 | * Reads the response from the agent and processes response attributes. 139 | * @param aiMessage - The response message from the AI 140 | * @param responseAttributes - Current response attributes 141 | */ 142 | async readResponse(aiMessage: AiResponseMessage, responseAttributes: Record): Promise { 143 | return; 144 | } 145 | 146 | /** 147 | * Resets the plugin to its initial state. 148 | */ 149 | async reset(): Promise { 150 | } 151 | 152 | /** 153 | * Returns the attributes that the plugin can provide for the agent to populate. 154 | * These attributes are used to generate additional data useful for the plugin. 155 | * @param context - Current agent context 156 | * @returns Array of attribute descriptors 157 | */ 158 | async attributes(nextMessage: NextMessage): Promise { 159 | return []; 160 | } 161 | 162 | /** 163 | * Returns the tools that the plugin provides to the agent. 164 | * @returns Array of agent tools, either synchronously or as a promise 165 | */ 166 | async tools(): Promise<(AgentTool)[]> { 167 | return []; 168 | } 169 | 170 | /** 171 | * Returns the commands that the plugin provides. 172 | * Commands are similar to Discord commands and are used to interact with the agent. 173 | * @returns Array of available commands 174 | */ 175 | async getCommands(): Promise { 176 | return []; 177 | } 178 | } 179 | 180 | /** 181 | * Represents the content of a command response. 182 | * Can be either user content or assistant content. 183 | */ 184 | export type CommandContent = { 185 | type: "user", 186 | content: ComplexMessageContent[] 187 | } | { 188 | type: "assistant", 189 | content: ComplexMessageContent[] 190 | } 191 | 192 | 193 | 194 | /** 195 | * Represents a system message that can be added by plugins. 196 | */ 197 | export type AgentSystemMessage = { 198 | content: ComplexMessageContent[] 199 | } 200 | -------------------------------------------------------------------------------- /agent-mimir/src/plugins/workspace.ts: -------------------------------------------------------------------------------- 1 | 2 | import { promises as fs } from 'fs'; 3 | import { AgentSystemMessage, AttributeDescriptor, AgentPlugin, PluginFactory, PluginContext } from "./index.js"; 4 | import { AgentWorkspace, InputAgentMessage, SharedFile } from "../agent-manager/index.js"; 5 | import { ComplexMessageContent } from '../schema.js'; 6 | 7 | export class WorkspacePluginFactory implements PluginFactory { 8 | name: string = "workspace"; 9 | 10 | async create(context: PluginContext): Promise { 11 | return new WorkspacePlugin(context.workspace); 12 | } 13 | } 14 | 15 | export class WorkspanceManager { 16 | 17 | private workspace: AgentWorkspace; 18 | 19 | constructor(workspace: AgentWorkspace) { 20 | this.workspace = workspace; 21 | } 22 | 23 | async loadFiles(sharedFiles: SharedFile[]): Promise { 24 | for (const file of sharedFiles ?? []) { 25 | await this.workspace.loadFileToWorkspace(file.fileName, file.url); 26 | } 27 | 28 | } 29 | 30 | async readAttributes(responseAttributes: Record): Promise { 31 | 32 | if (responseAttributes["workspaceFilesToShare"]) { 33 | const files = await Promise.all((JSON.parse(responseAttributes["workspaceFilesToShare"]) || []) 34 | .map(async (file: string) => ({ fileName: file, url: (await this.workspace.getUrlForFile(file))! }))); 35 | 36 | return files; 37 | } 38 | return [] 39 | } 40 | 41 | async additionalMessageContent(nextMessage: InputAgentMessage): Promise { 42 | 43 | if (nextMessage.sharedFiles && nextMessage.sharedFiles.length > 0) { 44 | const filesToSendMessage = nextMessage.sharedFiles.map((file: any) => `"${file.fileName}"`).join(", "); 45 | return [ 46 | { 47 | type: "text", 48 | text: `I am sending the following files into your workspace: ${filesToSendMessage} \n\n` 49 | } 50 | ] 51 | } 52 | return [] 53 | } 54 | 55 | 56 | } 57 | class WorkspacePlugin extends AgentPlugin { 58 | 59 | private workspace: AgentWorkspace; 60 | 61 | 62 | constructor(workspace: AgentWorkspace) { 63 | super(); 64 | this.workspace = workspace; 65 | } 66 | 67 | async init(): Promise { 68 | if (this.workspace.workingDirectory) { 69 | await fs.mkdir(this.workspace.workingDirectory, { recursive: true }); 70 | } 71 | } 72 | 73 | 74 | 75 | async getSystemMessages(): Promise { 76 | const files = (await this.workspace.listFiles()); 77 | const message = files.length > 0 ? `You have the following files in your workspace: ${files.join(", ")}` : "There are currently no files in your workspace."; 78 | return { 79 | content: [ 80 | { 81 | type: "text", 82 | text: message 83 | } 84 | ] 85 | } 86 | } 87 | 88 | async attributes(): Promise { 89 | return [ 90 | { 91 | attributeType: "string[]", 92 | description: "The list of files from your workspace you want to send back to the user. Respond back files you want to send back or the user has requested.", 93 | name: "workspaceFilesToShare", 94 | example: `["image.jpg", "textFile.txt", "movie.avi"]`, 95 | required: false, 96 | variableName: "workspaceFilesToShare" 97 | } 98 | ]; 99 | } 100 | 101 | } -------------------------------------------------------------------------------- /agent-mimir/src/schema.ts: -------------------------------------------------------------------------------- 1 | 2 | export type ImageType = { 3 | url: string, 4 | type: SupportedImageTypes 5 | } 6 | 7 | export type SupportedImageTypes = "url" | "jpeg" | "png"; 8 | 9 | export type TextMessageContent = { 10 | type: "text"; 11 | text: string; 12 | }; 13 | export type ImageMessageContent = { 14 | type: "image_url"; 15 | image_url: { 16 | url: string; 17 | type: SupportedImageTypes; 18 | }; 19 | }; 20 | 21 | export type ComplexMessageContent = TextMessageContent | ImageMessageContent 22 | 23 | -------------------------------------------------------------------------------- /agent-mimir/src/tools/image_view.ts: -------------------------------------------------------------------------------- 1 | import { SupportedImageTypes } from "../schema.js"; 2 | import { z } from "zod"; 3 | import { AgentTool, ToolResponse } from "./index.js"; 4 | import { AgentPlugin, PluginFactory, PluginContext } from "../plugins/index.js"; 5 | 6 | export class ViewPluginFactory implements PluginFactory { 7 | name: string = "viewImages"; 8 | async create(context: PluginContext): Promise { 9 | return new ViewPlugin(context); 10 | } 11 | 12 | } 13 | export class ViewPlugin extends AgentPlugin { 14 | constructor(private context: PluginContext) { 15 | super(); 16 | } 17 | 18 | async tools(): Promise { 19 | return [new ViewTool(this.context)]; 20 | } 21 | } 22 | export class ViewTool extends AgentTool { 23 | 24 | name: string = "seeImageFromWorkspace"; 25 | 26 | constructor(private context: PluginContext) { 27 | super(); 28 | } 29 | 30 | description: string = "Allows you you to see any image currently available in your workspace. This tool does not shares the image with the user, only for you to see.Use it when the given task requires you to understand the contents of an image."; 31 | schema = z.object({ 32 | fileName: z.string().describe("The name of the image file you want to see."), 33 | }); 34 | 35 | protected async _call(arg: z.input): Promise { 36 | const file = (await this.context.workspace.fileAsBuffer(arg.fileName)); 37 | if (file) { 38 | let imageType = arg.fileName.split('.').pop()!; 39 | return [ 40 | { 41 | type: "image_url", 42 | image_url: { 43 | type: imageType as SupportedImageTypes, 44 | url: file.toString("base64"), 45 | } 46 | } 47 | ]; 48 | } 49 | const response: ToolResponse = [ 50 | { 51 | type: "text", 52 | text: `The file named ${arg.fileName} does not exist in your workspace.` 53 | } 54 | ]; 55 | return response; 56 | } 57 | 58 | } -------------------------------------------------------------------------------- /agent-mimir/src/tools/index.ts: -------------------------------------------------------------------------------- 1 | 2 | import { z } from "zod"; 3 | import { ComplexMessageContent } from "../schema.js"; 4 | //import { JSONSchema } from "@langchain/core/utils/json_schema"; 5 | import { type JsonSchema7Type as JSONSchema } from "zod-to-json-schema"; 6 | export type ToolResponse = ComplexMessageContent[]; 7 | export type ZodObjectAny = z.ZodObject; 8 | export type ToolInputSchemaBase = ZodObjectAny | z.ZodEffects | JSONSchema; 9 | 10 | export type ToolInputSchemaOutputType = T extends z.ZodSchema ? z.output : T extends JSONSchema ? unknown : never; 11 | /** 12 | * Utility type that resolves the input type of a tool input schema. 13 | * 14 | * Input & Output types are a concept used with Zod schema, as Zod allows for transforms to occur 15 | * during parsing. When using JSONSchema, input and output types are the same. 16 | * 17 | * The input type for a given schema should match the structure of the arguments that the LLM 18 | * generates as part of its {@link ToolCall}. The output type will be the type that results from 19 | * applying any transforms defined in your schema. If there are no transforms, the input and output 20 | * types will be the same. 21 | */ 22 | export type ToolInputSchemaInputType = T extends z.ZodSchema ? z.input : T extends JSONSchema ? unknown : never; 23 | export type StructuredToolCallInput> = (ToolInputSchemaOutputType extends string ? string : never) | SchemaInputT; 24 | 25 | export abstract class AgentTool< 26 | SchemaT extends ToolInputSchemaBase = ZodObjectAny, 27 | SchemaOutputT = ToolInputSchemaOutputType, 28 | SchemaInputT = ToolInputSchemaInputType, 29 | O extends z.ZodTypeAny = z.ZodTypeAny> { 30 | abstract schema: SchemaT; 31 | 32 | outSchema: O | z.ZodEffects | undefined = undefined 33 | 34 | protected abstract _call( 35 | arg: SchemaOutputT, 36 | ): Promise; 37 | 38 | async invoke( 39 | input: StructuredToolCallInput 40 | ): Promise { 41 | return this.call(input); 42 | } 43 | 44 | /** 45 | * Calls the tool with the provided argument, configuration, and tags. It 46 | * parses the input according to the schema, handles any errors, and 47 | * manages callbacks. 48 | * @param arg The input argument for the tool. 49 | * @param configArg Optional configuration or callbacks for the tool. 50 | * @param tags Optional tags for the tool. 51 | * @returns A Promise that resolves with a string. 52 | */ 53 | async call( 54 | arg: StructuredToolCallInput, 55 | ): Promise { 56 | let parsed; 57 | try { 58 | parsed = await (this.schema as z.ZodSchema).parseAsync(arg); 59 | } catch (e) { 60 | throw new Error( 61 | `Received tool input did not match expected schema ${JSON.stringify(arg)}`, 62 | ); 63 | } 64 | 65 | let result; 66 | try { 67 | result = await this._call(parsed); 68 | } catch (e) { 69 | 70 | throw e; 71 | } 72 | return result; 73 | } 74 | 75 | abstract name: string; 76 | 77 | abstract description: string; 78 | } -------------------------------------------------------------------------------- /agent-mimir/src/tools/langchain.ts: -------------------------------------------------------------------------------- 1 | import { StructuredTool } from "langchain/tools"; 2 | import { AgentTool, ToolResponse } from "./index.js"; 3 | import { z } from "zod"; 4 | import { AgentPlugin, PluginFactory, PluginContext } from "../plugins/index.js"; 5 | import { lCmessageContentToContent } from "../agent-manager/message-utils.js"; 6 | import { MessageContent } from "@langchain/core/messages"; 7 | 8 | 9 | class LangchainToolToMimirTool extends AgentTool { 10 | 11 | schema = this.tool.schema; 12 | name: string = this.tool.name; 13 | description: string = this.tool.description; 14 | returnDirect: boolean = this.tool.returnDirect; 15 | 16 | constructor(private tool: StructuredTool) { 17 | super(); 18 | } 19 | 20 | protected async _call(arg: z.input): Promise { 21 | const response = await this.tool.invoke(arg); 22 | if (this.tool.responseFormat === "content_and_artifact") { 23 | return lCmessageContentToContent(response[0] as MessageContent) 24 | } 25 | return lCmessageContentToContent(response as MessageContent) 26 | 27 | } 28 | } 29 | 30 | /** 31 | * Factory class for creating LangChain tool wrapper plugins 32 | */ 33 | export class LangchainToolWrapperPluginFactory implements PluginFactory { 34 | readonly name: string; 35 | 36 | constructor(private readonly tool: StructuredTool) { 37 | this.name = tool.name; 38 | } 39 | 40 | async create(context: PluginContext): Promise { 41 | return new LangchainToolWrapper(this.tool); 42 | } 43 | } 44 | 45 | /** 46 | * Plugin class that wraps LangChain tools for use with Mimir 47 | */ 48 | class LangchainToolWrapper extends AgentPlugin { 49 | constructor(private readonly tool: StructuredTool) { 50 | super(); 51 | } 52 | 53 | async tools(): Promise { 54 | return [new LangchainToolToMimirTool(this.tool)]; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /agent-mimir/src/utils/format.ts: -------------------------------------------------------------------------------- 1 | import { MessageContent, MessageContentComplex, MessageContentText } from "@langchain/core/messages"; 2 | import { ComplexMessageContent, TextMessageContent, SupportedImageTypes } from "../schema.js"; 3 | 4 | export function extractTextContent(messageContent: MessageContent): string { 5 | if (typeof messageContent === "string") { 6 | return messageContent; 7 | } else if (Array.isArray(messageContent)) { 8 | return (messageContent as any).find((e: any) => e.type === "text")?.text ?? ""; 9 | } else { 10 | throw new Error(`Got unsupported text type: ${JSON.stringify(messageContent)}`); 11 | } 12 | } 13 | 14 | export function extractTextContentFromComplexMessageContent(messageContent: ComplexMessageContent[]): string { 15 | if (typeof messageContent === "string") { 16 | return messageContent; 17 | } else if (Array.isArray(messageContent)) { 18 | return (messageContent as any).find((e: any) => e.type === "text")?.text ?? ""; 19 | } else { 20 | throw new Error(`Got unsupported text type: ${JSON.stringify(messageContent)}`); 21 | } 22 | } 23 | 24 | 25 | export function complexResponseToLangchainMessageContent(toolResponse: ComplexMessageContent[]): MessageContent { 26 | const content = toolResponse.map((en) => { 27 | if (en.type === "text") { 28 | return { 29 | type: "text", 30 | text: en.text 31 | } satisfies MessageContentText 32 | } else if (en.type === "image_url") { 33 | return openAIImageHandler(en.image_url, "high") 34 | } 35 | throw new Error(`Unsupported type: ${JSON.stringify(en)}`) 36 | }) 37 | 38 | return mergeContent(content); 39 | } 40 | 41 | function mergeContent(agentSystemMessages: MessageContentComplex[]): MessageContent { 42 | 43 | const content = agentSystemMessages; 44 | const containsOnlyText = content.find((f) => f.type !== "text") === undefined; 45 | if (containsOnlyText) { 46 | const systemMessageText = content.reduce((prev, next) => { 47 | return prev + (next as MessageContentText).text 48 | }, ""); 49 | 50 | return systemMessageText; 51 | } 52 | return content; 53 | } 54 | 55 | 56 | 57 | 58 | export function extractAllTextFromComplexResponse(toolResponse: ComplexMessageContent[]): string { 59 | return toolResponse.filter((r) => r.type === "text").map((r) => (r as TextMessageContent).text).join("\n"); 60 | } 61 | 62 | export const openAIImageHandler = (image: { url: string, type: SupportedImageTypes }, detail: "high" | "low" = "high") => { 63 | let type = image.type as string; 64 | if (type === "jpg") { 65 | type = "jpeg" 66 | } 67 | const res = { 68 | type: "image_url" as const, 69 | image_url: { 70 | url: image.type === "url" ? image.url : `data:image/${type};base64,${image.url}`, 71 | detail: detail 72 | } 73 | } 74 | return res; 75 | } 76 | 77 | export function trimAndSanitizeMessageContent(inputArray: ComplexMessageContent[]): ComplexMessageContent[] { 78 | return inputArray.filter((c) => { 79 | const isEmpty = (c.type === "text" && c.text.length === 0); 80 | return !isEmpty; 81 | }) 82 | } 83 | 84 | 85 | export function isEmptyMessageContent(message: ComplexMessageContent): boolean { 86 | if (message.type === "text") { 87 | return message.text.trim().length === 0; 88 | } else if (message.type === "image_url") { 89 | return false; // Images are not considered empty 90 | } else { 91 | return false 92 | } 93 | } 94 | 95 | 96 | export function textComplexMessage(text: string): TextMessageContent { 97 | return { 98 | type: "text", 99 | text: text 100 | } satisfies TextMessageContent 101 | } -------------------------------------------------------------------------------- /agent-mimir/tsconfig.cjs.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "module": "commonjs", 5 | "declaration": false 6 | }, 7 | "exclude": [ 8 | "node_modules", 9 | "dist", 10 | "docs", 11 | "**/tests" 12 | ] 13 | } -------------------------------------------------------------------------------- /agent-mimir/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "ts-node": { 3 | "esm": true 4 | }, 5 | "include": [ 6 | "src/**/*" 7 | ], 8 | "exclude": [ 9 | "node_modules", 10 | "dist", 11 | "docs" 12 | ], 13 | "compilerOptions": { 14 | "target": "ES2021", 15 | "module": "NodeNext", 16 | "lib": [ 17 | "ES2021", 18 | "ES2022.Object", 19 | "DOM" 20 | ], 21 | "allowJs": true, 22 | "outDir": "dist", 23 | "rootDir": "src", 24 | "strict": true, 25 | "noImplicitAny": true, 26 | "declaration": true, 27 | "declarationDir": "dist", 28 | "esModuleInterop": true, 29 | "sourceMap": true, 30 | "moduleResolution": "nodenext", 31 | "skipLibCheck": true, 32 | "types": [ 33 | "node", 34 | ], 35 | } 36 | } -------------------------------------------------------------------------------- /assets/demo-discord.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altaflux/agent-mimir/959c8bd60c145a2e06a1440683941dc2acac3253/assets/demo-discord.png -------------------------------------------------------------------------------- /assets/demo-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altaflux/agent-mimir/959c8bd60c145a2e06a1440683941dc2acac3253/assets/demo-screenshot.png -------------------------------------------------------------------------------- /assets/discord/botIntents.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altaflux/agent-mimir/959c8bd60c145a2e06a1440683941dc2acac3253/assets/discord/botIntents.png -------------------------------------------------------------------------------- /assets/discord/newApp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altaflux/agent-mimir/959c8bd60c145a2e06a1440683941dc2acac3253/assets/discord/newApp.png -------------------------------------------------------------------------------- /assets/discord/tokenGenerate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altaflux/agent-mimir/959c8bd60c145a2e06a1440683941dc2acac3253/assets/discord/tokenGenerate.png -------------------------------------------------------------------------------- /assets/mimir_logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /lg-server/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # LangGraph API 3 | .langgraph_api 4 | -------------------------------------------------------------------------------- /lg-server/langgraph.json: -------------------------------------------------------------------------------- 1 | { 2 | "node_version": "20", 3 | "dockerfile_lines": [], 4 | "dependencies": ["."], 5 | "graphs": { 6 | "agent": "./src/index.ts:agent" 7 | }, 8 | "env": ".env" 9 | } 10 | -------------------------------------------------------------------------------- /lg-server/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lg-server", 3 | "version": "0.0.1", 4 | "packageManager": "yarn@1.22.22", 5 | "description": "A template containing a LangGraph.js ReAct agent.", 6 | "main": "src/index.ts", 7 | "author": "Your Name", 8 | "license": "MIT", 9 | "private": true, 10 | "type": "module", 11 | "scripts": { 12 | "build": "", 13 | "clean": "rm -rf dist", 14 | "start": "npx @langchain/langgraph-cli dev", 15 | "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathPattern=\\.test\\.ts$ --testPathIgnorePatterns=\\.int\\.test\\.ts$", 16 | "test:int": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathPattern=\\.int\\.test\\.ts$", 17 | "lint": "eslint src", 18 | "format": "prettier --write ." 19 | }, 20 | "dependencies": { 21 | "@langchain/anthropic": "^0.3.12", 22 | "@langchain/community": "^0.3.27", 23 | "@langchain/core": "^0.3.37", 24 | "@langchain/langgraph": "^0.2.43", 25 | "langchain": "^0.3.14", 26 | "agent-mimir": "*", 27 | "@agent-mimir/mcp-client": "*" 28 | }, 29 | "workspaces": [ 30 | "sub-project" 31 | ], 32 | "devDependencies": { 33 | "@eslint/eslintrc": "^3.1.0", 34 | "@eslint/js": "^9.9.1", 35 | "@tsconfig/recommended": "^1.0.7", 36 | "@types/jest": "^29.5.0", 37 | "@typescript-eslint/eslint-plugin": "^5.59.8", 38 | "@typescript-eslint/parser": "^5.59.8", 39 | "dotenv": "^16.4.5", 40 | "eslint": "^8.41.0", 41 | "eslint-config-prettier": "^8.8.0", 42 | "eslint-plugin-import": "^2.27.5", 43 | "eslint-plugin-no-instanceof": "^1.0.1", 44 | "eslint-plugin-prettier": "^4.2.1", 45 | "jest": "^29.7.0", 46 | "prettier": "^3.3.3", 47 | "ts-jest": "^29.1.0", 48 | "typescript": "^5.3.3" 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /lg-server/src/index.ts: -------------------------------------------------------------------------------- 1 | 2 | import { CodeAgentFactory, LocalPythonExecutor, createLgAgent as codeCreateLgAgent } from "agent-mimir/agent/code-agent"; 3 | import { FunctionAgentFactory, createLgAgent} from "agent-mimir/agent/tool-agent"; 4 | import { promises as fs } from 'fs'; 5 | import { FileSystemAgentWorkspace } from "agent-mimir/nodejs"; 6 | import os from 'os'; 7 | import path from "path"; 8 | import {ChatOpenAI} from '@langchain/openai'; 9 | import { AdditionalContent, AgentPlugin, PluginContext, PluginFactory } from "agent-mimir/plugins"; 10 | import { InputAgentMessage } from "agent-mimir/agent"; 11 | 12 | 13 | class DummyPluginFactory implements PluginFactory { 14 | name: string = "dummyPlugin"; 15 | async create(context: PluginContext): Promise { 16 | return new (class extends AgentPlugin { 17 | async additionalMessageContent(message: InputAgentMessage): Promise { 18 | 19 | return [ 20 | { 21 | displayOnCurrentMessage: true, 22 | saveToChatHistory: true, 23 | content: [ 24 | { 25 | type: "text", 26 | text: `The current time is: ${new Date().toISOString()}` 27 | } 28 | ] 29 | } 30 | ] 31 | } 32 | }) 33 | } 34 | } 35 | 36 | async function createAgent() { 37 | const McpClientPluginFactory = (await import('@agent-mimir/mcp-client')).McpClientPluginFactory; 38 | const StdioClientTransport = (await import('@agent-mimir/mcp-client')).StdioClientTransport; 39 | 40 | const workingDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'mimir-cli-')); 41 | const workspaceFactory = async (agentName: string) => { 42 | const tempDir = path.join(workingDirectory, agentName); 43 | await fs.mkdir(tempDir, { recursive: true }); 44 | const workspace = new FileSystemAgentWorkspace(tempDir); 45 | await fs.mkdir(workspace.workingDirectory, { recursive: true }); 46 | return workspace; 47 | } 48 | const chatModel = new ChatOpenAI({ 49 | openAIApiKey: process.env.OPENAI_API_KEY, 50 | temperature: 0.0, 51 | modelName: 'gpt-4.1-2025-04-14', 52 | }); 53 | 54 | return await codeCreateLgAgent({ 55 | name: "agent", 56 | description: "a helpful assistant", 57 | model: chatModel, 58 | profession: "a helpful assistant", 59 | workspaceFactory: workspaceFactory, 60 | visionSupport: "openai", 61 | codeExecutor: (workspace) => new LocalPythonExecutor({additionalPackages: [], workspace: workspace}), 62 | plugins:[ 63 | new DummyPluginFactory(), 64 | new McpClientPluginFactory({ 65 | servers: { 66 | // "sqlite": { 67 | // description: "Query a Northwind SQLite database.", 68 | // transport: new StdioClientTransport({ 69 | // "command": "docker", 70 | // "args": ["run", "-i", "--rm", "-v", "C:/AI:/mcp", "mcp/sqlite", "--db-path", "/mcp/northwind.db"] 71 | // }) 72 | // }, 73 | "brave-search": { 74 | description: "Brave Search", 75 | transport: new StdioClientTransport({ 76 | "command": "docker", 77 | "args": ["run", "-i", "--rm", "-e", "BRAVE_API_KEY", "mcp/brave-search"], 78 | "env": { 79 | "BRAVE_API_KEY": "" 80 | } 81 | }) 82 | } 83 | } 84 | }), 85 | ] 86 | }) 87 | } 88 | 89 | export const agent = (await createAgent()).graph; -------------------------------------------------------------------------------- /mimir-config.example/mimir-cfg.js: -------------------------------------------------------------------------------- 1 | 2 | const ChatOpenAI = require('langchain/chat_models/openai').ChatOpenAI; 3 | const OpenAIEmbeddings = require('langchain/embeddings/openai').OpenAIEmbeddings; 4 | 5 | const chatModel = new ChatOpenAI({ 6 | openAIApiKey: process.env.OPENAI_API_KEY, 7 | temperature: 0.0, 8 | modelName: 'gpt-4-0613' 9 | }); 10 | 11 | const summaryModel = new ChatOpenAI({ 12 | openAIApiKey: process.env.OPENAI_API_KEY, 13 | temperature: 0.0, 14 | modelName: 'gpt-3.5-turbo-16k-0613', 15 | }); 16 | 17 | const embeddings = new OpenAIEmbeddings({ 18 | openAIApiKey: process.env.OPENAI_API_KEY, 19 | }); 20 | 21 | module.exports = async function () { 22 | 23 | const CodeInterpreterPluginFactory = (await import('@agent-mimir/code-interpreter')).CodeInterpreterPluginFactory; 24 | const WebBrowserPluginFactory = (await import('@agent-mimir/selenium-browser')).WebBrowserPluginFactory; 25 | 26 | return { 27 | continuousMode: false, 28 | agents: { 29 | 'Assistant': { 30 | mainAgent: true, 31 | description: 'An assistant', 32 | definition: { 33 | agentType: 'openai-function-agent', 34 | chatModel: chatModel, 35 | chatHistory: { 36 | summaryModel: summaryModel, 37 | }, 38 | profession: 'an Assistant', 39 | plugins: [ 40 | new CodeInterpreterPluginFactory(), 41 | new WebBrowserPluginFactory({ browserConfig: { browserName: "chrome", disableHeadless: true }, maximumChunkSize: 5000, numberOfRelevantDocuments: 3 }, summaryModel, embeddings) 42 | ], 43 | langChainTools: [ 44 | 45 | ], 46 | } 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /mimir-config.example/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "agent-mimir-deps", 3 | "private": false, 4 | "scripts": {}, 5 | "dependencies": { 6 | "@langchain/openai": "0.3.16", 7 | "@agent-mimir/code-interpreter": "*" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "agent-mimir-pck", 3 | "private": false, 4 | "scripts": { 5 | "build": "turbo run build", 6 | "start-cli": "dotenv -- node scripts/config-setup.js && cd .temp_custom_deps && npm install && cd .. && cross-env MIMIR_CFG_PATH=$INIT_CWD/.temp_custom_deps npm run internalStart", 7 | "start-discord": "dotenv -- node scripts/config-setup.js && cd .temp_custom_deps && npm install && cd .. && cross-env MIMIR_CFG_PATH=$INIT_CWD/.temp_custom_deps npm run internalStartDiscordNoBuild", 8 | "start-lg": "npm run build && dotenv -- npm run start -w lg-server", 9 | "internalStartDiscord": "npm run build && dotenv -- npm run start -w agent-mimir-discord", 10 | "internalStartDiscordNoBuild": "dotenv -- npm run start -w agent-mimir-discord", 11 | "internalStart": "dotenv -- npm run start -w agent-mimir-cli", 12 | "test": "npm run test -w agent-mimir", 13 | "dev": "turbo run dev", 14 | "lint": "turbo run lint", 15 | "publish": "bash scripts/release-branch.sh && turbo run build lint && npm run release -w agent-mimir && echo '🔗 Open https://github.com/Altaflux/agent-mimir/compare/release?expand=1 and merge the release PR'", 16 | "format": "prettier --write \"**/*.{ts,tsx,md}\"" 17 | }, 18 | "author": "Altaflux", 19 | "repository": { 20 | "type": "git", 21 | "url": "https://github.com/Altaflux/agent-mimir.git" 22 | }, 23 | "devDependencies": { 24 | "cross-env": "^7.0.3", 25 | "dotenv-cli": "latest", 26 | "eslint": "^7.32.0", 27 | "eslint-config-custom": "*", 28 | "fs-extra": "^11.1.1", 29 | "prettier": "^2.5.1", 30 | "release-it": "^15.10.0", 31 | "turbo": "latest", 32 | "tsx": "^4.7.0", 33 | "typescript-transpile-only": "0.0.4" 34 | }, 35 | "packageManager": "npm@9.5.0", 36 | "workspaces": [ 37 | "agent-mimir", 38 | "agent-mimir-cli", 39 | "agent-mimir-discord", 40 | "lg-server", 41 | "tools/*", 42 | ".temp_custom_deps" 43 | ], 44 | "dependencies": { 45 | "@langchain/langgraph": "0.2.67", 46 | "@langchain/core": "0.3.50", 47 | "langchain": "0.3.24" 48 | } 49 | } -------------------------------------------------------------------------------- /scripts/config-setup.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs').promises; 2 | const fse = require('fs-extra'); 3 | const path = require('path'); 4 | const fileExists = async path => !!(await fs.stat(path).catch(e => false)); 5 | 6 | const target = '.temp_custom_deps'; 7 | const main = async () => { 8 | 9 | if (await fileExists(target)) { 10 | await fs.rm(target, { recursive: true, force: true }); 11 | } 12 | await fs.mkdir(target); 13 | 14 | const configLocation = process.env.CONFIG_LOCAION ?? path.join(process.env.INIT_CWD, 'mimir-config'); 15 | 16 | const exists = fse.existsSync(configLocation) 17 | if (await fileExists(configLocation)) { 18 | console.log(`Copying ${configLocation} to ${target}}`) 19 | await fse.copy(configLocation, target); 20 | } 21 | }; 22 | 23 | main() -------------------------------------------------------------------------------- /scripts/release-branch.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [[ $(git branch --show-current) == "main" ]]; then 4 | git checkout -B release 5 | git push -u origin release 6 | fi 7 | -------------------------------------------------------------------------------- /tool-examples/javascript-repl/mimir-cfg.js: -------------------------------------------------------------------------------- 1 | 2 | const ChatOpenAI = require('langchain/chat_models/openai').ChatOpenAI; 3 | const Tool = require('langchain/tools').Tool; 4 | var safeEval = require('safe-eval'); 5 | 6 | class JavascriptRepl extends Tool { 7 | 8 | async _call(arg) { 9 | return JSON.stringify(safeEval(arg)); 10 | } 11 | name = "javascriptRepl"; 12 | description = "Run a Javascript snippet of code and get the result back. The input must be the Javascript code to execute."; 13 | constructor() { 14 | super() 15 | } 16 | } 17 | 18 | const summaryModel = new ChatOpenAI({ 19 | openAIApiKey: process.env.OPENAI_API_KEY, 20 | temperature: 0.9, 21 | modelName: 'gpt-3.5-turbo-16k-0613', 22 | }); 23 | 24 | const chatModel = new ChatOpenAI({ 25 | openAIApiKey: process.env.OPENAI_API_KEY, 26 | temperature: 0.9, 27 | modelName: 'gpt-4', 28 | callbacks: callbackManager 29 | }); 30 | 31 | 32 | module.exports = function () { 33 | return { 34 | continuousMode: false, 35 | agents: { 36 | 'Assistant': { 37 | mainAgent: true, 38 | description: 'An assistant', 39 | definition: { 40 | chatModel: chatModel, 41 | chatHistory: { 42 | summaryModel: summaryModel 43 | }, 44 | profession: 'an Assistant', 45 | tools: [ 46 | new JavascriptRepl() 47 | ] 48 | } 49 | } 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /tool-examples/javascript-repl/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "agent-mimir-deps", 3 | "private": false, 4 | "scripts": {}, 5 | "dependencies": { 6 | "safe-eval": "0.4.1" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /tool-examples/powershell/mimir-cfg.js: -------------------------------------------------------------------------------- 1 | 2 | const ChatOpenAI = require('langchain/chat_models/openai').ChatOpenAI; 3 | const Tool = require('langchain/tools').Tool; 4 | const exec = require('child_process').exec; 5 | 6 | 7 | class PowerShell extends Tool { 8 | async _call(arg) { 9 | return new Promise((resolve, reject) => { 10 | exec(arg, { 'shell': 'powershell.exe' }, (error, stdout, stderr) => { 11 | if (error) { 12 | resolve(error.message) 13 | } 14 | else if (stderr) { 15 | resolve(stdout) 16 | } else { 17 | 18 | resolve(stdout) 19 | } 20 | }) 21 | }); 22 | } 23 | name = "windowsPowershell"; 24 | description = "Run a Powershell command in the human's Powershell terminal. The input must be the command to execute."; 25 | constructor() { 26 | super() 27 | } 28 | } 29 | 30 | const summaryModel = new ChatOpenAI({ 31 | openAIApiKey: process.env.OPENAI_API_KEY, 32 | temperature: 0.9, 33 | modelName: 'gpt-3.5-turbo-16k-0613', 34 | }); 35 | 36 | const chatModel = new ChatOpenAI({ 37 | openAIApiKey: process.env.OPENAI_API_KEY, 38 | temperature: 0.9, 39 | modelName: 'gpt-4', 40 | callbacks: callbackManager 41 | }); 42 | 43 | 44 | module.exports = function () { 45 | return { 46 | continuousMode: false, 47 | agents: { 48 | 'Assistant': { 49 | mainAgent: true, 50 | description: 'An assistant', 51 | definition: { 52 | chatModel: chatModel, 53 | chatHistory: { 54 | summaryModel: summaryModel 55 | }, 56 | profession: 'an Assistant', 57 | tools: [ 58 | new PowerShell() 59 | ] 60 | } 61 | } 62 | } 63 | } 64 | } -------------------------------------------------------------------------------- /tool-examples/powershell/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "agent-mimir-deps", 3 | "private": false, 4 | "scripts": {}, 5 | "dependencies": { 6 | "child_process": "1.0.2" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /tool-examples/shell/mimir-cfg.js: -------------------------------------------------------------------------------- 1 | 2 | const ChatOpenAI = require('langchain/chat_models/openai').ChatOpenAI; 3 | const Tool = require('langchain/tools').Tool; 4 | const exec = require('child_process').exec; 5 | 6 | 7 | class Shell extends Tool { 8 | async _call(arg) { 9 | return new Promise((resolve, reject) => { 10 | exec(arg, (error, stdout, stderr) => { 11 | if (error) { 12 | resolve(error.message) 13 | } 14 | else if (stderr) { 15 | resolve(stdout) 16 | } else { 17 | 18 | resolve(stdout) 19 | } 20 | }) 21 | }); 22 | } 23 | name = "linuxShell"; 24 | description = "Run a shell command in the human's terminal. The input must be the command to execute."; 25 | constructor() { 26 | super() 27 | } 28 | } 29 | 30 | const summaryModel = new ChatOpenAI({ 31 | openAIApiKey: process.env.OPENAI_API_KEY, 32 | temperature: 0.9, 33 | modelName: 'gpt-3.5-turbo-16k-0613', 34 | }); 35 | 36 | const chatModel = new ChatOpenAI({ 37 | openAIApiKey: process.env.OPENAI_API_KEY, 38 | temperature: 0.9, 39 | modelName: 'gpt-4', 40 | callbacks: callbackManager 41 | }); 42 | 43 | 44 | module.exports = function () { 45 | return { 46 | continuousMode: false, 47 | agents: { 48 | 'Assistant': { 49 | mainAgent: true, 50 | description: 'An assistant', 51 | definition: { 52 | chatModel: chatModel, 53 | chatHistory: { 54 | summaryModel: summaryModel 55 | }, 56 | profession: 'an Assistant', 57 | tools: [ 58 | new Shell() 59 | ] 60 | } 61 | } 62 | } 63 | } 64 | } -------------------------------------------------------------------------------- /tool-examples/shell/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "agent-mimir-deps", 3 | "private": false, 4 | "scripts": {}, 5 | "dependencies": { 6 | "child_process": "1.0.2" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /tools/code-interpreter/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@agent-mimir/code-interpreter", 3 | "description": "Python code interpreter tool for langchain", 4 | "main": "dist/index.js", 5 | "type": "module", 6 | "scripts": { 7 | "build": "tsc -p tsconfig.json " 8 | }, 9 | "author": "Altaflux", 10 | "keywords": [ 11 | "langchain" 12 | ], 13 | "devDependencies": { 14 | "@types/node": "^18.16.3", 15 | "rimraf": "^5.0.0", 16 | "typescript": "^5.3.3", 17 | "agent-mimir": "*" 18 | }, 19 | "dependencies": { 20 | 21 | }, 22 | "peerDependencies": { 23 | "agent-mimir": "*" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tools/code-interpreter/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021", 4 | "allowJs": true, 5 | "allowSyntheticDefaultImports": true, 6 | "baseUrl": "src", 7 | "declaration": true, 8 | "sourceMap": true, 9 | "inlineSourceMap": false, 10 | "moduleResolution": "nodenext", 11 | "rootDir": "src", 12 | "module": "NodeNext", /* Specify what module code is generated. */ 13 | "outDir": "dist", 14 | "lib": [ 15 | "ES2021", 16 | "ES2022.Object", 17 | "DOM" 18 | ], 19 | "types": [ 20 | "node", 21 | ], 22 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ 23 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ 24 | /* Type Checking */ 25 | "strict": true, /* Enable all strict type-checking options. */ 26 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 27 | }, 28 | "ts-node": { 29 | "esm": true 30 | } 31 | } -------------------------------------------------------------------------------- /tools/desktop-control/molmo/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | vllm-openai: 4 | deploy: 5 | resources: 6 | reservations: 7 | devices: 8 | - driver: nvidia 9 | count: all 10 | capabilities: 11 | - gpu 12 | volumes: 13 | - ~/.cache/huggingface:/root/.cache/huggingface 14 | - ./models:/models 15 | environment: 16 | - HUGGING_FACE_HUB_TOKEN=${HUGGING_FACE_HUB_TOKEN} 17 | ports: 18 | - "${PORT}:8000" 19 | ipc: host 20 | image: vllm/vllm-openai:latest 21 | command: --trust-remote-code --model allenai/Molmo-7B-D-0924 22 | -------------------------------------------------------------------------------- /tools/desktop-control/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@agent-mimir/desktop-control", 3 | "description": "Control your computer.", 4 | "main": "dist/index.js", 5 | "type": "module", 6 | "scripts": { 7 | "build": "tsc -p tsconfig.json ", 8 | "install": "node scripts/install.js " 9 | }, 10 | "author": "Altaflux", 11 | "keywords": [ 12 | "langchain" 13 | ], 14 | "devDependencies": { 15 | "@types/node": "^18.16.3", 16 | "rimraf": "^5.0.0", 17 | "typescript": "^5.3.3", 18 | "agent-mimir": "*", 19 | "@types/screenshot-desktop": "^1.12.3" 20 | }, 21 | "dependencies": { 22 | "screenshot-desktop": "1.15.0", 23 | "@nut-tree-fork/nut-js": "4.2.4", 24 | "sharp": "^0.32.6", 25 | "systeminformation": "5.21.18", 26 | "tesseract.js": "^5.0.4", 27 | "async-exit-hook": "^2.0.1", 28 | "env-paths": "^3.0.0", 29 | "fuse.js": "7.0.0" 30 | }, 31 | "peerDependencies": { 32 | "agent-mimir": "*" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /tools/desktop-control/python/mobile_sam.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altaflux/agent-mimir/959c8bd60c145a2e06a1440683941dc2acac3253/tools/desktop-control/python/mobile_sam.pt -------------------------------------------------------------------------------- /tools/desktop-control/python/requirementsCpu.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | mobile_sam @ git+https://github.com/ChaoningZhang/MobileSAM.git@c12dd83cbe26dffdcc6a0f9e7be2f6fb024df0ed 4 | flask 5 | opencv-python 6 | numpy 7 | timm 8 | 9 | keras==2.15.0 10 | keras-ocr==0.9.3 11 | Keras-Applications==1.0.8 12 | matplotlib 13 | tensorflow 14 | 15 | torch 16 | torchvision -------------------------------------------------------------------------------- /tools/desktop-control/python/requirementsCuda11.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | mobile_sam @ git+https://github.com/ChaoningZhang/MobileSAM.git@c12dd83cbe26dffdcc6a0f9e7be2f6fb024df0ed 4 | flask 5 | opencv-python 6 | numpy 7 | timm 8 | 9 | keras==2.15.0 10 | keras-ocr==0.9.3 11 | Keras-Applications==1.0.8 12 | matplotlib 13 | tensorflow 14 | 15 | 16 | --extra-index-ur https://download.pytorch.org/whl/cu118 17 | torch 18 | torchvision -------------------------------------------------------------------------------- /tools/desktop-control/python/requirementsCuda12.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | mobile_sam @ git+https://github.com/ChaoningZhang/MobileSAM.git@c12dd83cbe26dffdcc6a0f9e7be2f6fb024df0ed 4 | flask 5 | opencv-python 6 | numpy 7 | timm 8 | 9 | keras==2.15.0 10 | keras-ocr==0.9.3 11 | Keras-Applications==1.0.8 12 | matplotlib 13 | tensorflow 14 | 15 | 16 | --extra-index-ur https://download.pytorch.org/whl/cu126 17 | torch 18 | torchvision -------------------------------------------------------------------------------- /tools/desktop-control/python/server.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from flask import Flask, request, jsonify, send_file 3 | import cv2 4 | import numpy as np 5 | import torch 6 | import time 7 | import io 8 | import json 9 | import keras_ocr 10 | from mobile_sam import sam_model_registry, SamAutomaticMaskGenerator 11 | 12 | 13 | parser = argparse.ArgumentParser( 14 | description="Export the SAM prompt encoder and mask decoder to an ONNX model." 15 | ) 16 | 17 | parser.add_argument( 18 | "--port", 19 | type=int, 20 | required=True, 21 | default=5000, 22 | help="The port." 23 | ) 24 | parser.add_argument( 25 | "--model", 26 | type=str, 27 | required=True, 28 | help="Model path." 29 | ) 30 | 31 | 32 | app = Flask(__name__) 33 | 34 | @app.route('/calculate-boxes', methods=['POST']) 35 | def calculateBoxes(): 36 | 37 | 38 | start_time = time.time() 39 | file = request.files['image'].read() 40 | npimg = np.frombuffer(file,np.uint8) 41 | img = cv2.imdecode(npimg,cv2.IMREAD_COLOR) 42 | 43 | mask_generator = SamAutomaticMaskGenerator(mobile_sam, points_per_side=50, points_per_batch=100) 44 | masks = mask_generator.generate(img) 45 | end_time = time.time() 46 | execution_time = end_time - start_time 47 | print(f"Execution time: {execution_time:.5f} seconds") 48 | user_details = [{'coordinates': {'x': segment['point_coords'][0][0], 'y': segment['point_coords'][0][1] }} for i,segment in enumerate(masks)] 49 | response = { 50 | "masks": user_details 51 | } 52 | # Return the same string back 53 | return jsonify(response) 54 | 55 | ##### 56 | 57 | pipeline = keras_ocr.pipeline.Pipeline() 58 | def midpoint(x1, y1, x2, y2): 59 | x_mid = int((x1 + x2)/2) 60 | y_mid = int((y1 + y2)/2) 61 | return (x_mid, y_mid) 62 | 63 | 64 | class NumpyArrayEncoder(json.JSONEncoder): 65 | def default(self, obj): 66 | if isinstance(obj, np.integer): 67 | return int(obj) 68 | elif isinstance(obj, np.floating): 69 | return float(obj) 70 | elif isinstance(obj, np.ndarray): 71 | return obj.tolist() 72 | else: 73 | return super(NumpyArrayEncoder, self).default(obj) 74 | 75 | def extract_text(img, pipeline): 76 | 77 | prediction_groups = pipeline.recognize([img]) 78 | textAreas = [] 79 | for box in prediction_groups[0]: 80 | x0, y0 = box[1][0] 81 | x1, y1 = box[1][1] 82 | x2, y2 = box[1][2] 83 | x3, y3 = box[1][3] 84 | textAreas.append({ 85 | "text": box[0], 86 | "bbox": {'x0': x0, 'y0': y0, 'x1': x1, 'y1': y1, 'x2': x2, 'y2': y2, 'x3': x3, 'y3': y3} 87 | }) 88 | 89 | return textAreas 90 | 91 | @app.route('/obtainText', methods=['POST']) 92 | def obtainText(): 93 | 94 | file = request.files['image'].read() 95 | npimg = np.frombuffer(file,np.uint8) 96 | img = cv2.imdecode(npimg,cv2.IMREAD_UNCHANGED) 97 | textAreas = extract_text(img, pipeline) 98 | 99 | encodedNumpyData = json.dumps(textAreas, cls=NumpyArrayEncoder) 100 | decodedArrays = json.loads(encodedNumpyData) 101 | # Return the same string back 102 | return jsonify({'textBoxes': decodedArrays}) 103 | 104 | @app.route('/inpaint', methods=['POST']) 105 | def inpaint(): 106 | 107 | file = request.files['image'].read() 108 | img = cv2.imdecode(np.frombuffer(file,np.uint8), cv2.IMREAD_UNCHANGED) 109 | 110 | inpaintMaskFile = request.files['inpaintMask'].read() 111 | maskImg = cv2.imdecode(np.frombuffer(inpaintMaskFile,np.uint8),cv2.IMREAD_GRAYSCALE) 112 | 113 | 114 | finalImage = cv2.inpaint(img, maskImg, 7, cv2.INPAINT_NS) 115 | finalImageBuffer = cv2.imencode('.jpg', finalImage) 116 | 117 | return send_file( 118 | io.BytesIO(finalImageBuffer[1]), 119 | download_name='inpaint.jpg', 120 | mimetype='image/jpg' 121 | ) 122 | 123 | ############## 124 | if __name__ == '__main__': 125 | args = parser.parse_args() 126 | 127 | model_type = "vit_t" 128 | device = "cuda" if torch.cuda.is_available() else "cpu" 129 | print(f"Selected device: {device}") 130 | global mobile_sam 131 | mobile_sam = sam_model_registry[model_type](checkpoint=args.model) 132 | mobile_sam.to(device=device) 133 | mobile_sam.eval() 134 | 135 | app.run(debug=False, port=args.port) 136 | 137 | 138 | -------------------------------------------------------------------------------- /tools/desktop-control/scripts/install.js: -------------------------------------------------------------------------------- 1 | 2 | import { spawn, exec } from 'child_process'; 3 | import { promises as fs } from 'fs'; 4 | import path from "path"; 5 | import envPaths from 'env-paths'; 6 | 7 | async function start() { 8 | 9 | const dataDir = envPaths("mimir-desktop-control").data 10 | const pythonEnvironmentDir = path.join(dataDir, "pythonEnv"); 11 | 12 | console.debug(`Creating python virtual environment in ${pythonEnvironmentDir} ...`); 13 | await fs.mkdir(pythonEnvironmentDir, { recursive: true }); 14 | 15 | const scriptsDir = process.platform === "win32" ? 'Scripts' : 'bin'; 16 | const pyenv = await executeShellCommand(`cd ${pythonEnvironmentDir} && python -m venv .`); 17 | 18 | 19 | const activeScriptCall = process.platform === "win32" ? `activate` : `. ./activate`; 20 | const cudaVersion = await getCudaMajorVersion(); 21 | const requirementFile = cudaVersion === 12 ? 'requirementsCuda12.txt' : cudaVersion === 11 ? 'requirementsCuda11.txt' : 'requirementsCpu.txt'; 22 | 23 | const requirementsPath = path.resolve(process.cwd(), 'python', requirementFile); 24 | console.log("Installing requirements: " + requirementsPath); 25 | const result = await executeShellCommand(`cd ${path.join(pythonEnvironmentDir, scriptsDir)} && ${activeScriptCall} && python -m pip install -r ${requirementsPath}`) 26 | console.log(result.output) 27 | console.log("Finished installing requirements.") 28 | 29 | const pythonServerScript = path.resolve(process.cwd(), 'python', 'server.py'); 30 | await fs.copyFile(pythonServerScript, path.join(dataDir, "server.py")); 31 | 32 | 33 | const mobileSam = path.resolve(process.cwd(), 'python', 'mobile_sam.pt'); 34 | await fs.copyFile(mobileSam, path.join(dataDir, "mobile_sam.pt")); 35 | 36 | 37 | const dockerMolmo = path.resolve(process.cwd(), 'molmo', 'docker-compose.yml'); 38 | await fs.copyFile(dockerMolmo, path.join(dataDir, 'docker-compose.yml')); 39 | 40 | } 41 | 42 | 43 | async function getCudaMajorVersion() { 44 | return new Promise((resolve, reject) => { 45 | try { 46 | let command = 'nvcc --version'; 47 | 48 | exec(command, { encoding: 'utf8' }, (error, stdout, stderr) => { 49 | if (error) { 50 | console.error('CUDA is not installed or not found in the system PATH.'); 51 | resolve(null); 52 | return; 53 | } 54 | 55 | const output = stdout.trim(); 56 | const match = output.match(/release (\d+)/); 57 | if (match) { 58 | resolve(parseInt(match[1])); 59 | } else { 60 | console.error('Unable to determine CUDA major version.'); 61 | resolve(null); 62 | } 63 | }); 64 | } catch (error) { 65 | console.error('An error occurred while checking CUDA version:', error); 66 | reject(error); 67 | } 68 | }); 69 | } 70 | 71 | 72 | 73 | async function executeShellCommand(command) { 74 | return await new Promise((resolve, reject) => { 75 | let output = ''; 76 | const ls = spawn(command, [], { shell: true, env: process.env }); 77 | 78 | ls.stdout.on("data", data => { 79 | output += data; 80 | 81 | }); 82 | 83 | ls.stderr.on("data", data => { 84 | output += data; 85 | 86 | }); 87 | 88 | ls.on('error', (error) => { 89 | output += error.message; 90 | 91 | }); 92 | 93 | ls.on("close", code => { 94 | 95 | resolve({ 96 | exitCode: code ?? 0, 97 | output: output, 98 | }) 99 | }); 100 | }); 101 | } 102 | 103 | start() -------------------------------------------------------------------------------- /tools/desktop-control/src/molmo.ts: -------------------------------------------------------------------------------- 1 | 2 | import sharp from 'sharp'; 3 | import envPaths from 'env-paths'; 4 | import { ChildProcess, spawn } from 'child_process'; 5 | 6 | import exitHook from 'async-exit-hook'; 7 | import net, { AddressInfo } from "net"; 8 | 9 | 10 | 11 | export class MolmoServerControl { 12 | 13 | private process: ChildProcess | null = null; 14 | 15 | private port: number = 8000; 16 | 17 | async init() { 18 | console.log("Starting MOLMO server.") 19 | const dataDir = envPaths("mimir-desktop-control").data 20 | const port = await getPortFree(); 21 | this.port = port; 22 | const command = `cd ${dataDir} && docker-compose up `; 23 | const result = spawn(command, [], { 24 | shell: true, env: { 25 | ...process.env, 26 | PORT: port.toString() 27 | } 28 | }); 29 | 30 | result.stdout.pipe(process.stdout); 31 | result.stderr.pipe(process.stderr); 32 | 33 | this.process = result; 34 | 35 | exitHook(async (callback) => { 36 | console.log("Closing MOLMO server.") 37 | await this.close(); 38 | callback(); 39 | }); 40 | } 41 | 42 | async close() { 43 | if (this.process) { 44 | this.process.kill("SIGINT") 45 | this.process = null; 46 | } 47 | } 48 | 49 | 50 | async locateItem(screenshotImage: Buffer, description: string, imageWidth: number, imageHeight: number, currentPointer: { x: number, y: number}): Promise<{ 51 | x: number, 52 | y: number 53 | }> { 54 | 55 | console.log('Locating item:', description); 56 | console.log('Current pointer:', currentPointer); 57 | const requestBody = { 58 | model: "allenai/Molmo-7B-D-0924", 59 | messages: [ 60 | { 61 | role: "user", 62 | content: [ 63 | { 64 | type: "text", 65 | text: `Point to the position of: "${description}", it should be close or at point (x: ${currentPointer.x}, y: ${currentPointer.y}).` 66 | }, 67 | { 68 | type: "image_url", 69 | image_url: { 70 | url: `data:image/png;base64,${screenshotImage.toString("base64")}` 71 | } 72 | } 73 | ] 74 | } 75 | ] 76 | }; 77 | 78 | const url = `http://localhost:${this.port}/v1/chat/completions`; 79 | 80 | try { 81 | const response = await fetch(url, { 82 | method: 'POST', 83 | headers: { 84 | 'Content-Type': 'application/json', 85 | }, 86 | body: JSON.stringify(requestBody) 87 | }); 88 | 89 | if (!response.ok) { 90 | throw new Error(`HTTP error! status: ${response.status}`); 91 | } 92 | 93 | const data = await response.json(); 94 | 95 | // Extract the point coordinates from the response 96 | const message = data.choices[0].message; 97 | if (message.content) { 98 | // Parse the point coordinates using regex 99 | const pointMatch = message.content.match(/ { 147 | return new Promise(res => { 148 | const srv = net.createServer(); 149 | srv.listen(0, () => { 150 | const port = (srv.address()! as AddressInfo).port 151 | srv.close((err) => res(port)) 152 | }); 153 | }) 154 | } 155 | 156 | 157 | -------------------------------------------------------------------------------- /tools/desktop-control/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021", 4 | "allowJs": true, 5 | "allowSyntheticDefaultImports": true, 6 | "baseUrl": "src", 7 | "declaration": true, 8 | "sourceMap": true, 9 | "inlineSourceMap": false, 10 | 11 | "moduleResolution": "nodenext", 12 | "rootDir": "src", 13 | "module": "NodeNext", /* Specify what module code is generated. */ 14 | "outDir": "dist", 15 | "lib": [ 16 | "ES2021", 17 | "ES2022.Object", 18 | "DOM" 19 | ], 20 | "types": [ 21 | "node", 22 | ], 23 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ 24 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ 25 | /* Type Checking */ 26 | "strict": true, /* Enable all strict type-checking options. */ 27 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 28 | }, 29 | "include": [ 30 | "src/**/*" 31 | ], 32 | "ts-node": { 33 | "esm": true 34 | } 35 | } -------------------------------------------------------------------------------- /tools/gameboy-play/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@agent-mimir/gameboy-play", 3 | "description": "agent-mimir plugin for Gameboy emulator", 4 | "main": "dist/index.js", 5 | "type": "module", 6 | "scripts": { 7 | "build": "tsc -p tsconfig.json " 8 | }, 9 | "author": "Altaflux", 10 | "keywords": [ 11 | "langchain" 12 | ], 13 | "devDependencies": { 14 | "@types/node": "^18.16.3", 15 | "rimraf": "^5.0.0", 16 | "typescript": "^5.3.3", 17 | "agent-mimir": "*", 18 | "@types/screenshot-desktop": "^1.12.3" 19 | }, 20 | "dependencies": { 21 | "pixelmatch": "7.1.0", 22 | "screenshot-desktop": "1.15.0", 23 | "node-window-manager":"2.2.4", 24 | "@nut-tree-fork/nut-js": "4.2.4", 25 | "sharp": "^0.32.6", 26 | "systeminformation": "5.21.18", 27 | "tesseract.js": "^5.0.4", 28 | "async-exit-hook": "^2.0.1", 29 | "env-paths": "^3.0.0", 30 | "fuse.js": "7.0.0", 31 | "@jitsi/robotjs": "0.6.13" 32 | }, 33 | "peerDependencies": { 34 | "agent-mimir": "*" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /tools/gameboy-play/src/controller.ts: -------------------------------------------------------------------------------- 1 | import robotjs from '@jitsi/robotjs'; 2 | import { windowManager } from 'node-window-manager'; 3 | import screenshot from 'screenshot-desktop'; 4 | import sharp from 'sharp'; // For image manipulation (cropping, saving as JPEG) 5 | import * as fs from 'fs'; 6 | 7 | 8 | // Interfaces 9 | interface Controls { 10 | [key: string]: string; 11 | } 12 | 13 | interface Region { 14 | left: number; 15 | top: number; 16 | right: number; 17 | bottom: number; 18 | } 19 | 20 | interface Action { 21 | type: string; 22 | button?: string; 23 | buttons?: string[]; 24 | direction?: string; 25 | steps?: number; 26 | } 27 | 28 | class GameboyController { 29 | private windowTitle?: string; 30 | private region?: Region; 31 | private keyDelay: number; 32 | private controls: Controls; 33 | 34 | 35 | constructor(windowTitle?: string, region?: Region) { 36 | this.windowTitle = windowTitle; 37 | this.region = region; 38 | this.keyDelay = 100; // Delay in milliseconds (equivalent to 0.1s in Python) 39 | 40 | // GameBoy control mapping for robotjs 41 | this.controls = { 42 | 'up': 'up', 43 | 'down': 'down', 44 | 'left': 'left', 45 | 'right': 'right', 46 | 'a': 'x', // Typically X is mapped to A on emulators 47 | 'b': 'z', // Typically Z is mapped to B on emulators 48 | 'start': 'enter', 49 | 'select': 'backspace' 50 | }; 51 | } 52 | 53 | findWindow() { 54 | if (!this.windowTitle) { 55 | return null; 56 | } 57 | 58 | try { 59 | // Get all windows 60 | const windows = windowManager.getWindows(); 61 | 62 | // Find the window that matches our title 63 | for (const window of windows) { 64 | const title = window.getTitle(); 65 | if (title && title.includes(this.windowTitle)) { 66 | return window; 67 | } 68 | } 69 | 70 | console.log(`No window with title containing '${this.windowTitle}' found.`); 71 | } catch (e) { 72 | console.log(`Error finding window: ${e}`); 73 | } 74 | 75 | return null; 76 | } 77 | async captureScreen(): Promise { 78 | try { 79 | 80 | const window = this.findWindow(); 81 | // First, capture the entire screen 82 | const screenshotBuffer = await screenshot({screen: (window?.getMonitor() as any).id}); 83 | console.log("Captured entire screen"); 84 | 85 | // Process the full screenshot 86 | let imageProcessor = sharp(screenshotBuffer); 87 | 88 | // If we have a window, extract just that region 89 | 90 | if (window) { 91 | try { 92 | // Try to bring window to front 93 | window.bringToTop(); 94 | await new Promise(resolve => setTimeout(resolve, 200)); // Give window time to come to front 95 | 96 | const bounds = window.getBounds(); 97 | 98 | const mon = (window?.getMonitor() as any); 99 | // const scaleFactor = mon.getScaleFactor() || 1; 100 | const scaleFactor = (mon.getScaleFactor() * mon.getScaleFactor()) || 1; 101 | 102 | // Extract just the window region from the full screenshot 103 | imageProcessor = imageProcessor.extract({ 104 | left: Math.floor(bounds.x! * scaleFactor), 105 | top: Math.floor(bounds.y! * scaleFactor), 106 | width: Math.floor(bounds.width! * scaleFactor), 107 | height: Math.floor(bounds.height! * scaleFactor) 108 | }); 109 | 110 | console.log(`Extracted window: ${this.windowTitle}`); 111 | } catch (e) { 112 | console.log(`Error extracting window region: ${e}`); 113 | } 114 | } else if (this.region) { 115 | // Extract the specified region if no window was found but region is defined 116 | imageProcessor = imageProcessor.extract({ 117 | left: this.region.left, 118 | top: this.region.top, 119 | width: this.region.right - this.region.left, 120 | height: this.region.bottom - this.region.top 121 | }); 122 | console.log("Extracted specified region"); 123 | } 124 | 125 | // Save the processed image as JPEG 126 | // await imageProcessor.jpeg().toFile(filename); 127 | return await imageProcessor.toBuffer(); 128 | } catch (e) { 129 | console.error(`Error capturing screen: ${e}`); 130 | throw e; 131 | } 132 | } 133 | 134 | 135 | 136 | async pressButton(button: string, holdTime = 100): Promise { 137 | // Try to focus the window before pressing keys 138 | const window = this.findWindow(); 139 | if (window) { 140 | try { 141 | window.bringToTop(); 142 | await new Promise(resolve => setTimeout(resolve, 100)); // Short delay 143 | } catch (e) { 144 | console.log(`Error focusing window: ${e}`); 145 | } 146 | } 147 | 148 | if (button in this.controls) { 149 | const key = this.controls[button]; 150 | robotjs.keyToggle(key, 'down'); 151 | await new Promise(resolve => setTimeout(resolve, holdTime)); 152 | robotjs.keyToggle(key, 'up'); 153 | await new Promise(resolve => setTimeout(resolve, this.keyDelay)); // Short delay after button press 154 | return true; 155 | } else { 156 | console.log(`Unknown button: ${button}`); 157 | return false; 158 | } 159 | } 160 | 161 | async executeAction(action: Action): Promise { 162 | const actionType = action.type || ''; 163 | 164 | if (actionType === 'button_press' && action.button) { 165 | return await this.pressButton(action.button); 166 | } 167 | else if (actionType === 'sequence' && action.buttons) { 168 | // Execute a sequence of button presses 169 | for (const button of action.buttons) { 170 | await this.pressButton(button); 171 | } 172 | return true; 173 | } 174 | else if (actionType === 'navigate' && action.direction) { 175 | // Navigate in a direction 176 | const steps = action.steps || 1; 177 | 178 | for (let i = 0; i < steps; i++) { 179 | await this.pressButton(action.direction); 180 | } 181 | return true; 182 | } 183 | 184 | return false; 185 | } 186 | } 187 | 188 | function readImageToBase64(imagePath: string): string { 189 | try { 190 | const imageBuffer = fs.readFileSync(imagePath); 191 | return imageBuffer.toString('base64'); 192 | } catch (e) { 193 | throw new Error(`Error reading image file: ${e}`); 194 | } 195 | } 196 | 197 | // Export the class and function 198 | export { GameboyController as GameboyController, readImageToBase64 }; -------------------------------------------------------------------------------- /tools/gameboy-play/src/index.ts: -------------------------------------------------------------------------------- 1 | import { CallbackManagerForToolRun } from "@langchain/core/callbacks/manager"; 2 | import { z } from "zod"; 3 | 4 | import { ToolResponse } from "agent-mimir/tools"; 5 | import { AgentTool } from "agent-mimir/tools"; 6 | import { AdditionalContent, AgentPlugin, PluginFactory, NextMessageUser, PluginContext } from "agent-mimir/plugins"; 7 | import { InputAgentMessage } from "agent-mimir/agent"; 8 | import { GameboyController } from "./controller.js"; 9 | 10 | export class GameboyPluginFactory implements PluginFactory { 11 | constructor(private emulatorName: string) { } 12 | async create(context: PluginContext): Promise { 13 | return new GameboyPlugin(this.emulatorName); 14 | } 15 | 16 | name: string = "gameboy_controller_plugin"; 17 | } 18 | 19 | class GameboyPlugin extends AgentPlugin { 20 | 21 | private controller: GameboyController; 22 | 23 | constructor(emulatorName: string) { 24 | super(); 25 | this.controller = new GameboyController(emulatorName) 26 | } 27 | 28 | init(): Promise { 29 | 30 | return Promise.resolve(); 31 | } 32 | async readyToProceed(nextMessage: NextMessageUser): Promise { 33 | await new Promise(r => setTimeout(r, 500)); 34 | return Promise.resolve(); 35 | } 36 | 37 | async additionalMessageContent(message: InputAgentMessage): Promise { 38 | 39 | let screen = await this.controller.captureScreen(); 40 | return [ 41 | { 42 | saveToChatHistory: 10, 43 | displayOnCurrentMessage: true, 44 | content: [ 45 | { 46 | type: "image_url", 47 | image_url: { 48 | type: "jpeg", 49 | url: screen.toString("base64") 50 | }, 51 | } 52 | ] 53 | } 54 | ]; 55 | } 56 | 57 | async tools(): Promise<(AgentTool)[]> { 58 | return [ 59 | new GameboyControllerPlugin(this.controller), 60 | ]; 61 | } 62 | } 63 | 64 | class GameboyControllerPlugin extends AgentTool { 65 | 66 | schema = z.object({ 67 | action: z.enum(["a", "b", "start", "select", "up", "down", "left", "right"]).describe("The button to click on the gameboy."), 68 | stepsToTake: z.number().optional().describe("The number of steps to move the characted in the game, this only applies to the up, down, left and right buttons."), 69 | }) 70 | 71 | name: string = "gameboy_controller"; 72 | description: string = "Control the Pokémon game using button presses."; 73 | private controller: GameboyController; 74 | 75 | constructor(controller: GameboyController) { 76 | super(); 77 | this.controller = controller; 78 | } 79 | 80 | protected async _call(arg: z.input, runManager?: CallbackManagerForToolRun | undefined): Promise { 81 | if (arg.stepsToTake && ["up", "down", "left", "right"].includes(arg.action)) { 82 | for (let i = 0; i < arg.stepsToTake; i++) { 83 | await this.controller.pressButton(arg.action, 100); 84 | } 85 | } else { 86 | await this.controller.pressButton(arg.action, 100); 87 | } 88 | 89 | return [ 90 | { 91 | type: "text", 92 | text: `Pressed ${arg.action} on the gameboy.`, 93 | } 94 | ] 95 | } 96 | } -------------------------------------------------------------------------------- /tools/gameboy-play/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021", 4 | "allowJs": true, 5 | "allowSyntheticDefaultImports": true, 6 | "baseUrl": "src", 7 | "declaration": true, 8 | "sourceMap": true, 9 | "inlineSourceMap": false, 10 | 11 | "moduleResolution": "nodenext", 12 | "rootDir": "src", 13 | "module": "NodeNext", /* Specify what module code is generated. */ 14 | "outDir": "dist", 15 | "lib": [ 16 | "ES2021", 17 | "ES2022.Object", 18 | "DOM" 19 | ], 20 | "types": [ 21 | "node", 22 | ], 23 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ 24 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ 25 | /* Type Checking */ 26 | "strict": true, /* Enable all strict type-checking options. */ 27 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 28 | }, 29 | "include": [ 30 | "src/**/*" 31 | ], 32 | "ts-node": { 33 | "esm": true 34 | } 35 | } -------------------------------------------------------------------------------- /tools/javascript-code-runner/README.md: -------------------------------------------------------------------------------- 1 | # javascript-code-runner 2 | 3 | A langchain javascript code runner. Runs the javascript code in a sandboxed environment. 4 | 5 | ## Usage: 6 | ```javascript 7 | 8 | const JavascriptCodeRunner = require('@agent-mimir/javascript-code-runner').JavascriptCodeRunner; 9 | 10 | //Add to agents tool: 11 | tools: [ 12 | new JavascriptCodeRunner(), 13 | ], 14 | ``` -------------------------------------------------------------------------------- /tools/javascript-code-runner/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@agent-mimir/javascript-code-runner", 3 | "description": "Javascript code runner tool for langchain", 4 | "main": "dist/index.js", 5 | "scripts": { 6 | "build": "tsc -p tsconfig.json " 7 | }, 8 | "author": "Altaflux", 9 | "keywords": [ 10 | "langchain" 11 | ], 12 | "devDependencies": { 13 | "@types/node": "^18.16.3", 14 | "rimraf": "^5.0.0", 15 | "typescript": "^5.3.3" 16 | }, 17 | "dependencies": { 18 | "vm2": "^3.9.19" 19 | }, 20 | "peerDependencies": { 21 | "@langchain/core": "*" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /tools/javascript-code-runner/src/index.ts: -------------------------------------------------------------------------------- 1 | import { NodeVM } from "vm2"; 2 | import { StructuredTool } from "@langchain/core/tools"; 3 | 4 | import { z } from "zod"; 5 | 6 | export class JavascriptCodeRunner extends StructuredTool { 7 | schema = z.object({ 8 | code: z.string().describe("The javascript code to run. Always use a \"return\" statement to return the result."), 9 | }); 10 | 11 | protected async _call(arg: z.input): Promise { 12 | const vm = new NodeVM({ 13 | allowAsync: false, 14 | wrapper: "none", 15 | sandbox: {}, 16 | }); 17 | const result = vm.run(arg.code); 18 | if (result === undefined){ 19 | return "Return value was undefined, did you forget to use a \"return\" statement?"; 20 | } 21 | return JSON.stringify(result); 22 | } 23 | 24 | name: string = "javascript-code-runner"; 25 | description: string = "Runs javascript code in a sandboxed environment. Useful for when you need to do calculations, or run some code."; 26 | } -------------------------------------------------------------------------------- /tools/javascript-code-runner/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021", 4 | "allowJs": true, 5 | "allowSyntheticDefaultImports": true, 6 | "baseUrl": "src", 7 | "declaration": true, 8 | "sourceMap": true, 9 | "inlineSourceMap": false, 10 | "moduleResolution": "node", 11 | "rootDir": "src", 12 | "module": "commonjs", /* Specify what module code is generated. */ 13 | "outDir": "dist", 14 | "lib": [ 15 | "ES2021", 16 | "ES2022.Object", 17 | "DOM" 18 | ], 19 | "types": [ 20 | "node", 21 | ], 22 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ 23 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ 24 | /* Type Checking */ 25 | "strict": true, /* Enable all strict type-checking options. */ 26 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 27 | } 28 | } -------------------------------------------------------------------------------- /tools/mcp-client/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@agent-mimir/mcp-client", 3 | "description": "mcp-client", 4 | "main": "dist/index.js", 5 | "type": "module", 6 | "scripts": { 7 | "build": "tsc -p tsconfig.json " 8 | }, 9 | "author": "Altaflux", 10 | "keywords": [ 11 | "langchain" 12 | ], 13 | "devDependencies": { 14 | "@types/node": "^18.16.3", 15 | "rimraf": "^5.0.0", 16 | "typescript": "^5.3.3", 17 | "agent-mimir": "*" 18 | }, 19 | "dependencies": { 20 | "@modelcontextprotocol/sdk": "1.4.1", 21 | "@n8n/json-schema-to-zod": "1.1.0" 22 | 23 | }, 24 | "peerDependencies": { 25 | "agent-mimir": "*" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/index.ts: -------------------------------------------------------------------------------- 1 | export type * from './types.js'; 2 | export { jsonSchemaToZod } from './json-schema-to-zod.js'; 3 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/json-schema-to-zod.ts: -------------------------------------------------------------------------------- 1 | import type { z } from 'zod'; 2 | 3 | import { parseSchema } from './parsers/parse-schema.js'; 4 | import type { JsonSchemaToZodOptions, JsonSchema } from './types.js'; 5 | 6 | export const jsonSchemaToZod = ( 7 | schema: JsonSchema, 8 | options: JsonSchemaToZodOptions = {}, 9 | ): T => { 10 | return parseSchema(schema, { 11 | path: [], 12 | seen: new Map(), 13 | ...options, 14 | }) as T; 15 | }; 16 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-all-of.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import { parseSchema } from './parse-schema.js'; 4 | import type { JsonSchemaObject, JsonSchema, Refs } from '../types.js'; 5 | import { half } from '../utils/half.js'; 6 | 7 | const originalIndex = Symbol('Original index'); 8 | 9 | const ensureOriginalIndex = (arr: JsonSchema[]) => { 10 | const newArr = []; 11 | 12 | for (let i = 0; i < arr.length; i++) { 13 | const item = arr[i]; 14 | if (typeof item === 'boolean') { 15 | newArr.push(item ? { [originalIndex]: i } : { [originalIndex]: i, not: {} }); 16 | } else if (originalIndex in item) { 17 | return arr; 18 | } else { 19 | newArr.push({ ...item, [originalIndex]: i }); 20 | } 21 | } 22 | 23 | return newArr; 24 | }; 25 | 26 | export function parseAllOf( 27 | jsonSchema: JsonSchemaObject & { allOf: JsonSchema[] }, 28 | refs: Refs, 29 | ): z.ZodTypeAny { 30 | if (jsonSchema.allOf.length === 0) { 31 | return z.never(); 32 | } 33 | 34 | if (jsonSchema.allOf.length === 1) { 35 | const item = jsonSchema.allOf[0]; 36 | 37 | return parseSchema(item, { 38 | ...refs, 39 | path: [...refs.path, 'allOf', (item as never)[originalIndex]], 40 | }); 41 | } 42 | 43 | const [left, right] = half(ensureOriginalIndex(jsonSchema.allOf)); 44 | 45 | return z.intersection(parseAllOf({ allOf: left }, refs), parseAllOf({ allOf: right }, refs)); 46 | } 47 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-any-of.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import { parseSchema } from './parse-schema.js'; 4 | import type { JsonSchemaObject, JsonSchema, Refs } from '../types.js'; 5 | 6 | export const parseAnyOf = (jsonSchema: JsonSchemaObject & { anyOf: JsonSchema[] }, refs: Refs) => { 7 | return jsonSchema.anyOf.length 8 | ? jsonSchema.anyOf.length === 1 9 | ? parseSchema(jsonSchema.anyOf[0], { 10 | ...refs, 11 | path: [...refs.path, 'anyOf', 0], 12 | }) 13 | : z.union( 14 | jsonSchema.anyOf.map((schema, i) => 15 | parseSchema(schema, { ...refs, path: [...refs.path, 'anyOf', i] }), 16 | ) as [z.ZodTypeAny, z.ZodTypeAny], 17 | ) 18 | : z.any(); 19 | }; 20 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-array.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import { parseSchema } from './parse-schema.js'; 4 | import type { JsonSchemaObject, Refs } from '../types.js'; 5 | import { extendSchemaWithMessage } from '../utils/extend-schema.js'; 6 | 7 | export const parseArray = (jsonSchema: JsonSchemaObject & { type: 'array' }, refs: Refs) => { 8 | if (Array.isArray(jsonSchema.items)) { 9 | return z.tuple( 10 | jsonSchema.items.map((v, i) => 11 | parseSchema(v, { ...refs, path: [...refs.path, 'items', i] }), 12 | ) as [z.ZodTypeAny], 13 | ); 14 | } 15 | 16 | let zodSchema = !jsonSchema.items 17 | ? z.array(z.any()) 18 | : z.array(parseSchema(jsonSchema.items, { ...refs, path: [...refs.path, 'items'] })); 19 | 20 | zodSchema = extendSchemaWithMessage( 21 | zodSchema, 22 | jsonSchema, 23 | 'minItems', 24 | (zs, minItems, errorMessage) => zs.min(minItems, errorMessage), 25 | ); 26 | zodSchema = extendSchemaWithMessage( 27 | zodSchema, 28 | jsonSchema, 29 | 'maxItems', 30 | (zs, maxItems, errorMessage) => zs.max(maxItems, errorMessage), 31 | ); 32 | 33 | return zodSchema; 34 | }; 35 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-boolean.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import type { JsonSchemaObject } from '../types.js'; 4 | 5 | export const parseBoolean = (_jsonSchema: JsonSchemaObject & { type: 'boolean' }) => { 6 | return z.boolean(); 7 | }; 8 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-const.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import type { JsonSchemaObject, Serializable } from '../types.js'; 4 | 5 | export const parseConst = (jsonSchema: JsonSchemaObject & { const: Serializable }) => { 6 | return z.literal(jsonSchema.const as z.Primitive); 7 | }; 8 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-default.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import type { JsonSchemaObject } from '../types.js'; 4 | 5 | export const parseDefault = (_jsonSchema: JsonSchemaObject) => { 6 | return z.any(); 7 | }; 8 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-enum.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import type { JsonSchemaObject, Serializable } from '../types.js'; 4 | 5 | export const parseEnum = (jsonSchema: JsonSchemaObject & { enum: Serializable[] }) => { 6 | if (jsonSchema.enum.length === 0) { 7 | return z.never(); 8 | } 9 | 10 | if (jsonSchema.enum.length === 1) { 11 | // union does not work when there is only one element 12 | return z.literal(jsonSchema.enum[0] as z.Primitive); 13 | } 14 | 15 | if (jsonSchema.enum.every((x) => typeof x === 'string')) { 16 | return z.enum(jsonSchema.enum as [string]); 17 | } 18 | 19 | return z.union( 20 | jsonSchema.enum.map((x) => z.literal(x as z.Primitive)) as unknown as [ 21 | z.ZodTypeAny, 22 | z.ZodTypeAny, 23 | ], 24 | ); 25 | }; 26 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-if-then-else.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import { parseSchema } from './parse-schema.js'; 4 | import type { JsonSchemaObject, JsonSchema, Refs } from '../types.js'; 5 | 6 | export const parseIfThenElse = ( 7 | jsonSchema: JsonSchemaObject & { 8 | if: JsonSchema; 9 | then: JsonSchema; 10 | else: JsonSchema; 11 | }, 12 | refs: Refs, 13 | ) => { 14 | const $if = parseSchema(jsonSchema.if, { ...refs, path: [...refs.path, 'if'] }); 15 | const $then = parseSchema(jsonSchema.then, { 16 | ...refs, 17 | path: [...refs.path, 'then'], 18 | }); 19 | const $else = parseSchema(jsonSchema.else, { 20 | ...refs, 21 | path: [...refs.path, 'else'], 22 | }); 23 | 24 | return z.union([$then, $else]).superRefine((value, ctx) => { 25 | const result = $if.safeParse(value).success ? $then.safeParse(value) : $else.safeParse(value); 26 | 27 | if (!result.success) { 28 | result.error.errors.forEach((error) => ctx.addIssue(error)); 29 | } 30 | }); 31 | }; 32 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-multiple-type.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import { parseSchema } from './parse-schema.js'; 4 | import type { JsonSchema, JsonSchemaObject, Refs } from '../types.js'; 5 | 6 | export const parseMultipleType = ( 7 | jsonSchema: JsonSchemaObject & { type: string[] }, 8 | refs: Refs, 9 | ) => { 10 | return z.union( 11 | jsonSchema.type.map((type) => parseSchema({ ...jsonSchema, type } as JsonSchema, refs)) as [ 12 | z.ZodTypeAny, 13 | z.ZodTypeAny, 14 | ], 15 | ); 16 | }; 17 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-not.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import { parseSchema } from './parse-schema.js'; 4 | import type { JsonSchemaObject, JsonSchema, Refs } from '../types.js'; 5 | 6 | export const parseNot = (jsonSchema: JsonSchemaObject & { not: JsonSchema }, refs: Refs) => { 7 | return z.any().refine( 8 | (value) => 9 | !parseSchema(jsonSchema.not, { 10 | ...refs, 11 | path: [...refs.path, 'not'], 12 | }).safeParse(value).success, 13 | 'Invalid input: Should NOT be valid against schema', 14 | ); 15 | }; 16 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-null.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import type { JsonSchemaObject } from '../types.js'; 4 | 5 | export const parseNull = (_jsonSchema: JsonSchemaObject & { type: 'null' }) => { 6 | return z.null(); 7 | }; 8 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-nullable.ts: -------------------------------------------------------------------------------- 1 | import { parseSchema } from './parse-schema.js'; 2 | import type { JsonSchemaObject, Refs } from '../types.js'; 3 | import { omit } from '../utils/omit.js'; 4 | 5 | /** 6 | * For compatibility with open api 3.0 nullable 7 | */ 8 | export const parseNullable = (jsonSchema: JsonSchemaObject & { nullable: true }, refs: Refs) => { 9 | return parseSchema(omit(jsonSchema, 'nullable'), refs, true).nullable(); 10 | }; 11 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-number.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import type { JsonSchemaObject } from '../types.js'; 4 | import { extendSchemaWithMessage } from '../utils/extend-schema.js'; 5 | 6 | export const parseNumber = (jsonSchema: JsonSchemaObject & { type: 'number' | 'integer' }) => { 7 | let zodSchema = z.number(); 8 | 9 | let isInteger = false; 10 | if (jsonSchema.type === 'integer') { 11 | isInteger = true; 12 | zodSchema = extendSchemaWithMessage(zodSchema, jsonSchema, 'type', (zs, _, errorMsg) => 13 | zs.int(errorMsg), 14 | ); 15 | } else if (jsonSchema.format === 'int64') { 16 | isInteger = true; 17 | zodSchema = extendSchemaWithMessage(zodSchema, jsonSchema, 'format', (zs, _, errorMsg) => 18 | zs.int(errorMsg), 19 | ); 20 | } 21 | 22 | zodSchema = extendSchemaWithMessage( 23 | zodSchema, 24 | jsonSchema, 25 | 'multipleOf', 26 | (zs, multipleOf, errorMsg) => { 27 | if (multipleOf === 1) { 28 | if (isInteger) return zs; 29 | 30 | return zs.int(errorMsg); 31 | } 32 | 33 | return zs.multipleOf(multipleOf, errorMsg); 34 | }, 35 | ); 36 | 37 | if (typeof jsonSchema.minimum === 'number') { 38 | if (jsonSchema.exclusiveMinimum === true) { 39 | zodSchema = extendSchemaWithMessage( 40 | zodSchema, 41 | jsonSchema, 42 | 'minimum', 43 | (zs, minimum, errorMsg) => zs.gt(minimum, errorMsg), 44 | ); 45 | } else { 46 | zodSchema = extendSchemaWithMessage( 47 | zodSchema, 48 | jsonSchema, 49 | 'minimum', 50 | (zs, minimum, errorMsg) => zs.gte(minimum, errorMsg), 51 | ); 52 | } 53 | } else if (typeof jsonSchema.exclusiveMinimum === 'number') { 54 | zodSchema = extendSchemaWithMessage( 55 | zodSchema, 56 | jsonSchema, 57 | 'exclusiveMinimum', 58 | (zs, exclusiveMinimum, errorMsg) => zs.gt(exclusiveMinimum as number, errorMsg), 59 | ); 60 | } 61 | 62 | if (typeof jsonSchema.maximum === 'number') { 63 | if (jsonSchema.exclusiveMaximum === true) { 64 | zodSchema = extendSchemaWithMessage( 65 | zodSchema, 66 | jsonSchema, 67 | 'maximum', 68 | (zs, maximum, errorMsg) => zs.lt(maximum, errorMsg), 69 | ); 70 | } else { 71 | zodSchema = extendSchemaWithMessage( 72 | zodSchema, 73 | jsonSchema, 74 | 'maximum', 75 | (zs, maximum, errorMsg) => zs.lte(maximum, errorMsg), 76 | ); 77 | } 78 | } else if (typeof jsonSchema.exclusiveMaximum === 'number') { 79 | zodSchema = extendSchemaWithMessage( 80 | zodSchema, 81 | jsonSchema, 82 | 'exclusiveMaximum', 83 | (zs, exclusiveMaximum, errorMsg) => zs.lt(exclusiveMaximum as number, errorMsg), 84 | ); 85 | } 86 | 87 | return zodSchema; 88 | }; 89 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-one-of.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import { parseSchema } from './parse-schema.js'; 4 | import type { JsonSchemaObject, JsonSchema, Refs } from '../types.js'; 5 | 6 | export const parseOneOf = (jsonSchema: JsonSchemaObject & { oneOf: JsonSchema[] }, refs: Refs) => { 7 | if (!jsonSchema.oneOf.length) { 8 | return z.any(); 9 | } 10 | 11 | if (jsonSchema.oneOf.length === 1) { 12 | return parseSchema(jsonSchema.oneOf[0], { 13 | ...refs, 14 | path: [...refs.path, 'oneOf', 0], 15 | }); 16 | } 17 | 18 | return z.any().superRefine((x, ctx) => { 19 | const schemas = jsonSchema.oneOf.map((schema, i) => 20 | parseSchema(schema, { 21 | ...refs, 22 | path: [...refs.path, 'oneOf', i], 23 | }), 24 | ); 25 | 26 | const unionErrors = schemas.reduce( 27 | (errors, schema) => 28 | ((result) => (result.error ? [...errors, result.error] : errors))(schema.safeParse(x)), 29 | [], 30 | ); 31 | 32 | if (schemas.length - unionErrors.length !== 1) { 33 | ctx.addIssue({ 34 | path: ctx.path, 35 | code: 'invalid_union', 36 | unionErrors, 37 | message: 'Invalid input: Should pass single schema', 38 | }); 39 | } 40 | }); 41 | }; 42 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-schema.ts: -------------------------------------------------------------------------------- 1 | import * as z from 'zod'; 2 | 3 | import { parseAllOf } from './parse-all-of.js'; 4 | import { parseAnyOf } from './parse-any-of.js'; 5 | import { parseArray } from './parse-array.js'; 6 | import { parseBoolean } from './parse-boolean.js'; 7 | import { parseConst } from './parse-const.js'; 8 | import { parseDefault } from './parse-default.js'; 9 | import { parseEnum } from './parse-enum.js'; 10 | import { parseIfThenElse } from './parse-if-then-else.js'; 11 | import { parseMultipleType } from './parse-multiple-type.js'; 12 | import { parseNot } from './parse-not.js'; 13 | import { parseNull } from './parse-null.js'; 14 | import { parseNullable } from './parse-nullable.js'; 15 | import { parseNumber } from './parse-number.js'; 16 | import { parseObject } from './parse-object.js'; 17 | import { parseOneOf } from './parse-one-of.js'; 18 | import { parseString } from './parse-string.js'; 19 | import type { ParserSelector, Refs, JsonSchemaObject, JsonSchema } from '../types.js'; 20 | import { its } from '../utils/its.js'; 21 | 22 | const addDescribes = (jsonSchema: JsonSchemaObject, zodSchema: z.ZodTypeAny): z.ZodTypeAny => { 23 | if (jsonSchema.description) { 24 | zodSchema = zodSchema.describe(jsonSchema.description); 25 | } 26 | 27 | return zodSchema; 28 | }; 29 | 30 | const addDefaults = (jsonSchema: JsonSchemaObject, zodSchema: z.ZodTypeAny): z.ZodTypeAny => { 31 | if (jsonSchema.default !== undefined) { 32 | zodSchema = zodSchema.default(jsonSchema.default); 33 | } 34 | 35 | return zodSchema; 36 | }; 37 | 38 | const addAnnotations = (jsonSchema: JsonSchemaObject, zodSchema: z.ZodTypeAny): z.ZodTypeAny => { 39 | if (jsonSchema.readOnly) { 40 | zodSchema = zodSchema.readonly(); 41 | } 42 | 43 | return zodSchema; 44 | }; 45 | 46 | const selectParser: ParserSelector = (schema, refs) => { 47 | if (its.a.nullable(schema)) { 48 | return parseNullable(schema, refs); 49 | } else if (its.an.object(schema)) { 50 | return parseObject(schema, refs); 51 | } else if (its.an.array(schema)) { 52 | return parseArray(schema, refs); 53 | } else if (its.an.anyOf(schema)) { 54 | return parseAnyOf(schema, refs); 55 | } else if (its.an.allOf(schema)) { 56 | return parseAllOf(schema, refs); 57 | } else if (its.a.oneOf(schema)) { 58 | return parseOneOf(schema, refs); 59 | } else if (its.a.not(schema)) { 60 | return parseNot(schema, refs); 61 | } else if (its.an.enum(schema)) { 62 | return parseEnum(schema); //<-- needs to come before primitives 63 | } else if (its.a.const(schema)) { 64 | return parseConst(schema); 65 | } else if (its.a.multipleType(schema)) { 66 | return parseMultipleType(schema, refs); 67 | } else if (its.a.primitive(schema, 'string')) { 68 | return parseString(schema); 69 | } else if (its.a.primitive(schema, 'number') || its.a.primitive(schema, 'integer')) { 70 | return parseNumber(schema); 71 | } else if (its.a.primitive(schema, 'boolean')) { 72 | return parseBoolean(schema); 73 | } else if (its.a.primitive(schema, 'null')) { 74 | return parseNull(schema); 75 | } else if (its.a.conditional(schema)) { 76 | return parseIfThenElse(schema, refs); 77 | } else { 78 | return parseDefault(schema); 79 | } 80 | }; 81 | 82 | export const parseSchema = ( 83 | jsonSchema: JsonSchema, 84 | refs: Refs = { seen: new Map(), path: [] }, 85 | blockMeta?: boolean, 86 | ): z.ZodTypeAny => { 87 | if (typeof jsonSchema !== 'object') return jsonSchema ? z.any() : z.never(); 88 | 89 | if (refs.parserOverride) { 90 | const custom = refs.parserOverride(jsonSchema, refs); 91 | 92 | if (custom instanceof z.ZodType) { 93 | return custom; 94 | } 95 | } 96 | 97 | let seen = refs.seen.get(jsonSchema); 98 | 99 | if (seen) { 100 | if (seen.r !== undefined) { 101 | return seen.r; 102 | } 103 | 104 | if (refs.depth === undefined || seen.n >= refs.depth) { 105 | return z.any(); 106 | } 107 | 108 | seen.n += 1; 109 | } else { 110 | seen = { r: undefined, n: 0 }; 111 | refs.seen.set(jsonSchema, seen); 112 | } 113 | 114 | let parsedZodSchema = selectParser(jsonSchema, refs); 115 | if (!blockMeta) { 116 | if (!refs.withoutDescribes) { 117 | parsedZodSchema = addDescribes(jsonSchema, parsedZodSchema); 118 | } 119 | 120 | if (!refs.withoutDefaults) { 121 | parsedZodSchema = addDefaults(jsonSchema, parsedZodSchema); 122 | } 123 | 124 | parsedZodSchema = addAnnotations(jsonSchema, parsedZodSchema); 125 | } 126 | 127 | seen.r = parsedZodSchema; 128 | 129 | return parsedZodSchema; 130 | }; 131 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/parsers/parse-string.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | import type { JsonSchemaObject } from '../types.js'; 4 | import { extendSchemaWithMessage } from '../utils/extend-schema.js'; 5 | 6 | export const parseString = (jsonSchema: JsonSchemaObject & { type: 'string' }) => { 7 | let zodSchema = z.string(); 8 | 9 | zodSchema = extendSchemaWithMessage(zodSchema, jsonSchema, 'format', (zs, format, errorMsg) => { 10 | switch (format) { 11 | case 'email': 12 | return zs.email(errorMsg); 13 | case 'ip': 14 | return zs.ip(errorMsg); 15 | case 'ipv4': 16 | return zs.ip({ version: 'v4', message: errorMsg }); 17 | case 'ipv6': 18 | return zs.ip({ version: 'v6', message: errorMsg }); 19 | case 'uri': 20 | return zs.url(errorMsg); 21 | case 'uuid': 22 | return zs.uuid(errorMsg); 23 | case 'date-time': 24 | return zs.datetime({ offset: true, message: errorMsg }); 25 | case 'time': 26 | return zs.time(errorMsg); 27 | case 'date': 28 | return zs.date(errorMsg); 29 | case 'binary': 30 | return zs.base64(errorMsg); 31 | case 'duration': 32 | return zs.duration(errorMsg); 33 | default: 34 | return zs; 35 | } 36 | }); 37 | 38 | zodSchema = extendSchemaWithMessage(zodSchema, jsonSchema, 'contentEncoding', (zs, _, errorMsg) => 39 | zs.base64(errorMsg), 40 | ); 41 | zodSchema = extendSchemaWithMessage(zodSchema, jsonSchema, 'pattern', (zs, pattern, errorMsg) => 42 | zs.regex(new RegExp(pattern), errorMsg), 43 | ); 44 | zodSchema = extendSchemaWithMessage( 45 | zodSchema, 46 | jsonSchema, 47 | 'minLength', 48 | (zs, minLength, errorMsg) => zs.min(minLength, errorMsg), 49 | ); 50 | zodSchema = extendSchemaWithMessage( 51 | zodSchema, 52 | jsonSchema, 53 | 'maxLength', 54 | (zs, maxLength, errorMsg) => zs.max(maxLength, errorMsg), 55 | ); 56 | 57 | return zodSchema; 58 | }; 59 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/types.ts: -------------------------------------------------------------------------------- 1 | import type { ZodTypeAny } from 'zod'; 2 | 3 | export type Serializable = 4 | | { [key: string]: Serializable } 5 | | Serializable[] 6 | | string 7 | | number 8 | | boolean 9 | | null; 10 | 11 | export type JsonSchema = JsonSchemaObject | boolean; 12 | export type JsonSchemaObject = { 13 | // left permissive by design 14 | type?: string | string[]; 15 | 16 | // object 17 | properties?: { [key: string]: JsonSchema }; 18 | additionalProperties?: JsonSchema; 19 | unevaluatedProperties?: JsonSchema; 20 | patternProperties?: { [key: string]: JsonSchema }; 21 | minProperties?: number; 22 | maxProperties?: number; 23 | required?: string[] | boolean; 24 | propertyNames?: JsonSchema; 25 | 26 | // array 27 | items?: JsonSchema | JsonSchema[]; 28 | additionalItems?: JsonSchema; 29 | minItems?: number; 30 | maxItems?: number; 31 | uniqueItems?: boolean; 32 | 33 | // string 34 | minLength?: number; 35 | maxLength?: number; 36 | pattern?: string; 37 | format?: string; 38 | 39 | // number 40 | minimum?: number; 41 | maximum?: number; 42 | exclusiveMinimum?: number | boolean; 43 | exclusiveMaximum?: number | boolean; 44 | multipleOf?: number; 45 | 46 | // unions 47 | anyOf?: JsonSchema[]; 48 | allOf?: JsonSchema[]; 49 | oneOf?: JsonSchema[]; 50 | 51 | if?: JsonSchema; 52 | then?: JsonSchema; 53 | else?: JsonSchema; 54 | 55 | // shared 56 | const?: Serializable; 57 | enum?: Serializable[]; 58 | 59 | errorMessage?: { [key: string]: string | undefined }; 60 | 61 | description?: string; 62 | default?: Serializable; 63 | readOnly?: boolean; 64 | not?: JsonSchema; 65 | contentEncoding?: string; 66 | nullable?: boolean; 67 | }; 68 | 69 | export type ParserSelector = (schema: JsonSchemaObject, refs: Refs) => ZodTypeAny; 70 | export type ParserOverride = (schema: JsonSchemaObject, refs: Refs) => ZodTypeAny | undefined; 71 | 72 | export type JsonSchemaToZodOptions = { 73 | withoutDefaults?: boolean; 74 | withoutDescribes?: boolean; 75 | parserOverride?: ParserOverride; 76 | depth?: number; 77 | }; 78 | 79 | export type Refs = JsonSchemaToZodOptions & { 80 | path: Array; 81 | seen: Map; 82 | }; 83 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/utils/extend-schema.ts: -------------------------------------------------------------------------------- 1 | import type { z } from 'zod'; 2 | 3 | import type { JsonSchemaObject } from '../types.js'; 4 | 5 | export function extendSchemaWithMessage< 6 | TZod extends z.ZodTypeAny, 7 | TJson extends JsonSchemaObject, 8 | TKey extends keyof TJson, 9 | >( 10 | zodSchema: TZod, 11 | jsonSchema: TJson, 12 | key: TKey, 13 | extend: (zodSchema: TZod, value: NonNullable, errorMessage?: string) => TZod, 14 | ) { 15 | const value = jsonSchema[key]; 16 | 17 | if (value !== undefined) { 18 | const errorMessage = jsonSchema.errorMessage?.[key as string]; 19 | return extend(zodSchema, value as NonNullable, errorMessage); 20 | } 21 | 22 | return zodSchema; 23 | } 24 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/utils/half.ts: -------------------------------------------------------------------------------- 1 | export const half = (arr: T[]): [T[], T[]] => { 2 | return [arr.slice(0, arr.length / 2), arr.slice(arr.length / 2)]; 3 | }; 4 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/utils/its.ts: -------------------------------------------------------------------------------- 1 | import type { JsonSchema, JsonSchemaObject, Serializable } from '../types.js'; 2 | 3 | export const its = { 4 | an: { 5 | object: (x: JsonSchemaObject): x is JsonSchemaObject & { type: 'object' } => 6 | x.type === 'object', 7 | array: (x: JsonSchemaObject): x is JsonSchemaObject & { type: 'array' } => x.type === 'array', 8 | anyOf: ( 9 | x: JsonSchemaObject, 10 | ): x is JsonSchemaObject & { 11 | anyOf: JsonSchema[]; 12 | } => x.anyOf !== undefined, 13 | allOf: ( 14 | x: JsonSchemaObject, 15 | ): x is JsonSchemaObject & { 16 | allOf: JsonSchema[]; 17 | } => x.allOf !== undefined, 18 | enum: ( 19 | x: JsonSchemaObject, 20 | ): x is JsonSchemaObject & { 21 | enum: Serializable | Serializable[]; 22 | } => x.enum !== undefined, 23 | }, 24 | a: { 25 | nullable: (x: JsonSchemaObject): x is JsonSchemaObject & { nullable: true } => 26 | // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access 27 | (x as any).nullable === true, 28 | multipleType: (x: JsonSchemaObject): x is JsonSchemaObject & { type: string[] } => 29 | Array.isArray(x.type), 30 | not: ( 31 | x: JsonSchemaObject, 32 | ): x is JsonSchemaObject & { 33 | not: JsonSchema; 34 | } => x.not !== undefined, 35 | const: ( 36 | x: JsonSchemaObject, 37 | ): x is JsonSchemaObject & { 38 | const: Serializable; 39 | } => x.const !== undefined, 40 | primitive: ( 41 | x: JsonSchemaObject, 42 | p: T, 43 | ): x is JsonSchemaObject & { type: T } => x.type === p, 44 | conditional: ( 45 | x: JsonSchemaObject, 46 | ): x is JsonSchemaObject & { 47 | if: JsonSchema; 48 | then: JsonSchema; 49 | else: JsonSchema; 50 | } => Boolean('if' in x && x.if && 'then' in x && 'else' in x && x.then && x.else), 51 | oneOf: ( 52 | x: JsonSchemaObject, 53 | ): x is JsonSchemaObject & { 54 | oneOf: JsonSchema[]; 55 | } => x.oneOf !== undefined, 56 | }, 57 | }; 58 | -------------------------------------------------------------------------------- /tools/mcp-client/src/json-schema-to-zod/utils/omit.ts: -------------------------------------------------------------------------------- 1 | export const omit = (obj: T, ...keys: K[]): Omit => 2 | Object.keys(obj).reduce((acc: Record, key) => { 3 | if (!keys.includes(key as K)) { 4 | acc[key] = obj[key as K]; 5 | } 6 | 7 | return acc; 8 | }, {}) as Omit; 9 | -------------------------------------------------------------------------------- /tools/mcp-client/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021", 4 | "allowJs": true, 5 | "allowSyntheticDefaultImports": true, 6 | "baseUrl": "src", 7 | "declaration": true, 8 | "sourceMap": true, 9 | "inlineSourceMap": false, 10 | 11 | "moduleResolution": "nodenext", 12 | "rootDir": "src", 13 | "module": "NodeNext", /* Specify what module code is generated. */ 14 | "outDir": "dist", 15 | "lib": [ 16 | "ES2021", 17 | "ES2022.Object", 18 | "DOM" 19 | ], 20 | "types": [ 21 | "node", 22 | ], 23 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ 24 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ 25 | /* Type Checking */ 26 | "strict": true, /* Enable all strict type-checking options. */ 27 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 28 | }, 29 | "include": [ 30 | "src/**/*" 31 | ], 32 | "ts-node": { 33 | "esm": true 34 | } 35 | } -------------------------------------------------------------------------------- /tools/playwright-browser/README.md: -------------------------------------------------------------------------------- 1 | # langchain-selenium-browser 2 | 3 | A langchain web browser tool based on Selenium 4 | 5 | ## Usage: 6 | ```javascript 7 | const WebBrowserToolKit = require('@agent-mimir/selenium-browser').WebBrowserToolKit; 8 | const model = new ChatOpenAI({ 9 | openAIApiKey: process.env.OPENAI_API_KEY, 10 | temperature: 0.9, 11 | }); 12 | const embeddings = new OpenAIEmbeddings({ 13 | openAIApiKey: process.env.OPENAI_API_KEY, 14 | }); 15 | 16 | const webToolKit = new WebBrowserToolKit({ browserConfig: { browserName: "chrome" } }, model, embeddings); 17 | 18 | //Add to agents tool: 19 | tools: [ 20 | ...webToolKit.tools, 21 | ], 22 | ``` -------------------------------------------------------------------------------- /tools/playwright-browser/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@agent-mimir/playwright-browser", 3 | "description": "Playwright Web browser tool for langchain", 4 | "main": "dist/index.js", 5 | "type": "module", 6 | "scripts": { 7 | "build": "tsc -p tsconfig.json ", 8 | "install": "playwright install" 9 | }, 10 | "author": "Altaflux", 11 | "keywords": [ 12 | "langchain" 13 | ], 14 | "devDependencies": { 15 | "@types/async-exit-hook": "^2.0.0", 16 | "@types/jsdom": "^21.1.1", 17 | "@types/node": "^18.16.3", 18 | "@types/selenium-webdriver": "^4.1.14", 19 | "@types/turndown": "^5.0.1", 20 | "npm-run-all": "^4.1.5", 21 | "release-it": "^15.10.0", 22 | "rimraf": "^5.0.0", 23 | "typescript": "^5.3.3", 24 | "agent-mimir": "*" 25 | }, 26 | "dependencies": { 27 | "async-exit-hook": "^2.0.1", 28 | "cheerio": "^1.0.0-rc.12", 29 | "jsdom": "22.0.0", 30 | "sharp": "^0.32.6", 31 | "playwright": "1.50.1", 32 | "turndown": "^7.1.2", 33 | "gpt-3-encoder": "v1.1.4" 34 | }, 35 | "peerDependencies": { 36 | "agent-mimir": "*" 37 | } 38 | } -------------------------------------------------------------------------------- /tools/playwright-browser/src/prompt/combiner-prompt.ts: -------------------------------------------------------------------------------- 1 | import { PromptTemplate } from "@langchain/core/prompts"; 2 | 3 | //Combine the following two parts of a markdown document into one. 4 | const _DEFAULT_SUMMARIZER_TEMPLATE =`Combine the contents of "part 2" of a markdown document with "part 1" of the document. 5 | The document is a representation of the contents of a website. 6 | The title of the website is "{title}". 7 | 8 | The result should be in Markdown format. 9 | Prioritize keeping details related to what I intend to find in the site: "{focus}". 10 | Remove details that is not related to what I intend to find. 11 | 12 | Include any relevant html inputs, buttons, and link elements (always including all attributes and text). Do not rewrite or simplify the html elements! 13 | 14 | Summarize any large pieces of text while keeping key relevant pieces of information. 15 | 16 | If the combined document is too long, remove less relevant content from any of the parts of the document. 17 | 18 | If the contents of part 2 of the document is irrelevant to what I intend to find and you will not combine any content from it, respond only with the word "DISCARD" and nothing else! 19 | ------------------------- 20 | This is part 1 of the document: 21 | 22 | {document1} 23 | 24 | ------------------------- 25 | This is part 2 of the document: 26 | 27 | {document2} 28 | 29 | ------------------------- 30 | Combined Markdown document: 31 | `; 32 | export const COMBINE_PROMPT = /* #__PURE__ */ new PromptTemplate({ 33 | template: _DEFAULT_SUMMARIZER_TEMPLATE, 34 | inputVariables: ["title", "document1", "document2", "focus"], 35 | }); 36 | -------------------------------------------------------------------------------- /tools/playwright-browser/src/prompt/relevance-prompt.ts: -------------------------------------------------------------------------------- 1 | import { PromptTemplate } from "@langchain/core/prompts"; 2 | 3 | 4 | 5 | 6 | const IS_RELEVANT_PROMPT_TEMPLATE = `Respond me with a either the word "true" or "false" depending if the following piece of a website is relevant to: "{focus}". 7 | "true" if the piece of the website is relevant or is significantly related to the piece of the website, "false" if it is not. 8 | 9 | The title of the website is "{title}". 10 | 11 | Piece of website: 12 | ------------ 13 | {document} 14 | ------------ 15 | 16 | Was that piece of the website relevant?: 17 | `; 18 | export const IS_RELEVANT_PROMPT = /* #__PURE__ */ new PromptTemplate({ 19 | template: IS_RELEVANT_PROMPT_TEMPLATE, 20 | inputVariables: ["document", "focus", "title"], 21 | }); 22 | 23 | 24 | -------------------------------------------------------------------------------- /tools/playwright-browser/src/to-markdown.ts: -------------------------------------------------------------------------------- 1 | import TurndownService from 'turndown'; 2 | 3 | function buildAttribute(attributeName: string, attributeValue: string | null, defaultValue?: string) { 4 | if (attributeValue) { 5 | return `${attributeName}="${attributeValue}"`; 6 | } else if (defaultValue) { 7 | return `${attributeName}="${defaultValue}"`; 8 | } 9 | return ""; 10 | } 11 | 12 | function isEmptyOrSpaces(str: string | null) { 13 | return str === null || str.match(/^ *$/) !== null; 14 | } 15 | 16 | 17 | function getInputorLinkInfo(document: ParentNode, element: Element) { 18 | 19 | const ariaLabel = element.getAttribute('aria-label'); 20 | if (ariaLabel && ariaLabel !== '') { 21 | return ariaLabel! 22 | } 23 | const ariaLabelledBy = element.getAttribute('aria-labelledby'); 24 | if (ariaLabelledBy) { 25 | const labelledByElement = document.querySelector(`[id="${ariaLabelledBy}"]`); 26 | if (labelledByElement) { 27 | if (!labelledByElement.textContent) { 28 | console.log(`No text content for aria-labelledby ${ariaLabelledBy}`); 29 | } 30 | return labelledByElement.textContent?.trim() ?? null; 31 | } 32 | } 33 | 34 | if (!isEmptyOrSpaces(element.textContent?.trim() ?? null)) { 35 | return element.textContent?.trim() ?? null; 36 | } 37 | if (!isEmptyOrSpaces(element.getAttribute('placeholder')?.trim() ?? null)) { 38 | return element.getAttribute('placeholder')?.trim() ?? null; 39 | } 40 | 41 | return null; 42 | } 43 | 44 | function applyTurndownFixes(htmlDoc: Document) { 45 | //Turndown incorrectly does not mark ` 86 | } 87 | return ""; 88 | } 89 | }).addRule('img', { 90 | filter: ['img'], 91 | replacement: function (content, node, options) { 92 | let element = node as HTMLElement; 93 | var alt = cleanAttribute(element.getAttribute('alt')); 94 | if (alt === '') return ''; 95 | 96 | var src = element.getAttribute('src') || ''; 97 | var title = cleanAttribute(element.getAttribute('title')); 98 | var titlePart = title ? ' "' + title + '"' : ''; 99 | return src ? '![' + alt + ']' + '(' + '...' + src.slice(-15) + titlePart + ')' : ''; 100 | } 101 | }).addRule('removePicture', { 102 | filter: ['picture'], 103 | replacement: function () { 104 | return ""; 105 | } 106 | }).addRule('removeScript', { 107 | filter: ['script'], 108 | replacement: function () { 109 | return ""; 110 | } 111 | }).addRule('formatInput', { 112 | filter: ['input'], 113 | replacement: function (_, node) { 114 | let element = node as HTMLElement; 115 | const description = getInputorLinkInfo(htmlDoc, element); 116 | if (description) { 117 | return `` 118 | } 119 | return ""; 120 | } 121 | }); 122 | 123 | applyTurndownFixes(htmlDoc); 124 | const markdown = turndownService.turndown(htmlDoc.body.outerHTML); 125 | return markdown; 126 | } 127 | 128 | function cleanAttribute (attribute:string | null) { 129 | return attribute ? attribute.replace(/(\n+\s*)+/g, '\n') : '' 130 | } -------------------------------------------------------------------------------- /tools/playwright-browser/src/xpath.ts: -------------------------------------------------------------------------------- 1 | type XpathComponent = { 2 | name?: string, 3 | position?: number | null 4 | } 5 | 6 | export function getXPath(node: Element) { 7 | var comp, comps : XpathComponent[] = []; 8 | var xpath = ''; 9 | var getPos = function (node: Element) { 10 | var position = 1, curNode; 11 | if (node.nodeType == node.ATTRIBUTE_NODE) { 12 | return null; 13 | } 14 | for (curNode = node.previousSibling; curNode; curNode = curNode.previousSibling) { 15 | if (curNode.nodeName == node.nodeName) { 16 | ++position; 17 | } 18 | } 19 | return position; 20 | } 21 | let nodeToFind: Element | null = node; 22 | for (; nodeToFind && !(nodeToFind.nodeType === nodeToFind.DOCUMENT_NODE); nodeToFind = nodeToFind.nodeType == nodeToFind.ATTRIBUTE_NODE ? (nodeToFind as any).ownerElement : nodeToFind.parentNode) { 23 | 24 | let comp: XpathComponent | null = comps[comps.length] = {}; 25 | switch (nodeToFind.nodeType) { 26 | case nodeToFind.TEXT_NODE: 27 | comp.name = 'text()'; 28 | break; 29 | case nodeToFind.ATTRIBUTE_NODE: 30 | comp.name = '@' + nodeToFind.nodeName; 31 | break; 32 | case nodeToFind.PROCESSING_INSTRUCTION_NODE: 33 | comp.name = 'processing-instruction()'; 34 | break; 35 | case nodeToFind.COMMENT_NODE: 36 | comp.name = 'comment()'; 37 | break; 38 | case nodeToFind.ELEMENT_NODE: 39 | comp.name = nodeToFind.nodeName; 40 | if (nodeToFind.hasAttribute('id')) { 41 | comp.name = '/*[@id="' + nodeToFind.getAttribute('id') + '"]'; 42 | nodeToFind = null; 43 | } 44 | break; 45 | } 46 | if (!nodeToFind) { 47 | break; 48 | } else { 49 | comp.position = getPos(nodeToFind); 50 | } 51 | } 52 | 53 | for (var i = comps.length - 1; i >= 0; i--) { 54 | comp = comps[i]; 55 | xpath += '/' + comp.name; 56 | if (comp.position) { 57 | xpath += '[' + comp.position + ']'; 58 | } 59 | } 60 | 61 | return xpath; 62 | 63 | } 64 | -------------------------------------------------------------------------------- /tools/playwright-browser/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021", 4 | "allowJs": true, 5 | "allowSyntheticDefaultImports": true, 6 | "baseUrl": "src", 7 | "declaration": true, 8 | "sourceMap": true, 9 | "inlineSourceMap": false, 10 | "moduleResolution": "nodenext", 11 | "rootDir": "src", 12 | "module": "NodeNext", /* Specify what module code is generated. */ 13 | "outDir": "dist", 14 | "lib": [ 15 | "ES2021", 16 | "ES2022.Object", 17 | "DOM" 18 | ], 19 | "types": [ 20 | "node", 21 | ], 22 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ 23 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ 24 | /* Type Checking */ 25 | "strict": true, /* Enable all strict type-checking options. */ 26 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 27 | }, 28 | "ts-node": { 29 | "esm": true 30 | } 31 | } -------------------------------------------------------------------------------- /tools/selenium-browser/README.md: -------------------------------------------------------------------------------- 1 | # langchain-selenium-browser 2 | 3 | A langchain web browser tool based on Selenium 4 | 5 | ## Usage: 6 | ```javascript 7 | const WebBrowserToolKit = require('@agent-mimir/selenium-browser').WebBrowserToolKit; 8 | const model = new ChatOpenAI({ 9 | openAIApiKey: process.env.OPENAI_API_KEY, 10 | temperature: 0.9, 11 | }); 12 | const embeddings = new OpenAIEmbeddings({ 13 | openAIApiKey: process.env.OPENAI_API_KEY, 14 | }); 15 | 16 | const webToolKit = new WebBrowserToolKit({ browserConfig: { browserName: "chrome" } }, model, embeddings); 17 | 18 | //Add to agents tool: 19 | tools: [ 20 | ...webToolKit.tools, 21 | ], 22 | ``` -------------------------------------------------------------------------------- /tools/selenium-browser/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@agent-mimir/selenium-browser", 3 | "description": "Selenium Web browser tool for langchain", 4 | "main": "dist/index.js", 5 | "type": "module", 6 | "scripts": { 7 | "build": "tsc -p tsconfig.json " 8 | }, 9 | "author": "Altaflux", 10 | "keywords": [ 11 | "langchain" 12 | ], 13 | "devDependencies": { 14 | "@types/async-exit-hook": "^2.0.0", 15 | "@types/jsdom": "^21.1.1", 16 | "@types/node": "^18.16.3", 17 | "@types/selenium-webdriver": "^4.1.14", 18 | "@types/turndown": "^5.0.1", 19 | "npm-run-all": "^4.1.5", 20 | "release-it": "^15.10.0", 21 | "rimraf": "^5.0.0", 22 | "typescript": "^5.3.3", 23 | "agent-mimir": "*" 24 | }, 25 | "dependencies": { 26 | "async-exit-hook": "^2.0.1", 27 | "cheerio": "^1.0.0-rc.12", 28 | "jsdom": "22.0.0", 29 | "sharp": "^0.32.6", 30 | "selenium-webdriver": "4.9.0", 31 | "turndown": "^7.1.2", 32 | "webdriver-manager": "^13.0.2", 33 | "gpt-3-encoder": "v1.1.4" 34 | }, 35 | "peerDependencies": { 36 | "agent-mimir": "*" 37 | } 38 | } -------------------------------------------------------------------------------- /tools/selenium-browser/src/prompt/combiner-prompt.ts: -------------------------------------------------------------------------------- 1 | import { PromptTemplate } from "@langchain/core/prompts"; 2 | 3 | //Combine the following two parts of a markdown document into one. 4 | const _DEFAULT_SUMMARIZER_TEMPLATE =`Combine the contents of "part 2" of a markdown document with "part 1" of the document. 5 | The document is a representation of the contents of a website. 6 | The title of the website is "{title}". 7 | 8 | The result should be in Markdown format. 9 | Prioritize keeping details related to what I intend to find in the site: "{focus}". 10 | Remove details that is not related to what I intend to find. 11 | 12 | Include any relevant html inputs, buttons, and link elements (always including all attributes and text). Do not rewrite or simplify the html elements! 13 | 14 | Summarize any large pieces of text while keeping key relevant pieces of information. 15 | 16 | If the combined document is too long, remove less relevant content from any of the parts of the document. 17 | 18 | If the contents of part 2 of the document is irrelevant to what I intend to find and you will not combine any content from it, respond only with the word "DISCARD" and nothing else! 19 | ------------------------- 20 | This is part 1 of the document: 21 | 22 | {document1} 23 | 24 | ------------------------- 25 | This is part 2 of the document: 26 | 27 | {document2} 28 | 29 | ------------------------- 30 | Combined Markdown document: 31 | `; 32 | export const COMBINE_PROMPT = /* #__PURE__ */ new PromptTemplate({ 33 | template: _DEFAULT_SUMMARIZER_TEMPLATE, 34 | inputVariables: ["title", "document1", "document2", "focus"], 35 | }); 36 | -------------------------------------------------------------------------------- /tools/selenium-browser/src/prompt/relevance-prompt.ts: -------------------------------------------------------------------------------- 1 | import { PromptTemplate } from "@langchain/core/prompts"; 2 | 3 | 4 | 5 | 6 | const IS_RELEVANT_PROMPT_TEMPLATE = `Respond me with a either the word "true" or "false" depending if the following piece of a website is relevant to: "{focus}". 7 | "true" if the piece of the website is relevant or is significantly related to the piece of the website, "false" if it is not. 8 | 9 | The title of the website is "{title}". 10 | 11 | Piece of website: 12 | ------------ 13 | {document} 14 | ------------ 15 | 16 | Was that piece of the website relevant?: 17 | `; 18 | export const IS_RELEVANT_PROMPT = /* #__PURE__ */ new PromptTemplate({ 19 | template: IS_RELEVANT_PROMPT_TEMPLATE, 20 | inputVariables: ["document", "focus", "title"], 21 | }); 22 | 23 | 24 | -------------------------------------------------------------------------------- /tools/selenium-browser/src/to-markdown.ts: -------------------------------------------------------------------------------- 1 | import TurndownService from 'turndown'; 2 | 3 | function buildAttribute(attributeName: string, attributeValue: string | null, defaultValue?: string) { 4 | if (attributeValue) { 5 | return `${attributeName}="${attributeValue}"`; 6 | } else if (defaultValue) { 7 | return `${attributeName}="${defaultValue}"`; 8 | } 9 | return ""; 10 | } 11 | 12 | function isEmptyOrSpaces(str: string | null) { 13 | return str === null || str.match(/^ *$/) !== null; 14 | } 15 | 16 | 17 | function getInputorLinkInfo(document: ParentNode, element: Element) { 18 | 19 | const ariaLabel = element.getAttribute('aria-label'); 20 | if (ariaLabel && ariaLabel !== '') { 21 | return ariaLabel! 22 | } 23 | const ariaLabelledBy = element.getAttribute('aria-labelledby'); 24 | if (ariaLabelledBy) { 25 | const labelledByElement = document.querySelector(`[id="${ariaLabelledBy}"]`); 26 | if (labelledByElement) { 27 | if (!labelledByElement.textContent) { 28 | console.log(`No text content for aria-labelledby ${ariaLabelledBy}`); 29 | } 30 | return labelledByElement.textContent?.trim() ?? null; 31 | } 32 | } 33 | 34 | if (!isEmptyOrSpaces(element.textContent?.trim() ?? null)) { 35 | return element.textContent?.trim() ?? null; 36 | } 37 | return null; 38 | } 39 | 40 | function applyTurndownFixes(htmlDoc: Document) { 41 | //Turndown incorrectly does not mark ` 82 | } 83 | return ""; 84 | } 85 | }).addRule('img', { 86 | filter: ['img'], 87 | replacement: function (content, node, options) { 88 | let element = node as HTMLElement; 89 | var alt = cleanAttribute(element.getAttribute('alt')); 90 | if (alt === '') return ''; 91 | 92 | var src = element.getAttribute('src') || ''; 93 | var title = cleanAttribute(element.getAttribute('title')); 94 | var titlePart = title ? ' "' + title + '"' : ''; 95 | return src ? '![' + alt + ']' + '(' + '...' + src.slice(-15) + titlePart + ')' : ''; 96 | } 97 | }).addRule('removePicture', { 98 | filter: ['picture'], 99 | replacement: function () { 100 | return ""; 101 | } 102 | }).addRule('removeScript', { 103 | filter: ['script'], 104 | replacement: function () { 105 | return ""; 106 | } 107 | }).addRule('formatInput', { 108 | filter: ['input'], 109 | replacement: function (_, node) { 110 | let element = node as HTMLElement; 111 | const description = getInputorLinkInfo(htmlDoc, element); 112 | if (description) { 113 | return `` 114 | } 115 | return ""; 116 | } 117 | }); 118 | 119 | applyTurndownFixes(htmlDoc); 120 | const markdown = turndownService.turndown(htmlDoc.body.outerHTML); 121 | return markdown; 122 | } 123 | 124 | function cleanAttribute (attribute:string | null) { 125 | return attribute ? attribute.replace(/(\n+\s*)+/g, '\n') : '' 126 | } -------------------------------------------------------------------------------- /tools/selenium-browser/src/xpath.ts: -------------------------------------------------------------------------------- 1 | type XpathComponent = { 2 | name?: string, 3 | position?: number | null 4 | } 5 | 6 | export function getXPath(node: Element) { 7 | var comp, comps : XpathComponent[] = []; 8 | var xpath = ''; 9 | var getPos = function (node: Element) { 10 | var position = 1, curNode; 11 | if (node.nodeType == node.ATTRIBUTE_NODE) { 12 | return null; 13 | } 14 | for (curNode = node.previousSibling; curNode; curNode = curNode.previousSibling) { 15 | if (curNode.nodeName == node.nodeName) { 16 | ++position; 17 | } 18 | } 19 | return position; 20 | } 21 | let nodeToFind: Element | null = node; 22 | for (; nodeToFind && !(nodeToFind.nodeType === nodeToFind.DOCUMENT_NODE); nodeToFind = nodeToFind.nodeType == nodeToFind.ATTRIBUTE_NODE ? (nodeToFind as any).ownerElement : nodeToFind.parentNode) { 23 | 24 | let comp: XpathComponent | null = comps[comps.length] = {}; 25 | switch (nodeToFind.nodeType) { 26 | case nodeToFind.TEXT_NODE: 27 | comp.name = 'text()'; 28 | break; 29 | case nodeToFind.ATTRIBUTE_NODE: 30 | comp.name = '@' + nodeToFind.nodeName; 31 | break; 32 | case nodeToFind.PROCESSING_INSTRUCTION_NODE: 33 | comp.name = 'processing-instruction()'; 34 | break; 35 | case nodeToFind.COMMENT_NODE: 36 | comp.name = 'comment()'; 37 | break; 38 | case nodeToFind.ELEMENT_NODE: 39 | comp.name = nodeToFind.nodeName; 40 | if (nodeToFind.hasAttribute('id')) { 41 | comp.name = '/*[@id="' + nodeToFind.getAttribute('id') + '"]'; 42 | nodeToFind = null; 43 | } 44 | break; 45 | } 46 | if (!nodeToFind) { 47 | break; 48 | } else { 49 | comp.position = getPos(nodeToFind); 50 | } 51 | } 52 | 53 | for (var i = comps.length - 1; i >= 0; i--) { 54 | comp = comps[i]; 55 | xpath += '/' + comp.name; 56 | if (comp.position) { 57 | xpath += '[' + comp.position + ']'; 58 | } 59 | } 60 | 61 | return xpath; 62 | 63 | } 64 | -------------------------------------------------------------------------------- /tools/selenium-browser/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021", 4 | "allowJs": true, 5 | "allowSyntheticDefaultImports": true, 6 | "baseUrl": "src", 7 | "declaration": true, 8 | "sourceMap": true, 9 | "inlineSourceMap": false, 10 | "moduleResolution": "nodenext", 11 | "rootDir": "src", 12 | "module": "NodeNext", /* Specify what module code is generated. */ 13 | "outDir": "dist", 14 | "lib": [ 15 | "ES2021", 16 | "ES2022.Object", 17 | "DOM" 18 | ], 19 | "types": [ 20 | "node", 21 | ], 22 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ 23 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ 24 | /* Type Checking */ 25 | "strict": true, /* Enable all strict type-checking options. */ 26 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 27 | }, 28 | "ts-node": { 29 | "esm": true 30 | } 31 | } -------------------------------------------------------------------------------- /tools/serper-search/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@agent-mimir/serper-search", 3 | "description": "Serper search tool for langchain", 4 | "main": "dist/index.js", 5 | "scripts": { 6 | "build": "tsc -p tsconfig.json " 7 | }, 8 | "author": "Altaflux", 9 | "keywords": [ 10 | "langchain" 11 | ], 12 | "devDependencies": { 13 | "@types/node": "^18.16.3", 14 | "rimraf": "^5.0.0", 15 | "typescript": "^5.3.3" 16 | }, 17 | "dependencies": { 18 | }, 19 | "peerDependencies": { 20 | "@langchain/core": "*" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tools/serper-search/src/index.ts: -------------------------------------------------------------------------------- 1 | 2 | import { StructuredTool } from "@langchain/core/tools"; 3 | 4 | import { z } from "zod"; 5 | 6 | export type SerperParameters = { 7 | gl?: string; 8 | hl?: string; 9 | }; 10 | 11 | export type SerperResponseExtractor = (response: any) => Promise; 12 | 13 | /** 14 | * Wrapper around serper. 15 | * 16 | * You can create a free API key at https://serper.dev. 17 | * 18 | * To use, you should have the SERPER_API_KEY environment variable set. 19 | */ 20 | export class Serper extends StructuredTool { 21 | schema = z.object({ 22 | searchQuery: z.string().describe("The search query to run."), 23 | }); 24 | 25 | protected key: string; 26 | 27 | protected params: Partial; 28 | 29 | protected responseExtractor: SerperResponseExtractor; 30 | 31 | constructor( 32 | apiKey: string | undefined = process.env.SERPER_API_KEY, 33 | params: Partial = {}, 34 | responseExtractor: SerperResponseExtractor = defaultResponseExtractor 35 | ) { 36 | super(); 37 | 38 | if (!apiKey) { 39 | throw new Error( 40 | "Serper API key not set. You can set it as SERPER_API_KEY in your .env file, or pass it to Serper." 41 | ); 42 | } 43 | 44 | this.key = apiKey; 45 | this.params = params; 46 | this.responseExtractor = responseExtractor; 47 | } 48 | 49 | name = "search"; 50 | 51 | /** @ignore */ 52 | async _call(arg: z.input) { 53 | const options = { 54 | method: "POST", 55 | headers: { 56 | "X-API-KEY": this.key, 57 | "Content-Type": "application/json", 58 | }, 59 | body: JSON.stringify({ 60 | q: arg.searchQuery, 61 | ...this.params, 62 | }), 63 | }; 64 | 65 | const res = await fetch("https://google.serper.dev/search", options); 66 | 67 | if (!res.ok) { 68 | throw new Error(`Got ${res.status} error from serper: ${res.statusText}`); 69 | } 70 | 71 | const json = await res.json(); 72 | return await this.responseExtractor(json); 73 | } 74 | 75 | description = 76 | "a search engine. useful for when you need to answer questions about current events."; 77 | } 78 | 79 | async function defaultResponseExtractor(json: any) { 80 | const links = json.organic.slice(0, 3).map((link: any) => { 81 | return `--- Site Title: ${link.title}` + "\nUrl: " + link.link + "\nSnippet: " + link.snippet + "\n"; 82 | }).join("\n"); 83 | 84 | 85 | return `Result: ${json.answerBox?.answer ?? ""}\n${json.answerBox?.snippet ?? ""}\nLinks:\n` + links; 86 | } -------------------------------------------------------------------------------- /tools/serper-search/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021", 4 | "allowJs": true, 5 | "allowSyntheticDefaultImports": true, 6 | "baseUrl": "src", 7 | "declaration": true, 8 | "sourceMap": true, 9 | "inlineSourceMap": false, 10 | "moduleResolution": "node", 11 | "rootDir": "src", 12 | "module": "commonjs", /* Specify what module code is generated. */ 13 | "outDir": "dist", 14 | "lib": [ 15 | "ES2021", 16 | "ES2022.Object", 17 | "DOM" 18 | ], 19 | "types": [ 20 | "node", 21 | ], 22 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ 23 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ 24 | /* Type Checking */ 25 | "strict": true, /* Enable all strict type-checking options. */ 26 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 27 | } 28 | } -------------------------------------------------------------------------------- /tools/sqlite-tool/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@agent-mimir/sqlite-tool", 3 | "description": "SQLite tool", 4 | "main": "dist/index.js", 5 | "type": "module", 6 | "scripts": { 7 | "build": "tsc -p tsconfig.json " 8 | }, 9 | "author": "Altaflux", 10 | "keywords": [ 11 | "langchain" 12 | ], 13 | "devDependencies": { 14 | "@types/node": "^18.16.3", 15 | "@types/better-sqlite3": "^7.6.9", 16 | "rimraf": "^5.0.0", 17 | "typescript": "^5.3.3", 18 | "agent-mimir": "*" 19 | }, 20 | "dependencies": { 21 | "better-sqlite3": "^9.5.0" 22 | }, 23 | "peerDependencies": { 24 | "agent-mimir": "*" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /tools/sqlite-tool/src/index.ts: -------------------------------------------------------------------------------- 1 | import { AgentPlugin, PluginFactory, PluginContext, AgentSystemMessage } from "agent-mimir/plugins"; 2 | import { AgentTool, ToolResponse } from "agent-mimir/tools"; 3 | 4 | import { z } from "zod"; 5 | import Sqlite from 'better-sqlite3'; 6 | 7 | type SqliteConfig = { 8 | dbPath: string; 9 | description: string; 10 | } 11 | export class SqliteToolPluginFactory implements PluginFactory { 12 | name: string = "sqlite_tool_plugin"; 13 | constructor(private args: SqliteConfig) { 14 | } 15 | 16 | 17 | create(context: PluginContext): Promise { 18 | return Promise.resolve(new SqliteToolPlugin(this.args)); 19 | } 20 | } 21 | 22 | 23 | class SqliteToolPlugin extends AgentPlugin { 24 | private db: Sqlite.Database; 25 | 26 | name?: string | undefined = "SQLITE Database Access"; 27 | constructor(private args: SqliteConfig) { 28 | super(); 29 | this.db = new Sqlite(this.args.dbPath); 30 | } 31 | 32 | 33 | async getSystemMessages(): Promise { 34 | return { 35 | content: [ 36 | { 37 | type: "text", 38 | text: `You have access to a SQLite database which can be queried using sqlite tools. The description of the database is ${this.args.description}` 39 | } 40 | ] 41 | } 42 | } 43 | 44 | async tools(): Promise<(AgentTool)[]> { 45 | return Promise.resolve([ 46 | new DescribeTableCommand(this.db), 47 | new ListTableCommand(this.db), 48 | new ExecuteQueryCommand(this.db) 49 | ]); 50 | } 51 | } 52 | class DescribeTableCommand extends AgentTool { 53 | 54 | schema = z.object({ 55 | tableName: z.string().describe("The name of the table to describe."), 56 | }); 57 | 58 | constructor(private db: Sqlite.Database) { 59 | super(); 60 | } 61 | protected async _call(inputs: z.input): Promise { 62 | let tableName = inputs.tableName; 63 | if (!(tableName.startsWith("[") && tableName.endsWith("]"))) { 64 | tableName = `[${tableName}]`; 65 | } 66 | 67 | const sql = `PRAGMA table_info(${tableName});`; 68 | const stmt = this.db.prepare(sql); 69 | const columns = stmt.all(); // No parameters needed here 70 | return [ 71 | { 72 | type: "text", 73 | text: JSON.stringify(columns) 74 | } 75 | ]; 76 | } 77 | 78 | name: string = "sqlite_describe_table"; 79 | description: string = "Get the schema information for a specific table"; 80 | outSchema = z.array(z.object({ 81 | "cid": z.number(), 82 | "name": z.string(), 83 | "type": z.string(), 84 | "notnull": z.number(), 85 | "dflt_value": z.string().nullable(), 86 | "pk": z.number(), 87 | })) 88 | } 89 | 90 | class ListTableCommand extends AgentTool { 91 | 92 | schema = z.object({}); 93 | 94 | constructor(private db: Sqlite.Database) { 95 | super(); 96 | } 97 | protected async _call(inputs: z.input): Promise { 98 | const stmt = this.db.prepare(` 99 | SELECT name 100 | FROM sqlite_master 101 | WHERE type = 'table' AND name NOT LIKE 'sqlite_%' 102 | ORDER BY name; 103 | `); 104 | const tables = stmt.pluck().all(); 105 | 106 | return [ 107 | { 108 | type: "text", 109 | text: JSON.stringify(tables) 110 | } 111 | ]; 112 | } 113 | 114 | name: string = "sqlite_list_tables"; 115 | description: string = "List all tables in the SQLite database"; 116 | 117 | outSchema = z.array(z.string()).describe("The list of tables in the database."); 118 | 119 | } 120 | 121 | 122 | class ExecuteQueryCommand extends AgentTool { 123 | 124 | schema = z.object({ 125 | query: z.string().describe("The SQL query to execute."), 126 | }); 127 | 128 | constructor(private db: Sqlite.Database) { 129 | super(); 130 | } 131 | protected async _call(inputs: z.input): Promise { 132 | const stmt = this.db.prepare(inputs.query); 133 | const results = stmt.all([]); 134 | 135 | return [ 136 | { 137 | type: "text", 138 | text: JSON.stringify(results) 139 | } 140 | ]; 141 | } 142 | 143 | name: string = "sqlite_read_query"; 144 | description: string = "Execute a read query on the SQLite database"; 145 | 146 | outSchema = z.array(z.record(z.any())).describe("The results of the query."); 147 | } -------------------------------------------------------------------------------- /tools/sqlite-tool/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021", 4 | "allowJs": true, 5 | "allowSyntheticDefaultImports": true, 6 | "baseUrl": "src", 7 | "declaration": true, 8 | "sourceMap": true, 9 | "inlineSourceMap": false, 10 | "moduleResolution": "nodenext", 11 | "rootDir": "src", 12 | "module": "NodeNext", /* Specify what module code is generated. */ 13 | "outDir": "dist", 14 | "lib": [ 15 | "ES2021", 16 | "ES2022.Object", 17 | "DOM" 18 | ], 19 | "types": [ 20 | "node", 21 | ], 22 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ 23 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ 24 | /* Type Checking */ 25 | "strict": true, /* Enable all strict type-checking options. */ 26 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 27 | }, 28 | "ts-node": { 29 | "esm": true 30 | } 31 | } -------------------------------------------------------------------------------- /turbo.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://turbo.build/schema.json", 3 | "globalDependencies": ["**/.env.*local"], 4 | "pipeline": { 5 | "build": { 6 | "dependsOn": ["^build"], 7 | "outputs": ["dist/**", ".next/**", "!.next/cache/**"] 8 | }, 9 | "lint": {}, 10 | "dev": { 11 | "cache": false, 12 | "persistent": true 13 | } 14 | } 15 | } 16 | --------------------------------------------------------------------------------