├── .env.example ├── .github └── FUNDING.yml ├── .gitignore ├── .nvmrc ├── LICENSE ├── README.md ├── api.js ├── bot.js ├── cli.js ├── context ├── advanced.js ├── empty.js ├── index.js └── simple.js ├── package-lock.json ├── package.json └── skills └── sample.js /.env.example: -------------------------------------------------------------------------------- 1 | # See https://beta.openai.com/account/api-keys 2 | CODEX_API_KEY=123 -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: ['manekinekko'] 4 | custom: ['https://ecologi.com/wassimchegham?r=5facf70521660a001d024120', 'https://www.buymeacoffee.com/wassimchegham'] 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .env -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 16 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Wassim Chegham 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Minecraft OpenAI 2 | 3 | A proof of concept for controlling a Minecraft Non-Player Characters using OpenAI and GPT-3. 4 | 5 | ![playing-minecraft-with-ai-gpt-3-no-text](https://user-images.githubusercontent.com/1699357/165494902-122c763c-7b88-4158-9685-24f0bdecb6b8.jpg) 6 | 7 | ## Requirements 8 | 9 | - Clone this project on your local machine. 10 | - [Minecraft](https://www.minecraft.net/en-us/get-minecraft) (Java Edition) version 1.17 11 | - Node.js version 14+ 12 | - An [OpenAI](https://openai.com) account 13 | - Create a `.env` file and copy your [OpenAI API key](https://beta.openai.com/account/api-keys) and save it 14 | 15 | ``` 16 | CODEX_API_KEY= 17 | ``` 18 | 19 | ## Slide deck 20 | 21 | https://slides.com/wassimchegham/playing-minecraft-artificial-intelligence-open-ai-gpt-3-javascript 22 | 23 | ## How to use 24 | 25 | ### Start the Minecraft server 26 | 27 | Here is how to start the Minecraft server: 28 | 29 | 1. Choose a host computer. This computer should be fast enough to play Minecraft, while running a server for other players as well. 30 | 2. Launch the game and click **Single Player**. 31 | 3. Create a new world or open an existing one. 32 | 4. Inside that world, press the Esc key, and click **Open to LAN**. 33 | 5. Choose a game mode to set for the other players. 34 | 6. Choose **Creative mode** that allows you to fly and place an infinite number of blocks. 35 | 7. Click **Start LAN World**, and you'll see a message that a local game has been hosted. 36 | 8. Take note of the port number. 37 | 38 | ### Launch the bot 39 | 40 | From your terminal, run the following commands: 41 | 42 | ``` 43 | npm install 44 | npm start -- --port [PORT] 45 | ``` 46 | 47 | In a few seconds, you should see a message that the bot is running, and you should see the NPC pop up in Minecraft. 48 | 49 | ### Sending commands 50 | 51 | Inside the Minecraft client, press the `T` key to open the chat box. 52 | 53 | ### Loading context 54 | 55 | There are mulptiple supported contexts: 56 | 57 | 1. `empty`: An empty context (default). 58 | 2. `simple`: A basic context. 59 | 3. `advanced`: A more complex context. 60 | 61 | To load a context, type `load context [context_name]`. 62 | 63 | You can also reset the current context by typing `reset context`. 64 | 65 | ## Disclaimer 66 | 67 | This is a proof of concept. It is not intended to be used in production. 68 | 69 | ## Troubleshooting 70 | 71 | ### On WSL 72 | 73 | If you are using WSL, you may need to provide the host computer's IP address to the bot. 74 | 75 | ``` 76 | npm start -- --port [PORT] --host [HOST] 77 | ``` 78 | 79 | To get the IP address of your host computer, run the following command: 80 | 81 | ``` 82 | wsl.exe hostname -I 83 | ``` 84 | -------------------------------------------------------------------------------- /api.js: -------------------------------------------------------------------------------- 1 | import fetch from "isomorphic-fetch"; 2 | import dotenv from "dotenv"; 3 | dotenv.config(); 4 | 5 | import debug from "debug"; 6 | const error = debug("minecraft-openai.api:error"); 7 | const log = debug("minecraft-openai.api:log"); 8 | 9 | const STOP_WORD = "//"; 10 | const EOL = "\n"; 11 | 12 | /** 13 | * Call the OpenAI API with the previous context and the new user input. 14 | * 15 | * @param {string} input The user's input from the Minecraft chat. 16 | * @param {string} context The previous context to be sent to the OpenAI API 17 | * @returns {Pormise<{ id: string, object: string, created: number, mode: string, choices: Array<{ text: string, index: number, logprobs: any, finish_reason: text }> }>} 18 | */ 19 | export async function callOpenAI(input, context) { 20 | const openAIkey = process.env.CODEX_API_KEY; 21 | if (!openAIkey) { 22 | error("ERROR: CODEX_API_KEY is required."); 23 | process.exit(1); 24 | } 25 | 26 | const body = { 27 | prompt: `${context}${EOL}${STOP_WORD} ${input}${EOL}`, 28 | max_tokens: 300, 29 | temperature: 0, 30 | stop: STOP_WORD, 31 | n: 1, 32 | }; 33 | 34 | log("payload %o", body); 35 | log("context:\n", body.prompt); 36 | 37 | const response = await fetch("https://api.openai.com/v1/engines/davinci-codex/completions", { 38 | method: "POST", 39 | headers: { 40 | "Content-Type": "application/json", 41 | Authorization: `Bearer ${openAIkey}`, 42 | }, 43 | body: JSON.stringify(body), 44 | }); 45 | 46 | if (!response.ok) { 47 | error("api response failed with statis %s", response.statusText); 48 | return; 49 | } 50 | 51 | return await response.json(); 52 | } 53 | -------------------------------------------------------------------------------- /bot.js: -------------------------------------------------------------------------------- 1 | import debug from "debug"; 2 | import minecraftData from "minecraft-data"; 3 | import mineflayer from "mineflayer"; 4 | import { mineflayer as mineflayerViewer } from "prismarine-viewer"; 5 | import { callOpenAI } from "./api.js"; 6 | import { 7 | context, 8 | updateContext, 9 | loadContext, 10 | clearContext, 11 | } from "./context/index.js"; 12 | import collectBlock from "mineflayer-collectblock"; 13 | 14 | //@ts-ignore 15 | import mineflayerPathfinder from "mineflayer-pathfinder"; 16 | const { pathfinder } = mineflayerPathfinder; 17 | 18 | // load available skills 19 | //@ts-ignore 20 | import { watchPlayer, goToPlayer, mineBlock, giveToPlayer } from "./skills/sample.js"; 21 | 22 | const log = debug("minecraft-openai.bot:log"); 23 | const error = debug("minecraft-openai.bot:error"); 24 | 25 | // define global variable that will be used to craft items and interact with the world 26 | var mcData; 27 | var goToPlayerInterval; 28 | var watchInterval; 29 | var target; 30 | 31 | // a workaround to avoid Code from removing these variables 32 | goToPlayerInterval = null; 33 | watchInterval = null; 34 | mcData = null; 35 | target = null; 36 | 37 | export default async function bot(host, port, username) { 38 | const bot = mineflayer.createBot({ 39 | username: username || "OpenAI", 40 | host: host || "localhost", 41 | port, 42 | verbose: true, 43 | }); 44 | 45 | // on error 46 | bot.on("error", (err) => { 47 | error(err); 48 | }); 49 | 50 | bot.on("login", () => { 51 | log("bot joined the game"); 52 | }); 53 | 54 | bot.on("chat", async (username, input) => { 55 | if (username === bot.username) return; 56 | 57 | if (input.startsWith("load context")) { 58 | const contextName = input.replace("load context", "").trim(); 59 | if (contextName) { 60 | await loadContext(contextName); 61 | bot.chat(`Loaded context ${contextName}`); 62 | return; 63 | } 64 | } else if (input.startsWith("reset context")) { 65 | clearContext(); 66 | bot.chat("Cleared context"); 67 | return; 68 | } 69 | 70 | const previousContext = context(); 71 | log("input: %s", input); 72 | log("context: %s", previousContext); 73 | 74 | // call the OpenAI API 75 | const response = await callOpenAI(input, previousContext); 76 | target = bot.players[username].entity; 77 | 78 | if (response) { 79 | log("request: %s", response.id); 80 | log("codex: %s", response.model); 81 | log("choices: %o", response.choices); 82 | 83 | // extract code instructions from response 84 | const code = await response.choices 85 | .map((choice) => choice.text) 86 | .join("\n"); 87 | log("code: ", code); 88 | 89 | if (code === "") { 90 | bot.chat(`I am sorry, I don't understand.`); 91 | return; 92 | } 93 | 94 | try { 95 | // WARNING: this is a very dangerous way to execute code! Do you trust AI? 96 | // Note: the code is executed in the context of the bot entity 97 | await eval(`(async function inject() { 98 | try { 99 | ${code} 100 | } 101 | catch(err){ 102 | error("error: %s", err.message); 103 | bot.chat(\`error: \${err.message}\`); 104 | } 105 | })()`); 106 | 107 | // update the context for the next time 108 | // Note: we only update context if the code is valid 109 | updateContext(input, code); 110 | } catch (err) { 111 | error("error: %s", err.message); 112 | bot.chat(`error: ${err.message}`); 113 | } 114 | } else { 115 | log("OpenAI response was empty. Ignore."); 116 | } 117 | }); 118 | 119 | // Log errors and kick reasons: 120 | bot.on("kicked", log); 121 | bot.on("error", log); 122 | 123 | bot.once("spawn", () => { 124 | mcData = minecraftData(bot.version); 125 | log("Minecraft version: %s", bot.version); 126 | log("Minecraft protocol version: %s", bot.protocolVersion); 127 | 128 | // load all plugins 129 | bot.loadPlugin(collectBlock.plugin); 130 | bot.loadPlugin(pathfinder); 131 | 132 | // port is the minecraft server port, if first person is false, you get a bird's-eye view 133 | try { 134 | mineflayerViewer(bot, { port: 31337, firstPerson: true }); 135 | } catch (err) { 136 | error("error: %s", err.message); 137 | } 138 | }); 139 | } 140 | -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import debug from "debug"; 4 | import bot from "./bot.js"; 5 | import { program } from "commander"; 6 | 7 | const log = debug("minecraft-openai.cli:log"); 8 | const error = debug("minecraft-openai.cli:error"); 9 | 10 | program 11 | .name("minecraft-openai") 12 | .description("Playing Minecraft with OpenAI (proof of concept)") 13 | 14 | program.command('start') 15 | .description('start the bot') 16 | .option("--host ", "hostname of the minecraft server", "localhost") 17 | .option("--port ", "port of the minecraft server", 25565) 18 | .option("--username ", "username of the bot", "OpenAI") 19 | .action(async(options) => { 20 | log("starting bot"); 21 | await bot(options.host, options.port, "OpenAI").catch(error); 22 | }); 23 | 24 | 25 | program.parse(); 26 | 27 | -------------------------------------------------------------------------------- /context/advanced.js: -------------------------------------------------------------------------------- 1 | export default [ 2 | ` 3 | // Go forward 4 | bot.setControlState('forward', true); 5 | 6 | // Go back 7 | bot.setControlState('back', true); 8 | 9 | // jump 10 | bot.setControlState('jump', true); 11 | 12 | // Hi how are you? 13 | bot.chat("I'm fine, thanks!"); 14 | 15 | // What's your name? 16 | bot.chat("My name is " + bot.username); 17 | 18 | // This is the bot's current position 19 | bot.chat("My position is " + bot.entity.position); 20 | 21 | // Stop any movement 22 | bot.clearControlStates(); 23 | 24 | // Come to me 25 | goToPlayer(bot, 2, target); 26 | 27 | // find me 28 | goToPlayer(bot, 2, target); 29 | 30 | // join me 31 | goToPlayer(bot, 2, target); 32 | 33 | // Follow me 34 | goToPlayerInterval = setInterval(() => goToPlayer(bot, 2, target), 1000); 35 | 36 | // Keep following me 37 | goToPlayerInterval = setInterval(() => goToPlayer(bot, 2, target), 1000); 38 | 39 | // Stop following me 40 | clearInterval(goToPlayerInterval); 41 | 42 | // Look at me 43 | watchPlayer(target, bot); 44 | 45 | // keep looking at me 46 | watchInterval = setInterval(() => watchPlayer(target, bot), 1000); 47 | 48 | // stop looking at me 49 | clearInterval(watchInterval); 50 | 51 | // Mine 10 blocks of dirt 52 | mineBlock(bot, "dirt", mcData, 10); 53 | 54 | // Get 1 block of dirt 55 | mineBlock(bot, "dirt", mcData, 1); 56 | 57 | // Go get me 3 blocks of dirt 58 | mineBlock(bot, "dirt", mcData, 3); 59 | 60 | // Give me 5 dirt 61 | giveToPlayer (bot, "dirt", target, 5); 62 | 63 | // Give 4 dirt 64 | giveToPlayer (bot, "dirt", target, 4); 65 | 66 | // Drop 2 dirt 67 | giveToPlayer (bot, "dirt", target, 2); 68 | 69 | // Drop 1 dirt 70 | giveToPlayer (bot, "dirt", target, 1); 71 | 72 | // Get 4 oak logs 73 | mineBlock(bot, "oak_log", mcData, 4); 74 | 75 | `, 76 | ]; 77 | -------------------------------------------------------------------------------- /context/empty.js: -------------------------------------------------------------------------------- 1 | export default []; 2 | -------------------------------------------------------------------------------- /context/index.js: -------------------------------------------------------------------------------- 1 | import debug from "debug"; 2 | import empty from "./empty.js"; 3 | 4 | 5 | const warn = debug("minecraft-openai.context:warn"); 6 | let currentContext = empty; 7 | 8 | /** 9 | * Update the context with the latest user's input. 10 | * 11 | * @param {string} input The user's input from the Minecraft chat. 12 | * @param {string} code The code example to be inserted in the last context. 13 | */ 14 | export const updateContext = (input, code) => { 15 | currentContext.push(`// ${input}`); 16 | currentContext.push(code); 17 | }; 18 | 19 | /** 20 | * Get the current context as a string. 21 | * 22 | * @returns {string} The new context to be sent to the OpenAI API. 23 | */ 24 | export const context = () => { 25 | if (currentContext.length) { 26 | return currentContext.join("\n"); 27 | } 28 | warn("No context available: %o", currentContext); 29 | return ""; 30 | }; 31 | 32 | /** 33 | * Load the context from a context file. 34 | * 35 | * @param {string[]} context The context to be loaded. 36 | */ 37 | export const loadContext = async (context) => { 38 | try { 39 | currentContext = (await import(`./${context}.js`)).default; 40 | } catch (err) { 41 | warn(err.message); 42 | currentContext = (await import(`./empty.js`)).default; 43 | warn("Loading empty context."); 44 | } 45 | }; 46 | 47 | /** 48 | * Clear the current context. 49 | */ 50 | export const clearContext = () => { 51 | currentContext = empty; 52 | }; 53 | -------------------------------------------------------------------------------- /context/simple.js: -------------------------------------------------------------------------------- 1 | export default [ 2 | ` 3 | // Go forward 4 | bot.setControlState('forward', true); 5 | 6 | // Go back 7 | bot.setControlState('back', true); 8 | 9 | // jump 10 | bot.setControlState('jump', true); 11 | 12 | // Hello ! 13 | bot.chat("Hello friend!"); 14 | 15 | // Hi how are you ? 16 | bot.chat("I'm fine, thanks!"); 17 | 18 | // What's your name ? 19 | bot.chat("My name is " + bot.username); 20 | 21 | // What's your favorite color ? 22 | bot.chat("I like red"); 23 | 24 | // What's your favorite conference? 25 | bot.chat("Devoxx France, of course!"); 26 | ` 27 | ]; 28 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "minecraft-openai", 3 | "version": "1.0.0", 4 | "description": "Controlling a Minecraft bot using OpenAI and GPT-3", 5 | "type": "module", 6 | "scripts": { 7 | "start": "cross-env DEBUG=minecraft-openai* nodemon --inspect cli.js start" 8 | }, 9 | "keywords": [ 10 | "minecraft", 11 | "openai", 12 | "gpt-3", 13 | "bot", 14 | "ai", 15 | "minecraft-bot", 16 | "minecraft-ai", 17 | "minecraft-openai" 18 | ], 19 | "author": "Wassim Chegham ", 20 | "license": "MIT", 21 | "dependencies": { 22 | "commander": "^9.2.0", 23 | "cross-env": "^7.0.3", 24 | "debug": "^4.3.4", 25 | "dotenv": "^16.0.0", 26 | "isomorphic-fetch": "^3.0.0", 27 | "minecraft-data": "^3.0.0", 28 | "mineflayer": "^4.3.0 ", 29 | "mineflayer-pathfinder": "^2.0.0", 30 | "nodemon": "^2.0.15", 31 | "prismarine-viewer": "^1.21.0", 32 | "mineflayer-collectblock": "^1.3.4" 33 | }, 34 | "engines": { 35 | "node": ">=16.0.0" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /skills/sample.js: -------------------------------------------------------------------------------- 1 | import pkg from 'mineflayer-pathfinder'; 2 | const { goals } = pkg; 3 | 4 | export async function watchPlayer(target, bot) { 5 | await bot.lookAt(target.position.offset(0, target.height, 0)); 6 | } 7 | 8 | export async function goToPlayer(bot, range, target) { 9 | await bot.pathfinder.setGoal(new goals.GoalFollow(target, range)); 10 | } 11 | 12 | export async function giveToPlayer (bot, name, target, amount = 1) { 13 | await goToPlayer(bot, 2, target); 14 | bot.once('goal_reached', () => { 15 | const items = bot.inventory.items(); 16 | const item = items.filter(item => item.name === name)[0]; 17 | if (!item) { 18 | bot.chat(`I have no ${ name }`); 19 | return false; 20 | } else if (amount) { 21 | bot.toss(item.type, null, amount); 22 | bot.chat("Here you go!"); 23 | } 24 | }); 25 | } 26 | 27 | export async function mineBlock (bot, type, mcData, count = 1) { 28 | const blockType = mcData.blocksByName[type]; 29 | if (!blockType) { 30 | bot.chat(`Unknown block type: ${type}`); 31 | return; 32 | } 33 | 34 | const blocks = bot.findBlocks({ 35 | matching: blockType.id, 36 | maxDistance: 128, 37 | count: count 38 | }); 39 | 40 | if (blocks.length === 0) { 41 | bot.chat("I don't see that block nearby."); 42 | return; 43 | } 44 | 45 | const targets = []; 46 | for (let i = 0; i < count; i++) { 47 | targets.push(bot.blockAt(blocks[i])); 48 | } 49 | 50 | bot.chat(`I found ${targets.length} ${type} blocks`); 51 | 52 | try { 53 | await bot.collectBlock.collect(targets); 54 | bot.chat('Done'); 55 | 56 | } catch (err) { 57 | bot.chat(err.message); 58 | } 59 | } 60 | --------------------------------------------------------------------------------