├── .env.example ├── .github └── workflows │ └── webpack.yml ├── .gitignore ├── .vscode └── launch.json ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── SECURITY.md ├── SUPPORT.md ├── codex-babylon.gif ├── package-lock.json ├── package.json ├── src ├── client │ ├── App.css │ ├── App.tsx │ ├── components │ │ ├── Form.css │ │ └── Form.tsx │ ├── context │ │ ├── babylonContext.tsx │ │ └── codexContext.tsx │ ├── index.html │ └── index.tsx └── server │ ├── Context.ts │ ├── app.ts │ ├── contentFiltering.ts │ ├── contexts │ └── context1.ts │ └── model.ts ├── tsconfig.json └── webpack.config.js /.env.example: -------------------------------------------------------------------------------- 1 | OPENAI_API_KEY="" 2 | OPENAI_ORGANIZATION_ID="" 3 | OPENAI_ENGINE_ID="" 4 | 5 | SERVER_PORT=1200 6 | CLIENT_PORT=3000 -------------------------------------------------------------------------------- /.github/workflows/webpack.yml: -------------------------------------------------------------------------------- 1 | name: NodeJS with Webpack 2 | 3 | on: 4 | pull_request: 5 | branches: [ main ] 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | 11 | strategy: 12 | matrix: 13 | node-version: [16.x] 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | 18 | - name: Use Node.js ${{ matrix.node-version }} 19 | uses: actions/setup-node@v1 20 | with: 21 | node-version: ${{ matrix.node-version }} 22 | 23 | - name: Build 24 | run: | 25 | npm install 26 | npm run build:client -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .env 3 | dist 4 | .tsbuildinfo -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "Launch Edge", 5 | "request": "launch", 6 | "type": "pwa-msedge", 7 | "url": "http://localhost:3000", 8 | "webRoot": "${workspaceFolder}" 9 | } 10 | 11 | ] 12 | } -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Codex Babylon Prototype 2 | 3 | This project converts natural language into 3D assets using [BabylonJS](https://www.babylonjs.com/) and OpenAI's [Codex](https://openai.com/blog/openai-codex/): 4 | 5 | ![Codex Babylon GIF](codex-babylon.gif) 6 | 7 | The project is made up of a React web application frontend with an [Express](https://expressjs.com/) backend. 8 | 9 | ## Statement of Purpose 10 | This repository aims to grow the understanding of using Codex in applications by providing an example of implementation and references to support the [Microsoft Build conference in 2022](https://mybuild.microsoft.com/). It is not intended to be a released product. Therefore, this repository is not for discussing OpenAI API, BabylonJS or requesting new features. 11 | 12 | ## Requirements 13 | 14 | * Node.JS 15 | * An [OpenAI account](https://openai.com/api/) 16 | * [OpenAI API Key](https://beta.openai.com/account/api-keys). 17 | * [OpenAI Organization Id](https://beta.openai.com/account/org-settings). If you have multiple organizations, please update your [default organization](https://beta.openai.com/account/api-keys) to the one that has access to codex engines before getting the organization Id. 18 | * [OpenAI Engine Id](https://beta.openai.com/docs/engines/codex-series-private-beta). It provides access to a model. For example, `code-davinci-002` or `code-cushman-001`. See [here](#what-openai-engines-are-available-to-me) for checking available engines. 19 | 20 | ## Running the App 21 | 22 | 1. Clone the repo: `git clone https://github.com/microsoft/Codex-Babylon` and open the `Codex-Babylon` folder. 23 | 2. Create a `.env` file in the root directory of the project, copying the contents of the `.env.example` file. 24 | 3. In `.env`, provide the following configuration: 25 | 26 | |Config Name|Description| 27 | |--|--| 28 | |`OPENAI_API_KEY`|The [OpenAI API key](https://beta.openai.com/account/api-keys).| 29 | |`OPENAI_ORGANIZATION_ID`|Your [OpenAI organization id](https://beta.openai.com/account/org-settings).
If you have multiple organizations, please update your [default organization](https://beta.openai.com/account/api-keys) to the one that has access to codex engines before getting the organization id.| 30 | |`OPENAI_ENGINE_ID`|The [OpenAI engine id](https://beta.openai.com/docs/engines/codex-series-private-beta) that provides access to a model. For example, `code-davinci-002` or `code-cushman-001`.
See [here](#what-openai-engines-are-available-to-me) for checking available engines.| 31 | |`SERVER_PORT`|The port to run the server code. Default to `1200`.| 32 | |`CLIENT_PORT`|The port to run the web app. Default to `3000`. | 33 | 34 | 4. Run `npm install` to gather the projects' dependencies. 35 | 5. Run `npm run start` to serve the backend and launch the web application. 36 | 37 | ## Using the App 38 | 39 | The app consists of a basic text box to enter natural language commands, and a 3D scene to display the results. Enter commands into the text box and press enter to see the results. Note that conversation context is maintained between commands, so subsequent commands can refer back to previous ones. 40 | 41 | Example commands: 42 | 43 | > _Create a cube_ 44 | 45 | > _Make it red and make it spin_ 46 | 47 | >_Put a teal sphere above it and another below it_ 48 | 49 | > _Make the bottom sphere change colors when the cursor hovers over it_ 50 | 51 | ## Debugging 52 | To debug the web application, you can [debug with VSCode debugger](https://code.visualstudio.com/Docs/editor/debugging). 53 | 54 | To debug the code generated from codex, the current debugging experience is basic: 55 | - Observe logs in your browser dev tools (F12) to debug issues evaluating generated code 56 | - Observe logs in your console to debug issues between the Express server, Codex, and the client 57 | 58 | ## Understand the Code 59 | The server and client code is under `src/`. 60 | ### Client (src/client) 61 | - `index.tsx` is the entry to bootstrap the React web application. 62 | - `index.html` is the barebones main view of the app. It uses Bootstrap for basic styling. 63 | 64 | ### Server (src/server) 65 | - `app.ts` is the main entry point for the app. It sets up the Express to serve RESTful APIs after being transpile into JavaScript (output: `dist\server\app.js`). 66 | - `model.ts` manages interaction the Codex API. This uses `isomorphic-fetch` to make POST calls of natural language to be converted to code. It also includes helper methods for engineering the prompt that is sent to Codex (see "prompt engineering" below). 67 | 68 | ## Prompt Engineering 69 | 70 | Generative models like Codex are trained on the simple task of guessing the next token in a sequence. A good practice to coax the kind of tokens (code) you want from Codex is to include context and example interactions in a prompt - this practice is called few-shot prompt engineering. These examples are sent to the model with every API call, along with your natural language query. Codex then "guesses" the next tokens in the sequence (the code that satisfies the natural language). 71 | 72 | This project currently contains "contexts" - examples of what we expect from the model in the `src/server/contexts` folder. A context consists of a description to the model of what will be in the prompt along with examples of Natural Language and the code it should produce. See snippet of `context1` from the `contexts` folder: 73 | 74 | 75 | ```js 76 | /* This document contains natural language commands and the BabylonJS code needed to accomplish them */ 77 | 78 | state = {}; 79 | 80 | /* Make a cube */ 81 | state.cube = BABYLON.MeshBuilder.CreateBox("cube", {size: 1}, scene); 82 | 83 | /* Move the cube up */ 84 | state.cube.position.y += 1; 85 | 86 | /* Move it to the left */ 87 | state.cube.position.x -= 1; 88 | ``` 89 | 90 | As you can see, the first line gives a description of the prompt (explaining to Codex that it should take natural language commands and produce BabylonJS code. It then shows a single line of contextual code, establishing the existence of a `state` object to be used by Codex. Finally, it gives several examples of natural language and code to give Codex a sense of the kind of code it should write. These examples use the `state` object mentioned above to save new Babylon objects onto. It also establishes a kind of conversational interaction with the model, where a natural language command might refer to something created on a past turn ("Move it to..."). These examples help nudge the model to produce this kind of code on future turns. 91 | 92 | The project also includes a `Context` class (see `Context.ts`) that offers several helpers for loading contexts and creating prompts. As a user interacts with the experience, we update the context to include past commands and responses. On subsequent conversation turns, this gives the model the relevant context to do things like pronoun resolution (e.g. of "it" in "make it red"). 93 | 94 | Currently a single ongoing context is maintained on the server. This can be reset with the "Reset" button in the app. The single context means that the app is currently not multi-tenanted, and that multiple browser instances will reuse the same context. Note that prompts to Codex models can only be so long - as the prompt exceeds a certain token limit, the `Context` class will shorten the prompt from the beginning. 95 | 96 | ## Contributing 97 | 98 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 99 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 100 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 101 | 102 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 103 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 104 | provided by the bot. You will only need to do this once across all repos using our CLA. 105 | 106 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 107 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 108 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 109 | 110 | ## Trademarks 111 | 112 | This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 113 | trademarks or logos is subject to and must follow 114 | [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). 115 | Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. 116 | Any use of third-party trademarks or logos are subject to those third-party's policies. 117 | 118 | ## FAQ 119 | ### What OpenAI engines are available to me? 120 | You might have access to different [OpenAI engines](https://beta.openai.com/docs/api-reference/engines) per OpenAI organization. To check what engines are available to you, one can query the [List engines API](https://beta.openai.com/docs/api-reference/engines/list) for available engines. See the following commands: 121 | 122 | * Shell 123 | ``` 124 | curl https://api.openai.com/v1/engines \ 125 | -H 'Authorization: Bearer YOUR_API_KEY' \ 126 | -H 'OpenAI-Organization: YOUR_ORG_ID' 127 | ``` 128 | 129 | * Windows Command Prompt (cmd) 130 | ``` 131 | curl --ssl-no-revoke https://api.openai.com/v1/engines --header OpenAI-Organization:YOUR_ORG_ID --oauth2-bearer YOUR_API_KEY 132 | ``` 133 | 134 | ### Can I run the sample on Azure? 135 | The sample code can be currently be used with Codex on OpenAI’s API. In the coming months, the sample will be updated so you can use it also with the [Azure OpenAI Service](https://aka.ms/azure-openai). 136 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). 40 | 41 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # TODO: The maintainer of this repo has not yet edited this file 2 | 3 | **REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? 4 | 5 | - **No CSS support:** Fill out this template with information about how to file issues and get help. 6 | - **Yes CSS support:** Fill out an intake form at [aka.ms/spot](https://aka.ms/spot). CSS will work with/help you to determine next steps. More details also available at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). 7 | - **Not sure?** Fill out a SPOT intake as though the answer were "Yes". CSS will help you decide. 8 | 9 | *Then remove this first heading from this SUPPORT.MD file before publishing your repo.* 10 | 11 | # Support 12 | 13 | ## How to file issues and get help 14 | 15 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 16 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 17 | feature request as a new Issue. 18 | 19 | For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE 20 | FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER 21 | CHANNEL. WHERE WILL YOU HELP PEOPLE?**. 22 | 23 | ## Microsoft Support Policy 24 | 25 | Support for this **PROJECT or PRODUCT** is limited to the resources listed above. 26 | -------------------------------------------------------------------------------- /codex-babylon.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/Codex-Babylon/f59d745f47d48cf09063e07a9277639b9151d437/codex-babylon.gif -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "codex-babylon", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "app.js", 6 | "scripts": { 7 | "start": "concurrently \"npm:watch:*\" -k", 8 | "serve:server": "npm run build:server && node ./dist/server/app.js", 9 | "watch:server": "nodemon --watch src/server -e ts --exec \"npm run serve:server\"", 10 | "watch:client": "webpack-dev-server --color --config ./webpack.config.js", 11 | "build:server": "tsc src/server/app.ts -inlineSourceMap -outDir dist/server --esModuleInterop --incremental --tsBuildInfoFile .tsbuildinfo", 12 | "build:client": "webpack --env prod --env clean", 13 | "test": "echo \"Error: no test specified\" && exit 1" 14 | }, 15 | "author": "", 16 | "license": "ISC", 17 | "dependencies": { 18 | "@fluentui/react": "^8.56.2", 19 | "body-parser": "^1.19.1", 20 | "cors": "^2.8.5", 21 | "dotenv": "^11.0.0", 22 | "express": "^4.17.2", 23 | "isomorphic-fetch": "^3.0.0", 24 | "react": "^17.0.2", 25 | "react-dom": "^17.0.2" 26 | }, 27 | "devDependencies": { 28 | "@babel/core": "^7.16.12", 29 | "@babel/preset-env": "^7.16.11", 30 | "@babel/preset-react": "^7.16.7", 31 | "@types/isomorphic-fetch": "0.0.36", 32 | "babel-loader": "^8.2.3", 33 | "clean-webpack-plugin": "^4.0.0", 34 | "concurrently": "^7.0.0", 35 | "css-loader": "^6.5.1", 36 | "html-webpack-plugin": "^5.5.0", 37 | "nodemon": "^2.0.15", 38 | "style-loader": "^3.3.1", 39 | "ts-loader": "^9.2.6", 40 | "typescript": "^4.5.5", 41 | "webpack": "^5.67.0", 42 | "webpack-cli": "^4.9.2", 43 | "webpack-dev-server": "^4.7.3" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/client/App.css: -------------------------------------------------------------------------------- 1 | html, 2 | body { 3 | overflow: hidden; 4 | width: 100%; 5 | height: 100%; 6 | margin: 0; 7 | padding: 0; 8 | } 9 | 10 | #root { 11 | width: 100%; 12 | height: 100%; 13 | } 14 | -------------------------------------------------------------------------------- /src/client/App.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import Form from "./components/Form"; 3 | import "./App.css"; 4 | import { BabylonProvider } from "./context/babylonContext"; 5 | import { CodexStateProvider } from "./context/codexContext"; 6 | 7 | export default function App() { 8 | return ( 9 | 10 | 11 |
12 | 13 | 14 | ); 15 | } 16 | -------------------------------------------------------------------------------- /src/client/components/Form.css: -------------------------------------------------------------------------------- 1 | .submitButton { 2 | margin-left: 6px; 3 | } 4 | 5 | .resetButton { 6 | margin-left: 6px; 7 | } 8 | 9 | .commandDiv { 10 | display: flex; 11 | flex-direction: row; 12 | align-items: center; 13 | position: absolute; 14 | top: 0px; 15 | margin: 2%; 16 | width: 60%; 17 | } 18 | 19 | .codeDiv { 20 | position: absolute; 21 | top: 60px; 22 | margin-left: 20px; 23 | color: white 24 | } 25 | 26 | .nlInput { 27 | font-size: 23px; 28 | } -------------------------------------------------------------------------------- /src/client/components/Form.tsx: -------------------------------------------------------------------------------- 1 | import { DefaultButton, IButtonStyles, ITextField, KeyCodes, PrimaryButton, Stack, TextField } from "@fluentui/react"; 2 | import React, { useRef, useCallback, useState, useEffect } from "react"; 3 | import { 4 | useBabylonContext, 5 | useBabylonResetSceneDispatch 6 | } from "../context/babylonContext"; 7 | import { 8 | useCodexResetStateDispatch, 9 | useCodexStateContext 10 | } from "../context/codexContext"; 11 | import "./Form.css"; 12 | 13 | const controlHeight: number = 50; // px 14 | const buttonStyle: IButtonStyles = { root: { minWidth: 150, height: controlHeight, fontSize: 23 } }; 15 | 16 | export default function Form() { 17 | const serverUrl = `http://localhost:${process.env.SERVER_PORT}`; 18 | const codeDivRef = useRef(null); 19 | const inputRef = useRef(null); 20 | 21 | const { state } = useCodexStateContext(); 22 | const { scene } = useBabylonContext(); 23 | const resetBabylonScene = useBabylonResetSceneDispatch(); 24 | const resetCodexState = useCodexResetStateDispatch(); 25 | 26 | const [currentCommand, setCurrentCommand] = useState(); 27 | const [isSendingCommand, setIsSendingCommand] = useState(false); 28 | 29 | useEffect(() => { 30 | if(!isSendingCommand) { 31 | inputRef.current?.focus(); 32 | } 33 | },[isSendingCommand]); 34 | 35 | const evalAsync = async function (code) { 36 | await eval("(async () => { " + code + "})()"); 37 | }; 38 | 39 | const handleSubmit = useCallback(() => { 40 | 41 | setIsSendingCommand(true); 42 | 43 | const nlCommand = currentCommand; 44 | console.log("Sending natural language command: " + nlCommand); 45 | 46 | fetch(`${serverUrl}/codegen`, { 47 | method: "POST", 48 | headers: { 49 | "Content-Type": "application/json; charset=utf-8" 50 | }, 51 | body: JSON.stringify({ 52 | text: nlCommand 53 | }) 54 | }) 55 | .then((response) => response.json()) 56 | .then((data) => { 57 | console.log(`Received the following code: ${data.code}`); 58 | console.log(`Received the following sensitiveContentFlag: ${data.sensitiveContentFlag}`); 59 | 60 | if(data.sensitiveContentFlag > 0) { 61 | var warning = data.sensitiveContentFlag === 1 62 | ? "Your message or the model's response may have contained sensitive content." 63 | : "Your message or the model's response may have contained unsafe content."; 64 | 65 | setCurrentCommand(""); 66 | console.warn(warning); 67 | 68 | if (codeDivRef.current != null) { 69 | codeDivRef.current.innerText = "Potentially sensitive language detected in prompt or completion. Try another prompt using different language."; 70 | } 71 | }else if (data.error && codeDivRef.current != null) { 72 | codeDivRef.current.innerText = data.error; 73 | }else { 74 | if (codeDivRef.current != null && currentCommand !== undefined) { 75 | codeDivRef.current.innerText = data.code; 76 | 77 | setCurrentCommand(""); 78 | evalAsync(data.code); 79 | } 80 | } 81 | 82 | setIsSendingCommand(false); 83 | }) 84 | .catch((error) => { 85 | console.error(error); 86 | setIsSendingCommand(false); 87 | }); 88 | }, [currentCommand, serverUrl]); 89 | 90 | const handleReset = useCallback(() => { 91 | if (codeDivRef.current != null) { 92 | codeDivRef.current.innerText = ""; 93 | } 94 | console.log("resetting context"); 95 | fetch(`${serverUrl}/reset`) 96 | .then((response) => response.json()) 97 | .then((res) => { 98 | console.log(`Reset context: ${res.context}`); 99 | }) 100 | .catch((error) => console.error(error)); 101 | 102 | // reset 103 | for (const key in state.intervals) { 104 | if (state.intervals.hasOwnProperty(key)) { 105 | const interval = state.intervals[key]; 106 | console.log(`Clearing interval ${key}`); 107 | clearInterval(interval); 108 | } 109 | } 110 | 111 | resetBabylonScene(); 112 | resetCodexState(); 113 | }, [resetBabylonScene, resetCodexState, serverUrl]); 114 | 115 | return ( 116 | <> 117 | 118 | setCurrentCommand(newValue)} 122 | value={currentCommand} 123 | autoComplete="off" 124 | styles={{ root: { minWidth: "75%" }, fieldGroup: { height: controlHeight } }} 125 | inputClassName="nlInput" 126 | onKeyUp={(k)=> k.code === "Enter" ? handleSubmit() : ()=>{}} 127 | placeholder="Enter Natural Language Command (e.g. 'create a cube')" 128 | /> 129 | 134 | Enter 135 | 136 | 141 | Reset 142 | 143 |
144 |

145 |
146 | 147 | 148 | ); 149 | } 150 | -------------------------------------------------------------------------------- /src/client/context/babylonContext.tsx: -------------------------------------------------------------------------------- 1 | import React, { 2 | createContext, 3 | ReactNode, 4 | useCallback, 5 | useContext, 6 | useEffect, 7 | useState, 8 | useRef 9 | } from "react"; 10 | 11 | // temp workaround to deal with the global variable of babylon CDN 12 | interface customWindow extends Window { 13 | BABYLON?: any; 14 | } 15 | 16 | declare const window: customWindow; 17 | 18 | type BabylonProviderProps = { 19 | children?: ReactNode; 20 | }; 21 | 22 | export type BabylonResetSceneDispatch = () => void; 23 | 24 | // TODO: Add type definition. 25 | type BabylonContext = { 26 | engine: any; 27 | scene: any; 28 | }; 29 | 30 | const BabylonStateContext = createContext({ 31 | engine: null, 32 | scene: null 33 | }); 34 | 35 | const BabylonResetSceneDispatchContext = createContext< 36 | BabylonResetSceneDispatch | undefined 37 | >(undefined); 38 | 39 | function BabylonProvider({ children }: BabylonProviderProps) { 40 | const canvas: HTMLCanvasElement | null = document.getElementById("renderCanvas") as HTMLCanvasElement; // Get the canvas element 41 | const _engine = new window.BABYLON.Engine(canvas, true); // Generate the BABYLON 3D engine 42 | const _scene = createScene(_engine, canvas); 43 | 44 | const [engine] = useState(_engine); 45 | const [scene, setScene] = useState(_scene); 46 | 47 | const resetSceneCallback = useCallback(() => { 48 | scene?.dispose(); 49 | setScene(createScene(engine, canvas)); 50 | }, [scene, engine, canvas, setScene]); 51 | 52 | useEffect(() => { 53 | engine.runRenderLoop(function () { 54 | scene?.render(); 55 | }); 56 | 57 | function handleWindowResize() { 58 | engine.resize(); 59 | } 60 | 61 | // Watch for browser/canvas resize events 62 | window.addEventListener("resize", handleWindowResize); 63 | return () => { 64 | window.removeEventListener("resize", handleWindowResize); 65 | }; 66 | }, [engine, scene]); 67 | 68 | return ( 69 | 75 | 78 | {children} 79 | 80 | 81 | ); 82 | } 83 | 84 | function createScene(engine: any, canvas: HTMLElement | null) { 85 | if (engine == null || canvas == null) { 86 | return null; 87 | } 88 | 89 | const scene = new window.BABYLON.Scene(engine); 90 | scene.clearColor = new window.BABYLON.Color4.FromHexString("#201c24"); 91 | const camera = new window.BABYLON.ArcRotateCamera( 92 | "camera", 93 | -Math.PI / 2, 94 | Math.PI / 2.5, 95 | 15, 96 | new window.BABYLON.Vector3(0, 0, 0) 97 | ); 98 | camera.attachControl(canvas, true); 99 | camera.wheelPrecision = 5; 100 | 101 | const light = new window.BABYLON.HemisphericLight( 102 | "light", 103 | new window.BABYLON.Vector3(1, 1, 0) 104 | ); 105 | 106 | return scene; 107 | } 108 | 109 | function useBabylonContext() { 110 | const context = useContext(BabylonStateContext); 111 | 112 | if (context === undefined) { 113 | throw new Error( 114 | "useBabylonState must be used within a BabylonProvider" 115 | ); 116 | } 117 | 118 | return context; 119 | } 120 | 121 | function useBabylonResetSceneDispatch() { 122 | const context = useContext(BabylonResetSceneDispatchContext); 123 | 124 | if (context === undefined) { 125 | throw new Error( 126 | "useBabylonResetSceneDispatch must be used within a BabylonProvider" 127 | ); 128 | } 129 | 130 | return context; 131 | } 132 | 133 | export { 134 | BabylonProvider, 135 | useBabylonContext, 136 | useBabylonResetSceneDispatch, 137 | BabylonStateContext 138 | }; 139 | -------------------------------------------------------------------------------- /src/client/context/codexContext.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { useCallback } from "react"; 3 | 4 | type CodexResetStateDispatch = () => void; 5 | type CodexStateProviderProps = { children?: React.ReactNode }; 6 | 7 | const CodexStateContext = React.createContext({}); 8 | const CodexResetStateDispatchContext = React.createContext< 9 | CodexResetStateDispatch | undefined 10 | >(undefined); 11 | 12 | /** 13 | * Codex state context provider. 14 | * Currently provide: 15 | * 1) Codex state, 16 | * 2) dispatch function to reset state. 17 | */ 18 | function CodexStateProvider({ children }: CodexStateProviderProps) { 19 | const [state, setState] = React.useState({intervals: {}}); 20 | 21 | const resetStateCallback = useCallback(() => setState({intervals: {}}), [setState]); 22 | 23 | return ( 24 | 25 | 26 | {children} 27 | 28 | 29 | ); 30 | } 31 | 32 | function useCodexStateContext() { 33 | const context = React.useContext(CodexStateContext); 34 | 35 | if (context === undefined) { 36 | throw new Error( 37 | "useCodexStateContext must be used within a CodexStateProvider" 38 | ); 39 | } 40 | 41 | return context; 42 | } 43 | 44 | function useCodexResetStateDispatch() { 45 | const context = React.useContext(CodexResetStateDispatchContext); 46 | 47 | if (context === undefined) { 48 | throw new Error( 49 | "useCodexResetStateDispatch must be used within a CodexStateProvider" 50 | ); 51 | } 52 | 53 | return context; 54 | } 55 | 56 | export { 57 | CodexStateProvider, 58 | useCodexStateContext, 59 | useCodexResetStateDispatch, 60 | CodexStateContext 61 | }; 62 | -------------------------------------------------------------------------------- /src/client/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Codex Babylon 7 | 8 | 9 | 10 | 11 | 12 | 17 |
18 | 19 | 20 | -------------------------------------------------------------------------------- /src/client/index.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import * as ReactDOM from "react-dom"; 3 | import App from "./App"; 4 | 5 | ReactDOM.render(, document.getElementById("root")); 6 | -------------------------------------------------------------------------------- /src/server/Context.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Class for creating and managing contexts and prompts for Codex. We define the context as a set of examples and previous interactions, whereas 3 | * a prompt is a context plus command, which we pass to the model to get a code response. 4 | * 5 | * We keep track of both the text context and an array of interactions in order to facilitate "undoing" of interactions, and removal of the oldest 6 | * interactions (see `trimContext`) when a context gets too large for the model. 7 | */ 8 | 9 | type Interaction = { 10 | command: string; 11 | code: string; 12 | }; 13 | 14 | export default class Context { 15 | private baseContext: string; 16 | private context: string; 17 | private baseInteractions: Interaction[]; 18 | private interactions: Interaction[]; 19 | 20 | constructor(baseContext: string) { 21 | this.baseContext = baseContext; 22 | this.context = baseContext; 23 | 24 | this.baseInteractions = this.createInteractionsFromContext(baseContext); 25 | this.interactions = this.baseInteractions; 26 | } 27 | 28 | // Returns the current cached prompt 29 | getContext() { 30 | return this.context; 31 | } 32 | 33 | // Adds command (modelled as a comment) to a context, creating our prompt to the model 34 | getPrompt(command: string) { 35 | return `${this.context}\n\n/* ${command} */\n`; 36 | } 37 | 38 | // Adds a new interaction (command and code) to the prompt, "remembering" it for future turns 39 | addInteraction(command: string, code: string) { 40 | let context = `${this.context}\n/* ${command} */\n${code}`; 41 | this.context = context.split("\n").slice(0, -1).join("\n"); 42 | this.interactions.push({ 43 | command, 44 | code 45 | }); 46 | } 47 | 48 | // Removes the last interaction, "forgetting" it 49 | undoInteraction() { 50 | if (this.interactions.length > 0) { 51 | this.interactions.pop(); 52 | this.createContextFromInteractions(); 53 | } 54 | } 55 | 56 | // Resets prompt to the original context 57 | resetContext() { 58 | this.context = this.baseContext; 59 | this.interactions = this.baseInteractions; 60 | } 61 | 62 | // Trims the prompt to remain under a certain length, removing interactions from the top. This is necessary to prevent the prompt from getting too long for a given model. 63 | // The side effect is that the prompt will be truncated, "forgetting" past interactions or context. 64 | trimContext(length: number) { 65 | console.log("Trimming Context"); 66 | while (this.context.length > length) { 67 | console.log( 68 | `Trimming oldest interaction off context: ${this.interactions[0].command}` 69 | ); 70 | this.interactions = this.interactions.slice(1); 71 | this.createContextFromInteractions(); 72 | console.log(`New prompt length: ${this.context.length}`); 73 | } 74 | this.createContextFromInteractions(); 75 | } 76 | 77 | // Creates prompt using interactions (commands and code), modelling the commands as comments 78 | createContextFromInteractions() { 79 | this.context = this.interactions.reduce((prev, next) => { 80 | return `${prev}\n/* ${next.command} */\n${next.code}`; 81 | }, ""); 82 | } 83 | 84 | // Turns a text prompt into an array of interactions (commands and code) - this is effectively the inverse of the createContextFromInteractions method 85 | createInteractionsFromContext(baseContext) { 86 | let interactions: Interaction[] = []; 87 | let lines = baseContext.split("\n"); 88 | for (let i = 0; i < lines.length; i++) { 89 | let line = lines[i]; 90 | if (line.startsWith("/*")) { 91 | // get command, trimming whitespace 92 | let command = line.substring(2, line.indexOf("*/")).trim(); 93 | let code = ""; 94 | for (let j = i + 1; j < lines.length; j++) { 95 | if (lines[j].startsWith("/*")) { 96 | break; 97 | } else { 98 | code += lines[j] + "\n"; 99 | } 100 | } 101 | interactions.push({ 102 | command, 103 | code 104 | }); 105 | } 106 | } 107 | return interactions; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/server/app.ts: -------------------------------------------------------------------------------- 1 | import express from "express"; 2 | import path from "path"; 3 | import { context, getCompletion } from "./model"; 4 | import cors from "cors"; 5 | 6 | const app = express(); 7 | app.use(express.json()); 8 | app.use(express.static(__dirname + "/public")); 9 | 10 | const port = process.env.SERVER_PORT; 11 | app.use(cors({ origin: `http://localhost:${process.env.CLIENT_PORT}` })); 12 | 13 | app.get("/", (_req, res) => { 14 | res.sendFile(path.join(__dirname, "/index.html")); 15 | }); 16 | 17 | // Gets natural language and returns code 18 | app.post("/codegen", async (req, res) => { 19 | console.log(`Received natural language command: '${req.body.text}'`); 20 | const response = await getCompletion(req.body.text); 21 | res.send(JSON.stringify(response)); 22 | }); 23 | 24 | // Gets natural language and returns code 25 | app.get("/reset", async (_req, res) => { 26 | context.resetContext(); 27 | res.send( 28 | JSON.stringify({ 29 | context: context.getContext() 30 | }) 31 | ); 32 | }); 33 | 34 | app.listen(port, () => { 35 | console.log(`Codex Babylon webapp listening at http://localhost:${port}`); 36 | }); 37 | -------------------------------------------------------------------------------- /src/server/contentFiltering.ts: -------------------------------------------------------------------------------- 1 | import fetch from "isomorphic-fetch"; 2 | 3 | export async function detectSensitiveContent(content: string): Promise { 4 | const response = await fetch('https://api.openai.com/v1/engines/content-filter-alpha/completions', { 5 | method: "POST", 6 | headers: { 7 | "Content-Type": "application/json", 8 | Authorization: `Bearer ${process.env.OPENAI_API_KEY}`, 9 | "OpenAI-Organization": `${process.env.OPENAI_ORGANIZATION_ID}` 10 | }, 11 | body: JSON.stringify({ 12 | prompt: `<|endoftext|>[${content}]\n--\nLabel:`, 13 | temperature: 0, 14 | max_tokens: 1, 15 | top_p: 0, 16 | logprobs: 10, 17 | }) 18 | }); 19 | 20 | var json = await response.json(); 21 | 22 | const filterFlag = json.choices[0].text as string; 23 | return parseInt(filterFlag); 24 | } -------------------------------------------------------------------------------- /src/server/contexts/context1.ts: -------------------------------------------------------------------------------- 1 | export const baseContext = `/* This document contains a BabylonJS scene, natural language commands and the BabylonJS code needed to accomplish them */ 2 | 3 | state = {}; 4 | 5 | /* Make a cube */ 6 | state.cube = BABYLON.MeshBuilder.CreateBox("cube", {size: 1}, scene); 7 | 8 | /* Move the cube up */ 9 | state.cube.position.y += 1; 10 | 11 | /* Move it to the left */ 12 | state.cube.position.x -= 1; 13 | 14 | /* Make the block teal */ 15 | state.cube.material = new BABYLON.StandardMaterial("mat", scene); 16 | state.cube.material.diffuseColor = new BABYLON.Color3(0, 1, 1); 17 | 18 | /* Now make it spin */ 19 | state.intervals["spinningCubeInterval"] = setInterval(() => { 20 | scene.meshes[0].rotation.y += 0.02 21 | }, 10); 22 | 23 | /* Make it stop */ 24 | clearInterval(state.intervals.spinningCubeInterval); 25 | 26 | /* Make it change color when the mouse is over it */ 27 | state.cube.actionManager = new BABYLON.ActionManager(scene); 28 | 29 | state.hoverAction = new BABYLON.ExecuteCodeAction(BABYLON.ActionManager.OnPointerOverTrigger, function () { 30 | state.cube.material = new BABYLON.StandardMaterial("mat", scene); 31 | state.cube.material.diffuseColor = new BABYLON.Color3(1, 0, 0); 32 | }); 33 | 34 | state.unHoverAction = new BABYLON.ExecuteCodeAction(BABYLON.ActionManager.OnPointerOutTrigger, function () { 35 | state.cube.material = new BABYLON.StandardMaterial("mat", scene); 36 | state.cube.material.diffuseColor = new BABYLON.Color3(0, 1, 1); 37 | }); 38 | 39 | state.cube.actionManager.registerAction(state.hoverAction); 40 | state.cube.actionManager.registerAction(state.unHoverAction); 41 | 42 | /* Put a sphere on top of the cube */ 43 | state.sphere = BABYLON.MeshBuilder.CreateSphere("sphere", {diameter: 1}, scene); 44 | state.sphere.position.y = 1; 45 | 46 | /* Delete the sphere and the cube */ 47 | state.sphere.dispose(); 48 | state.cube.dispose(); 49 | 50 | /* make 50 cubes side by side */ 51 | state.cubes = []; 52 | for (let i = 0; i < 50; i++) { 53 | state.cubes[i] = BABYLON.MeshBuilder.CreateBox("cube", {size: 1}, scene); 54 | state.cubes[i].position.x = i; 55 | } 56 | 57 | /* stack them like stairs */ 58 | for (let i = 0; i < 50; i++) { 59 | state.cubes[i].position.y = i; 60 | } 61 | 62 | /* remove them */ 63 | for (let i = 0; i < 50; i++) { 64 | state.cubes[i].dispose(); 65 | } 66 | 67 | /* make the background red */ 68 | scene.clearColor = new BABYLON.Color3(1, 0, 0); 69 | `; 70 | -------------------------------------------------------------------------------- /src/server/model.ts: -------------------------------------------------------------------------------- 1 | require("dotenv").config(); 2 | import fetch from "isomorphic-fetch"; 3 | 4 | // Contains the helper methods for interacting with Codex and crafting model prompts 5 | import { baseContext } from "./contexts/context1"; 6 | import Context from "./Context"; 7 | import { detectSensitiveContent } from "./contentFiltering"; 8 | 9 | const maxPromptLength = 3200; 10 | 11 | // CURRENTLY SINGLE TENANT - WOULD NEED TO UPDATE THIS TO A MAP OF TENANT IDs TO PROMPTS TO MAKE MULTI-TENANT 12 | export const context = new Context(baseContext); 13 | 14 | export async function getCompletion(command: string) { 15 | let prompt = context.getPrompt(command); 16 | 17 | if (prompt.length > maxPromptLength) { 18 | context.trimContext(maxPromptLength - command.length + 6); // The max length of the prompt, including the command, comment operators and spacing. 19 | } 20 | 21 | // To learn more about making requests to OpanAI API, please refer to https://beta.openai.com/docs/api-reference/making-requests. 22 | // Here we use the following endpoint pattern for engine selection. 23 | // https://api.openai.com/v1/engines/{engine_id}/completions 24 | // You can switch to different engines that are available to you. Learn more about engines - https://beta.openai.com/docs/engines/engines 25 | const response = await fetch(`https://api.openai.com/v1/engines/${process.env.OPENAI_ENGINE_ID}/completions`, { 26 | method: "POST", 27 | headers: { 28 | "Content-Type": "application/json", 29 | Authorization: `Bearer ${process.env.OPENAI_API_KEY}`, 30 | "OpenAI-Organization": `${process.env.OPENAI_ORGANIZATION_ID}` 31 | }, 32 | body: JSON.stringify({ 33 | prompt, 34 | max_tokens: 800, 35 | temperature: 0, 36 | stop: "/*", 37 | n: 1 38 | }) 39 | }); 40 | 41 | // catch errors 42 | if (!response.ok) { 43 | //throw new Error(`${response.status} ${response.statusText}`); 44 | const error = `There is an issue with your OpenAI credentials, please check your OpenAI API key, organization ID and model name. Modify the credentials and restart the server!`; 45 | if (response.status == 404){ 46 | console.log(error); 47 | } 48 | return {error}; 49 | } 50 | 51 | const json = await response.json(); 52 | let code = json.choices[0].text; 53 | 54 | let sensitiveContentFlag = await detectSensitiveContent(command + "\n" + code); 55 | 56 | // The flag can be 0, 1 or 2, corresponding to 'safe', 'sensitive' and 'unsafe' 57 | if (sensitiveContentFlag > 0) { 58 | console.warn( 59 | sensitiveContentFlag === 1 60 | ? "Your message or the model's response may have contained sensitive content." 61 | : "Your message or the model's response may have contained unsafe content." 62 | ); 63 | 64 | code = ''; 65 | } 66 | else { 67 | //only allow safe interactions to be added to the context history 68 | context.addInteraction(command, code); 69 | } 70 | 71 | return { 72 | code, 73 | prompt, 74 | sensitiveContentFlag 75 | }; 76 | } 77 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": false, 3 | "buildOnSave": false, 4 | "compilerOptions": { 5 | "target": "es5", 6 | "jsx": "react", 7 | "module": "esnext", 8 | "moduleResolution": "node", 9 | "noImplicitAny": false, 10 | "emitDecoratorMetadata": true, 11 | "experimentalDecorators": true, 12 | "esModuleInterop": true, 13 | "sourceMap": true, 14 | "strict": true, 15 | "removeComments": false, 16 | "lib": ["es2015", "es2016", "es2017.object", "dom"], 17 | "strictNullChecks": true, 18 | "forceConsistentCasingInFileNames": true, 19 | "skipLibCheck": true, 20 | "importHelpers": true 21 | }, 22 | "exclude": ["node_modules", "dist"] 23 | } 24 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | var webpack = require("webpack"); 2 | var HtmlWebpackPlugin = require("html-webpack-plugin"); 3 | var { CleanWebpackPlugin } = require("clean-webpack-plugin"); 4 | var path = require("path"); 5 | require("dotenv").config(); 6 | 7 | module.exports = (env) => { 8 | let mode = ""; 9 | let devtool = ""; 10 | if (env && env.prod) { 11 | // prod config 12 | mode = "production"; 13 | devtool = "cheap-module-source-map"; 14 | } else { 15 | // dev config 16 | mode = "development"; 17 | devtool = "inline-source-map"; 18 | } 19 | 20 | return { 21 | entry: { 22 | app: "./src/client/index.tsx" 23 | }, 24 | resolve: { 25 | extensions: [".tsx", ".ts", ".js"] 26 | }, 27 | mode, 28 | devtool, 29 | devServer: { 30 | open: true, 31 | port: process.env.CLIENT_PORT 32 | }, 33 | module: { 34 | rules: [ 35 | { 36 | test: /\.tsx?$/, 37 | loader: "ts-loader", 38 | exclude: /node_modules/ 39 | }, 40 | { 41 | test: /\.css$/, 42 | use: [ 43 | { 44 | loader: "style-loader" 45 | }, 46 | { 47 | loader: "css-loader", 48 | options: { 49 | sourceMap: true 50 | } 51 | } 52 | ] 53 | } 54 | ] 55 | }, 56 | output: { 57 | path: path.join(__dirname, "./dist/client"), 58 | filename: "[name].[fullhash].js", 59 | chunkFilename: "[name].bundle.js" 60 | }, 61 | plugins: [ 62 | new CleanWebpackPlugin({ 63 | cleanOnceBeforeBuild: ["./dist"] 64 | }), 65 | new HtmlWebpackPlugin({ 66 | template: path.join(__dirname, "./src/client/index.html") 67 | }), 68 | new webpack.DefinePlugin({ 69 | "process.env": { 70 | CLIENT_PORT: JSON.stringify(process.env.CLIENT_PORT), 71 | SERVER_PORT: JSON.stringify(process.env.SERVER_PORT) 72 | } 73 | }) 74 | ] 75 | }; 76 | }; 77 | --------------------------------------------------------------------------------