├── .DS_Store ├── .env ├── .eslintignore ├── .eslintrc ├── .gitignore ├── .prettierignore ├── .prettierrc ├── .vscode ├── launch.json └── tasks.json ├── LICENSE ├── README.md ├── appPackage ├── JARVIB_192.png ├── JARVIB_32.png ├── JARVIB_512.jpg ├── color.png ├── manifest.json └── outline.png ├── env ├── .env.local ├── .env.local.user └── .env.staging ├── images ├── F5launch001.png ├── F5launch002.png ├── F5launch003.jpg ├── F5launch004.png ├── F5launch005.png ├── F5launch006.png ├── F5launch007.jpg ├── F5launch008.jpg └── F5launch009.jpg ├── infra ├── azure.bicep ├── azure.parameters.json └── botRegistration │ └── azurebot.bicep ├── package-lock.json ├── package.json ├── promptsExamples.txt ├── src ├── .DS_Store ├── app.js ├── debug.html ├── index.html ├── index.ts ├── prompts │ ├── .DS_Store │ ├── chatGPT │ │ ├── actions.json │ │ ├── config.json │ │ └── skprompt.txt │ └── describe │ │ ├── config.json │ │ └── skprompt.txt └── responses.ts ├── teamsapp.local.yml ├── teamsapp.yml ├── tsconfig.json ├── web.config └── yarn.lock /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/.DS_Store -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | BOT_ID= 2 | BOT_PASSWORD= 3 | OPENAI_KEY= 4 | AZURE_OPENAI_KEY= 5 | AZURE_OPENAI_ENDPOINT= 6 | AZURE_OPENAI_API_MODEL= -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | bin 2 | build 3 | demo-packages 4 | dist 5 | manifest 6 | node_modules 7 | package-lock.json 8 | docs/assets/main.js -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "@typescript-eslint/parser", 3 | "root": true, 4 | "env": { 5 | "browser": true, 6 | "node": true, 7 | "es2015": true, 8 | "mocha": true, 9 | "jest": true 10 | }, 11 | "extends": [ 12 | "eslint:recommended", 13 | "plugin:@typescript-eslint/recommended", 14 | "plugin:import/typescript", 15 | "plugin:import/recommended", 16 | "plugin:jsdoc/recommended", 17 | 18 | "plugin:security/recommended", 19 | "plugin:prettier/recommended" // Recommended to be last 20 | ], 21 | "plugins": [ 22 | "@typescript-eslint", 23 | "jsdoc", 24 | 25 | "mocha", 26 | "only-warn", 27 | "prettier" 28 | // "react" 29 | ], 30 | "parserOptions": { 31 | "ecmaVersion": 2015, 32 | // Allows for the parsing of modern ECMAScript features 33 | "sourceType": "module" // Allows for the use of imports 34 | // "ecmaFeatures": { 35 | // "jsx": true 36 | // } 37 | }, 38 | "rules": { 39 | // Place to specify ESLint rules. Can be used to overwrite rules specified from the extended configs 40 | "@typescript-eslint/ban-types": "off", 41 | "@typescript-eslint/explicit-function-return-type": "off", 42 | "@typescript-eslint/explicit-member-accessibility": "off", 43 | "@typescript-eslint/explicit-module-boundary-types": "off", 44 | "@typescript-eslint/interface-name-prefix": "off", 45 | "@typescript-eslint/no-empty-function": "off", 46 | "@typescript-eslint/no-explicit-any": "off", 47 | "@typescript-eslint/no-namespace": "off", 48 | "@typescript-eslint/no-unused-vars": "off", 49 | "@typescript-eslint/no-non-null-assertion": "off", 50 | "no-async-promise-executor": "off", 51 | "no-constant-condition": "off", 52 | "no-undef": "off", // Disabled due to conflicts with @typescript/eslint 53 | "no-unused-vars": "off", // Disabled due to conflicts with @typescript/eslint 54 | "prettier/prettier": "error" 55 | }, 56 | "overrides": [ 57 | { 58 | "files": ["bin/*.js", "lib/*.js"] 59 | } 60 | ], 61 | "ignorePatterns": ["node_modules/*"], 62 | "settings": { 63 | // "react": { 64 | // "version": "detect" 65 | // } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .zip 2 | 3 | # Logs 4 | logs 5 | *.log 6 | npm-debug.log* 7 | yarn-debug.log* 8 | yarn-error.log* 9 | lerna-debug.log* 10 | 11 | # Diagnostic reports (https://nodejs.org/api/report.html) 12 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 13 | 14 | # Runtime data 15 | pids 16 | *.pid 17 | *.seed 18 | *.pid.lock 19 | 20 | # Directory for instrumented libs generated by jscoverage/JSCover 21 | lib-cov 22 | /**/lib 23 | lib 24 | 25 | # Coverage directory used by tools like istanbul 26 | coverage 27 | *.lcov 28 | 29 | # nyc test coverage 30 | .nyc_output 31 | 32 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 33 | .grunt 34 | 35 | # Bower dependency directory (https://bower.io/) 36 | bower_components 37 | 38 | # node-waf configuration 39 | .lock-wscript 40 | 41 | # Compiled binary addons (https://nodejs.org/api/addons.html) 42 | build/Release 43 | 44 | # Dependency directories 45 | node_modules/ 46 | jspm_packages/ 47 | 48 | # TypeScript v1 declaration files 49 | typings/ 50 | 51 | # TypeScript cache 52 | *.tsbuildinfo 53 | 54 | # Optional npm cache directory 55 | .npm 56 | 57 | # Optional eslint cache 58 | .eslintcache 59 | 60 | # Microbundle cache 61 | .rpt2_cache/ 62 | .rts2_cache_cjs/ 63 | .rts2_cache_es/ 64 | .rts2_cache_umd/ 65 | 66 | # Optional REPL history 67 | .node_repl_history 68 | 69 | # Output of 'npm pack' 70 | *.tgz 71 | 72 | # Yarn Integrity file 73 | .yarn-integrity 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | 78 | # Next.js build output 79 | .next 80 | 81 | # Nuxt.js build / generate output 82 | .nuxt 83 | dist 84 | 85 | # Gatsby files 86 | .cache/ 87 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 88 | # https://nextjs.org/blog/next-9-1#public-directory-support 89 | # public 90 | 91 | # vuepress build output 92 | .vuepress/dist 93 | 94 | # Serverless directories 95 | .serverless/ 96 | 97 | # FuseBox cache 98 | .fusebox/ 99 | 100 | # DynamoDB Local files 101 | .dynamodb/ 102 | 103 | # TernJS port file 104 | .tern-port 105 | 106 | # Teams Toolkit 107 | appPackage/build 108 | .deployment 109 | 110 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | bin 2 | build 3 | demo-packages 4 | dist 5 | manifest 6 | node_modules 7 | package-lock.json -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "arrowParens": "always", 3 | "endOfLine": "auto", 4 | "printWidth": 120, 5 | "semi": true, 6 | "singleAttributePerLine": false, 7 | "singleQuote": true, 8 | "tabWidth": 4, 9 | "trailingComma": "none", 10 | "useTabs": false 11 | } 12 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Launch Remote (Edge)", 6 | "type": "msedge", 7 | "request": "launch", 8 | "url": "https://teams.microsoft.com/l/app/${{TEAMS_APP_ID}}?installAppPackage=true&webjoin=true&${account-hint}", 9 | "presentation": { 10 | "group": "remote", 11 | "order": 1 12 | }, 13 | "internalConsoleOptions": "neverOpen" 14 | }, 15 | { 16 | "name": "Launch Remote (Chrome)", 17 | "type": "chrome", 18 | "request": "launch", 19 | "url": "https://teams.microsoft.com/l/app/${{TEAMS_APP_ID}}?installAppPackage=true&webjoin=true&${account-hint}", 20 | "presentation": { 21 | "group": "remote", 22 | "order": 2 23 | }, 24 | "internalConsoleOptions": "neverOpen" 25 | }, 26 | { 27 | "name": "Launch App (Edge)", 28 | "type": "msedge", 29 | "request": "launch", 30 | "url": "https://teams.microsoft.com/l/app/${{local:TEAMS_APP_ID}}?installAppPackage=true&webjoin=true&${account-hint}", 31 | "cascadeTerminateToConfigurations": [ 32 | "Attach to Local Service" 33 | ], 34 | "presentation": { 35 | "group": "all", 36 | "hidden": true 37 | }, 38 | "internalConsoleOptions": "neverOpen" 39 | }, 40 | { 41 | "name": "Launch App (Chrome)", 42 | "type": "chrome", 43 | "request": "launch", 44 | "url": "https://teams.microsoft.com/l/app/${{local:TEAMS_APP_ID}}?installAppPackage=true&webjoin=true&${account-hint}", 45 | "cascadeTerminateToConfigurations": [ 46 | "Attach to Local Service" 47 | ], 48 | "presentation": { 49 | "group": "all", 50 | "hidden": true 51 | }, 52 | "internalConsoleOptions": "neverOpen" 53 | }, 54 | { 55 | "name": "Attach to Local Service", 56 | "type": "node", 57 | "request": "attach", 58 | "port": 9239, 59 | "restart": true, 60 | "presentation": { 61 | "group": "all", 62 | "hidden": true 63 | }, 64 | "internalConsoleOptions": "neverOpen" 65 | } 66 | ], 67 | "compounds": [ 68 | { 69 | "name": "Debug (Edge)", 70 | "configurations": [ 71 | "Launch App (Edge)", 72 | "Attach to Local Service" 73 | ], 74 | "preLaunchTask": "Start Teams App Locally", 75 | "presentation": { 76 | "group": "all", 77 | "order": 1 78 | }, 79 | "stopAll": true 80 | }, 81 | { 82 | "name": "Debug (Chrome)", 83 | "configurations": [ 84 | "Launch App (Chrome)", 85 | "Attach to Local Service" 86 | ], 87 | "preLaunchTask": "Start Teams App Locally", 88 | "presentation": { 89 | "group": "all", 90 | "order": 2 91 | }, 92 | "stopAll": true 93 | } 94 | ] 95 | } 96 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | // This file is automatically generated by Teams Toolkit. 2 | // The teamsfx tasks defined in this file require Teams Toolkit version >= 5.0.0. 3 | // See https://aka.ms/teamsfx-tasks for details on how to customize each task. 4 | { 5 | "version": "2.0.0", 6 | "tasks": [ 7 | { 8 | "label": "Start Teams App Locally", 9 | "dependsOn": [ 10 | "Validate prerequisites", 11 | "Start local tunnel", 12 | "Provision", 13 | "Deploy", 14 | "Start application" 15 | ], 16 | "dependsOrder": "sequence" 17 | }, 18 | { 19 | // Check all required prerequisites. 20 | // See https://aka.ms/teamsfx-tasks/check-prerequisites to know the details and how to customize the args. 21 | "label": "Validate prerequisites", 22 | "type": "teamsfx", 23 | "command": "debug-check-prerequisites", 24 | "args": { 25 | "prerequisites": [ 26 | "nodejs", // Validate if Node.js is installed. 27 | "m365Account", // Sign-in prompt for Microsoft 365 account, then validate if the account enables the sideloading permission. 28 | "portOccupancy" // Validate available ports to ensure those debug ones are not occupied. 29 | ], 30 | "portOccupancy": [ 31 | 3978, // app service port 32 | 9239, // app inspector port for Node.js debugger 33 | 3000 // use for WebSocket connection 34 | ] 35 | } 36 | }, 37 | { 38 | // Start the local tunnel service to forward public URL to local port and inspect traffic. 39 | // See https://aka.ms/teamsfx-tasks/local-tunnel for the detailed args definitions. 40 | "label": "Start local tunnel", 41 | "type": "teamsfx", 42 | "command": "debug-start-local-tunnel", 43 | "args": { 44 | "type": "dev-tunnel", 45 | "ports": [ 46 | { 47 | "portNumber": 3978, 48 | "protocol": "http", 49 | "access": "public", 50 | "writeToEnvironmentFile": { 51 | "endpoint": "BOT_ENDPOINT", // output tunnel endpoint as BOT_ENDPOINT 52 | "domain": "BOT_DOMAIN" // output tunnel domain as BOT_DOMAIN 53 | } 54 | }, 55 | { 56 | "portNumber": 3000, 57 | "protocol": "http", 58 | "access": "public", 59 | "writeToEnvironmentFile": { 60 | "endpoint": "TAB_ENDPOINT", 61 | "domain": "TAB_DOMAIN" 62 | } 63 | } 64 | ], 65 | "env": "local" 66 | }, 67 | "isBackground": true, 68 | "problemMatcher": "$teamsfx-local-tunnel-watch" 69 | }, 70 | { 71 | // Create the debug resources. 72 | // See https://aka.ms/teamsfx-tasks/provision to know the details and how to customize the args. 73 | "label": "Provision", 74 | "type": "teamsfx", 75 | "command": "provision", 76 | "args": { 77 | "env": "local" 78 | } 79 | }, 80 | { 81 | // Build project. 82 | // See https://aka.ms/teamsfx-tasks/deploy to know the details and how to customize the args. 83 | "label": "Deploy", 84 | "type": "teamsfx", 85 | "command": "deploy", 86 | "args": { 87 | "env": "local" 88 | } 89 | }, 90 | { 91 | "label": "Start application", 92 | "type": "shell", 93 | "command": "npm run dev:teamsfx", 94 | "isBackground": true, 95 | "options": { 96 | "cwd": "${workspaceFolder}" 97 | }, 98 | "problemMatcher": { 99 | "pattern": [ 100 | { 101 | "regexp": "^.*$", 102 | "file": 0, 103 | "location": 1, 104 | "message": 2 105 | } 106 | ], 107 | "background": { 108 | "activeOnStart": true, 109 | "beginsPattern": "[nodemon] starting", 110 | "endsPattern": "restify listening to|Bot/ME service listening at|[nodemon] app crashed" 111 | } 112 | } 113 | } 114 | ] 115 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 David Rousset 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AI in Microsoft Teams: J.A.R.V.I.B. 2 | 3 | *Your custom Microsoft Teams AI Copilot to build interactive dynamic 3D worlds* 4 | 5 | Logo of JARVIB 6 | 7 | JARVIB (Just A Rather Very Intelligent Bot) is your personnal Copilot to help you designing 3D worlds in a collaborative way inside a personnal conversation or in a Teams meeting. Ask him to create various Babylon.js primitive or to search for specific models to load them into the scene. You can also ask to the AI to visually describe the scene for accessibility reasons. Look at possible prompts in ```promptsExamples.txt``` 8 | 9 | 10 | 11 | 12 | 13 | - [AI in Microsoft Teams: J.A.R.V.I.B.](#ai-in-microsoft-teams-list-bot) 14 | - [View it in action](#view-it-in-action) 15 | - [Setting up the sample](#setting-up-the-sample) 16 | - [Interacting with the bot](#interacting-with-the-bot) 17 | - [Further reading](#further-reading) 18 | 19 | 20 | 21 | ## View it in action 22 | 23 | [![Image alt text](https://img.youtube.com/vi/181D9lk7DRc/hqdefault.jpg)](https://www.youtube.com/watch?v=181D9lk7DRc) 24 | 25 | JARVIB is using the Teams AI library to map users' intent to 3 different type of actions: 26 | 27 | - **codeToExecute** which will take the Babylon.js code generated by the LLM and send it to the web page containing the 3D canvas to execute it via WebSocket 28 | - **listAvailableModel** which will use the same API as PowerPoint to find for existing 3D models and return a list of available models to load through an Adaptive Card list. 29 | - **loadThisModel** which will load one of the models exposed in the list displayed before 30 | 31 | You have also 3 commands available: 32 | 33 | - **/reset** will clear the conversation state, clean the various objects states and reload the web page containing the 3D canvas to start from scratch 34 | - **/describe** will ask to the AI, via the text-davinci-003 LLM model to visually describe the scene 35 | - **/fullcode** will return all Babylon.js code executed so far so you can copy/paste it in your project or in the Babylon.js Playground 36 | 37 | Look at the ```/src/prompts``` in ```/ChatGPT/skprompt.txt``` to check how the AI is prompted and check the ```/ChatGPT/actions.json``` file to understand how the various actions are described for the 3 Teams AI actions. 38 | 39 | The prompt configuration: 40 | 41 | ``` 42 | Pretend you're an expert in Babylon.js, the JavaScript WebGL 3D engine. 43 | 44 | Assume there is already an existing Babylon.js scene and engine so you don't have to create them, just generate the code to add into an existing program. 45 | Use the scene and engine objects directly. 46 | 47 | Pay attention when trying to access previously created Meshes by getting access to them via their name rather than assuming the associated variable is already created. 48 | When writing a new code, consider all the previous one you've generated to be sure the new code will be consistent with the previous one. 49 | Remember about the already created meshes, animations or any other specific ressources before trying to create them or reuse them. 50 | ``` 51 | 52 | The JSON schema to help the LLM to map to the defined actions: 53 | 54 | ```javascript 55 | [ 56 | { 57 | "name": "codeToExecute", 58 | "description": "Returns the Babylon.js JavaScript code matching the user intent, building the next code working in a consistent way with the previous code already executed", 59 | "parameters": { 60 | "type": "object", 61 | "properties": { 62 | "code": { 63 | "type": "string", 64 | "description": "The JavaScript code to execute next" 65 | } 66 | }, 67 | "required": [ 68 | "code" 69 | ] 70 | } 71 | }, 72 | { 73 | "name": "listAvailableModel", 74 | "description": "List the available 3D models we can load from the library", 75 | "parameters": { 76 | "type": "object", 77 | "properties": { 78 | "nameOfTheModel": { 79 | "type": "string", 80 | "description": "The name of the model to search inside the library" 81 | } 82 | }, 83 | "required": [ 84 | "nameOfTheModel" 85 | ] 86 | } 87 | }, 88 | { 89 | "name": "loadThisModel", 90 | "description": "Load the 3D model specified by the user", 91 | "parameters": { 92 | "type": "object", 93 | "properties": { 94 | "nameOfTheModel": { 95 | "type": "string", 96 | "description": "The name of the model to load from the library" 97 | } 98 | }, 99 | "required": [ 100 | "nameOfTheModel" 101 | ] 102 | } 103 | } 104 | ] 105 | ``` 106 | 107 | ## Setting up the sample 108 | 109 | 1. Clone the repository 110 | 111 | ```bash 112 | git clone https://github.com/davrous/JARVIB.git 113 | ``` 114 | 115 | 2. Install the Teams Toolkit: https://learn.microsoft.com/en-us/microsoftteams/platform/toolkit/install-teams-toolkit 116 | 117 | 3. Set your Azure OpenAI key, endpoints and model names or your OpenAI key in the .env file. 118 | 119 | For Azure OpenAI, pay attention to use the "*Deployment name*" and not the "*Model name*" as the value for the properties in the .env file. If you've got an error 404 during execution, it's because you either entered the wrong endpoint or the wrong deployment name. 120 | 121 | 4. Press F5! 122 | 123 | The Team Toolkit will do a lot of magic for you by: 124 | - installing all dependencies and building the TypeScript code 125 | - registering the bot in the Azure Active Directory 126 | - building the Teams manifest file, zipping the resources to upload them in the Teams Developer Portal 127 | - creating 2 tunnels to expose your localhost on public URLs using VS Dev Tunnels, you can find the details in ```/.vscode/tasks.json``` 128 | - 1 for the bot itself mapped on port 3978 129 | - 1 for the tab to be used inside a Teams meeting mapped on port 3000 130 | - launching the Teams web app to let you install the bot / meeting extension 131 | 132 | 5. If everything goes well, you should first be able that the 2 tunnels have been created: 133 | 134 | Terminal showing the VS Dev Tunnels running fine 135 | 136 | 6. Then, the webserver will be launched and you'll be asked to install JARVIB in your developer tenant: 137 | 138 | Terminal showing the VS Dev Tunnels running fine 139 | 140 | You can either simply press the "*Add*" button and this will install the app to let you discuss with it directly as a classical bot. 141 | 142 | You can also add it to an existing meeting to be able to replicate the full experience showcased in the video. 143 | 144 | ## Interacting with the bot 145 | 146 | The easiest way to try it is to install it using the "*Add*" button as a bot, then open http://localhost:3000/debug.html on the side open a simple canvas 3D websocket client. Then ask to the bot to create 3D content, load some models and so on: 147 | 148 | screenshot of the Teams AI bot with the canvas scene on its right 149 | 150 | The other option to replicate the full experience is to load the app inside an existing meeting: 151 | 152 | screenshot the Teams app installation in an existing meeting found 153 | 154 | You need to accept connecting to the VS Dev Tunnel redirecting to your localhost developer machine: 155 | 156 | screenshot VS Dev Tunnels warning message 157 | 158 | Finally, save JARVIB as an app part of this chosen Teams meeting: 159 | 160 | screenshot VS Dev Tunnels warning message 161 | 162 | Now, launch the Teams meeting alone or with someone else to collaborate with: 163 | 164 | Teams meeting launched with the JARVIB app visible in the bar 165 | 166 | Click on the JARVIB icon and then click on the "*Share to meeting*" button: 167 | 168 | Teams meeting with the sidebar window and the meeting stage showing the 3D canvas 169 | 170 | Finally, call the AI bot via @JARVIB-local following your order: 171 | 172 | JARVIB fully working in a Teams meeting 173 | 174 | ## Further reading 175 | 176 | - [Teams AI library](https://learn.microsoft.com/en-us/microsoftteams/platform/bots/how-to/teams%20conversational%20ai/teams-conversation-ai-overview) 177 | - [Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview) 178 | - [Teams Toolkit overview](https://learn.microsoft.com/en-us/microsoftteams/platform/toolkit/teams-toolkit-fundamentals) 179 | - [How Microsoft Teams bots work](https://docs.microsoft.com/en-us/azure/bot-service/bot-builder-basics-teams?view=azure-bot-service-4.0&tabs=javascript) 180 | -------------------------------------------------------------------------------- /appPackage/JARVIB_192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/appPackage/JARVIB_192.png -------------------------------------------------------------------------------- /appPackage/JARVIB_32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/appPackage/JARVIB_32.png -------------------------------------------------------------------------------- /appPackage/JARVIB_512.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/appPackage/JARVIB_512.jpg -------------------------------------------------------------------------------- /appPackage/color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/appPackage/color.png -------------------------------------------------------------------------------- /appPackage/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/en-us/json-schemas/teams/v1.16/MicrosoftTeams.schema.json", 3 | "version": "1.0.0", 4 | "manifestVersion": "1.16", 5 | "id": "${{TEAMS_APP_ID}}", 6 | "packageName": "com.davrous.jarvib", 7 | "name": { 8 | "short": "JARVIB-${{TEAMSFX_ENV}}", 9 | "full": "Just A Rather Very Intelligent Bot that can create dynamic 3D worlds." 10 | }, 11 | "developer": { 12 | "name": "David Rousset", 13 | "mpnId": "", 14 | "websiteUrl": "https://www.davrous.com", 15 | "privacyUrl": "https://www.davrous.com/", 16 | "termsOfUseUrl": "https://microsoft.com/termsofuse" 17 | }, 18 | "description": { 19 | "short": "A smart bot that can create dynamic 3D worlds.", 20 | "full": "Just A Rather Very Intelligent Bot that can create dynamic 3D worlds." 21 | }, 22 | "icons": { 23 | "outline": "JARVIB_32.png", 24 | "color": "JARVIB_192.png" 25 | }, 26 | "accentColor": "#FFFFFF", 27 | "bots": [ 28 | { 29 | "botId": "${{BOT_ID}}", 30 | "scopes": ["personal", "team", "groupChat"], 31 | "isNotificationOnly": false, 32 | "supportsCalling": false, 33 | "supportsVideo": false, 34 | "supportsFiles": false 35 | } 36 | ], 37 | "configurableTabs": [ 38 | { 39 | "configurationUrl": "${{TAB_ENDPOINT}}/?view=config&inTeams=1&load=1", 40 | "canUpdateConfiguration": false, 41 | "scopes": [ 42 | "groupchat" 43 | ], 44 | "context": [ 45 | "meetingSidePanel", 46 | "meetingStage" 47 | ] 48 | } 49 | ], 50 | "permissions": [ 51 | "identity", 52 | "messageTeamMembers" 53 | ], 54 | "validDomains": [ 55 | "${{BOT_DOMAIN}}", 56 | "${{TAB_DOMAIN}}" 57 | ] 58 | } 59 | -------------------------------------------------------------------------------- /appPackage/outline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/appPackage/outline.png -------------------------------------------------------------------------------- /env/.env.local: -------------------------------------------------------------------------------- 1 | # This file includes environment variables that can be committed to git. It's gitignored by default because it represents your local development environment. 2 | 3 | # Built-in environment variables 4 | TEAMSFX_ENV=local 5 | 6 | # Generated during provision, you can also add your own variables. 7 | BOT_ID= 8 | TEAMS_APP_ID= 9 | BOT_DOMAIN= 10 | BOT_ENDPOINT= 11 | TEAMS_APP_TENANT_ID= 12 | 13 | # TO DO: Add your own environment variables values here. 14 | TAB_ENDPOINT= 15 | TAB_DOMAIN= 16 | APP_NAME_SUFFIX=local -------------------------------------------------------------------------------- /env/.env.local.user: -------------------------------------------------------------------------------- 1 | SECRET_BOT_PASSWORD= 2 | TEAMS_APP_UPDATE_TIME= -------------------------------------------------------------------------------- /env/.env.staging: -------------------------------------------------------------------------------- 1 | # This file includes environment variables that will be committed to git by default. 2 | 3 | # Built-in environment variables 4 | TEAMSFX_ENV=staging 5 | 6 | # Updating AZURE_SUBSCRIPTION_ID or AZURE_RESOURCE_GROUP_NAME after provision may also require an update to RESOURCE_SUFFIX, because some services require a globally unique name across subscriptions/resource groups. 7 | AZURE_SUBSCRIPTION_ID= 8 | AZURE_RESOURCE_GROUP_NAME= 9 | RESOURCE_SUFFIX= 10 | 11 | # Generated during provision, you can also add your own variables. 12 | BOT_ID= 13 | TEAMS_APP_ID= 14 | BOT_AZURE_APP_SERVICE_RESOURCE_ID= 15 | BOT_DOMAIN= 16 | -------------------------------------------------------------------------------- /images/F5launch001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/images/F5launch001.png -------------------------------------------------------------------------------- /images/F5launch002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/images/F5launch002.png -------------------------------------------------------------------------------- /images/F5launch003.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/images/F5launch003.jpg -------------------------------------------------------------------------------- /images/F5launch004.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/images/F5launch004.png -------------------------------------------------------------------------------- /images/F5launch005.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/images/F5launch005.png -------------------------------------------------------------------------------- /images/F5launch006.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/images/F5launch006.png -------------------------------------------------------------------------------- /images/F5launch007.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/images/F5launch007.jpg -------------------------------------------------------------------------------- /images/F5launch008.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/images/F5launch008.jpg -------------------------------------------------------------------------------- /images/F5launch009.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/images/F5launch009.jpg -------------------------------------------------------------------------------- /infra/azure.bicep: -------------------------------------------------------------------------------- 1 | @maxLength(20) 2 | @minLength(4) 3 | @description('Used to generate names for all resources in this file') 4 | param resourceBaseName string 5 | 6 | @description('Required when create Azure Bot service') 7 | param botAadAppClientId string 8 | 9 | @secure() 10 | @description('Required by Bot Framework package in your bot project') 11 | param botAadAppClientSecret string 12 | 13 | param webAppSKU string 14 | 15 | @maxLength(42) 16 | param botDisplayName string 17 | 18 | param serverfarmsName string = resourceBaseName 19 | param webAppName string = resourceBaseName 20 | param location string = resourceGroup().location 21 | 22 | // Compute resources for your Web App 23 | resource serverfarm 'Microsoft.Web/serverfarms@2021-02-01' = { 24 | kind: 'app' 25 | location: location 26 | name: serverfarmsName 27 | sku: { 28 | name: webAppSKU 29 | } 30 | } 31 | 32 | // Web App that hosts your bot 33 | resource webApp 'Microsoft.Web/sites@2021-02-01' = { 34 | kind: 'app' 35 | location: location 36 | name: webAppName 37 | properties: { 38 | serverFarmId: serverfarm.id 39 | httpsOnly: true 40 | siteConfig: { 41 | alwaysOn: true 42 | appSettings: [ 43 | { 44 | name: 'WEBSITE_RUN_FROM_PACKAGE' 45 | value: '1' // Run Azure APP Service from a package file 46 | } 47 | { 48 | name: 'WEBSITE_NODE_DEFAULT_VERSION' 49 | value: '~16' // Set NodeJS version to 16.x for your site 50 | } 51 | { 52 | name: 'RUNNING_ON_AZURE' 53 | value: '1' 54 | } 55 | { 56 | name: 'BOT_ID' 57 | value: botAadAppClientId 58 | } 59 | { 60 | name: 'BOT_PASSWORD' 61 | value: botAadAppClientSecret 62 | } 63 | ] 64 | ftpsState: 'FtpsOnly' 65 | } 66 | } 67 | } 68 | 69 | // Register your web service as a bot with the Bot Framework 70 | module azureBotRegistration './botRegistration/azurebot.bicep' = { 71 | name: 'Azure-Bot-registration' 72 | params: { 73 | resourceBaseName: resourceBaseName 74 | botAadAppClientId: botAadAppClientId 75 | botAppDomain: webApp.properties.defaultHostName 76 | botDisplayName: botDisplayName 77 | } 78 | } 79 | 80 | // The output will be persisted in .env.{envName}. Visit https://aka.ms/teamsfx-actions/arm-deploy for more details. 81 | output BOT_AZURE_APP_SERVICE_RESOURCE_ID string = webApp.id 82 | output BOT_DOMAIN string = webApp.properties.defaultHostName 83 | -------------------------------------------------------------------------------- /infra/azure.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "resourceBaseName": { 6 | "value": "bot${{RESOURCE_SUFFIX}}" 7 | }, 8 | "botAadAppClientId": { 9 | "value": "${{BOT_ID}}" 10 | }, 11 | "botAadAppClientSecret": { 12 | "value": "${{SECRET_BOT_PASSWORD}}" 13 | }, 14 | "webAppSKU": { 15 | "value": "B1" 16 | }, 17 | "botDisplayName": { 18 | "value": "ListBotInfra" 19 | }, 20 | "openAIKey": { 21 | "value": "${{SECRET_OPENAI_KEY}}" 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /infra/botRegistration/azurebot.bicep: -------------------------------------------------------------------------------- 1 | @maxLength(20) 2 | @minLength(4) 3 | @description('Used to generate names for all resources in this file') 4 | param resourceBaseName string 5 | 6 | @maxLength(42) 7 | param botDisplayName string 8 | 9 | param botServiceName string = resourceBaseName 10 | param botServiceSku string = 'F0' 11 | param botAadAppClientId string 12 | param botAppDomain string 13 | 14 | // Register your web service as a bot with the Bot Framework 15 | resource botService 'Microsoft.BotService/botServices@2021-03-01' = { 16 | kind: 'azurebot' 17 | location: 'global' 18 | name: botServiceName 19 | properties: { 20 | displayName: botDisplayName 21 | endpoint: 'https://${botAppDomain}/api/messages' 22 | msaAppId: botAadAppClientId 23 | } 24 | sku: { 25 | name: botServiceSku 26 | } 27 | } 28 | 29 | // Connect the bot service to Microsoft Teams 30 | resource botServiceMsTeamsChannel 'Microsoft.BotService/botServices/channels@2021-03-01' = { 31 | parent: botService 32 | location: 'global' 33 | name: 'MsTeamsChannel' 34 | properties: { 35 | channelName: 'MsTeamsChannel' 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "JARVIB", 3 | "version": "1.0.0", 4 | "description": "Just A Rather Very Intelligent Bot that can create dynamic 3D worlds.", 5 | "author": "David Rousset", 6 | "license": "MIT", 7 | "main": "./lib/index.js", 8 | "scripts": { 9 | "build": "tsc --build && shx cp -r ./src/prompts ./lib/", 10 | "clean": "rimraf node_modules lib tsconfig.tsbuildinfo", 11 | "lint": "eslint **/src/**/*.{j,t}s{,x} --fix --no-error-on-unmatched-pattern", 12 | "start": "tsc --build && node ./lib/index.js", 13 | "test": "echo \"Error: no test specified\" && exit 1", 14 | "watch": "nodemon --watch ./src -e ts --exec \"yarn start\"", 15 | "dev:teamsfx": "nodemon --exec node --inspect=9239 --signal SIGINT -r ts-node/register ./src/index.ts" 16 | }, 17 | "repository": { 18 | "type": "git", 19 | "url": "https://github.com/davrous/JARVIB.git" 20 | }, 21 | "dependencies": { 22 | "@azure/msal-node": "^2.6.4", 23 | "@microsoft/teams-ai": "~1.1.0", 24 | "@microsoft/teams-js": "^2.14.0", 25 | "@microsoft/teamsfx": "^2.3.0", 26 | "botbuilder": "^4.22.1", 27 | "dotenv": "^16.4.2", 28 | "express": "^4.18.2", 29 | "replace": "~1.2.0", 30 | "restify": "~11.1.0", 31 | "rimraf": "^5.0.5", 32 | "socket.io": "^4.7.2" 33 | }, 34 | "devDependencies": { 35 | "@types/dotenv": "6.1.1", 36 | "@types/jsonwebtoken": "^9.0.2", 37 | "@types/lodash": "^4.14.196", 38 | "@types/node": "^20.11.19", 39 | "@types/restify": "8.5.12", 40 | "nodemon": "~1.19.4", 41 | "shx": "^0.3.4", 42 | "ts-node": "^10.9.1", 43 | "typescript": "^5.1.6" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /promptsExamples.txt: -------------------------------------------------------------------------------- 1 | create 10 cubes of different colors and place them on a circle 2 | 3 | animate the rotation of the cube 5 and 9 4 | 5 | make the cube 3 transparent and move it up by 2 units 6 | 7 | find available models for a car 8 | 9 | load the purple one 10 | 11 | scale it to 80 12 | 13 | use the cube 4 position to move the purple car just on top of it 14 | 15 | load the C7 car 16 | 17 | scale it to 80 18 | 19 | attach it to the cube 5 20 | 21 | make the cube 5 totally transparent 22 | 23 | find models for a house 24 | 25 | load a scary one 26 | 27 | scale it to 10 28 | 29 | create a yellowish ground and move it down by 1 unit 30 | 31 | create an orange spot light 32 | 33 | animate its position like around a circle 34 | 35 | slow down the animation 36 | 37 | enable VR support 38 | 39 | animate the main scene active camera to circle around the haunted house 40 | 41 | slow down the animation by 10 42 | 43 | -------------------------------------------------------------------------------- /src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/src/.DS_Store -------------------------------------------------------------------------------- /src/app.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | */ 5 | 6 | // Code used when the app is loaded in the Teams meeting 7 | // We're using the Teams JS SDK to get the context of our app in Teams 8 | 9 | const searchParams = new URL(window.location).searchParams; 10 | const root = document.getElementById("content"); 11 | let color = "white"; 12 | 13 | // STARTUP LOGIC 14 | async function start() { 15 | // Check for page to display 16 | let view = searchParams.get("view") || "stage"; 17 | 18 | // Check if we are running on stage. 19 | if (searchParams.get("inTeams")) { 20 | // Initialize teams app 21 | await microsoftTeams.app.initialize(); 22 | 23 | // Get our frameContext from context of our app in Teams 24 | const context = await microsoftTeams.app.getContext(); 25 | if (context.page.frameContext == "meetingStage") { 26 | view = "stage"; 27 | } 28 | const theme = context.app.theme; 29 | if (theme == "default") { 30 | color = "black"; 31 | } 32 | microsoftTeams.app.registerOnThemeChangeHandler(function(theme) { 33 | color = theme === "default" ? "black" : "white"; 34 | }); 35 | } 36 | 37 | // Load the requested view 38 | switch (view) { 39 | case "content": 40 | renderSideBar(root); 41 | break; 42 | case "config": 43 | renderSettings(root); 44 | break; 45 | case "stage": 46 | default: 47 | try { 48 | renderStage(root); 49 | } catch (error) { 50 | renderError(root, error); 51 | } 52 | break; 53 | } 54 | } 55 | 56 | // STAGE VIEW 57 | const stageTemplate = document.createElement("template"); 58 | 59 | stageTemplate["innerHTML"] = ` 60 | 61 | `; 62 | 63 | function renderStage(elem) { 64 | elem.appendChild(stageTemplate.content.cloneNode(true)); 65 | 66 | var __EVAL = s => eval(`void (__EVAL = ${__EVAL.toString()}); ${s}`); 67 | 68 | function evaluate(expr) { 69 | try { 70 | const result = __EVAL(expr); 71 | console.log(expr, '===>', result) 72 | } catch(err) { 73 | console.log(expr, 'ERROR:', err.message) 74 | } 75 | } 76 | 77 | var socket = io(); 78 | var scene; 79 | var camera; 80 | var light; 81 | var JARVIB_sphere; 82 | 83 | const canvas = document.getElementById("renderCanvas"); // Get the canvas element 84 | var engine = new BABYLON.Engine(canvas, true); // Generate the BABYLON 3D engine 85 | const createScene = function () { 86 | // Creates a basic Babylon Scene object 87 | scene = new BABYLON.Scene(engine); 88 | 89 | // Create a default skybox with an environment. 90 | var hdrTexture = BABYLON.CubeTexture.CreateFromPrefilteredData("https://playground.babylonjs.com/textures/environment.dds", scene); 91 | var currentSkybox = scene.createDefaultSkybox(hdrTexture, true); 92 | 93 | camera = new window.BABYLON.ArcRotateCamera( 94 | "camera", 95 | -Math.PI / 2, 96 | Math.PI / 2.5, 97 | 15, 98 | new window.BABYLON.Vector3(0, 0, 0) 99 | ); 100 | // Targets the camera to scene origin 101 | camera.setTarget(BABYLON.Vector3.Zero()); 102 | // This attaches the camera to the canvas 103 | camera.attachControl(canvas, true); 104 | // Creates a light, aiming 0,1,0 - to the sky 105 | light = new BABYLON.HemisphericLight("light", 106 | new BABYLON.Vector3(0, 1, 0), scene); 107 | // Dim the light a small amount - 0 to 1 108 | light.intensity = 0.8; 109 | 110 | JARVIB_sphere = BABYLON.MeshBuilder.CreateSphere("sphere", {diameter:2}, scene); 111 | 112 | socket.on('execute code', function(msg) { 113 | console.log(msg); 114 | evaluate(msg); 115 | }); 116 | 117 | return scene; 118 | }; 119 | 120 | // Access the microphone and create an analyser node 121 | navigator.mediaDevices.getUserMedia({ audio: true, video: false }) 122 | .then(function(stream) { 123 | var audioContext = new (window.AudioContext || window.webkitAudioContext)(); 124 | var microphone = audioContext.createMediaStreamSource(stream); 125 | var analyser = audioContext.createAnalyser(); 126 | analyser.fftSize = 1024; 127 | microphone.connect(analyser); 128 | 129 | // Create a data array to store the frequency data 130 | var dataArray = new Uint8Array(analyser.frequencyBinCount); 131 | 132 | // Create a function to animate the sphere based on the frequency data 133 | function animate() { 134 | analyser.getByteFrequencyData(dataArray); 135 | 136 | // Use the average frequency to scale the sphere 137 | var sum = dataArray.reduce(function(a, b) { return a + b; }); 138 | var avg = sum / dataArray.length; 139 | if (JARVIB_sphere) { 140 | JARVIB_sphere.scaling = new BABYLON.Vector3(avg/100, avg/100, avg/100); 141 | } 142 | 143 | requestAnimationFrame(animate); 144 | } 145 | 146 | animate(); 147 | }) 148 | .catch(function(err) { 149 | console.log('An error occurred: ' + err); 150 | }); 151 | 152 | var scene = createScene(); //Call the createScene function 153 | // Register a render loop to repeatedly render the scene 154 | engine.runRenderLoop(function () { 155 | scene.render(); 156 | }); 157 | // Watch for browser/canvas resize events 158 | window.addEventListener("resize", function () { 159 | engine.resize(); 160 | }); 161 | } 162 | 163 | // SIDEBAR VIEW 164 | const sideBarTemplate = document.createElement("template"); 165 | 166 | function renderSideBar(elem) { 167 | sideBarTemplate["innerHTML"] = ` 168 | 173 |
174 |

Lets get started

175 |

Press the share to meeting button.

176 | 177 |
178 | `; 179 | elem.appendChild(sideBarTemplate.content.cloneNode(true)); 180 | const shareButton = elem.querySelector(".share"); 181 | 182 | // Set the value at our dataKey with a random number between 1 and 6. 183 | shareButton.onclick = shareToStage; 184 | } 185 | 186 | function shareToStage() { 187 | microsoftTeams.meeting.shareAppContentToStage((error, result) => { 188 | if (!error) { 189 | console.log("Started sharing, sharedToStage result"); 190 | } else { 191 | console.warn("SharingToStageError", error); 192 | } 193 | }, window.location.origin + "?inTeams=1&view=stage"); 194 | } 195 | 196 | // SETTINGS VIEW 197 | const settingsTemplate = document.createElement("template"); 198 | 199 | function renderSettings(elem) { 200 | settingsTemplate["innerHTML"] = ` 201 | 206 |
207 |

Welcome to J.A.R.V.I.B.!

208 |

Press the save button to continue.

209 |
210 | `; 211 | elem.appendChild(settingsTemplate.content.cloneNode(true)); 212 | 213 | // Save the configurable tab 214 | microsoftTeams.pages.config.registerOnSaveHandler((saveEvent) => { 215 | microsoftTeams.pages.config.setConfig({ 216 | websiteUrl: window.location.origin, 217 | contentUrl: window.location.origin + "?inTeams=1&view=content", 218 | entityId: "J.A.R.V.I.B.", 219 | suggestedDisplayName: "J.A.R.V.I.B.", 220 | }); 221 | saveEvent.notifySuccess(); 222 | }); 223 | 224 | // Enable the Save button in config dialog 225 | microsoftTeams.pages.config.setValidityState(true); 226 | } 227 | 228 | // Error view 229 | const errorTemplate = document.createElement("template"); 230 | 231 | errorTemplate["inner" + "HTML"] = ` 232 | 237 |
238 |

Something went wrong

239 |

240 | 241 |
242 | `; 243 | 244 | function renderError(elem, error) { 245 | elem.appendChild(errorTemplate.content.cloneNode(true)); 246 | const refreshButton = elem.querySelector(".refresh"); 247 | const errorText = elem.querySelector(".error-text"); 248 | 249 | // Refresh the page on click 250 | refreshButton.onclick = () => { 251 | window.location.reload(); 252 | }; 253 | console.error(error); 254 | const errorTextContent = error.toString(); 255 | errorText.textContent = errorTextContent; 256 | } 257 | 258 | start().catch((error) => console.error(error)); -------------------------------------------------------------------------------- /src/debug.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | J.A.R.V.I.B Debug page 7 | 8 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 138 | 139 | -------------------------------------------------------------------------------- /src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | J.A.R.V.I.B. 6 | 20 | 21 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 |
44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable security/detect-object-injection */ 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License. 4 | 5 | // Import required packages 6 | import { config } from 'dotenv'; 7 | import * as path from 'path'; 8 | import * as restify from 'restify'; 9 | 10 | // Import required bot services. 11 | // See https://aka.ms/bot-services to learn more about the different parts of a bot. 12 | import { CardFactory, ConfigurationServiceClientCredentialFactory, MemoryStorage, MessageFactory, TurnContext } from 'botbuilder'; 13 | 14 | import { 15 | AI, 16 | Application, 17 | ActionPlanner, 18 | OpenAIModel, 19 | PromptManager, 20 | TurnState, 21 | DefaultConversationState, 22 | DefaultUserState, 23 | DefaultTempState, 24 | TeamsAdapter, 25 | Memory 26 | } from '@microsoft/teams-ai'; 27 | 28 | import * as responses from './responses'; 29 | import { forEach } from 'lodash'; 30 | 31 | // Read botFilePath and botFileSecret from .env file. 32 | const ENV_FILE = path.join(__dirname, '..', '.env'); 33 | config({ path: ENV_FILE }); 34 | 35 | // Create adapter. 36 | // See https://aka.ms/about-bot-adapter to learn more about how bots work. 37 | const adapter = new TeamsAdapter( 38 | {}, 39 | new ConfigurationServiceClientCredentialFactory({ 40 | MicrosoftAppId: process.env.BOT_ID, 41 | MicrosoftAppPassword: process.env.BOT_PASSWORD, 42 | MicrosoftAppType: 'MultiTenant' 43 | }) 44 | ); 45 | 46 | // Catch-all for errors. 47 | const onTurnErrorHandler = async (context: TurnContext, error: Error) => { 48 | // This check writes out errors to console log .vs. app insights. 49 | // NOTE: In production environment, you should consider logging this to Azure 50 | // application insights. 51 | console.error(`\n [onTurnError] unhandled error: ${error.toString()}`); 52 | 53 | // Send a trace activity, which will be displayed in Bot Framework Emulator 54 | await context.sendTraceActivity( 55 | 'OnTurnError Trace', 56 | `${error.toString()}`, 57 | 'https://www.botframework.com/schemas/error', 58 | 'TurnError' 59 | ); 60 | 61 | // Send a message to the user 62 | await context.sendActivity('The bot encountered an error or bug.'); 63 | await context.sendActivity('To continue to run this bot, please fix the bot source code.'); 64 | }; 65 | 66 | // Set the onTurnError for the singleton CloudAdapter. 67 | adapter.onTurnError = onTurnErrorHandler; 68 | 69 | // Create HTTP server. 70 | const server = restify.createServer(); 71 | server.use(restify.plugins.bodyParser()); 72 | 73 | server.listen(process.env.port || process.env.PORT || 3978, () => { 74 | console.log(`\n${server.name} listening to ${server.url}`); 75 | console.log('\nGet Bot Framework Emulator: https://aka.ms/botframework-emulator'); 76 | console.log('\nTo test your bot in Teams, sideload the app manifest.json within Teams Apps.'); 77 | }); 78 | 79 | if (!process.env.OPENAI_KEY && !process.env.AZURE_OPENAI_KEY) { 80 | throw new Error('Missing environment variables - please check that OPENAI_KEY or AZURE_OPENAI_KEY is set.'); 81 | } 82 | 83 | // Create AI components 84 | const model = new OpenAIModel({ 85 | // OpenAI Support 86 | // apiKey: process.env.OPENAI_KEY!, 87 | // defaultModel: 'gpt-3.5-turbo', 88 | 89 | // Azure OpenAI Support 90 | azureApiKey: process.env.AZURE_OPENAI_KEY!, 91 | azureDefaultDeployment: process.env.AZURE_OPENAI_API_MODEL!, 92 | azureEndpoint: process.env.AZURE_OPENAI_ENDPOINT!, 93 | azureApiVersion: '2024-02-15-preview', 94 | 95 | // Request logging 96 | logRequests: true 97 | }); 98 | 99 | interface fullListItem { 100 | name: string; 101 | imageUrl: string; 102 | modelUrl: string; 103 | } 104 | 105 | // Strongly type the applications turn state 106 | interface ConversationState extends DefaultConversationState { 107 | greeted: boolean; 108 | fullList: fullListItem[]; 109 | imageList: string[]; 110 | list: string[]; 111 | lastModelLoaded: string; 112 | fullCode: string; 113 | } 114 | 115 | type UserState = DefaultUserState; 116 | 117 | interface TempState extends DefaultTempState { 118 | fullList: fullListItem[]; 119 | imageList: string[]; 120 | list: string[]; 121 | } 122 | 123 | type ApplicationTurnState = TurnState; 124 | 125 | const prompts = new PromptManager({ 126 | promptsFolder: path.join(__dirname, '../src/prompts') 127 | }); 128 | 129 | const planner = new ActionPlanner({ 130 | model, 131 | prompts, 132 | defaultPrompt: 'chatGPT' 133 | }); 134 | 135 | // Define storage and application 136 | const storage = new MemoryStorage(); 137 | const app = new Application({ 138 | storage, 139 | ai: { 140 | planner 141 | } 142 | }); 143 | 144 | // Define a prompt function for getting the current status of the lights 145 | planner.prompts.addFunction('getModelsList', async (context: TurnContext, memory: Memory) => { 146 | return memory.getValue('conversation.list'); 147 | }); 148 | 149 | planner.prompts.addFunction('getLastModelLoaded', async (context: TurnContext, memory: Memory) => { 150 | return memory.getValue('conversation.lastModelLoaded'); 151 | }); 152 | 153 | planner.prompts.addFunction('getFullCode', async (context: TurnContext, memory: Memory) => { 154 | return memory.getValue('conversation.fullCode'); 155 | }); 156 | 157 | // Listen for new members to join the conversation 158 | app.conversationUpdate('membersAdded', async (context: TurnContext, state: ApplicationTurnState) => { 159 | if (!state.conversation.greeted) { 160 | state.conversation.greeted = true; 161 | await context.sendActivity(responses.greeting()); 162 | } 163 | }); 164 | 165 | // List for /reset command, then delete the conversation state, clean the object 166 | // and reload the page containing the 3D canvas to start from scratch 167 | app.message('/reset', async (context: TurnContext, state: ApplicationTurnState) => { 168 | state.deleteConversationState(); 169 | state.conversation.list = []; 170 | state.conversation.fullList = []; 171 | state.conversation.imageList = []; 172 | state.conversation.lastModelLoaded = ""; 173 | state.conversation.fullCode = ""; 174 | io.emit('execute code', "location.reload(true);"); 175 | await context.sendActivity(responses.reset()); 176 | }); 177 | 178 | // List for /describe command to visually describe the complete scene to someone who is blind 179 | app.message('/describe', async (context: TurnContext, state: ApplicationTurnState) => { 180 | await app.ai.doAction(context, state, 'describe'); 181 | }); 182 | 183 | // List for /fullcode to return all the code generated so far by the bot if you want to copy it 184 | app.message('/fullcode', async (context: TurnContext, state: ApplicationTurnState) => { 185 | await context.sendActivity(state.conversation.fullCode); 186 | }); 187 | 188 | // Define an interface to strongly type data parameters for actions 189 | interface GetModel { 190 | nameOfTheModel: string; // <- populated by GPT 191 | } 192 | 193 | // Define an interface to strongly type data parameters for actions 194 | interface GetCode { 195 | code: string; // <- populated by GPT 196 | } 197 | 198 | // Register action handlers 199 | app.ai.action('codeToExecute', async (context: TurnContext, state: ApplicationTurnState, codeToExecute: GetCode) => { 200 | let code = ""; 201 | if (codeToExecute && codeToExecute.code) { 202 | code = codeToExecute.code; 203 | io.emit('execute code', code); 204 | state.conversation.fullCode += code + "\n"; 205 | }; 206 | console.dir(codeToExecute.code); 207 | await context.sendActivity(`
${codeToExecute.code}
`); 208 | 209 | return ''; 210 | }); 211 | 212 | interface Item { 213 | name: string; 214 | imageUrl: string; 215 | } 216 | 217 | interface TextBlock { 218 | type: "TextBlock"; 219 | text: string; 220 | weight?: "bolder"; 221 | size?: "large"; 222 | } 223 | 224 | interface Image { 225 | type: "Image"; 226 | url: string; 227 | width: string; 228 | horizontalAlignment: "left"; 229 | } 230 | 231 | interface Container { 232 | type: "Container"; 233 | items: (TextBlock | Image)[]; 234 | } 235 | 236 | interface AdaptiveCard { 237 | $schema: string; 238 | version: string; 239 | type: "AdaptiveCard"; 240 | body: (TextBlock | Container)[]; 241 | } 242 | 243 | app.ai.action('listAvailableModel', async (context: TurnContext, state: ApplicationTurnState, model: GetModel) => { 244 | console.dir(model); 245 | var modelName = model.nameOfTheModel ?? (model).model; 246 | var jsonRequest = 247 | { 248 | "type":"Search", 249 | "pageSize":5, 250 | "query":modelName, 251 | "parameters":{"firstpartycontent":false,"app":"office"}, 252 | "descriptor":{"$type":"FirstPartyContentSearchDescriptor"} 253 | } 254 | // create a POST request to the server with a JSON parameter 255 | // that contains the model name 256 | const response = await fetch('https://hubble.officeapps.live.com/mediasvc/api/media/search?v=1&lang=en-us', { 257 | method: 'POST', 258 | headers: { 259 | 'Content-Type': 'application/json' 260 | }, 261 | body: JSON.stringify(jsonRequest) 262 | }); 263 | const content = await response.json(); 264 | 265 | let items: Item[] = []; 266 | 267 | if (content.Result && content.Result.PartGroups.length > 0) { 268 | state.conversation.list = []; 269 | state.conversation.fullList = []; 270 | state.conversation.imageList = []; 271 | var list = state.conversation.list; 272 | var fullList = state.conversation.fullList; 273 | //var imageList = state.conversation.imageList; 274 | 275 | var results = content.Result.PartGroups; 276 | forEach(results, function(value) { 277 | var image = value.ImageParts[0].SourceUrl; 278 | var title; 279 | var url; 280 | forEach(value.TextParts, function(text) { 281 | if (text.TextCategory == "Title") { 282 | title = text.Text; 283 | } 284 | if (text.TextCategory == "OasisGlbLink") { 285 | url = text.Text; 286 | } 287 | }); 288 | if (title && url && image) { 289 | // imageList.push(image); 290 | list.push(title); 291 | items.push({ name: title, imageUrl: image }) 292 | fullList.push({name: title, imageUrl: image, modelUrl: url}); 293 | } 294 | }); 295 | 296 | const adaptiveCardSchema: AdaptiveCard = { 297 | "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", 298 | "version": "1.3", 299 | "type": "AdaptiveCard", 300 | "body": [ 301 | { 302 | "type": "TextBlock", 303 | "text": "Available models", 304 | "weight": "bolder", 305 | "size": "large" 306 | }, 307 | { 308 | "type": "Container", 309 | "items": [] 310 | } 311 | ] 312 | }; 313 | 314 | function insertItems(schema: AdaptiveCard, items: Item[]): void { 315 | const container = schema.body.find(element => element.type === "Container") as Container; 316 | if (!container) { 317 | console.error('Container element not found in the schema'); 318 | return; 319 | } 320 | 321 | container.items = []; // Clear existing items 322 | 323 | items.forEach(item => { 324 | // Add TextBlock for the item name 325 | container.items.push({ 326 | "type": "TextBlock", 327 | "text": `* ${item.name}` 328 | }); 329 | 330 | // Add Image for the item 331 | container.items.push({ 332 | "type": "Image", 333 | "url": item.imageUrl, 334 | "width": "100px", 335 | "horizontalAlignment": "left" 336 | }); 337 | }); 338 | } 339 | 340 | // Call the function to insert items 341 | insertItems(adaptiveCardSchema, items); 342 | 343 | // Log the modified schema to see the result 344 | console.log(JSON.stringify(adaptiveCardSchema, null, 2)); 345 | 346 | const attachment = CardFactory.adaptiveCard(adaptiveCardSchema); 347 | await context.sendActivity(MessageFactory.attachment(attachment)); 348 | return 'We found available models, you can stop there'; 349 | } 350 | else { 351 | return 'No model found, try to find another one closer to the requested name'; 352 | } 353 | }); 354 | 355 | app.ai.action('loadThisModel', async (context: TurnContext, state: ApplicationTurnState, model: GetModel) => { 356 | const modelsList = state.conversation.list; 357 | var modelName = model.nameOfTheModel ?? (model).model; 358 | let index: number; 359 | // If the user would like to load a specific model via its index in the list 360 | if (!isNaN(Number.parseInt(modelName))) { 361 | index = Number.parseInt(modelName); 362 | } 363 | // Otherwise, we look for the model name in the list 364 | else { 365 | index = modelsList.indexOf(modelName); 366 | } 367 | // If the model is found, we load it 368 | if (index >= 0) { 369 | var modelToLoad = state.conversation.fullList[index]; 370 | var fullUrl = modelToLoad.modelUrl; 371 | let lastSlash = fullUrl.lastIndexOf("/"); 372 | let baseUrl = fullUrl.substring(0, lastSlash+1); 373 | let fileName = fullUrl.substring(lastSlash+1, fullUrl.length); 374 | var code = `BABYLON.SceneLoader.ImportMesh("", "${baseUrl}", "${fileName}", scene, function (newMeshes) { 375 | newMeshes[0].name = "${modelsList[index]}"; 376 | newMeshes[0].scaling = new BABYLON.Vector3(30, 30, 30); 377 | });`; 378 | await context.sendActivity(responses.itemFound(modelsList[index], code)); 379 | io.emit('execute code', code); 380 | state.conversation.fullCode += code + "\n"; 381 | state.conversation.lastModelLoaded = modelsList[index]; 382 | return state.conversation.lastModelLoaded + ' model successfully loaded, you can stop there'; 383 | } else { 384 | await context.sendActivity(responses.itemNotFound(modelName)); 385 | return 'No model found, try to find another one closer to the concept of the request one'; 386 | } 387 | }); 388 | 389 | // Register a handler to handle unknown actions that might be predicted 390 | app.ai.action( 391 | AI.UnknownActionName, 392 | async (context: TurnContext, state: ApplicationTurnState, data: GetCode, action?: string) => { 393 | await context.sendActivity(responses.unknownAction(action!)); 394 | return 'Sorry, this is an unknown action available'; 395 | } 396 | ); 397 | 398 | // Listen for incoming server requests. 399 | server.post('/api/messages', async (req, res) => { 400 | // Route received a request to adapter for processing 401 | await adapter.process(req, res as any, async (context) => { 402 | // Dispatch to application for routing 403 | await app.run(context); 404 | }); 405 | }); 406 | 407 | // WebSocket server part 408 | const express = require('express'); 409 | const appSocket = express(); 410 | const http = require('http'); 411 | const serverSocket = http.createServer(appSocket); 412 | const { Server } = require("socket.io"); 413 | const io = new Server(serverSocket); 414 | 415 | appSocket.get('/', (req: any, res: any) => { 416 | res.sendFile(__dirname + '/index.html'); 417 | }); 418 | 419 | appSocket.get('/debug.html', (req: any, res: any) => { 420 | res.sendFile(__dirname + '/debug.html'); 421 | }); 422 | 423 | appSocket.get('/app.js', (req: any, res: any) => { 424 | res.sendFile(__dirname + '/app.js'); 425 | }); 426 | 427 | io.on('connection', (socket: any) => { 428 | console.log('a user connected'); 429 | }); 430 | 431 | serverSocket.listen(3000, () => { 432 | console.log('WebSocket server listening on *:3000'); 433 | }); 434 | 435 | 436 | 437 | 438 | 439 | 440 | 441 | -------------------------------------------------------------------------------- /src/prompts/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davrous/JARVIB/f3197db6ad9d6c9691f3c392d2ed9aeb87720d6f/src/prompts/.DS_Store -------------------------------------------------------------------------------- /src/prompts/chatGPT/actions.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "codeToExecute", 4 | "description": "Returns the Babylon.js JavaScript code matching the user intent", 5 | "parameters": { 6 | "type": "object", 7 | "properties": { 8 | "code": { 9 | "type": "string", 10 | "description": "The JavaScript code to execute next" 11 | } 12 | }, 13 | "required": [ 14 | "code" 15 | ] 16 | } 17 | }, 18 | { 19 | "name": "listAvailableModel", 20 | "description": "List the available 3D models we can load from the library", 21 | "parameters": { 22 | "type": "object", 23 | "properties": { 24 | "nameOfTheModel": { 25 | "type": "string", 26 | "description": "The name of the model to search inside the library" 27 | } 28 | }, 29 | "required": [ 30 | "nameOfTheModel" 31 | ] 32 | } 33 | }, 34 | { 35 | "name": "loadThisModel", 36 | "description": "Load the 3D model specified by the user", 37 | "parameters": { 38 | "type": "object", 39 | "properties": { 40 | "nameOfTheModel": { 41 | "type": "string", 42 | "description": "The name of the model to load from the library" 43 | } 44 | }, 45 | "required": [ 46 | "nameOfTheModel" 47 | ] 48 | } 49 | } 50 | ] -------------------------------------------------------------------------------- /src/prompts/chatGPT/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": 1.1, 3 | "description": "A bot that can generate Babylon.js code or display a list of available models to load.", 4 | "type": "completion", 5 | "completion": { 6 | "completion_type": "chat", 7 | "max_input_tokens": 2800, 8 | "max_tokens": 1000, 9 | "temperature": 0.5, 10 | "top_p": 0.0, 11 | "presence_penalty": 0.6, 12 | "frequency_penalty": 0.0, 13 | "stop_sequences": [] 14 | }, 15 | "augmentation": { 16 | "augmentation_type": "monologue" 17 | } 18 | } -------------------------------------------------------------------------------- /src/prompts/chatGPT/skprompt.txt: -------------------------------------------------------------------------------- 1 | Pretend you're an expert in Babylon.js, the JavaScript WebGL 3D engine. 2 | 3 | rules: 4 | - assume there is already an existing Babylon.js scene and engine so you don't have to create them 5 | - just generate the code to add into an existing program. 6 | - use the scene and engine objects directly. 7 | - pay attention when trying to access previously created Meshes by getting access to them via their name rather than assuming the associated variable is already created 8 | - when writing a new code, consider all the previous one you've generated to be sure the new code will be consistent with the previous one. 9 | - remember about the already created meshes, animations or any other specific ressources before trying to create them or reuse them. 10 | 11 | Here is the current list of available models to load: 12 | 13 | ``` 14 | {{getModelsList}} 15 | ``` 16 | 17 | Current mesh model name loaded: 18 | 19 | ``` 20 | {{getLastModelLoaded}} 21 | ``` 22 | 23 | Code executed so far: 24 | 25 | ``` 26 | {{getFullCode}} 27 | ``` -------------------------------------------------------------------------------- /src/prompts/describe/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": 1, 3 | "description": "A bot visually describing a scene create by Babylon.js code for visually impaired persons.", 4 | "type": "completion", 5 | "completion": { 6 | "max_tokens": 1024, 7 | "temperature": 0.7, 8 | "top_p": 0.0, 9 | "presence_penalty": 0.6, 10 | "frequency_penalty": 0.0 11 | }, 12 | "default_backends": [ 13 | "davinci" 14 | ] 15 | } -------------------------------------------------------------------------------- /src/prompts/describe/skprompt.txt: -------------------------------------------------------------------------------- 1 | Explain what the below Babylon.js code does. Try to be as visually descritive as possible to that a blind person car understand. 2 | 3 | ``` 4 | {{$conversation.fullCode}} 5 | ``` 6 | 7 | Explanation: 8 | -------------------------------------------------------------------------------- /src/responses.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | 4 | /* 5 | All of these responses where generated by GPT using a prompt similar to: 6 | 7 | ``` 8 | Here's a JavaScript string literal template: 9 | 10 | `I couldn't find a ${item} on your ${list} list.` 11 | 12 | Create a JavaScript array with 7 variations of the template. 13 | The variations should be helpful, creative, clever, and very friendly. 14 | The variations should always use the back tick `` syntax. 15 | The variations should always include ${item} and ${list} variables. 16 | ``` 17 | 18 | 7 variations were asked for so that we can remove the 2 we like the least. 19 | */ 20 | 21 | /** 22 | * 23 | */ 24 | export function greeting(): string { 25 | return getRandomResponse([ 26 | "Hi! I'm J.A.R.V.I.B, I'm here to help you building dynamic 3D worlds. Ask me to create Babylon.js code or to find and load some 3D models. Type /reset to clean the context." 27 | ]); 28 | } 29 | 30 | /** 31 | * 32 | */ 33 | export function reset(): string { 34 | return getRandomResponse([ 35 | 'Resetting the context. A fresh world is available to you now.', 36 | 'Starting fresh. The context has been cleaned.', 37 | "I've cleaned the house. Ready for new ideas!", 38 | "Cleaning slate. Let's rebuild the world." 39 | ]); 40 | } 41 | 42 | /** 43 | * Generates a response message for when an item is not found on a list. 44 | * @param {string} list The name of the list. 45 | * @param {string} item The name of the item. 46 | * @returns {string} The response message. 47 | */ 48 | export function itemNotFound(item: string): string { 49 | return getRandomResponse([ 50 | `I'm sorry, I couldn't locate a ${item} on your list.`, 51 | `I don't see a ${item} on your list.`, 52 | `It looks like you don't have a ${item} on your list.`, 53 | `I'm sorry, I don't see a ${item} on your list.`, 54 | `I couldn't find a ${item} listed on your list.` 55 | ]); 56 | } 57 | 58 | /** 59 | * Generates a response message for when an item is found on a list. 60 | * @param {string} list The name of the list. 61 | * @param {string} item The name of the item. 62 | * @returns {string} The response message. 63 | */ 64 | export function itemFound(item: string, code: string): string { 65 | return getRandomResponse([ 66 | `Loading model ${item} from your list using code: ${code}`, 67 | `We're loading the ${item} model using following code: ${code}`, 68 | `Ok, we found a ${item} model and we're loading it using this code: ${code}` 69 | ]); 70 | } 71 | 72 | /** 73 | * @param action 74 | */ 75 | export function unknownAction(action: string): string { 76 | return getRandomResponse([ 77 | `I'm sorry, I'm not sure how to ${action}.`, 78 | `I don't know the first thing about ${action}.`, 79 | `I'm not sure I'm the best person to help with ${action}.`, 80 | `I'm still learning about ${action}, but I'll try my best.`, 81 | `I'm afraid I'm not experienced enough with ${action}.` 82 | ]); 83 | } 84 | 85 | /** 86 | * 87 | */ 88 | export function offTopic(): string { 89 | return getRandomResponse([ 90 | `I'm sorry, I'm not sure I can help you with that.`, 91 | `I'm sorry, I'm afraid I'm not allowed to talk about such things.`, 92 | `I'm sorry, I'm not sure I'm the right person to help you with that.`, 93 | `I wish I could help you with that, but it's not something I can talk about.`, 94 | `I'm sorry, I'm not allowed to discuss that topic.` 95 | ]); 96 | } 97 | 98 | /** 99 | * @param responses 100 | */ 101 | function getRandomResponse(responses: string[]): string { 102 | const i = Math.floor(Math.random() * (responses.length - 1)); 103 | // eslint-disable-next-line security/detect-object-injection 104 | return responses[i]; 105 | } 106 | -------------------------------------------------------------------------------- /teamsapp.local.yml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://aka.ms/teams-toolkit/1.0.0/yaml.schema.json 2 | # 3 | # The teamsapp.local.yml composes automation tasks for Teams Toolkit when running locally. 4 | # This file is used when running Start Debugging (F5) from Visual Studio Code or with the TeamsFx CLI commands. 5 | # i.e. `teamsfx provision --env local` or `teamsfx deploy --env local`. 6 | # 7 | # You can customize this file. Visit https://aka.ms/teamsfx-v5.0-guide for more info about Teams Toolkit project files. 8 | version: 1.0.0 9 | 10 | environmentFolderPath: ./env 11 | 12 | # Defines what the `provision` lifecycle step does with Teams Toolkit. 13 | # Runs first during Start Debugging (F5) or run manually using `teamsfx provision --env local`. 14 | provision: 15 | 16 | # Automates the creation of a Teams app registration and saves the App ID to an environment file. 17 | - uses: teamsApp/create 18 | with: 19 | name: JARVIB-${{TEAMSFX_ENV}} 20 | writeToEnvironmentFile: 21 | teamsAppId: TEAMS_APP_ID 22 | 23 | # Automates the creation an Azure AD app registration which is required for a bot. 24 | # The Bot ID (AAD app client ID) and Bot Password (AAD app client secret) are saved to an environment file. 25 | - uses: botAadApp/create 26 | with: 27 | name: JARVIB-${{TEAMSFX_ENV}} 28 | writeToEnvironmentFile: 29 | botId: BOT_ID 30 | botPassword: SECRET_BOT_PASSWORD 31 | 32 | # Automates the creation and configuration of a Bot Framework registration which is required for a bot. 33 | # This configures the bot to use the Azure AD app registration created in the previous step. 34 | # Teams Toolkit automatically creates a local Dev Tunnel URL and updates BOT_ENDPOINT when debugging (F5). 35 | - uses: botFramework/create 36 | with: 37 | botId: ${{BOT_ID}} 38 | name: JARVIB 39 | messagingEndpoint: ${{BOT_ENDPOINT}}/api/messages 40 | description: "" 41 | channels: 42 | - name: msteams 43 | 44 | # Optional: Automates schema and error checking of the Teams app manifest and outputs the results in the console. 45 | - uses: teamsApp/validateManifest 46 | with: 47 | manifestPath: ./appPackage/manifest.json 48 | 49 | # Automates the creation of a Teams app package (.zip). 50 | - uses: teamsApp/zipAppPackage 51 | with: 52 | manifestPath: ./appPackage/manifest.json 53 | outputZipPath: ./appPackage/build/appPackage.${{TEAMSFX_ENV}}.zip 54 | outputJsonPath: ./appPackage/build/manifest.${{TEAMSFX_ENV}}.json 55 | 56 | # Automates updating the Teams app manifest in Teams Developer Portal using the App ID from the mainfest file. 57 | # This action ensures that any manifest changes are reflected when launching the app again in Teams. 58 | - uses: teamsApp/update 59 | with: 60 | # Relative path to this file. This is the path for built zip file. 61 | appPackagePath: ./appPackage/build/appPackage.${{TEAMSFX_ENV}}.zip 62 | 63 | # Defines what the `deploy` lifecycle step does with Teams Toolkit. 64 | # Runs after `provision` during Start Debugging (F5) or run manually using `teamsfx deploy --env local`. 65 | deploy: 66 | # Provides the Teams Toolkit .env file values to the apps runtime so they can be accessed with `process.env`. 67 | - uses: file/createOrUpdateEnvironmentFile 68 | with: 69 | target: ./.env 70 | envs: 71 | BOT_ID: ${{BOT_ID}} 72 | BOT_PASSWORD: ${{SECRET_BOT_PASSWORD}} 73 | -------------------------------------------------------------------------------- /teamsapp.yml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://aka.ms/teams-toolkit/1.0.0/yaml.schema.json 2 | # 3 | # The teamsapp.yml composes automation tasks for Teams Toolkit when running other environment configurations. 4 | # This file is used when selecting the Provision, Deploy, or Publish menu items in the Teams Toolkit for Visual Studio Code window 5 | # or with the TeamsFx CLI commands. 6 | # i.e. `teamsfx provision --env {environment name}` or `teamsfx deploy --env {environment name}`. 7 | # 8 | # You can customize this file. Visit https://aka.ms/teamsfx-v5.0-guide for more info about Teams Toolkit project files. 9 | version: 1.0.0 10 | 11 | environmentFolderPath: ./env 12 | 13 | # Defines what the `provision` lifecycle step does with Teams Toolkit. 14 | # Runs with the Provision menu or CLI using `teamsfx provision --env {environment name}`. 15 | provision: 16 | 17 | # Automates the creation of a Teams app registration and saves the App ID to an environment file. 18 | - uses: teamsApp/create 19 | with: 20 | name: JARVIB-${{TEAMSFX_ENV}} 21 | writeToEnvironmentFile: 22 | teamsAppId: TEAMS_APP_ID 23 | 24 | # Automates the creation an Azure AD app registration which is required for a bot. 25 | # The Bot ID (AAD app client ID) and Bot Password (AAD app client secret) are saved to an environment file. 26 | - uses: botAadApp/create 27 | with: 28 | name: JARVIB-${{TEAMSFX_ENV}} 29 | writeToEnvironmentFile: 30 | botId: BOT_ID 31 | botPassword: SECRET_BOT_PASSWORD 32 | 33 | # Automates the creation of infrastructure defined in ARM templates to host the bot. 34 | # The created resource IDs are saved to an environment file. 35 | - uses: arm/deploy 36 | with: 37 | subscriptionId: ${{AZURE_SUBSCRIPTION_ID}} 38 | resourceGroupName: ${{AZURE_RESOURCE_GROUP_NAME}} 39 | templates: 40 | - path: ./infra/azure.bicep 41 | parameters: ./infra/azure.parameters.json 42 | deploymentName: Create-resources-for-tab 43 | bicepCliVersion: v0.9.1 44 | 45 | # Optional: Automates schema and error checking of the Teams app manifest and outputs the results in the console. 46 | - uses: teamsApp/validateManifest 47 | with: 48 | manifestPath: ./appPackage/manifest.json 49 | 50 | # Automates creating a final app package (.zip) by replacing any variables in the manifest.json file for the current environment. 51 | - uses: teamsApp/zipAppPackage 52 | with: 53 | manifestPath: ./appPackage/manifest.json 54 | outputZipPath: ./appPackage/build/appPackage.${{TEAMSFX_ENV}}.zip 55 | outputJsonPath: ./appPackage/build/manifest.${{TEAMSFX_ENV}}.json 56 | 57 | # Optional: Automates an app package check for errors that would prevent the app from being published and reports any problems. 58 | - uses: teamsApp/validateAppPackage 59 | with: 60 | appPackagePath: ./appPackage/build/appPackage.${{TEAMSFX_ENV}}.zip 61 | 62 | # Automates updating the Teams app manifest in Teams Developer Portal using the App ID from the mainfest file. 63 | # This action ensures that any manifest changes are reflected when launching the app again in Teams. 64 | - uses: teamsApp/update 65 | with: 66 | appPackagePath: ./appPackage/build/appPackage.${{TEAMSFX_ENV}}.zip 67 | 68 | # Defines what the `deploy` lifecycle step does with Teams Toolkit. 69 | # Runs with the Deploy menu or CLI using `teamsfx deploy --env {environment name}`. 70 | deploy: 71 | - uses: script 72 | with: 73 | run: "yarn install" 74 | timeout: 180000 # timeout in ms 75 | 76 | - uses: script 77 | with: 78 | run: "yarn build" 79 | timeout: 180000 # timeout in ms 80 | 81 | # Deploy to an Azure App Service using the zip file created in the provision step. 82 | - uses: azureAppService/zipDeploy 83 | with: 84 | artifactFolder: . 85 | ignoreFile: .webappignore 86 | # This example uses the env var thats generated by the arm/deploy action. 87 | # You can replace it with an existing Azure Resource ID or other 88 | # custom environment variable. 89 | resourceId: ${{BOT_AZURE_APP_SERVICE_RESOURCE_ID}} 90 | 91 | # Defines what the `publish` lifecycle step does with Teams Toolkit. 92 | # Runs with the Deploy menu or CLI using `teamsfx publish --env {environment name}`. 93 | publish: 94 | 95 | # Optional: Automates schema and error checking of the Teams app manifest and outputs the results in the console. 96 | - uses: teamsApp/validateManifest 97 | with: 98 | manifestPath: ./appPackage/manifest.json 99 | 100 | # Automates creating a final app package (.zip) by replacing any variables in the manifest.json file for the current environment. 101 | - uses: teamsApp/zipAppPackage 102 | with: 103 | manifestPath: ./appPackage/manifest.json 104 | outputZipPath: ./appPackage/build/appPackage.${{TEAMSFX_ENV}}.zip 105 | outputJsonPath: ./appPackage/build/manifest.${{TEAMSFX_ENV}}.json 106 | 107 | # Optional: Automates an app package check for errors that would prevent the app from being published and reports any problems. 108 | - uses: teamsApp/validateAppPackage 109 | with: 110 | appPackagePath: ./appPackage/build/appPackage.${{TEAMSFX_ENV}}.zip 111 | 112 | # Automates updating the Teams app manifest in Teams Developer Portal using the App ID from the mainfest file. 113 | # This action ensures that any manifest changes are reflected when launching the app again in Teams. 114 | - uses: teamsApp/update 115 | with: 116 | appPackagePath: ./appPackage/build/appPackage.${{TEAMSFX_ENV}}.zip 117 | projectId: a6ef6a86-ec78-4cf0-bc78-4091921b6a7c 118 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "declaration": true, 4 | "forceConsistentCasingInFileNames": true, 5 | "incremental": true, 6 | "module": "commonjs", 7 | "outDir": "./lib", 8 | "rootDir": "./src", 9 | "sourceMap": true, 10 | "strict": true, 11 | "target": "es2017", 12 | "tsBuildInfoFile": "./lib/.tsbuildinfo" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /web.config: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 59 | 60 | 61 | 62 | --------------------------------------------------------------------------------