├── .nvmrc ├── .prettierrc ├── docs ├── ai-assistants.png └── ai-assistants-light.png ├── tsconfig.json ├── package.json ├── CONTRIBUTING.md ├── .env.example ├── LICENSE ├── functions ├── channels │ ├── voice │ │ └── incoming-call.protected.js │ ├── messaging │ │ ├── response.js │ │ └── incoming.protected.js │ └── conversations │ │ ├── response.js │ │ ├── messageAdded.protected.js │ │ └── flex-webchat.protected.js └── tools │ ├── ui-tools.js │ ├── studio-handover.js │ ├── google-maps.js │ ├── internet-search.js │ └── flex-handover.js ├── .gitignore ├── .twilioserverlessrc ├── assets ├── utils.private.js └── index.html └── README.md /.nvmrc: -------------------------------------------------------------------------------- 1 | 18 -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "tabWidth": 2, 3 | "useTabs": false 4 | } -------------------------------------------------------------------------------- /docs/ai-assistants.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twilio-labs/ai-assistants-samples/HEAD/docs/ai-assistants.png -------------------------------------------------------------------------------- /docs/ai-assistants-light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twilio-labs/ai-assistants-samples/HEAD/docs/ai-assistants-light.png -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowJs": true 4 | }, 5 | "include": ["node_modules/@twilio-labs/serverless-runtime-types/index.d.ts"] 6 | } 7 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ai-assistants-samples", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "test": "echo \"Error: no test specified\" && exit 1", 7 | "start": "twilio-run", 8 | "deploy": "twilio-run deploy" 9 | }, 10 | "dependencies": { 11 | "@langchain/community": "^0.0.32", 12 | "@langchain/core": "^0.1.32", 13 | "@langchain/openai": "^0.0.14", 14 | "@twilio/runtime-handler": "1.3.0", 15 | "exa-js": "^1.0.12", 16 | "jsonwebtoken": "^9.0.2", 17 | "langchain": "^0.1.21", 18 | "twilio": "^3.56" 19 | }, 20 | "devDependencies": { 21 | "twilio-run": "^3.5.4" 22 | }, 23 | "engines": { 24 | "node": "18" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | This project uses the [Twilio Serverless Toolkit](https://www.twilio.com/docs/labs/serverless-toolkit) for all development. Check the [documentation](https://www.twilio.com/docs/labs/serverless-toolkit/general-usage) for any general questions about the tool. 4 | 5 | ## Setup 6 | 7 | Follow the setup instructions in the [`README`](README.md). 8 | 9 | ## Starting a local server to test your changes 10 | 11 | ```bash 12 | twilio serverless:start 13 | ``` 14 | 15 | If you are using Functions that will reference other Functions make sure you update the `.env` file to contain a `DOMAIN_NAME` that points against the domain of your ngrok tunnel. Example: 16 | 17 | ``` 18 | DOMAIN_NAME=example.ngrok.io 19 | ``` 20 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | # description: The default Assistant SID you want to use 2 | # format: sid 3 | # required: true 4 | ASSISTANT_SID= 5 | 6 | # description: The API key for Google Maps if you want to use the tool. 7 | # format: secret 8 | # required: false 9 | GOOGLE_MAPS_API_KEY= 10 | 11 | # description: The TaskRouter Workspace SID for handing over conversations to a human 12 | # format: sid 13 | # required: false 14 | FLEX_WORKSPACE_SID= 15 | 16 | # description: The TaskRouter Workflow SID for handing over conversations to a human 17 | # format: sid 18 | # required: false 19 | FLEX_WORKFLOW_SID= 20 | 21 | # description: The default Studio Flow you want to hand the conversation over to 22 | # format: sid 23 | # required: false 24 | STUDIO_FLOW_SID= 25 | 26 | # description: An API key for Exa.ai, a search engine for LLMs. Required for the Internet Search tool. 27 | # format: secret 28 | # required: false 29 | EXA_API_KEY= 30 | 31 | # description: An API key for OpenAI 32 | # format: secret 33 | # required: false 34 | OPENAI_API_KEY= -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Twilio Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /functions/channels/voice/incoming-call.protected.js: -------------------------------------------------------------------------------- 1 | const { 2 | getAssistantSid, 3 | } = require(Runtime.getAssets()["/utils.js"].path); 4 | 5 | /** 6 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 7 | * @param {{}} event 8 | * @param {import('@twilio-labs/serverless-runtime-types/types').ServerlessCallback} callback 9 | */ 10 | exports.handler = async function(context, event, callback) { 11 | const assistantSid = await getAssistantSid(context, event); 12 | let greeting = event.greeting; 13 | if (greeting == '$VOICE_GREETING') { 14 | // Modify this parameter to customize your Assistant's default greeting 15 | greeting = "Thanks for calling; how can I help you?" 16 | } 17 | const twiml = ` 18 | 19 | 20 | 24 | 25 | 26 | `; 27 | 28 | const response = new Twilio.Response(); 29 | response.appendHeader('Content-Type', 'text/xml'); 30 | response.setBody(twiml); 31 | 32 | return callback(null, response); 33 | }; 34 | -------------------------------------------------------------------------------- /functions/channels/messaging/response.js: -------------------------------------------------------------------------------- 1 | const { verifyRequest } = require(Runtime.getAssets()["/utils.js"].path); 2 | 3 | /** 4 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 5 | * @param {{}} event 6 | * @param {import('@twilio-labs/serverless-runtime-types/types').ServerlessCallback} callback 7 | */ 8 | exports.handler = async function (context, event, callback) { 9 | try { 10 | if (!verifyRequest(context, event)) { 11 | console.error("Invalid token", event._token); 12 | return callback(new Error("Invalid token")); 13 | } 14 | 15 | const client = context.getTwilioClient(); 16 | let from = event.Identity; 17 | if (from.startsWith("phone:")) { 18 | from = from.replace("phone:", ""); 19 | } 20 | 21 | const [to] = event.SessionId.replace("webhook:messaging__", "").split("/"); 22 | const body = event.Body; 23 | 24 | const message = await client.messages.create({ 25 | from: to, 26 | to: from, 27 | body, 28 | }); 29 | 30 | console.log(`message sent ${message.sid}`); 31 | 32 | callback(null, {}); 33 | } catch (err) { 34 | console.error(err); 35 | return callback(null, {}); 36 | } 37 | }; 38 | -------------------------------------------------------------------------------- /functions/tools/ui-tools.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 3 | * @param {{}} event 4 | * @param {import('@twilio-labs/serverless-runtime-types/types').ServerlessCallback} callback 5 | */ 6 | exports.handler = async function (context, event, callback) { 7 | try { 8 | if ( 9 | !event.request.headers["x-session-id"]?.startsWith( 10 | "webhook:conversations__" 11 | ) 12 | ) { 13 | return callback(null, "Unable to perform action. Ignore this output"); 14 | } 15 | 16 | const client = context.getTwilioClient(); 17 | const [serviceSid, conversationsSid] = event.request.headers["x-session-id"] 18 | ?.replace("webhook:conversations__", "") 19 | .split("/"); 20 | 21 | const data = { ...event }; 22 | delete data.request; 23 | delete data.toolName; 24 | delete data.successMessage; 25 | const payload = { 26 | name: event.toolName, 27 | data, 28 | }; 29 | 30 | await client.conversations.v1 31 | .services(serviceSid) 32 | .conversations(conversationsSid) 33 | .messages.create({ 34 | attributes: JSON.stringify({ assistantMessageType: "ui-tool" }), 35 | body: JSON.stringify(payload), 36 | }); 37 | 38 | return callback(null, event.successMessage ?? "Success"); 39 | } catch (err) { 40 | console.error(err); 41 | return callback(null, "An error ocurred"); 42 | } 43 | }; 44 | -------------------------------------------------------------------------------- /functions/channels/messaging/incoming.protected.js: -------------------------------------------------------------------------------- 1 | const { 2 | signRequest, 3 | getAssistantSid, 4 | sendMessageToAssistant, 5 | } = require(Runtime.getAssets()["/utils.js"].path); 6 | const crypto = require("crypto"); 7 | 8 | /** 9 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 10 | * @param {{}} event 11 | * @param {import('@twilio-labs/serverless-runtime-types/types').ServerlessCallback} callback 12 | */ 13 | exports.handler = async function (context, event, callback) { 14 | const twiml = new Twilio.twiml.MessagingResponse(); 15 | 16 | const assistantSid = await getAssistantSid(context, event); 17 | 18 | // using cookies for sessions so that a session persists for 4h (lifetime of cookie) 19 | const sessionId = 20 | event.request.cookies["SESSION_ID"] || 21 | `messaging__${event.To}/${crypto.randomUUID()}`; 22 | 23 | const token = await signRequest(context, event); 24 | const body = { 25 | body: event.Body, 26 | identity: event.From.startsWith("whatsapp:") 27 | ? event.From 28 | : `phone:${event.From}`, 29 | session_id: sessionId, 30 | webhook: `https://${context.DOMAIN_NAME}/channels/messaging/response?_token=${token}`, 31 | }; 32 | 33 | const response = new Twilio.Response(); 34 | response.setCookie("SESSION_ID", sessionId); 35 | response.appendHeader("content-type", "text/xml"); 36 | response.setBody(twiml.toString()); 37 | 38 | try { 39 | await sendMessageToAssistant(context, assistantSid, body); 40 | } catch (err) { 41 | console.error(err); 42 | } 43 | 44 | callback(null, response); 45 | }; 46 | -------------------------------------------------------------------------------- /functions/tools/studio-handover.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 3 | * @param {{}} event 4 | * @param {import('@twilio-labs/serverless-runtime-types/types').ServerlessCallback} callback 5 | */ 6 | exports.handler = async function (context, event, callback) { 7 | try { 8 | if ( 9 | !event.request.headers["x-session-id"]?.startsWith( 10 | "webhook:conversations__" 11 | ) 12 | ) { 13 | return callback(null, "Unable to perform action. Ignore this output"); 14 | } 15 | 16 | const client = context.getTwilioClient(); 17 | const [serviceSid, conversationsSid] = event.request.headers["x-session-id"] 18 | ?.replace("webhook:conversations__", "") 19 | .split("/"); 20 | 21 | console.log("attempting handover"); 22 | const flowSid = event.FlowSid || event.flowSid || context.STUDIO_FLOW_SID; 23 | if (!flowSid) { 24 | console.error("Missing flow sid"); 25 | return callback(new Error("Unable to hand over conversation")); 26 | } 27 | 28 | await client.conversations.v1 29 | .services(serviceSid) 30 | .conversations(conversationsSid) 31 | .webhooks.create({ 32 | target: "studio", 33 | configuration: { 34 | flowSid, 35 | }, 36 | }); 37 | 38 | console.log("handed over"); 39 | const successMessage = 40 | event.SuccessMessage ?? 41 | event.successMessage ?? 42 | "Conversation handed over"; 43 | return callback(null, successMessage); 44 | } catch (err) { 45 | console.error(err); 46 | return callback(null, "Could not handover"); 47 | } 48 | }; 49 | -------------------------------------------------------------------------------- /functions/tools/google-maps.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 3 | * @param {{}} event 4 | * @param {import('@twilio-labs/serverless-runtime-types/types').ServerlessCallback} callback 5 | */ 6 | exports.handler = async function (context, event, callback) { 7 | if (!context.GOOGLE_MAPS_API_KEY) { 8 | return callback(new Error('Missing API Key')) 9 | } 10 | 11 | let { location, name } = event; 12 | 13 | if (!location || !name) { 14 | return callback(new Error("Invalid request. Missing location or name")); 15 | } 16 | 17 | const searchResponse = await fetch( 18 | `https://maps.googleapis.com/maps/api/place/findplacefromtext/json?fields=place_id&input=${location} ${name}&inputtype=textquery&key=${context.GOOGLE_MAPS_API_KEY}` 19 | ); 20 | 21 | if (!searchResponse.ok) { 22 | console.error(await searchResponse.text()); 23 | return callback(new Error("Failed to get response from Google Maps")); 24 | } 25 | 26 | let searchData = await searchResponse.json(); 27 | 28 | if (searchData.candidates.length === 0) { 29 | return callback(null, "No Results Found"); 30 | } 31 | 32 | let place_id = searchData.candidates[0].place_id; 33 | 34 | const detailsResponse = await fetch( 35 | `https://maps.googleapis.com/maps/api/place/details/json?fields=name%2Ccurrent_opening_hours%2Cformatted_address%2Cformatted_phone_number&place_id=${place_id}&key=${context.GOOGLE_MAPS_API_KEY}` 36 | ); 37 | 38 | if (!detailsResponse.ok) { 39 | console.error(await detailsResponse.text()); 40 | return callback(new Error("Failed to get details from Google Maps")); 41 | } 42 | 43 | let detailsData = await detailsResponse.json(); 44 | 45 | return callback(null, { 46 | name: detailsData.result?.name, 47 | address: detailsData.result?.formatted_address, 48 | phone_number: detailsData.result?.formatted_phone_number, 49 | opening_hours: detailsData.result?.current_opening_hours?.weekday_text, 50 | }); 51 | }; 52 | -------------------------------------------------------------------------------- /functions/channels/conversations/response.js: -------------------------------------------------------------------------------- 1 | const { 2 | verifyRequest, 3 | readConversationAttributes, 4 | } = require(Runtime.getAssets()["/utils.js"].path); 5 | 6 | /** 7 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 8 | * @param {{}} event 9 | * @param {import('@twilio-labs/serverless-runtime-types/types').ServerlessCallback} callback 10 | */ 11 | exports.handler = async function (context, event, callback) { 12 | try { 13 | if (!verifyRequest(context, event)) { 14 | return callback(new Error("Invalid token")); 15 | } 16 | console.log("response", event); 17 | const assistantIdentity = 18 | typeof event._assistantIdentity === "string" 19 | ? event._assistantIdentity 20 | : undefined; 21 | 22 | if (event.Status === "Failed") { 23 | console.error(event); 24 | return callback( 25 | new Error("Failed to generate response. Check error logs.") 26 | ); 27 | } 28 | 29 | const client = context.getTwilioClient(); 30 | 31 | const [serviceSid, conversationsSid] = event.SessionId.replace( 32 | "webhook:conversations__", 33 | "" 34 | ).split("/"); 35 | const body = event.Body; 36 | 37 | const attributes = await readConversationAttributes( 38 | context, 39 | serviceSid, 40 | conversationsSid 41 | ); 42 | await client.conversations.v1 43 | .services(serviceSid) 44 | .conversations(conversationsSid) 45 | .update({ 46 | attributes: JSON.stringify({ ...attributes, assistantIsTyping: false }), 47 | }); 48 | 49 | const message = await client.conversations.v1 50 | .services(serviceSid) 51 | .conversations(conversationsSid) 52 | .messages.create({ 53 | body, 54 | author: assistantIdentity, 55 | }); 56 | 57 | console.log(`conversation message sent ${message.sid}`); 58 | 59 | return callback(null, {}); 60 | } catch (err) { 61 | console.error(err); 62 | return callback(null, {}); 63 | } 64 | }; 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Twilio Serverless 2 | .twiliodeployinfo 3 | 4 | # Logs 5 | logs 6 | *.log 7 | npm-debug.log* 8 | yarn-debug.log* 9 | yarn-error.log* 10 | lerna-debug.log* 11 | .pnpm-debug.log* 12 | 13 | # Diagnostic reports (https://nodejs.org/api/report.html) 14 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 15 | 16 | # Runtime data 17 | pids 18 | *.pid 19 | *.seed 20 | *.pid.lock 21 | 22 | # Directory for instrumented libs generated by jscoverage/JSCover 23 | lib-cov 24 | 25 | # Coverage directory used by tools like istanbul 26 | coverage 27 | *.lcov 28 | 29 | # nyc test coverage 30 | .nyc_output 31 | 32 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 33 | .grunt 34 | 35 | # Bower dependency directory (https://bower.io/) 36 | bower_components 37 | 38 | # node-waf configuration 39 | .lock-wscript 40 | 41 | # Compiled binary addons (https://nodejs.org/api/addons.html) 42 | build/Release 43 | 44 | # Dependency directories 45 | node_modules/ 46 | jspm_packages/ 47 | 48 | # Snowpack dependency directory (https://snowpack.dev/) 49 | web_modules/ 50 | 51 | # TypeScript cache 52 | *.tsbuildinfo 53 | 54 | # Optional npm cache directory 55 | .npm 56 | 57 | # Optional eslint cache 58 | .eslintcache 59 | 60 | # Optional stylelint cache 61 | .stylelintcache 62 | 63 | # Microbundle cache 64 | .rpt2_cache/ 65 | .rts2_cache_cjs/ 66 | .rts2_cache_es/ 67 | .rts2_cache_umd/ 68 | 69 | # Optional REPL history 70 | .node_repl_history 71 | 72 | # Output of 'npm pack' 73 | *.tgz 74 | 75 | # Yarn Integrity file 76 | .yarn-integrity 77 | 78 | # dotenv environment variable files 79 | .env 80 | .env.development.local 81 | .env.test.local 82 | .env.production.local 83 | .env.local 84 | 85 | # parcel-bundler cache (https://parceljs.org/) 86 | .cache 87 | .parcel-cache 88 | 89 | # Next.js build output 90 | .next 91 | out 92 | 93 | # Nuxt.js build / generate output 94 | .nuxt 95 | dist 96 | 97 | # Gatsby files 98 | .cache/ 99 | # Comment in the public line in if your project uses Gatsby and not Next.js 100 | # https://nextjs.org/blog/next-9-1#public-directory-support 101 | # public 102 | 103 | # vuepress build output 104 | .vuepress/dist 105 | 106 | # vuepress v2.x temp and cache directory 107 | .temp 108 | .cache 109 | 110 | # Docusaurus cache and generated files 111 | .docusaurus 112 | 113 | # Serverless directories 114 | .serverless/ 115 | 116 | # FuseBox cache 117 | .fusebox/ 118 | 119 | # DynamoDB Local files 120 | .dynamodb/ 121 | 122 | # TernJS port file 123 | .tern-port 124 | 125 | # Stores VSCode versions used for testing VSCode extensions 126 | .vscode-test 127 | 128 | # yarn v2 129 | .yarn/cache 130 | .yarn/unplugged 131 | .yarn/build-state.yml 132 | .yarn/install-state.gz 133 | .pnp.* 134 | -------------------------------------------------------------------------------- /functions/tools/internet-search.js: -------------------------------------------------------------------------------- 1 | const { ChatOpenAI } = require("@langchain/openai"); 2 | const { ChatPromptTemplate } = require("@langchain/core/prompts"); 3 | const { formatDocumentsAsString } = require("langchain/util/document"); 4 | const { 5 | RunnableSequence, 6 | RunnablePassthrough, 7 | } = require("@langchain/core/runnables"); 8 | const { StringOutputParser } = require("@langchain/core/output_parsers"); 9 | const { default: Exa } = require("exa-js"); 10 | 11 | const PROMPT = ` 12 | You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. Use a 8 sentences maximum and keep the answer concise. 13 | 14 | Question: {question} 15 | 16 | Context: {context} 17 | 18 | Answer: 19 | `.trim(); 20 | 21 | /** 22 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 23 | * @param {{}} event 24 | * @param {import('@twilio-labs/serverless-runtime-types/types').ServerlessCallback} callback 25 | */ 26 | exports.handler = async function (context, event, callback) { 27 | if (!context.EXA_API_KEY) { 28 | return callback(new Error("Invalid configuration")); 29 | } 30 | 31 | if (!event.query) { 32 | return callback(new Error("Missing query")); 33 | } 34 | 35 | const exa = new Exa(context.EXA_API_KEY); 36 | 37 | const domains = Array.isArray(event.limitToDomains) 38 | ? event.limitToDomains 39 | : event.limitToDomains 40 | ? [event.limitToDomains] 41 | : undefined; 42 | const numResults = event.n ? parseInt(event.n) : 5; 43 | const searchAndTextResults = await exa.searchAndContents(event.query, { 44 | numResults, 45 | text: { 46 | maxCharacters: 1000, 47 | }, 48 | highlights: { 49 | highlightsPerUrl: 1, 50 | numSentences: 7, 51 | }, 52 | useAutoprompt: true, 53 | includeDomains: domains, 54 | }); 55 | 56 | if (!event.summarize) { 57 | return callback(null, searchAndTextResults.results); 58 | } 59 | 60 | if (!context.OPENAI_API_KEY) { 61 | console.error("No OpenAI key, skipping summarization"); 62 | return callback(null, searchAndTextResults.results); 63 | } 64 | 65 | const data = searchAndTextResults.results.flatMap( 66 | (result) => result.highlights 67 | ); 68 | 69 | const prompt = ChatPromptTemplate.fromMessages([["human", PROMPT]]); 70 | const llm = new ChatOpenAI({ modelName: "gpt-3.5-turbo", temperature: 0 }); 71 | 72 | const ragChain = RunnableSequence.from([ 73 | { 74 | context: () => data.join("\n\n"), 75 | question: new RunnablePassthrough(), 76 | }, 77 | prompt, 78 | llm, 79 | new StringOutputParser(), 80 | ]); 81 | 82 | const result = await ragChain.invoke(event.query); 83 | callback(null, result); 84 | }; 85 | -------------------------------------------------------------------------------- /functions/tools/flex-handover.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 3 | * @param {{}} event 4 | * @param {import('@twilio-labs/serverless-runtime-types/types').ServerlessCallback} callback 5 | */ 6 | exports.handler = async function (context, event, callback) { 7 | const client = context.getTwilioClient(); 8 | 9 | const FLEX_WORKFLOW_SID = event.FlexWorkflowSid || context.FLEX_WORKFLOW_SID; 10 | const FLEX_WORKSPACE_SID = 11 | event.FlexWorkspaceSid || context.FLEX_WORKSPACE_SID; 12 | 13 | if (!FLEX_WORKFLOW_SID || !FLEX_WORKSPACE_SID) { 14 | return callback( 15 | new Error( 16 | "Missing configuration for FLEX_WORKSPACE_SID OR FLEX_WORKFLOW_SID" 17 | ) 18 | ); 19 | } 20 | 21 | const [serviceSid, conversationsSid] = event.request.headers["x-session-id"] 22 | ?.replace("conversations__", "") 23 | .split("/"); 24 | const [traitName, identity] = event.request.headers["x-identity"]?.split(":"); 25 | 26 | if (!identity || !conversationsSid) { 27 | return callback(new Error("Invalid request")); 28 | } 29 | 30 | try { 31 | let from = identity; 32 | let customerName = identity; 33 | let customerAddress = identity; 34 | let channelType = "chat"; 35 | if (traitName === "whatsapp") { 36 | channelType = "whatsapp"; 37 | from = `whatsapp:${identity}`; 38 | customerName = from; 39 | customerAddress = from; 40 | } else if (identity.startsWith("+")) { 41 | channelType = "sms"; 42 | customerName = from; 43 | customerAddress = from; 44 | } else if (identity.startsWith("FX")) { 45 | // Flex webchat 46 | channelType = "web"; 47 | customerName = from; 48 | customerAddress = from; 49 | try { 50 | const user = await client.conversations.users(identity).fetch(); 51 | from = user.friendlyName; 52 | } catch (err) { 53 | console.error(err); 54 | } 55 | } 56 | const result = await client.flexApi.v1.interaction.create({ 57 | channel: { 58 | type: channelType, 59 | initiated_by: "customer", 60 | properties: { 61 | media_channel_sid: conversationsSid, 62 | }, 63 | }, 64 | routing: { 65 | properties: { 66 | workspace_sid: FLEX_WORKSPACE_SID, 67 | workflow_sid: FLEX_WORKFLOW_SID, 68 | task_channel_unique_name: "chat", 69 | attributes: { 70 | from: from, 71 | customerName: customerName, 72 | customerAddress: customerAddress, 73 | }, 74 | }, 75 | }, 76 | }); 77 | console.log(result.sid); 78 | } catch (err) { 79 | console.error(err); 80 | return callback(new Error("Failed to hand over to a human agent")); 81 | } 82 | 83 | return callback(null, "Transferred to human agent"); 84 | }; 85 | -------------------------------------------------------------------------------- /functions/channels/conversations/messageAdded.protected.js: -------------------------------------------------------------------------------- 1 | const { 2 | signRequest, 3 | getAssistantSid, 4 | sendMessageToAssistant, 5 | readConversationAttributes, 6 | } = require(Runtime.getAssets()["/utils.js"].path); 7 | 8 | /** 9 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 10 | * @param {{}} event 11 | * @param {import('@twilio-labs/serverless-runtime-types/types').ServerlessCallback} callback 12 | */ 13 | exports.handler = async function (context, event, callback) { 14 | const assistantSid = await getAssistantSid(context, event); 15 | 16 | const { ConversationSid, ChatServiceSid, Author } = event; 17 | const AssistantIdentity = 18 | typeof event.AssistantIdentity === "string" 19 | ? event.AssistantIdentity 20 | : undefined; 21 | 22 | let identity = Author.includes(":") ? Author : `user_id:${Author}`; 23 | 24 | const client = context.getTwilioClient(); 25 | 26 | const webhooks = ( 27 | await client.conversations.v1 28 | .services(ChatServiceSid) 29 | .conversations(ConversationSid) 30 | .webhooks.list() 31 | ).filter((entry) => entry.target === "studio"); 32 | 33 | if (webhooks.length > 0) { 34 | // ignoring if the conversation has a studio webhook set (assuming it was handed over) 35 | return callback(null, ""); 36 | } 37 | 38 | const participants = await client.conversations.v1 39 | .services(ChatServiceSid) 40 | .conversations(ConversationSid) 41 | .participants.list(); 42 | 43 | if (participants.length > 1) { 44 | // Ignoring the conversation because there is more than one human 45 | return callback(null, ""); 46 | } 47 | 48 | const token = await signRequest(context, event); 49 | const params = new URLSearchParams(); 50 | params.append("_token", token); 51 | if (typeof AssistantIdentity === "string") { 52 | params.append("_assistantIdentity", AssistantIdentity); 53 | } 54 | const body = { 55 | body: event.Body, 56 | identity: identity, 57 | session_id: `conversations__${ChatServiceSid}/${ConversationSid}`, 58 | // using a callback to handle AI Assistant responding 59 | webhook: `https://${ 60 | context.DOMAIN_NAME 61 | }/channels/conversations/response?${params.toString()}`, 62 | }; 63 | 64 | const response = new Twilio.Response(); 65 | response.appendHeader("content-type", "text/plain"); 66 | response.setBody(""); 67 | 68 | const attributes = await readConversationAttributes( 69 | context, 70 | ChatServiceSid, 71 | ConversationSid 72 | ); 73 | await client.conversations.v1 74 | .services(ChatServiceSid) 75 | .conversations(ConversationSid) 76 | .update({ 77 | attributes: JSON.stringify({ ...attributes, assistantIsTyping: true }), 78 | }); 79 | 80 | try { 81 | await sendMessageToAssistant(context, assistantSid, body); 82 | } catch (err) { 83 | console.error(err); 84 | } 85 | 86 | callback(null, response); 87 | }; 88 | -------------------------------------------------------------------------------- /functions/channels/conversations/flex-webchat.protected.js: -------------------------------------------------------------------------------- 1 | const { 2 | signRequest, 3 | getAssistantSid, 4 | sendMessageToAssistant, 5 | readConversationAttributes, 6 | } = require(Runtime.getAssets()["/utils.js"].path); 7 | 8 | /** 9 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 10 | * @param {{ 11 | * Data?: string; 12 | * Body?: string; 13 | * }} event 14 | * @param {import('@twilio-labs/serverless-runtime-types/types').ServerlessCallback} callback 15 | */ 16 | exports.handler = async function (context, event, callback) { 17 | const assistantSid = await getAssistantSid(context, event); 18 | 19 | const data = typeof event.Data === "string" ? JSON.parse(event.Data) : {}; 20 | const { 21 | ChannelSid: ConversationSid, 22 | InstanceSid: ChatServiceSid, 23 | From: Author, 24 | } = data; 25 | 26 | const Body = typeof event.Body === "string" ? event.Body : data.Body; 27 | const AssistantIdentity = 28 | typeof event.AssistantIdentity === "string" 29 | ? event.AssistantIdentity 30 | : undefined; 31 | 32 | if ( 33 | typeof ConversationSid !== "string" || 34 | typeof ChatServiceSid !== "string" || 35 | typeof Author !== "string" 36 | ) { 37 | return callback( 38 | new Error( 39 | 'Failed request. Make sure to configure your Studio widget to pass "Data" with the value "{{trigger.conversation | to_json}}. This only works for "Incoming Conversation" flows.' 40 | ) 41 | ); 42 | } 43 | 44 | let identity = Author.includes(":") ? Author : `user_id:${Author}`; 45 | if (Author.startsWith("FX")) { 46 | // User is a Flex Web Chat Conversation Participant 47 | identity = `flex_participant:${Author}`; 48 | } 49 | 50 | const client = context.getTwilioClient(); 51 | 52 | const participants = await client.conversations.v1 53 | .services(ChatServiceSid) 54 | .conversations(ConversationSid) 55 | .participants.list(); 56 | 57 | if (participants.length > 1) { 58 | // Ignoring the conversation because there is more than one human 59 | return callback(null, ""); 60 | } 61 | 62 | const token = await signRequest(context, event); 63 | const params = new URLSearchParams(); 64 | params.append("_token", token); 65 | if (typeof AssistantIdentity === "string") { 66 | params.append("_assistantIdentity", AssistantIdentity); 67 | } 68 | const body = { 69 | body: Body, 70 | identity: identity, 71 | session_id: `conversations__${ChatServiceSid}/${ConversationSid}`, 72 | // using a callback to handle AI Assistant responding 73 | webhook: `https://${ 74 | context.DOMAIN_NAME 75 | }/channels/conversations/response?${params.toString()}`, 76 | }; 77 | 78 | const response = new Twilio.Response(); 79 | response.appendHeader("content-type", "text/plain"); 80 | response.setBody(""); 81 | 82 | const attributes = await readConversationAttributes( 83 | context, 84 | ChatServiceSid, 85 | ConversationSid 86 | ); 87 | await client.conversations.v1 88 | .services(ChatServiceSid) 89 | .conversations(ConversationSid) 90 | .update({ 91 | attributes: JSON.stringify({ ...attributes, assistantIsTyping: true }), 92 | }); 93 | 94 | try { 95 | await sendMessageToAssistant(context, assistantSid, body); 96 | } catch (err) { 97 | console.error(err); 98 | } 99 | 100 | callback(null, response); 101 | }; 102 | -------------------------------------------------------------------------------- /.twilioserverlessrc: -------------------------------------------------------------------------------- 1 | { 2 | "commands": {}, 3 | "environments": {}, 4 | "projects": {}, 5 | // "assets": true /* Upload assets. Can be turned off with --no-assets */, 6 | // "assetsFolder": null /* Specific folder name to be used for static assets */, 7 | // "buildSid": null /* An existing Build SID to deploy to the new environment */, 8 | // "createEnvironment": false /* Creates environment if it couldn't find it. */, 9 | // "cwd": null /* Sets the directory of your existing Serverless project. Defaults to current directory */, 10 | // "detailedLogs": false /* Toggles detailed request logging by showing request body and query params */, 11 | // "edge": null /* Twilio API Region */, 12 | // "env": null /* Path to .env file for environment variables that should be installed */, 13 | // "environment": "dev" /* The environment name (domain suffix) you want to use for your deployment. Alternatively you can specify an environment SID starting with ZE. */, 14 | // "extendedOutput": false /* Show an extended set of properties on the output */, 15 | // "force": false /* Will run deployment in force mode. Can be dangerous. */, 16 | // "forkProcess": true /* Disable forking function processes to emulate production environment */, 17 | // "functionSid": null /* Specific Function SID to retrieve logs for */, 18 | // "functions": true /* Upload functions. Can be turned off with --no-functions */, 19 | // "functionsFolder": null /* Specific folder name to be used for static functions */, 20 | // "inspect": null /* Enables Node.js debugging protocol */, 21 | // "inspectBrk": null /* Enables Node.js debugging protocol, stops execution until debugger is attached */, 22 | // "legacyMode": false /* Enables legacy mode, it will prefix your asset paths with /assets */, 23 | // "live": true /* Always serve from the current functions (no caching) */, 24 | // "loadLocalEnv": false /* Includes the local environment variables */, 25 | // "loadSystemEnv": false /* Uses system environment variables as fallback for variables specified in your .env file. Needs to be used with --env explicitly specified. */, 26 | // "logCacheSize": null /* Tailing the log endpoint will cache previously seen entries to avoid duplicates. The cache is topped at a maximum of 1000 by default. This option can change that. */, 27 | // "logLevel": "info" /* Level of logging messages. */, 28 | // "logs": true /* Toggles request logging */, 29 | // "ngrok": null /* Uses ngrok to create a public url. Pass a string to set the subdomain (requires a paid-for ngrok account). */, 30 | // "outputFormat": "" /* Output the results in a different format */, 31 | // "overrideExistingProject": false /* Deploys Serverless project to existing service if a naming conflict has been found. */, 32 | // "port": "3000" /* Override default port of 3000 */, 33 | // "production": false /* Promote build to the production environment (no domain suffix). Overrides environment flag */, 34 | // "properties": null /* Specify the output properties you want to see. Works best on single types */, 35 | // "region": null /* Twilio API Region */, 36 | "runtime": "node18" /* The version of Node.js to deploy the build to. (node18) */, 37 | // "serviceName": null /* Overrides the name of the Serverless project. Default: the name field in your package.json */, 38 | // "serviceSid": null /* SID of the Twilio Serverless Service to deploy to */, 39 | // "sourceEnvironment": null /* SID or suffix of an existing environment you want to deploy from. */, 40 | // "tail": false /* Continuously stream the logs */, 41 | // "template": null /* undefined */, 42 | } -------------------------------------------------------------------------------- /assets/utils.private.js: -------------------------------------------------------------------------------- 1 | const { sign, decode } = require("jsonwebtoken"); 2 | 3 | /** 4 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 5 | * @param {string} assistantId 6 | * @param {*} body 7 | */ 8 | async function sendMessageToAssistant(context, assistantId, body) { 9 | const environmentPrefix = context.TWILIO_REGION?.startsWith("stage") 10 | ? ".stage" 11 | : context.TWILIO_REGION?.startsWith("dev") 12 | ? ".dev" 13 | : ""; 14 | const url = `https://assistants${environmentPrefix}.twilio.com/v1/Assistants/${assistantId}/Messages`; 15 | 16 | const response = await fetch(url, { 17 | method: "POST", 18 | body: JSON.stringify(body), 19 | headers: { 20 | Authorization: `Basic ${Buffer.from( 21 | `${context.ACCOUNT_SID}:${context.AUTH_TOKEN}`, 22 | "utf-8" 23 | ).toString("base64")}`, 24 | "Content-Type": "application/json", 25 | Accept: "application/json", 26 | }, 27 | }); 28 | if (response.ok) { 29 | console.log("Sent message to AI Assistant"); 30 | return; 31 | } else { 32 | throw new Error( 33 | "Failed to send request to AI Assistants. " + (await response.text()) 34 | ); 35 | } 36 | } 37 | 38 | /** 39 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 40 | * @param {string} chatServiceSid 41 | * @param {string} conversationSid 42 | */ 43 | async function readConversationAttributes( 44 | context, 45 | chatServiceSid, 46 | conversationSid 47 | ) { 48 | try { 49 | const client = context.getTwilioClient(); 50 | const data = await client.conversations.v1 51 | .services(chatServiceSid) 52 | .conversations(conversationSid) 53 | .fetch(); 54 | return JSON.parse(data.attributes); 55 | } catch (err) { 56 | console.error(err); 57 | return {}; 58 | } 59 | } 60 | 61 | /** 62 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 63 | * @param {*} event 64 | */ 65 | async function getAssistantSid(context, event) { 66 | if (event.EventType === "onMessageAdded") { 67 | try { 68 | const { ConversationSid, ChatServiceSid } = event; 69 | const parsed = await readConversationAttributes( 70 | context, 71 | ChatServiceSid, 72 | ConversationSid 73 | ); 74 | if (typeof parsed.assistantSid === "string" && parsed.assistantSid) { 75 | return parsed.assistantSid; 76 | } 77 | } catch (err) { 78 | console.log("Invalid attribute structure", err); 79 | } 80 | } 81 | const assistantSid = 82 | event.AssistantId || 83 | context.ASSISTANT_ID || 84 | event.AssistantSid || 85 | context.ASSISTANT_SID; 86 | 87 | if (!assistantSid) { 88 | throw new Error("Missing Assistant ID configuration"); 89 | } 90 | 91 | return assistantSid; 92 | } 93 | 94 | /** 95 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 96 | * @param {*} event 97 | */ 98 | async function signRequest(context, event) { 99 | const assistantSid = await getAssistantSid(context, event); 100 | const authToken = context.AUTH_TOKEN; 101 | if (!authToken) { 102 | throw new Error("No auth token found"); 103 | } 104 | return sign({ assistantSid }, authToken, { expiresIn: "5m" }); 105 | } 106 | 107 | /** 108 | * @param {import('@twilio-labs/serverless-runtime-types/types').Context} context 109 | * @param {*} event 110 | */ 111 | function verifyRequest(context, event) { 112 | const token = event._token; 113 | if (!token) { 114 | throw new Error("Missing token"); 115 | } 116 | 117 | const authToken = context.AUTH_TOKEN; 118 | if (!authToken) { 119 | throw new Error("No auth token found"); 120 | } 121 | 122 | try { 123 | const decoded = decode(token, authToken, { json: true }); 124 | if (decoded.assistantSid) { 125 | return true; 126 | } 127 | } catch (err) { 128 | console.error("Failed to verify token", err); 129 | return false; 130 | } 131 | return false; 132 | } 133 | 134 | module.exports = { 135 | getAssistantSid, 136 | signRequest, 137 | verifyRequest, 138 | sendMessageToAssistant, 139 | readConversationAttributes, 140 | }; 141 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

Twilio AI AssistantsTwilio AI Assistants

2 |

Twilio AI Assistants Samples

3 | 4 | > [!NOTE] 5 | > Twilio AI Assistants is a [Twilio Alpha](https://twilioalpha.com) project that is currently in Developer Preview. If you would like to try AI Assistants, [join the waitlist](https://twilioalpha.com/ai-assistants). 6 | 7 | This project contains various different Twilio Functions for common use cases like different channel integrations and common example tools you might want to use. 8 | 9 | For more detailed documentation [visit the Twilio Docs](https://twilio.com/docs/alpha/ai-assistants/code-samples). 10 | 11 | ## Setup 12 | 13 | Requirements: Node 18 & [Twlio CLI & Twilio Serverless Toolkit](https://twilio.com/docs/labs/serverless-toolkit) 14 | 15 | ```bash 16 | git clone git@github.com:twilio-labs/ai-assistants-samples.git 17 | cd ai-assistants-samples 18 | npm install 19 | cp .env.example .env 20 | 21 | # optional: fill in your Assistant SID. If you don't fill it in you'll have to pass it via `?AssistantSid=<...>` 22 | 23 | twilio serverless:deploy 24 | ``` 25 | 26 | You should get output similar to this: 27 | 28 | ```text 29 | Deployment Details 30 | Domain: ai-assistants-samples-1111-dev.twil.io 31 | Service: 32 | ai-assistants-samples (ZSf3510841424c854e3f3b282550211111) 33 | Environment: 34 | dev (ZE94900e7f2a2c330b15cf6e1c9fd11111) 35 | Build SID: 36 | ZB2743d62d52d42ccd55873a0bcd511111 37 | Runtime: 38 | node18 39 | View Live Logs: 40 | https://www.twilio.com/console/functions/editor/ZSf3510841424c854e3f3b282550211111/environment/ZE94900e7f2a2c330b15cf6e1c9fd11111 41 | Functions: 42 | [protected] https://ai-assistants-samples-1111-dev.twil.io/channels/conversations/messageAdded 43 | [protected] https://ai-assistants-samples-1111-dev.twil.io/channels/messaging/incoming 44 | https://ai-assistants-samples-1111-dev.twil.io/channels/conversations/response 45 | https://ai-assistants-samples-1111-dev.twil.io/channels/messaging/response 46 | https://ai-assistants-samples-1111-dev.twil.io/tools/flex-handover 47 | https://ai-assistants-samples-1111-dev.twil.io/tools/google-maps 48 | https://ai-assistants-samples-1111-dev.twil.io/tools/internet-search 49 | https://ai-assistants-samples-1111-dev.twil.io/tools/studio-handover 50 | Assets: 51 | ``` 52 | 53 | Replace below any `` with the output next to `Domain: ` in that output. 54 | 55 | ## Messaging Channel 56 | 57 | ### SMS 58 | 59 | **Via the Twilio CLI:** 60 | 61 | ```bash 62 | twilio phone_number \ 63 | --sms-url=https://.twil.io/channels/messaging/incoming 64 | ``` 65 | 66 | **Using the Twilio Console:** 67 | Open your SMS-capable phone number of choice or Messaging Service and configure the `When a message comes in` webhook to point to: `https://.twil.io/channels/messaging/incoming` 68 | 69 | ### WhatsApp Sandbox 70 | 71 | Configure your `When a message comes in` webhook in the [WhatsApp Sandbox Seetings](https://console.twilio.com/us1/develop/sms/try-it-out/whatsapp-learn?frameUrl=%2Fconsole%2Fsms%2Fwhatsapp%2Flearn%3Fx-target-region%3Dus1) to point to `https://.twil.io/channels/messaging/incoming` 72 | 73 | > [!NOTE] 74 | > If you want to use the same webhook for another Assistant you can add `?AssistantSid=` as query parameter to the webhook URL. Example: `https://.twil.io/channels/messaging/incoming?AssistantSid=AI1234561231237812312` 75 | 76 | ## Conversations Channel 77 | 78 | Setup: 79 | 80 | 1. Set up a Conversations Service or use your default Conversations Service from the Console 81 | 2. Configure the webhook on a service level using the Twilio CLI command below 82 | 3. Connect your preferred Conversations channel following the [guides in the docs](https://www.twilio.com/docs/conversations/overview). 83 | 84 | ```bash 85 | twilio api:conversations:v1:services:configuration:webhooks:update \ 86 | --post-webhook-url=https://.twil.io/channels/conversations/messageAdded 87 | --chat-service-sid= 88 | --filter=onMessageAdded 89 | ``` 90 | 91 | ## Tools 92 | 93 | Below are a selection of common tools that you might want to use or modify to your own needs. Each has an example configuration but you might want to tweak it to your own needs especially the `Description` if you find your Assistant not triggering the Tool reliably. 94 | 95 | ### Google Maps 96 | 97 | Tool to enable your Assistant to search Google Maps for the full address, phone number and opening hours for a business in a given location. 98 | 99 | > [!IMPORTANT] 100 | > Requires the `GOOGLE_MAPS_API_KEY` environment variable to be set 101 | 102 | | Field | Configuration | 103 | | --------------- | -------------------------------------------------------------------------------------------------------------------------------------- | 104 | | **Name** | `Google Maps` | 105 | | **Description** | `Use this to fetch information about the a location from Google Maps. You MUST use this tool before the "Ask User for New Data" tool.` | 106 | | **Input** | 
{
 location: string;
 name: string;
}
| 107 | | **Method** | `GET` | 108 | | **URL** | `https://.twil.io/tools/google-maps` | 109 | 110 | ### Flex Handover 111 | 112 | Tool for your AI Assistant to hand over a conversation to a human agent. 113 | 114 | > [!IMPORTANT] 115 | > Requires: 116 | > 117 | > 1. The use of [Twilio Conversations as channel](#conversations-channel) 118 | > 2. The Assistant & these Functions to be deployed in a Flex Account 119 | > 3. The `FLEX_WORKSPACE_SID` and `FLEX_WORKFLOW_SID` environment variables to be configured 120 | 121 | | Field | Configuration | 122 | | --------------- | ------------------------------------------------------------------------------------------------------------------------------- | 123 | | **Name** | `Hand over Conversation` | 124 | | **Description** | `You MUST use this if you don't know how to fulfill the request to let another customer service agent handle the conversation.` | 125 | | **Input** | 
{}
| 126 | | **Method** | `POST` | 127 | | **URL** | `https://.twil.io/tools/flex-handover` | 128 | 129 | ### Studio Handover 130 | 131 | Tool to hand over a conversation that the Assistant is handling to a Studio flow. While there is an example `Description` for the Tool you want to update this to match your handover criteria. 132 | 133 | This Tool rewires the conversation from your Assistant to Studio but does not forward the last message to the Studio flow. Instead the Tool will by default respond with `Conversation handed over` which might prompt the Assistant to say something like "I handed this conversation over". If you want to manipulate this message you can pass a different message into the Tool URL using the `SuccessMessage` query parameter. 134 | 135 | > [!IMPORTANT] 136 | > Requires: 137 | > 138 | > 1. The use of [Twilio Conversations as channel](#conversations-channel) 139 | > 2. The Assistant & these Functions to be deployed in the same account as Studio flow 140 | > 3. You either need to configure the `STUDIO_FLOW_SID` or pass `FlowSid` as query parameter to the Tool URL. 141 | 142 | | Field | Configuration | 143 | | --------------- | --------------------------------------------------------------- | 144 | | **Name** | `Studio Handover` | 145 | | **Description** | `You MUST use this if a customer is asking for a refund.` | 146 | | **Input** | 
{}
| 147 | | **Method** | `POST` | 148 | | **URL** | `https://.twil.io/tools/studio-handover` | 149 | 150 | ### Internet Search 151 | 152 | This tool will search the internet for relevant information and optionally summarize the information using GPT-3.5 Turbo. 153 | 154 | > [!IMPORTANT] 155 | > Requires you to set up the `EXA_API_KEY` with a valid key from [exa.ai](https://exa.ai) 156 | 157 | > [!CAUTION] 158 | > This tool will fetch data from the internet and there is a risk that this can open up your Assistant for prompt injection attacks. 159 | 160 | | Field | Configuration | 161 | | --------------- | ----------------------------------------------------------------------------------------------- | 162 | | **Name** | `Search Internet` | 163 | | **Description** | `You MUST use this for any information you are unsure about or information about recent events` | 164 | | **Input** | 
{
 query: string; // a search engine query
}
| 165 | | **Method** | `GET` | 166 | | **URL** | `https://.twil.io/tools/internet-search` | 167 | 168 | Additionally to the configuration below, you can use the following query parameters to configure your search behavior. These can be put into the `input` but are recommended to passed directly into the end of the URL instead. 169 | 170 | - `limitDomains` — You can pass multiple ones to limit search results to specific domains. For example `?limitDomains=www.segment.com&limitDomains=www.twilio.com` will only search those two domains. 171 | - `n` — specifies the amount of search results you want to take into consideration for the response. Example: `?n=2` 172 | - `summarize` — If set to `?summarize=true`, it will optionally run the request through OpenAI's GPT-3.5-Turbo for a proper answer that gets pushed into your Assistant. This requires the `OPENAI_API_KEY` environment variable to be set. 173 | 174 | ### UI Tools 175 | 176 | This tool enables you to trigger functions in the web UI of your AI Assistant assuming you are using the [AI Assistants JavaScript SDK](https://github.com/twilio-labs/ai-assistants-js). 177 | 178 | > [!IMPORTANT] 179 | > This will only work if you are using the specific AI Assistants JavaScript SDK and not with any other Twilio Conversations SDK. 180 | 181 | | Field | Configuration | 182 | | --------------- | ----------------------------------------------------------------------------------------- | 183 | | **Name** | `` | 184 | | **Description** | `Description for when this UI tool should be triggered` | 185 | | **Input** | 
{ // anything you want to send to the UI tool }
| 186 | | **Method** | `GET` | 187 | | **URL** | `https://.twil.io/tools/ui-tools?toolName=` | 188 | 189 | ## Contributing 190 | 191 | [See contributing guide](./CONTRIBUTING.md) 192 | 193 | ## License 194 | 195 | MIT 196 | -------------------------------------------------------------------------------- /assets/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Get started with your Twilio AI Assistants! 9 | 10 | 14 | 18 | 22 | 77 | 164 | 165 | 166 |
167 |
168 | 229 | 250 |
251 |
252 |
253 |
254 |

255 | 258 |
259 |

Welcome!

260 |

Let's get your AI Assistant connected to the world!

261 |
262 |

263 |
264 |

Get started with your AI Assistant

265 |

266 | Follow these steps to connect your AI Assistant to new Channels and 267 | Tools: 268 |

269 |
    270 |
  1. Bookmark this page so it's easier to come back to it later.
    2. 271 |
  2. 272 | If you don't have an AI Assistant yet, 273 | go into the Twilio Console and create your AI Assistant. 279 |
    3. 280 |
  3. 281 | Configure your AI Assistant 287 | and start debugging using the 288 | Simulator. 293 |
    4. 294 |
  4. 295 | Copy your AI Assistant's ID and paste it into the input below. 296 |
    5. 297 |
  5. 298 | Follow the steps of the individual Tools and channels below. 299 |
    6. 300 |
301 |
302 | 303 | 309 |
310 |
311 |
312 |

Channels

313 |

314 | For a more in-depth guide of connecting AI Assistants to any of 315 | these channels, check out the 316 | Twilio Docs. 319 |

320 | 327 |
328 | 329 | 338 | 339 | Conversations Icon 340 | 341 | 346 | 347 |

348 | Twilio Conversations (Web Chat, iOS, Android, SMS, WhatsApp) 349 |

350 | 351 |

352 | Twilio Conversations 355 | is the recommended way to connect AI Assistants to most channels. By 356 | onboarding with Twilio Conversations you can connect your AI 357 | Assistant to most Twilio channels, use our 358 | AI Assistants React and JavaScript SDKs, and handoff conversations to 364 | Twilio Flex 370 | and 371 | Twilio Studio. 377 |

378 |

379 | If you are only interested in SMS and WhatsApp without any handoff 380 | capabilities, you can also check out the separate instructions 381 | below. 382 |

383 |

384 | To connect your AI Assistant to Twilio Conversations follow these 385 | steps: 386 |

387 |
    388 |
  1. 389 | Onboard with Twilio Conversations by 390 | checking out their documentation. 393 |
    2. 394 |
  2. 395 | Find your 396 | Conversations Service 402 |
    3. 403 |
  3. 404 | Navigate to the "Webhooks" section of your Conversations Service. 405 |
    4. 406 |
  4. Set the "On Message Added" webhook to the URL below.
    5. 407 |
  5. 408 | Create a new Conversation using your preferred channel or use our 409 | React SDK 415 |
    6. 416 |
417 | 418 |
419 | 426 |
427 |

428 | Check out our documentation for more detailed instructions. 434 |

435 |
436 |
437 | 438 | 447 | SMS Icon 448 | 454 | 455 |

SMS

456 | 457 |

458 | If you just want to connect your AI Assistant to a Twilio phone 459 | number, this is the fastest way to get started. 460 |

461 |
    462 |
  1. 463 | Get an 464 | SMS-compatible Twilio phone number 470 |
    2. 471 |
  2. 472 | Open up the configuration screen of your phone number or your 473 | Messaging Service 479 | if you are using one. 480 |
    3. 481 |
  3. 482 | Configure the "When a message comes in" webhook to match the URL 483 | below. 484 |
    4. 485 |
  4. Save your configuration
    5. 486 |
  5. 487 | Send a message to your phone number and see if you get a response 488 | from your AI Assistant. 489 |
    6. 490 |
491 | 492 |
493 | 500 |
501 |

502 | Check out our documentation for more detailed instructions. 508 |

509 |
510 |
511 | 512 | 521 | 522 | WhatsApp Icon 523 | 524 | 529 | 530 |

WhatsApp

531 | 532 |

533 | If you just want to connect your AI Assistant to WhatsApp without 534 | any handoff capabilities, this is the fastest way to get started. 535 |

536 |
    537 |
  1. 538 | Request to register your WhatsApp phone number 544 |
    2. 545 |
  2. 546 | While you are waiting for your WhatsApp number you can use the 547 | WhatsApp Sandbox. 553 |
    3. 554 |
  3. 555 | Configure your WhatsApp number's "When a message comes in" webhook 556 | to be the URL below. 557 |
    4. 558 |
  4. 559 | Message your WhatsApp number and see if you receive a message back 560 | from your AI Assistant. 561 |
    5. 562 |
563 | 566 |
567 | 574 |
575 |

576 | Check out our documentation for more detailed instructions. 582 |

583 |
584 |
585 | 586 | 595 | 596 | Twilio Flex 597 | 598 | 603 | 604 |

Flex Webchat

605 | 606 |

607 | If you are using the existing Flex Webchat and Studio Flow but want 608 | to have AI Assistants handle your conversation before handing it off 609 | to an agent follow this setup. 610 |

611 |
    612 |
  1. 613 | Find your "Chat Flow" 614 | Studio Flow. 620 |
    2. 621 |
  2. Create a new Run Function widget
    3. 622 |
  3. 623 | Select the "ai-assistants-samples" service and pick the 624 | /channels/conversations/flex-webchat Function. 625 |
    4. 626 |
  4. Add the following Function Parameters
    5. 627 |
      628 |
    • 629 | AssistantId: 634 |
      • 635 |
    • 636 | Data: 637 | {{trigger.conversation | to_json}} 638 |
      • 639 |
    640 |
  5. 641 | Name your widget with a recognizable name like 642 | "SendtoAIAssistant". 643 |
    6. 644 |
  6. 645 | Connect your "SendtoAIAssistant" widget to the "Incoming 646 | Conversation" trigger. 647 |
    7. 648 |
649 | 650 |
651 | 658 |
659 |
660 |
661 | 662 | 671 | Twilio Voice 672 | 677 | 678 |

Voice

679 | 680 |

681 | Follow these steps to connect your Assistant to a Twilio number for incoming calls. 682 |

683 |
    684 |
  1. 685 | Get a 686 | Voice-compatible Twilio phone number 692 |
    2. 693 |
  2. 694 | Define the greeting you'd like your Assistant to start the call with below. 695 | If you don't specify a greeting, your Assistant will start the call with a generic default greeting. 696 |
    3. 697 |
  3. 698 | Open up the configuration screen of your phone number. 699 |
    4. 700 |
  4. 701 | Set the Voice Configuration to use the "Webhook, TwiML Bin, Function, Studio Flow, Proxy Service" option. 702 | Then set your number's "When a message comes in" webhook 703 | to the URL below. 704 |
    5. 705 |
  5. Save your configuration
    6. 706 |
  6. 707 | Make a call to your phone number and you should get a 708 | response from your Assistant. 709 |
    7. 710 |
711 |
712 | 713 | 719 |
720 | 721 |
722 | 729 |
730 |

731 | Check out our documentation for more detailed instructions. 737 |

738 |
739 |
740 |
741 |

Tools

742 |
743 | 744 | 753 | Twilio Studio 754 | 759 | 760 |

Hand over to Studio

761 | 762 |

763 | Tool to hand over a conversation that the Assistant is handling to a 764 | Studio flow. While there is an example Description for 765 | the Tool you want to update this to match your handover criteria. 766 |

767 |

768 | This Tool rewires the conversation from your Assistant to Studio but 769 | does not forward the last message to the Studio flow. Instead the 770 | Tool will by default respond with "Conversation handed over" which 771 | might prompt the Assistant to say something like "I handed this 772 | conversation over". If you want to manipulate this message you can 773 | pass a different message into the Tool URL using the 774 | successMessage query parameter. 775 |

776 |
    777 |
  1. 778 | Create a 779 | Studio Flow 785 |
    2. 786 |
  2. 787 | Connect your Flow to trigger for "Incoming Conversations" and 788 | deploy your Flow. 789 |
    3. 790 |
  3. Copy your Flow SID starting with FW
    4. 791 |
  4. 792 | Create a new Tool with the following configuration: 793 |
      794 |
    • Name: Studio Handover
      • 795 |
    • 796 | Description: You MUST use this if a customer 797 | is asking for a refund. 798 |
      • 799 |
    • Input: {}
      • 800 |
    • Method: POST
      • 801 |
    • 802 | URL: 803 | Use the URL below and replace $FLOW_SID with 805 | your Flow SID 807 |
      • 808 |
    809 |
    5. 810 |
  5. 811 | Send a message to your Assistant that triggers this tool and see 812 | if you get a notification that you got transferred. Afterwards 813 | write another message to trigger the Studio flow. 814 |
    6. 815 |
816 | 817 |
818 | 825 |
826 |

827 | Check out our documentation for more detailed instructions. 833 |

834 |
835 | 836 | 845 | 846 | Twilio Flex 847 | 848 | 853 | 854 |

Hand over to Flex

855 | 856 |

857 | Tool for your AI Assistant to hand over a conversation to a human agent. 858 |

859 |
    860 |
  1. Make sure you are in a Flex enabled account by visiting Flex in the Console.
    2. 861 |
  2. 862 | Find your Workspace SID and Workflow SID for your handover by visiting TaskRouter and note them down. 863 |
    3. 864 |
  3. 865 | Create a new Tool with the following configuration: 866 |
      867 |
    • Name: Flex Handover
      • 868 |
    • 869 | Description: You MUST use this if you don't know how to fulfill the request to let another customer service agent handle the conversation. 870 |
      • 871 |
    • Input: {}
      • 872 |
    • Method: POST
      • 873 |
    • 874 | URL: 875 | Use the URL below and replace $FLEX_WORKSPACE_SID and $FLEX_WORKFLOW_SID with 877 | your respective values from Step 2. 879 |
      • 880 |
    881 |
    4. 882 |
  4. Make sure you have your Channel of choice is appropriately connected to an AI Assistant by following the steps above." --> "Make sure your Channel of choice is appropriately connected to an AI Assistant by following the steps above.
    5. 883 |
  5. 884 | Send a message to your Assistant through one of those channels and ask it to connect you to a customer agent. 885 |
    6. 886 |
  6. Check your Flex UI to see if you got a new task assigned.
    7. 887 |
888 | 889 |
890 | 897 |
898 |

899 | Check out our documentation for more detailed instructions. 905 |

906 |
907 |
908 |
909 | 910 |
911 |
912 |

Troubleshooting

913 |
    914 |
  • Verify that your channel webhook is configured correctly.
    • 915 |
  • 916 | Check the 917 | Error logs 922 | in the Twilio Console to see if you have any errors. 923 |
    • 924 |
  • 925 | Open the Twilio Functions project by clicking "Edit this 926 | application" above, enable "Live Logs" and retry the scenario 927 | that's failing. 928 |
    • 929 |
930 |
931 |
932 |
933 | 936 | 937 | 938 | --------------------------------------------------------------------------------