├── .gitignore ├── README.md ├── config.new.js ├── package.json ├── run.js ├── tj_raspberrypi ├── package.json └── run_raspberry.js ├── tutorial ├── step1_stt.js ├── step2_tone.js ├── step3_conversation.js └── step4_tts.js └── workspace.json /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | config.js 3 | output.wav 4 | 5 | # Logs 6 | logs 7 | *.log 8 | npm-debug.log* 9 | 10 | # Runtime data 11 | pids 12 | *.pid 13 | *.seed 14 | 15 | # Directory for instrumented libs generated by jscoverage/JSCover 16 | lib-cov 17 | 18 | # Coverage directory used by tools like istanbul 19 | coverage 20 | 21 | # nyc test coverage 22 | .nyc_output 23 | 24 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 25 | .grunt 26 | 27 | # node-waf configuration 28 | .lock-wscript 29 | 30 | # Compiled binary addons (http://nodejs.org/api/addons.html) 31 | build/Release 32 | 33 | # Dependency directories 34 | node_modules 35 | jspm_packages 36 | 37 | # Optional npm cache directory 38 | .npm 39 | 40 | # Optional REPL history 41 | .node_repl_history 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tjbot-raspberrypi-nodejs 2 | A chatbot that cares. 3 | 4 | ## tutorial: 5 | *Part 1:* https://medium.com/ibm-watson-developer-cloud/build-a-chatbot-that-cares-part-1-d1c273e17a63#.r9g1q6e4l 6 | 7 | *Part 2:* https://medium.com/ibm-watson-developer-cloud/build-a-chatbot-that-cares-part-2-66367cf26e4b#.5j6t75sr4 -------------------------------------------------------------------------------- /config.new.js: -------------------------------------------------------------------------------- 1 | // The attention word to wake up the robot. 2 | exports.attentionWord ='watson'; 3 | 4 | // You can change the voice of the robot to your favorite voice. 5 | exports.voice = 'en-US_MichaelVoice'; 6 | // Some of the available options are: 7 | // en-US_AllisonVoice 8 | // en-US_LisaVoice 9 | // en-US_MichaelVoice (the default) 10 | 11 | // Credentials for Watson Speech to Text service 12 | exports.STTUsername = ''; 13 | exports.STTPassword = ''; 14 | 15 | // Credentials for Watson Conversation service 16 | exports.ConUsername = ''; 17 | exports.ConPassword = ''; 18 | exports.ConWorkspace = '' 19 | 20 | //Credentials for Watson Tone Analyzer service 21 | exports.ToneUsername = ''; 22 | exports.TonePassword = ''; 23 | 24 | //Credentials for Watson Text to Speech service 25 | exports.TTSUsername = ''; 26 | exports.TTSPassword = ''; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tjbot-raspberrypi-nodejs", 3 | "version": "1.0.0", 4 | "description": "TJBot has feelings.", 5 | "main": "run.js", 6 | "scripts": { 7 | "start": "node run.js", 8 | "test": "echo \"Error: no test specified\" && exit 1" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git@github.com:boxcarton/tjbot-raspberrypi-nodejs.git" 13 | }, 14 | "author": "Josh Zheng", 15 | "dependencies": { 16 | "mic": "^2.1.1", 17 | "node-ffprobe": "^1.2.2", 18 | "play-sound": "^1.1.1", 19 | "prompt": "^1.0.0", 20 | "raspicam": "git+https://github.com/troyth/node-raspicam.git", 21 | "watson-developer-cloud": "^2.2.0" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /run.js: -------------------------------------------------------------------------------- 1 | const watson = require('watson-developer-cloud'); 2 | const config = require('./config.js') 3 | const fs = require('fs'); 4 | const mic = require('mic'); 5 | const player = require('play-sound')(opts = {}) 6 | const probe = require('node-ffprobe'); 7 | 8 | const attentionWord = config.attentionWord; 9 | 10 | /****************************************************************************** 11 | * Create Watson Services 12 | *******************************************************************************/ 13 | const speechToText = watson.speech_to_text({ 14 | username: config.STTUsername, 15 | password: config.STTPassword, 16 | version: 'v1' 17 | }); 18 | 19 | const toneAnalyzer = watson.tone_analyzer({ 20 | username: config.ToneUsername, 21 | password: config.TonePassword, 22 | version: 'v3', 23 | version_date: '2016-05-19' 24 | }); 25 | 26 | const conversation = watson.conversation({ 27 | username: config.ConUsername, 28 | password: config.ConPassword, 29 | version: 'v1', 30 | version_date: '2016-07-11' 31 | }); 32 | 33 | const textToSpeech = watson.text_to_speech({ 34 | username: config.TTSUsername, 35 | password: config.TTSPassword, 36 | version: 'v1' 37 | }); 38 | 39 | /****************************************************************************** 40 | * Configuring the Microphone 41 | *******************************************************************************/ 42 | const micParams = { 43 | rate: 44100, 44 | channels: 2, 45 | debug: false, 46 | exitOnSilence: 6 47 | } 48 | const micInstance = mic(micParams); 49 | const micInputStream = micInstance.getAudioStream(); 50 | 51 | let pauseDuration = 0; 52 | micInputStream.on('pauseComplete', ()=> { 53 | console.log('Microphone paused for', pauseDuration, 'seconds.'); 54 | setTimeout(function() { 55 | micInstance.resume(); 56 | console.log('Microphone resumed.') 57 | }, Math.round(pauseDuration * 1000)); //Stop listening when speaker is talking 58 | }); 59 | 60 | micInstance.start(); 61 | console.log('TJ is listening, you may speak now.'); 62 | 63 | /****************************************************************************** 64 | * Speech To Text 65 | *******************************************************************************/ 66 | const textStream = micInputStream.pipe( 67 | speechToText.createRecognizeStream({ 68 | content_type: 'audio/l16; rate=44100; channels=2', 69 | })).setEncoding('utf8'); 70 | 71 | /****************************************************************************** 72 | * Get Emotional Tone 73 | *******************************************************************************/ 74 | const getEmotion = (text) => { 75 | return new Promise((resolve) => { 76 | let maxScore = 0; 77 | let emotion = null; 78 | toneAnalyzer.tone({text: text}, (err, tone) => { 79 | let tones = tone.document_tone.tone_categories[0].tones; 80 | for (let i=0; i maxScore){ 82 | maxScore = tones[i].score; 83 | emotion = tones[i].tone_id; 84 | } 85 | } 86 | resolve({emotion, maxScore}); 87 | }) 88 | }) 89 | }; 90 | 91 | /****************************************************************************** 92 | * Text To Speech 93 | *******************************************************************************/ 94 | const speakResponse = (text) => { 95 | const params = { 96 | text: text, 97 | voice: config.voice, 98 | accept: 'audio/wav' 99 | }; 100 | textToSpeech.synthesize(params) 101 | .pipe(fs.createWriteStream('output.wav')) 102 | .on('close', () => { 103 | probe('output.wav', function(err, probeData) { 104 | pauseDuration = probeData.format.duration + 0.2; 105 | micInstance.pause(); 106 | player.play('output.wav'); 107 | }); 108 | }); 109 | } 110 | 111 | /****************************************************************************** 112 | * Conversation 113 | ******************************************************************************/ 114 | let start_dialog = false; 115 | let context = {}; 116 | let watson_response = ''; 117 | 118 | speakResponse('Hi there, I am awake.'); 119 | textStream.on('data', (user_speech_text) => { 120 | user_speech_text = user_speech_text.toLowerCase(); 121 | console.log('Watson hears: ', user_speech_text); 122 | if (user_speech_text.indexOf(attentionWord.toLowerCase()) >= 0) { 123 | start_dialog = true; 124 | } 125 | 126 | if (start_dialog) { 127 | getEmotion(user_speech_text).then((detectedEmotion) => { 128 | context.emotion = detectedEmotion.emotion; 129 | conversation.message({ 130 | workspace_id: config.ConWorkspace, 131 | input: {'text': user_speech_text}, 132 | context: context 133 | }, (err, response) => { 134 | context = response.context; 135 | watson_response = response.output.text[0]; 136 | speakResponse(watson_response); 137 | console.log('Watson says:', watson_response); 138 | if(context.system.dialog_turn_counter == 2) { 139 | context = {}; 140 | start_dialog = false; 141 | } 142 | }); 143 | }); 144 | } else { 145 | console.log('Waiting to hear the word "', attentionWord, '"'); 146 | } 147 | }); 148 | -------------------------------------------------------------------------------- /tj_raspberrypi/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tjbot-raspberrypi-nodejs", 3 | "version": "1.0.0", 4 | "description": "TJBot has feelings.", 5 | "main": "run.js", 6 | "scripts": { 7 | "start": "node run.js", 8 | "test": "echo \"Error: no test specified\" && exit 1" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git@github.com:boxcarton/tjbot-raspberrypi-nodejs.git" 13 | }, 14 | "author": "Josh Zheng", 15 | "dependencies": { 16 | "mic": "^2.1.1", 17 | "node-ffprobe": "^1.2.2", 18 | "watson-developer-cloud": "^2.2.0" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /tj_raspberrypi/run_raspberry.js: -------------------------------------------------------------------------------- 1 | const watson = require('watson-developer-cloud'); 2 | const config = require('./config.js') 3 | const exec = require('child_process').exec; 4 | const fs = require('fs'); 5 | const mic = require('mic'); 6 | const probe = require('node-ffprobe'); 7 | 8 | const attentionWord = config.attentionWord; 9 | 10 | /****************************************************************************** 11 | * Create Watson Services 12 | *******************************************************************************/ 13 | const speechToText = watson.speech_to_text({ 14 | username: config.STTUsername, 15 | password: config.STTPassword, 16 | version: 'v1' 17 | }); 18 | 19 | const toneAnalyzer = watson.tone_analyzer({ 20 | username: config.ToneUsername, 21 | password: config.TonePassword, 22 | version: 'v3', 23 | version_date: '2016-05-19' 24 | }); 25 | 26 | const conversation = watson.conversation({ 27 | username: config.ConUsername, 28 | password: config.ConPassword, 29 | version: 'v1', 30 | version_date: '2016-07-11' 31 | }); 32 | 33 | const textToSpeech = watson.text_to_speech({ 34 | username: config.TTSUsername, 35 | password: config.TTSPassword, 36 | version: 'v1' 37 | }); 38 | 39 | /****************************************************************************** 40 | * Configuring the Microphone 41 | *******************************************************************************/ 42 | const micParams = { 43 | rate: 44100, 44 | channels: 2, 45 | debug: false, 46 | exitOnSilence: 6 47 | } 48 | const micInstance = mic(micParams); 49 | const micInputStream = micInstance.getAudioStream(); 50 | 51 | let pauseDuration = 0; 52 | micInputStream.on('pauseComplete', ()=> { 53 | console.log('Microphone paused for', pauseDuration, 'seconds.'); 54 | setTimeout(function() { 55 | micInstance.resume(); 56 | console.log('Microphone resumed.') 57 | }, Math.round(pauseDuration * 1000)); //Stop listening when speaker is talking 58 | }); 59 | 60 | micInstance.start(); 61 | console.log('TJ is listening, you may speak now.'); 62 | 63 | /****************************************************************************** 64 | * Speech To Text 65 | *******************************************************************************/ 66 | const textStream = micInputStream.pipe( 67 | speechToText.createRecognizeStream({ 68 | content_type: 'audio/l16; rate=44100; channels=2', 69 | })).setEncoding('utf8'); 70 | 71 | /****************************************************************************** 72 | * Get Emotional Tone 73 | *******************************************************************************/ 74 | const getEmotion = (text) => { 75 | return new Promise((resolve) => { 76 | let maxScore = 0; 77 | let emotion = null; 78 | toneAnalyzer.tone({text: text}, (err, tone) => { 79 | let tones = tone.document_tone.tone_categories[0].tones; 80 | for (let i=0; i maxScore){ 82 | maxScore = tones[i].score; 83 | emotion = tones[i].tone_id; 84 | } 85 | } 86 | resolve({emotion, maxScore}); 87 | }) 88 | }) 89 | }; 90 | 91 | 92 | /****************************************************************************** 93 | * Text To Speech 94 | *******************************************************************************/ 95 | const speakResponse = (text) => { 96 | const params = { 97 | text: text, 98 | voice: config.voice, 99 | accept: 'audio/wav' 100 | }; 101 | textToSpeech.synthesize(params) 102 | .pipe(fs.createWriteStream('output.wav')) 103 | .on('close', () => { 104 | probe('output.wav', (err, probeData) => { 105 | pauseDuration = probeData.format.duration + 0.2; 106 | micInstance.pause(); 107 | exec('aplay output.wav', (error, stdout, stderr) => { 108 | if (error !== null) { 109 | console.log('exec error: ' + error); 110 | } 111 | }); 112 | }); 113 | }); 114 | } 115 | 116 | /****************************************************************************** 117 | * Conversation 118 | ******************************************************************************/ 119 | let start_dialog = false; 120 | let context = {}; 121 | let watson_response = ''; 122 | 123 | speakResponse('Hi there, I am awake.'); 124 | textStream.on('data', (user_speech_text) => { 125 | user_speech_text = user_speech_text.toLowerCase(); 126 | console.log('Watson hears: ', user_speech_text); 127 | if (user_speech_text.indexOf(attentionWord.toLowerCase()) >= 0) { 128 | start_dialog = true; 129 | } 130 | 131 | if (start_dialog) { 132 | getEmotion(user_speech_text).then((detectedEmotion) => { 133 | context.emotion = detectedEmotion.emotion; 134 | conversation.message({ 135 | workspace_id: config.ConWorkspace, 136 | input: {'text': user_speech_text}, 137 | context: context 138 | }, (err, response) => { 139 | context = response.context; 140 | watson_response = response.output.text[0]; 141 | speakResponse(watson_response); 142 | console.log('Watson says:', watson_response); 143 | if (context.system.dialog_turn_counter == 2) { 144 | context = {}; 145 | start_dialog = false; 146 | } 147 | }); 148 | }); 149 | } else { 150 | console.log('Waiting to hear the word "', attentionWord, '"'); 151 | } 152 | }); -------------------------------------------------------------------------------- /tutorial/step1_stt.js: -------------------------------------------------------------------------------- 1 | const watson = require('watson-developer-cloud'); 2 | const mic = require('mic'); 3 | const config = require('../config.js'); 4 | 5 | const micParams = { 6 | rate: 44100, 7 | channels: 2, 8 | debug: false, 9 | exitOnSilence: 6 10 | } 11 | const micInstance = mic(micParams); 12 | const micInputStream = micInstance.getAudioStream(); 13 | micInstance.start(); 14 | 15 | console.log('Watson is listening, you may speak now.'); 16 | 17 | const speechToText = watson.speech_to_text({ 18 | username: config.STTUsername, 19 | password: config.STTPassword, 20 | version: 'v1' 21 | }); 22 | 23 | const textStream = micInputStream.pipe( 24 | speechToText.createRecognizeStream({ 25 | content_type: 'audio/l16; rate=44100; channels=2' 26 | })).setEncoding('utf8'); 27 | 28 | textStream.on('data', (user_speech_text) => { 29 | console.log('Watson hears:', user_speech_text); 30 | }); -------------------------------------------------------------------------------- /tutorial/step2_tone.js: -------------------------------------------------------------------------------- 1 | const watson = require('watson-developer-cloud'); 2 | const config = require('../config.js'); 3 | 4 | const tone_analyzer = watson.tone_analyzer({ 5 | username: config.ToneUsername, 6 | password: config.TonePassword, 7 | version: 'v3', 8 | version_date: '2016-05-19' 9 | }); 10 | 11 | let text = 'I love watson'; 12 | tone_analyzer.tone({text: text}, (err, tone) => { 13 | console.log(JSON.stringify(tone, null, 2)); 14 | }); -------------------------------------------------------------------------------- /tutorial/step3_conversation.js: -------------------------------------------------------------------------------- 1 | const watson = require('watson-developer-cloud'); //to connect to Watson developer cloud 2 | const config = require('../config.js') // to get our credentials and the attention word from the config.js files 3 | const prompt = require('prompt'); 4 | 5 | const conversation = watson.conversation({ 6 | username: config.ConUsername, 7 | password: config.ConPassword, 8 | version: 'v1', 9 | version_date: '2016-07-11' 10 | }); 11 | 12 | prompt.start(); 13 | 14 | let context = {}; 15 | let converse = () => 16 | prompt.get('input', (err, result) => { 17 | 18 | context.emotion = 'sadness'; //replace with results from Tone Analzyer 19 | conversation.message({ 20 | workspace_id: config.ConWorkspace, 21 | input: {'text': result.input}, 22 | context: context 23 | }, (err, response) => { 24 | context = response.context; 25 | watson_response = response.output.text[0]; 26 | console.log('Watson says:', watson_response); 27 | }); 28 | 29 | converse(); 30 | }) 31 | 32 | converse(); -------------------------------------------------------------------------------- /tutorial/step4_tts.js: -------------------------------------------------------------------------------- 1 | const watson = require('watson-developer-cloud'); //to connect to Watson developer cloud 2 | const config = require('../config.js') // to get our credentials and the attention word from the config.js files 3 | const fs = require('fs'); 4 | const player = require('play-sound')(opts = {}) 5 | 6 | const text_to_speech = watson.text_to_speech({ 7 | username: config.TTSUsername, 8 | password: config.TTSPassword, 9 | version: 'v1' 10 | }); 11 | 12 | let text = 'Hey guys, I am Watson' 13 | 14 | const params = { 15 | text: text, 16 | voice: config.voice, 17 | accept: 'audio/wav' 18 | }; 19 | 20 | text_to_speech.synthesize(params) 21 | .pipe(fs.createWriteStream('output.wav')) 22 | .on('close', () => { 23 | player.play('output.wav'); 24 | }); -------------------------------------------------------------------------------- /workspace.json: -------------------------------------------------------------------------------- 1 | {"name":"TJBot","created":"2016-11-23T04:41:17.435Z","intents":[],"updated":"2016-12-04T00:47:46.356Z","entities":[{"type":null,"entity":"watson","source":null,"values":[{"value":"watson","created":"2016-12-01T18:49:38.772Z","updated":"2016-12-01T18:49:43.308Z","metadata":null,"synonyms":["Watson"]}],"created":"2016-12-01T18:40:56.389Z","updated":"2016-12-01T18:40:56.389Z","open_list":false,"description":null}],"language":"en","metadata":{"runtime_version":"2016-09-20"},"description":"Voice interface + tone analyzer","dialog_nodes":[{"go_to":null,"output":{"text":"Don't be scared. Life will get better soon. This too shall pass."},"parent":"node_3_1479876818619","context":null,"created":"2016-12-01T08:10:48.544Z","updated":"2016-12-02T03:44:39.646Z","metadata":null,"conditions":"$emotion == \"fear\"","description":null,"dialog_node":"node_3_1480579848344","previous_sibling":"node_6_1480581990022"},{"go_to":null,"output":{"text":"Hey, how are you feeling today?"},"parent":null,"context":null,"created":"2016-11-23T04:53:38.805Z","updated":"2016-12-01T18:46:48.144Z","metadata":null,"conditions":"@watson","description":null,"dialog_node":"node_3_1479876818619","previous_sibling":null},{"go_to":null,"output":{"text":"I hope whatever's making you angry gets fixed soon!"},"parent":"node_3_1479876818619","context":null,"created":"2016-12-01T08:10:16.725Z","updated":"2016-12-01T18:35:40.710Z","metadata":null,"conditions":"$emotion == \"anger\"","description":null,"dialog_node":"node_2_1480579816329","previous_sibling":"node_3_1480579848344"},{"go_to":null,"output":{"text":"I'm sorry, I only respond to Watson and feelings."},"parent":null,"context":null,"created":"2016-11-23T04:53:51.016Z","updated":"2016-12-04T00:47:46.356Z","metadata":null,"conditions":"anything_else","description":null,"dialog_node":"node_4_1479876830818","previous_sibling":"node_3_1479876818619"},{"go_to":null,"output":{"text":"I'm sorry you're feeling down, I hope you feel better soon."},"parent":"node_3_1479876818619","context":null,"created":"2016-11-30T18:30:57.457Z","updated":"2016-12-01T08:09:54.048Z","metadata":null,"conditions":"$emotion == \"sadness\"","description":null,"dialog_node":"node_1_1480530657191","previous_sibling":"node_1_1480530555300"},{"go_to":null,"output":{"text":"Yay, you're happy, That makes me happy."},"parent":"node_3_1479876818619","context":null,"created":"2016-11-30T18:29:15.573Z","updated":"2016-12-02T08:28:44.070Z","metadata":null,"conditions":"$emotion == \"joy\"","description":null,"dialog_node":"node_1_1480530555300","previous_sibling":"node_2_1480579816329"},{"go_to":null,"output":{"text":"Yikes, sorry to hear you're disgusted by that."},"parent":"node_3_1479876818619","context":null,"created":"2016-12-01T08:46:30.333Z","updated":"2016-12-01T18:35:31.072Z","metadata":null,"conditions":"$emotion == \"disgust\"","description":null,"dialog_node":"node_6_1480581990022","previous_sibling":null}],"workspace_id":"804dfe69-89f7-402e-97ef-054ffa0864ab","counterexamples":[]} --------------------------------------------------------------------------------