├── ExampleStreaming.cs ├── ExampleStreaming2.cs ├── ExampleStreaming3RobotEmotion.cs ├── ExampleStreaming4RobotEmotion.cs ├── ExampleStreaming5RobotEmotion.cs ├── ExampleStreaming6.cs ├── ExampleStreaming7.cs ├── ExampleStreaming_plus_Text_to_Speech_expressive.cs ├── LT2.zip ├── LanguageTranslatorDemo.cs ├── README.md ├── apples.cs ├── blackie.zip ├── cameraperspective.cs ├── colorbands.cs ├── colors.cs ├── eeg_emotiv_key_mapping.cs ├── examplestreaming_Plus_rainbow_octopusprotoype.cs ├── moveobject.cs ├── objects.png ├── octopus_color.cs ├── octopus_color2.cs ├── orb.zip └── sizeobject.cs /ExampleStreaming.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 IBM Corp. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | using UnityEngine; 19 | using System.Collections; 20 | using IBM.Watson.DeveloperCloud.Logging; 21 | using IBM.Watson.DeveloperCloud.Services.SpeechToText.v1; 22 | using IBM.Watson.DeveloperCloud.Utilities; 23 | using IBM.Watson.DeveloperCloud.DataTypes; 24 | using System.Collections.Generic; 25 | using UnityEngine.UI; 26 | 27 | public class ExampleStreaming : MonoBehaviour 28 | { 29 | private string _username = "d8406233-74d3-####-9792-f#####"; 30 | private string _password = "zFP######"; 31 | private string _url = "https://stream.watsonplatform.net/speech-to-text/api"; 32 | 33 | public Text ResultsField; 34 | public Text ResponseTextField_en; 35 | 36 | public LanguageTranslatorDemo LanguageTranslatorDemo; 37 | 38 | private int _recordingRoutine = 0; 39 | private string _microphoneID = null; 40 | private AudioClip _recording = null; 41 | private int _recordingBufferSize = 1; 42 | private int _recordingHZ = 22050; 43 | 44 | private SpeechToText _speechToText; 45 | 46 | void Start() 47 | { 48 | LogSystem.InstallDefaultReactors(); 49 | 50 | // Create credential and instantiate service 51 | Credentials credentials = new Credentials(_username, _password, _url); 52 | 53 | _speechToText = new SpeechToText(credentials); 54 | Active = true; 55 | 56 | StartRecording(); 57 | } 58 | 59 | public bool Active 60 | { 61 | get { return _speechToText.IsListening; } 62 | set 63 | { 64 | if (value && !_speechToText.IsListening) 65 | { 66 | _speechToText.DetectSilence = true; 67 | _speechToText.EnableWordConfidence = true; 68 | _speechToText.EnableTimestamps = true; 69 | _speechToText.SilenceThreshold = 0.01f; 70 | _speechToText.MaxAlternatives = 0; 71 | _speechToText.EnableInterimResults = true; 72 | _speechToText.OnError = OnError; 73 | _speechToText.InactivityTimeout = -1; 74 | _speechToText.ProfanityFilter = false; 75 | _speechToText.SmartFormatting = true; 76 | _speechToText.SpeakerLabels = false; 77 | _speechToText.WordAlternativesThreshold = null; 78 | _speechToText.StartListening(OnRecognize, OnRecognizeSpeaker); 79 | } 80 | else if (!value && _speechToText.IsListening) 81 | { 82 | _speechToText.StopListening(); 83 | } 84 | } 85 | } 86 | 87 | private void StartRecording() 88 | { 89 | if (_recordingRoutine == 0) 90 | { 91 | UnityObjectUtil.StartDestroyQueue(); 92 | _recordingRoutine = Runnable.Run(RecordingHandler()); 93 | } 94 | } 95 | 96 | private void StopRecording() 97 | { 98 | if (_recordingRoutine != 0) 99 | { 100 | Microphone.End(_microphoneID); 101 | Runnable.Stop(_recordingRoutine); 102 | _recordingRoutine = 0; 103 | } 104 | } 105 | 106 | private void OnError(string error) 107 | { 108 | Active = false; 109 | 110 | Log.Debug("ExampleStreaming.OnError()", "Error! {0}", error); 111 | } 112 | 113 | private IEnumerator RecordingHandler() 114 | { 115 | Log.Debug("ExampleStreaming.RecordingHandler()", "devices: {0}", Microphone.devices); 116 | _recording = Microphone.Start(_microphoneID, true, _recordingBufferSize, _recordingHZ); 117 | yield return null; // let _recordingRoutine get set.. 118 | 119 | if (_recording == null) 120 | { 121 | StopRecording(); 122 | yield break; 123 | } 124 | 125 | bool bFirstBlock = true; 126 | int midPoint = _recording.samples / 2; 127 | float[] samples = null; 128 | 129 | while (_recordingRoutine != 0 && _recording != null) 130 | { 131 | int writePos = Microphone.GetPosition(_microphoneID); 132 | if (writePos > _recording.samples || !Microphone.IsRecording(_microphoneID)) 133 | { 134 | Log.Error("ExampleStreaming.RecordingHandler()", "Microphone disconnected."); 135 | 136 | StopRecording(); 137 | yield break; 138 | } 139 | 140 | if ((bFirstBlock && writePos >= midPoint) 141 | || (!bFirstBlock && writePos < midPoint)) 142 | { 143 | // front block is recorded, make a RecordClip and pass it onto our callback. 144 | samples = new float[midPoint]; 145 | _recording.GetData(samples, bFirstBlock ? 0 : midPoint); 146 | 147 | AudioData record = new AudioData(); 148 | record.MaxLevel = Mathf.Max(Mathf.Abs(Mathf.Min(samples)), Mathf.Max(samples)); 149 | record.Clip = AudioClip.Create("Recording", midPoint, _recording.channels, _recordingHZ, false); 150 | record.Clip.SetData(samples, 0); 151 | 152 | _speechToText.OnListen(record); 153 | 154 | bFirstBlock = !bFirstBlock; 155 | } 156 | else 157 | { 158 | // calculate the number of samples remaining until we ready for a block of audio, 159 | // and wait that amount of time it will take to record. 160 | int remaining = bFirstBlock ? (midPoint - writePos) : (_recording.samples - writePos); 161 | float timeRemaining = (float)remaining / (float)_recordingHZ; 162 | 163 | yield return new WaitForSeconds(timeRemaining); 164 | } 165 | 166 | } 167 | 168 | yield break; 169 | } 170 | 171 | private void OnRecognize(SpeechRecognitionEvent result) 172 | { 173 | if (result != null && result.results.Length > 0) 174 | { 175 | foreach (var res in result.results) 176 | { 177 | foreach (var alt in res.alternatives) 178 | { 179 | string text = string.Format("{0} ({1}, {2:0.00})\n", alt.transcript, res.final ? "Final" : "Interim", alt.confidence); 180 | Log.Debug("ExampleStreaming.OnRecognize()", text); 181 | // ResultsField.text = text; 182 | ResponseTextField_en.text = text; 183 | LanguageTranslatorDemo.Translate_es(alt.transcript); 184 | LanguageTranslatorDemo.Translate_fr(alt.transcript); 185 | LanguageTranslatorDemo.Translate_it(alt.transcript); 186 | LanguageTranslatorDemo.Translate_de(alt.transcript); 187 | LanguageTranslatorDemo.Translate_pt(alt.transcript); 188 | LanguageTranslatorDemo.Translate_ar(alt.transcript); 189 | LanguageTranslatorDemo.Translate_ja(alt.transcript); 190 | 191 | } 192 | 193 | if (res.keywords_result != null && res.keywords_result.keyword != null) 194 | { 195 | foreach (var keyword in res.keywords_result.keyword) 196 | { 197 | Log.Debug("ExampleStreaming.OnRecognize()", "keyword: {0}, confidence: {1}, start time: {2}, end time: {3}", keyword.normalized_text, keyword.confidence, keyword.start_time, keyword.end_time); 198 | } 199 | } 200 | 201 | if (res.word_alternatives != null) 202 | { 203 | foreach (var wordAlternative in res.word_alternatives) 204 | { 205 | Log.Debug("ExampleStreaming.OnRecognize()", "Word alternatives found. Start time: {0} | EndTime: {1}", wordAlternative.start_time, wordAlternative.end_time); 206 | foreach(var alternative in wordAlternative.alternatives) 207 | Log.Debug("ExampleStreaming.OnRecognize()", "\t word: {0} | confidence: {1}", alternative.word, alternative.confidence); 208 | } 209 | } 210 | } 211 | } 212 | } 213 | 214 | private void OnRecognizeSpeaker(SpeakerRecognitionEvent result) 215 | { 216 | if (result != null) 217 | { 218 | foreach (SpeakerLabelsResult labelResult in result.speaker_labels) 219 | { 220 | Log.Debug("ExampleStreaming.OnRecognize()", string.Format("speaker result: {0} | confidence: {3} | from: {1} | to: {2}", labelResult.speaker, labelResult.from, labelResult.to, labelResult.confidence)); 221 | } 222 | } 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /ExampleStreaming2.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * ############# HELLO VIRTUAL WORLD 4 | * Copyright 2015 IBM Corp. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * THIS IS VERY VERY VERY ROUGH CODE - WARNING :) 18 | */ 19 | 20 | using UnityEngine; 21 | using System.Collections; 22 | using IBM.Watson.DeveloperCloud.Logging; 23 | using IBM.Watson.DeveloperCloud.Services.SpeechToText.v1; 24 | using IBM.Watson.DeveloperCloud.Utilities; 25 | using IBM.Watson.DeveloperCloud.DataTypes; 26 | using System.Collections.Generic; 27 | using UnityEngine.UI; 28 | 29 | // added this from the TONE ANALYZER . CS file 30 | using IBM.Watson.DeveloperCloud.Services.ToneAnalyzer.v3; 31 | using IBM.Watson.DeveloperCloud.Connection; 32 | 33 | // and for Regex 34 | using System.Text.RegularExpressions; 35 | 36 | 37 | public class ExampleStreaming : MonoBehaviour 38 | { 39 | private string _username_STT = "d###############a7"; 40 | private string _password_STT = "#############"; 41 | private string _url_STT = "https://stream.watsonplatform.net/speech-to-text/api"; 42 | 43 | public Text ResultsField; 44 | 45 | private SpeechToText _speechToText; 46 | 47 | // TONE ZONE 48 | private string _username_TONE = "3###########################9e"; 49 | private string _password_TONE = "###########"; 50 | private string _url_TONE = "https://gateway.watsonplatform.net/tone-analyzer/api"; 51 | 52 | private ToneAnalyzer _toneAnalyzer; 53 | private string _toneAnalyzerVersionDate = "2017-05-26"; 54 | private string _stringToTestTone1 = "START AND TEST - But this totally sucks! I hate beans and liver!"; 55 | private string _stringToTestTone2 = "SECOND TEST - Failed Test Sucks"; 56 | private bool _analyzeToneTested = false; 57 | 58 | 59 | // magic 60 | //public GameObject sphere_rad; 61 | public MeshRenderer sphereMeshRenderer; 62 | public MeshRenderer cubeMeshRenderer; 63 | 64 | public MeshRenderer bar1JoyRenderer; 65 | public MeshRenderer bar2SadnessRenderer; 66 | public MeshRenderer bar3FearRenderer; 67 | public MeshRenderer bar4DisgustRenderer; 68 | public MeshRenderer bar5AngerRenderer; 69 | 70 | 71 | public Material original_material; 72 | public Material red_material; 73 | public Material blue_material; 74 | 75 | public Material yellow_material; 76 | public Material green_material; 77 | public Material purple_material; 78 | 79 | private int _recordingRoutine = 0; 80 | private string _microphoneID = null; 81 | private AudioClip _recording = null; 82 | private int _recordingBufferSize = 1; 83 | private int _recordingHZ = 22050; 84 | 85 | 86 | 87 | void Start() 88 | { 89 | LogSystem.InstallDefaultReactors(); 90 | 91 | // Create credential and instantiate service 92 | Credentials credentials_STT = new Credentials(_username_STT, _password_STT, _url_STT); 93 | 94 | _speechToText = new SpeechToText(credentials_STT); 95 | Active = true; 96 | 97 | StartRecording(); 98 | 99 | 100 | // TONE ZONE 101 | Credentials credentials_TONE = new Credentials(_username_TONE, _password_TONE, _url_TONE); 102 | _toneAnalyzer = new ToneAnalyzer(credentials_TONE); 103 | _toneAnalyzer.VersionDate = _toneAnalyzerVersionDate; 104 | 105 | //This is a "on first run" test 106 | //Runnable.Run(Examples()); // example on pump prime 107 | } 108 | 109 | public bool Active 110 | { 111 | get { return _speechToText.IsListening; } 112 | set 113 | { 114 | if (value && !_speechToText.IsListening) 115 | { 116 | _speechToText.DetectSilence = true; 117 | _speechToText.EnableWordConfidence = true; 118 | _speechToText.EnableTimestamps = true; 119 | _speechToText.SilenceThreshold = 0.01f; 120 | _speechToText.MaxAlternatives = 0; 121 | _speechToText.EnableInterimResults = true; 122 | _speechToText.OnError = OnError; 123 | _speechToText.InactivityTimeout = -1; 124 | _speechToText.ProfanityFilter = false; 125 | _speechToText.SmartFormatting = true; 126 | _speechToText.SpeakerLabels = false; 127 | _speechToText.WordAlternativesThreshold = null; 128 | _speechToText.StartListening(OnRecognize, OnRecognizeSpeaker); 129 | } 130 | else if (!value && _speechToText.IsListening) 131 | { 132 | _speechToText.StopListening(); 133 | } 134 | } 135 | } 136 | 137 | private void StartRecording() 138 | { 139 | if (_recordingRoutine == 0) 140 | { 141 | UnityObjectUtil.StartDestroyQueue(); 142 | _recordingRoutine = Runnable.Run(RecordingHandler()); 143 | } 144 | } 145 | 146 | private void StopRecording() 147 | { 148 | if (_recordingRoutine != 0) 149 | { 150 | Microphone.End(_microphoneID); 151 | Runnable.Stop(_recordingRoutine); 152 | _recordingRoutine = 0; 153 | } 154 | } 155 | 156 | private void OnError(string error) 157 | { 158 | Active = false; 159 | 160 | Log.Debug("ExampleStreaming.OnError()", "Error! {0}", error); 161 | } 162 | 163 | private IEnumerator RecordingHandler() 164 | { 165 | Log.Debug("ExampleStreaming.RecordingHandler()", "devices: {0}", Microphone.devices); 166 | _recording = Microphone.Start(_microphoneID, true, _recordingBufferSize, _recordingHZ); 167 | yield return null; // let _recordingRoutine get set.. 168 | 169 | if (_recording == null) 170 | { 171 | StopRecording(); 172 | yield break; 173 | } 174 | 175 | bool bFirstBlock = true; 176 | int midPoint = _recording.samples / 2; 177 | float[] samples = null; 178 | 179 | while (_recordingRoutine != 0 && _recording != null) 180 | { 181 | int writePos = Microphone.GetPosition(_microphoneID); 182 | if (writePos > _recording.samples || !Microphone.IsRecording(_microphoneID)) 183 | { 184 | Log.Error("ExampleStreaming.RecordingHandler()", "Microphone disconnected."); 185 | 186 | StopRecording(); 187 | yield break; 188 | } 189 | 190 | if ((bFirstBlock && writePos >= midPoint) 191 | || (!bFirstBlock && writePos < midPoint)) 192 | { 193 | // front block is recorded, make a RecordClip and pass it onto our callback. 194 | samples = new float[midPoint]; 195 | _recording.GetData(samples, bFirstBlock ? 0 : midPoint); 196 | 197 | AudioData record = new AudioData(); 198 | record.MaxLevel = Mathf.Max(Mathf.Abs(Mathf.Min(samples)), Mathf.Max(samples)); 199 | record.Clip = AudioClip.Create("Recording", midPoint, _recording.channels, _recordingHZ, false); 200 | record.Clip.SetData(samples, 0); 201 | 202 | _speechToText.OnListen(record); 203 | 204 | bFirstBlock = !bFirstBlock; 205 | } 206 | else 207 | { 208 | // calculate the number of samples remaining until we ready for a block of audio, 209 | // and wait that amount of time it will take to record. 210 | int remaining = bFirstBlock ? (midPoint - writePos) : (_recording.samples - writePos); 211 | float timeRemaining = (float)remaining / (float)_recordingHZ; 212 | 213 | yield return new WaitForSeconds(timeRemaining); 214 | } 215 | 216 | } 217 | 218 | yield break; 219 | } 220 | 221 | 222 | 223 | // TONE ZONE 224 | private IEnumerator Examples() 225 | { 226 | // Analyze tone 227 | if (!_toneAnalyzer.GetToneAnalyze(OnGetToneAnalyze, OnFail, _stringToTestTone1)) 228 | Log.Debug("ExampleToneAnalyzer.Examples()", "Failed to analyze!"); 229 | 230 | while (!_analyzeToneTested) 231 | yield return null; 232 | 233 | Log.Debug("ExampleToneAnalyzer.Examples()", "Tone analyzer examples complete."); 234 | } 235 | 236 | 237 | // TESTING 238 | private void OnGetToneAnalyze(ToneAnalyzerResponse resp, Dictionary customData) 239 | { 240 | Log.Debug("ExampleToneAnalyzer.OnGetToneAnalyze()", "{0}", customData["json"].ToString()); 241 | 242 | //ResultsField.text = (customData["json"].ToString()); // works but long and cannot read 243 | 244 | //string XYZ = string.Format("XYZ"); // works 1/2 245 | //ResultsField.text = XYZ; // works 2/2 246 | 247 | //string XYZ = string.Concat("hello ", "world \n", "ABCDE"); // works 1/2 248 | //ResultsField.text = XYZ; // works 2/2 249 | 250 | string RAW = (customData["json"].ToString()); // works but long and cannot read 251 | //RAW = string.Concat("Tone Response \n", RAW); 252 | RAW = Regex.Replace(RAW, "tone_categories", " \\\n"); 253 | RAW = Regex.Replace(RAW, "}", "} \\\n"); 254 | RAW = Regex.Replace(RAW, "tone_id", " "); 255 | RAW = Regex.Replace(RAW, "tone_name", " "); 256 | RAW = Regex.Replace(RAW, "score", " "); 257 | RAW = Regex.Replace(RAW, @"[{\\},:]", ""); 258 | RAW = Regex.Replace(RAW, "\"", ""); 259 | ResultsField.text = RAW; 260 | 261 | _analyzeToneTested = true; 262 | } 263 | 264 | private void OnFail(RESTConnector.Error error, Dictionary customData) 265 | { 266 | Log.Error("ExampleRetrieveAndRank.OnFail()", "Error received: {0}", error.ToString()); 267 | } 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | private void OnRecognize(SpeechRecognitionEvent result) 276 | { 277 | if (result != null && result.results.Length > 0) 278 | { 279 | foreach (var res in result.results) 280 | { 281 | foreach (var alt in res.alternatives) 282 | { 283 | string text = string.Format("{0} ({1}, {2:0.00})\n", alt.transcript, res.final ? "Final" : "Interim", alt.confidence); 284 | Log.Debug("ExampleStreaming.OnRecognize()", text); 285 | ResultsField.text = text; 286 | 287 | // Magic happens here (Hello World Magic!) 288 | // FOR NOW HARD WIRING THE EXPLICIT UTTERANCES TO COLOR STATE CHANGES 289 | // LATER WILL USE LOOKUPS; THRESHOLDES; AND STATEHOLDERS 290 | if (alt.transcript.Contains("red")) { 291 | sphereMeshRenderer.material = red_material; 292 | } 293 | if (alt.transcript.Contains("blue")) { 294 | sphereMeshRenderer.material = blue_material; 295 | } 296 | if (alt.transcript.Contains("green")) { 297 | cubeMeshRenderer.material = green_material; 298 | } 299 | if (alt.transcript.Contains("yellow")) { 300 | cubeMeshRenderer.material = yellow_material; 301 | } 302 | 303 | // Here is the Emotional Zone - GREP For now 304 | if (alt.transcript.Contains("happy") | alt.transcript.Contains("joy")) { 305 | bar1JoyRenderer.material = yellow_material; 306 | } 307 | if (alt.transcript.Contains("sad") | alt.transcript.Contains("depressed")) { 308 | bar2SadnessRenderer.material = blue_material; 309 | } 310 | if (alt.transcript.Contains("scared") | alt.transcript.Contains("fear")) { 311 | bar3FearRenderer.material = purple_material; 312 | } 313 | if (alt.transcript.Contains("yucky") | alt.transcript.Contains("gross")) { 314 | bar4DisgustRenderer.material = green_material; 315 | } 316 | if (alt.transcript.Contains("mad") | alt.transcript.Contains("angry")) { 317 | bar5AngerRenderer.material = red_material; 318 | } 319 | 320 | // ENTERING THE TONE ZONE - when the utterance contains this word 321 | if (alt.transcript.Contains("tone") | alt.transcript.Contains("emotion")) { 322 | // if the utterance 323 | // Runnable.Run(Examples()); // this compiled - it's simply the same test from startup 324 | 325 | 326 | string GHI = alt.transcript; 327 | if (!_toneAnalyzer.GetToneAnalyze(OnGetToneAnalyze, OnFail, GHI)) 328 | Log.Debug("ExampleToneAnalyzer.Examples()", "Failed to analyze!"); 329 | 330 | // TEST START 331 | // Analyze tone 332 | // if (!_toneAnalyzer.GetToneAnalyze(OnGetToneAnalyze, OnFail, _stringToTestTone2)) 333 | // Log.Debug("ExampleToneAnalyzer.Examples()", "Failed to analyze!"); 334 | 335 | Log.Debug("ExampleToneAnalyzer.Examples()", "NESTED TONE ZONE branch complete."); 336 | //ResultsField.text = "tone analyzed! 111"; 337 | // TEST END 338 | 339 | } 340 | 341 | 342 | 343 | } 344 | 345 | if (res.keywords_result != null && res.keywords_result.keyword != null) 346 | { 347 | foreach (var keyword in res.keywords_result.keyword) 348 | { 349 | Log.Debug("ExampleStreaming.OnRecognize()", "keyword: {0}, confidence: {1}, start time: {2}, end time: {3}", keyword.normalized_text, keyword.confidence, keyword.start_time, keyword.end_time); 350 | //ResultsField.text = "tone analyzed! 222"; 351 | } 352 | } 353 | 354 | if (res.word_alternatives != null) 355 | { 356 | foreach (var wordAlternative in res.word_alternatives) 357 | { 358 | Log.Debug("ExampleStreaming.OnRecognize()", "Word alternatives found. Start time: {0} | EndTime: {1}", wordAlternative.start_time, wordAlternative.end_time); 359 | foreach(var alternative in wordAlternative.alternatives) 360 | Log.Debug("ExampleStreaming.OnRecognize()", "\t word: {0} | confidence: {1}", alternative.word, alternative.confidence); 361 | } 362 | } 363 | } 364 | } 365 | } 366 | 367 | private void OnRecognizeSpeaker(SpeakerRecognitionEvent result) 368 | { 369 | if (result != null) 370 | { 371 | foreach (SpeakerLabelsResult labelResult in result.speaker_labels) 372 | { 373 | Log.Debug("ExampleStreaming.OnRecognize()", string.Format("speaker result: {0} | confidence: {3} | from: {1} | to: {2}", labelResult.speaker, labelResult.from, labelResult.to, labelResult.confidence)); 374 | } 375 | } 376 | } 377 | } 378 | -------------------------------------------------------------------------------- /ExampleStreaming3RobotEmotion.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * ############# HELLO VIRTUAL WORLD 4 | * Copyright 2015 IBM Corp. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | 18 | * THIS IS VERY VERY VERY ROUGH CODE - WARNING :) 19 | * THIS IS VERY VERY VERY ROUGH CODE - WARNING :) 20 | * THIS IS VERY VERY VERY ROUGH CODE - WARNING :) 21 | * THIS IS VERY VERY VERY ROUGH CODE - WARNING :) 22 | 23 | */ 24 | 25 | using UnityEngine; 26 | using System.Collections; 27 | using IBM.Watson.DeveloperCloud.Logging; 28 | using IBM.Watson.DeveloperCloud.Services.SpeechToText.v1; 29 | using IBM.Watson.DeveloperCloud.Utilities; 30 | using IBM.Watson.DeveloperCloud.DataTypes; 31 | using System.Collections.Generic; 32 | using UnityEngine.UI; 33 | 34 | // added this from the TONE ANALYZER . CS file 35 | using IBM.Watson.DeveloperCloud.Services.ToneAnalyzer.v3; 36 | using IBM.Watson.DeveloperCloud.Connection; 37 | 38 | // and for Regex 39 | using System.Text.RegularExpressions; 40 | 41 | 42 | public class ExampleStreaming : MonoBehaviour 43 | { 44 | 45 | private string _username_STT = "####"; 46 | private string _password_STT = "####"; 47 | private string _url_STT = "https://stream.watsonplatform.net/speech-to-text/api"; 48 | 49 | public Text ResultsField; 50 | public Text CactusField; 51 | 52 | public float emotion_threshold; 53 | 54 | private SpeechToText _speechToText; 55 | 56 | // TONE ZONE 57 | private string _username_TONE = "####"; 58 | private string _password_TONE = "####"; 59 | private string _url_TONE = "https://gateway.watsonplatform.net/tone-analyzer/api"; 60 | 61 | 62 | private ToneAnalyzer _toneAnalyzer; 63 | private string _toneAnalyzerVersionDate = "2017-05-26"; 64 | private string _stringToTestTone1 = "START AND TEST - But this totally sucks! I hate beans and liver!"; 65 | private string _stringToTestTone2 = "SECOND TEST - Failed Test Sucks"; 66 | private bool _analyzeToneTested = false; 67 | 68 | 69 | 70 | // magic 71 | //public GameObject sphere_rad; 72 | public MeshRenderer sphereMeshRenderer; 73 | public MeshRenderer cubeMeshRenderer; 74 | 75 | public MeshRenderer bar1JoyRenderer; 76 | public MeshRenderer bar2SadnessRenderer; 77 | public MeshRenderer bar3FearRenderer; 78 | public MeshRenderer bar4DisgustRenderer; 79 | public MeshRenderer bar5AngerRenderer; 80 | 81 | public MeshRenderer DroidRenderer; 82 | 83 | public MeshRenderer cyl1AnalyticalRenderer; 84 | public MeshRenderer cyl2ConfidentRenderer; 85 | public MeshRenderer cyl3TentativeRenderer; 86 | 87 | public Material original_material; 88 | public Material red_material; 89 | public Material blue_material; 90 | public Material yellow_material; 91 | public Material green_material; 92 | public Material purple_material; 93 | public Material white_material; 94 | 95 | private int _recordingRoutine = 0; 96 | private string _microphoneID = null; 97 | private AudioClip _recording = null; 98 | private int _recordingBufferSize = 1; 99 | private int _recordingHZ = 22050; 100 | 101 | 102 | 103 | void Start() 104 | { 105 | LogSystem.InstallDefaultReactors(); 106 | 107 | // Create credential and instantiate service 108 | Credentials credentials_STT = new Credentials(_username_STT, _password_STT, _url_STT); 109 | 110 | _speechToText = new SpeechToText(credentials_STT); 111 | Active = true; 112 | 113 | StartRecording(); 114 | 115 | 116 | // TONE ZONE 117 | Credentials credentials_TONE = new Credentials(_username_TONE, _password_TONE, _url_TONE); 118 | _toneAnalyzer = new ToneAnalyzer(credentials_TONE); 119 | _toneAnalyzer.VersionDate = _toneAnalyzerVersionDate; 120 | 121 | bar1JoyRenderer.material = yellow_material; 122 | bar2SadnessRenderer.material = blue_material; 123 | bar3FearRenderer.material = purple_material; 124 | bar4DisgustRenderer.material = green_material; 125 | bar5AngerRenderer.material = red_material; 126 | 127 | emotion_threshold = 0.75f; // for loose demo - above 75% seems to work well - may vary by signal 128 | 129 | //This is a "on first run" test 130 | //Runnable.Run(Examples()); // example on pump prime 131 | } 132 | 133 | public bool Active 134 | { 135 | get { return _speechToText.IsListening; } 136 | set 137 | { 138 | if (value && !_speechToText.IsListening) 139 | { 140 | _speechToText.DetectSilence = true; 141 | _speechToText.EnableWordConfidence = true; 142 | _speechToText.EnableTimestamps = true; 143 | _speechToText.SilenceThreshold = 0.01f; 144 | _speechToText.MaxAlternatives = 0; 145 | _speechToText.EnableInterimResults = true; 146 | _speechToText.OnError = OnError; 147 | _speechToText.InactivityTimeout = -1; 148 | _speechToText.ProfanityFilter = false; 149 | _speechToText.SmartFormatting = true; 150 | _speechToText.SpeakerLabels = false; 151 | _speechToText.WordAlternativesThreshold = null; 152 | _speechToText.StartListening(OnRecognize, OnRecognizeSpeaker); 153 | } 154 | else if (!value && _speechToText.IsListening) 155 | { 156 | _speechToText.StopListening(); 157 | } 158 | } 159 | } 160 | 161 | private void StartRecording() 162 | { 163 | if (_recordingRoutine == 0) 164 | { 165 | UnityObjectUtil.StartDestroyQueue(); 166 | _recordingRoutine = Runnable.Run(RecordingHandler()); 167 | } 168 | } 169 | 170 | private void StopRecording() 171 | { 172 | if (_recordingRoutine != 0) 173 | { 174 | Microphone.End(_microphoneID); 175 | Runnable.Stop(_recordingRoutine); 176 | _recordingRoutine = 0; 177 | } 178 | } 179 | 180 | private void OnError(string error) 181 | { 182 | Active = false; 183 | 184 | Log.Debug("ExampleStreaming.OnError()", "Error! {0}", error); 185 | } 186 | 187 | private IEnumerator RecordingHandler() 188 | { 189 | Log.Debug("ExampleStreaming.RecordingHandler()", "devices: {0}", Microphone.devices); 190 | _recording = Microphone.Start(_microphoneID, true, _recordingBufferSize, _recordingHZ); 191 | yield return null; // let _recordingRoutine get set.. 192 | 193 | if (_recording == null) 194 | { 195 | StopRecording(); 196 | yield break; 197 | } 198 | 199 | bool bFirstBlock = true; 200 | int midPoint = _recording.samples / 2; 201 | float[] samples = null; 202 | 203 | while (_recordingRoutine != 0 && _recording != null) 204 | { 205 | int writePos = Microphone.GetPosition(_microphoneID); 206 | if (writePos > _recording.samples || !Microphone.IsRecording(_microphoneID)) 207 | { 208 | Log.Error("ExampleStreaming.RecordingHandler()", "Microphone disconnected."); 209 | 210 | StopRecording(); 211 | yield break; 212 | } 213 | 214 | if ((bFirstBlock && writePos >= midPoint) 215 | || (!bFirstBlock && writePos < midPoint)) 216 | { 217 | // front block is recorded, make a RecordClip and pass it onto our callback. 218 | samples = new float[midPoint]; 219 | _recording.GetData(samples, bFirstBlock ? 0 : midPoint); 220 | 221 | AudioData record = new AudioData(); 222 | record.MaxLevel = Mathf.Max(Mathf.Abs(Mathf.Min(samples)), Mathf.Max(samples)); 223 | record.Clip = AudioClip.Create("Recording", midPoint, _recording.channels, _recordingHZ, false); 224 | record.Clip.SetData(samples, 0); 225 | 226 | _speechToText.OnListen(record); 227 | 228 | bFirstBlock = !bFirstBlock; 229 | } 230 | else 231 | { 232 | // calculate the number of samples remaining until we ready for a block of audio, 233 | // and wait that amount of time it will take to record. 234 | int remaining = bFirstBlock ? (midPoint - writePos) : (_recording.samples - writePos); 235 | float timeRemaining = (float)remaining / (float)_recordingHZ; 236 | 237 | yield return new WaitForSeconds(timeRemaining); 238 | } 239 | 240 | } 241 | 242 | yield break; 243 | } 244 | 245 | 246 | 247 | // TONE ZONE 248 | private IEnumerator Examples() 249 | { 250 | // Analyze tone 251 | if (!_toneAnalyzer.GetToneAnalyze(OnGetToneAnalyze, OnFail, _stringToTestTone1)) 252 | Log.Debug("ExampleToneAnalyzer.Examples()", "Failed to analyze!"); 253 | 254 | while (!_analyzeToneTested) 255 | yield return null; 256 | 257 | Log.Debug("ExampleToneAnalyzer.Examples()", "Tone analyzer examples complete."); 258 | } 259 | 260 | 261 | // TESTING 262 | private void OnGetToneAnalyze(ToneAnalyzerResponse resp, Dictionary customData) 263 | { 264 | Log.Debug("ExampleToneAnalyzer.OnGetToneAnalyze()", "{0}", customData["json"].ToString()); 265 | 266 | ResultsField.text = (customData["json"].ToString()); // works but long and cannot read 267 | 268 | 269 | // CACTUS EMOTION DETECTION FOR DROID EMOTION RESPONSE 270 | //var bar1Object = GameObject.Find("bar1"); 271 | //var renderer = gameObject.GetComponent(); 272 | 273 | //CactusField.text = ResultsField.text; // works 274 | 275 | Log.Debug ("$$$$$ TONE LOG 0 ANGER", "{0}",resp.document_tone.tone_categories[0].tones[0].score); // ANGER resp.document_tone.tone_categories [0].tones [0].score); 276 | Log.Debug ("$$$$$ TONE LOG 1 DISGUST", "{0}",resp.document_tone.tone_categories[0].tones[1].score); // DISGUST 277 | Log.Debug ("$$$$$ TONE LOG 2 FEAR", "{0}",resp.document_tone.tone_categories[0].tones[2].score); // FEAR 278 | Log.Debug ("$$$$$ TONE LOG 3 JOY", "{0}",resp.document_tone.tone_categories[0].tones[3].score); // JOY 279 | Log.Debug ("$$$$$ TONE LOG 4 SAD", "{0}",resp.document_tone.tone_categories[0].tones[4].score); // SADNESS 280 | 281 | Log.Debug ("$$$$$ TONE ANALYTICAL", "{0}",resp.document_tone.tone_categories[1].tones[0].score); // ANALYTICAL 282 | Log.Debug ("$$$$$ TONE CONFIDENT", "{0}",resp.document_tone.tone_categories[1].tones[1].score); // CONFIDENT 283 | Log.Debug ("$$$$$ TONE TENTATIVE", "{0}",resp.document_tone.tone_categories[1].tones[2].score); // TENTATIVE 284 | 285 | // EMOTION 286 | if (resp.document_tone.tone_categories [0].tones [0].score > emotion_threshold) { 287 | DroidRenderer.material = red_material; 288 | CactusField.text = "Anger"; 289 | bar5AngerRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 290 | } 291 | else if (resp.document_tone.tone_categories [0].tones [1].score > emotion_threshold) { 292 | DroidRenderer.material = green_material; 293 | CactusField.text = "Disgust"; 294 | bar4DisgustRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 295 | } 296 | else if (resp.document_tone.tone_categories [0].tones [2].score > emotion_threshold) { 297 | DroidRenderer.material = purple_material; 298 | CactusField.text = "Fear"; 299 | bar3FearRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 300 | } 301 | else if (resp.document_tone.tone_categories [0].tones [3].score > emotion_threshold) { 302 | DroidRenderer.material = yellow_material; 303 | CactusField.text = "Joy"; 304 | bar1JoyRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 305 | } 306 | else if (resp.document_tone.tone_categories [0].tones [4].score > emotion_threshold) { 307 | DroidRenderer.material = blue_material; 308 | CactusField.text = "Sadness"; 309 | bar2SadnessRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 310 | } 311 | 312 | // Language tone - https://www.ibm.com/watson/developercloud/tone-analyzer/api/v3/ 313 | else if (resp.document_tone.tone_categories[1].tones[0].score > emotion_threshold) { 314 | DroidRenderer.material = white_material; 315 | CactusField.text = "Analytical"; 316 | cyl1AnalyticalRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 317 | } 318 | else if (resp.document_tone.tone_categories[1].tones[1].score > emotion_threshold) { 319 | DroidRenderer.material = white_material; 320 | CactusField.text = "Confident"; 321 | cyl2ConfidentRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 322 | } 323 | else if (resp.document_tone.tone_categories [1].tones[2].score > emotion_threshold) { 324 | DroidRenderer.material = white_material; 325 | CactusField.text = "Tentative"; 326 | cyl3TentativeRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 327 | } 328 | 329 | 330 | // OTHER TEXT - Formatting for On Screen dump - LATER - pretty this up to use standard DESERIALIZE methods and table 331 | string RAW = (customData["json"].ToString()); // works but long and cannot read 332 | //RAW = string.Concat("Tone Response \n", RAW); 333 | RAW = Regex.Replace(RAW, "tone_categories", " \\\n"); 334 | RAW = Regex.Replace(RAW, "}", "} \\\n"); 335 | RAW = Regex.Replace(RAW, "tone_id", " "); 336 | RAW = Regex.Replace(RAW, "tone_name", " "); 337 | RAW = Regex.Replace(RAW, "score", " "); 338 | RAW = Regex.Replace(RAW, @"[{\\},:]", ""); 339 | RAW = Regex.Replace(RAW, "\"", ""); 340 | ResultsField.text = RAW; 341 | 342 | _analyzeToneTested = true; 343 | } 344 | 345 | 346 | private void OnFail(RESTConnector.Error error, Dictionary customData) 347 | { 348 | Log.Error("ExampleRetrieveAndRank.OnFail()", "Error received: {0}", error.ToString()); 349 | } 350 | 351 | 352 | 353 | 354 | private void OnRecognize(SpeechRecognitionEvent result) 355 | { 356 | if (result != null && result.results.Length > 0) 357 | { 358 | foreach (var res in result.results) 359 | { 360 | foreach (var alt in res.alternatives) { 361 | string text = string.Format ("{0} ({1}, {2:0.00})\n", alt.transcript, res.final ? "Final" : "Interim", alt.confidence); 362 | Log.Debug ("ExampleStreaming.OnRecognize()", text); 363 | ResultsField.text = text; 364 | 365 | // Magic happens here (Hello World Magic!) 366 | // FOR NOW HARD WIRING THE EXPLICIT UTTERANCES TO COLOR STATE CHANGES 367 | // LATER WILL USE LOOKUPS; THRESHOLDES; AND STATEHOLDERS 368 | // if (alt.transcript.Contains("red")) { 369 | // sphereMeshRenderer.material = red_material; 370 | // //DroidRenderer.material = red_material; 371 | // } 372 | // if (alt.transcript.Contains("blue")) { 373 | // sphereMeshRenderer.material = blue_material; 374 | // //DroidRenderer.material = blue_material; 375 | // } 376 | // if (alt.transcript.Contains("green")) { 377 | // cubeMeshRenderer.material = green_material; // disable for cactus 378 | // DroidRenderer.material = green_material; 379 | // } 380 | // if (alt.transcript.Contains("yellow")) { 381 | // cubeMeshRenderer.material = yellow_material; // disable for cactus 382 | // //DroidRenderer.material = yellow_material; 383 | // } 384 | 385 | // Here is the Emotional Zone - GREP For now 386 | // if (alt.transcript.Contains("happy") | alt.transcript.Contains("joy")) { 387 | // bar1JoyRenderer.material = yellow_material; 388 | // //DroidRenderer.material = yellow_material; 389 | // } 390 | // if (alt.transcript.Contains("sad") | alt.transcript.Contains("depressed")) { 391 | // bar2SadnessRenderer.material = blue_material; 392 | // //DroidRenderer.material = blue_material; 393 | // } 394 | // if (alt.transcript.Contains("scared") | alt.transcript.Contains("fear")) { 395 | // bar3FearRenderer.material = purple_material; 396 | // //DroidRenderer.material = purple_material; 397 | // } 398 | // if (alt.transcript.Contains("yucky") | alt.transcript.Contains("gross")) { 399 | // bar4DisgustRenderer.material = green_material; 400 | // //DroidRenderer.material = green_material; 401 | // } 402 | // if (alt.transcript.Contains("mad") | alt.transcript.Contains("angry")) { 403 | // bar5AngerRenderer.material = red_material; 404 | // //DroidRenderer.material = red_material; 405 | // } 406 | 407 | // ENTERING THE TONE ZONE - when the utterance contains this word 408 | if (alt.transcript.Contains ("feel") | alt.transcript.Contains ("you") | alt.transcript.Contains ("robot")) { 409 | // if the utterance 410 | // Runnable.Run(Examples()); // this compiled - it's simply the same test from startup 411 | 412 | 413 | string GHI = alt.transcript; 414 | if (!_toneAnalyzer.GetToneAnalyze (OnGetToneAnalyze, OnFail, GHI)) 415 | Log.Debug ("ExampleToneAnalyzer.Examples()", "Failed to analyze!"); 416 | 417 | // TEST START 418 | // Analyze tone 419 | // if (!_toneAnalyzer.GetToneAnalyze(OnGetToneAnalyze, OnFail, _stringToTestTone2)) 420 | // Log.Debug("ExampleToneAnalyzer.Examples()", "Failed to analyze!"); 421 | 422 | Log.Debug ("ExampleToneAnalyzer.Examples()", "NESTED TONE ZONE branch complete."); 423 | //ResultsField.text = "tone analyzed! 111"; 424 | // TEST END 425 | 426 | } 427 | 428 | // ENTERING THE TONE ZONE - when the utterance contains this word 429 | else if (alt.transcript.Contains ("reset")) { 430 | cyl1AnalyticalRenderer.transform.localScale = new Vector3 (1F, 1F, 1F); 431 | cyl2ConfidentRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 432 | cyl3TentativeRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 433 | bar1JoyRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 434 | bar2SadnessRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 435 | bar3FearRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 436 | bar4DisgustRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 437 | bar5AngerRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 438 | DroidRenderer.material = white_material; 439 | } 440 | 441 | } 442 | 443 | if (res.keywords_result != null && res.keywords_result.keyword != null) 444 | { 445 | foreach (var keyword in res.keywords_result.keyword) 446 | { 447 | Log.Debug("ExampleStreaming.OnRecognize()", "keyword: {0}, confidence: {1}, start time: {2}, end time: {3}", keyword.normalized_text, keyword.confidence, keyword.start_time, keyword.end_time); 448 | //ResultsField.text = "tone analyzed! 222"; 449 | } 450 | } 451 | 452 | if (res.word_alternatives != null) 453 | { 454 | foreach (var wordAlternative in res.word_alternatives) 455 | { 456 | Log.Debug("ExampleStreaming.OnRecognize()", "Word alternatives found. Start time: {0} | EndTime: {1}", wordAlternative.start_time, wordAlternative.end_time); 457 | foreach(var alternative in wordAlternative.alternatives) 458 | Log.Debug("ExampleStreaming.OnRecognize()", "\t word: {0} | confidence: {1}", alternative.word, alternative.confidence); 459 | } 460 | } 461 | } 462 | } 463 | } 464 | 465 | private void OnRecognizeSpeaker(SpeakerRecognitionEvent result) 466 | { 467 | if (result != null) 468 | { 469 | foreach (SpeakerLabelsResult labelResult in result.speaker_labels) 470 | { 471 | Log.Debug("ExampleStreaming.OnRecognize()", string.Format("speaker result: {0} | confidence: {3} | from: {1} | to: {2}", labelResult.speaker, labelResult.from, labelResult.to, labelResult.confidence)); 472 | } 473 | } 474 | } 475 | } 476 | -------------------------------------------------------------------------------- /ExampleStreaming4RobotEmotion.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * ############# HELLO VIRTUAL WORLD 4 | * Copyright 2015 IBM Corp. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * THIS IS VERY VERY VERY VERY VERY VERY VERY VERY VERY VERY VERY VERY VERY VERY VERY VERY VERY VERY ROUGH CODE - WARNING :) 18 | */ 19 | 20 | using UnityEngine; 21 | using System.Collections; 22 | using IBM.Watson.DeveloperCloud.Logging; 23 | using IBM.Watson.DeveloperCloud.Services.SpeechToText.v1; 24 | using IBM.Watson.DeveloperCloud.Utilities; 25 | using IBM.Watson.DeveloperCloud.DataTypes; 26 | using System.Collections.Generic; 27 | using UnityEngine.UI; 28 | 29 | // added this from the TONE ANALYZER . CS file 30 | using IBM.Watson.DeveloperCloud.Services.ToneAnalyzer.v3; 31 | using IBM.Watson.DeveloperCloud.Connection; 32 | 33 | // and for Regex 34 | using System.Text.RegularExpressions; 35 | 36 | // and for Ethan's body 37 | using UnityStandardAssets; 38 | using UnityStandardAssets.Characters.ThirdPerson; 39 | 40 | 41 | public class ExampleStreaming : MonoBehaviour 42 | { 43 | 44 | private string _username_STT = "XXXX"; 45 | private string _password_STT = "XXXX"; 46 | private string _url_STT = "https://stream.watsonplatform.net/speech-to-text/api"; 47 | 48 | public Text ResultsField; 49 | public Text CactusField; 50 | 51 | public float emotion_threshold; 52 | 53 | private SpeechToText _speechToText; 54 | 55 | // TONE ZONE 56 | private string _username_TONE = "XXXX"; 57 | private string _password_TONE = "XXXX"; 58 | private string _url_TONE = "https://gateway.watsonplatform.net/tone-analyzer/api"; 59 | 60 | 61 | private ToneAnalyzer _toneAnalyzer; 62 | private string _toneAnalyzerVersionDate = "2017-05-26"; 63 | private string _stringToTestTone1 = "START AND TEST - But this totally sucks! I hate beans and liver!"; 64 | private string _stringToTestTone2 = "SECOND TEST - Failed Test Sucks"; 65 | private bool _analyzeToneTested = false; 66 | 67 | 68 | 69 | // magic 70 | //public GameObject sphere_rad; 71 | public MeshRenderer sphereMeshRenderer; 72 | public MeshRenderer cubeMeshRenderer; 73 | 74 | public MeshRenderer bar1JoyRenderer; 75 | public MeshRenderer bar2SadnessRenderer; 76 | public MeshRenderer bar3FearRenderer; 77 | public MeshRenderer bar4DisgustRenderer; 78 | public MeshRenderer bar5AngerRenderer; 79 | 80 | public MeshRenderer DroidRenderer; // droid R2D2 wannabe 81 | 82 | // Over the shoulder emotional spheres 83 | public MeshRenderer sphere_emo_joyRenderer; 84 | public MeshRenderer sphere_emo_angerRenderer; 85 | public MeshRenderer sphere_emo_fearRenderer; 86 | public MeshRenderer sphere_emo_disgustRenderer; 87 | public MeshRenderer sphere_emo_sadnessRenderer; 88 | 89 | 90 | // Tin man is ethanbot 91 | public MeshRenderer TinManRenderer; // tinman - ethan stock character with metal body - for the SCALE and LOCAION 92 | public MeshRenderer TinManHelmet; // tinman just the helm - for color 93 | 94 | public MeshRenderer cyl1AnalyticalRenderer; 95 | public MeshRenderer cyl2ConfidentRenderer; 96 | public MeshRenderer cyl3TentativeRenderer; 97 | 98 | public Material original_material; 99 | public Material red_material; 100 | public Material blue_material; 101 | public Material yellow_material; 102 | public Material green_material; 103 | public Material purple_material; 104 | public Material white_material; 105 | 106 | private int _recordingRoutine = 0; 107 | private string _microphoneID = null; 108 | private AudioClip _recording = null; 109 | private int _recordingBufferSize = 1; 110 | private int _recordingHZ = 22050; 111 | 112 | 113 | 114 | void Start() 115 | { 116 | 117 | 118 | LogSystem.InstallDefaultReactors(); 119 | 120 | // Create credential and instantiate service 121 | Credentials credentials_STT = new Credentials(_username_STT, _password_STT, _url_STT); 122 | 123 | _speechToText = new SpeechToText(credentials_STT); 124 | Active = true; 125 | 126 | StartRecording(); 127 | 128 | 129 | // TONE ZONE 130 | Credentials credentials_TONE = new Credentials(_username_TONE, _password_TONE, _url_TONE); 131 | _toneAnalyzer = new ToneAnalyzer(credentials_TONE); 132 | _toneAnalyzer.VersionDate = _toneAnalyzerVersionDate; 133 | 134 | bar1JoyRenderer.material = yellow_material; 135 | bar2SadnessRenderer.material = blue_material; 136 | bar3FearRenderer.material = purple_material; 137 | bar4DisgustRenderer.material = green_material; 138 | bar5AngerRenderer.material = red_material; 139 | 140 | emotion_threshold = 0.75f; // for loose demo - above 75% seems to work well - may vary by signal 141 | 142 | //This is a "on first run" test 143 | //Runnable.Run(Examples()); // example on pump prime 144 | } 145 | 146 | public bool Active 147 | { 148 | get { return _speechToText.IsListening; } 149 | set 150 | { 151 | if (value && !_speechToText.IsListening) 152 | { 153 | _speechToText.DetectSilence = true; 154 | _speechToText.EnableWordConfidence = true; 155 | _speechToText.EnableTimestamps = true; 156 | _speechToText.SilenceThreshold = 0.01f; 157 | _speechToText.MaxAlternatives = 0; 158 | _speechToText.EnableInterimResults = true; 159 | _speechToText.OnError = OnError; 160 | _speechToText.InactivityTimeout = -1; 161 | _speechToText.ProfanityFilter = false; 162 | _speechToText.SmartFormatting = true; 163 | _speechToText.SpeakerLabels = false; 164 | _speechToText.WordAlternativesThreshold = null; 165 | _speechToText.StartListening(OnRecognize, OnRecognizeSpeaker); 166 | } 167 | else if (!value && _speechToText.IsListening) 168 | { 169 | _speechToText.StopListening(); 170 | } 171 | } 172 | } 173 | 174 | private void StartRecording() 175 | { 176 | if (_recordingRoutine == 0) 177 | { 178 | UnityObjectUtil.StartDestroyQueue(); 179 | _recordingRoutine = Runnable.Run(RecordingHandler()); 180 | } 181 | } 182 | 183 | private void StopRecording() 184 | { 185 | if (_recordingRoutine != 0) 186 | { 187 | Microphone.End(_microphoneID); 188 | Runnable.Stop(_recordingRoutine); 189 | _recordingRoutine = 0; 190 | } 191 | } 192 | 193 | private void OnError(string error) 194 | { 195 | Active = false; 196 | 197 | Log.Debug("ExampleStreaming.OnError()", "Error! {0}", error); 198 | } 199 | 200 | private IEnumerator RecordingHandler() 201 | { 202 | Log.Debug("ExampleStreaming.RecordingHandler()", "devices: {0}", Microphone.devices); 203 | _recording = Microphone.Start(_microphoneID, true, _recordingBufferSize, _recordingHZ); 204 | yield return null; // let _recordingRoutine get set.. 205 | 206 | if (_recording == null) 207 | { 208 | StopRecording(); 209 | yield break; 210 | } 211 | 212 | bool bFirstBlock = true; 213 | int midPoint = _recording.samples / 2; 214 | float[] samples = null; 215 | 216 | while (_recordingRoutine != 0 && _recording != null) 217 | { 218 | int writePos = Microphone.GetPosition(_microphoneID); 219 | if (writePos > _recording.samples || !Microphone.IsRecording(_microphoneID)) 220 | { 221 | Log.Error("ExampleStreaming.RecordingHandler()", "Microphone disconnected."); 222 | 223 | StopRecording(); 224 | yield break; 225 | } 226 | 227 | if ((bFirstBlock && writePos >= midPoint) 228 | || (!bFirstBlock && writePos < midPoint)) 229 | { 230 | // front block is recorded, make a RecordClip and pass it onto our callback. 231 | samples = new float[midPoint]; 232 | _recording.GetData(samples, bFirstBlock ? 0 : midPoint); 233 | 234 | AudioData record = new AudioData(); 235 | record.MaxLevel = Mathf.Max(Mathf.Abs(Mathf.Min(samples)), Mathf.Max(samples)); 236 | record.Clip = AudioClip.Create("Recording", midPoint, _recording.channels, _recordingHZ, false); 237 | record.Clip.SetData(samples, 0); 238 | 239 | _speechToText.OnListen(record); 240 | 241 | bFirstBlock = !bFirstBlock; 242 | } 243 | else 244 | { 245 | // calculate the number of samples remaining until we ready for a block of audio, 246 | // and wait that amount of time it will take to record. 247 | int remaining = bFirstBlock ? (midPoint - writePos) : (_recording.samples - writePos); 248 | float timeRemaining = (float)remaining / (float)_recordingHZ; 249 | 250 | yield return new WaitForSeconds(timeRemaining); 251 | } 252 | 253 | } 254 | 255 | yield break; 256 | } 257 | 258 | 259 | 260 | // TONE ZONE 261 | private IEnumerator Examples() 262 | { 263 | // Analyze tone 264 | if (!_toneAnalyzer.GetToneAnalyze(OnGetToneAnalyze, OnFail, _stringToTestTone1)) 265 | Log.Debug("ExampleToneAnalyzer.Examples()", "Failed to analyze!"); 266 | 267 | while (!_analyzeToneTested) 268 | yield return null; 269 | 270 | Log.Debug("ExampleToneAnalyzer.Examples()", "Tone analyzer examples complete."); 271 | } 272 | 273 | 274 | // TESTING 275 | private void OnGetToneAnalyze(ToneAnalyzerResponse resp, Dictionary customData) 276 | { 277 | Log.Debug("ExampleToneAnalyzer.OnGetToneAnalyze()", "{0}", customData["json"].ToString()); 278 | 279 | ResultsField.text = (customData["json"].ToString()); // works but long and cannot read 280 | 281 | 282 | // CACTUS EMOTION DETECTION FOR DROID EMOTION RESPONSE 283 | //var bar1Object = GameObject.Find("bar1"); 284 | //var renderer = gameObject.GetComponent(); 285 | 286 | //CactusField.text = ResultsField.text; // works 287 | 288 | Log.Debug ("$$$$$ TONE LOG 0 ANGER", "{0}",resp.document_tone.tone_categories[0].tones[0].score); // ANGER resp.document_tone.tone_categories [0].tones [0].score); 289 | Log.Debug ("$$$$$ TONE LOG 1 DISGUST", "{0}",resp.document_tone.tone_categories[0].tones[1].score); // DISGUST 290 | Log.Debug ("$$$$$ TONE LOG 2 FEAR", "{0}",resp.document_tone.tone_categories[0].tones[2].score); // FEAR 291 | Log.Debug ("$$$$$ TONE LOG 3 JOY", "{0}",resp.document_tone.tone_categories[0].tones[3].score); // JOY 292 | Log.Debug ("$$$$$ TONE LOG 4 SAD", "{0}",resp.document_tone.tone_categories[0].tones[4].score); // SADNESS 293 | 294 | Log.Debug ("$$$$$ TONE ANALYTICAL", "{0}",resp.document_tone.tone_categories[1].tones[0].score); // ANALYTICAL 295 | Log.Debug ("$$$$$ TONE CONFIDENT", "{0}",resp.document_tone.tone_categories[1].tones[1].score); // CONFIDENT 296 | Log.Debug ("$$$$$ TONE TENTATIVE", "{0}",resp.document_tone.tone_categories[1].tones[2].score); // TENTATIVE 297 | 298 | 299 | 300 | 301 | // EMOTION 302 | if (resp.document_tone.tone_categories [0].tones [0].score > emotion_threshold) { 303 | DroidRenderer.material = red_material; 304 | CactusField.text = "Anger"; 305 | bar5AngerRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 306 | TinManRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 307 | TinManHelmet.material = red_material; 308 | 309 | sphere_emo_angerRenderer.transform.localScale += new Vector3(0.025F, 0.025F, 0.025F); 310 | 311 | 312 | } 313 | else if (resp.document_tone.tone_categories [0].tones [1].score > emotion_threshold) { 314 | DroidRenderer.material = green_material; 315 | CactusField.text = "Disgust"; 316 | bar4DisgustRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 317 | 318 | TinManRenderer.transform.localScale -= new Vector3(0.1F, 0.1F, 0.1F); 319 | TinManHelmet.material = green_material; 320 | 321 | sphere_emo_disgustRenderer.transform.localScale += new Vector3(0.025F, 0.025F, 0.025F); 322 | 323 | } 324 | else if (resp.document_tone.tone_categories [0].tones [2].score > emotion_threshold) { 325 | DroidRenderer.material = purple_material; 326 | CactusField.text = "Fear"; 327 | bar3FearRenderer.transform.localScale += new Vector3 (0.1F, 0.1F, 0.1F); 328 | 329 | TinManRenderer.transform.localScale -= new Vector3(0.1F, 0.1F, 0.1F); 330 | TinManHelmet.material = purple_material; 331 | 332 | sphere_emo_fearRenderer.transform.localScale += new Vector3(0.025F, 0.025F, 0.025F); 333 | 334 | } 335 | else if (resp.document_tone.tone_categories [0].tones [3].score > emotion_threshold) { 336 | DroidRenderer.material = yellow_material; 337 | TinManRenderer.material = yellow_material; 338 | CactusField.text = "Joy"; 339 | bar1JoyRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 340 | 341 | TinManRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 342 | TinManHelmet.material = yellow_material; 343 | 344 | sphere_emo_joyRenderer.transform.localScale += new Vector3(0.025F, 0.025F, 0.025F); 345 | 346 | } 347 | else if (resp.document_tone.tone_categories [0].tones [4].score > emotion_threshold) { 348 | DroidRenderer.material = blue_material; 349 | CactusField.text = "Sadness"; 350 | bar2SadnessRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 351 | 352 | TinManRenderer.transform.localScale -= new Vector3(0.1F, 0.1F, 0.1F); 353 | TinManRenderer.material = blue_material; 354 | TinManHelmet.material = blue_material; 355 | 356 | sphere_emo_sadnessRenderer.transform.localScale += new Vector3(0.025F, 0.025F, 0.025F); 357 | 358 | } 359 | 360 | // Language tone - https://www.ibm.com/watson/developercloud/tone-analyzer/api/v3/ 361 | else if (resp.document_tone.tone_categories[1].tones[0].score > emotion_threshold) { 362 | DroidRenderer.material = white_material; 363 | CactusField.text = "Analytical"; 364 | cyl1AnalyticalRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 365 | } 366 | else if (resp.document_tone.tone_categories[1].tones[1].score > emotion_threshold) { 367 | DroidRenderer.material = white_material; 368 | CactusField.text = "Confident"; 369 | cyl2ConfidentRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 370 | } 371 | else if (resp.document_tone.tone_categories [1].tones[2].score > emotion_threshold) { 372 | DroidRenderer.material = white_material; 373 | CactusField.text = "Tentative"; 374 | cyl3TentativeRenderer.transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 375 | } 376 | 377 | 378 | // OTHER TEXT - Formatting for On Screen dump - LATER - pretty this up to use standard DESERIALIZE methods and table 379 | string RAW = (customData["json"].ToString()); // works but long and cannot read 380 | //RAW = string.Concat("Tone Response \n", RAW); 381 | RAW = Regex.Replace(RAW, "tone_categories", " \\\n"); 382 | RAW = Regex.Replace(RAW, "}", "} \\\n"); 383 | RAW = Regex.Replace(RAW, "tone_id", " "); 384 | RAW = Regex.Replace(RAW, "tone_name", " "); 385 | RAW = Regex.Replace(RAW, "score", " "); 386 | RAW = Regex.Replace(RAW, @"[{\\},:]", ""); 387 | RAW = Regex.Replace(RAW, "\"", ""); 388 | ResultsField.text = RAW; 389 | 390 | _analyzeToneTested = true; 391 | } 392 | 393 | 394 | private void OnFail(RESTConnector.Error error, Dictionary customData) 395 | { 396 | Log.Error("ExampleRetrieveAndRank.OnFail()", "Error received: {0}", error.ToString()); 397 | } 398 | 399 | 400 | 401 | 402 | private void OnRecognize(SpeechRecognitionEvent result) 403 | { 404 | if (result != null && result.results.Length > 0) 405 | { 406 | foreach (var res in result.results) 407 | { 408 | foreach (var alt in res.alternatives) { 409 | string text = string.Format ("{0} ({1}, {2:0.00})\n", alt.transcript, res.final ? "Final" : "Interim", alt.confidence); 410 | Log.Debug ("ExampleStreaming.OnRecognize()", text); 411 | ResultsField.text = text; 412 | 413 | // Magic happens here (Hello World Magic!) 414 | // FOR NOW HARD WIRING THE EXPLICIT UTTERANCES TO COLOR STATE CHANGES 415 | // LATER WILL USE LOOKUPS; THRESHOLDES; AND STATEHOLDERS 416 | // if (alt.transcript.Contains("red")) { 417 | // sphereMeshRenderer.material = red_material; 418 | // //DroidRenderer.material = red_material; 419 | // } 420 | // if (alt.transcript.Contains("blue")) { 421 | // sphereMeshRenderer.material = blue_material; 422 | // //DroidRenderer.material = blue_material; 423 | // } 424 | // if (alt.transcript.Contains("green")) { 425 | // cubeMeshRenderer.material = green_material; // disable for cactus 426 | // DroidRenderer.material = green_material; 427 | // } 428 | // if (alt.transcript.Contains("yellow")) { 429 | // cubeMeshRenderer.material = yellow_material; // disable for cactus 430 | // //DroidRenderer.material = yellow_material; 431 | // } 432 | 433 | // Here is the Emotional Zone - GREP For now 434 | // if (alt.transcript.Contains("happy") | alt.transcript.Contains("joy")) { 435 | // bar1JoyRenderer.material = yellow_material; 436 | // //DroidRenderer.material = yellow_material; 437 | // } 438 | // if (alt.transcript.Contains("sad") | alt.transcript.Contains("depressed")) { 439 | // bar2SadnessRenderer.material = blue_material; 440 | // //DroidRenderer.material = blue_material; 441 | // } 442 | // if (alt.transcript.Contains("scared") | alt.transcript.Contains("fear")) { 443 | // bar3FearRenderer.material = purple_material; 444 | // //DroidRenderer.material = purple_material; 445 | // } 446 | // if (alt.transcript.Contains("yucky") | alt.transcript.Contains("gross")) { 447 | // bar4DisgustRenderer.material = green_material; 448 | // //DroidRenderer.material = green_material; 449 | // } 450 | // if (alt.transcript.Contains("mad") | alt.transcript.Contains("angry")) { 451 | // bar5AngerRenderer.material = red_material; 452 | // //DroidRenderer.material = red_material; 453 | // } 454 | 455 | // ENTERING THE TONE ZONE - when the utterance contains this word 456 | //if (alt.transcript.Contains ("feel") | alt.transcript.Contains ("you") | alt.transcript.Contains ("Jimmy") | alt.transcript.Contains ("robot")) { 457 | if (true) { 458 | // if the utterance 459 | // Runnable.Run(Examples()); // this compiled - it's simply the same test from startup 460 | 461 | 462 | string GHI = alt.transcript; 463 | if (!_toneAnalyzer.GetToneAnalyze (OnGetToneAnalyze, OnFail, GHI)) 464 | Log.Debug ("ExampleToneAnalyzer.Examples()", "Failed to analyze!"); 465 | 466 | // TEST START 467 | // Analyze tone 468 | // if (!_toneAnalyzer.GetToneAnalyze(OnGetToneAnalyze, OnFail, _stringToTestTone2)) 469 | // Log.Debug("ExampleToneAnalyzer.Examples()", "Failed to analyze!"); 470 | 471 | Log.Debug ("ExampleToneAnalyzer.Examples()", "NESTED TONE ZONE branch complete."); 472 | //ResultsField.text = "tone analyzed! 111"; 473 | // TEST END 474 | 475 | } 476 | 477 | // ENTERING THE TONE ZONE - when the utterance contains this word 478 | if (alt.transcript.Contains ("reset")) { 479 | cyl1AnalyticalRenderer.transform.localScale = new Vector3 (1F, 1F, 1F); 480 | cyl2ConfidentRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 481 | cyl3TentativeRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 482 | bar1JoyRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 483 | bar2SadnessRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 484 | bar3FearRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 485 | bar4DisgustRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 486 | bar5AngerRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 487 | DroidRenderer.material = white_material; 488 | 489 | sphere_emo_sadnessRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 490 | sphere_emo_angerRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 491 | sphere_emo_disgustRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 492 | sphere_emo_fearRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 493 | sphere_emo_joyRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 494 | 495 | TinManRenderer.transform.localScale = new Vector3(10F, 10F, 10F); 496 | } 497 | 498 | if (alt.transcript.Contains ("panic attack")) { 499 | sphere_emo_sadnessRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 500 | sphere_emo_angerRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 501 | sphere_emo_disgustRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 502 | sphere_emo_fearRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 503 | sphere_emo_joyRenderer.transform.localScale = new Vector3(1F, 1F, 1F); 504 | } 505 | 506 | if (alt.transcript.Contains ("mouse")) { 507 | TinManRenderer.transform.localScale = new Vector3(2F, 2F, 2F); 508 | } 509 | 510 | if (alt.transcript.Contains ("elephant")) { 511 | TinManRenderer.transform.localScale = new Vector3(20F, 20F, 20F); 512 | } 513 | 514 | } 515 | 516 | if (res.keywords_result != null && res.keywords_result.keyword != null) 517 | { 518 | foreach (var keyword in res.keywords_result.keyword) 519 | { 520 | Log.Debug("ExampleStreaming.OnRecognize()", "keyword: {0}, confidence: {1}, start time: {2}, end time: {3}", keyword.normalized_text, keyword.confidence, keyword.start_time, keyword.end_time); 521 | //ResultsField.text = "tone analyzed! 222"; 522 | } 523 | } 524 | 525 | if (res.word_alternatives != null) 526 | { 527 | foreach (var wordAlternative in res.word_alternatives) 528 | { 529 | Log.Debug("ExampleStreaming.OnRecognize()", "Word alternatives found. Start time: {0} | EndTime: {1}", wordAlternative.start_time, wordAlternative.end_time); 530 | foreach(var alternative in wordAlternative.alternatives) 531 | Log.Debug("ExampleStreaming.OnRecognize()", "\t word: {0} | confidence: {1}", alternative.word, alternative.confidence); 532 | } 533 | } 534 | } 535 | } 536 | } 537 | 538 | private void OnRecognizeSpeaker(SpeakerRecognitionEvent result) 539 | { 540 | if (result != null) 541 | { 542 | foreach (SpeakerLabelsResult labelResult in result.speaker_labels) 543 | { 544 | Log.Debug("ExampleStreaming.OnRecognize()", string.Format("speaker result: {0} | confidence: {3} | from: {1} | to: {2}", labelResult.speaker, labelResult.from, labelResult.to, labelResult.confidence)); 545 | } 546 | } 547 | } 548 | } 549 | -------------------------------------------------------------------------------- /ExampleStreaming5RobotEmotion.cs: -------------------------------------------------------------------------------- 1 | /** 2 | ExampleStreaming5RobotEmotion.cs - May 18 - uses several Free assets from Unity - Ethan, Nature Starter Kit, SuperCyan, 3 | **/ 4 | 5 | using UnityEngine; 6 | using System.Collections; 7 | using IBM.Watson.DeveloperCloud.Logging; 8 | using IBM.Watson.DeveloperCloud.Services.SpeechToText.v1; 9 | using IBM.Watson.DeveloperCloud.Utilities; 10 | using IBM.Watson.DeveloperCloud.DataTypes; 11 | using IBM.Watson.DeveloperCloud.Connection; 12 | 13 | using System.Collections.Generic; 14 | using UnityEngine.UI; 15 | 16 | using IBM.Watson.DeveloperCloud.Services.TextToSpeech.v1; 17 | 18 | 19 | // added this from the TONE ANALYZER . CS file 20 | using IBM.Watson.DeveloperCloud.Services.ToneAnalyzer.v3; 21 | 22 | // and for Regex 23 | using System.Text.RegularExpressions; 24 | 25 | // and for Ethan's body 26 | using UnityStandardAssets; 27 | using UnityStandardAssets.Characters.ThirdPerson; 28 | 29 | 30 | public class ExampleStreaming : MonoBehaviour 31 | { 32 | 33 | // STT - BURNER CREDS - DELETE AFTER RECORDING 34 | private string _username_STT = "YOUR-CREDS"; 35 | private string _password_STT = "YOUR-CREDS"; 36 | private string _url_STT = "https://stream.watsonplatform.net/speech-to-text/api"; 37 | 38 | 39 | public Text ResultsField; 40 | public Text ResultsAnalysis; 41 | 42 | private int _recordingRoutine = 0; 43 | private string _microphoneID = null; 44 | private AudioClip _recording = null; 45 | private int _recordingBufferSize = 1; 46 | private int _recordingHZ = 22050; 47 | 48 | private SpeechToText _speechToText; 49 | 50 | // TEXT TO SPEECH - BURNER CREDENTIALS FOR PUBLIC DEMO I WILL DELETE AFTER RECORDING 51 | private string _username_TTS = "YOUR-CREDS"; 52 | private string _password_TTS = "YOUR-CREDS"; 53 | private string _url_TTS = "https://stream.watsonplatform.net/text-to-speech/api"; 54 | 55 | TextToSpeech _textToSpeech; 56 | 57 | 58 | // TONE ZONE 59 | // TONE ZONE 60 | private string _username_TONE = "YOUR-CREDS"; 61 | private string _password_TONE = "YOUR-CREDS"; 62 | private string _url_TONE = "https://gateway.watsonplatform.net/tone-analyzer/api"; 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | public float emotion_threshold; 74 | 75 | private ToneAnalyzer _toneAnalyzer; 76 | private string _toneAnalyzerVersionDate = "2017-05-26"; 77 | private string _stringToTestTone1 = "START AND TEST - But this totally sucks! I hate beans and liver!"; 78 | private string _stringToTestTone2 = "SECOND TEST - Failed Test Sucks"; 79 | private bool _analyzeToneTested = false; 80 | 81 | 82 | public MeshRenderer MacKenzieRenderer; // main digital human 83 | 84 | // Over the shoulder emotional spheres 85 | public MeshRenderer sphere_emo_joyRenderer; 86 | public MeshRenderer sphere_emo_angerRenderer; 87 | public MeshRenderer sphere_emo_fearRenderer; 88 | public MeshRenderer sphere_emo_disgustRenderer; 89 | public MeshRenderer sphere_emo_sadnessRenderer; 90 | 91 | public Material original_material; 92 | public Material red_material; 93 | public Material blue_material; 94 | public Material yellow_material; 95 | public Material green_material; 96 | public Material purple_material; 97 | public Material white_material; 98 | 99 | //string _testString = "I'm sorry. This is Text to Speech!I'm sorry. This is Text to Speech!"; 100 | 101 | /// TEST STRINGS OK 102 | 103 | // Pitch Shifting 104 | //string _testString = "This is Text to Speech!"; 105 | //string _testString = "This is Text to Speech!"; 106 | //string _testString = "This is Text to Speech!"; 107 | //string _testString = "hi"; 108 | 109 | // Good news and sorrow and uncertainty - ref https://console.bluemix.net/docs/services/text-to-speech/SSML-expressive.html#expressive 110 | // This is Text to Speech! 111 | string _testString = "Welcome!"; 112 | //string _testString = "Hello! Good News! Text to Speech is Working!"; 113 | //string _testString = "I am terribly sorry for the quality of service you have received."; 114 | //string _testString = "Can you please explain it again? I am not sure I understand."; 115 | 116 | //string _testString = "Can you please explain it again? I am confused and I'm not sure I understand."; 117 | 118 | 119 | string _createdCustomizationId; 120 | CustomVoiceUpdate _customVoiceUpdate; 121 | string _customizationName = "unity-example-customization"; 122 | string _customizationLanguage = "en-US"; 123 | string _customizationDescription = "A text to speech voice customization created within Unity."; 124 | string _testWord = "Watson"; 125 | 126 | private bool _synthesizeTested = false; 127 | private bool _getVoicesTested = false; 128 | private bool _getVoiceTested = false; 129 | private bool _getPronuciationTested = false; 130 | private bool _getCustomizationsTested = false; 131 | private bool _createCustomizationTested = false; 132 | private bool _deleteCustomizationTested = false; 133 | private bool _getCustomizationTested = false; 134 | private bool _updateCustomizationTested = false; 135 | private bool _getCustomizationWordsTested = false; 136 | private bool _addCustomizationWordsTested = false; 137 | private bool _deleteCustomizationWordTested = false; 138 | private bool _getCustomizationWordTested = false; 139 | 140 | 141 | 142 | void Start() 143 | { 144 | LogSystem.InstallDefaultReactors(); 145 | 146 | // Create credential and instantiate service 147 | Credentials credentials_STT = new Credentials(_username_STT, _password_STT, _url_STT); 148 | Credentials credentials_TTS = new Credentials(_username_TTS, _password_TTS, _url_TTS); 149 | Credentials credentials_TONE = new Credentials(_username_TONE, _password_TONE, _url_TONE); 150 | 151 | 152 | _speechToText = new SpeechToText(credentials_STT); 153 | _textToSpeech = new TextToSpeech(credentials_TTS); 154 | _toneAnalyzer = new ToneAnalyzer(credentials_TONE); 155 | 156 | 157 | /// TONE 158 | _toneAnalyzer.VersionDate = _toneAnalyzerVersionDate; 159 | 160 | 161 | 162 | // Over the shoulder emotional spheres 163 | sphere_emo_joyRenderer.material = yellow_material; 164 | sphere_emo_angerRenderer.material = red_material; 165 | sphere_emo_fearRenderer.material = purple_material; 166 | sphere_emo_disgustRenderer.material = green_material; 167 | sphere_emo_sadnessRenderer.material = blue_material; 168 | 169 | sphere_emo_joyRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 170 | sphere_emo_angerRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 171 | sphere_emo_fearRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 172 | sphere_emo_disgustRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 173 | sphere_emo_sadnessRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 174 | 175 | emotion_threshold = 0.75f; // for loose demo - above 75% seems to work well - may vary by signal 176 | 177 | Active = true; 178 | StartRecording(); 179 | 180 | Runnable.Run(Examples()); 181 | 182 | 183 | 184 | 185 | } 186 | 187 | public bool Active 188 | { 189 | get { return _speechToText.IsListening; } 190 | set 191 | { 192 | if (value && !_speechToText.IsListening) 193 | { 194 | _speechToText.DetectSilence = true; 195 | _speechToText.EnableWordConfidence = true; 196 | _speechToText.EnableTimestamps = true; 197 | _speechToText.SilenceThreshold = 0.01f; 198 | _speechToText.MaxAlternatives = 0; 199 | _speechToText.EnableInterimResults = true; 200 | _speechToText.OnError = OnError; 201 | _speechToText.InactivityTimeout = -1; 202 | _speechToText.ProfanityFilter = false; 203 | _speechToText.SmartFormatting = true; 204 | _speechToText.SpeakerLabels = false; 205 | _speechToText.WordAlternativesThreshold = null; 206 | _speechToText.StartListening(OnRecognize, OnRecognizeSpeaker); 207 | } 208 | else if (!value && _speechToText.IsListening) 209 | { 210 | _speechToText.StopListening(); 211 | } 212 | } 213 | } 214 | 215 | private void StartRecording() 216 | { 217 | if (_recordingRoutine == 0) 218 | { 219 | UnityObjectUtil.StartDestroyQueue(); 220 | _recordingRoutine = Runnable.Run(RecordingHandler()); 221 | } 222 | } 223 | 224 | private void StopRecording() 225 | { 226 | if (_recordingRoutine != 0) 227 | { 228 | Microphone.End(_microphoneID); 229 | Runnable.Stop(_recordingRoutine); 230 | _recordingRoutine = 0; 231 | } 232 | } 233 | 234 | private void OnError(string error) 235 | { 236 | Active = false; 237 | 238 | Log.Debug("ExampleStreaming.OnError()", "Error! {0}", error); 239 | } 240 | 241 | private IEnumerator RecordingHandler() 242 | { 243 | Log.Debug("ExampleStreaming.RecordingHandler()", "devices: {0}", Microphone.devices); 244 | _recording = Microphone.Start(_microphoneID, true, _recordingBufferSize, _recordingHZ); 245 | yield return null; // let _recordingRoutine get set.. 246 | 247 | if (_recording == null) 248 | { 249 | StopRecording(); 250 | yield break; 251 | } 252 | 253 | bool bFirstBlock = true; 254 | int midPoint = _recording.samples / 2; 255 | float[] samples = null; 256 | 257 | while (_recordingRoutine != 0 && _recording != null) 258 | { 259 | int writePos = Microphone.GetPosition(_microphoneID); 260 | if (writePos > _recording.samples || !Microphone.IsRecording(_microphoneID)) 261 | { 262 | Log.Error("ExampleStreaming.RecordingHandler()", "Microphone disconnected."); 263 | 264 | StopRecording(); 265 | yield break; 266 | } 267 | 268 | if ((bFirstBlock && writePos >= midPoint) 269 | || (!bFirstBlock && writePos < midPoint)) 270 | { 271 | // front block is recorded, make a RecordClip and pass it onto our callback. 272 | samples = new float[midPoint]; 273 | _recording.GetData(samples, bFirstBlock ? 0 : midPoint); 274 | 275 | AudioData record = new AudioData(); 276 | record.MaxLevel = Mathf.Max(Mathf.Abs(Mathf.Min(samples)), Mathf.Max(samples)); 277 | record.Clip = AudioClip.Create("Recording", midPoint, _recording.channels, _recordingHZ, false); 278 | record.Clip.SetData(samples, 0); 279 | 280 | _speechToText.OnListen(record); 281 | 282 | bFirstBlock = !bFirstBlock; 283 | } 284 | else 285 | { 286 | // calculate the number of samples remaining until we ready for a block of audio, 287 | // and wait that amount of time it will take to record. 288 | int remaining = bFirstBlock ? (midPoint - writePos) : (_recording.samples - writePos); 289 | float timeRemaining = (float)remaining / (float)_recordingHZ; 290 | 291 | yield return new WaitForSeconds(timeRemaining); 292 | } 293 | 294 | } 295 | 296 | yield break; 297 | } 298 | 299 | private void OnRecognize(SpeechRecognitionEvent result) 300 | { 301 | if (result != null && result.results.Length > 0) 302 | { 303 | foreach (var res in result.results) 304 | { 305 | foreach (var alt in res.alternatives) 306 | { 307 | string text = string.Format("{0} ({1}, {2:0.00})\n", alt.transcript, res.final ? "Final" : "Interim", alt.confidence); 308 | Log.Debug("ExampleStreaming.OnRecognize()", text); 309 | ResultsField.text = text; 310 | 311 | //// 312 | /// // ENTERING THE TONE ZONE - when the utterance contains this word 313 | //if (alt.transcript.Contains ("feel") | alt.transcript.Contains ("you") | alt.transcript.Contains ("Jimmy") | alt.transcript.Contains ("robot")) { 314 | if (true) { 315 | string GHI = alt.transcript; 316 | if (!_toneAnalyzer.GetToneAnalyze (OnGetToneAnalyze, OnFail, GHI)) 317 | Log.Debug ("ExampleToneAnalyzer.Examples()", "Failed to analyze!"); 318 | 319 | Log.Debug ("ExampleToneAnalyzer.Examples()", "NESTED TONE ZONE branch complete."); 320 | //ResultsField.text = "tone analyzed! 111"; 321 | // TEST END 322 | 323 | } 324 | 325 | 326 | ///// 327 | /// 328 | /// // ENTERING THE TONE ZONE - when the utterance contains this word 329 | if (alt.transcript.Contains ("reset")) { 330 | 331 | 332 | sphere_emo_joyRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 333 | sphere_emo_angerRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 334 | sphere_emo_fearRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 335 | sphere_emo_disgustRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 336 | sphere_emo_sadnessRenderer.transform.localScale = new Vector3(.075F, .075F, .075F); 337 | MacKenzieRenderer.transform.localScale = new Vector3(5F, 5F, 5F); 338 | 339 | } 340 | 341 | 342 | if (alt.transcript.Contains ("mouse") || alt.transcript.Contains ("tiny") ) { 343 | MacKenzieRenderer.transform.localScale = new Vector3(2F, 2F, 2F); 344 | } 345 | 346 | if (alt.transcript.Contains ("elephant") || alt.transcript.Contains ("giant") ) { 347 | MacKenzieRenderer.transform.localScale = new Vector3(10F, 10F, 10F); 348 | } 349 | 350 | 351 | //// Text to Speak Trigger zone 352 | 353 | if (alt.transcript.Contains("blue") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 354 | { 355 | _testString = "The color of the SKy. I am a little under the weather"; 356 | Runnable.Run(Examples()); 357 | 358 | } 359 | if (alt.transcript.Contains("yellow") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 360 | { 361 | _testString = "Oh Yes! The color of daisies! It makes me feel delighted to be alive! Good things happen with such a bright color."; 362 | Runnable.Run(Examples()); 363 | } // Cannot ECHO the trigger condition (or be ready for loop 364 | 365 | if (alt.transcript.Contains("happy") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 366 | { 367 | _testString = "That's so nice! You bring me great joy as well!"; 368 | Runnable.Run(Examples()); 369 | } // Cannot ECHO the trigger condition (or be ready for loop 370 | 371 | if (alt.transcript.Contains("mistake") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 372 | { 373 | _testString = "I am so sorry. I'll try to do better next time."; 374 | Runnable.Run(Examples()); 375 | } // Cannot ECHO the trigger condition (or be ready for loop 376 | 377 | if (alt.transcript.Contains("quantum") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 378 | { 379 | _testString = "I dont really know how to answer that."; 380 | Runnable.Run(Examples()); 381 | } // Cannot ECHO the trigger condition (or be ready for loop 382 | 383 | 384 | if (alt.transcript.Contains("despair") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 385 | { 386 | _testString = "Yes, hopeless"; 387 | Runnable.Run(Examples()); 388 | } // Cannot ECHO the trigger condition (or be ready for loop 389 | 390 | if (alt.transcript.Contains("bears") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 391 | { 392 | _testString = "Large carnivores terrify me"; 393 | Runnable.Run(Examples()); 394 | } // Cannot ECHO the trigger condition (or be ready for loop 395 | 396 | 397 | 398 | if (alt.transcript.Contains("goodbye") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 399 | { 400 | _testString = "Bye bye! And thank you!"; 401 | Runnable.Run(Examples()); 402 | } // Cannot ECHO the trigger condition (or be ready for loop 403 | 404 | 405 | 406 | } 407 | 408 | if (res.keywords_result != null && res.keywords_result.keyword != null) 409 | { 410 | foreach (var keyword in res.keywords_result.keyword) 411 | { 412 | Log.Debug("ExampleStreaming.OnRecognize()", "keyword: {0}, confidence: {1}, start time: {2}, end time: {3}", keyword.normalized_text, keyword.confidence, keyword.start_time, keyword.end_time); 413 | } 414 | } 415 | 416 | if (res.word_alternatives != null) 417 | { 418 | foreach (var wordAlternative in res.word_alternatives) 419 | { 420 | Log.Debug("ExampleStreaming.OnRecognize()", "Word alternatives found. Start time: {0} | EndTime: {1}", wordAlternative.start_time, wordAlternative.end_time); 421 | foreach(var alternative in wordAlternative.alternatives) 422 | Log.Debug("ExampleStreaming.OnRecognize()", "\t word: {0} | confidence: {1}", alternative.word, alternative.confidence); 423 | } 424 | } 425 | } 426 | } 427 | } 428 | 429 | private void OnRecognizeSpeaker(SpeakerRecognitionEvent result) 430 | { 431 | if (result != null) 432 | { 433 | foreach (SpeakerLabelsResult labelResult in result.speaker_labels) 434 | { 435 | Log.Debug("ExampleStreaming.OnRecognize()", string.Format("speaker result: {0} | confidence: {3} | from: {1} | to: {2}", labelResult.speaker, labelResult.from, labelResult.to, labelResult.confidence)); 436 | } 437 | } 438 | } 439 | 440 | 441 | // TTS CODE 442 | private IEnumerator Examples() 443 | { 444 | /// TONE 445 | /// // Analyze tone 446 | if (!_toneAnalyzer.GetToneAnalyze(OnGetToneAnalyze, OnFail, _stringToTestTone1)) 447 | Log.Debug("ExampleToneAnalyzer.Examples()", "Failed to analyze!"); 448 | 449 | while (!_analyzeToneTested) 450 | yield return null; 451 | 452 | Log.Debug("ExampleToneAnalyzer.Examples()", "Tone analyzer examples complete."); 453 | 454 | 455 | 456 | 457 | 458 | 459 | // Synthesize 460 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting synthesize."); 461 | _textToSpeech.Voice = VoiceType.en_US_Allison; 462 | _textToSpeech.ToSpeech(HandleToSpeechCallback, OnFail, _testString, true); 463 | while (!_synthesizeTested) 464 | yield return null; 465 | 466 | // Get Voices 467 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get voices."); 468 | _textToSpeech.GetVoices(OnGetVoices, OnFail); 469 | while (!_getVoicesTested) 470 | yield return null; 471 | 472 | // Get Voice 473 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get voice {0}.", VoiceType.en_US_Allison); 474 | _textToSpeech.GetVoice(OnGetVoice, OnFail, VoiceType.en_US_Allison); 475 | while (!_getVoiceTested) 476 | yield return null; 477 | 478 | // Get Pronunciation 479 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get pronunciation of {0}", _testWord); 480 | _textToSpeech.GetPronunciation(OnGetPronunciation, OnFail, _testWord, VoiceType.en_US_Allison); 481 | while (!_getPronuciationTested) 482 | yield return null; 483 | 484 | // Get Customizations 485 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get a list of customizations"); 486 | // _textToSpeech.GetCustomizations(OnGetCustomizations, OnFail); 487 | // while (!_getCustomizationsTested) 488 | // yield return null; 489 | 490 | // Create Customization 491 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to create a customization"); 492 | // _textToSpeech.CreateCustomization(OnCreateCustomization, OnFail, _customizationName, _customizationLanguage, _customizationDescription); 493 | // while (!_createCustomizationTested) 494 | // yield return null; 495 | 496 | // Get Customization 497 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get a customization"); 498 | // if (!_textToSpeech.GetCustomization(OnGetCustomization, OnFail, _createdCustomizationId)) 499 | // Log.Debug("ExampleTextToSpeech.Examples()", "Failed to get custom voice model!"); 500 | // while (!_getCustomizationTested) 501 | // yield return null; 502 | 503 | // Update Customization 504 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to update a customization"); 505 | // Word[] wordsToUpdateCustomization = 506 | // { 507 | // new Word() 508 | // { 509 | // word = "hello", 510 | // translation = "hullo" 511 | // }, 512 | // new Word() 513 | // { 514 | // word = "goodbye", 515 | // translation = "gbye" 516 | // }, 517 | // new Word() 518 | // { 519 | // word = "hi", 520 | // translation = "ohioooo" 521 | // } 522 | // }; 523 | 524 | // _customVoiceUpdate = new CustomVoiceUpdate() 525 | // { 526 | // words = wordsToUpdateCustomization, 527 | // description = "My updated description", 528 | // name = "My updated name" 529 | // }; 530 | 531 | if (!_textToSpeech.UpdateCustomization(OnUpdateCustomization, OnFail, _createdCustomizationId, _customVoiceUpdate)) 532 | Log.Debug("ExampleTextToSpeech.Examples()", "Failed to update customization!"); 533 | while (!_updateCustomizationTested) 534 | yield return null; 535 | 536 | // Get Customization Words 537 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get a customization's words"); 538 | // if (!_textToSpeech.GetCustomizationWords(OnGetCustomizationWords, OnFail, _createdCustomizationId)) 539 | // Log.Debug("ExampleTextToSpeech.GetCustomizationWords()", "Failed to get {0} words!", _createdCustomizationId); 540 | // while (!_getCustomizationWordsTested) 541 | // yield return null; 542 | 543 | // Add Customization Words 544 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to add words to a customization"); 545 | // Word[] wordArrayToAddToCustomization = 546 | // { 547 | // new Word() 548 | // { 549 | // word = "bananna", 550 | // translation = "arange" 551 | // }, 552 | // new Word() 553 | // { 554 | // word = "orange", 555 | // translation = "gbye" 556 | // }, 557 | // new Word() 558 | // { 559 | // word = "tomato", 560 | // translation = "tomahto" 561 | // } 562 | // }; 563 | 564 | // Words wordsToAddToCustomization = new Words() 565 | // { 566 | // words = wordArrayToAddToCustomization 567 | // }; 568 | 569 | // if (!_textToSpeech.AddCustomizationWords(OnAddCustomizationWords, OnFail, _createdCustomizationId, wordsToAddToCustomization)) 570 | // Log.Debug("ExampleTextToSpeech.AddCustomizationWords()", "Failed to add words to {0}!", _createdCustomizationId); 571 | // while (!_addCustomizationWordsTested) 572 | // yield return null; 573 | 574 | // Get Customization Word 575 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get the translation of a custom voice model's word."); 576 | // string customIdentifierWord = wordsToUpdateCustomization[0].word; 577 | // if (!_textToSpeech.GetCustomizationWord(OnGetCustomizationWord, OnFail, _createdCustomizationId, customIdentifierWord)) 578 | // Log.Debug("ExampleTextToSpeech.GetCustomizationWord()", "Failed to get the translation of {0} from {1}!", customIdentifierWord, _createdCustomizationId); 579 | // while (!_getCustomizationWordTested) 580 | // yield return null; 581 | 582 | // Delete Customization Word 583 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to delete customization word from custom voice model."); 584 | string wordToDelete = "goodbye"; 585 | if (!_textToSpeech.DeleteCustomizationWord(OnDeleteCustomizationWord, OnFail, _createdCustomizationId, wordToDelete)) 586 | Log.Debug("ExampleTextToSpeech.DeleteCustomizationWord()", "Failed to delete {0} from {1}!", wordToDelete, _createdCustomizationId); 587 | while (!_deleteCustomizationWordTested) 588 | yield return null; 589 | 590 | // Delete Customization 591 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to delete a customization"); 592 | if (!_textToSpeech.DeleteCustomization(OnDeleteCustomization, OnFail, _createdCustomizationId)) 593 | Log.Debug("ExampleTextToSpeech.DeleteCustomization()", "Failed to delete custom voice model!"); 594 | while (!_deleteCustomizationTested) 595 | yield return null; 596 | 597 | Log.Debug("ExampleTextToSpeech.Examples()", "Text to Speech examples complete."); 598 | } 599 | 600 | 601 | 602 | 603 | 604 | 605 | void HandleToSpeechCallback(AudioClip clip, Dictionary customData = null) 606 | { 607 | PlayClip(clip); 608 | } 609 | 610 | private void PlayClip(AudioClip clip) 611 | { 612 | if (Application.isPlaying && clip != null) 613 | { 614 | GameObject audioObject = new GameObject("AudioObject"); 615 | AudioSource source = audioObject.AddComponent(); 616 | source.spatialBlend = 0.0f; 617 | source.loop = false; 618 | source.clip = clip; 619 | source.Play(); 620 | 621 | Destroy(audioObject, clip.length); 622 | 623 | _synthesizeTested = true; 624 | } 625 | } 626 | 627 | private void OnGetVoices(Voices voices, Dictionary customData = null) 628 | { 629 | Log.Debug("ExampleTextToSpeech.OnGetVoices()", "Text to Speech - Get voices response: {0}", customData["json"].ToString()); 630 | _getVoicesTested = true; 631 | } 632 | 633 | private void OnGetVoice(Voice voice, Dictionary customData = null) 634 | { 635 | Log.Debug("ExampleTextToSpeech.OnGetVoice()", "Text to Speech - Get voice response: {0}", customData["json"].ToString()); 636 | _getVoiceTested = true; 637 | } 638 | 639 | private void OnGetPronunciation(Pronunciation pronunciation, Dictionary customData = null) 640 | { 641 | Log.Debug("ExampleTextToSpeech.OnGetPronunciation()", "Text to Speech - Get pronunciation response: {0}", customData["json"].ToString()); 642 | _getPronuciationTested = true; 643 | } 644 | 645 | // private void OnGetCustomizations(Customizations customizations, Dictionary customData = null) 646 | // { 647 | // Log.Debug("ExampleTextToSpeech.OnGetCustomizations()", "Text to Speech - Get customizations response: {0}", customData["json"].ToString()); 648 | // _getCustomizationsTested = true; 649 | // } 650 | 651 | // private void OnCreateCustomization(CustomizationID customizationID, Dictionary customData = null) 652 | // { 653 | // Log.Debug("ExampleTextToSpeech.OnCreateCustomization()", "Text to Speech - Create customization response: {0}", customData["json"].ToString()); 654 | // _createdCustomizationId = customizationID.customization_id; 655 | // _createCustomizationTested = true; 656 | // } 657 | 658 | private void OnDeleteCustomization(bool success, Dictionary customData = null) 659 | { 660 | Log.Debug("ExampleTextToSpeech.OnDeleteCustomization()", "Text to Speech - Delete customization response: {0}", customData["json"].ToString()); 661 | _createdCustomizationId = null; 662 | _deleteCustomizationTested = true; 663 | } 664 | 665 | // private void OnGetCustomization(Customization customization, Dictionary customData = null) 666 | // { 667 | // Log.Debug("ExampleTextToSpeech.OnGetCustomization()", "Text to Speech - Get customization response: {0}", customData["json"].ToString()); 668 | // _getCustomizationTested = true; 669 | // } 670 | 671 | private void OnUpdateCustomization(bool success, Dictionary customData = null) 672 | { 673 | Log.Debug("ExampleTextToSpeech.OnUpdateCustomization()", "Text to Speech - Update customization response: {0}", customData["json"].ToString()); 674 | _updateCustomizationTested = true; 675 | } 676 | 677 | // private void OnGetCustomizationWords(Words words, Dictionary customData = null) 678 | // { 679 | // Log.Debug("ExampleTextToSpeech.OnGetCustomizationWords()", "Text to Speech - Get customization words response: {0}", customData["json"].ToString()); 680 | // _getCustomizationWordsTested = true; 681 | // } 682 | 683 | private void OnAddCustomizationWords(bool success, Dictionary customData = null) 684 | { 685 | Log.Debug("ExampleTextToSpeech.OnAddCustomizationWords()", "Text to Speech - Add customization words response: {0}", customData["json"].ToString()); 686 | _addCustomizationWordsTested = true; 687 | } 688 | 689 | private void OnDeleteCustomizationWord(bool success, Dictionary customData = null) 690 | { 691 | Log.Debug("ExampleTextToSpeech.OnDeleteCustomizationWord()", "Text to Speech - Delete customization word response: {0}", customData["json"].ToString()); 692 | _deleteCustomizationWordTested = true; 693 | } 694 | 695 | private void OnGetCustomizationWord(Translation translation, Dictionary customData = null) 696 | { 697 | Log.Debug("ExampleTextToSpeech.OnGetCustomizationWord()", "Text to Speech - Get customization word response: {0}", customData["json"].ToString()); 698 | _getCustomizationWordTested = true; 699 | } 700 | 701 | private void OnFail(RESTConnector.Error error, Dictionary customData) 702 | { 703 | Log.Error("ExampleTextToSpeech.OnFail()", "Error received: {0}", error.ToString()); 704 | } 705 | 706 | 707 | 708 | // TESTING 709 | private void OnGetToneAnalyze(ToneAnalyzerResponse resp, Dictionary customData) 710 | { 711 | Log.Debug("ExampleToneAnalyzer.OnGetToneAnalyze()", "{0}", customData["json"].ToString()); 712 | 713 | ResultsAnalysis.text = (customData["json"].ToString()); // works but long and cannot read 714 | 715 | /// Logging 716 | Log.Debug ("$$$$$ TONE LOG 0 ANGER", "{0}",resp.document_tone.tone_categories[0].tones[0].score); // ANGER resp.document_tone.tone_categories [0].tones [0].score); 717 | Log.Debug ("$$$$$ TONE LOG 1 DISGUST", "{0}",resp.document_tone.tone_categories[0].tones[1].score); // DISGUST 718 | Log.Debug ("$$$$$ TONE LOG 2 FEAR", "{0}",resp.document_tone.tone_categories[0].tones[2].score); // FEAR 719 | Log.Debug ("$$$$$ TONE LOG 3 JOY", "{0}",resp.document_tone.tone_categories[0].tones[3].score); // JOY 720 | Log.Debug ("$$$$$ TONE LOG 4 SAD", "{0}",resp.document_tone.tone_categories[0].tones[4].score); // SADNESS 721 | 722 | Log.Debug ("$$$$$ TONE ANALYTICAL", "{0}",resp.document_tone.tone_categories[1].tones[0].score); // ANALYTICAL 723 | Log.Debug ("$$$$$ TONE CONFIDENT", "{0}",resp.document_tone.tone_categories[1].tones[1].score); // CONFIDENT 724 | Log.Debug ("$$$$$ TONE TENTATIVE", "{0}",resp.document_tone.tone_categories[1].tones[2].score); // TENTATIVE 725 | 726 | 727 | // EMOTION 728 | if (resp.document_tone.tone_categories [0].tones [0].score > emotion_threshold) { 729 | sphere_emo_angerRenderer.transform.localScale += new Vector3(0.025F, 0.025F, 0.025F); 730 | 731 | } 732 | else if (resp.document_tone.tone_categories [0].tones [1].score > emotion_threshold) { 733 | sphere_emo_disgustRenderer.transform.localScale += new Vector3(0.025F, 0.025F, 0.025F); 734 | 735 | } 736 | else if (resp.document_tone.tone_categories [0].tones [2].score > emotion_threshold) { 737 | sphere_emo_fearRenderer.transform.localScale += new Vector3(0.025F, 0.025F, 0.025F); 738 | 739 | } 740 | else if (resp.document_tone.tone_categories [0].tones [3].score > emotion_threshold) { 741 | sphere_emo_joyRenderer.transform.localScale += new Vector3(0.025F, 0.025F, 0.025F); 742 | 743 | } 744 | else if (resp.document_tone.tone_categories [0].tones [4].score > emotion_threshold) { 745 | sphere_emo_sadnessRenderer.transform.localScale += new Vector3(0.025F, 0.025F, 0.025F); 746 | 747 | } 748 | 749 | 750 | 751 | // OTHER TEXT - Formatting for On Screen dump - LATER - pretty this up to use standard DESERIALIZE methods and table 752 | string RAW = (customData["json"].ToString()); // works but long and cannot read 753 | //RAW = string.Concat("Tone Response \n", RAW); 754 | RAW = Regex.Replace(RAW, "tone_categories", " \\\n"); 755 | RAW = Regex.Replace(RAW, "}", "} \\\n"); 756 | RAW = Regex.Replace(RAW, "tone_id", " "); 757 | RAW = Regex.Replace(RAW, "tone_name", " "); 758 | RAW = Regex.Replace(RAW, "score", " "); 759 | RAW = Regex.Replace(RAW, @"[{\\},:]", ""); 760 | RAW = Regex.Replace(RAW, "\"", ""); 761 | ResultsAnalysis.text = RAW; 762 | 763 | _analyzeToneTested = true; 764 | } 765 | 766 | 767 | 768 | 769 | } 770 | -------------------------------------------------------------------------------- /ExampleStreaming6.cs: -------------------------------------------------------------------------------- 1 | /** 2 | demo here https://youtu.be/mrKzq-z7h14 3 | Part 1 4 | Implemented some basic LOGGING so i can start to do more ground truth capture with voice in my lab. 5 | Each utterance is captured and classified as mode/state. 6 | Voice attributes of Text to Speech are changeable using Verbal command and Control. 7 | */ 8 | 9 | using UnityEngine; 10 | using System.Collections; 11 | using IBM.Watson.DeveloperCloud.Logging; 12 | using IBM.Watson.DeveloperCloud.Services.SpeechToText.v1; 13 | using IBM.Watson.DeveloperCloud.Utilities; 14 | using IBM.Watson.DeveloperCloud.DataTypes; 15 | using System.Collections.Generic; 16 | using UnityEngine.UI; 17 | 18 | // For Read Write to Logs and Memory Files 19 | using System; 20 | using System.IO; 21 | 22 | // For Text to speech (not sure if we should declare here or only in ETTS 23 | using IBM.Watson.DeveloperCloud.Services.TextToSpeech.v1; 24 | using IBM.Watson.DeveloperCloud.Connection; 25 | 26 | 27 | 28 | public class ExampleStreaming : MonoBehaviour 29 | { 30 | #region PLEASE SET THESE VARIABLES IN THE INSPECTOR 31 | [SerializeField] 32 | private string _username_STT; 33 | [SerializeField] 34 | private string _password_STT; 35 | [SerializeField] 36 | private string _url_STT; 37 | #endregion 38 | 39 | #region PLEASE SET THESE VARIABLES IN THE INSPECTOR 40 | [SerializeField] 41 | private string _username_TTS; 42 | [SerializeField] 43 | private string _password_TTS; 44 | [SerializeField] 45 | private string _url_TTS; 46 | #endregion 47 | 48 | public Text ResultsField; // Maps to Text Streaming - from Raw and Original Example file 49 | public Text System_Mode; /// System Mode - (1) Diagnostics (2) Teach_and_Learn (3) Factoid (4) Conversation 50 | public Text Agent_Status; /// Agent Status and Emotional state (confused, apologetic, listening) 51 | public Text Voicebox_Status; 52 | 53 | // https://console.bluemix.net/docs/services/text-to-speech/SSML-transformation.html#transformation 54 | 55 | public int voicebox_prosody_pitch = 0; // -100% to 100% 56 | public int voicebox_pitch_range = 0; // -100% to 100% 57 | public int voicebox_glottal_tension = 0; // -100% to 100% 58 | public int voicebox_breathiness = 0; // -100% to 100% 59 | public int voicebox_rate = 0; // -100% to 100% 60 | //public int voicebox_timbre // "Sunrise" or "Breeze" 61 | public int voicebox_timbre_extent = 50; // -0% to 100% 62 | 63 | 64 | 65 | private int _recordingRoutine = 0; 66 | private string _microphoneID = null; 67 | private AudioClip _recording = null; 68 | private int _recordingBufferSize = 1; 69 | private int _recordingHZ = 22050; 70 | 71 | private SpeechToText _speechToText; 72 | private TextToSpeech _textToSpeech; 73 | 74 | 75 | // To enable TTS calls from this code 76 | string _testString = "Great news!"; 77 | 78 | string _createdCustomizationId; 79 | CustomVoiceUpdate _customVoiceUpdate; 80 | string _customizationName = "unity-example-customization"; 81 | string _customizationLanguage = "en-US"; 82 | string _customizationDescription = "A text to speech voice customization created within Unity."; 83 | string _testWord = "Watson"; 84 | 85 | private bool _synthesizeTested = false; 86 | private bool _getVoicesTested = false; 87 | private bool _getVoiceTested = false; 88 | private bool _getPronuciationTested = false; 89 | private bool _getCustomizationsTested = false; 90 | private bool _createCustomizationTested = false; 91 | private bool _deleteCustomizationTested = false; 92 | private bool _getCustomizationTested = false; 93 | private bool _updateCustomizationTested = false; 94 | private bool _getCustomizationWordsTested = false; 95 | private bool _addCustomizationWordsTested = false; 96 | private bool _deleteCustomizationWordTested = false; 97 | private bool _getCustomizationWordTested = false; 98 | 99 | 100 | 101 | 102 | 103 | 104 | void Start() 105 | { 106 | 107 | /// TTS TEST ON START 108 | LogSystem.InstallDefaultReactors(); 109 | // Create credential and instantiate service 110 | 111 | Credentials credentials_TTS = new Credentials(_username_TTS, _password_TTS, _url_TTS); 112 | _textToSpeech = new TextToSpeech(credentials_TTS); 113 | 114 | _testString = "Hello! Welcome to IBM Watson Unity Voicebox!"; 115 | Runnable.Run (Examples ()); 116 | 117 | /// STT BEGIN 118 | Credentials credentials_STT = new Credentials(_username_STT, _password_STT, _url_STT); 119 | _speechToText = new SpeechToText(credentials_STT); 120 | Active = true; 121 | 122 | StartRecording(); 123 | 124 | Agent_Status.text = "Listening"; 125 | System_Mode.text = "mode: listen-learn"; 126 | 127 | voicebox_prosody_pitch = 150; // 150 hZ to 350 hz 128 | voicebox_pitch_range = 0; // -100% to 100% 129 | voicebox_glottal_tension = 25; // -100% to 100% 130 | voicebox_breathiness = 50; // -100% to 100% 131 | voicebox_rate = 0; // -100% to 100% 132 | voicebox_timbre_extent = 50; // -0% to 100% 133 | //voicebox_timbre // "Sunrise" or "Breeze" 134 | 135 | 136 | } 137 | 138 | 139 | /// TEMP CODE 140 | public void TalkieTalkie(){ 141 | 142 | //Log.Debug("red", ""); 143 | //Runnable.Run (Examples ("blue")); 144 | } 145 | 146 | public void TextTalk(string str){ 147 | //Runnable.Run (Examples (_testString2)); 148 | //Runnable.Run (Examples ("blue")); 149 | } 150 | 151 | 152 | 153 | public bool Active 154 | { 155 | get { return _speechToText.IsListening; } 156 | set 157 | { 158 | if (value && !_speechToText.IsListening) 159 | { 160 | _speechToText.DetectSilence = true; 161 | _speechToText.EnableWordConfidence = true; 162 | _speechToText.EnableTimestamps = true; 163 | _speechToText.SilenceThreshold = 0.01f; 164 | _speechToText.MaxAlternatives = 0; 165 | _speechToText.EnableInterimResults = true; 166 | _speechToText.OnError = OnError; 167 | _speechToText.InactivityTimeout = -1; 168 | _speechToText.ProfanityFilter = false; 169 | _speechToText.SmartFormatting = true; 170 | _speechToText.SpeakerLabels = false; 171 | _speechToText.WordAlternativesThreshold = null; 172 | _speechToText.StartListening(OnRecognize, OnRecognizeSpeaker); 173 | } 174 | else if (!value && _speechToText.IsListening) 175 | { 176 | _speechToText.StopListening(); 177 | } 178 | } 179 | } 180 | 181 | private void StartRecording() 182 | { 183 | if (_recordingRoutine == 0) 184 | { 185 | UnityObjectUtil.StartDestroyQueue(); 186 | _recordingRoutine = Runnable.Run(RecordingHandler()); 187 | } 188 | } 189 | 190 | private void StopRecording() 191 | { 192 | if (_recordingRoutine != 0) 193 | { 194 | Microphone.End(_microphoneID); 195 | Runnable.Stop(_recordingRoutine); 196 | _recordingRoutine = 0; 197 | } 198 | } 199 | 200 | private void OnError(string error) 201 | { 202 | Active = false; 203 | 204 | Log.Debug("ExampleStreaming.OnError()", "Error! {0}", error); 205 | } 206 | 207 | private IEnumerator RecordingHandler() 208 | { 209 | Log.Debug("ExampleStreaming.RecordingHandler()", "devices: {0}", Microphone.devices); 210 | _recording = Microphone.Start(_microphoneID, true, _recordingBufferSize, _recordingHZ); 211 | yield return null; // let _recordingRoutine get set.. 212 | 213 | if (_recording == null) 214 | { 215 | StopRecording(); 216 | yield break; 217 | } 218 | 219 | bool bFirstBlock = true; 220 | int midPoint = _recording.samples / 2; 221 | float[] samples = null; 222 | 223 | while (_recordingRoutine != 0 && _recording != null) 224 | { 225 | int writePos = Microphone.GetPosition(_microphoneID); 226 | if (writePos > _recording.samples || !Microphone.IsRecording(_microphoneID)) 227 | { 228 | Log.Error("ExampleStreaming.RecordingHandler()", "Microphone disconnected."); 229 | 230 | StopRecording(); 231 | yield break; 232 | } 233 | 234 | if ((bFirstBlock && writePos >= midPoint) 235 | || (!bFirstBlock && writePos < midPoint)) 236 | { 237 | // front block is recorded, make a RecordClip and pass it onto our callback. 238 | samples = new float[midPoint]; 239 | _recording.GetData(samples, bFirstBlock ? 0 : midPoint); 240 | 241 | AudioData record = new AudioData(); 242 | record.MaxLevel = Mathf.Max(Mathf.Abs(Mathf.Min(samples)), Mathf.Max(samples)); 243 | record.Clip = AudioClip.Create("Recording", midPoint, _recording.channels, _recordingHZ, false); 244 | record.Clip.SetData(samples, 0); 245 | 246 | _speechToText.OnListen(record); 247 | 248 | bFirstBlock = !bFirstBlock; 249 | } 250 | else 251 | { 252 | // calculate the number of samples remaining until we ready for a block of audio, 253 | // and wait that amount of time it will take to record. 254 | int remaining = bFirstBlock ? (midPoint - writePos) : (_recording.samples - writePos); 255 | float timeRemaining = (float)remaining / (float)_recordingHZ; 256 | 257 | yield return new WaitForSeconds(timeRemaining); 258 | } 259 | 260 | } 261 | 262 | yield break; 263 | } 264 | 265 | private void OnRecognize(SpeechRecognitionEvent result, Dictionary customData) 266 | { 267 | if (result != null && result.results.Length > 0) 268 | { 269 | foreach (var res in result.results) 270 | { 271 | foreach (var alt in res.alternatives) { 272 | string text = string.Format ("{0} ({1}, {2:0.00})\n", alt.transcript, res.final ? "Final" : "Interim", alt.confidence); 273 | Log.Debug ("ExampleStreaming.OnRecognize()", text); 274 | ResultsField.text = text; 275 | 276 | 277 | 278 | 279 | ///// HACK LOG INSERT BEGINS 280 | 281 | 282 | if (ResultsField.text.Contains ("Final")) // needs to be final or ECHO happens 283 | { 284 | 285 | if (alt.transcript.Contains ("echo") || alt.transcript.Contains ("repeat") ) 286 | { 287 | _testString = alt.transcript; 288 | _testString = _testString.Replace ("echo", ""); 289 | _testString = _testString.Replace ("repeat", ""); 290 | Runnable.Run (Examples ()); 291 | } 292 | 293 | else if (alt.transcript.Contains ("voicebox")) 294 | { 295 | _testString = "Voice traits have been modified."; 296 | 297 | if ( alt.transcript.Contains ("increase") && alt.transcript.Contains ("pitch") ) 298 | { 299 | voicebox_prosody_pitch = voicebox_prosody_pitch + 25; // 150 hZ to 350 hz 300 | voicebox_pitch_range = voicebox_pitch_range + 25; // -100% to 100% 301 | 302 | } 303 | else if (alt.transcript.Contains ("increase") && alt.transcript.Contains ("rate") ) 304 | { 305 | voicebox_rate = voicebox_rate + 25; // -100% to 100% 306 | } 307 | else if (alt.transcript.Contains ("increase") && alt.transcript.Contains ("tension") ) 308 | { 309 | voicebox_glottal_tension = voicebox_glottal_tension + 25; // -100% to 100% 310 | voicebox_breathiness = voicebox_breathiness + 25; // -100% to 100% 311 | voicebox_timbre_extent = voicebox_timbre_extent + 10; // -0% to 100% 312 | } 313 | else if (alt.transcript.Contains ("decrease") && alt.transcript.Contains ("pitch") ) 314 | { 315 | voicebox_prosody_pitch = voicebox_prosody_pitch - 25; // 150 hZ to 350 hz 316 | voicebox_pitch_range = voicebox_pitch_range - 25; // -100% to 100% 317 | 318 | } 319 | else if (alt.transcript.Contains ("decrease") && alt.transcript.Contains ("rate") ) 320 | { 321 | voicebox_rate = voicebox_rate - 25; // -100% to 100% 322 | } 323 | else if (alt.transcript.Contains ("decrease") && alt.transcript.Contains ("tension") ) 324 | { 325 | voicebox_glottal_tension = voicebox_glottal_tension - 25; // -100% to 100% 326 | voicebox_breathiness = voicebox_breathiness - 25; // -100% to 100% 327 | voicebox_timbre_extent = voicebox_timbre_extent - 10; // -0% to 100% 328 | } 329 | else if (alt.transcript.Contains ("reset") ) 330 | { 331 | voicebox_prosody_pitch = 0; // -100% to 100% 332 | voicebox_pitch_range = 0; // -100% to 100% 333 | voicebox_glottal_tension = 0; // -100% to 100% 334 | voicebox_breathiness = 0; // -100% to 100% 335 | voicebox_rate = 0; // -100% to 100% 336 | voicebox_timbre_extent = 50; // -0% to 100% 337 | } 338 | 339 | else 340 | { 341 | _testString = "My voice is my passport"; // if no changes 342 | } 343 | 344 | Voicebox_Status.text = 345 | "voicebox:" + 346 | "\n pitch" + 347 | voicebox_prosody_pitch + 348 | "hz \n pitch_range" + 349 | voicebox_pitch_range + 350 | "\n glottal_tension" + 351 | voicebox_glottal_tension + 352 | "\n breathiness" + 353 | voicebox_breathiness + 354 | "\n rate" + 355 | voicebox_rate + 356 | "\n timbre_extent" + 357 | voicebox_timbre_extent; 358 | 359 | 360 | _testString = "" + 361 | "" + 364 | " " + 374 | _testString + 375 | "" + 376 | "" + 377 | ""; 378 | Runnable.Run (Examples ()); 379 | 380 | 381 | 382 | } // end of voicebox 383 | 384 | else if (alt.transcript.Contains ("status")) 385 | { 386 | 387 | /// Count Transcript lines 388 | System.IO.StreamReader reader = System.IO.File.OpenText(@"MacKenzie_Conversation_Log.txt"); 389 | int iLineCount = 0; 390 | while (reader.Peek() >= 0) 391 | { reader.ReadLine(); 392 | ++iLineCount; } 393 | reader.Close(); 394 | 395 | _testString = "" + 396 | "I am in " + 397 | System_Mode.text + 398 | " and, " + 399 | "There are "+ iLineCount + 400 | " utterances in the transcript log." + 401 | " Voicebox settings. Pitch is " + 402 | voicebox_prosody_pitch + 403 | " hertz. The speech rate is " + 404 | voicebox_rate + 405 | ". And the glottal tension is " + 406 | voicebox_glottal_tension + 407 | ""; 408 | Runnable.Run (Examples ()); 409 | } 410 | 411 | 412 | 413 | // check if user is changing SYSTEM MODE 414 | else if ((alt.transcript.Contains ("switch") || alt.transcript.Contains ("change")) && alt.transcript.Contains ("mode")) 415 | { 416 | if (ResultsField.text.Contains ("diagnostic")) { 417 | System_Mode.text = "mode: diagnostic"; 418 | _testString = "Diagnostic Mode Engaged"; 419 | Runnable.Run (Examples ()); 420 | } 421 | else if (ResultsField.text.Contains ("listen") || ResultsField.text.Contains ("learn")) { 422 | _testString = "Listen Learn Mode Engaged"; 423 | Runnable.Run (Examples ()); 424 | System_Mode.text = "mode: listen-learn"; 425 | } 426 | else if (ResultsField.text.Contains ("conversation")) { 427 | System_Mode.text = "mode: conversation"; 428 | _testString = "Conversation Mode Engaged!"; 429 | Runnable.Run (Examples ()); 430 | 431 | } 432 | else if (ResultsField.text.Contains ("factoid")) { 433 | _testString = "Factoid Mode Engaged"; 434 | Runnable.Run (Examples ()); 435 | System_Mode.text = "mode: factoid"; 436 | } 437 | } 438 | 439 | 440 | 441 | //string _testString = "This is Text to Speech!"; 442 | //string _testString = "hi"; 443 | // Good news and sorrow and uncertainty - ref https://console.bluemix.net/docs/services/text-to-speech/SSML-expressive.html#expressive 444 | // This is Text to Speech! 445 | //string _testString = "I am terribly sorry for the quality of service you have received."; 446 | //string _testString = "Can you please explain it again? I am not sure I understand."; 447 | //string _testString = "Can you please explain it again? I am confused and I'm not sure I understand."; 448 | 449 | 450 | //// Check if file exists (later, we dont need to do this all the time, but here for now. OPTIMIZE LATER 451 | string path = @"MacKenzie_Conversation_Log.txt"; 452 | if (!File.Exists (path)) { 453 | // Create a file to write to if the file does not exist 454 | using (StreamWriter sw = File.CreateText (path)) { 455 | sw.WriteLine (string.Concat ("timestamp,","mode,","transcript")); 456 | sw.WriteLine (string.Concat (System.DateTime.Now.ToLongTimeString (), "-", System.DateTime.Now.ToString ("MM/dd/yyyy"), ",log_initialized,", "MacKenzie_Conversation_Log Initialized")); 457 | } 458 | } 459 | 460 | 461 | 462 | //// WRITE TO LOG FILE - time-date and MODE and transcript (final) 463 | using (StreamWriter sw = File.AppendText (path)) { 464 | sw.WriteLine (string.Concat (System.DateTime.Now.ToLongTimeString (), "-", System.DateTime.Now.ToString ("MM/dd/yyyy"), ",",System_Mode.text,",", alt.transcript)); 465 | } 466 | 467 | } 468 | ////// HACK LOG INSERT ENDS 469 | 470 | 471 | } 472 | 473 | if (res.keywords_result != null && res.keywords_result.keyword != null) 474 | { 475 | foreach (var keyword in res.keywords_result.keyword) 476 | { 477 | Log.Debug("ExampleStreaming.OnRecognize()", "keyword: {0}, confidence: {1}, start time: {2}, end time: {3}", keyword.normalized_text, keyword.confidence, keyword.start_time, keyword.end_time); 478 | } 479 | } 480 | 481 | if (res.word_alternatives != null) 482 | { 483 | foreach (var wordAlternative in res.word_alternatives) 484 | { 485 | Log.Debug("ExampleStreaming.OnRecognize()", "Word alternatives found. Start time: {0} | EndTime: {1}", wordAlternative.start_time, wordAlternative.end_time); 486 | foreach(var alternative in wordAlternative.alternatives) 487 | Log.Debug("ExampleStreaming.OnRecognize()", "\t word: {0} | confidence: {1}", alternative.word, alternative.confidence); 488 | } 489 | } 490 | } 491 | } 492 | } 493 | 494 | private void OnRecognizeSpeaker(SpeakerRecognitionEvent result, Dictionary customData) 495 | { 496 | if (result != null) 497 | { 498 | foreach (SpeakerLabelsResult labelResult in result.speaker_labels) 499 | { 500 | Log.Debug("ExampleStreaming.OnRecognize()", string.Format("speaker result: {0} | confidence: {3} | from: {1} | to: {2}", labelResult.speaker, labelResult.from, labelResult.to, labelResult.confidence)); 501 | } 502 | } 503 | } 504 | 505 | 506 | 507 | 508 | 509 | 510 | 511 | 512 | public IEnumerator Examples() ///// RYAN CHECK - changed to Public / from private and may need to declare call examplestts.exampels from other code 513 | { 514 | //_testString = str; 515 | 516 | // Synthesize 517 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting synthesize."); 518 | _textToSpeech.Voice = VoiceType.en_US_Allison; 519 | _textToSpeech.ToSpeech(HandleToSpeechCallback, OnFail, _testString, true); 520 | while (!_synthesizeTested) 521 | yield return null; 522 | 523 | // Get Voices 524 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get voices."); 525 | _textToSpeech.GetVoices(OnGetVoices, OnFail); 526 | while (!_getVoicesTested) 527 | yield return null; 528 | 529 | // Get Voice 530 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get voice {0}.", VoiceType.en_US_Allison); 531 | _textToSpeech.GetVoice(OnGetVoice, OnFail, VoiceType.en_US_Allison); 532 | while (!_getVoiceTested) 533 | yield return null; 534 | 535 | // Get Pronunciation 536 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get pronunciation of {0}", _testWord); 537 | _textToSpeech.GetPronunciation(OnGetPronunciation, OnFail, _testWord, VoiceType.en_US_Allison); 538 | while (!_getPronuciationTested) 539 | yield return null; 540 | // 541 | // // Get Customizations 542 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get a list of customizations"); 543 | // _textToSpeech.GetCustomizations(OnGetCustomizations, OnFail); 544 | // while (!_getCustomizationsTested) 545 | // yield return null; 546 | // 547 | // // Create Customization 548 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to create a customization"); 549 | // _textToSpeech.CreateCustomization(OnCreateCustomization, OnFail, _customizationName, _customizationLanguage, _customizationDescription); 550 | // while (!_createCustomizationTested) 551 | // yield return null; 552 | // 553 | // // Get Customization 554 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get a customization"); 555 | // if (!_textToSpeech.GetCustomization(OnGetCustomization, OnFail, _createdCustomizationId)) 556 | // Log.Debug("ExampleTextToSpeech.Examples()", "Failed to get custom voice model!"); 557 | // while (!_getCustomizationTested) 558 | // yield return null; 559 | 560 | // // Update Customization 561 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to update a customization"); 562 | // Word[] wordsToUpdateCustomization = 563 | // { 564 | // new Word() 565 | // { 566 | // word = "hello", 567 | // translation = "hullo" 568 | // }, 569 | // new Word() 570 | // { 571 | // word = "goodbye", 572 | // translation = "gbye" 573 | // }, 574 | // new Word() 575 | // { 576 | // word = "hi", 577 | // translation = "ohioooo" 578 | // } 579 | // }; 580 | // 581 | // _customVoiceUpdate = new CustomVoiceUpdate() 582 | // { 583 | // words = wordsToUpdateCustomization, 584 | // description = "My updated description", 585 | // name = "My updated name" 586 | // }; 587 | 588 | if (!_textToSpeech.UpdateCustomization(OnUpdateCustomization, OnFail, _createdCustomizationId, _customVoiceUpdate)) 589 | Log.Debug("ExampleTextToSpeech.Examples()", "Failed to update customization!"); 590 | while (!_updateCustomizationTested) 591 | yield return null; 592 | 593 | // // Get Customization Words 594 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get a customization's words"); 595 | // if (!_textToSpeech.GetCustomizationWords(OnGetCustomizationWords, OnFail, _createdCustomizationId)) 596 | // Log.Debug("ExampleTextToSpeech.GetCustomizationWords()", "Failed to get {0} words!", _createdCustomizationId); 597 | // while (!_getCustomizationWordsTested) 598 | // yield return null; 599 | // 600 | // // Add Customization Words 601 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to add words to a customization"); 602 | // Word[] wordArrayToAddToCustomization = 603 | // { 604 | // new Word() 605 | // { 606 | // word = "bananna", 607 | // translation = "arange" 608 | // }, 609 | // new Word() 610 | // { 611 | // word = "orange", 612 | // translation = "gbye" 613 | // }, 614 | // new Word() 615 | // { 616 | // word = "tomato", 617 | // translation = "tomahto" 618 | // } 619 | // }; 620 | // 621 | // Words wordsToAddToCustomization = new Words() 622 | // { 623 | // words = wordArrayToAddToCustomization 624 | // }; 625 | // 626 | // if (!_textToSpeech.AddCustomizationWords(OnAddCustomizationWords, OnFail, _createdCustomizationId, wordsToAddToCustomization)) 627 | // Log.Debug("ExampleTextToSpeech.AddCustomizationWords()", "Failed to add words to {0}!", _createdCustomizationId); 628 | // while (!_addCustomizationWordsTested) 629 | // yield return null; 630 | // 631 | // // Get Customization Word 632 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get the translation of a custom voice model's word."); 633 | // string customIdentifierWord = wordsToUpdateCustomization[0].word; 634 | // if (!_textToSpeech.GetCustomizationWord(OnGetCustomizationWord, OnFail, _createdCustomizationId, customIdentifierWord)) 635 | // Log.Debug("ExampleTextToSpeech.GetCustomizationWord()", "Failed to get the translation of {0} from {1}!", customIdentifierWord, _createdCustomizationId); 636 | // while (!_getCustomizationWordTested) 637 | // yield return null; 638 | 639 | // Delete Customization Word 640 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to delete customization word from custom voice model."); 641 | string wordToDelete = "goodbye"; 642 | if (!_textToSpeech.DeleteCustomizationWord(OnDeleteCustomizationWord, OnFail, _createdCustomizationId, wordToDelete)) 643 | Log.Debug("ExampleTextToSpeech.DeleteCustomizationWord()", "Failed to delete {0} from {1}!", wordToDelete, _createdCustomizationId); 644 | while (!_deleteCustomizationWordTested) 645 | yield return null; 646 | 647 | // Delete Customization 648 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to delete a customization"); 649 | if (!_textToSpeech.DeleteCustomization(OnDeleteCustomization, OnFail, _createdCustomizationId)) 650 | Log.Debug("ExampleTextToSpeech.DeleteCustomization()", "Failed to delete custom voice model!"); 651 | while (!_deleteCustomizationTested) 652 | yield return null; 653 | 654 | Log.Debug("ExampleTextToSpeech.Examples()", "Text to Speech examples complete."); 655 | } 656 | 657 | void HandleToSpeechCallback(AudioClip clip, Dictionary customData = null) 658 | { 659 | PlayClip(clip); 660 | } 661 | 662 | private void PlayClip(AudioClip clip) 663 | { 664 | if (Application.isPlaying && clip != null) 665 | { 666 | GameObject audioObject = new GameObject("AudioObject"); 667 | AudioSource source = audioObject.AddComponent(); 668 | source.spatialBlend = 0.0f; 669 | source.loop = false; 670 | source.clip = clip; 671 | source.Play(); 672 | 673 | Destroy(audioObject, clip.length); 674 | 675 | _synthesizeTested = true; 676 | } 677 | } 678 | 679 | private void OnGetVoices(Voices voices, Dictionary customData = null) 680 | { 681 | Log.Debug("ExampleTextToSpeech.OnGetVoices()", "Text to Speech - Get voices response: {0}", customData["json"].ToString()); 682 | _getVoicesTested = true; 683 | } 684 | 685 | private void OnGetVoice(Voice voice, Dictionary customData = null) 686 | { 687 | Log.Debug("ExampleTextToSpeech.OnGetVoice()", "Text to Speech - Get voice response: {0}", customData["json"].ToString()); 688 | _getVoiceTested = true; 689 | } 690 | 691 | private void OnGetPronunciation(Pronunciation pronunciation, Dictionary customData = null) 692 | { 693 | Log.Debug("ExampleTextToSpeech.OnGetPronunciation()", "Text to Speech - Get pronunciation response: {0}", customData["json"].ToString()); 694 | _getPronuciationTested = true; 695 | } 696 | 697 | // private void OnGetCustomizations(Customizations customizations, Dictionary customData = null) 698 | // { 699 | // Log.Debug("ExampleTextToSpeech.OnGetCustomizations()", "Text to Speech - Get customizations response: {0}", customData["json"].ToString()); 700 | // _getCustomizationsTested = true; 701 | // } 702 | // 703 | // private void OnCreateCustomization(CustomizationID customizationID, Dictionary customData = null) 704 | // { 705 | // Log.Debug("ExampleTextToSpeech.OnCreateCustomization()", "Text to Speech - Create customization response: {0}", customData["json"].ToString()); 706 | // _createdCustomizationId = customizationID.customization_id; 707 | // _createCustomizationTested = true; 708 | // } 709 | 710 | private void OnDeleteCustomization(bool success, Dictionary customData = null) 711 | { 712 | Log.Debug("ExampleTextToSpeech.OnDeleteCustomization()", "Text to Speech - Delete customization response: {0}", customData["json"].ToString()); 713 | _createdCustomizationId = null; 714 | _deleteCustomizationTested = true; 715 | } 716 | 717 | // private void OnGetCustomization(Customization customization, Dictionary customData = null) 718 | // { 719 | // Log.Debug("ExampleTextToSpeech.OnGetCustomization()", "Text to Speech - Get customization response: {0}", customData["json"].ToString()); 720 | // _getCustomizationTested = true; 721 | // } 722 | 723 | private void OnUpdateCustomization(bool success, Dictionary customData = null) 724 | { 725 | Log.Debug("ExampleTextToSpeech.OnUpdateCustomization()", "Text to Speech - Update customization response: {0}", customData["json"].ToString()); 726 | _updateCustomizationTested = true; 727 | } 728 | 729 | // private void OnGetCustomizationWords(Words words, Dictionary customData = null) 730 | // { 731 | // Log.Debug("ExampleTextToSpeech.OnGetCustomizationWords()", "Text to Speech - Get customization words response: {0}", customData["json"].ToString()); 732 | // _getCustomizationWordsTested = true; 733 | // } 734 | 735 | private void OnAddCustomizationWords(bool success, Dictionary customData = null) 736 | { 737 | Log.Debug("ExampleTextToSpeech.OnAddCustomizationWords()", "Text to Speech - Add customization words response: {0}", customData["json"].ToString()); 738 | _addCustomizationWordsTested = true; 739 | } 740 | 741 | private void OnDeleteCustomizationWord(bool success, Dictionary customData = null) 742 | { 743 | Log.Debug("ExampleTextToSpeech.OnDeleteCustomizationWord()", "Text to Speech - Delete customization word response: {0}", customData["json"].ToString()); 744 | _deleteCustomizationWordTested = true; 745 | } 746 | 747 | private void OnGetCustomizationWord(Translation translation, Dictionary customData = null) 748 | { 749 | Log.Debug("ExampleTextToSpeech.OnGetCustomizationWord()", "Text to Speech - Get customization word response: {0}", customData["json"].ToString()); 750 | _getCustomizationWordTested = true; 751 | } 752 | 753 | private void OnFail(RESTConnector.Error error, Dictionary customData) 754 | { 755 | Log.Error("ExampleTextToSpeech.OnFail()", "Error received: {0}", error.ToString()); 756 | } 757 | } 758 | -------------------------------------------------------------------------------- /ExampleStreaming7.cs: -------------------------------------------------------------------------------- 1 | /** 2 | Hi. this is REALLY Rough code. Beware. vid here; https://youtu.be/JXY_zJetxQM 3 | */ 4 | 5 | using UnityEngine; 6 | using System.Collections; 7 | using IBM.Watson.DeveloperCloud.Logging; 8 | using IBM.Watson.DeveloperCloud.Services.SpeechToText.v1; 9 | using IBM.Watson.DeveloperCloud.Utilities; 10 | using IBM.Watson.DeveloperCloud.DataTypes; 11 | using System.Collections.Generic; 12 | using UnityEngine.UI; 13 | 14 | // For Read Write to Logs and Memory Files 15 | using System; 16 | using System.IO; 17 | 18 | // For Text to speech (not sure if we should declare here or only in ETTS 19 | using IBM.Watson.DeveloperCloud.Services.TextToSpeech.v1; 20 | using IBM.Watson.DeveloperCloud.Connection; 21 | 22 | 23 | 24 | public class ExampleStreaming : MonoBehaviour 25 | { 26 | #region PLEASE SET THESE VARIABLES IN THE INSPECTOR 27 | [SerializeField] 28 | private string _username_STT; 29 | [SerializeField] 30 | private string _password_STT; 31 | [SerializeField] 32 | private string _url_STT; 33 | #endregion 34 | 35 | #region PLEASE SET THESE VARIABLES IN THE INSPECTOR 36 | [SerializeField] 37 | private string _username_TTS; 38 | [SerializeField] 39 | private string _password_TTS; 40 | [SerializeField] 41 | private string _url_TTS; 42 | #endregion 43 | 44 | 45 | 46 | // https://console.bluemix.net/docs/services/text-to-speech/SSML-transformation.html#transformation 47 | 48 | public int voicebox_prosody_pitch; // = 200; // HZ 49 | public int voicebox_pitch_range; // = 0; // -100% to 100% 50 | public int voicebox_glottal_tension; // = 0; // -100% to 100% 51 | public int voicebox_breathiness; // = 0; // -100% to 100% 52 | public int voicebox_rate = 0; // -100% to 100% 53 | //public int voicebox_timbre // "Sunrise" or "Breeze" 54 | public int voicebox_timbre_extent; // = 50; // -0% to 100% 55 | 56 | 57 | 58 | private int _recordingRoutine = 0; 59 | private string _microphoneID = null; 60 | private AudioClip _recording = null; 61 | private int _recordingBufferSize = 1; 62 | private int _recordingHZ = 22050; 63 | 64 | private SpeechToText _speechToText; 65 | private TextToSpeech _textToSpeech; 66 | 67 | // Used for System to update LARGE FONT Text in the UX on operation 68 | public Text ResultsField; // Maps to Text Streaming - from Raw and Original Example file 69 | public Text System_Mode; /// System Mode - (1) Diagnostics (2) Teach_and_Learn (3) Factoid (4) Conversation 70 | public Text Agent_Status; /// Agent Status and Emotional state (confused, apologetic, listening) 71 | public Text Voicebox_Status; // Voicebox defaults 72 | 73 | // To enable TTS calls from this code 74 | string _testString = "Great news!"; 75 | 76 | string _createdCustomizationId; 77 | CustomVoiceUpdate _customVoiceUpdate; 78 | string _customizationName = "unity-example-customization"; 79 | string _customizationLanguage = "en-US"; 80 | string _customizationDescription = "A text to speech voice customization created within Unity."; 81 | string _testWord = "Watson"; 82 | 83 | private bool _synthesizeTested = false; 84 | private bool _getVoicesTested = false; 85 | private bool _getVoiceTested = false; 86 | private bool _getPronuciationTested = false; 87 | private bool _getCustomizationsTested = false; 88 | private bool _createCustomizationTested = false; 89 | private bool _deleteCustomizationTested = false; 90 | private bool _getCustomizationTested = false; 91 | private bool _updateCustomizationTested = false; 92 | private bool _getCustomizationWordsTested = false; 93 | private bool _addCustomizationWordsTested = false; 94 | private bool _deleteCustomizationWordTested = false; 95 | private bool _getCustomizationWordTested = false; 96 | 97 | 98 | 99 | 100 | 101 | 102 | void Start() 103 | { 104 | 105 | /// TTS TEST ON START 106 | LogSystem.InstallDefaultReactors(); 107 | // Create credential and instantiate service 108 | 109 | Credentials credentials_TTS = new Credentials(_username_TTS, _password_TTS, _url_TTS); 110 | _textToSpeech = new TextToSpeech(credentials_TTS); 111 | 112 | _testString = "Hi! I'm MacKenzie! A Unity demo for the IBM Watson Software Development Kit "; 113 | Runnable.Run (Examples ()); 114 | 115 | /// STT BEGIN 116 | Credentials credentials_STT = new Credentials(_username_STT, _password_STT, _url_STT); 117 | _speechToText = new SpeechToText(credentials_STT); 118 | Active = true; 119 | 120 | StartRecording(); 121 | 122 | Agent_Status.text = "Listening"; 123 | System_Mode.text = "mode: listen-learn"; 124 | 125 | voicebox_prosody_pitch = 200; // 150 hZ to 350 hz 126 | voicebox_pitch_range = 0; // -100% to 100% 127 | voicebox_glottal_tension = 25; // -100% to 100% 128 | voicebox_breathiness = 50; // -100% to 100% 129 | voicebox_rate = 0; // -100% to 100% 130 | voicebox_timbre_extent = 50; // -0% to 100% 131 | //voicebox_timbre // "Sunrise" or "Breeze" 132 | 133 | 134 | 135 | //// Check if LOG file exists (later, we dont need to do this all the time, but here for now. OPTIMIZE LATER 136 | string path = @"MacKenzie_Conversation_Log.txt"; 137 | if (!File.Exists (path)) { 138 | // Create a file to write to if the file does not exist 139 | using (StreamWriter sw = File.CreateText (path)) { 140 | sw.WriteLine (string.Concat ("timestamp,","mode,","transcript")); 141 | sw.WriteLine (string.Concat (System.DateTime.Now.ToLongTimeString (), "-", System.DateTime.Now.ToString ("MM/dd/yyyy"), ",log_initialized,", "MacKenzie_Conversation_Log Initialized")); 142 | } 143 | } 144 | 145 | 146 | } 147 | 148 | 149 | /// TEMP CODE 150 | public void TalkieTalkie(){ 151 | 152 | //Log.Debug("red", ""); 153 | //Runnable.Run (Examples ("blue")); 154 | } 155 | 156 | public void TextTalk(string str){ 157 | //Runnable.Run (Examples (_testString2)); 158 | //Runnable.Run (Examples ("blue")); 159 | } 160 | 161 | 162 | 163 | public bool Active 164 | { 165 | get { return _speechToText.IsListening; } 166 | set 167 | { 168 | if (value && !_speechToText.IsListening) 169 | { 170 | _speechToText.DetectSilence = true; 171 | _speechToText.EnableWordConfidence = true; 172 | _speechToText.EnableTimestamps = true; 173 | _speechToText.SilenceThreshold = 0.01f; 174 | _speechToText.MaxAlternatives = 0; 175 | _speechToText.EnableInterimResults = true; 176 | _speechToText.OnError = OnError; 177 | _speechToText.InactivityTimeout = -1; 178 | _speechToText.ProfanityFilter = false; 179 | _speechToText.SmartFormatting = true; 180 | _speechToText.SpeakerLabels = false; 181 | _speechToText.WordAlternativesThreshold = null; 182 | _speechToText.StartListening(OnRecognize, OnRecognizeSpeaker); 183 | } 184 | else if (!value && _speechToText.IsListening) 185 | { 186 | _speechToText.StopListening(); 187 | } 188 | } 189 | } 190 | 191 | private void StartRecording() 192 | { 193 | if (_recordingRoutine == 0) 194 | { 195 | UnityObjectUtil.StartDestroyQueue(); 196 | _recordingRoutine = Runnable.Run(RecordingHandler()); 197 | } 198 | } 199 | 200 | private void StopRecording() 201 | { 202 | if (_recordingRoutine != 0) 203 | { 204 | Microphone.End(_microphoneID); 205 | Runnable.Stop(_recordingRoutine); 206 | _recordingRoutine = 0; 207 | } 208 | } 209 | 210 | private void OnError(string error) 211 | { 212 | Active = false; 213 | 214 | Log.Debug("ExampleStreaming.OnError()", "Error! {0}", error); 215 | } 216 | 217 | private IEnumerator RecordingHandler() 218 | { 219 | Log.Debug("ExampleStreaming.RecordingHandler()", "devices: {0}", Microphone.devices); 220 | _recording = Microphone.Start(_microphoneID, true, _recordingBufferSize, _recordingHZ); 221 | yield return null; // let _recordingRoutine get set.. 222 | 223 | if (_recording == null) 224 | { 225 | StopRecording(); 226 | yield break; 227 | } 228 | 229 | bool bFirstBlock = true; 230 | int midPoint = _recording.samples / 2; 231 | float[] samples = null; 232 | 233 | while (_recordingRoutine != 0 && _recording != null) 234 | { 235 | int writePos = Microphone.GetPosition(_microphoneID); 236 | if (writePos > _recording.samples || !Microphone.IsRecording(_microphoneID)) 237 | { 238 | Log.Error("ExampleStreaming.RecordingHandler()", "Microphone disconnected."); 239 | 240 | StopRecording(); 241 | yield break; 242 | } 243 | 244 | if ((bFirstBlock && writePos >= midPoint) 245 | || (!bFirstBlock && writePos < midPoint)) 246 | { 247 | // front block is recorded, make a RecordClip and pass it onto our callback. 248 | samples = new float[midPoint]; 249 | _recording.GetData(samples, bFirstBlock ? 0 : midPoint); 250 | 251 | AudioData record = new AudioData(); 252 | record.MaxLevel = Mathf.Max(Mathf.Abs(Mathf.Min(samples)), Mathf.Max(samples)); 253 | record.Clip = AudioClip.Create("Recording", midPoint, _recording.channels, _recordingHZ, false); 254 | record.Clip.SetData(samples, 0); 255 | 256 | _speechToText.OnListen(record); 257 | 258 | bFirstBlock = !bFirstBlock; 259 | } 260 | else 261 | { 262 | // calculate the number of samples remaining until we ready for a block of audio, 263 | // and wait that amount of time it will take to record. 264 | int remaining = bFirstBlock ? (midPoint - writePos) : (_recording.samples - writePos); 265 | float timeRemaining = (float)remaining / (float)_recordingHZ; 266 | 267 | yield return new WaitForSeconds(timeRemaining); 268 | } 269 | 270 | } 271 | 272 | yield break; 273 | } 274 | 275 | private void OnRecognize(SpeechRecognitionEvent result, Dictionary customData) 276 | { 277 | if (result != null && result.results.Length > 0) 278 | { 279 | foreach (var res in result.results) 280 | { 281 | foreach (var alt in res.alternatives) { 282 | string text = string.Format ("{0} ({1}, {2:0.00})\n", alt.transcript, res.final ? "Final" : "Interim", alt.confidence); 283 | Log.Debug ("ExampleStreaming.OnRecognize()", text); 284 | ResultsField.text = text; 285 | 286 | 287 | 288 | 289 | ///// HACK LOG INSERT BEGINS 290 | 291 | 292 | if (ResultsField.text.Contains ("Final")) // needs to be final or ECHO happens 293 | { 294 | 295 | if (alt.transcript.Contains ("echo") || alt.transcript.Contains ("repeat") ) 296 | { 297 | _testString = alt.transcript; 298 | _testString = _testString.Replace ("echo", ""); 299 | _testString = _testString.Replace ("repeat", ""); 300 | Runnable.Run (Examples ()); 301 | } 302 | 303 | 304 | else if (alt.transcript.Contains ("connect") && (System_Mode.text == "mode: conversation")) { 305 | /// Count Transcript lines 306 | 307 | _testString = "" + 308 | " No conversation endoints are currently enabled. If you enable them, we can talk more." + // Later - CONNECT to conversation service 309 | ""; 310 | Runnable.Run (Examples ()); 311 | } 312 | 313 | else if (alt.transcript.Contains ("question") && (System_Mode.text == "mode: factoid")) { 314 | /// Count Transcript lines 315 | 316 | _testString = "" + 317 | " No factoid endoints are currently enabled. If you enable them, we can do some Q&A." + // Later - CONNECT to factoid service 318 | ""; 319 | Runnable.Run (Examples ()); 320 | } 321 | 322 | 323 | /// DIAGNOSTIC ZONE - for checking on Agent status 324 | /// 325 | else if (alt.transcript.Contains ("status") && (System_Mode.text == "mode: diagnostic")) { 326 | /// Count Transcript lines 327 | System.IO.StreamReader reader = System.IO.File.OpenText(@"MacKenzie_Conversation_Log.txt"); 328 | int iLineCount = 0; 329 | while (reader.Peek() >= 0) 330 | { reader.ReadLine(); 331 | ++iLineCount; } 332 | reader.Close(); 333 | 334 | _testString = "" + 335 | "I am in " + 336 | System_Mode.text + 337 | " and, " + 338 | "There are "+ iLineCount + 339 | " utterances in my transcript log." + 340 | " This session has been active for " + 341 | Mathf.Round(Time.realtimeSinceStartup) + 342 | " seconds." + 343 | " Emotionally I am feeling Neutral" + // FOR NOW HARDWIRE, later, do a movign window of this https://www.youtube.com/watch?v=fOfFrGsNwHo 344 | ""; 345 | Runnable.Run (Examples ()); 346 | } 347 | 348 | 349 | else if (alt.transcript.Contains ("utterance summary") && (System_Mode.text == "mode: diagnostic")) { 350 | /// Count Transcript lines 351 | System.IO.StreamReader reader = System.IO.File.OpenText(@"MacKenzie_Conversation_Log.txt"); 352 | 353 | // Number of lines in log 354 | // int iLineCount = 0; 355 | // while (reader.Peek() >= 0) 356 | // { reader.ReadLine(); 357 | // ++iLineCount; 358 | // } 359 | // reader.Close(); 360 | 361 | // Number of lines in log (merge with above later) 362 | int iTypeCount_ll = 0; // counter of the thing we care about 363 | int iTypeCount_diag = 0; // counter of the thing we care about 364 | int iTypeCount_conv = 0; // counter of the thing we care about 365 | string reader_string; // place holder for each string from each line 366 | 367 | while ((reader_string = reader.ReadLine()) != null) 368 | { 369 | if (reader_string.Contains("listen-learn")) 370 | { 371 | ++iTypeCount_ll; 372 | } 373 | if (reader_string.Contains("conversation")) 374 | { 375 | ++iTypeCount_conv; 376 | } 377 | if (reader_string.Contains("diagnostic")) 378 | { 379 | ++iTypeCount_diag; 380 | } 381 | } 382 | reader.Close(); 383 | 384 | 385 | _testString = "" + 386 | " For utterance type listen-learn, I counted " + 387 | iTypeCount_ll + 388 | " occurances." + 389 | " There are " + 390 | iTypeCount_conv + 391 | " conversation utterances. There are also " + 392 | iTypeCount_diag + 393 | " dialog utterances" + 394 | ""; 395 | Runnable.Run (Examples ()); 396 | } 397 | 398 | // HARD WIRED FOR NOW 399 | // HARD WIRED FOR NOW - ICE CREAM but later Focus "FOCUS" keyword and can get Sentiment 400 | else if (alt.transcript.Contains ("log query") && alt.transcript.Contains ("ice cream") && (System_Mode.text == "mode: diagnostic")) { 401 | /// Count Transcript lines 402 | System.IO.StreamReader reader = System.IO.File.OpenText(@"MacKenzie_Conversation_Log.txt"); 403 | int iTypeCount_focus = 0; // counter of the thing we care about 404 | string reader_string; // place holder for each string from each line 405 | 406 | while ((reader_string = reader.ReadLine()) != null) 407 | { 408 | if (reader_string.Contains("ice cream")) 409 | { 410 | ++iTypeCount_focus; 411 | } 412 | // later do Sentiment by Keyword or Entity in FOCUS 413 | } 414 | reader.Close(); 415 | 416 | 417 | _testString = "" + 418 | " For FOCUS keyword Ice Cream, I counted " + 419 | iTypeCount_focus + 420 | " occurances. And the sentiment was very positive" + // LATER - call Sentiment by Keyword/ENtitty (hardwired now) 421 | ""; 422 | Runnable.Run (Examples ()); 423 | } 424 | 425 | 426 | 427 | else if (alt.transcript.Contains ("emotional state") && (System_Mode.text == "mode: diagnostic")) { 428 | /// Count Transcript lines 429 | 430 | _testString = "" + 431 | " Emotionally I am feeling Neutral." + // Later - CONNECT to emotional levels (volumetric representation of state) moving window 432 | ""; 433 | Runnable.Run (Examples ()); 434 | } 435 | 436 | 437 | 438 | // https://console.bluemix.net/docs/services/text-to-speech/SSML-elements.html#elements 439 | // https://console.bluemix.net/docs/services/text-to-speech/SSML-expressive.html#expressive 440 | 441 | else if (alt.transcript.Contains ("voicebox") || alt.transcript.Contains ("voice box")) 442 | { 443 | _testString = "Voice traits have been modified."; 444 | 445 | if ( alt.transcript.Contains ("increase") && (alt.transcript.Contains ("pitch") || alt.transcript.Contains ("frequency") )) 446 | { 447 | voicebox_prosody_pitch = voicebox_prosody_pitch + 50; // 150 hZ to 350 hz 448 | voicebox_pitch_range = voicebox_pitch_range + 25; // -100% to 100% 449 | 450 | } 451 | else if (alt.transcript.Contains ("increase") && alt.transcript.Contains ("rate") ) 452 | { 453 | voicebox_rate = voicebox_rate + 25; // -100% to 100% 454 | } 455 | else if (alt.transcript.Contains ("increase") && alt.transcript.Contains ("tension") ) 456 | { 457 | voicebox_glottal_tension = voicebox_glottal_tension + 25; // -100% to 100% 458 | voicebox_breathiness = voicebox_breathiness + 25; // -100% to 100% 459 | voicebox_timbre_extent = voicebox_timbre_extent + 10; // -0% to 100% 460 | } 461 | else if (alt.transcript.Contains ("decrease") && (alt.transcript.Contains ("pitch") || alt.transcript.Contains ("frequency") )) 462 | { 463 | voicebox_prosody_pitch = voicebox_prosody_pitch - 50; // 150 hZ to 350 hz 464 | voicebox_pitch_range = voicebox_pitch_range - 25; // -100% to 100% 465 | 466 | } 467 | else if (alt.transcript.Contains ("decrease") && alt.transcript.Contains ("rate") ) 468 | { 469 | voicebox_rate = voicebox_rate - 25; // -100% to 100% 470 | } 471 | else if (alt.transcript.Contains ("decrease") && alt.transcript.Contains ("tension") ) 472 | { 473 | voicebox_glottal_tension = voicebox_glottal_tension - 25; // -100% to 100% 474 | voicebox_breathiness = voicebox_breathiness - 25; // -100% to 100% 475 | voicebox_timbre_extent = voicebox_timbre_extent - 10; // -0% to 100% 476 | } 477 | else if (alt.transcript.Contains ("reset") ) 478 | { 479 | voicebox_prosody_pitch = 250; // HZ 480 | voicebox_pitch_range = 0; // -100% to 100% 481 | voicebox_glottal_tension = 0; // -100% to 100% 482 | voicebox_breathiness = 0; // -100% to 100% 483 | voicebox_rate = 0; // -100% to 100% 484 | voicebox_timbre_extent = 50; // -0% to 100% 485 | } 486 | 487 | else if (alt.transcript.Contains ("settings") ) 488 | { 489 | _testString = 490 | " Voicebox settings. Pitch is " + 491 | voicebox_prosody_pitch + 492 | " hertz. The speech rate is " + 493 | voicebox_rate + 494 | ". And the glottal tension is " + 495 | voicebox_glottal_tension; 496 | } 497 | 498 | else 499 | { 500 | _testString = "My voice is my passport"; // if no changes 501 | } 502 | 503 | 504 | 505 | Voicebox_Status.text = 506 | "voicebox:" + 507 | "\n pitch" + 508 | voicebox_prosody_pitch + 509 | "hz \n pitch_range" + 510 | voicebox_pitch_range + 511 | "\n glottal_tension" + 512 | voicebox_glottal_tension + 513 | "\n breathiness" + 514 | voicebox_breathiness + 515 | "\n rate" + 516 | voicebox_rate + 517 | "\n timbre_extent" + 518 | voicebox_timbre_extent; 519 | 520 | 521 | /// WRAP TEST STRING IN THE SSML/TTS FIXINS THAT MAKE IT SPECIAL 522 | _testString = "" + 523 | "" + 526 | " " + 536 | _testString + 537 | "" + 538 | "" + 539 | ""; 540 | /// AND FIRE TTS THAT USES PUBLIC _testString 541 | Runnable.Run (Examples ()); 542 | 543 | 544 | 545 | } // end of voicebox 546 | 547 | 548 | 549 | 550 | 551 | // check if user is changing SYSTEM MODE 552 | else if ((alt.transcript.Contains ("switch") || alt.transcript.Contains ("change")) && alt.transcript.Contains ("mode")) 553 | { 554 | if (ResultsField.text.Contains ("diagnostic")) { 555 | System_Mode.text = "mode: diagnostic"; 556 | _testString = "Diagnostic Mode Engaged"; 557 | Runnable.Run (Examples ()); 558 | } 559 | else if (ResultsField.text.Contains ("listen") || ResultsField.text.Contains ("learn")) { 560 | _testString = "Listen Learn Mode Engaged"; 561 | Runnable.Run (Examples ()); 562 | System_Mode.text = "mode: listen-learn"; 563 | } 564 | else if (ResultsField.text.Contains ("conversation")) { 565 | System_Mode.text = "mode: conversation"; 566 | _testString = "Conversation Mode Engaged!"; 567 | Runnable.Run (Examples ()); 568 | 569 | } 570 | else if (ResultsField.text.Contains ("factoid")) { 571 | _testString = "Factoid Mode Engaged"; 572 | Runnable.Run (Examples ()); 573 | System_Mode.text = "mode: factoid"; 574 | } 575 | } 576 | 577 | 578 | 579 | //string _testString = "This is Text to Speech!"; 580 | //string _testString = "hi"; 581 | // Good news and sorrow and uncertainty - ref https://console.bluemix.net/docs/services/text-to-speech/SSML-expressive.html#expressive 582 | // This is Text to Speech! 583 | //string _testString = "I am terribly sorry for the quality of service you have received."; 584 | //string _testString = "Can you please explain it again? I am not sure I understand."; 585 | //string _testString = "Can you please explain it again? I am confused and I'm not sure I understand."; 586 | 587 | 588 | //// WRITE TO LOG FILE - time-date and MODE and transcript (final) - we're also logging her responses right now. may mute later (or tag) 589 | string path = @"MacKenzie_Conversation_Log.txt"; 590 | using (StreamWriter sw = File.AppendText (path)) { 591 | sw.WriteLine (string.Concat (System.DateTime.Now.ToLongTimeString (), "-", System.DateTime.Now.ToString ("MM/dd/yyyy"), ",",System_Mode.text,",", alt.transcript)); 592 | } 593 | 594 | } 595 | ////// HACK LOG INSERT ENDS 596 | 597 | 598 | } 599 | 600 | if (res.keywords_result != null && res.keywords_result.keyword != null) 601 | { 602 | foreach (var keyword in res.keywords_result.keyword) 603 | { 604 | Log.Debug("ExampleStreaming.OnRecognize()", "keyword: {0}, confidence: {1}, start time: {2}, end time: {3}", keyword.normalized_text, keyword.confidence, keyword.start_time, keyword.end_time); 605 | } 606 | } 607 | 608 | if (res.word_alternatives != null) 609 | { 610 | foreach (var wordAlternative in res.word_alternatives) 611 | { 612 | Log.Debug("ExampleStreaming.OnRecognize()", "Word alternatives found. Start time: {0} | EndTime: {1}", wordAlternative.start_time, wordAlternative.end_time); 613 | foreach(var alternative in wordAlternative.alternatives) 614 | Log.Debug("ExampleStreaming.OnRecognize()", "\t word: {0} | confidence: {1}", alternative.word, alternative.confidence); 615 | } 616 | } 617 | } 618 | } 619 | } 620 | 621 | private void OnRecognizeSpeaker(SpeakerRecognitionEvent result, Dictionary customData) 622 | { 623 | if (result != null) 624 | { 625 | foreach (SpeakerLabelsResult labelResult in result.speaker_labels) 626 | { 627 | Log.Debug("ExampleStreaming.OnRecognize()", string.Format("speaker result: {0} | confidence: {3} | from: {1} | to: {2}", labelResult.speaker, labelResult.from, labelResult.to, labelResult.confidence)); 628 | } 629 | } 630 | } 631 | 632 | 633 | 634 | 635 | 636 | 637 | 638 | 639 | public IEnumerator Examples() ///// RYAN CHECK - changed to Public / from private and may need to declare call examplestts.exampels from other code 640 | { 641 | //_testString = str; 642 | 643 | // Synthesize 644 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting synthesize."); 645 | _textToSpeech.Voice = VoiceType.en_US_Allison; 646 | _textToSpeech.ToSpeech(HandleToSpeechCallback, OnFail, _testString, true); 647 | while (!_synthesizeTested) 648 | yield return null; 649 | 650 | // Get Voices 651 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get voices."); 652 | _textToSpeech.GetVoices(OnGetVoices, OnFail); 653 | while (!_getVoicesTested) 654 | yield return null; 655 | 656 | // Get Voice 657 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get voice {0}.", VoiceType.en_US_Allison); 658 | _textToSpeech.GetVoice(OnGetVoice, OnFail, VoiceType.en_US_Allison); 659 | while (!_getVoiceTested) 660 | yield return null; 661 | 662 | // Get Pronunciation 663 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get pronunciation of {0}", _testWord); 664 | _textToSpeech.GetPronunciation(OnGetPronunciation, OnFail, _testWord, VoiceType.en_US_Allison); 665 | while (!_getPronuciationTested) 666 | yield return null; 667 | // 668 | // // Get Customizations 669 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get a list of customizations"); 670 | // _textToSpeech.GetCustomizations(OnGetCustomizations, OnFail); 671 | // while (!_getCustomizationsTested) 672 | // yield return null; 673 | // 674 | // // Create Customization 675 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to create a customization"); 676 | // _textToSpeech.CreateCustomization(OnCreateCustomization, OnFail, _customizationName, _customizationLanguage, _customizationDescription); 677 | // while (!_createCustomizationTested) 678 | // yield return null; 679 | // 680 | // // Get Customization 681 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get a customization"); 682 | // if (!_textToSpeech.GetCustomization(OnGetCustomization, OnFail, _createdCustomizationId)) 683 | // Log.Debug("ExampleTextToSpeech.Examples()", "Failed to get custom voice model!"); 684 | // while (!_getCustomizationTested) 685 | // yield return null; 686 | 687 | // // Update Customization 688 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to update a customization"); 689 | // Word[] wordsToUpdateCustomization = 690 | // { 691 | // new Word() 692 | // { 693 | // word = "hello", 694 | // translation = "hullo" 695 | // }, 696 | // new Word() 697 | // { 698 | // word = "goodbye", 699 | // translation = "gbye" 700 | // }, 701 | // new Word() 702 | // { 703 | // word = "hi", 704 | // translation = "ohioooo" 705 | // } 706 | // }; 707 | // 708 | // _customVoiceUpdate = new CustomVoiceUpdate() 709 | // { 710 | // words = wordsToUpdateCustomization, 711 | // description = "My updated description", 712 | // name = "My updated name" 713 | // }; 714 | 715 | if (!_textToSpeech.UpdateCustomization(OnUpdateCustomization, OnFail, _createdCustomizationId, _customVoiceUpdate)) 716 | Log.Debug("ExampleTextToSpeech.Examples()", "Failed to update customization!"); 717 | while (!_updateCustomizationTested) 718 | yield return null; 719 | 720 | // // Get Customization Words 721 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get a customization's words"); 722 | // if (!_textToSpeech.GetCustomizationWords(OnGetCustomizationWords, OnFail, _createdCustomizationId)) 723 | // Log.Debug("ExampleTextToSpeech.GetCustomizationWords()", "Failed to get {0} words!", _createdCustomizationId); 724 | // while (!_getCustomizationWordsTested) 725 | // yield return null; 726 | // 727 | // // Add Customization Words 728 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to add words to a customization"); 729 | // Word[] wordArrayToAddToCustomization = 730 | // { 731 | // new Word() 732 | // { 733 | // word = "bananna", 734 | // translation = "arange" 735 | // }, 736 | // new Word() 737 | // { 738 | // word = "orange", 739 | // translation = "gbye" 740 | // }, 741 | // new Word() 742 | // { 743 | // word = "tomato", 744 | // translation = "tomahto" 745 | // } 746 | // }; 747 | // 748 | // Words wordsToAddToCustomization = new Words() 749 | // { 750 | // words = wordArrayToAddToCustomization 751 | // }; 752 | // 753 | // if (!_textToSpeech.AddCustomizationWords(OnAddCustomizationWords, OnFail, _createdCustomizationId, wordsToAddToCustomization)) 754 | // Log.Debug("ExampleTextToSpeech.AddCustomizationWords()", "Failed to add words to {0}!", _createdCustomizationId); 755 | // while (!_addCustomizationWordsTested) 756 | // yield return null; 757 | // 758 | // // Get Customization Word 759 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get the translation of a custom voice model's word."); 760 | // string customIdentifierWord = wordsToUpdateCustomization[0].word; 761 | // if (!_textToSpeech.GetCustomizationWord(OnGetCustomizationWord, OnFail, _createdCustomizationId, customIdentifierWord)) 762 | // Log.Debug("ExampleTextToSpeech.GetCustomizationWord()", "Failed to get the translation of {0} from {1}!", customIdentifierWord, _createdCustomizationId); 763 | // while (!_getCustomizationWordTested) 764 | // yield return null; 765 | 766 | // Delete Customization Word 767 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to delete customization word from custom voice model."); 768 | string wordToDelete = "goodbye"; 769 | if (!_textToSpeech.DeleteCustomizationWord(OnDeleteCustomizationWord, OnFail, _createdCustomizationId, wordToDelete)) 770 | Log.Debug("ExampleTextToSpeech.DeleteCustomizationWord()", "Failed to delete {0} from {1}!", wordToDelete, _createdCustomizationId); 771 | while (!_deleteCustomizationWordTested) 772 | yield return null; 773 | 774 | // Delete Customization 775 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to delete a customization"); 776 | if (!_textToSpeech.DeleteCustomization(OnDeleteCustomization, OnFail, _createdCustomizationId)) 777 | Log.Debug("ExampleTextToSpeech.DeleteCustomization()", "Failed to delete custom voice model!"); 778 | while (!_deleteCustomizationTested) 779 | yield return null; 780 | 781 | Log.Debug("ExampleTextToSpeech.Examples()", "Text to Speech examples complete."); 782 | } 783 | 784 | void HandleToSpeechCallback(AudioClip clip, Dictionary customData = null) 785 | { 786 | PlayClip(clip); 787 | } 788 | 789 | private void PlayClip(AudioClip clip) 790 | { 791 | if (Application.isPlaying && clip != null) 792 | { 793 | GameObject audioObject = new GameObject("AudioObject"); 794 | AudioSource source = audioObject.AddComponent(); 795 | source.spatialBlend = 0.0f; 796 | source.loop = false; 797 | source.clip = clip; 798 | source.Play(); 799 | 800 | Destroy(audioObject, clip.length); 801 | 802 | _synthesizeTested = true; 803 | } 804 | } 805 | 806 | private void OnGetVoices(Voices voices, Dictionary customData = null) 807 | { 808 | Log.Debug("ExampleTextToSpeech.OnGetVoices()", "Text to Speech - Get voices response: {0}", customData["json"].ToString()); 809 | _getVoicesTested = true; 810 | } 811 | 812 | private void OnGetVoice(Voice voice, Dictionary customData = null) 813 | { 814 | Log.Debug("ExampleTextToSpeech.OnGetVoice()", "Text to Speech - Get voice response: {0}", customData["json"].ToString()); 815 | _getVoiceTested = true; 816 | } 817 | 818 | private void OnGetPronunciation(Pronunciation pronunciation, Dictionary customData = null) 819 | { 820 | Log.Debug("ExampleTextToSpeech.OnGetPronunciation()", "Text to Speech - Get pronunciation response: {0}", customData["json"].ToString()); 821 | _getPronuciationTested = true; 822 | } 823 | 824 | // private void OnGetCustomizations(Customizations customizations, Dictionary customData = null) 825 | // { 826 | // Log.Debug("ExampleTextToSpeech.OnGetCustomizations()", "Text to Speech - Get customizations response: {0}", customData["json"].ToString()); 827 | // _getCustomizationsTested = true; 828 | // } 829 | // 830 | // private void OnCreateCustomization(CustomizationID customizationID, Dictionary customData = null) 831 | // { 832 | // Log.Debug("ExampleTextToSpeech.OnCreateCustomization()", "Text to Speech - Create customization response: {0}", customData["json"].ToString()); 833 | // _createdCustomizationId = customizationID.customization_id; 834 | // _createCustomizationTested = true; 835 | // } 836 | 837 | private void OnDeleteCustomization(bool success, Dictionary customData = null) 838 | { 839 | Log.Debug("ExampleTextToSpeech.OnDeleteCustomization()", "Text to Speech - Delete customization response: {0}", customData["json"].ToString()); 840 | _createdCustomizationId = null; 841 | _deleteCustomizationTested = true; 842 | } 843 | 844 | // private void OnGetCustomization(Customization customization, Dictionary customData = null) 845 | // { 846 | // Log.Debug("ExampleTextToSpeech.OnGetCustomization()", "Text to Speech - Get customization response: {0}", customData["json"].ToString()); 847 | // _getCustomizationTested = true; 848 | // } 849 | 850 | private void OnUpdateCustomization(bool success, Dictionary customData = null) 851 | { 852 | Log.Debug("ExampleTextToSpeech.OnUpdateCustomization()", "Text to Speech - Update customization response: {0}", customData["json"].ToString()); 853 | _updateCustomizationTested = true; 854 | } 855 | 856 | // private void OnGetCustomizationWords(Words words, Dictionary customData = null) 857 | // { 858 | // Log.Debug("ExampleTextToSpeech.OnGetCustomizationWords()", "Text to Speech - Get customization words response: {0}", customData["json"].ToString()); 859 | // _getCustomizationWordsTested = true; 860 | // } 861 | 862 | private void OnAddCustomizationWords(bool success, Dictionary customData = null) 863 | { 864 | Log.Debug("ExampleTextToSpeech.OnAddCustomizationWords()", "Text to Speech - Add customization words response: {0}", customData["json"].ToString()); 865 | _addCustomizationWordsTested = true; 866 | } 867 | 868 | private void OnDeleteCustomizationWord(bool success, Dictionary customData = null) 869 | { 870 | Log.Debug("ExampleTextToSpeech.OnDeleteCustomizationWord()", "Text to Speech - Delete customization word response: {0}", customData["json"].ToString()); 871 | _deleteCustomizationWordTested = true; 872 | } 873 | 874 | private void OnGetCustomizationWord(Translation translation, Dictionary customData = null) 875 | { 876 | Log.Debug("ExampleTextToSpeech.OnGetCustomizationWord()", "Text to Speech - Get customization word response: {0}", customData["json"].ToString()); 877 | _getCustomizationWordTested = true; 878 | } 879 | 880 | private void OnFail(RESTConnector.Error error, Dictionary customData) 881 | { 882 | Log.Error("ExampleTextToSpeech.OnFail()", "Error received: {0}", error.ToString()); 883 | } 884 | } 885 | -------------------------------------------------------------------------------- /ExampleStreaming_plus_Text_to_Speech_expressive.cs: -------------------------------------------------------------------------------- 1 | /** 2 | 3 | **/ 4 | 5 | using UnityEngine; 6 | using System.Collections; 7 | using IBM.Watson.DeveloperCloud.Logging; 8 | using IBM.Watson.DeveloperCloud.Services.SpeechToText.v1; 9 | using IBM.Watson.DeveloperCloud.Utilities; 10 | using IBM.Watson.DeveloperCloud.DataTypes; 11 | using System.Collections; 12 | using System.Collections.Generic; 13 | using UnityEngine.UI; 14 | 15 | using IBM.Watson.DeveloperCloud.Services.TextToSpeech.v1; 16 | using IBM.Watson.DeveloperCloud.Connection; 17 | 18 | public class ExampleStreaming : MonoBehaviour 19 | { 20 | 21 | // STT - BURNER CREDS - DELETE AFTER RECORDING 22 | private string _username_STT = "555555555555555555"; 23 | private string _password_STT = "555555555"; 24 | private string _url_STT = "https://stream.watsonplatform.net/speech-to-text/api"; 25 | 26 | 27 | public Text ResultsField; 28 | 29 | private int _recordingRoutine = 0; 30 | private string _microphoneID = null; 31 | private AudioClip _recording = null; 32 | private int _recordingBufferSize = 1; 33 | private int _recordingHZ = 22050; 34 | 35 | private SpeechToText _speechToText; 36 | 37 | // TEXT TO SPEECH - BURNER CREDENTIALS FOR PUBLIC DEMO I WILL DELETE AFTER RECORDING 38 | private string _username_TTS = "5555555555555555555555555"; 39 | private string _password_TTS = "5555555555"; 40 | private string _url_TTS = "https://stream.watsonplatform.net/text-to-speech/api"; 41 | 42 | TextToSpeech _textToSpeech; 43 | 44 | //string _testString = "I'm sorry. This is Text to Speech!I'm sorry. This is Text to Speech!"; 45 | 46 | /// TEST STRINGS OK 47 | 48 | // Pitch Shifting 49 | //string _testString = "This is Text to Speech!"; 50 | //string _testString = "This is Text to Speech!"; 51 | //string _testString = "This is Text to Speech!"; 52 | //string _testString = "hi"; 53 | 54 | // Good news and sorrow and uncertainty - ref https://console.bluemix.net/docs/services/text-to-speech/SSML-expressive.html#expressive 55 | // This is Text to Speech! 56 | string _testString = "Hello! Good News! Text to Speech is Working!"; 57 | //string _testString = "I am terribly sorry for the quality of service you have received."; 58 | //string _testString = "Can you please explain it again? I am not sure I understand."; 59 | 60 | //string _testString = "Can you please explain it again? I am confused and I'm not sure I understand."; 61 | 62 | 63 | string _createdCustomizationId; 64 | CustomVoiceUpdate _customVoiceUpdate; 65 | string _customizationName = "unity-example-customization"; 66 | string _customizationLanguage = "en-US"; 67 | string _customizationDescription = "A text to speech voice customization created within Unity."; 68 | string _testWord = "Watson"; 69 | 70 | private bool _synthesizeTested = false; 71 | private bool _getVoicesTested = false; 72 | private bool _getVoiceTested = false; 73 | private bool _getPronuciationTested = false; 74 | private bool _getCustomizationsTested = false; 75 | private bool _createCustomizationTested = false; 76 | private bool _deleteCustomizationTested = false; 77 | private bool _getCustomizationTested = false; 78 | private bool _updateCustomizationTested = false; 79 | private bool _getCustomizationWordsTested = false; 80 | private bool _addCustomizationWordsTested = false; 81 | private bool _deleteCustomizationWordTested = false; 82 | private bool _getCustomizationWordTested = false; 83 | 84 | 85 | 86 | void Start() 87 | { 88 | LogSystem.InstallDefaultReactors(); 89 | 90 | // Create credential and instantiate service 91 | Credentials credentials_STT = new Credentials(_username_STT, _password_STT, _url_STT); 92 | Credentials credentials_TTS = new Credentials(_username_TTS, _password_TTS, _url_TTS); 93 | 94 | _speechToText = new SpeechToText(credentials_STT); 95 | _textToSpeech = new TextToSpeech(credentials_TTS); 96 | 97 | Active = true; 98 | StartRecording(); 99 | 100 | Runnable.Run(Examples()); 101 | 102 | } 103 | 104 | public bool Active 105 | { 106 | get { return _speechToText.IsListening; } 107 | set 108 | { 109 | if (value && !_speechToText.IsListening) 110 | { 111 | _speechToText.DetectSilence = true; 112 | _speechToText.EnableWordConfidence = true; 113 | _speechToText.EnableTimestamps = true; 114 | _speechToText.SilenceThreshold = 0.01f; 115 | _speechToText.MaxAlternatives = 0; 116 | _speechToText.EnableInterimResults = true; 117 | _speechToText.OnError = OnError; 118 | _speechToText.InactivityTimeout = -1; 119 | _speechToText.ProfanityFilter = false; 120 | _speechToText.SmartFormatting = true; 121 | _speechToText.SpeakerLabels = false; 122 | _speechToText.WordAlternativesThreshold = null; 123 | _speechToText.StartListening(OnRecognize, OnRecognizeSpeaker); 124 | } 125 | else if (!value && _speechToText.IsListening) 126 | { 127 | _speechToText.StopListening(); 128 | } 129 | } 130 | } 131 | 132 | private void StartRecording() 133 | { 134 | if (_recordingRoutine == 0) 135 | { 136 | UnityObjectUtil.StartDestroyQueue(); 137 | _recordingRoutine = Runnable.Run(RecordingHandler()); 138 | } 139 | } 140 | 141 | private void StopRecording() 142 | { 143 | if (_recordingRoutine != 0) 144 | { 145 | Microphone.End(_microphoneID); 146 | Runnable.Stop(_recordingRoutine); 147 | _recordingRoutine = 0; 148 | } 149 | } 150 | 151 | private void OnError(string error) 152 | { 153 | Active = false; 154 | 155 | Log.Debug("ExampleStreaming.OnError()", "Error! {0}", error); 156 | } 157 | 158 | private IEnumerator RecordingHandler() 159 | { 160 | Log.Debug("ExampleStreaming.RecordingHandler()", "devices: {0}", Microphone.devices); 161 | _recording = Microphone.Start(_microphoneID, true, _recordingBufferSize, _recordingHZ); 162 | yield return null; // let _recordingRoutine get set.. 163 | 164 | if (_recording == null) 165 | { 166 | StopRecording(); 167 | yield break; 168 | } 169 | 170 | bool bFirstBlock = true; 171 | int midPoint = _recording.samples / 2; 172 | float[] samples = null; 173 | 174 | while (_recordingRoutine != 0 && _recording != null) 175 | { 176 | int writePos = Microphone.GetPosition(_microphoneID); 177 | if (writePos > _recording.samples || !Microphone.IsRecording(_microphoneID)) 178 | { 179 | Log.Error("ExampleStreaming.RecordingHandler()", "Microphone disconnected."); 180 | 181 | StopRecording(); 182 | yield break; 183 | } 184 | 185 | if ((bFirstBlock && writePos >= midPoint) 186 | || (!bFirstBlock && writePos < midPoint)) 187 | { 188 | // front block is recorded, make a RecordClip and pass it onto our callback. 189 | samples = new float[midPoint]; 190 | _recording.GetData(samples, bFirstBlock ? 0 : midPoint); 191 | 192 | AudioData record = new AudioData(); 193 | record.MaxLevel = Mathf.Max(Mathf.Abs(Mathf.Min(samples)), Mathf.Max(samples)); 194 | record.Clip = AudioClip.Create("Recording", midPoint, _recording.channels, _recordingHZ, false); 195 | record.Clip.SetData(samples, 0); 196 | 197 | _speechToText.OnListen(record); 198 | 199 | bFirstBlock = !bFirstBlock; 200 | } 201 | else 202 | { 203 | // calculate the number of samples remaining until we ready for a block of audio, 204 | // and wait that amount of time it will take to record. 205 | int remaining = bFirstBlock ? (midPoint - writePos) : (_recording.samples - writePos); 206 | float timeRemaining = (float)remaining / (float)_recordingHZ; 207 | 208 | yield return new WaitForSeconds(timeRemaining); 209 | } 210 | 211 | } 212 | 213 | yield break; 214 | } 215 | 216 | private void OnRecognize(SpeechRecognitionEvent result) 217 | { 218 | if (result != null && result.results.Length > 0) 219 | { 220 | foreach (var res in result.results) 221 | { 222 | foreach (var alt in res.alternatives) 223 | { 224 | string text = string.Format("{0} ({1}, {2:0.00})\n", alt.transcript, res.final ? "Final" : "Interim", alt.confidence); 225 | Log.Debug("ExampleStreaming.OnRecognize()", text); 226 | ResultsField.text = text; 227 | 228 | if (alt.transcript.Contains("blue") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 229 | { 230 | _testString = "I love the color of the sky too!"; 231 | Runnable.Run(Examples()); 232 | 233 | } 234 | if (alt.transcript.Contains("yellow") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 235 | { 236 | _testString = "Oh Yes! The color of daisies and bananas!"; 237 | Runnable.Run(Examples()); 238 | } // Cannot ECHO the trigger condition (or be ready for loop 239 | 240 | if (alt.transcript.Contains("happy") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 241 | { 242 | _testString = "That's so nice! You bring me great joy as well!"; 243 | Runnable.Run(Examples()); 244 | } // Cannot ECHO the trigger condition (or be ready for loop 245 | 246 | if (alt.transcript.Contains("mistake") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 247 | { 248 | _testString = "I am so sorry. I'll try to do better next time."; 249 | Runnable.Run(Examples()); 250 | } // Cannot ECHO the trigger condition (or be ready for loop 251 | 252 | if (alt.transcript.Contains("quantum") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 253 | { 254 | _testString = "I dont really know how to answer that."; 255 | Runnable.Run(Examples()); 256 | } // Cannot ECHO the trigger condition (or be ready for loop 257 | 258 | if (alt.transcript.Contains("goodbye") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 259 | { 260 | _testString = "Bye bye! And thank you!"; 261 | Runnable.Run(Examples()); 262 | } // Cannot ECHO the trigger condition (or be ready for loop 263 | 264 | 265 | 266 | } 267 | 268 | if (res.keywords_result != null && res.keywords_result.keyword != null) 269 | { 270 | foreach (var keyword in res.keywords_result.keyword) 271 | { 272 | Log.Debug("ExampleStreaming.OnRecognize()", "keyword: {0}, confidence: {1}, start time: {2}, end time: {3}", keyword.normalized_text, keyword.confidence, keyword.start_time, keyword.end_time); 273 | } 274 | } 275 | 276 | if (res.word_alternatives != null) 277 | { 278 | foreach (var wordAlternative in res.word_alternatives) 279 | { 280 | Log.Debug("ExampleStreaming.OnRecognize()", "Word alternatives found. Start time: {0} | EndTime: {1}", wordAlternative.start_time, wordAlternative.end_time); 281 | foreach(var alternative in wordAlternative.alternatives) 282 | Log.Debug("ExampleStreaming.OnRecognize()", "\t word: {0} | confidence: {1}", alternative.word, alternative.confidence); 283 | } 284 | } 285 | } 286 | } 287 | } 288 | 289 | private void OnRecognizeSpeaker(SpeakerRecognitionEvent result) 290 | { 291 | if (result != null) 292 | { 293 | foreach (SpeakerLabelsResult labelResult in result.speaker_labels) 294 | { 295 | Log.Debug("ExampleStreaming.OnRecognize()", string.Format("speaker result: {0} | confidence: {3} | from: {1} | to: {2}", labelResult.speaker, labelResult.from, labelResult.to, labelResult.confidence)); 296 | } 297 | } 298 | } 299 | 300 | 301 | // TTS CODE 302 | private IEnumerator Examples() 303 | { 304 | // Synthesize 305 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting synthesize."); 306 | _textToSpeech.Voice = VoiceType.en_US_Allison; 307 | _textToSpeech.ToSpeech(HandleToSpeechCallback, OnFail, _testString, true); 308 | while (!_synthesizeTested) 309 | yield return null; 310 | 311 | // Get Voices 312 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get voices."); 313 | _textToSpeech.GetVoices(OnGetVoices, OnFail); 314 | while (!_getVoicesTested) 315 | yield return null; 316 | 317 | // Get Voice 318 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get voice {0}.", VoiceType.en_US_Allison); 319 | _textToSpeech.GetVoice(OnGetVoice, OnFail, VoiceType.en_US_Allison); 320 | while (!_getVoiceTested) 321 | yield return null; 322 | 323 | // Get Pronunciation 324 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get pronunciation of {0}", _testWord); 325 | _textToSpeech.GetPronunciation(OnGetPronunciation, OnFail, _testWord, VoiceType.en_US_Allison); 326 | while (!_getPronuciationTested) 327 | yield return null; 328 | 329 | // Get Customizations 330 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get a list of customizations"); 331 | // _textToSpeech.GetCustomizations(OnGetCustomizations, OnFail); 332 | // while (!_getCustomizationsTested) 333 | // yield return null; 334 | 335 | // Create Customization 336 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to create a customization"); 337 | // _textToSpeech.CreateCustomization(OnCreateCustomization, OnFail, _customizationName, _customizationLanguage, _customizationDescription); 338 | // while (!_createCustomizationTested) 339 | // yield return null; 340 | 341 | // Get Customization 342 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get a customization"); 343 | // if (!_textToSpeech.GetCustomization(OnGetCustomization, OnFail, _createdCustomizationId)) 344 | // Log.Debug("ExampleTextToSpeech.Examples()", "Failed to get custom voice model!"); 345 | // while (!_getCustomizationTested) 346 | // yield return null; 347 | 348 | // Update Customization 349 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to update a customization"); 350 | // Word[] wordsToUpdateCustomization = 351 | // { 352 | // new Word() 353 | // { 354 | // word = "hello", 355 | // translation = "hullo" 356 | // }, 357 | // new Word() 358 | // { 359 | // word = "goodbye", 360 | // translation = "gbye" 361 | // }, 362 | // new Word() 363 | // { 364 | // word = "hi", 365 | // translation = "ohioooo" 366 | // } 367 | // }; 368 | 369 | // _customVoiceUpdate = new CustomVoiceUpdate() 370 | // { 371 | // words = wordsToUpdateCustomization, 372 | // description = "My updated description", 373 | // name = "My updated name" 374 | // }; 375 | 376 | if (!_textToSpeech.UpdateCustomization(OnUpdateCustomization, OnFail, _createdCustomizationId, _customVoiceUpdate)) 377 | Log.Debug("ExampleTextToSpeech.Examples()", "Failed to update customization!"); 378 | while (!_updateCustomizationTested) 379 | yield return null; 380 | 381 | // Get Customization Words 382 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get a customization's words"); 383 | // if (!_textToSpeech.GetCustomizationWords(OnGetCustomizationWords, OnFail, _createdCustomizationId)) 384 | // Log.Debug("ExampleTextToSpeech.GetCustomizationWords()", "Failed to get {0} words!", _createdCustomizationId); 385 | // while (!_getCustomizationWordsTested) 386 | // yield return null; 387 | 388 | // Add Customization Words 389 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to add words to a customization"); 390 | // Word[] wordArrayToAddToCustomization = 391 | // { 392 | // new Word() 393 | // { 394 | // word = "bananna", 395 | // translation = "arange" 396 | // }, 397 | // new Word() 398 | // { 399 | // word = "orange", 400 | // translation = "gbye" 401 | // }, 402 | // new Word() 403 | // { 404 | // word = "tomato", 405 | // translation = "tomahto" 406 | // } 407 | // }; 408 | 409 | // Words wordsToAddToCustomization = new Words() 410 | // { 411 | // words = wordArrayToAddToCustomization 412 | // }; 413 | 414 | // if (!_textToSpeech.AddCustomizationWords(OnAddCustomizationWords, OnFail, _createdCustomizationId, wordsToAddToCustomization)) 415 | // Log.Debug("ExampleTextToSpeech.AddCustomizationWords()", "Failed to add words to {0}!", _createdCustomizationId); 416 | // while (!_addCustomizationWordsTested) 417 | // yield return null; 418 | 419 | // Get Customization Word 420 | // Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to get the translation of a custom voice model's word."); 421 | // string customIdentifierWord = wordsToUpdateCustomization[0].word; 422 | // if (!_textToSpeech.GetCustomizationWord(OnGetCustomizationWord, OnFail, _createdCustomizationId, customIdentifierWord)) 423 | // Log.Debug("ExampleTextToSpeech.GetCustomizationWord()", "Failed to get the translation of {0} from {1}!", customIdentifierWord, _createdCustomizationId); 424 | // while (!_getCustomizationWordTested) 425 | // yield return null; 426 | 427 | // Delete Customization Word 428 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to delete customization word from custom voice model."); 429 | string wordToDelete = "goodbye"; 430 | if (!_textToSpeech.DeleteCustomizationWord(OnDeleteCustomizationWord, OnFail, _createdCustomizationId, wordToDelete)) 431 | Log.Debug("ExampleTextToSpeech.DeleteCustomizationWord()", "Failed to delete {0} from {1}!", wordToDelete, _createdCustomizationId); 432 | while (!_deleteCustomizationWordTested) 433 | yield return null; 434 | 435 | // Delete Customization 436 | Log.Debug("ExampleTextToSpeech.Examples()", "Attempting to delete a customization"); 437 | if (!_textToSpeech.DeleteCustomization(OnDeleteCustomization, OnFail, _createdCustomizationId)) 438 | Log.Debug("ExampleTextToSpeech.DeleteCustomization()", "Failed to delete custom voice model!"); 439 | while (!_deleteCustomizationTested) 440 | yield return null; 441 | 442 | Log.Debug("ExampleTextToSpeech.Examples()", "Text to Speech examples complete."); 443 | } 444 | 445 | 446 | 447 | 448 | 449 | 450 | void HandleToSpeechCallback(AudioClip clip, Dictionary customData = null) 451 | { 452 | PlayClip(clip); 453 | } 454 | 455 | private void PlayClip(AudioClip clip) 456 | { 457 | if (Application.isPlaying && clip != null) 458 | { 459 | GameObject audioObject = new GameObject("AudioObject"); 460 | AudioSource source = audioObject.AddComponent(); 461 | source.spatialBlend = 0.0f; 462 | source.loop = false; 463 | source.clip = clip; 464 | source.Play(); 465 | 466 | Destroy(audioObject, clip.length); 467 | 468 | _synthesizeTested = true; 469 | } 470 | } 471 | 472 | private void OnGetVoices(Voices voices, Dictionary customData = null) 473 | { 474 | Log.Debug("ExampleTextToSpeech.OnGetVoices()", "Text to Speech - Get voices response: {0}", customData["json"].ToString()); 475 | _getVoicesTested = true; 476 | } 477 | 478 | private void OnGetVoice(Voice voice, Dictionary customData = null) 479 | { 480 | Log.Debug("ExampleTextToSpeech.OnGetVoice()", "Text to Speech - Get voice response: {0}", customData["json"].ToString()); 481 | _getVoiceTested = true; 482 | } 483 | 484 | private void OnGetPronunciation(Pronunciation pronunciation, Dictionary customData = null) 485 | { 486 | Log.Debug("ExampleTextToSpeech.OnGetPronunciation()", "Text to Speech - Get pronunciation response: {0}", customData["json"].ToString()); 487 | _getPronuciationTested = true; 488 | } 489 | 490 | // private void OnGetCustomizations(Customizations customizations, Dictionary customData = null) 491 | // { 492 | // Log.Debug("ExampleTextToSpeech.OnGetCustomizations()", "Text to Speech - Get customizations response: {0}", customData["json"].ToString()); 493 | // _getCustomizationsTested = true; 494 | // } 495 | 496 | // private void OnCreateCustomization(CustomizationID customizationID, Dictionary customData = null) 497 | // { 498 | // Log.Debug("ExampleTextToSpeech.OnCreateCustomization()", "Text to Speech - Create customization response: {0}", customData["json"].ToString()); 499 | // _createdCustomizationId = customizationID.customization_id; 500 | // _createCustomizationTested = true; 501 | // } 502 | 503 | private void OnDeleteCustomization(bool success, Dictionary customData = null) 504 | { 505 | Log.Debug("ExampleTextToSpeech.OnDeleteCustomization()", "Text to Speech - Delete customization response: {0}", customData["json"].ToString()); 506 | _createdCustomizationId = null; 507 | _deleteCustomizationTested = true; 508 | } 509 | 510 | // private void OnGetCustomization(Customization customization, Dictionary customData = null) 511 | // { 512 | // Log.Debug("ExampleTextToSpeech.OnGetCustomization()", "Text to Speech - Get customization response: {0}", customData["json"].ToString()); 513 | // _getCustomizationTested = true; 514 | // } 515 | 516 | private void OnUpdateCustomization(bool success, Dictionary customData = null) 517 | { 518 | Log.Debug("ExampleTextToSpeech.OnUpdateCustomization()", "Text to Speech - Update customization response: {0}", customData["json"].ToString()); 519 | _updateCustomizationTested = true; 520 | } 521 | 522 | // private void OnGetCustomizationWords(Words words, Dictionary customData = null) 523 | // { 524 | // Log.Debug("ExampleTextToSpeech.OnGetCustomizationWords()", "Text to Speech - Get customization words response: {0}", customData["json"].ToString()); 525 | // _getCustomizationWordsTested = true; 526 | // } 527 | 528 | private void OnAddCustomizationWords(bool success, Dictionary customData = null) 529 | { 530 | Log.Debug("ExampleTextToSpeech.OnAddCustomizationWords()", "Text to Speech - Add customization words response: {0}", customData["json"].ToString()); 531 | _addCustomizationWordsTested = true; 532 | } 533 | 534 | private void OnDeleteCustomizationWord(bool success, Dictionary customData = null) 535 | { 536 | Log.Debug("ExampleTextToSpeech.OnDeleteCustomizationWord()", "Text to Speech - Delete customization word response: {0}", customData["json"].ToString()); 537 | _deleteCustomizationWordTested = true; 538 | } 539 | 540 | private void OnGetCustomizationWord(Translation translation, Dictionary customData = null) 541 | { 542 | Log.Debug("ExampleTextToSpeech.OnGetCustomizationWord()", "Text to Speech - Get customization word response: {0}", customData["json"].ToString()); 543 | _getCustomizationWordTested = true; 544 | } 545 | 546 | private void OnFail(RESTConnector.Error error, Dictionary customData) 547 | { 548 | Log.Error("ExampleTextToSpeech.OnFail()", "Error received: {0}", error.ToString()); 549 | } 550 | 551 | 552 | 553 | } 554 | -------------------------------------------------------------------------------- /LT2.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustyoldrake/ibm_watson_unity/dea53276734ce88672dc40512cb9196eb39af3d1/LT2.zip -------------------------------------------------------------------------------- /LanguageTranslatorDemo.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Generic; 3 | using UnityEngine; 4 | using UnityEngine.UI; 5 | using IBM.Watson.DeveloperCloud.Services.LanguageTranslator.v2; 6 | using IBM.Watson.DeveloperCloud.Logging; 7 | using IBM.Watson.DeveloperCloud.Utilities; 8 | using IBM.Watson.DeveloperCloud.Connection; 9 | 10 | public class LanguageTranslatorDemo : MonoBehaviour 11 | { 12 | 13 | public Text ResponseTextField_es; 14 | public Text ResponseTextField_fr; 15 | public Text ResponseTextField_it; 16 | public Text ResponseTextField_de; 17 | public Text ResponseTextField_pt; 18 | public Text ResponseTextField_ar; 19 | public Text ResponseTextField_ja; 20 | 21 | 22 | 23 | private LanguageTranslator _languageTranslator; 24 | private string _translationModel_en_es = "en-es"; //spanish 25 | private string _translationModel_en_fr = "en-fr"; //french 26 | private string _translationModel_en_it = "en-it"; //italian 27 | private string _translationModel_en_de = "en-de"; //german 28 | private string _translationModel_en_pt = "en-pt"; //portuguese 29 | private string _translationModel_en_ar = "en-ar"; //arabic 30 | private string _translationModel_en_ja = "en-ja"; // japanese 31 | 32 | 33 | // All the language options here https://console.bluemix.net/docs/services/language-translator/translation-models.html 34 | // Use this for initialization 35 | void Start () 36 | { 37 | LogSystem.InstallDefaultReactors(); 38 | 39 | Credentials languageTranslatorCredentials = new Credentials () 40 | { 41 | Username = "f57f599c-45c5-402b-a998-######", 42 | Password = "62#####", 43 | Url = "https://gateway.watsonplatform.net/language-translator/api" 44 | }; 45 | 46 | _languageTranslator = new LanguageTranslator(languageTranslatorCredentials); 47 | 48 | //Translate("Where is the library?"); 49 | } 50 | 51 | 52 | public void Translate_es(string text) 53 | { 54 | _languageTranslator.GetTranslation(OnTranslate_es, OnFail, text, _translationModel_en_es); 55 | } 56 | 57 | private void OnTranslate_es(Translations response, Dictionary customData) 58 | { 59 | ResponseTextField_es.text = response.translations[0].translation; 60 | } 61 | 62 | 63 | // yes, i'm a terrible person for doing the code this way 64 | public void Translate_fr(string text) 65 | { 66 | _languageTranslator.GetTranslation(OnTranslate_fr, OnFail, text, _translationModel_en_fr); 67 | } 68 | 69 | private void OnTranslate_fr(Translations response, Dictionary customData) 70 | { 71 | ResponseTextField_fr.text = response.translations[0].translation; 72 | } 73 | 74 | 75 | // yes, i'm a terrible person for doing the code this way 76 | public void Translate_it(string text) 77 | { 78 | _languageTranslator.GetTranslation(OnTranslate_it, OnFail, text, _translationModel_en_it); 79 | } 80 | 81 | private void OnTranslate_it(Translations response, Dictionary customData) 82 | { 83 | ResponseTextField_it.text = response.translations[0].translation; 84 | } 85 | 86 | 87 | // yes, i'm a terrible person for doing the code this way 88 | public void Translate_de(string text) 89 | { 90 | _languageTranslator.GetTranslation(OnTranslate_de, OnFail, text, _translationModel_en_de); 91 | } 92 | 93 | private void OnTranslate_de(Translations response, Dictionary customData) 94 | { 95 | ResponseTextField_de.text = response.translations[0].translation; 96 | } 97 | 98 | 99 | 100 | 101 | // yes, i'm a terrible person for doing the code this way 102 | public void Translate_pt(string text) 103 | { 104 | _languageTranslator.GetTranslation(OnTranslate_pt, OnFail, text, _translationModel_en_pt); 105 | } 106 | 107 | private void OnTranslate_pt(Translations response, Dictionary customData) 108 | { 109 | ResponseTextField_pt.text = response.translations[0].translation; 110 | } 111 | 112 | 113 | // yes, i'm a terrible person for doing the code this way 114 | public void Translate_ar(string text) 115 | { 116 | _languageTranslator.GetTranslation(OnTranslate_ar, OnFail, text, _translationModel_en_ar); 117 | } 118 | 119 | private void OnTranslate_ar(Translations response, Dictionary customData) 120 | { 121 | ResponseTextField_ar.text = response.translations[0].translation; 122 | } 123 | 124 | // yes, i'm a terrible person for doing the code this way 125 | public void Translate_ja(string text) 126 | { 127 | _languageTranslator.GetTranslation(OnTranslate_ja, OnFail, text, _translationModel_en_ja); 128 | } 129 | 130 | private void OnTranslate_ja(Translations response, Dictionary customData) 131 | { 132 | ResponseTextField_ja.text = response.translations[0].translation; 133 | } 134 | 135 | 136 | 137 | 138 | private void OnFail(RESTConnector.Error error, Dictionary customData) 139 | { 140 | Log.Debug ("LanguageTranslatorDemo.OnFail()", "Error: {0}", error.ToString ()); 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ibm_watson_unity 2 | Loose collection of pieces related to IBM Watson SDK and Unity Package 3 | -------------------------------------------------------------------------------- /apples.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Generic; 3 | using UnityEngine; 4 | 5 | 6 | public class apples : MonoBehaviour { 7 | 8 | public IdleChanger _idlechanger; 9 | 10 | // Use this for initialization 11 | void Start () { 12 | 13 | //_idlechanger.gameObject.transform.localScale += new Vector3(.1F, .1F, .1F); 14 | 15 | } 16 | 17 | // Update is called once per frame 18 | void Update () { 19 | 20 | 21 | if (Input.GetKeyDown(KeyCode.Q)) 22 | { 23 | 24 | 25 | } 26 | 27 | 28 | // Update is called once per frame 29 | // PUMP UP SCALE 30 | if (Input.GetKeyDown(KeyCode.A)) 31 | { 32 | _idlechanger.gameObject.transform.localScale += new Vector3(.1F, .1F, .1F); 33 | 34 | } 35 | // SHRINK SCALE 36 | if (Input.GetKeyDown(KeyCode.Z)) 37 | { 38 | _idlechanger.gameObject.transform.localScale -= new Vector3(.1F, .1F, .1F); 39 | } 40 | 41 | 42 | // Rotate Clockwise 43 | if (Input.GetKeyDown(KeyCode.LeftArrow)) 44 | { 45 | _idlechanger.transform.localEulerAngles += new Vector3(0,10,0); 46 | 47 | } 48 | // Rotate CounterClockwise 49 | if (Input.GetKeyDown(KeyCode.RightArrow)) 50 | { 51 | _idlechanger.transform.localEulerAngles -= new Vector3(0,10,0); 52 | 53 | } 54 | 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /blackie.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustyoldrake/ibm_watson_unity/dea53276734ce88672dc40512cb9196eb39af3d1/blackie.zip -------------------------------------------------------------------------------- /cameraperspective.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Generic; 3 | using UnityEngine; 4 | 5 | public class camera_perspective : MonoBehaviour { 6 | 7 | // Use this for initialization 8 | void Start () { 9 | 10 | } 11 | 12 | // Update is called once per frame 13 | void Update () { 14 | 15 | // CHANGE CAMERA 16 | if (Input.GetKeyDown(KeyCode.A)) 17 | { 18 | Vector3 position = this.transform.position; 19 | position.x--; 20 | this.transform.position = position; 21 | } 22 | if (Input.GetKeyDown(KeyCode.D)) 23 | { 24 | Vector3 position = this.transform.position; 25 | position.x++; 26 | this.transform.position = position; 27 | } 28 | if (Input.GetKeyDown(KeyCode.W)) 29 | { 30 | Vector3 position = this.transform.position; 31 | position.y++; 32 | this.transform.position = position; 33 | } 34 | if (Input.GetKeyDown(KeyCode.S)) 35 | { 36 | Vector3 position = this.transform.position; 37 | position.y--; 38 | this.transform.position = position; 39 | } 40 | 41 | 42 | 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /colorbands.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Generic; 3 | using UnityEngine; 4 | 5 | 6 | public class color_bands : MonoBehaviour { 7 | 8 | 9 | // declare 10 | public MeshRenderer sphereMeshRenderer; 11 | 12 | public Material original_material; 13 | public Material color_black; 14 | public Material color_brown; 15 | public Material color_red; 16 | public Material color_orange; 17 | public Material color_yellow; 18 | public Material color_green; 19 | public Material color_blue; 20 | public Material color_violet; 21 | public Material color_grey; 22 | public Material color_white; 23 | 24 | //private int _numbertest = 1; 25 | 26 | 27 | // Use this for initialization 28 | void Start () { 29 | 30 | } 31 | 32 | // Update is called once per frame 33 | void Update () { 34 | 35 | // press a Key - get a color 36 | // this maps to RESISTOR LOOKUP in IEEE / Electronics 37 | // 38 | // -----(=|=\=|=|)----- 39 | // 40 | //Black 0 //Brown 1 41 | //Red 2 //Orange 3 42 | //Yellow 4 //Green 5 43 | //Blue 6 //Violet 7 44 | //Grey 8 //White 9 45 | 46 | // later you can do a CASE or SWITCH - but for now.... 47 | 48 | if (Input.GetKeyDown(KeyCode.Alpha0)) 49 | { sphereMeshRenderer.material = color_black; } 50 | 51 | if (Input.GetKeyDown(KeyCode.Alpha1)) 52 | { sphereMeshRenderer.material = color_brown; } 53 | 54 | if (Input.GetKeyDown(KeyCode.Alpha2)) 55 | { sphereMeshRenderer.material = color_red; } 56 | 57 | if (Input.GetKeyDown(KeyCode.Alpha3)) 58 | { sphereMeshRenderer.material = color_orange; } 59 | 60 | if (Input.GetKeyDown(KeyCode.Alpha4)) 61 | { sphereMeshRenderer.material = color_yellow; } 62 | 63 | if (Input.GetKeyDown(KeyCode.Alpha5)) 64 | { sphereMeshRenderer.material = color_green; } 65 | 66 | if (Input.GetKeyDown(KeyCode.Alpha6)) 67 | { sphereMeshRenderer.material = color_blue; } 68 | 69 | if (Input.GetKeyDown(KeyCode.Alpha7)) 70 | { sphereMeshRenderer.material = color_violet; } 71 | 72 | if (Input.GetKeyDown(KeyCode.Alpha8)) 73 | { sphereMeshRenderer.material = color_grey; } 74 | 75 | if (Input.GetKeyDown(KeyCode.Alpha9)) 76 | { sphereMeshRenderer.material = color_white; } 77 | 78 | 79 | 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /colors.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Generic; 3 | using UnityEngine; 4 | 5 | public class colors : MonoBehaviour { 6 | 7 | // Use this for initialization 8 | void Start () 9 | { 10 | 11 | } 12 | 13 | // Update is called once per frame 14 | void Update () 15 | { 16 | if (Input.GetKeyDown (KeyCode.R)) { 17 | gameObject.GetComponent().material.color = Color.red; 18 | } 19 | if (Input.GetKeyDown (KeyCode.G)) { 20 | gameObject.GetComponent().material.color = Color.green; 21 | } 22 | if (Input.GetKeyDown (KeyCode.B)) { 23 | gameObject.GetComponent().material.color = Color.blue; 24 | } 25 | if (Input.GetKeyDown (KeyCode.M)) { 26 | gameObject.GetComponent().material.color = Color.magenta; 27 | } 28 | 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /eeg_emotiv_key_mapping.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Generic; 3 | using UnityEngine; 4 | 5 | public class controller : MonoBehaviour { 6 | 7 | public float speed = 0.5f; 8 | public float force = 200; 9 | 10 | void Start () { 11 | 12 | } 13 | 14 | // Update is called once per frame 15 | void Update () { 16 | 17 | if (Input.GetKeyDown (KeyCode.LeftArrow) | Input.GetKeyDown (KeyCode.A) ) { this.GetComponent ().AddForce (new Vector3 (-force, 0, 0)); } 18 | if (Input.GetKeyDown (KeyCode.RightArrow) | Input.GetKeyDown (KeyCode.D) ) { this.GetComponent ().AddForce (new Vector3 (force, 0, 0)); } 19 | if (Input.GetKeyDown (KeyCode.UpArrow) | Input.GetKeyDown (KeyCode.W) ) { this.GetComponent ().AddForce (new Vector3 (0, 0, force)); } 20 | if (Input.GetKeyDown (KeyCode.DownArrow) | Input.GetKeyDown (KeyCode.S) ) { this.GetComponent ().AddForce (new Vector3 (0, 0, -force)); } 21 | if (Input.GetKeyDown (KeyCode.Space)) { this.GetComponent ().AddForce (new Vector3 (0, force, 0)); } 22 | if (Input.GetKeyDown (KeyCode.A)) { this.GetComponent ().AddForce (new Vector3 (0, 0, 0)); } 23 | } 24 | } 25 | 26 | 27 | 28 | // some basic code for mapping Emotive EEG brain signal detecter to keystrokes to input for my objects in unity 29 | -------------------------------------------------------------------------------- /examplestreaming_Plus_rainbow_octopusprotoype.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 IBM Corp. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | using UnityEngine; 19 | using System.Collections; 20 | using IBM.Watson.DeveloperCloud.Logging; 21 | using IBM.Watson.DeveloperCloud.Services.SpeechToText.v1; 22 | using IBM.Watson.DeveloperCloud.Utilities; 23 | using IBM.Watson.DeveloperCloud.DataTypes; 24 | using System.Collections.Generic; 25 | using UnityEngine.UI; 26 | 27 | public class ExampleStreaming : MonoBehaviour 28 | { 29 | #region PLEASE SET THESE VARIABLES IN THE INSPECTOR 30 | [SerializeField] 31 | private string _username; 32 | [SerializeField] 33 | private string _password; 34 | [SerializeField] 35 | private string _url; 36 | #endregion 37 | 38 | public Text ResultsField; 39 | 40 | private int _recordingRoutine = 0; 41 | private string _microphoneID = null; 42 | private AudioClip _recording = null; 43 | private int _recordingBufferSize = 1; 44 | private int _recordingHZ = 22050; 45 | 46 | private SpeechToText _speechToText; 47 | 48 | 49 | 50 | public MeshRenderer octopusMeshRenderer; 51 | 52 | public Material color_red; 53 | public Material color_green; 54 | public Material color_blue; 55 | public Material color_yellow; 56 | public Material color_white; 57 | 58 | public float speed; 59 | public float roto; 60 | public float moto_x; 61 | public float moto_y; 62 | 63 | 64 | 65 | 66 | void Start() 67 | { 68 | LogSystem.InstallDefaultReactors(); 69 | 70 | // Create credential and instantiate service 71 | Credentials credentials = new Credentials(_username, _password, _url); 72 | 73 | _speechToText = new SpeechToText(credentials); 74 | Active = true; 75 | 76 | StartRecording(); 77 | 78 | 79 | transform.position = new Vector3(0.0f, 0.0f, 0.0f); 80 | roto = 0.0f; 81 | moto_x = 0.0f; 82 | moto_y = 0.0f; 83 | Vector3 orginalPosition = octopusMeshRenderer.transform.position; // for reset 84 | 85 | 86 | 87 | } 88 | 89 | public bool Active 90 | { 91 | get { return _speechToText.IsListening; } 92 | set 93 | { 94 | if (value && !_speechToText.IsListening) 95 | { 96 | _speechToText.DetectSilence = true; 97 | _speechToText.EnableWordConfidence = true; 98 | _speechToText.EnableTimestamps = true; 99 | _speechToText.SilenceThreshold = 0.01f; 100 | _speechToText.MaxAlternatives = 0; 101 | _speechToText.EnableInterimResults = true; 102 | _speechToText.OnError = OnError; 103 | _speechToText.InactivityTimeout = -1; 104 | _speechToText.ProfanityFilter = false; 105 | _speechToText.SmartFormatting = true; 106 | _speechToText.SpeakerLabels = false; 107 | _speechToText.WordAlternativesThreshold = null; 108 | _speechToText.StartListening(OnRecognize, OnRecognizeSpeaker); 109 | } 110 | else if (!value && _speechToText.IsListening) 111 | { 112 | _speechToText.StopListening(); 113 | } 114 | } 115 | } 116 | 117 | private void StartRecording() 118 | { 119 | if (_recordingRoutine == 0) 120 | { 121 | UnityObjectUtil.StartDestroyQueue(); 122 | _recordingRoutine = Runnable.Run(RecordingHandler()); 123 | } 124 | } 125 | 126 | private void StopRecording() 127 | { 128 | if (_recordingRoutine != 0) 129 | { 130 | Microphone.End(_microphoneID); 131 | Runnable.Stop(_recordingRoutine); 132 | _recordingRoutine = 0; 133 | } 134 | } 135 | 136 | private void OnError(string error) 137 | { 138 | Active = false; 139 | 140 | Log.Debug("ExampleStreaming.OnError()", "Error! {0}", error); 141 | } 142 | 143 | private IEnumerator RecordingHandler() 144 | { 145 | Log.Debug("ExampleStreaming.RecordingHandler()", "devices: {0}", Microphone.devices); 146 | _recording = Microphone.Start(_microphoneID, true, _recordingBufferSize, _recordingHZ); 147 | yield return null; // let _recordingRoutine get set.. 148 | 149 | if (_recording == null) 150 | { 151 | StopRecording(); 152 | yield break; 153 | } 154 | 155 | bool bFirstBlock = true; 156 | int midPoint = _recording.samples / 2; 157 | float[] samples = null; 158 | 159 | while (_recordingRoutine != 0 && _recording != null) 160 | { 161 | int writePos = Microphone.GetPosition(_microphoneID); 162 | if (writePos > _recording.samples || !Microphone.IsRecording(_microphoneID)) 163 | { 164 | Log.Error("ExampleStreaming.RecordingHandler()", "Microphone disconnected."); 165 | 166 | StopRecording(); 167 | yield break; 168 | } 169 | 170 | if ((bFirstBlock && writePos >= midPoint) 171 | || (!bFirstBlock && writePos < midPoint)) 172 | { 173 | // front block is recorded, make a RecordClip and pass it onto our callback. 174 | samples = new float[midPoint]; 175 | _recording.GetData(samples, bFirstBlock ? 0 : midPoint); 176 | 177 | AudioData record = new AudioData(); 178 | record.MaxLevel = Mathf.Max(Mathf.Abs(Mathf.Min(samples)), Mathf.Max(samples)); 179 | record.Clip = AudioClip.Create("Recording", midPoint, _recording.channels, _recordingHZ, false); 180 | record.Clip.SetData(samples, 0); 181 | 182 | _speechToText.OnListen(record); 183 | 184 | bFirstBlock = !bFirstBlock; 185 | } 186 | else 187 | { 188 | // calculate the number of samples remaining until we ready for a block of audio, 189 | // and wait that amount of time it will take to record. 190 | int remaining = bFirstBlock ? (midPoint - writePos) : (_recording.samples - writePos); 191 | float timeRemaining = (float)remaining / (float)_recordingHZ; 192 | 193 | yield return new WaitForSeconds(timeRemaining); 194 | } 195 | 196 | } 197 | 198 | yield break; 199 | } 200 | 201 | private void OnRecognize(SpeechRecognitionEvent result, Dictionary customData) 202 | { 203 | if (result != null && result.results.Length > 0) 204 | { 205 | foreach (var res in result.results) 206 | { 207 | foreach (var alt in res.alternatives) 208 | { 209 | string text = string.Format("{0} ({1}, {2:0.00})\n", alt.transcript, res.final ? "Final" : "Interim", alt.confidence); 210 | Log.Debug("ExampleStreaming.OnRecognize()", text); 211 | ResultsField.text = text; 212 | 213 | 214 | if (alt.transcript.Contains("green") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 215 | { 216 | octopusMeshRenderer.material = color_green; 217 | } 218 | 219 | if (alt.transcript.Contains("red") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 220 | { 221 | octopusMeshRenderer.material = color_red; 222 | } 223 | 224 | if (alt.transcript.Contains("blue") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 225 | { 226 | octopusMeshRenderer.material = color_blue; 227 | } 228 | if (alt.transcript.Contains("yellow") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 229 | { 230 | octopusMeshRenderer.material = color_yellow; 231 | } 232 | 233 | if (alt.transcript.Contains("rotate") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 234 | { 235 | roto = 1.0f; 236 | octopusMeshRenderer.transform.Rotate(Time.deltaTime, roto, 0); 237 | //octopusMeshRenderer.transform.Translate(moto_y, 0,0); 238 | 239 | //roto = -1.0f; 240 | 241 | } 242 | 243 | if (alt.transcript.Contains("stop") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 244 | { 245 | roto = 0.0f; 246 | octopusMeshRenderer.transform.Rotate(Time.deltaTime, roto, 0); 247 | //octopusMeshRenderer.transform.Translate(moto_y, 0,0); 248 | } 249 | 250 | if (alt.transcript.Contains("reset") && ResultsField.text.Contains("Final")) // needs to be final or ECHO happens 251 | { 252 | octopusMeshRenderer.transform.localPosition = new Vector3(0, 0, 0); 253 | 254 | //gameObject.transform.Rotate(0, -90, 0); // works 255 | octopusMeshRenderer.transform.Rotate(0, -90, 0); // works (ish) 256 | 257 | roto = 0.0f; 258 | moto_x = 0.0f; 259 | moto_y = 0.0f; 260 | 261 | } 262 | 263 | 264 | 265 | 266 | 267 | } 268 | 269 | if (res.keywords_result != null && res.keywords_result.keyword != null) 270 | { 271 | foreach (var keyword in res.keywords_result.keyword) 272 | { 273 | Log.Debug("ExampleStreaming.OnRecognize()", "keyword: {0}, confidence: {1}, start time: {2}, end time: {3}", keyword.normalized_text, keyword.confidence, keyword.start_time, keyword.end_time); 274 | } 275 | } 276 | 277 | if (res.word_alternatives != null) 278 | { 279 | foreach (var wordAlternative in res.word_alternatives) 280 | { 281 | Log.Debug("ExampleStreaming.OnRecognize()", "Word alternatives found. Start time: {0} | EndTime: {1}", wordAlternative.start_time, wordAlternative.end_time); 282 | foreach(var alternative in wordAlternative.alternatives) 283 | Log.Debug("ExampleStreaming.OnRecognize()", "\t word: {0} | confidence: {1}", alternative.word, alternative.confidence); 284 | } 285 | } 286 | } 287 | } 288 | } 289 | 290 | private void OnRecognizeSpeaker(SpeakerRecognitionEvent result, Dictionary customData) 291 | { 292 | if (result != null) 293 | { 294 | foreach (SpeakerLabelsResult labelResult in result.speaker_labels) 295 | { 296 | Log.Debug("ExampleStreaming.OnRecognize()", string.Format("speaker result: {0} | confidence: {3} | from: {1} | to: {2}", labelResult.speaker, labelResult.from, labelResult.to, labelResult.confidence)); 297 | } 298 | } 299 | } 300 | 301 | // Update is called once per frame 302 | void Update () { 303 | 304 | 305 | //float step = speed * Time.deltaTime; 306 | //transform.position = Vector3.MoveTowards(transform.position, target.position, step); 307 | 308 | /// COLOR 309 | if (Input.GetKeyDown (KeyCode.R)) { 310 | octopusMeshRenderer.material = color_red; 311 | } 312 | if (Input.GetKeyDown (KeyCode.G)) { 313 | octopusMeshRenderer.material = color_green; 314 | } 315 | if (Input.GetKeyDown (KeyCode.B)) { 316 | octopusMeshRenderer.material = color_blue; 317 | } 318 | if (Input.GetKeyDown (KeyCode.Y)) { 319 | octopusMeshRenderer.material = color_yellow; 320 | } 321 | 322 | 323 | 324 | /// MOVEMENT 325 | 326 | 327 | /// Navigation 328 | /// https://www.youtube.com/watch?v=mC9BfAqwU2Q 329 | float horizontal = Input.GetAxisRaw ("Horizontal"); 330 | float vertical = Input.GetAxisRaw ("Vertical"); 331 | Vector3 direction = new Vector3 (vertical, 0, horizontal); 332 | octopusMeshRenderer.transform.Translate (direction.normalized * Time.deltaTime * speed); 333 | 334 | /// Rotataion Update 335 | octopusMeshRenderer.transform.Rotate(Time.deltaTime, roto, 0); 336 | octopusMeshRenderer.transform.Translate(moto_y, 0,0); 337 | 338 | 339 | // Rotate the object around its local X axis at 1 degree per second 340 | //transform.Rotate(Time.deltaTime, 0, 0); 341 | // ...also rotate around the World's Y axis 342 | //transform.Rotate(0, Time.deltaTime, 0, Space.World); 343 | 344 | 345 | // Keystrokes to toggle on/off rotation 346 | if (Input.GetKeyDown (KeyCode.Alpha1)) { 347 | roto = 1.0f; 348 | } 349 | if (Input.GetKeyDown (KeyCode.Alpha2)) { 350 | roto = 0.0f; 351 | } 352 | if (Input.GetKeyDown (KeyCode.Alpha3)) { 353 | roto = -1.0f; 354 | } 355 | if (Input.GetKeyDown (KeyCode.D)) { 356 | moto_y = 0.1f; 357 | } 358 | if (Input.GetKeyDown (KeyCode.C)) { 359 | moto_y = -0.1f; 360 | } 361 | 362 | /// REset octopus 363 | if (Input.GetKeyDown (KeyCode.Space)) { 364 | 365 | 366 | octopusMeshRenderer.transform.localPosition = new Vector3(0, 0, 0); 367 | 368 | //gameObject.transform.Rotate(0, -90, 0); // works 369 | octopusMeshRenderer.transform.Rotate(0, -90, 0); // works (ish) 370 | 371 | roto = 0.0f; 372 | moto_x = 0.0f; 373 | moto_y = 0.0f; 374 | 375 | } 376 | } // update 377 | 378 | } 379 | -------------------------------------------------------------------------------- /moveobject.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Generic; 3 | using UnityEngine; 4 | 5 | public class moveobject : MonoBehaviour { 6 | 7 | // Use this for initialization 8 | void Start () { 9 | 10 | } 11 | 12 | // Update is called once per frame 13 | void Update () { 14 | 15 | if (Input.GetKeyDown(KeyCode.LeftArrow)) 16 | { 17 | Vector3 position = this.transform.position; 18 | position.x--; 19 | this.transform.position = position; 20 | } 21 | if (Input.GetKeyDown(KeyCode.RightArrow)) 22 | { 23 | Vector3 position = this.transform.position; 24 | position.x++; 25 | this.transform.position = position; 26 | } 27 | if (Input.GetKeyDown(KeyCode.UpArrow)) 28 | { 29 | Vector3 position = this.transform.position; 30 | position.y++; 31 | this.transform.position = position; 32 | } 33 | if (Input.GetKeyDown(KeyCode.DownArrow)) 34 | { 35 | Vector3 position = this.transform.position; 36 | position.y--; 37 | this.transform.position = position; 38 | } 39 | 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /objects.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustyoldrake/ibm_watson_unity/dea53276734ce88672dc40512cb9196eb39af3d1/objects.png -------------------------------------------------------------------------------- /octopus_color.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Generic; 3 | using UnityEngine; 4 | 5 | public class octopus_color : MonoBehaviour { 6 | 7 | 8 | public MeshRenderer octopusMeshRenderer; 9 | 10 | public Material color_red; 11 | public Material color_green; 12 | public Material color_blue; 13 | public Material color_yellow; 14 | 15 | 16 | // Use this for initialization 17 | void Start () { 18 | 19 | } 20 | 21 | // Update is called once per frame 22 | void Update () { 23 | 24 | 25 | if (Input.GetKeyDown (KeyCode.R)) { 26 | octopusMeshRenderer.material = color_red; 27 | } 28 | if (Input.GetKeyDown (KeyCode.G)) { 29 | octopusMeshRenderer.material = color_green; 30 | } 31 | if (Input.GetKeyDown (KeyCode.B)) { 32 | octopusMeshRenderer.material = color_blue; 33 | } 34 | if (Input.GetKeyDown (KeyCode.Y)) { 35 | octopusMeshRenderer.material = color_yellow; 36 | } 37 | 38 | 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /octopus_color2.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Generic; 3 | using UnityEngine; 4 | 5 | public class octopus_color : MonoBehaviour { 6 | 7 | 8 | public MeshRenderer octopusMeshRenderer; 9 | 10 | public Material color_red; 11 | public Material color_green; 12 | public Material color_blue; 13 | public Material color_yellow; 14 | public Material color_white; 15 | 16 | public float speed; 17 | public float roto; 18 | public float moto_x; 19 | public float moto_y; 20 | 21 | 22 | // Use this for initialization 23 | void Start () { 24 | 25 | 26 | 27 | transform.position = new Vector3(0.0f, 0.0f, 0.0f); 28 | roto = 0.0f; 29 | moto_x = 0.0f; 30 | moto_y = 0.0f; 31 | Vector3 orginalPosition = octopusMeshRenderer.transform.position; // for reset 32 | 33 | } 34 | 35 | // Update is called once per frame 36 | void Update () { 37 | 38 | 39 | //float step = speed * Time.deltaTime; 40 | //transform.position = Vector3.MoveTowards(transform.position, target.position, step); 41 | 42 | /// COLOR 43 | if (Input.GetKeyDown (KeyCode.R)) { 44 | octopusMeshRenderer.material = color_red; 45 | } 46 | if (Input.GetKeyDown (KeyCode.G)) { 47 | octopusMeshRenderer.material = color_green; 48 | } 49 | if (Input.GetKeyDown (KeyCode.B)) { 50 | octopusMeshRenderer.material = color_blue; 51 | } 52 | if (Input.GetKeyDown (KeyCode.Y)) { 53 | octopusMeshRenderer.material = color_yellow; 54 | } 55 | 56 | 57 | 58 | /// MOVEMENT 59 | 60 | 61 | /// Navigation 62 | /// https://www.youtube.com/watch?v=mC9BfAqwU2Q 63 | float horizontal = Input.GetAxisRaw ("Horizontal"); 64 | float vertical = Input.GetAxisRaw ("Vertical"); 65 | Vector3 direction = new Vector3 (vertical, 0, horizontal); 66 | octopusMeshRenderer.transform.Translate (direction.normalized * Time.deltaTime * speed); 67 | 68 | /// Rotataion Update 69 | transform.Rotate(Time.deltaTime, roto, 0); 70 | octopusMeshRenderer.transform.Translate(moto_y, 0,0); 71 | 72 | 73 | // Rotate the object around its local X axis at 1 degree per second 74 | //transform.Rotate(Time.deltaTime, 0, 0); 75 | // ...also rotate around the World's Y axis 76 | //transform.Rotate(0, Time.deltaTime, 0, Space.World); 77 | 78 | 79 | // Keystrokes to toggle on/off rotation 80 | if (Input.GetKeyDown (KeyCode.Alpha1)) { 81 | roto = 1.0f; 82 | } 83 | if (Input.GetKeyDown (KeyCode.Alpha2)) { 84 | roto = 0.0f; 85 | } 86 | if (Input.GetKeyDown (KeyCode.Alpha3)) { 87 | roto = -1.0f; 88 | } 89 | if (Input.GetKeyDown (KeyCode.D)) { 90 | moto_y = 0.1f; 91 | } 92 | if (Input.GetKeyDown (KeyCode.C)) { 93 | moto_y = -0.1f; 94 | } 95 | 96 | /// REset octopus 97 | if (Input.GetKeyDown (KeyCode.Space)) { 98 | 99 | 100 | octopusMeshRenderer.transform.localPosition = new Vector3(0, 0, 0); 101 | 102 | //gameObject.transform.Rotate(0, -90, 0); // works 103 | octopusMeshRenderer.transform.Rotate(0, -90, 0); // works (ish) 104 | 105 | roto = 0.0f; 106 | moto_x = 0.0f; 107 | moto_y = 0.0f; 108 | 109 | } 110 | 111 | 112 | 113 | } // update 114 | 115 | 116 | } // class 117 | -------------------------------------------------------------------------------- /orb.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustyoldrake/ibm_watson_unity/dea53276734ce88672dc40512cb9196eb39af3d1/orb.zip -------------------------------------------------------------------------------- /sizeobject.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Generic; 3 | using UnityEngine; 4 | 5 | public class sizeobject : MonoBehaviour { 6 | 7 | // Use this for initialization 8 | void Start () { 9 | 10 | } 11 | 12 | // Update is called once per frame 13 | void Update () { 14 | 15 | // PUMP UP SCALE 16 | if (Input.GetKeyDown(KeyCode.Period)) 17 | { 18 | transform.localScale += new Vector3(0.1F, 0.1F, 0.1F); 19 | 20 | } 21 | // SHRINK SCALE 22 | if (Input.GetKeyDown(KeyCode.Comma)) 23 | { 24 | transform.localScale -= new Vector3(0.1F, 0.1F, 0.1F); 25 | } 26 | 27 | 28 | 29 | } 30 | } 31 | --------------------------------------------------------------------------------