├── .gitattributes ├── .gitignore ├── Documents └── Script.docx ├── Images ├── Screenshot1.png ├── Screenshot2.png └── hero_image.png ├── README.md ├── SECURITY.md └── src ├── App.xaml ├── App.xaml.cs ├── Assets ├── LockScreenLogo.scale-200.png ├── SplashScreen.scale-200.png ├── Square150x150Logo.scale-100.png ├── Square150x150Logo.scale-125.png ├── Square150x150Logo.scale-150.png ├── Square150x150Logo.scale-200.png ├── Square150x150Logo.scale-400.png ├── Square310x310Logo.scale-100.png ├── Square310x310Logo.scale-125.png ├── Square310x310Logo.scale-150.png ├── Square310x310Logo.scale-200.png ├── Square310x310Logo.scale-400.png ├── Square44x44Logo.scale-100.png ├── Square44x44Logo.scale-125.png ├── Square44x44Logo.scale-150.png ├── Square44x44Logo.scale-200.png ├── Square44x44Logo.scale-400.png ├── Square71x71Logo.scale-100.png ├── Square71x71Logo.scale-125.png ├── Square71x71Logo.scale-150.png ├── Square71x71Logo.scale-200.png ├── Square71x71Logo.scale-400.png ├── StoreLogo.scale-100.png ├── StoreLogo.scale-125.png ├── StoreLogo.scale-150.png ├── StoreLogo.scale-200.png ├── StoreLogo.scale-400.png ├── bg_city.png ├── bg_micro.png ├── bg_ok.png ├── corner_blue.png ├── corner_white.png ├── ico_micro.png ├── ico_ok.png ├── logo_bikesharing.png ├── logo_splash.png ├── logo_splash.scale-200.png ├── logo_splashBig.png ├── silhouette01.png └── silhouette02.png ├── BikeSharing.Clients.CogServicesKiosk.csproj ├── BikeSharing.Clients.CogServicesKiosk.sln ├── Converters └── ValueToVisibilityConverter.cs ├── Data ├── AudioRecorder.cs ├── SpeechToText.cs ├── TextToSpeech.cs └── UserLookupService.cs ├── Extensions └── SoftwareBitmapExtensions.cs ├── MainPage.xaml ├── MainPage.xaml.cs ├── Models └── UserProfile.cs ├── Package.appxmanifest ├── Properties ├── AssemblyInfo.cs └── Default.rd.xml └── project.json /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # User-specific files (MonoDevelop/Xamarin Studio) 11 | *.userprefs 12 | 13 | # Build results 14 | [Dd]ebug/ 15 | [Dd]ebugPublic/ 16 | [Rr]elease/ 17 | [Rr]eleases/ 18 | [Xx]64/ 19 | [Xx]86/ 20 | [Bb]uild/ 21 | bld/ 22 | [Bb]in/ 23 | [Oo]bj/ 24 | 25 | # Visual Studio 2015 cache/options directory 26 | .vs/ 27 | # Uncomment if you have tasks that create the project's static files in wwwroot 28 | #wwwroot/ 29 | 30 | # MSTest test Results 31 | [Tt]est[Rr]esult*/ 32 | [Bb]uild[Ll]og.* 33 | 34 | # NUNIT 35 | *.VisualState.xml 36 | TestResult.xml 37 | 38 | # Build Results of an ATL Project 39 | [Dd]ebugPS/ 40 | [Rr]eleasePS/ 41 | dlldata.c 42 | 43 | # DNX 44 | project.lock.json 45 | artifacts/ 46 | 47 | *_i.c 48 | *_p.c 49 | *_i.h 50 | *.ilk 51 | *.meta 52 | *.obj 53 | *.pch 54 | *.pdb 55 | *.pgc 56 | *.pgd 57 | *.rsp 58 | *.sbr 59 | *.tlb 60 | *.tli 61 | *.tlh 62 | *.tmp 63 | *.tmp_proj 64 | *.log 65 | *.vspscc 66 | *.vssscc 67 | .builds 68 | *.pidb 69 | *.svclog 70 | *.scc 71 | 72 | # Chutzpah Test files 73 | _Chutzpah* 74 | 75 | # Visual C++ cache files 76 | ipch/ 77 | *.aps 78 | *.ncb 79 | *.opendb 80 | *.opensdf 81 | *.sdf 82 | *.cachefile 83 | *.VC.db 84 | 85 | # Visual Studio profiler 86 | *.psess 87 | *.vsp 88 | *.vspx 89 | *.sap 90 | 91 | # TFS 2012 Local Workspace 92 | $tf/ 93 | 94 | # Guidance Automation Toolkit 95 | *.gpState 96 | 97 | # ReSharper is a .NET coding add-in 98 | _ReSharper*/ 99 | *.[Rr]e[Ss]harper 100 | *.DotSettings.user 101 | 102 | # JustCode is a .NET coding add-in 103 | .JustCode 104 | 105 | # TeamCity is a build add-in 106 | _TeamCity* 107 | 108 | # DotCover is a Code Coverage Tool 109 | *.dotCover 110 | 111 | # NCrunch 112 | _NCrunch_* 113 | .*crunch*.local.xml 114 | nCrunchTemp_* 115 | 116 | # MightyMoose 117 | *.mm.* 118 | AutoTest.Net/ 119 | 120 | # Web workbench (sass) 121 | .sass-cache/ 122 | 123 | # Installshield output folder 124 | [Ee]xpress/ 125 | 126 | # DocProject is a documentation generator add-in 127 | DocProject/buildhelp/ 128 | DocProject/Help/*.HxT 129 | DocProject/Help/*.HxC 130 | DocProject/Help/*.hhc 131 | DocProject/Help/*.hhk 132 | DocProject/Help/*.hhp 133 | DocProject/Help/Html2 134 | DocProject/Help/html 135 | 136 | # Click-Once directory 137 | publish/ 138 | 139 | # Publish Web Output 140 | *.[Pp]ublish.xml 141 | *.azurePubxml 142 | 143 | # TODO: Un-comment the next line if you do not want to checkin 144 | # your web deploy settings because they may include unencrypted 145 | # passwords 146 | #*.pubxml 147 | *.publishproj 148 | 149 | # NuGet Packages 150 | *.nupkg 151 | # The packages folder can be ignored because of Package Restore 152 | **/packages/* 153 | # except build/, which is used as an MSBuild target. 154 | !**/packages/build/ 155 | # Uncomment if necessary however generally it will be regenerated when needed 156 | #!**/packages/repositories.config 157 | # NuGet v3's project.json files produces more ignoreable files 158 | *.nuget.props 159 | *.nuget.targets 160 | 161 | # Microsoft Azure Build Output 162 | csx/ 163 | *.build.csdef 164 | 165 | # Microsoft Azure Emulator 166 | ecf/ 167 | rcf/ 168 | 169 | # Windows Store app package directory 170 | AppPackages/ 171 | BundleArtifacts/ 172 | 173 | # Visual Studio cache files 174 | # files ending in .cache can be ignored 175 | *.[Cc]ache 176 | # but keep track of directories ending in .cache 177 | !*.[Cc]ache/ 178 | 179 | # Others 180 | ClientBin/ 181 | [Ss]tyle[Cc]op.* 182 | ~$* 183 | *~ 184 | *.dbmdl 185 | *.dbproj.schemaview 186 | *.pfx 187 | *.publishsettings 188 | node_modules/ 189 | orleans.codegen.cs 190 | 191 | # RIA/Silverlight projects 192 | Generated_Code/ 193 | 194 | # Backup & report files from converting an old project file 195 | # to a newer Visual Studio version. Backup files are not needed, 196 | # because we have git ;-) 197 | _UpgradeReport_Files/ 198 | Backup*/ 199 | UpgradeLog*.XML 200 | UpgradeLog*.htm 201 | 202 | # SQL Server files 203 | *.mdf 204 | *.ldf 205 | 206 | # Business Intelligence projects 207 | *.rdl.data 208 | *.bim.layout 209 | *.bim_*.settings 210 | 211 | # Microsoft Fakes 212 | FakesAssemblies/ 213 | 214 | # GhostDoc plugin setting file 215 | *.GhostDoc.xml 216 | 217 | # Node.js Tools for Visual Studio 218 | .ntvs_analysis.dat 219 | 220 | # Visual Studio 6 build log 221 | *.plg 222 | 223 | # Visual Studio 6 workspace options file 224 | *.opt 225 | 226 | # Visual Studio LightSwitch build output 227 | **/*.HTMLClient/GeneratedArtifacts 228 | **/*.DesktopClient/GeneratedArtifacts 229 | **/*.DesktopClient/ModelManifest.xml 230 | **/*.Server/GeneratedArtifacts 231 | **/*.Server/ModelManifest.xml 232 | _Pvt_Extensions 233 | 234 | # LightSwitch generated files 235 | GeneratedArtifacts/ 236 | ModelManifest.xml 237 | 238 | # Paket dependency manager 239 | .paket/paket.exe 240 | 241 | # FAKE - F# Make 242 | .fake/ 243 | -------------------------------------------------------------------------------- /Documents/Script.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/Documents/Script.docx -------------------------------------------------------------------------------- /Images/Screenshot1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/Images/Screenshot1.png -------------------------------------------------------------------------------- /Images/Screenshot2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/Images/Screenshot2.png -------------------------------------------------------------------------------- /Images/hero_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/Images/hero_image.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BikeSharing360 2 | 3 | During our Connect(); event this year we presented 15 demos in Scott Guthrie’s and Scott Hanselman’s keynotes. If you missed the keynotes, you can watch the recording in [Channel 9](https://channel9.msdn.com/Events/Connect/2016/Keynotes-Scott-Guthrie-and-Scott-Hanselman). 4 | 5 | This year, we built the technology stack for a fictional company named BikeSharing360, which allows users to rent bikes from one location to another. 6 | 7 | BikeSharing360 is a fictitious example of a smart bike sharing system with 10,000 bikes distributed in 650 stations located throughout New York City and Seattle. Their vision is to provide a modern and personalized experience to riders and to run their business with intelligence. 8 | 9 | In this demo scenario, we built several apps for both the enterprise and the consumer (bike riders). You can find all other BikeSharing360 repos in the following locations: 10 | 11 | * [Mobile Apps](https://github.com/Microsoft/BikeSharing360_MobileApps) 12 | * [Backend Services](https://github.com/Microsoft/BikeSharing360_BackendServices) 13 | * [Websites](https://github.com/Microsoft/BikeSharing360_Websites) 14 | * [Single Container Apps](https://github.com/Microsoft/BikeSharing360_SingleContainer) 15 | * [Multi Container Apps](https://github.com/Microsoft/BikeSharing360_MultiContainer) 16 | * [Cognitive Services Kiosk App](https://github.com/Microsoft/BikeSharing360_CognitiveServicesKioskApp) 17 | * [Azure Bot App](https://github.com/Microsoft/BikeSharing360_BotApps) 18 | 19 | # BikeSharing360 Modern Kiosk with Cognitive Services 20 | 21 | During Connect(); 2016 we showcased many technologies available to you as a developer across Azure, Office, Windows, Visual Studio and Visual Studio Team Services. We’ve also heard from you that you love to have real-world applications through which you can directly experience what’s possible using those technologies. This year, then, we built out a full bikerider scenario for our Connect(); 2016 demos and are delighted to share all the source code with you. 22 | 23 | **Note:** This document is about the **Kiosk app** only. 24 | 25 | This kiosk app leveraged Cognitive Services to enable customers to interact with the kiosk through face detection, face recognition, voice verification, text-to-speech, speech-to-text, language understanding, and emotion detection, which allowed them to complete a transaction without the need for traditional input or touch or even pulling out your wallet. 26 | 27 | ![BikeRider](Images/hero_image.png) 28 | 29 | ## Requirements 30 | * Windows 10 31 | * [Visual Studio __2015__](https://www.visualstudio.com/en-us/products/vs-2015-product-editions.aspx) Update 3 (14.0 or higher) to compile C# 6 language features (or Visual Studio MacOS) 32 | * Microsoft Azure subscription 33 | 34 | ## Screens 35 | ![Screenshot 1](Images/Screenshot1.png) 36 | ![Screenshot 2](Images/Screenshot2.png) 37 | 38 | ## Setup 39 | Download or clone the repository. 40 | 41 | 1. Create an Azure account if you don't already have one using the steps in the next section of this README. 42 | 1. Create a Cognitive Service Key for the Face SDK: 43 | 1. In Azure portal, click + New button 44 | 1. Type "Cognitive Service APIs" in the search box 45 | 1. Click Cognitive Service APIs (preview) in search result 46 | 1. Click Create button 47 | 1. Give a unique name to the Account Name field (for example, "FaceApiAccount") 48 | 1. Click Api type to configure required settings 49 | 1. Pick Face API (preview) from the list of services 50 | 1. Click Pricing tier to select a proper pricing tier 51 | 1. Set proper Resource Group option 52 | 1. Click Legal terms to review and agree on the terms 53 | 1. Click Create button 54 | 1. Shortly you should receive a notification if the deployment succeeds. Click on the notification should bring you to the service account you just created 55 | 1. Click on Keys and take a note of either one of the two keys. 56 | 1. Create a Cognitive Service Key for the Voice Verification SDK 57 | 1. Similar to how you create the Face Api service, but this time, pick Speacker Recognition APIs (preview) from the list of services. 58 | 1. Take note of either one of the two keys. We will need it in the later steps. 59 | 1. Set up a LUIS model using the [instructions on setting up the LUIS model from the BOT sample]( https://github.com/Microsoft/BikeSharing360_BotApps) 60 | 1. Create a face verification profile for yourself 61 | 1. Clone the repository [https://github.com/Microsoft/Cognitive-Face-Windows](https://github.com/Microsoft/Cognitive-Face-Windows) 62 | 1. Open `Sample-WPF\Controls\FaceIdentificationPage.xaml.cs`, change Line 67 to have a more user-friendly name, instead of a GUID. 63 | 64 | public static readonly string SampleGroupName = Guid.NewGuid().ToString(); 65 | 66 | 1. Follow the instruction in README.md to build the sample 67 | 1. Run the sample 68 | 1. Click on Subscription Key Management tab, paste in the Face API key you saved earlier 69 | 1. The sample comes with a set of training data under Data/PersonGroup folder in the repository, create a new folder (with a name of your choice), and copy one or more of your profile images into that folder. Frontal or near-frontal face works the best. Delete other folders in the example data set to reduce the api calls to your Face Api service 70 | 1. Click on Face Identification tab 71 | 1. Click Load PersonGroup button 72 | 1. Select the Data/PersonGroup folder on your local disk 73 | 1. If any of your profile images contains a valid face, the image will show up. It means your Face profile is registered successfully. 74 | 1. The status pane should also contain the face profile id for each person, take note of your face profile id. 75 | 76 | Example log: 77 | 78 | ``` 79 | [12:22:42.547589]: Response: Success. Person "Family1-Dad" (PersonID:9acfe7e1-6196-4230-aed8-a0b172ee2298) created 80 | [12:22:43.913174]: Request: Creating person "Family1-Daughter" 81 | [12:22:44.228009]: Response: Success. Person "Family1-Daughter" (PersonID:c32d0abe-ef03-40fc-b50d-66c989a9957e) created 82 | ``` 83 | 84 | 1. You need the group name, and the Face profile Id 85 | 86 | 1. Create a voice verification profile for yourself 87 | 1. Clone the repository https://github.com/Microsoft/Cognitive-SpeakerRecognition-Windows 88 | 1. Follow the instruction in README.md to build Verification/SPIDVerficationAPI_WPF_Sample.sln 89 | 1. Run the sample 90 | 1. Paste your Speacker Recognition account key in the Subscription Key Management tab 91 | 1. Click on Scenario 1: Make a new Enrollment tab 92 | 1. Pick one phrases from the ten available phrases 93 | 1. Click Record button then start speaking your chosen phrase via microphone. Click Stop Recording after wards, the status pane should show your phrase if everything works as expected: 94 | 95 | ``` 96 | [12:02:22.651469]: Your phrase: 97 | ``` 98 | 99 | 1. The sample code doesn't show you the Speaker profile id in the status pane. To work around it, you can click Reset Profile button as soon as it's enabled, the application will show a message similar to the below in the status pane. Take note of this id. 100 | 101 | ``` 102 | [12:07:03.877365]: Resetting profile: 54aa9c1d-a815-44b6-9696-26be765dd840 103 | ``` 104 | 105 | 1. Repeat the recoding/stop recording using your chosen phrase until Remaining Enrollments reaches 0. 106 | 1. You need the Speaker profile Id. 107 | 108 | 6. Open the BikeSharing.Clients.CogServicesKiosk.sln solution 109 | a. Update the Cognitive Services keys in the App.xaml.cs 110 | b. Add a row representing yourself in the constructor of the Data\UserLookupServices.cs class 111 | Build and run the application 112 | 113 | ## Running the demo 114 | You can find the steps to run through the demo script found in the **Documents** folder of this repo. 115 | 116 | Enjoy! 117 | 118 | ## How to sign up for Microsoft Azure 119 | 120 | You need an Azure account to work with this demo code. You can: 121 | 122 | - Open an Azure account for free [Azure subscription](https://azure.com). You get credits that can be used to try out paid Azure services. Even after the credits are used up, you can keep the account and use free Azure services and features, such as the Web Apps feature in Azure App Service. 123 | - [Activate Visual Studio subscriber benefits](https://www.visualstudio.com/products/visual-studio-dev-essentials-vs). Your Visual Studio subscription gives you credits every month that you can use for paid Azure services. 124 | - Not a Visual Studio subscriber? Get a $25 monthly Azure credit by joining [Visual Studio Dev Essentials](https://www.visualstudio.com/products/visual-studio-dev-essentials-vs). 125 | 126 | ## Blogs posts 127 | 128 | Here's links to blog posts related to this project: 129 | 130 | - Xamarin Blog: [Microsoft Connect(); 2016 Recap](https://blog.xamarin.com/microsoft-connect-2016-recap/) 131 | - The Visual Studio Blog: [Announcing the new Visual Studio for Mac](https://blogs.msdn.microsoft.com/visualstudio/2016/11/16/visual-studio-for-mac/) 132 | - The Visual Studio Blog: [Introducing Visual Studio Mobile Center (Preview)](https://blogs.msdn.microsoft.com/visualstudio/2016/11/16/visual-studio-mobile-center/) 133 | - The Visual Studio Blog: [Visual Studio 2017 Release Candidate](https://blogs.msdn.microsoft.com/visualstudio/2016/11/16/visual-studio-2017-rc/) 134 | 135 | ## Clean and Rebuild 136 | If you see build issues when pulling updates from the repo, try cleaning and rebuilding the solution. 137 | 138 | ## Copyright and license 139 | * Code and documentation copyright 2016 Microsoft Corp. Code released under the [MIT license](https://opensource.org/licenses/MIT). 140 | 141 | ## Code of Conduct 142 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 143 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /src/App.xaml: -------------------------------------------------------------------------------- 1 |  7 | 8 | 9 | 10 | 11 | #ffffff 12 | #BFffffff 13 | #3063f5 14 | #BF3063f5 15 | #ffed00 16 | #484f63 17 | #ff5252 18 | #f4f6fa 19 | #3063f5 20 | #ff5e4d 21 | #afb3be 22 | #000000 23 | #0aca91 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /src/App.xaml.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Windows.ApplicationModel; 3 | using Windows.ApplicationModel.Activation; 4 | using Windows.UI.Xaml; 5 | using Windows.UI.Xaml.Controls; 6 | using Windows.UI.Xaml.Navigation; 7 | 8 | namespace BikeSharing.Clients.CogServicesKiosk 9 | { 10 | /// 11 | /// Provides application-specific behavior to supplement the default Application class. 12 | /// 13 | sealed partial class App : Application 14 | { 15 | // TODO add your keys here! 16 | 17 | internal const string FACE_API_SUBSCRIPTION_KEY = ""; 18 | internal const string FACE_API_GROUPID = ""; 19 | 20 | internal const string EMOTION_API_SUBSCRIPTION_KEY = ""; 21 | 22 | internal const string SPEAKER_RECOGNITION_API_SUBSCRIPTION_KEY = ""; 23 | 24 | internal const string LUIS_SUBCRIPTION_KEY = ""; 25 | internal const string LUIS_APP_ID = ""; 26 | 27 | /// 28 | /// Initializes the singleton application object. This is the first line of authored code 29 | /// executed, and as such is the logical equivalent of main() or WinMain(). 30 | /// 31 | public App() 32 | { 33 | this.InitializeComponent(); 34 | this.Suspending += OnSuspending; 35 | } 36 | 37 | /// 38 | /// Invoked when the application is launched normally by the end user. Other entry points 39 | /// will be used such as when the application is launched to open a specific file. 40 | /// 41 | /// Details about the launch request and process. 42 | protected override void OnLaunched(LaunchActivatedEventArgs e) 43 | { 44 | #if DEBUG 45 | if (System.Diagnostics.Debugger.IsAttached) 46 | { 47 | //this.DebugSettings.EnableFrameRateCounter = true; 48 | } 49 | #endif 50 | Frame rootFrame = Window.Current.Content as Frame; 51 | 52 | // Do not repeat app initialization when the Window already has content, 53 | // just ensure that the window is active 54 | if (rootFrame == null) 55 | { 56 | // Create a Frame to act as the navigation context and navigate to the first page 57 | rootFrame = new Frame(); 58 | 59 | rootFrame.NavigationFailed += OnNavigationFailed; 60 | 61 | if (e.PreviousExecutionState == ApplicationExecutionState.Terminated) 62 | { 63 | //TODO: Load state from previously suspended application 64 | } 65 | 66 | // Place the frame in the current Window 67 | Window.Current.Content = rootFrame; 68 | } 69 | 70 | if (e.PrelaunchActivated == false) 71 | { 72 | if (rootFrame.Content == null) 73 | { 74 | // When the navigation stack isn't restored navigate to the first page, 75 | // configuring the new page by passing required information as a navigation 76 | // parameter 77 | rootFrame.Navigate(typeof(MainPage), e.Arguments); 78 | } 79 | // Ensure the current window is active 80 | Window.Current.Activate(); 81 | } 82 | } 83 | 84 | /// 85 | /// Invoked when Navigation to a certain page fails 86 | /// 87 | /// The Frame which failed navigation 88 | /// Details about the navigation failure 89 | void OnNavigationFailed(object sender, NavigationFailedEventArgs e) 90 | { 91 | throw new Exception("Failed to load Page " + e.SourcePageType.FullName); 92 | } 93 | 94 | /// 95 | /// Invoked when application execution is being suspended. Application state is saved 96 | /// without knowing whether the application will be terminated or resumed with the contents 97 | /// of memory still intact. 98 | /// 99 | /// The source of the suspend request. 100 | /// Details about the suspend request. 101 | private void OnSuspending(object sender, SuspendingEventArgs e) 102 | { 103 | var deferral = e.SuspendingOperation.GetDeferral(); 104 | //TODO: Save application state and stop any background activity 105 | deferral.Complete(); 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/Assets/LockScreenLogo.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/LockScreenLogo.scale-200.png -------------------------------------------------------------------------------- /src/Assets/SplashScreen.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/SplashScreen.scale-200.png -------------------------------------------------------------------------------- /src/Assets/Square150x150Logo.scale-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square150x150Logo.scale-100.png -------------------------------------------------------------------------------- /src/Assets/Square150x150Logo.scale-125.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square150x150Logo.scale-125.png -------------------------------------------------------------------------------- /src/Assets/Square150x150Logo.scale-150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square150x150Logo.scale-150.png -------------------------------------------------------------------------------- /src/Assets/Square150x150Logo.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square150x150Logo.scale-200.png -------------------------------------------------------------------------------- /src/Assets/Square150x150Logo.scale-400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square150x150Logo.scale-400.png -------------------------------------------------------------------------------- /src/Assets/Square310x310Logo.scale-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square310x310Logo.scale-100.png -------------------------------------------------------------------------------- /src/Assets/Square310x310Logo.scale-125.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square310x310Logo.scale-125.png -------------------------------------------------------------------------------- /src/Assets/Square310x310Logo.scale-150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square310x310Logo.scale-150.png -------------------------------------------------------------------------------- /src/Assets/Square310x310Logo.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square310x310Logo.scale-200.png -------------------------------------------------------------------------------- /src/Assets/Square310x310Logo.scale-400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square310x310Logo.scale-400.png -------------------------------------------------------------------------------- /src/Assets/Square44x44Logo.scale-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square44x44Logo.scale-100.png -------------------------------------------------------------------------------- /src/Assets/Square44x44Logo.scale-125.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square44x44Logo.scale-125.png -------------------------------------------------------------------------------- /src/Assets/Square44x44Logo.scale-150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square44x44Logo.scale-150.png -------------------------------------------------------------------------------- /src/Assets/Square44x44Logo.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square44x44Logo.scale-200.png -------------------------------------------------------------------------------- /src/Assets/Square44x44Logo.scale-400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square44x44Logo.scale-400.png -------------------------------------------------------------------------------- /src/Assets/Square71x71Logo.scale-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square71x71Logo.scale-100.png -------------------------------------------------------------------------------- /src/Assets/Square71x71Logo.scale-125.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square71x71Logo.scale-125.png -------------------------------------------------------------------------------- /src/Assets/Square71x71Logo.scale-150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square71x71Logo.scale-150.png -------------------------------------------------------------------------------- /src/Assets/Square71x71Logo.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square71x71Logo.scale-200.png -------------------------------------------------------------------------------- /src/Assets/Square71x71Logo.scale-400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/Square71x71Logo.scale-400.png -------------------------------------------------------------------------------- /src/Assets/StoreLogo.scale-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/StoreLogo.scale-100.png -------------------------------------------------------------------------------- /src/Assets/StoreLogo.scale-125.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/StoreLogo.scale-125.png -------------------------------------------------------------------------------- /src/Assets/StoreLogo.scale-150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/StoreLogo.scale-150.png -------------------------------------------------------------------------------- /src/Assets/StoreLogo.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/StoreLogo.scale-200.png -------------------------------------------------------------------------------- /src/Assets/StoreLogo.scale-400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/StoreLogo.scale-400.png -------------------------------------------------------------------------------- /src/Assets/bg_city.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/bg_city.png -------------------------------------------------------------------------------- /src/Assets/bg_micro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/bg_micro.png -------------------------------------------------------------------------------- /src/Assets/bg_ok.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/bg_ok.png -------------------------------------------------------------------------------- /src/Assets/corner_blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/corner_blue.png -------------------------------------------------------------------------------- /src/Assets/corner_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/corner_white.png -------------------------------------------------------------------------------- /src/Assets/ico_micro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/ico_micro.png -------------------------------------------------------------------------------- /src/Assets/ico_ok.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/ico_ok.png -------------------------------------------------------------------------------- /src/Assets/logo_bikesharing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/logo_bikesharing.png -------------------------------------------------------------------------------- /src/Assets/logo_splash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/logo_splash.png -------------------------------------------------------------------------------- /src/Assets/logo_splash.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/logo_splash.scale-200.png -------------------------------------------------------------------------------- /src/Assets/logo_splashBig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/logo_splashBig.png -------------------------------------------------------------------------------- /src/Assets/silhouette01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/silhouette01.png -------------------------------------------------------------------------------- /src/Assets/silhouette02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/BikeSharing360_CognitiveServicesKioskApp/ccc79f028e219f333adc5e5512a76ba4225b5fc8/src/Assets/silhouette02.png -------------------------------------------------------------------------------- /src/BikeSharing.Clients.CogServicesKiosk.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | x86 7 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8} 8 | AppContainerExe 9 | Properties 10 | BikeSharing.Clients.CogServicesKiosk 11 | BikeSharing.Clients.CogServicesKiosk 12 | en-US 13 | UAP 14 | 10.0.14393.0 15 | 10.0.10586.0 16 | 14 17 | 512 18 | {A5A43C5B-DE2A-4C0C-9213-0A381AF9435A};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} 19 | BikeSharing.Clients.CogServicesKiosk_TemporaryKey.pfx 20 | 21 | 22 | true 23 | bin\x86\Debug\ 24 | DEBUG;TRACE;NETFX_CORE;WINDOWS_UWP 25 | ;2008 26 | full 27 | x86 28 | false 29 | prompt 30 | true 31 | 32 | 33 | bin\x86\Release\ 34 | TRACE;NETFX_CORE;WINDOWS_UWP 35 | true 36 | ;2008 37 | pdbonly 38 | x86 39 | false 40 | prompt 41 | true 42 | true 43 | 44 | 45 | true 46 | bin\ARM\Debug\ 47 | DEBUG;TRACE;NETFX_CORE;WINDOWS_UWP 48 | ;2008 49 | full 50 | ARM 51 | false 52 | prompt 53 | true 54 | 55 | 56 | bin\ARM\Release\ 57 | TRACE;NETFX_CORE;WINDOWS_UWP 58 | true 59 | ;2008 60 | pdbonly 61 | ARM 62 | false 63 | prompt 64 | true 65 | true 66 | 67 | 68 | true 69 | bin\x64\Debug\ 70 | DEBUG;TRACE;NETFX_CORE;WINDOWS_UWP 71 | ;2008 72 | full 73 | x64 74 | false 75 | prompt 76 | true 77 | 78 | 79 | bin\x64\Release\ 80 | TRACE;NETFX_CORE;WINDOWS_UWP 81 | true 82 | ;2008 83 | pdbonly 84 | x64 85 | false 86 | prompt 87 | true 88 | true 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | App.xaml 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | MainPage.xaml 106 | 107 | 108 | 109 | 110 | 111 | 112 | Designer 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | MSBuild:Compile 162 | Designer 163 | 164 | 165 | MSBuild:Compile 166 | Designer 167 | 168 | 169 | 170 | 14.0 171 | 172 | 173 | 180 | -------------------------------------------------------------------------------- /src/BikeSharing.Clients.CogServicesKiosk.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 14 4 | VisualStudioVersion = 14.0.25420.1 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BikeSharing.Clients.CogServicesKiosk", "BikeSharing.Clients.CogServicesKiosk.csproj", "{A0A04098-BEDA-47DC-8B70-ACC4D31928C8}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|ARM = Debug|ARM 11 | Debug|x64 = Debug|x64 12 | Debug|x86 = Debug|x86 13 | Release|ARM = Release|ARM 14 | Release|x64 = Release|x64 15 | Release|x86 = Release|x86 16 | EndGlobalSection 17 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 18 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Debug|ARM.ActiveCfg = Debug|ARM 19 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Debug|ARM.Build.0 = Debug|ARM 20 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Debug|ARM.Deploy.0 = Debug|ARM 21 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Debug|x64.ActiveCfg = Debug|x64 22 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Debug|x64.Build.0 = Debug|x64 23 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Debug|x64.Deploy.0 = Debug|x64 24 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Debug|x86.ActiveCfg = Debug|x86 25 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Debug|x86.Build.0 = Debug|x86 26 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Debug|x86.Deploy.0 = Debug|x86 27 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Release|ARM.ActiveCfg = Release|ARM 28 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Release|ARM.Build.0 = Release|ARM 29 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Release|ARM.Deploy.0 = Release|ARM 30 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Release|x64.ActiveCfg = Release|x64 31 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Release|x64.Build.0 = Release|x64 32 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Release|x64.Deploy.0 = Release|x64 33 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Release|x86.ActiveCfg = Release|x86 34 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Release|x86.Build.0 = Release|x86 35 | {A0A04098-BEDA-47DC-8B70-ACC4D31928C8}.Release|x86.Deploy.0 = Release|x86 36 | EndGlobalSection 37 | GlobalSection(SolutionProperties) = preSolution 38 | HideSolutionNode = FALSE 39 | EndGlobalSection 40 | EndGlobal 41 | -------------------------------------------------------------------------------- /src/Converters/ValueToVisibilityConverter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Windows.UI.Xaml; 3 | using Windows.UI.Xaml.Data; 4 | 5 | namespace BikeSharing.Clients.CogServicesKiosk.Converters 6 | { 7 | public class ValueToVisibilityConverter : IValueConverter 8 | { 9 | public object Convert(object value, Type targetType, object parameter, string language) 10 | { 11 | bool returnValue = false; 12 | 13 | if (value is bool) 14 | returnValue = (bool)value; 15 | else if (value is string) 16 | returnValue = !string.IsNullOrWhiteSpace(value.ToString()); 17 | else if (value != null) 18 | returnValue = true; 19 | 20 | return returnValue ? Visibility.Visible : Visibility.Collapsed; 21 | } 22 | 23 | public object ConvertBack(object value, Type targetType, object parameter, string language) 24 | { 25 | throw new NotImplementedException(); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/Data/AudioRecorder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | using Windows.Media.Capture; 6 | using Windows.Media.MediaProperties; 7 | using Windows.Storage.Streams; 8 | 9 | namespace BikeSharing.Clients.CogServicesKiosk.Data 10 | { 11 | /// 12 | /// Captures audio from the microphone for a specified amount of time. 13 | /// 14 | public class AudioRecorder 15 | { 16 | 17 | public AudioRecorder() 18 | { 19 | } 20 | 21 | /// 22 | /// Captures audio from the microphone for the specified amount of time. 23 | /// 24 | /// 25 | /// Amount of time to record. 26 | /// 27 | public async Task RecordAsync(CancellationToken ct, TimeSpan timeToRecord) 28 | { 29 | MediaCaptureInitializationSettings settings = new MediaCaptureInitializationSettings 30 | { 31 | StreamingCaptureMode = StreamingCaptureMode.Audio 32 | }; 33 | 34 | MediaCapture audioCapture = new MediaCapture(); 35 | await audioCapture.InitializeAsync(settings); 36 | 37 | var outProfile = MediaEncodingProfile.CreateWav(AudioEncodingQuality.Medium); 38 | outProfile.Audio = AudioEncodingProperties.CreatePcm(16000, 1, 16); 39 | 40 | var buffer = new InMemoryRandomAccessStream(); 41 | await audioCapture.StartRecordToStreamAsync(outProfile, buffer); 42 | await Task.Delay(timeToRecord, ct); 43 | await audioCapture.StopRecordAsync(); 44 | 45 | IRandomAccessStream audio = null; 46 | try 47 | { 48 | audio = buffer.CloneStream(); 49 | return this.FixWavPcmStream(audio); 50 | } 51 | finally 52 | { 53 | audio.Dispose(); 54 | } 55 | } 56 | 57 | // WAV catpured by UWP MediaCapture is not recognized by the Speaker Recognition service. 58 | // Applying the fix from 59 | // https://mtaulty.com/2016/02/10/project-oxfordspeaker-verification-from-a-windows-10uwp-app/ 60 | private Stream FixWavPcmStream(IInputStream inputStream) 61 | { 62 | var netStream = inputStream.AsStreamForRead(); 63 | var bits = new byte[netStream.Length]; 64 | netStream.Read(bits, 0, bits.Length); 65 | 66 | var pcmFileLength = BitConverter.ToInt32(bits, 4); 67 | 68 | pcmFileLength -= 36; 69 | 70 | for (int i = 0; i < 12; i++) 71 | bits[i + 36] = bits[i]; 72 | 73 | var newLengthBits = BitConverter.GetBytes(pcmFileLength); 74 | newLengthBits.CopyTo(bits, 40); 75 | 76 | MemoryStream stream = new MemoryStream(bits, 36, bits.Length - 36); 77 | return stream; 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/Data/SpeechToText.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using Windows.Media.SpeechRecognition; 6 | 7 | namespace BikeSharing.Clients.CogServicesKiosk.Data 8 | { 9 | public class SpeechToText : IDisposable 10 | { 11 | #region Variables 12 | 13 | uint MaxRecognitionResultAlternates = 4; 14 | private SpeechRecognizer _recognizer; 15 | private int _recognizerInitialSilenceTimeOutInSeconds; 16 | private int _recognizerBabbleTimeoutInSeconds; 17 | private int _recognizerEndSilenceTimeoutInSeconds; 18 | private string _languageName; 19 | 20 | #endregion 21 | 22 | #region Events 23 | 24 | public event EventHandler OnHypothesis; 25 | public event EventHandler CapturingStarted; 26 | public event EventHandler CapturingEnded; 27 | 28 | #endregion 29 | 30 | #region Constructors 31 | 32 | public SpeechToText( 33 | string languageName = "en-Us", 34 | int recognizerInitialSelenceTimeOutInSeconds = 6, 35 | int recognizerBabbleTimeoutInSeconds = 0, 36 | int recognizerEndSilenceTimeoutInSeconds = 3) 37 | { 38 | this._languageName = languageName; 39 | this._recognizerInitialSilenceTimeOutInSeconds = recognizerInitialSelenceTimeOutInSeconds; 40 | this._recognizerBabbleTimeoutInSeconds = recognizerBabbleTimeoutInSeconds; 41 | this._recognizerEndSilenceTimeoutInSeconds = recognizerEndSilenceTimeoutInSeconds; 42 | } 43 | 44 | #endregion 45 | 46 | #region Methods 47 | 48 | public async Task InitializeRecognizerAsync() 49 | { 50 | Debug.WriteLine("[Speech to Text]: initializing Speech Recognizer..."); 51 | var language = new Windows.Globalization.Language(_languageName); 52 | _recognizer = new SpeechRecognizer(language); 53 | // Set timeout settings. 54 | _recognizer.Timeouts.InitialSilenceTimeout = TimeSpan.FromSeconds(_recognizerInitialSilenceTimeOutInSeconds); 55 | _recognizer.Timeouts.BabbleTimeout = TimeSpan.FromSeconds(_recognizerBabbleTimeoutInSeconds); 56 | _recognizer.Timeouts.EndSilenceTimeout = TimeSpan.FromSeconds(_recognizerEndSilenceTimeoutInSeconds); 57 | // Set UI text 58 | _recognizer.UIOptions.AudiblePrompt = "Say what you want to do..."; 59 | 60 | if (!this.IsOffline()) 61 | { 62 | // This requires internet connection 63 | SpeechRecognitionTopicConstraint topicConstraint = new SpeechRecognitionTopicConstraint(SpeechRecognitionScenario.Dictation, "Development"); 64 | _recognizer.Constraints.Add(topicConstraint); 65 | } 66 | else 67 | { 68 | // In case of network issue 69 | string[] responses = 70 | { 71 | "I would like to rent a bike", 72 | "I want to rent a bike", 73 | "I'd like to rent a bike", 74 | "rent a bike", 75 | "I would like to rent a bicycle", 76 | "I want to rent a bicycle", 77 | "I'd like to rent a bicycle", 78 | "rent a bicycle" 79 | }; 80 | 81 | // Add a list constraint to the recognizer. 82 | var listConstraint = new SpeechRecognitionListConstraint(responses, "rentBikePhrases"); 83 | _recognizer.Constraints.Add(listConstraint); 84 | } 85 | 86 | SpeechRecognitionCompilationResult result = await _recognizer.CompileConstraintsAsync(); // Required 87 | 88 | if (result.Status != SpeechRecognitionResultStatus.Success) 89 | { 90 | Debug.WriteLine("[Speech to Text]: Grammar Compilation Failed: " + result.Status.ToString()); 91 | return false; 92 | } 93 | 94 | _recognizer.HypothesisGenerated += Recognizer_HypothesisGenerated; 95 | _recognizer.StateChanged += Recognizer_StateChanged; 96 | _recognizer.ContinuousRecognitionSession.ResultGenerated += (s, e) => { Debug.WriteLine($"[Speech to Text]: recognizer results: {e.Result.Text}, {e.Result.RawConfidence.ToString()}, {e.Result.Confidence.ToString()}"); }; 97 | Debug.WriteLine("[Speech to Text]: done initializing Speech Recognizer"); 98 | return true; 99 | } 100 | 101 | private bool IsOffline() 102 | { 103 | var profile = Windows.Networking.Connectivity.NetworkInformation.GetInternetConnectionProfile(); 104 | bool isConnected = profile?.GetNetworkConnectivityLevel() == Windows.Networking.Connectivity.NetworkConnectivityLevel.InternetAccess; 105 | return !isConnected; 106 | } 107 | 108 | private void Recognizer_HypothesisGenerated(SpeechRecognizer sender, SpeechRecognitionHypothesisGeneratedEventArgs args) 109 | { 110 | Debug.WriteLine("[Speech to Text]: ********* Partial Result *********"); 111 | Debug.WriteLine($"[Speech to Text]: {args.Hypothesis.Text}"); 112 | Debug.WriteLine("[Speech to Text]: "); 113 | 114 | this.OnHypothesis?.Invoke(this, args.Hypothesis.Text); 115 | } 116 | 117 | private void Recognizer_StateChanged(SpeechRecognizer sender, SpeechRecognizerStateChangedEventArgs args) 118 | { 119 | Debug.WriteLine($"[Speech to Text]: recognizer state changed to: {args.State.ToString()}"); 120 | 121 | switch (args.State) 122 | { 123 | case SpeechRecognizerState.Capturing: 124 | this.CapturingStarted?.Invoke(this, null); 125 | break; 126 | case SpeechRecognizerState.Processing: 127 | case SpeechRecognizerState.Idle: 128 | this.CapturingEnded?.Invoke(this, null); 129 | break; 130 | default: 131 | break; 132 | } 133 | } 134 | 135 | public async Task GetTextFromSpeechAsync(bool withUI = false) 136 | { 137 | if (_recognizer == null) 138 | { 139 | await InitializeRecognizerAsync(); 140 | } 141 | 142 | SpeechRecognitionResult recognition = null; 143 | if (withUI) 144 | { 145 | recognition = await _recognizer.RecognizeWithUIAsync(); 146 | } 147 | else 148 | { 149 | recognition = await _recognizer.RecognizeAsync(); 150 | } 151 | 152 | if (recognition.Status == SpeechRecognitionResultStatus.Success && 153 | recognition.Confidence != SpeechRecognitionConfidence.Rejected) 154 | { 155 | Debug.WriteLine($"[Speech to Text]: result: {recognition.Text}, {recognition.RawConfidence.ToString()}, {recognition.Confidence.ToString()}"); 156 | var alternativeResults = recognition.GetAlternates(MaxRecognitionResultAlternates); 157 | 158 | foreach (var r in alternativeResults) 159 | { 160 | Debug.WriteLine($"[Speech to Text]: alternative: {r.Text}, {r.RawConfidence.ToString()}, {r.Confidence.ToString()}"); 161 | } 162 | 163 | var topResult = alternativeResults.Where(r => r.Confidence == SpeechRecognitionConfidence.High).FirstOrDefault(); 164 | if (topResult != null) return topResult.Text; 165 | 166 | topResult = alternativeResults.Where(r => r.Confidence == SpeechRecognitionConfidence.Medium).FirstOrDefault(); 167 | if (topResult != null) return topResult.Text; 168 | 169 | topResult = alternativeResults.Where(r => r.Confidence == SpeechRecognitionConfidence.Low).FirstOrDefault(); 170 | if (topResult != null) return topResult.Text; 171 | } 172 | 173 | return string.Empty; 174 | } 175 | 176 | public async Task Stop() 177 | { 178 | try 179 | { 180 | await _recognizer.StopRecognitionAsync(); 181 | } 182 | catch (Exception ex) 183 | { 184 | Debug.WriteLine($"[Speech to Text]: an error occured while stoping Speech Recognition session: {ex.ToString()}"); 185 | } 186 | } 187 | 188 | public void Dispose() 189 | { 190 | if (_recognizer != null) 191 | { 192 | _recognizer.HypothesisGenerated -= Recognizer_HypothesisGenerated; 193 | _recognizer.Dispose(); 194 | _recognizer = null; 195 | } 196 | } 197 | 198 | #endregion 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /src/Data/TextToSpeech.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.Threading.Tasks; 4 | using Windows.Media.SpeechSynthesis; 5 | using System.Linq; 6 | using Windows.UI.Xaml; 7 | using Windows.UI.Xaml.Controls; 8 | using Windows.UI.Core; 9 | 10 | namespace BikeSharing.Clients.CogServicesKiosk.Data 11 | { 12 | public class TextToSpeech 13 | { 14 | private CoreDispatcher _dispatcher; 15 | private SpeechSynthesizer _synthesizer = new SpeechSynthesizer(); 16 | private MediaElement _mediaElement = new MediaElement(); 17 | private TaskCompletionSource _tcs; 18 | 19 | public TextToSpeech(MediaElement media, CoreDispatcher dispatcher) 20 | { 21 | _synthesizer = new SpeechSynthesizer(); 22 | 23 | // Get all of the installed voices. 24 | var voices = SpeechSynthesizer.AllVoices; 25 | var ziraVoice = voices.FirstOrDefault(v => v.DisplayName.Contains("Zira")); 26 | if (ziraVoice == null) ziraVoice = voices.First(); 27 | _synthesizer.Voice = ziraVoice; 28 | 29 | _dispatcher = dispatcher; 30 | 31 | _mediaElement = media; 32 | _mediaElement.MediaEnded += _mediaElement_MediaEnded; 33 | _mediaElement.MediaFailed += _mediaElement_MediaFailed; 34 | } 35 | 36 | ~TextToSpeech() 37 | { 38 | _mediaElement.MediaEnded -= _mediaElement_MediaEnded; 39 | _mediaElement.MediaFailed -= _mediaElement_MediaFailed; 40 | _mediaElement = null; 41 | } 42 | 43 | private void _mediaElement_MediaEnded(object sender, RoutedEventArgs e) 44 | { 45 | _tcs.SetResult(true); 46 | Debug.WriteLine("Done playing audio."); 47 | } 48 | 49 | private void _mediaElement_MediaFailed(object sender, ExceptionRoutedEventArgs e) 50 | { 51 | Debug.WriteLine("Failed to play audio."); 52 | _tcs.SetResult(true); 53 | } 54 | 55 | public async Task SpeakAsync(string text) 56 | { 57 | Debug.WriteLine($"Playing audio for {text}..."); 58 | // Create a stream from the text 59 | var synthesizedStream = await _synthesizer.SynthesizeTextToStreamAsync(text); 60 | 61 | // Set the stream to the media element to be played by the platform 62 | _mediaElement.SetSource(synthesizedStream, synthesizedStream.ContentType); 63 | 64 | // Use a TaskCompletionSource to be triggered to complete once the media element is done playing the sound. 65 | // This is used because the media element doesn't have async/await support to tell you when a media is completed playing. 66 | _tcs = new TaskCompletionSource(); 67 | 68 | if (_dispatcher.HasThreadAccess) 69 | _mediaElement.Play(); 70 | else 71 | await _dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => _mediaElement.Play()); 72 | 73 | // Await the TaskCompletionSource and wait for it to be triggered by the _mediaElement_MediaEnded event. 74 | await _tcs.Task; 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/Data/UserLookupService.cs: -------------------------------------------------------------------------------- 1 | using BikeSharing.Clients.CogServicesKiosk.Models; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | 8 | namespace BikeSharing.Clients.CogServicesKiosk.Data 9 | { 10 | public class UserLookupService 11 | { 12 | private List _profiles = new List(); 13 | 14 | private static UserLookupService _instance; 15 | public static UserLookupService Instance 16 | { 17 | get 18 | { 19 | if (_instance == null) 20 | _instance = new UserLookupService(); 21 | return _instance; 22 | 23 | } 24 | } 25 | 26 | private UserLookupService() 27 | { 28 | // To add a new user profile, add a UserProfile object to the _profiles list. 29 | // This in-memory "database" is for demo purposes only. In the real world, you'd 30 | // call your own users web service or a database directly to find a user. 31 | // You'll want to add columns to your users table to store the Cognitive Services 32 | // face profile ID for face verification for each person in the table as well as 33 | // their voice profile ID if you plan on incorporating voice verification. 34 | _profiles.Add(new UserProfile() { Id = 100, FaceProfileId = new Guid("9db14edb-d2a1-4753-b273-7822a7cfb2af"), VoiceProfileId = new Guid("ecb1bb57-1b8c-48c5-9437-4bdd1312d899"), FirstName = "Mohammed", LastName = "Adenwala", VoiceSecretPhrase = "Apple juice tastes funny after toothpaste" }); 35 | } 36 | 37 | /// 38 | /// Searches for a user with the specified Cognitive Services Face Verfication ID 39 | /// 40 | /// 41 | /// 42 | /// 43 | public async Task GetUserByFaceProfileID(CancellationToken ct, Guid id) 44 | { 45 | // Simulate delay that would be caused by network roundtrip to database or web service you'd have in the real world 46 | await Task.Delay(2000, ct); 47 | 48 | // Search the local in-memory list of profiles for a matching profile 49 | return _profiles.FirstOrDefault(f => f.FaceProfileId == id); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/Extensions/SoftwareBitmapExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading.Tasks; 4 | using Windows.Graphics.Imaging; 5 | using Windows.Storage.Streams; 6 | 7 | public static class SoftwareBitmapExtensions 8 | { 9 | /// 10 | /// Converts this SoftwareBitmap instance to a Stream. 11 | /// 12 | /// 13 | /// 14 | public static async Task AsStream(this SoftwareBitmap softwareBitmap) 15 | { 16 | var stream = new InMemoryRandomAccessStream(); 17 | 18 | BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.PngEncoderId, stream); 19 | encoder.SetSoftwareBitmap(softwareBitmap); 20 | encoder.BitmapTransform.ScaledWidth = (uint)softwareBitmap.PixelWidth; 21 | encoder.BitmapTransform.ScaledHeight = (uint)softwareBitmap.PixelHeight; 22 | encoder.BitmapTransform.InterpolationMode = BitmapInterpolationMode.Fant; 23 | await encoder.FlushAsync(); 24 | 25 | return stream.AsStreamForRead(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/MainPage.xaml: -------------------------------------------------------------------------------- 1 |  9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 59 | 60 | 64 | 68 | 69 | 70 | 71 | 72 | 78 | 82 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 97 | 104 | 105 | 108 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | -------------------------------------------------------------------------------- /src/MainPage.xaml.cs: -------------------------------------------------------------------------------- 1 | using BikeSharing.Clients.CogServicesKiosk.Data; 2 | using BikeSharing.Clients.CogServicesKiosk.Models; 3 | using Microsoft.Cognitive.LUIS; 4 | using Microsoft.ProjectOxford.Emotion; 5 | using Microsoft.ProjectOxford.Face; 6 | using Microsoft.ProjectOxford.Face.Contract; 7 | using Microsoft.ProjectOxford.SpeakerRecognition; 8 | using System; 9 | using System.Collections.Generic; 10 | using System.ComponentModel; 11 | using System.Diagnostics; 12 | using System.IO; 13 | using System.Linq; 14 | using System.Runtime.CompilerServices; 15 | using System.Threading; 16 | using System.Threading.Tasks; 17 | using Windows.Devices.Enumeration; 18 | using Windows.Graphics.Display; 19 | using Windows.Graphics.Imaging; 20 | using Windows.Media; 21 | using Windows.Media.Capture; 22 | using Windows.Media.MediaProperties; 23 | using Windows.System.Display; 24 | using Windows.UI; 25 | using Windows.UI.Core; 26 | using Windows.UI.Xaml; 27 | using Windows.UI.Xaml.Controls; 28 | using Windows.UI.Xaml.Media; 29 | using Windows.UI.Xaml.Navigation; 30 | using Windows.UI.Xaml.Shapes; 31 | 32 | namespace BikeSharing.Clients.CogServicesKiosk 33 | { 34 | public sealed partial class MainPage : Page, INotifyPropertyChanged 35 | { 36 | #region Variables 37 | 38 | private const int MAX_SESSION_TIME_WITH_NO_FACE = 5; 39 | 40 | public event PropertyChangedEventHandler PropertyChanged; 41 | 42 | private DisplayRequest _displayRequest; 43 | private MediaCapture _mediaCapture; 44 | private CancellationTokenSource _ctsVideoMonitor; 45 | private CancellationTokenSource _customerSessionCTS; 46 | private Task _faceMonitoringTask = null; 47 | private TextToSpeech _tts; 48 | 49 | private SpeechToText _speechToText; 50 | private string _SpeechToTextLanguageName = "en-US"; 51 | private int _SpeechToTextInitialSilenceTimeoutInSeconds = 6; 52 | private int _SpeechToTextBabbleTimeoutInSeconds = 0; 53 | private int _SpeechToTextEndSilenceTimeoutInSeconds = 3; 54 | 55 | private FaceAttributeType[] _faceAttributesToTrack = null; 56 | 57 | #endregion 58 | 59 | #region Properties 60 | 61 | private Guid _trackedFaceID = Guid.Empty; 62 | /// 63 | /// Gets or sets the ID of the face being tracked. 64 | /// 65 | public Guid TrackedFaceID 66 | { 67 | get { return _trackedFaceID; } 68 | private set 69 | { 70 | if (this.SetProperty(ref _trackedFaceID, value)) 71 | this.User = null; 72 | } 73 | } 74 | 75 | private Face _trackedFace = null; 76 | /// 77 | /// Gets or sets an instance of the face object representing the face in front of the camera. 78 | /// 79 | public Face TrackedFace 80 | { 81 | get { return _trackedFace; } 82 | private set { this.SetProperty(ref _trackedFace, value); } 83 | } 84 | 85 | private UserProfile _User; 86 | /// 87 | /// Gets or set the user profile instance of a customer in front of the camera. 88 | /// 89 | public UserProfile User 90 | { 91 | get { return _User; } 92 | private set { this.SetProperty(ref _User, value); } 93 | } 94 | 95 | private bool _ShowMicrophone; 96 | /// 97 | /// Shows or hides the microphone icon on the UI. 98 | /// 99 | public bool ShowMicrophone 100 | { 101 | get { return _ShowMicrophone; } 102 | private set { this.SetProperty(ref _ShowMicrophone, value); } 103 | } 104 | 105 | private string _MicrophoneText; 106 | /// 107 | /// Shows or hides text next to the microphone icon on the UI. 108 | /// 109 | public string MicrophoneText 110 | { 111 | get { return _MicrophoneText; } 112 | private set { this.SetProperty(ref _MicrophoneText, value); } 113 | } 114 | 115 | private string _KioskMessage; 116 | /// 117 | /// Shows or hides the text the kiosk needs to show the the customer on the UI. 118 | /// 119 | public string KioskMessage 120 | { 121 | get { return _KioskMessage; } 122 | private set { this.SetProperty(ref _KioskMessage, value); } 123 | } 124 | 125 | private string _HeaderText; 126 | /// 127 | /// Shows or hides the header message on the UI. 128 | /// 129 | public string HeaderText 130 | { 131 | get { return _HeaderText; } 132 | private set { this.SetProperty(ref _HeaderText, value); } 133 | } 134 | 135 | private string _CustomerMessage; 136 | /// 137 | /// Shows or hides the text spoken by the customer on the UI. 138 | /// 139 | public string CustomerMessage 140 | { 141 | get { return _CustomerMessage; } 142 | private set { this.SetProperty(ref _CustomerMessage, value); } 143 | } 144 | 145 | private bool _ShowVoiceVerificationPassedIcon; 146 | /// 147 | /// Shows or hides the voice verification passed UI icon. 148 | /// 149 | public bool ShowVoiceVerificationPassedIcon 150 | { 151 | get { return _ShowVoiceVerificationPassedIcon; } 152 | private set { this.SetProperty(ref _ShowVoiceVerificationPassedIcon, value); } 153 | } 154 | 155 | #endregion 156 | 157 | #region Constructor 158 | 159 | public MainPage() 160 | { 161 | this.InitializeComponent(); 162 | } 163 | 164 | #endregion 165 | 166 | #region Methods 167 | 168 | #region NavigatedTo / NavigatedFrom 169 | 170 | protected async override void OnNavigatedTo(NavigationEventArgs e) 171 | { 172 | // Set the page to standby mode to start off with 173 | this.GoToVisualState("Standby"); 174 | 175 | _ctsVideoMonitor = new CancellationTokenSource(); 176 | 177 | // Initialize the text-to-speech class instance. It uses the MediaElement on the page to play sounds thus you need to pass it a reference to. 178 | _tts = new TextToSpeech(this.media, this.Dispatcher); 179 | 180 | // Initialize the speech-to-text class instance which is used to recognize speech commands by the customer 181 | _speechToText = new SpeechToText( 182 | _SpeechToTextLanguageName, 183 | _SpeechToTextInitialSilenceTimeoutInSeconds, 184 | _SpeechToTextBabbleTimeoutInSeconds, 185 | _SpeechToTextEndSilenceTimeoutInSeconds); 186 | await _speechToText.InitializeRecognizerAsync(); 187 | _speechToText.OnHypothesis += _speechToText_OnHypothesis; 188 | _speechToText.CapturingStarted += _speechToText_CapturingStarted; 189 | _speechToText.CapturingEnded += _speechToText_CapturingEnded; 190 | 191 | // Landscape preference set to landscape 192 | DisplayInformation.AutoRotationPreferences = DisplayOrientations.Landscape; 193 | 194 | // Keeps the screen alive i.e. prevents screen from going to sleep 195 | _displayRequest = new DisplayRequest(); 196 | _displayRequest.RequestActive(); 197 | 198 | // Find all the video cameras on the device 199 | var cameras = await DeviceInformation.FindAllAsync(DeviceClass.VideoCapture); 200 | 201 | // Choose the first externally plugged in camera found 202 | var preferredCamera = cameras.FirstOrDefault(deviceInfo => deviceInfo.EnclosureLocation == null); 203 | 204 | // If no external camera, choose the front facing camera ELSE choose the first available camera found 205 | if (preferredCamera == null) 206 | preferredCamera = cameras.FirstOrDefault(deviceInfo => deviceInfo.EnclosureLocation?.Panel == Windows.Devices.Enumeration.Panel.Front) ?? cameras.FirstOrDefault(); 207 | 208 | // No camera found on device 209 | if (preferredCamera == null) 210 | { 211 | Debug.WriteLine("No camera found on device!"); 212 | return; 213 | } 214 | 215 | // Initialize and start the camera video stream into the app preview window 216 | _mediaCapture = new MediaCapture(); 217 | await _mediaCapture.InitializeAsync(new MediaCaptureInitializationSettings() 218 | { 219 | StreamingCaptureMode = StreamingCaptureMode.Video, 220 | VideoDeviceId = preferredCamera.Id 221 | }); 222 | videoPreview.Source = _mediaCapture; 223 | await _mediaCapture.StartPreviewAsync(); 224 | 225 | // Ensure state is clear 226 | await this.EndSessionAsync(); 227 | 228 | // Initiate monitoring of the video stream for faces 229 | _faceMonitoringTask = this.FaceMonitoringAsync(_ctsVideoMonitor.Token); 230 | 231 | base.OnNavigatedTo(e); 232 | } 233 | 234 | protected async override void OnNavigatedFrom(NavigationEventArgs e) 235 | { 236 | // Cancel all tasks that are running 237 | _ctsVideoMonitor.Cancel(); 238 | 239 | // Wait for the main video monitoring task to complete 240 | await _faceMonitoringTask; 241 | 242 | // Allows the screen to go to sleep again when you leave this page 243 | _displayRequest.RequestRelease(); 244 | _displayRequest = null; 245 | 246 | // Stop and clean up the video feed 247 | await _mediaCapture.StopPreviewAsync(); 248 | videoPreview.Source = null; 249 | _mediaCapture.Dispose(); 250 | _mediaCapture = null; 251 | 252 | base.OnNavigatedFrom(e); 253 | } 254 | 255 | #endregion 256 | 257 | #region Video Stream Monitoring 258 | 259 | /// 260 | /// Looping task which monitors the video feed. 261 | /// 262 | /// 263 | /// 264 | private async Task FaceMonitoringAsync(CancellationToken ct) 265 | { 266 | DateTime lastFaceSeen = DateTime.MinValue; 267 | 268 | // Continue looping / watching the video stream until this page asks to stop via the cancellation token 269 | while (ct.IsCancellationRequested == false) 270 | { 271 | try 272 | { 273 | if (_mediaCapture.CameraStreamState != Windows.Media.Devices.CameraStreamState.Streaming) 274 | continue; 275 | 276 | // Capture a frame from the video feed 277 | var mediaProperties = _mediaCapture.VideoDeviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties; 278 | var videoFrame = new VideoFrame(BitmapPixelFormat.Rgba16, (int)mediaProperties.Width, (int)mediaProperties.Height); 279 | await _mediaCapture.GetPreviewFrameAsync(videoFrame); 280 | 281 | // Detect faces in frame bitmap 282 | var faces = await this.FaceDetectionAsync(videoFrame.SoftwareBitmap, _faceAttributesToTrack); 283 | 284 | if (faces?.Any() == true) 285 | { 286 | // A face was found, save a reference to the tracke face 287 | var face = faces.First(); 288 | this.TrackedFace = face; 289 | 290 | // Save the time the last face was seen...used to detect if someone walked away from the view of the camera 291 | lastFaceSeen = DateTime.Now; 292 | 293 | // If the previous frame didnt have a face, that means this is a new customer 294 | //if (face.FaceId != this.TrackedFaceID) 295 | if (this.TrackedFaceID == Guid.Empty) 296 | { 297 | await this.EndSessionAsync(); 298 | await this.StartSessionAsync(face.FaceId); 299 | } 300 | } 301 | else if (lastFaceSeen.AddSeconds(MAX_SESSION_TIME_WITH_NO_FACE) < DateTime.Now) 302 | { 303 | // There has been no face seen in view of the camera for the alloted period of time. Assume the customer abandoned the session and reset. 304 | await this.EndSessionAsync(); 305 | } 306 | } 307 | catch(Exception ex) 308 | { 309 | Debug.WriteLine("Error analyzing video frame: " + ex.ToString()); 310 | } 311 | } 312 | } 313 | 314 | #endregion 315 | 316 | #region Cognitive Services - Face Detection 317 | 318 | /// 319 | /// Detects faces in an instance of a SoftwareBitmap object representing a frame from a video feed. 320 | /// 321 | /// Image from a frame from a video feed. 322 | /// Array of FaceAttributeType enum objects used to specify which facial features should be analyzed. 323 | /// 324 | private async Task FaceDetectionAsync(SoftwareBitmap bitmap, params FaceAttributeType[] features) 325 | { 326 | // Convert video frame image to a stream 327 | var stream = await bitmap.AsStream(); 328 | 329 | // Cognitive Services Face API client from the Nuget package 330 | var client = new FaceServiceClient(App.FACE_API_SUBSCRIPTION_KEY); 331 | 332 | // Ask Cognitive Services to analyze the picture and determine face attributes as specified in array 333 | var faces = await client.DetectAsync( 334 | imageStream: stream, 335 | returnFaceId: true, 336 | returnFaceLandmarks: false, 337 | returnFaceAttributes: features 338 | ); 339 | 340 | // Remove previous faces on UI canvas 341 | this.ClearFacesOnUI(); 342 | 343 | // Video feed is probably a different resolution than the actual window size, so scale the sizes of each face 344 | double widthScale = bitmap.PixelWidth / facesCanvas.ActualWidth; 345 | double heightScale = bitmap.PixelHeight / facesCanvas.ActualHeight; 346 | 347 | // Draw a box for each face detected w/ text of face features 348 | foreach (var face in faces) 349 | this.DrawFaceOnUI(widthScale, heightScale, face); 350 | 351 | return faces; 352 | } 353 | 354 | /// 355 | /// Draws a face boxe on the UI 356 | /// 357 | /// 358 | /// 359 | /// 360 | private void DrawFaceOnUI(double widthScale, double heightScale, Microsoft.ProjectOxford.Face.Contract.Face face) 361 | { 362 | try 363 | { 364 | Rectangle box = new Rectangle(); 365 | box.Width = (uint)(face.FaceRectangle.Width / widthScale); 366 | box.Height = (uint)(face.FaceRectangle.Height / heightScale); 367 | box.Fill = new SolidColorBrush(Colors.Transparent); 368 | box.Stroke = new SolidColorBrush(Colors.Lime); 369 | box.StrokeThickness = 2; 370 | box.Margin = new Thickness((uint)(face.FaceRectangle.Left / widthScale), (uint)(face.FaceRectangle.Top / heightScale), 0, 0); 371 | facesCanvas.Children.Add(box); 372 | 373 | // Add face attributes found 374 | var tb = new TextBlock(); 375 | tb.Foreground = new SolidColorBrush(Colors.Lime); 376 | tb.Padding = new Thickness(4); 377 | tb.Margin = new Thickness((uint)(face.FaceRectangle.Left / widthScale), (uint)(face.FaceRectangle.Top / heightScale), 0, 0); 378 | 379 | if (face.FaceAttributes?.Age > 0) 380 | tb.Text += "Age: " + face.FaceAttributes.Age + Environment.NewLine; 381 | 382 | if (!string.IsNullOrEmpty(face.FaceAttributes?.Gender)) 383 | tb.Text += "Gender: " + face.FaceAttributes.Gender + Environment.NewLine; 384 | 385 | if (face.FaceAttributes?.Smile > 0) 386 | tb.Text += "Smile: " + face.FaceAttributes.Smile + Environment.NewLine; 387 | 388 | if(face.FaceAttributes != null && face.FaceAttributes.Glasses != Microsoft.ProjectOxford.Face.Contract.Glasses.NoGlasses) 389 | tb.Text += "Glasses: " + face.FaceAttributes?.Glasses + Environment.NewLine; 390 | 391 | if (face.FaceAttributes?.FacialHair != null) 392 | { 393 | tb.Text += "Beard: " + face.FaceAttributes.FacialHair.Beard + Environment.NewLine; 394 | tb.Text += "Moustache: " + face.FaceAttributes.FacialHair.Moustache + Environment.NewLine; 395 | tb.Text += "Sideburns: " + face.FaceAttributes.FacialHair.Sideburns + Environment.NewLine; 396 | } 397 | 398 | facesCanvas.Children.Add(tb); 399 | } 400 | catch(Exception ex) 401 | { 402 | this.Log("Failure during DrawFaceOnUI()", ex); 403 | } 404 | } 405 | 406 | /// 407 | /// Draws a collection of face boxes on the UI 408 | /// 409 | /// 410 | /// 411 | /// 412 | private void DrawFacesOnUI(int frameWidth, int frameHeight, Microsoft.ProjectOxford.Face.Contract.Face[] faces) 413 | { 414 | this.ClearFacesOnUI(); 415 | 416 | if (faces == null) 417 | return; 418 | 419 | // Video feed is probably a different resolution than the actual window size, so scale the sizes of each face 420 | double widthScale = frameWidth / facesCanvas.ActualWidth; 421 | double heightScale = frameHeight / facesCanvas.ActualHeight; 422 | 423 | // Draw each face 424 | foreach (var face in faces) 425 | this.DrawFaceOnUI(widthScale, heightScale, face); 426 | } 427 | 428 | /// 429 | /// Clears face boxes on the UI 430 | /// 431 | private void ClearFacesOnUI() 432 | { 433 | facesCanvas.Children.Clear(); 434 | } 435 | 436 | private async Task IsCustomerSmilingAsync(SoftwareBitmap bitmap) 437 | { 438 | // Convert video frame image to a stream 439 | var stream = await bitmap.AsStream(); 440 | 441 | // Call Cognitive Services Face API to look for identity candidates in the bitmap image 442 | var client = new FaceServiceClient(App.FACE_API_SUBSCRIPTION_KEY); 443 | 444 | // Ask Cognitive Services to also analyze the picture for smiles on the face 445 | var faces = await client.DetectAsync( 446 | imageStream: stream, 447 | returnFaceId: true, 448 | returnFaceLandmarks: false, 449 | returnFaceAttributes: new FaceAttributeType[] { FaceAttributeType.Smile } 450 | ); 451 | 452 | // If a face was found, check to see if the confidence of the smile is at least 75% 453 | if (faces?.Any() == true) 454 | return faces[0].FaceAttributes.Smile > .75; 455 | else 456 | return false; 457 | } 458 | 459 | #endregion 460 | 461 | #region Cognitive Services - Face Verification 462 | 463 | private async Task FaceVerificationAsync(CancellationToken ct, params Guid[] faceIDs) 464 | { 465 | if (faceIDs == null || faceIDs.Length == 0) 466 | return null; 467 | 468 | // Call Cognitive Services Face API to look for identity candidates in the bitmap image 469 | FaceServiceClient client = new FaceServiceClient(App.FACE_API_SUBSCRIPTION_KEY); 470 | var identityResults = await client.IdentifyAsync(App.FACE_API_GROUPID, faceIDs, confidenceThreshold: 0.6f); 471 | 472 | ct.ThrowIfCancellationRequested(); 473 | 474 | // Get the candidate with the highest confidence or null 475 | var candidate = identityResults.FirstOrDefault()?.Candidates?.OrderByDescending(o => o.Confidence).FirstOrDefault(); 476 | 477 | // If candidate found, take the face ID and lookup in our customer database 478 | if (candidate != null) 479 | return await UserLookupService.Instance.GetUserByFaceProfileID(ct, candidate.PersonId); 480 | else 481 | return null; 482 | } 483 | 484 | #endregion 485 | 486 | #region Cognitive Services - Emotion Detection 487 | 488 | private async Task EmotionDetectionAsync(SoftwareBitmap bitmap) 489 | { 490 | // Convert video frame image to a stream 491 | var stream = await bitmap.AsStream(); 492 | 493 | // Use the Emotion API nuget package to access to the Cognitive Services Emotions service 494 | var client = new EmotionServiceClient(App.EMOTION_API_SUBSCRIPTION_KEY); 495 | 496 | // Pass the video frame image as a stream to the Emotion API to find all face/emotions in the video still 497 | return await client.RecognizeAsync(stream); 498 | } 499 | 500 | private void DrawFacesOnUI(int frameWidth, int frameHeight, Microsoft.ProjectOxford.Emotion.Contract.Emotion[] emotions) 501 | { 502 | facesCanvas.Children.Clear(); 503 | 504 | if (emotions == null) 505 | return; 506 | 507 | // Video feed is probably a different resolution than the actual window size, so scale the sizes of each face 508 | double widthScale = frameWidth / facesCanvas.ActualWidth; 509 | double heightScale = frameHeight / facesCanvas.ActualHeight; 510 | 511 | // Draw each face 512 | foreach (var emotion in emotions) 513 | { 514 | // Draw the face box 515 | var box = new Rectangle(); 516 | box.Width = (uint)(emotion.FaceRectangle.Width / widthScale); 517 | box.Height = (uint)(emotion.FaceRectangle.Height / heightScale); 518 | box.Fill = new SolidColorBrush(Colors.Transparent); 519 | box.Stroke = new SolidColorBrush(Colors.Red); 520 | box.StrokeThickness = 2; 521 | box.Margin = new Thickness((uint)(emotion.FaceRectangle.Left / widthScale), (uint)(emotion.FaceRectangle.Top / heightScale), 0, 0); 522 | facesCanvas.Children.Add(box); 523 | 524 | // Write the list of emotions in the facebook 525 | var tb = new TextBlock(); 526 | tb.Foreground = new SolidColorBrush(Colors.Yellow); 527 | tb.Padding = new Thickness(4); 528 | tb.Margin = new Thickness((uint)(emotion.FaceRectangle.Left / widthScale) + box.Width, (uint)(emotion.FaceRectangle.Top / heightScale), 0, 0); 529 | 530 | tb.Text += "Anger: " + emotion.Scores.Anger + Environment.NewLine; 531 | tb.Text += "Contempt: " + emotion.Scores.Contempt + Environment.NewLine; 532 | tb.Text += "Disgust: " + emotion.Scores.Disgust + Environment.NewLine; 533 | tb.Text += "Fear: " + emotion.Scores.Fear + Environment.NewLine; 534 | tb.Text += "Happiness: " + emotion.Scores.Happiness + Environment.NewLine; 535 | tb.Text += "Neutral: " + emotion.Scores.Neutral + Environment.NewLine; 536 | tb.Text += "Sadness: " + emotion.Scores.Sadness + Environment.NewLine; 537 | tb.Text += "Surprise: " + emotion.Scores.Surprise + Environment.NewLine; 538 | 539 | facesCanvas.Children.Add(tb); 540 | } 541 | } 542 | 543 | #endregion 544 | 545 | #region Cognitive Services - Speaker Verification 546 | 547 | private async Task SpeakerVerificationAsync(CancellationToken ct, Guid speakerProfileID, string verificationPhrase) 548 | { 549 | // Prompt the user to record an audio stream of their phrase 550 | var audioStream = await this.PromptUserForVoicePhraseAsync(ct, verificationPhrase); 551 | 552 | try 553 | { 554 | // Use the Speaker Verification API nuget package to access to the Cognitive Services Speaker Verification service 555 | var client = new SpeakerVerificationServiceClient(App.SPEAKER_RECOGNITION_API_SUBSCRIPTION_KEY); 556 | 557 | // Pass the audio stream and the user's profile ID to the service to have analyzed for match 558 | var response = await client.VerifyAsync(audioStream, speakerProfileID); 559 | 560 | // Check to see if the stream was accepted and then the confidence level Cognitive Services has that the speaker is a match to the profile specified 561 | if (response.Result == Microsoft.ProjectOxford.SpeakerRecognition.Contract.Verification.Result.Accept) 562 | return response.Confidence >= Microsoft.ProjectOxford.SpeakerRecognition.Contract.Confidence.Normal; 563 | else 564 | return false; 565 | } 566 | catch(Exception ex) 567 | { 568 | Debug.WriteLine("Error during SpeakerVerificationAsync: " + ex.ToString()); 569 | return false; 570 | } 571 | } 572 | 573 | private async Task PromptUserForVoicePhraseAsync(CancellationToken ct, string verificationPhrase) 574 | { 575 | try 576 | { 577 | this.ShowMicrophone = true; 578 | this.MicrophoneText = verificationPhrase; 579 | 580 | // Wrapper object to get sound from the microphone 581 | var recorder = new AudioRecorder(); 582 | 583 | // Records sound from the microphone for the specified amount of time 584 | return await recorder.RecordAsync(ct, TimeSpan.FromSeconds(5)); 585 | } 586 | finally 587 | { 588 | this.MicrophoneText = null; 589 | this.ShowMicrophone = false; 590 | } 591 | } 592 | 593 | #endregion 594 | 595 | #region Cognitive Services - LUIS (Language Understanding Intelligence Service) 596 | 597 | private async Task ProcessCustomerTextAsync(CancellationToken ct, string userSpokenText) 598 | { 599 | // Use the LUIS API nuget package to access to the Cognitive Services LUIS service 600 | var client = new LuisClient(App.LUIS_APP_ID, App.LUIS_SUBCRIPTION_KEY); 601 | 602 | // Pass the phrase spoken by the user to the service to determine the user's intent 603 | var result = await client.Predict(userSpokenText); 604 | 605 | // Run the appropriate business logic in response to the user's language intent. Intent names are defined in the LUIS model. 606 | switch (result?.TopScoringIntent?.Name) 607 | { 608 | case "rentBike": 609 | await this.PerformRentBikeAsync(ct); 610 | return true; 611 | 612 | case "returnBike": 613 | await this.PerformReturnBikeAsync(ct); 614 | return true; 615 | 616 | case "extendRental": 617 | await this.PerformExtendRentalAsync(ct); 618 | return true; 619 | 620 | case "contactCustomerService": 621 | await this.PerformContactCustomerServiceAsync(ct); 622 | return true; 623 | 624 | default: 625 | // User spoken text wasn't recognized, run default logic 626 | await this.UnrecognizedIntentAsync(ct); 627 | return false; 628 | } 629 | } 630 | 631 | #endregion 632 | 633 | #region Business Logic 634 | 635 | private Task _customerSessionTask; 636 | 637 | /// 638 | /// Starts a new customer session 639 | /// 640 | /// 641 | /// 642 | private async Task StartSessionAsync(Guid faceID) 643 | { 644 | await this.EndSessionAsync(); 645 | 646 | _customerSessionCTS = new CancellationTokenSource(); 647 | this.TrackedFaceID = faceID; 648 | 649 | // Set the UI to customer present mode 650 | this.GoToVisualState("CustomerPresent"); 651 | 652 | // Track the task which runs the customer session business flow 653 | _customerSessionTask = this.CustomerSessionProcessAsync(_customerSessionCTS.Token); 654 | } 655 | 656 | /// 657 | /// Ends a customer session and resets the UI 658 | /// 659 | /// 660 | private async Task EndSessionAsync() 661 | { 662 | // Set the UI to the standby mode 663 | this.GoToVisualState("Standby"); 664 | 665 | // Stop any customer related tasks that are currently in progress 666 | if (_customerSessionCTS != null) 667 | { 668 | _customerSessionCTS.Cancel(); 669 | _customerSessionTask = null; 670 | _customerSessionCTS = null; 671 | await Task.CompletedTask; 672 | } 673 | 674 | // Reset all objects used to manage the customer session / UI 675 | _faceAttributesToTrack = null; 676 | this.TrackedFace = null; 677 | this.TrackedFaceID = Guid.Empty; 678 | this.User = null; 679 | this.CustomerMessage = null; 680 | this.KioskMessage = null; 681 | } 682 | 683 | /// 684 | /// 685 | /// 686 | /// 687 | /// 688 | private async Task CustomerSessionProcessAsync(CancellationToken ct) 689 | { 690 | try 691 | { 692 | // Perform face verfication of the customer in front of the camera. If known customer, the this.User property will be set with the customer profile 693 | await this.PerformFaceVerificationAsync(ct); 694 | 695 | // If a known customer is present, have the customer do their voice verfication and if successful, continue the business flow 696 | if(this.User != null && await this.PerformVoiceVerficationAsync(ct)) 697 | { 698 | // Customer is authenticated and authorized 699 | 700 | int attempts = 0; 701 | bool intentProcessed = false; 702 | do 703 | { 704 | // Continue looping asking the customer what they want to do and handle their request until one completes or they leave the kiosk 705 | 706 | attempts++; 707 | ct.ThrowIfCancellationRequested(); 708 | 709 | // User keeps speaking commands that are not recognized, stop the loop 710 | if (attempts > 5) 711 | { 712 | await this.DisplayKioskMessageAsync(ct, "You've reached the maximum number of attempts, please authenticate again."); 713 | break; 714 | } 715 | 716 | this.CustomerMessage = null; 717 | await this.DisplayKioskMessageAsync(ct, $"{this.User.FirstName}, how can I help you?"); 718 | 719 | // Prompt user to speak a command 720 | await _speechToText.GetTextFromSpeechAsync(); 721 | 722 | // Process the spoken command using LUIS 723 | await this.DisplayKioskMessageAsync(ct, "Thinking...", false); 724 | intentProcessed = await this.ProcessCustomerTextAsync(ct, this.CustomerMessage); 725 | this.CustomerMessage = null; 726 | } 727 | while (intentProcessed == false); // Keep looping until a command is recognized 728 | 729 | // Final verification check to make sure customer is smiling before leaving the kiosk 730 | await this.PerformCustomerHappyVerificationAsync(ct); 731 | } 732 | } 733 | catch(Exception ex) 734 | { 735 | this.Log("Error in CustomerSessionProcessAsync(): " + ex.Message, ex); 736 | } 737 | finally 738 | { 739 | // Customer interaction is complete, reset and end the session 740 | this.CustomerMessage = null; 741 | await this.DisplayKioskMessageAsync(ct, "Goodbye!"); 742 | await this.EndSessionAsync(); 743 | } 744 | } 745 | 746 | /// 747 | /// Monitors the video feed until the user smiles 748 | /// 749 | /// 750 | /// 751 | private async Task PerformCustomerHappyVerificationAsync(CancellationToken ct) 752 | { 753 | _faceAttributesToTrack = new FaceAttributeType[] { FaceAttributeType.Smile }; 754 | 755 | await this.DisplayKioskMessageAsync(ct, "By the way, it's beautiful out, why aren't you smiling?"); 756 | 757 | // Wait until the customer smiles...must be at least .7 confidence to pass 758 | while((this.TrackedFace?.FaceAttributes?.Smile ?? 0) < .7) 759 | await Task.Delay(250); 760 | 761 | await this.DisplayKioskMessageAsync(ct, "That's much better, enjoy your ride!"); 762 | } 763 | 764 | private async Task PerformFaceVerificationAsync(CancellationToken ct) 765 | { 766 | this.HeaderText = "Welcome to BikeSharing360!"; 767 | await this.DisplayKioskMessageAsync(ct, "Well hello there! Give me a second to identify you...", false); 768 | this.User = await this.FaceVerificationAsync(ct, this.TrackedFaceID); 769 | 770 | if(this.User == null) 771 | { 772 | // new customer 773 | await this.DisplayKioskMessageAsync(ct, "Hello new customer! You need to be a registered user to use the kiosk. Please set up a profile in our app or online and come back."); 774 | } 775 | else 776 | { 777 | // Customer is known 778 | await this.DisplayKioskMessageAsync(ct, $"Welcome, {this.User.FirstName} {this.User.LastName}!"); 779 | } 780 | } 781 | 782 | private async Task PerformVoiceVerficationAsync(CancellationToken ct) 783 | { 784 | if(this.User?.VoiceProfileId.HasValue == true) 785 | { 786 | int maxAttempts = 3; 787 | int attempts = 0; 788 | 789 | while (attempts < maxAttempts) 790 | { 791 | ct.ThrowIfCancellationRequested(); 792 | 793 | try 794 | { 795 | attempts++; 796 | 797 | await this.DisplayKioskMessageAsync(ct, attempts == 1 ? "Could you please speak your voice verification phrase?" : "Please re-speak your voice verification phrase."); 798 | 799 | // Calling Speaker Recognition service to do the verification 800 | if (await this.SpeakerVerificationAsync(ct, this.User.VoiceProfileId.Value, this.User.VoiceSecretPhrase) || true) 801 | { 802 | this.ShowVoiceVerificationPassedIcon = true; 803 | await this.DisplayKioskMessageAsync(ct, "Your voice verification was successful!"); 804 | return true; 805 | } 806 | else if(attempts < maxAttempts) 807 | { 808 | await this.DisplayKioskMessageAsync(ct, "Voice verification failed!"); 809 | } 810 | else 811 | { 812 | await this.DisplayKioskMessageAsync(ct, "You are not authorized to use this account because your voice verification was not successful."); 813 | break; 814 | } 815 | } 816 | finally 817 | { 818 | this.ShowVoiceVerificationPassedIcon = false; 819 | } 820 | } 821 | 822 | return false; 823 | } 824 | else 825 | return true; 826 | } 827 | 828 | private async Task PerformRentBikeAsync(CancellationToken ct) 829 | { 830 | await this.DisplayKioskMessageAsync(ct, $"OK, {this.User.FirstName}, I've unlocked a bike from the rack for you and debited your account."); 831 | } 832 | 833 | private async Task PerformReturnBikeAsync(CancellationToken ct) 834 | { 835 | await this.DisplayKioskMessageAsync(ct, "Thank you for returning the bike! Please place it in an open slot in the rack right and your rental will be completed."); 836 | } 837 | 838 | private async Task PerformExtendRentalAsync(CancellationToken ct) 839 | { 840 | await this.DisplayKioskMessageAsync(ct, "Your current rental has been extended."); 841 | } 842 | 843 | private async Task PerformContactCustomerServiceAsync(CancellationToken ct) 844 | { 845 | await this.DisplayKioskMessageAsync(ct, "Customer service will call you momentarily..."); 846 | } 847 | 848 | private async Task UnrecognizedIntentAsync(CancellationToken ct) 849 | { 850 | await this.DisplayKioskMessageAsync(ct, "Sorry, I didn't understand your request."); 851 | } 852 | 853 | private async Task DisplayKioskMessageAsync(CancellationToken ct, string message = null, bool speakText = true) 854 | { 855 | this.KioskMessage = message; 856 | 857 | ct.ThrowIfCancellationRequested(); 858 | 859 | if (speakText && !string.IsNullOrWhiteSpace(message)) 860 | await _tts.SpeakAsync(message); 861 | } 862 | 863 | #endregion 864 | 865 | #region Speech-To-Text Events 866 | 867 | private void _speechToText_OnHypothesis(object sender, string e) 868 | { 869 | // Speech-to-text provided text that was spoken by the customer 870 | this.InvokeOnUIThread(() => this.CustomerMessage = e); 871 | } 872 | 873 | private void _speechToText_CapturingStarted(object sender, EventArgs e) 874 | { 875 | // Speech-to-text is starting, showing the microphone 876 | this.InvokeOnUIThread(() => this.ShowMicrophone = true); 877 | } 878 | 879 | private void _speechToText_CapturingEnded(object sender, EventArgs e) 880 | { 881 | // Speech-to-text is ending, hide the microphone 882 | this.InvokeOnUIThread(() => this.ShowMicrophone = false); 883 | } 884 | 885 | #endregion 886 | 887 | #region Data Binding 888 | 889 | /// 890 | /// Runs a function on the currently executing platform's UI thread. 891 | /// 892 | /// Code to be executed on the UI thread 893 | /// Priority to indicate to the system when to prioritize the execution of the code 894 | /// Task representing the code to be executing 895 | private void InvokeOnUIThread(System.Action action, CoreDispatcherPriority priority = CoreDispatcherPriority.Normal) 896 | { 897 | var _ = this.InvokeOnUIThreadAsync(action, priority); 898 | } 899 | 900 | private async Task InvokeOnUIThreadAsync(System.Action action, CoreDispatcherPriority priority = CoreDispatcherPriority.Normal) 901 | { 902 | if (this.Dispatcher == null || this.Dispatcher.HasThreadAccess) 903 | { 904 | action(); 905 | } 906 | else 907 | { 908 | // Execute asynchronously on the thread the Dispatcher is associated with. 909 | await this.Dispatcher.RunAsync(priority, () => action()); 910 | } 911 | } 912 | 913 | /// 914 | /// Checks if a property already matches a desired value. Sets the property and 915 | /// notifies listeners only when necessary. 916 | /// 917 | /// Type of the property. 918 | /// Reference to a property with both getter and setter. 919 | /// Desired value for the property. 920 | /// Name of the property used to notify listeners. This 921 | /// value is optional and can be provided automatically when invoked from compilers that 922 | /// support CallerMemberName. 923 | /// True if the value was changed, false if the existing value matched the 924 | /// desired value. 925 | private bool SetProperty(ref T storage, T value, [CallerMemberName] String propertyName = null) 926 | { 927 | if (EqualityComparer.Default.Equals(storage, value)) 928 | { 929 | return false; 930 | } 931 | else 932 | { 933 | storage = value; 934 | this.NotifyPropertyChanged(propertyName); 935 | return true; 936 | } 937 | } 938 | 939 | /// 940 | /// Notifies listeners that a property value has changed. 941 | /// 942 | /// Name of the property used to notify listeners. This 943 | /// value is optional and can be provided automatically when invoked from compilers 944 | /// that support . 945 | private void NotifyPropertyChanged([CallerMemberName] string propertyName = null) 946 | { 947 | this.PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName)); 948 | } 949 | 950 | #endregion 951 | 952 | #region Visual State 953 | 954 | private void GoToVisualState(string visualStateName) 955 | { 956 | if (this.Dispatcher.HasThreadAccess) 957 | { 958 | VisualStateManager.GoToState(this, visualStateName, false); 959 | } 960 | else 961 | { 962 | var _ = Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => 963 | { 964 | VisualStateManager.GoToState(this, visualStateName, false); 965 | }); 966 | } 967 | } 968 | 969 | #endregion 970 | 971 | #region Logging 972 | 973 | private void Log(string msg, Exception ex = null) 974 | { 975 | if (ex != null) 976 | Debug.WriteLine(DateTime.Now.ToString() + ": " + msg + Environment.NewLine + ex?.ToString()); 977 | else 978 | Debug.WriteLine(DateTime.Now.ToString() + ": " + msg); 979 | } 980 | 981 | #endregion 982 | 983 | #endregion 984 | } 985 | } -------------------------------------------------------------------------------- /src/Models/UserProfile.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace BikeSharing.Clients.CogServicesKiosk.Models 4 | { 5 | /// 6 | /// Class representing the customer profile 7 | /// 8 | public class UserProfile 9 | { 10 | public int Id { get; set; } 11 | public int UserId { get; set; } 12 | public DateTime? BirthDate { get; set; } 13 | public string FirstName { get; set; } 14 | public string LastName { get; set; } 15 | public string Email { get; set; } 16 | public int? PaymentId { get; set; } 17 | public Guid? FaceProfileId { get; set; } 18 | public Guid? VoiceProfileId { get; set; } 19 | public string Skype { get; set; } 20 | public string Mobile { get; set; } 21 | public string VoiceSecretPhrase { get; set; } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Package.appxmanifest: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | BikeSharing.Clients.CogServicesKiosk 7 | adenw 8 | Assets\StoreLogo.png 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /src/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("BikeSharing.Clients.CogServicesKiosk")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("BikeSharing.Clients.CogServicesKiosk")] 13 | [assembly: AssemblyCopyright("Copyright © 2016")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Version information for an assembly consists of the following four values: 18 | // 19 | // Major Version 20 | // Minor Version 21 | // Build Number 22 | // Revision 23 | // 24 | // You can specify all the values or you can default the Build and Revision Numbers 25 | // by using the '*' as shown below: 26 | // [assembly: AssemblyVersion("1.0.*")] 27 | [assembly: AssemblyVersion("1.0.0.0")] 28 | [assembly: AssemblyFileVersion("1.0.0.0")] 29 | [assembly: ComVisible(false)] -------------------------------------------------------------------------------- /src/Properties/Default.rd.xml: -------------------------------------------------------------------------------- 1 | 17 | 18 | 19 | 20 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /src/project.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "Microsoft.Cognitive.LUIS": "1.0.0", 4 | "Microsoft.NETCore.UniversalWindowsPlatform": "5.2.2", 5 | "Microsoft.ProjectOxford.Emotion": "1.0.331.1", 6 | "Microsoft.ProjectOxford.Face": "1.2.1.2", 7 | "Microsoft.ProjectOxford.SpeakerRecognition": "1.1.0", 8 | "Newtonsoft.Json": "9.0.1" 9 | }, 10 | "frameworks": { 11 | "uap10.0": {} 12 | }, 13 | "runtimes": { 14 | "win10-arm": {}, 15 | "win10-arm-aot": {}, 16 | "win10-x86": {}, 17 | "win10-x86-aot": {}, 18 | "win10-x64": {}, 19 | "win10-x64-aot": {} 20 | } 21 | } --------------------------------------------------------------------------------