├── app.config ├── VirtualDesktop └── FaceTracking │ ├── Vector3.cs │ ├── Pose.cs │ ├── Quaternion.cs │ ├── FaceState.cs │ └── Expressions.cs ├── Properties └── AssemblyInfo.cs ├── README.md ├── VDFaceTracking.sln ├── VDFaceTracking.csproj ├── VDFaceTracking.cs ├── FBExpression.cs ├── .gitignore └── VDProxy.cs /app.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /VirtualDesktop/FaceTracking/Vector3.cs: -------------------------------------------------------------------------------- 1 | namespace VirtualDesktop.FaceTracking 2 | { 3 | public struct Vector3 4 | { 5 | public float X; 6 | public float Y; 7 | public float Z; 8 | } 9 | } -------------------------------------------------------------------------------- /VirtualDesktop/FaceTracking/Pose.cs: -------------------------------------------------------------------------------- 1 | namespace VirtualDesktop.FaceTracking 2 | { 3 | public struct Pose 4 | { 5 | public static readonly Pose Identity = new Pose() 6 | { 7 | Orientation = Quaternion.Identity 8 | }; 9 | public Quaternion Orientation; 10 | public Vector3 Position; 11 | } 12 | } -------------------------------------------------------------------------------- /Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | 3 | [assembly: AssemblyTitle("VDFaceTracking")] 4 | [assembly: AssemblyProduct("VDFaceTracking")] 5 | [assembly: AssemblyDescription("Virtual Desktop face tracking mod for Resonite.")] 6 | [assembly: AssemblyCompany("Zeith & dfgHiatus & Geenz & Earthmark & Delta")] 7 | [assembly: AssemblyCopyright("Copyright © 2025")] 8 | [assembly: AssemblyVersion(VDFaceTracking.VDFaceTracking.VERSION_CONSTANT)] 9 | [assembly: AssemblyFileVersion(VDFaceTracking.VDFaceTracking.VERSION_CONSTANT)] -------------------------------------------------------------------------------- /VirtualDesktop/FaceTracking/Quaternion.cs: -------------------------------------------------------------------------------- 1 | namespace VirtualDesktop.FaceTracking 2 | { 3 | public struct Quaternion 4 | { 5 | public static readonly Quaternion Identity = new Quaternion(0.0f, 0.0f, 0.0f, 1f); 6 | public float X; 7 | public float Y; 8 | public float Z; 9 | public float W; 10 | 11 | public Quaternion(float x, float y, float z, float w) 12 | { 13 | this.X = x; 14 | this.Y = y; 15 | this.Z = z; 16 | this.W = w; 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # VDFaceTracking 2 | 3 | This is a Resonite mod, which lets the game read facial expressions and eye gaze directly from VirtualDesktop (v1.30 or newer). 4 | 5 | This was patched together from [dfgHiatus' QuestPro4Neos mod](https://github.com/dfgHiatus/QuestPro4Neos) and [VirtualDesktop](https://www.vrdesktop.net/) VRCFT module. 6 | 7 | To install this mod go grab the release [over here](https://github.com/Zeitheron/VDFaceTracking/releases/latest) and put it into `Resonite\rml_mods\` folder. 8 | 9 | You are going to need [Resonite Mod Loader](https://github.com/resonite-modding-group/ResoniteModLoader), read how to install it there. -------------------------------------------------------------------------------- /VDFaceTracking.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.2.32526.322 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VDFaceTracking", "VDFaceTracking.csproj", "{39d944a3-538c-4a2d-8e53-8db606c64233}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {39d944a3-538c-4a2d-8e53-8db606c64233}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {39d944a3-538c-4a2d-8e53-8db606c64233}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {39d944a3-538c-4a2d-8e53-8db606c64233}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {39d944a3-538c-4a2d-8e53-8db606c64233}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | GlobalSection(ExtensibilityGlobals) = postSolution 23 | SolutionGuid = {0322B2EF-7452-479D-BAE2-FCAB75033337} 24 | EndGlobalSection 25 | EndGlobal -------------------------------------------------------------------------------- /VirtualDesktop/FaceTracking/FaceState.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.InteropServices; 2 | 3 | namespace VirtualDesktop.FaceTracking 4 | { 5 | public struct FaceState 6 | { 7 | #region Constants 8 | public const int ExpressionCount = 70; 9 | public const int ConfidenceCount = 2; 10 | #endregion 11 | 12 | #region Static Fields 13 | public static readonly FaceState Identity = new FaceState { LeftEyePose = Pose.Identity, RightEyePose = Pose.Identity }; 14 | #endregion 15 | 16 | #region Fields 17 | [MarshalAs(UnmanagedType.I1)] 18 | public bool FaceIsValid; 19 | 20 | [MarshalAs(UnmanagedType.I1)] 21 | public bool IsEyeFollowingBlendshapesValid; 22 | 23 | public unsafe fixed float ExpressionWeights[ExpressionCount]; 24 | 25 | public unsafe fixed float ExpressionConfidences[ConfidenceCount]; 26 | 27 | [MarshalAs(UnmanagedType.I1)] 28 | public bool LeftEyeIsValid; 29 | 30 | [MarshalAs(UnmanagedType.I1)] 31 | public bool RightEyeIsValid; 32 | 33 | public Pose LeftEyePose; 34 | public Pose RightEyePose; 35 | 36 | public float LeftEyeConfidence; 37 | public float RightEyeConfidence; 38 | #endregion 39 | } 40 | } -------------------------------------------------------------------------------- /VirtualDesktop/FaceTracking/Expressions.cs: -------------------------------------------------------------------------------- 1 | namespace VirtualDesktop.FaceTracking 2 | { 3 | public enum Expressions 4 | { 5 | BrowLowererL = 0, 6 | BrowLowererR = 1, 7 | CheekPuffL = 2, 8 | CheekPuffR = 3, 9 | CheekRaiserL = 4, 10 | CheekRaiserR = 5, 11 | CheekSuckL = 6, 12 | CheekSuckR = 7, 13 | ChinRaiserB = 8, 14 | ChinRaiserT = 9, 15 | DimplerL = 10, 16 | DimplerR = 11, 17 | EyesClosedL = 12, 18 | EyesClosedR = 13, 19 | EyesLookDownL = 14, 20 | EyesLookDownR = 15, 21 | EyesLookLeftL = 16, 22 | EyesLookLeftR = 17, 23 | EyesLookRightL = 18, 24 | EyesLookRightR = 19, 25 | EyesLookUpL = 20, 26 | EyesLookUpR = 21, 27 | InnerBrowRaiserL = 22, 28 | InnerBrowRaiserR = 23, 29 | JawDrop = 24, 30 | JawSidewaysLeft = 25, 31 | JawSidewaysRight = 26, 32 | JawThrust = 27, 33 | LidTightenerL = 28, 34 | LidTightenerR = 29, 35 | LipCornerDepressorL = 30, 36 | LipCornerDepressorR = 31, 37 | LipCornerPullerL = 32, 38 | LipCornerPullerR = 33, 39 | LipFunnelerLb = 34, 40 | LipFunnelerLt = 35, 41 | LipFunnelerRb = 36, 42 | LipFunnelerRt = 37, 43 | LipPressorL = 38, 44 | LipPressorR = 39, 45 | LipPuckerL = 40, 46 | LipPuckerR = 41, 47 | LipStretcherL = 42, 48 | LipStretcherR = 43, 49 | LipSuckLb = 44, 50 | LipSuckLt = 45, 51 | LipSuckRb = 46, 52 | LipSuckRt = 47, 53 | LipTightenerL = 48, 54 | LipTightenerR = 49, 55 | LipsToward = 50, 56 | LowerLipDepressorL = 51, 57 | LowerLipDepressorR = 52, 58 | MouthLeft = 53, 59 | MouthRight = 54, 60 | NoseWrinklerL = 55, 61 | NoseWrinklerR = 56, 62 | OuterBrowRaiserL = 57, 63 | OuterBrowRaiserR = 58, 64 | UpperLidRaiserL = 59, 65 | UpperLidRaiserR = 60, 66 | UpperLipRaiserL = 61, 67 | UpperLipRaiserR = 62, 68 | 69 | // new tongue tracking data 70 | TongueTipInterdental = 63, 71 | TongueTipAlveolar = 64, 72 | TongueFrontDorsalPalate = 65, 73 | TongueMidDorsalPalate = 66, 74 | TongueBackDorsalVelar = 67, 75 | TongueOut = 68, 76 | TongueRetreat = 69 77 | } 78 | } -------------------------------------------------------------------------------- /VDFaceTracking.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | VDFaceTracking 4 | VDFaceTracking 5 | false 6 | net900 7 | 512 8 | latest 9 | enable 10 | true 11 | 12 | true 13 | embedded 14 | true 15 | 16 | 17 | 18 | 19 | $(MSBuildThisFileDirectory)Resonite/ 20 | C:\Program Files (x86)\Steam\steamapps\common\Resonite\ 21 | $(HOME)/.steam/steam/steamapps/common/Resonite/ 22 | E:\SteamLibrary\steamapps\common\Resonite\ 23 | 24 | 25 | 26 | 27 | $(ResonitePath)Libraries\ResoniteModLoader.dll 28 | False 29 | 30 | 31 | $(ResonitePath)rml_libs\0Harmony.dll 32 | False 33 | 34 | 35 | $(ResonitePath)\FrooxEngine.dll 36 | False 37 | 38 | 39 | $(ResonitePath)\Elements.Core.dll 40 | False 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /VDFaceTracking.cs: -------------------------------------------------------------------------------- 1 | using FrooxEngine; 2 | using ResoniteModLoader; 3 | using System; 4 | 5 | namespace VDFaceTracking 6 | { 7 | public class VDFaceTracking : ResoniteMod 8 | { 9 | [AutoRegisterConfigKey] 10 | private static readonly ModConfigurationKey EyeOpennessExponent = 11 | new("quest_pro_eye_open_exponent", 12 | "Exponent to apply to eye openness. Can be updated at runtime. Useful for applying different curves for how open your eyes are.", 13 | () => 1.0f); 14 | 15 | [AutoRegisterConfigKey] 16 | private static readonly ModConfigurationKey EyeWideMultiplier = 17 | new("quest_pro_eye_wide_multiplier", 18 | "Multiplier to apply to eye wideness. Can be updated at runtime. Useful for multiplying the amount your eyes can widen by.", 19 | () => 1.0f); 20 | 21 | [AutoRegisterConfigKey] 22 | private static readonly ModConfigurationKey EyeMovementMultiplier = 23 | new("quest_pro_eye_movement_multiplier", 24 | "Multiplier to adjust the movement range of the user's eyes. Can be updated at runtime.", () => 1.0f); 25 | 26 | [AutoRegisterConfigKey] 27 | private static readonly ModConfigurationKey EyeExpressionMultiplier = 28 | new("quest_pro_eye_expression_multiplier", 29 | "Multiplier to adjust the range of the user's eye expressions. Can be updated at runtime.", () => 1.0f); 30 | 31 | private static ModConfiguration _config; 32 | 33 | internal const string VERSION_CONSTANT = "1.1.3"; 34 | public override string Name => "VDFaceTracking"; 35 | public override string Author => "Zeith & dfgHiatus & Geenz & Earthmark & Delta"; 36 | public override string Version => VERSION_CONSTANT; 37 | 38 | public static VDProxy proxy; 39 | 40 | public static float EyeOpenExponent = 1.0f; 41 | public static float EyeWideMult = 1.0f; 42 | public static float EyeMoveMult = 1.0f; 43 | public static float EyeExpressionMult = 1.0f; 44 | 45 | public override void OnEngineInit() 46 | { 47 | _config = GetConfiguration(); 48 | _config.OnThisConfigurationChanged += OnConfigurationChanged; 49 | 50 | Engine.Current.RunPostInit(() => 51 | { 52 | proxy = new VDProxy(); 53 | if (!proxy.Initialize()) { return; } 54 | 55 | Engine.Current.InputInterface.RegisterInputDriver(proxy); 56 | Engine.Current.OnShutdown += () => proxy.Teardown(); 57 | }); 58 | } 59 | 60 | private void OnConfigurationChanged(ConfigurationChangedEvent @event) 61 | { 62 | if (@event.Key == EyeOpennessExponent) 63 | { 64 | if (@event.Config.TryGetValue(EyeOpennessExponent, out var openExp)) 65 | { 66 | EyeOpenExponent = openExp; 67 | } 68 | } 69 | 70 | if (@event.Key == EyeWideMultiplier) 71 | { 72 | if (@event.Config.TryGetValue(EyeWideMultiplier, out var wideMulti)) 73 | { 74 | EyeWideMult = wideMulti; 75 | } 76 | } 77 | 78 | if (@event.Key == EyeMovementMultiplier) 79 | { 80 | if (@event.Config.TryGetValue(EyeMovementMultiplier, out var moveMulti)) 81 | { 82 | EyeMoveMult = moveMulti; 83 | } 84 | } 85 | 86 | if (@event.Key == EyeExpressionMultiplier) 87 | { 88 | if (@event.Config.TryGetValue(EyeExpressionMultiplier, out var eyeExpressionMulti)) 89 | { 90 | EyeExpressionMult = eyeExpressionMulti; 91 | } 92 | } 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /FBExpression.cs: -------------------------------------------------------------------------------- 1 | namespace VDFaceTracking 2 | { 3 | public static class FBExpression 4 | { 5 | public const int Brow_Lowerer_L = 0; 6 | public const int Brow_Lowerer_R = 1; 7 | public const int Cheek_Puff_L = 2; 8 | public const int Cheek_Puff_R = 3; 9 | public const int Cheek_Raiser_L = 4; 10 | public const int Cheek_Raiser_R = 5; 11 | public const int Cheek_Suck_L = 6; 12 | public const int Cheek_Suck_R = 7; 13 | public const int Chin_Raiser_B = 8; 14 | public const int Chin_Raiser_T = 9; 15 | public const int Dimpler_L = 10; 16 | public const int Dimpler_R = 11; 17 | public const int Eyes_Closed_L = 12; 18 | public const int Eyes_Closed_R = 13; 19 | public const int Eyes_Look_Down_L = 14; 20 | public const int Eyes_Look_Down_R = 15; 21 | public const int Eyes_Look_Left_L = 16; 22 | public const int Eyes_Look_Left_R = 17; 23 | public const int Eyes_Look_Right_L = 18; 24 | public const int Eyes_Look_Right_R = 19; 25 | public const int Eyes_Look_Up_L = 20; 26 | public const int Eyes_Look_Up_R = 21; 27 | public const int Inner_Brow_Raiser_L = 22; 28 | public const int Inner_Brow_Raiser_R = 23; 29 | public const int Jaw_Drop = 24; 30 | public const int Jaw_Sideways_Left = 25; 31 | public const int Jaw_Sideways_Right = 26; 32 | public const int Jaw_Thrust = 27; 33 | public const int Lid_Tightener_L = 28; 34 | public const int Lid_Tightener_R = 29; 35 | public const int Lip_Corner_Depressor_L = 30; 36 | public const int Lip_Corner_Depressor_R = 31; 37 | public const int Lip_Corner_Puller_L = 32; 38 | public const int Lip_Corner_Puller_R = 33; 39 | public const int Lip_Funneler_LB = 34; 40 | public const int Lip_Funneler_LT = 35; 41 | public const int Lip_Funneler_RB = 36; 42 | public const int Lip_Funneler_RT = 37; 43 | public const int Lip_Pressor_L = 38; 44 | public const int Lip_Pressor_R = 39; 45 | public const int Lip_Pucker_L = 40; 46 | public const int Lip_Pucker_R = 41; 47 | public const int Lip_Stretcher_L = 42; 48 | public const int Lip_Stretcher_R = 43; 49 | public const int Lip_Suck_LB = 44; 50 | public const int Lip_Suck_LT = 45; 51 | public const int Lip_Suck_RB = 46; 52 | public const int Lip_Suck_RT = 47; 53 | public const int Lip_Tightener_L = 48; 54 | public const int Lip_Tightener_R = 49; 55 | public const int Lips_Toward = 50; 56 | public const int Lower_Lip_Depressor_L = 51; 57 | public const int Lower_Lip_Depressor_R = 52; 58 | public const int Mouth_Left = 53; 59 | public const int Mouth_Right = 54; 60 | public const int Nose_Wrinkler_L = 55; 61 | public const int Nose_Wrinkler_R = 56; 62 | public const int Outer_Brow_Raiser_L = 57; 63 | public const int Outer_Brow_Raiser_R = 58; 64 | public const int Upper_Lid_Raiser_L = 59; 65 | public const int Upper_Lid_Raiser_R = 60; 66 | public const int Upper_Lip_Raiser_L = 61; 67 | public const int Upper_Lip_Raiser_R = 62; 68 | public const int TongueTipInterdental = 63; 69 | public const int TongueTipAlveolar = 64; 70 | public const int TongueFrontDorsalPalate = 65; 71 | public const int TongueMidDorsalPalate = 66; 72 | public const int TongueBackDorsalVelar = 67; 73 | public const int TongueOut = 68; 74 | public const int TongueRetreat = 69; 75 | public const int Max = 70; 76 | 77 | // Above are the natural expressions tracked by the Quest Pro 78 | // Below is the eye tracking information 79 | public const int LeftRot_x = 71; 80 | public const int LeftRot_y = 72; 81 | public const int LeftRot_z = 73; 82 | public const int LeftRot_w = 74; 83 | public const int LeftPos_x = 75; 84 | public const int LeftPos_y = 76; // Flipped, need to convert RHS to LHS 85 | public const int LeftPos_z = 77; 86 | // public const int 71 is unused 87 | public const int RightRot_x = 78; 88 | public const int RightRot_y = 79; 89 | public const int RightRot_z = 80; 90 | public const int RightRot_w = 81; 91 | public const int RightPos_x = 82; 92 | public const int RightPos_y = 83; // Flipped, need to convert RHS to LHS 93 | public const int RightPos_z = 84; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. ## ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore # User-specific files *.rsuser *.suo *.user *.userosscache *.sln.docstates # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs # Mono auto generated files mono_crash.* # Build results [Dd]ebug/ [Dd]ebugPublic/ [Rr]elease/ [Rr]eleases/ x64/ x86/ [Ww][Ii][Nn]32/ [Aa][Rr][Mm]/ [Aa][Rr][Mm]64/ bld/ [Bb]in/ [Oo]bj/ [Oo]ut/ [Ll]og/ [Ll]ogs/ # Visual Studio 2015/2017 cache/options directory /.vs/ /obj/ /bin/ # Uncomment if you have tasks that create the project's static files in wwwroot #wwwroot/ # Visual Studio 2017 auto generated files Generated\ Files/ # MSTest test Results [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* # NUnit *.VisualState.xml TestResult.xml nunit-*.xml # Build Results of an ATL Project [Dd]ebugPS/ [Rr]eleasePS/ dlldata.c # Benchmark Results BenchmarkDotNet.Artifacts/ # .NET Core project.lock.json project.fragment.lock.json artifacts/ # ASP.NET Scaffolding ScaffoldingReadMe.txt # StyleCop StyleCopReport.xml # Files built by Visual Studio *_i.c *_p.c *_h.h *.ilk *.meta *.obj *.iobj *.pch *.pdb *.ipdb *.pgc *.pgd *.rsp *.sbr *.tlb *.tli *.tlh *.tmp *.tmp_proj *_wpftmp.csproj *.log *.vspscc *.vssscc .builds *.pidb *.svclog *.scc # Chutzpah Test files _Chutzpah* # Visual C++ cache files ipch/ *.aps *.ncb *.opendb *.opensdf *.sdf *.cachefile *.VC.db *.VC.VC.opendb # Visual Studio profiler *.psess *.vsp *.vspx *.sap # Visual Studio Trace Files *.e2e # TFS 2012 Local Workspace $tf/ # Guidance Automation Toolkit *.gpState # ReSharper is a .NET coding add-in _ReSharper*/ *.[Rr]e[Ss]harper *.DotSettings.user # TeamCity is a build add-in _TeamCity* # DotCover is a Code Coverage Tool *.dotCover # AxoCover is a Code Coverage Tool .axoCover/* !.axoCover/settings.json # Coverlet is a free, cross platform Code Coverage Tool coverage*.json coverage*.xml coverage*.info # Visual Studio code coverage results *.coverage *.coveragexml # NCrunch _NCrunch_* .*crunch*.local.xml nCrunchTemp_* # MightyMoose *.mm.* AutoTest.Net/ # Web workbench (sass) .sass-cache/ # Installshield output folder [Ee]xpress/ # DocProject is a documentation generator add-in DocProject/buildhelp/ DocProject/Help/*.HxT DocProject/Help/*.HxC DocProject/Help/*.hhc DocProject/Help/*.hhk DocProject/Help/*.hhp DocProject/Help/Html2 DocProject/Help/html # Click-Once directory publish/ # Publish Web Output *.[Pp]ublish.xml *.azurePubxml # Note: Comment the next line if you want to checkin your web deploy settings, # but database connection strings (with potential passwords) will be unencrypted *.pubxml *.publishproj # Microsoft Azure Web App publish settings. Comment the next line if you want to # checkin your Azure Web App publish settings, but sensitive information contained # in these scripts will be unencrypted PublishScripts/ # NuGet Packages *.nupkg # NuGet Symbol Packages *.snupkg # The packages folder can be ignored because of Package Restore **/[Pp]ackages/* # except build/, which is used as an MSBuild target. !**/[Pp]ackages/build/ # Uncomment if necessary however generally it will be regenerated when needed #!**/[Pp]ackages/repositories.config # NuGet v3's project.json files produces more ignorable files *.nuget.props *.nuget.targets # Microsoft Azure Build Output csx/ *.build.csdef # Microsoft Azure Emulator ecf/ rcf/ # Windows Store app package directories and files AppPackages/ BundleArtifacts/ Package.StoreAssociation.xml _pkginfo.txt *.appx *.appxbundle *.appxupload # Visual Studio cache files # files ending in .cache can be ignored *.[Cc]ache # but keep track of directories ending in .cache !?*.[Cc]ache/ # Others ClientBin/ ~$* *~ *.dbmdl *.dbproj.schemaview *.jfm *.pfx *.publishsettings orleans.codegen.cs # Including strong name files can present a security risk # (https://github.com/github/gitignore/pull/2483#issue-259490424) #*.snk # Since there are multiple workflows, uncomment next line to ignore bower_components # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) #bower_components/ # RIA/Silverlight projects Generated_Code/ # Backup & report files from converting an old project file # to a newer Visual Studio version. Backup files are not needed, # because we have git ;-) _UpgradeReport_Files/ Backup*/ UpgradeLog*.XML UpgradeLog*.htm ServiceFabricBackup/ *.rptproj.bak # SQL Server files *.mdf *.ldf *.ndf # Business Intelligence projects *.rdl.data *.bim.layout *.bim_*.settings *.rptproj.rsuser *- [Bb]ackup.rdl *- [Bb]ackup ([0-9]).rdl *- [Bb]ackup ([0-9][0-9]).rdl # Microsoft Fakes FakesAssemblies/ # GhostDoc plugin setting file *.GhostDoc.xml # Node.js Tools for Visual Studio .ntvs_analysis.dat node_modules/ # Visual Studio 6 build log *.plg # Visual Studio 6 workspace options file *.opt # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) *.vbw # Visual Studio LightSwitch build output **/*.HTMLClient/GeneratedArtifacts **/*.DesktopClient/GeneratedArtifacts **/*.DesktopClient/ModelManifest.xml **/*.Server/GeneratedArtifacts **/*.Server/ModelManifest.xml _Pvt_Extensions # Paket dependency manager .paket/paket.exe paket-files/ # FAKE - F# Make .fake/ # CodeRush personal settings .cr/personal # Python Tools for Visual Studio (PTVS) __pycache__/ *.pyc # Cake - Uncomment if you are using it # tools/** # !tools/packages.config # Tabs Studio *.tss # Telerik's JustMock configuration file *.jmconfig # BizTalk build output *.btp.cs *.btm.cs *.odx.cs *.xsd.cs # OpenCover UI analysis results OpenCover/ # Azure Stream Analytics local run output ASALocalRun/ # MSBuild Binary and Structured Log *.binlog # NVidia Nsight GPU debugger configuration file *.nvuser # MFractors (Xamarin productivity tool) working folder .mfractor/ # Local History for Visual Studio .localhistory/ # BeatPulse healthcheck temp database healthchecksdb # Backup folder for Package Reference Convert tool in Visual Studio 2017 MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder .ionide/ # Fody - auto-generated XML schema FodyWeavers.xsd 2 | /.vs/* 3 | /bin/* 4 | /obj/* 5 | -------------------------------------------------------------------------------- /VDProxy.cs: -------------------------------------------------------------------------------- 1 | using Elements.Core; 2 | using FrooxEngine; 3 | using System; 4 | using System.IO.MemoryMappedFiles; 5 | using System.Runtime.InteropServices; 6 | using System.Threading; 7 | using VirtualDesktop.FaceTracking; 8 | 9 | namespace VDFaceTracking 10 | { 11 | public class VDProxy : IInputDriver 12 | { 13 | private const string BodyStateMapName = "VirtualDesktop.BodyState"; 14 | private const string BodyStateEventName = "VirtualDesktop.BodyStateEvent"; 15 | private MemoryMappedFile _mappedFile; 16 | private MemoryMappedViewAccessor _mappedView; 17 | private unsafe FaceState* _faceState; 18 | private EventWaitHandle _faceStateEvent; 19 | 20 | private CancellationTokenSource cancellationTokenSource; 21 | 22 | private bool? _isTracking; 23 | 24 | private Thread thread; 25 | 26 | private const int NATURAL_EXPRESSIONS_COUNT = FBExpression.Max; 27 | private const float SRANIPAL_NORMALIZER = 0.75f; 28 | private float[] expressions = new float[NATURAL_EXPRESSIONS_COUNT + (8 * 2)]; 29 | 30 | private double pitch_L, yaw_L, pitch_R, yaw_R; // Eye rotations 31 | 32 | #region RESONITE VARIABLES 33 | private InputInterface _input; 34 | public int UpdateOrder => 100; 35 | private Mouth mouth; 36 | private Eyes eyes; 37 | #endregion 38 | 39 | private bool? IsTracking 40 | { 41 | get => this._isTracking; 42 | set 43 | { 44 | bool? nullable = value; 45 | bool? isTracking = this._isTracking; 46 | if (nullable.GetValueOrDefault() == isTracking.GetValueOrDefault() & nullable.HasValue == isTracking.HasValue) 47 | return; 48 | this._isTracking = value; 49 | if (value.Value) 50 | VDFaceTracking.Msg("Tracking is now active!"); 51 | else 52 | VDFaceTracking.Msg("Tracking is not active. Make sure you are connected to your computer, a VR game or SteamVR is launched and 'Forward tracking data' is enabled in the Streaming tab."); 53 | } 54 | } 55 | 56 | public virtual void UpdateThread() 57 | { 58 | while (!cancellationTokenSource.IsCancellationRequested) 59 | { 60 | try 61 | { 62 | Update(); 63 | } 64 | catch 65 | { 66 | } 67 | } 68 | } 69 | 70 | public virtual unsafe void Update() 71 | { 72 | if (this._faceStateEvent.WaitOne(50)) 73 | { 74 | this.UpdateTracking(); 75 | } 76 | else 77 | { 78 | FaceState* faceState = this._faceState; 79 | this.IsTracking = new bool?((IntPtr)faceState != IntPtr.Zero && (faceState->LeftEyeIsValid || faceState->RightEyeIsValid || faceState->IsEyeFollowingBlendshapesValid || faceState->FaceIsValid)); 80 | } 81 | } 82 | 83 | private unsafe void UpdateTracking() 84 | { 85 | bool flag = false; 86 | FaceState* faceState = this._faceState; 87 | if ((IntPtr)faceState != IntPtr.Zero) 88 | { 89 | float* expressionWeights = faceState->ExpressionWeights; 90 | 91 | if (faceState->LeftEyeIsValid) 92 | { 93 | Pose leftEyePose = faceState->LeftEyePose; 94 | 95 | expressions[FBExpression.LeftRot_x] = leftEyePose.Orientation.X; 96 | expressions[FBExpression.LeftRot_y] = leftEyePose.Orientation.Y; 97 | expressions[FBExpression.LeftRot_z] = leftEyePose.Orientation.Z; 98 | expressions[FBExpression.LeftRot_w] = leftEyePose.Orientation.W; 99 | 100 | expressions[FBExpression.LeftPos_x] = leftEyePose.Position.X; 101 | expressions[FBExpression.LeftPos_y] = leftEyePose.Position.Y; 102 | expressions[FBExpression.LeftPos_z] = leftEyePose.Position.Z; 103 | 104 | flag = true; 105 | } 106 | 107 | if(faceState->RightEyeIsValid) 108 | { 109 | Pose rightEyePose = faceState->RightEyePose; 110 | 111 | expressions[FBExpression.RightRot_x] = rightEyePose.Orientation.X; 112 | expressions[FBExpression.RightRot_y] = rightEyePose.Orientation.Y; 113 | expressions[FBExpression.RightRot_z] = rightEyePose.Orientation.Z; 114 | expressions[FBExpression.RightRot_w] = rightEyePose.Orientation.W; 115 | 116 | expressions[FBExpression.RightPos_x] = rightEyePose.Position.X; 117 | expressions[FBExpression.RightPos_y] = rightEyePose.Position.Y; 118 | expressions[FBExpression.RightPos_z] = rightEyePose.Position.Z; 119 | 120 | flag = true; 121 | } 122 | 123 | if (faceState->FaceIsValid && faceState->IsEyeFollowingBlendshapesValid) 124 | { 125 | for(int i = 0; i < NATURAL_EXPRESSIONS_COUNT; ++i) 126 | expressions[i] = expressionWeights[i]; 127 | 128 | flag = true; 129 | } 130 | } 131 | this.IsTracking = new bool?(flag); 132 | } 133 | 134 | internal unsafe bool Initialize() 135 | { 136 | // MemoryMappedFile is supported only on Windows, therefore we should handle it gracefully. 137 | if (!OperatingSystem.IsWindows()) 138 | { 139 | VDFaceTracking.Warn("This mod can not run on non-windows OS. Virtual Desktop proxy initialization is going to be skipped."); 140 | return false; 141 | } 142 | 143 | try 144 | { 145 | int size = Marshal.SizeOf(); 146 | this._mappedFile = MemoryMappedFile.OpenExisting(BodyStateMapName, MemoryMappedFileRights.ReadWrite); 147 | this._mappedView = this._mappedFile.CreateViewAccessor(0L, (long)size); 148 | 149 | byte* numPtr = null; 150 | _mappedView.SafeMemoryMappedViewHandle.AcquirePointer(ref numPtr); 151 | this._faceState = (FaceState*) numPtr; 152 | this._faceStateEvent = EventWaitHandle.OpenExisting(BodyStateEventName); 153 | VDFaceTracking.Msg("Opened MemoryMappedFile. Everything should be working!"); 154 | 155 | cancellationTokenSource = new CancellationTokenSource(); 156 | thread = new Thread(UpdateThread); 157 | thread.Start(); 158 | 159 | return true; 160 | } 161 | catch 162 | { 163 | VDFaceTracking.Error("Failed to open MemoryMappedFile. Make sure the Virtual Desktop Streamer (v1.30 or later) is running."); 164 | return false; 165 | } 166 | } 167 | 168 | private void PrepareUpdate() 169 | { 170 | // Eye Expressions 171 | 172 | double q_x = expressions[FBExpression.LeftRot_x]; 173 | double q_y = expressions[FBExpression.LeftRot_y]; 174 | double q_z = expressions[FBExpression.LeftRot_z]; 175 | double q_w = expressions[FBExpression.LeftRot_w]; 176 | 177 | double yaw = Math.Atan2(2.0 * (q_y * q_z + q_w * q_x), q_w * q_w - q_x * q_x - q_y * q_y + q_z * q_z); 178 | double pitch = Math.Asin(-2.0 * (q_x * q_z - q_w * q_y)); 179 | // Not needed for eye tracking 180 | // double roll = Math.Atan2(2.0 * (q_x * q_y + q_w * q_z), q_w * q_w + q_x * q_x - q_y * q_y - q_z * q_z); 181 | 182 | // From radians 183 | pitch_L = 180.0 / Math.PI * pitch; 184 | yaw_L = 180.0 / Math.PI * yaw; 185 | 186 | q_x = expressions[FBExpression.RightRot_x]; 187 | q_y = expressions[FBExpression.RightRot_y]; 188 | q_z = expressions[FBExpression.RightRot_z]; 189 | q_w = expressions[FBExpression.RightRot_w]; 190 | 191 | yaw = Math.Atan2(2.0 * (q_y * q_z + q_w * q_x), q_w * q_w - q_x * q_x - q_y * q_y + q_z * q_z); 192 | pitch = Math.Asin(-2.0 * (q_x * q_z - q_w * q_y)); 193 | 194 | // From radians 195 | pitch_R = 180.0 / Math.PI * pitch; 196 | yaw_R = 180.0 / Math.PI * yaw; 197 | 198 | // Face Expressions 199 | 200 | // Eyelid edge case, eyes are actually closed now 201 | if (expressions[FBExpression.Eyes_Look_Down_L] == expressions[FBExpression.Eyes_Look_Up_L] && expressions[FBExpression.Eyes_Closed_L] > 0.25f) 202 | { 203 | expressions[FBExpression.Eyes_Closed_L] = 0; // 0.9f - (expressions[FBExpression.Lid_Tightener_L] * 3); 204 | } 205 | else 206 | { 207 | expressions[FBExpression.Eyes_Closed_L] = 0.9f - ((expressions[FBExpression.Eyes_Closed_L] * 3) / (1 + expressions[FBExpression.Eyes_Look_Down_L] * 3)); 208 | } 209 | 210 | // Another eyelid edge case 211 | if (expressions[FBExpression.Eyes_Look_Down_R] == expressions[FBExpression.Eyes_Look_Up_R] && expressions[FBExpression.Eyes_Closed_R] > 0.25f) 212 | { 213 | expressions[FBExpression.Eyes_Closed_R] = 0; // 0.9f - (expressions[FBExpression.Lid_Tightener_R] * 3); 214 | } 215 | else 216 | { 217 | expressions[FBExpression.Eyes_Closed_R] = 0.9f - ((expressions[FBExpression.Eyes_Closed_R] * 3) / (1 + expressions[FBExpression.Eyes_Look_Down_R] * 3)); 218 | } 219 | 220 | //expressions[FBExpression.Lid_Tightener_L = 0.8f-expressions[FBExpression.Eyes_Closed_L]; // Sad: fix combined param instead 221 | //expressions[FBExpression.Lid_Tightener_R = 0.8f-expressions[FBExpression.Eyes_Closed_R]; // Sad: fix combined param instead 222 | 223 | if (1 - expressions[FBExpression.Eyes_Closed_L] < expressions[FBExpression.Lid_Tightener_L]) 224 | expressions[FBExpression.Lid_Tightener_L] = (1 - expressions[FBExpression.Eyes_Closed_L]) - 0.01f; 225 | 226 | if (1 - expressions[FBExpression.Eyes_Closed_R] < expressions[FBExpression.Lid_Tightener_R]) 227 | expressions[FBExpression.Lid_Tightener_R] = (1 - expressions[FBExpression.Eyes_Closed_R]) - 0.01f; 228 | 229 | //expressions[FBExpression.Lid_Tightener_L = Math.Max(0, expressions[FBExpression.Lid_Tightener_L] - 0.15f); 230 | //expressions[FBExpression.Lid_Tightener_R = Math.Max(0, expressions[FBExpression.Lid_Tightener_R] - 0.15f); 231 | 232 | expressions[FBExpression.Upper_Lid_Raiser_L] = Math.Max(0, expressions[FBExpression.Upper_Lid_Raiser_L] - 0.5f); 233 | expressions[FBExpression.Upper_Lid_Raiser_R] = Math.Max(0, expressions[FBExpression.Upper_Lid_Raiser_R] - 0.5f); 234 | 235 | expressions[FBExpression.Lid_Tightener_L] = Math.Max(0, expressions[FBExpression.Lid_Tightener_L] - 0.5f); 236 | expressions[FBExpression.Lid_Tightener_R] = Math.Max(0, expressions[FBExpression.Lid_Tightener_R] - 0.5f); 237 | 238 | expressions[FBExpression.Inner_Brow_Raiser_L] = Math.Min(1, expressions[FBExpression.Inner_Brow_Raiser_L] * 3f); // * 4; 239 | expressions[FBExpression.Brow_Lowerer_L] = Math.Min(1, expressions[FBExpression.Brow_Lowerer_L] * 3f); // * 4; 240 | expressions[FBExpression.Outer_Brow_Raiser_L] = Math.Min(1, expressions[FBExpression.Outer_Brow_Raiser_L] * 3f); // * 4; 241 | 242 | expressions[FBExpression.Inner_Brow_Raiser_R] = Math.Min(1, expressions[FBExpression.Inner_Brow_Raiser_R] * 3f); // * 4; 243 | expressions[FBExpression.Brow_Lowerer_R] = Math.Min(1, expressions[FBExpression.Brow_Lowerer_R] * 3f); // * 4; 244 | expressions[FBExpression.Outer_Brow_Raiser_R] = Math.Min(1, expressions[FBExpression.Outer_Brow_Raiser_R] * 3f); // * 4; 245 | 246 | expressions[FBExpression.Eyes_Look_Up_L] = expressions[FBExpression.Eyes_Look_Up_L] * 0.55f; 247 | expressions[FBExpression.Eyes_Look_Up_R] = expressions[FBExpression.Eyes_Look_Up_R] * 0.55f; 248 | expressions[FBExpression.Eyes_Look_Down_L] = expressions[FBExpression.Eyes_Look_Down_L] * 1.5f; 249 | expressions[FBExpression.Eyes_Look_Down_R] = expressions[FBExpression.Eyes_Look_Down_R] * 1.5f; 250 | 251 | expressions[FBExpression.Eyes_Look_Left_L] = expressions[FBExpression.Eyes_Look_Left_L] * 0.85f; 252 | expressions[FBExpression.Eyes_Look_Right_L] = expressions[FBExpression.Eyes_Look_Right_L] * 0.85f; 253 | expressions[FBExpression.Eyes_Look_Left_R] = expressions[FBExpression.Eyes_Look_Left_R] * 0.85f; 254 | expressions[FBExpression.Eyes_Look_Right_R] = expressions[FBExpression.Eyes_Look_Right_R] * 0.85f; 255 | 256 | // Hack: turn rots to looks 257 | // Yitch = 29(left)-- > -29(right) 258 | // Yaw = -27(down)-- > 27(up) 259 | 260 | if (pitch_L > 0) 261 | { 262 | expressions[FBExpression.Eyes_Look_Left_L] = Math.Min(1, (float)(pitch_L / 29.0)) * SRANIPAL_NORMALIZER; 263 | expressions[FBExpression.Eyes_Look_Right_L] = 0; 264 | } 265 | else 266 | { 267 | expressions[FBExpression.Eyes_Look_Left_L] = 0; 268 | expressions[FBExpression.Eyes_Look_Right_L] = Math.Min(1, (float)((-pitch_L) / 29.0)) * SRANIPAL_NORMALIZER; 269 | } 270 | 271 | if (yaw_L > 0) 272 | { 273 | expressions[FBExpression.Eyes_Look_Up_L] = Math.Min(1, (float)(yaw_L / 27.0)) * SRANIPAL_NORMALIZER; 274 | expressions[FBExpression.Eyes_Look_Down_L] = 0; 275 | } 276 | else 277 | { 278 | expressions[FBExpression.Eyes_Look_Up_L] = 0; 279 | expressions[FBExpression.Eyes_Look_Down_L] = Math.Min(1, (float)((-yaw_L) / 27.0)) * SRANIPAL_NORMALIZER; 280 | } 281 | 282 | 283 | if (pitch_R > 0) 284 | { 285 | expressions[FBExpression.Eyes_Look_Left_R] = Math.Min(1, (float)(pitch_R / 29.0)) * SRANIPAL_NORMALIZER; 286 | expressions[FBExpression.Eyes_Look_Right_R] = 0; 287 | } 288 | else 289 | { 290 | expressions[FBExpression.Eyes_Look_Left_R] = 0; 291 | expressions[FBExpression.Eyes_Look_Right_R] = Math.Min(1, (float)((-pitch_R) / 29.0)) * SRANIPAL_NORMALIZER; 292 | } 293 | 294 | if (yaw_R > 0) 295 | { 296 | expressions[FBExpression.Eyes_Look_Up_R] = Math.Min(1, (float)(yaw_R / 27.0)) * SRANIPAL_NORMALIZER; 297 | expressions[FBExpression.Eyes_Look_Down_R] = 0; 298 | } 299 | else 300 | { 301 | expressions[FBExpression.Eyes_Look_Up_R] = 0; 302 | expressions[FBExpression.Eyes_Look_Down_R] = Math.Min(1, (float)((-yaw_R) / 27.0)) * SRANIPAL_NORMALIZER; 303 | } 304 | } 305 | 306 | internal unsafe void Teardown() 307 | { 308 | cancellationTokenSource.Cancel(); 309 | 310 | if (thread != null) 311 | thread.Interrupt(); 312 | 313 | cancellationTokenSource.Dispose(); 314 | 315 | if ((IntPtr)_faceState != IntPtr.Zero) 316 | { 317 | _faceState = (FaceState*)null; 318 | if (_mappedView != null) 319 | { 320 | _mappedView.Dispose(); 321 | _mappedView = null; 322 | } 323 | if (_mappedFile != null) 324 | { 325 | _mappedFile.Dispose(); 326 | _mappedFile = null; 327 | } 328 | } 329 | if (_faceStateEvent != null) 330 | { 331 | _faceStateEvent.Dispose(); 332 | _faceStateEvent = null; 333 | } 334 | _isTracking = new bool?(); 335 | } 336 | 337 | bool IsValid(float3 value) => IsValid(value.x) && IsValid(value.y) && IsValid(value.z); 338 | bool IsValid(floatQ value) => IsValid(value.x) && IsValid(value.y) && IsValid(value.z) && IsValid(value.w) && InRange(value.x, new float2(1, -1)) && InRange(value.y, new float2(1, -1)) && InRange(value.z, new float2(1, -1)) && InRange(value.w, new float2(1, -1)); 339 | 340 | bool IsValid(float value) => !float.IsInfinity(value) && !float.IsNaN(value); 341 | 342 | bool InRange(float value, float2 range) => (value <= range.x && value >= range.y); 343 | 344 | public struct EyeGazeData 345 | { 346 | public bool isValid; 347 | public float3 position; 348 | public floatQ rotation; 349 | public float open; 350 | public float squeeze; 351 | public float wide; 352 | public float gazeConfidence; 353 | } 354 | 355 | public EyeGazeData GetEyeData(FBEye fbEye) 356 | { 357 | EyeGazeData eyeRet = new EyeGazeData(); 358 | switch (fbEye) 359 | { 360 | case FBEye.Left: 361 | eyeRet.position = new float3(expressions[FBExpression.LeftPos_x], -expressions[FBExpression.LeftPos_y], expressions[FBExpression.LeftPos_z]); 362 | eyeRet.rotation = new floatQ(-expressions[FBExpression.LeftRot_x], -expressions[FBExpression.LeftRot_y], -expressions[FBExpression.LeftRot_z], expressions[FBExpression.LeftRot_w]); 363 | eyeRet.open = MathX.Max(0, expressions[FBExpression.Eyes_Closed_L]); 364 | eyeRet.squeeze = expressions[FBExpression.Lid_Tightener_L]; 365 | eyeRet.wide = expressions[FBExpression.Upper_Lid_Raiser_L]; 366 | eyeRet.isValid = IsValid(eyeRet.position); 367 | return eyeRet; 368 | case FBEye.Right: 369 | eyeRet.position = new float3(expressions[FBExpression.RightPos_x], -expressions[FBExpression.RightPos_y], expressions[FBExpression.RightPos_z]); 370 | eyeRet.rotation = new floatQ(-expressions[FBExpression.LeftRot_x], -expressions[FBExpression.LeftRot_y], -expressions[FBExpression.LeftRot_z], expressions[FBExpression.RightRot_w]); 371 | eyeRet.open = MathX.Max(0, expressions[FBExpression.Eyes_Closed_R]); 372 | eyeRet.squeeze = expressions[FBExpression.Lid_Tightener_R]; 373 | eyeRet.wide = expressions[FBExpression.Upper_Lid_Raiser_R]; 374 | eyeRet.isValid = IsValid(eyeRet.position); 375 | return eyeRet; 376 | default: 377 | throw new Exception($"Invalid eye argument: {fbEye}"); 378 | } 379 | } 380 | 381 | public void GetEyeExpressions(FBEye fbEye, Eye frooxEye) 382 | { 383 | frooxEye.PupilDiameter = 0.004f; 384 | 385 | switch (fbEye) 386 | { 387 | case FBEye.Left: 388 | frooxEye.UpdateWithRotation(new floatQ(-expressions[FBExpression.LeftRot_x], -expressions[FBExpression.LeftRot_z], -expressions[FBExpression.LeftRot_y], expressions[FBExpression.LeftRot_w])); 389 | frooxEye.RawPosition = new float3(expressions[FBExpression.LeftPos_x], expressions[FBExpression.LeftPos_y], expressions[FBExpression.LeftPos_z]); 390 | frooxEye.Openness = MathX.Max(0, expressions[FBExpression.Eyes_Closed_L]); 391 | frooxEye.Squeeze = expressions[FBExpression.Lid_Tightener_L]; 392 | frooxEye.Widen = expressions[FBExpression.Upper_Lid_Raiser_L]; 393 | frooxEye.Frown = expressions[FBExpression.Lip_Corner_Puller_L] - expressions[FBExpression.Lip_Corner_Depressor_L]; 394 | break; 395 | case FBEye.Right: 396 | frooxEye.UpdateWithRotation(new floatQ(-expressions[FBExpression.RightRot_x], -expressions[FBExpression.RightRot_z], -expressions[FBExpression.RightRot_y], expressions[FBExpression.RightRot_w])); 397 | frooxEye.RawPosition = new float3(expressions[FBExpression.RightPos_x], expressions[FBExpression.RightPos_y], expressions[FBExpression.RightPos_z]); 398 | frooxEye.Openness = MathX.Max(0, expressions[FBExpression.Eyes_Closed_R]); 399 | frooxEye.Squeeze = expressions[FBExpression.Lid_Tightener_R]; 400 | frooxEye.Widen = expressions[FBExpression.Upper_Lid_Raiser_R]; 401 | frooxEye.Frown = expressions[FBExpression.Lip_Corner_Puller_R] - expressions[FBExpression.Lip_Corner_Depressor_R]; 402 | break; 403 | case FBEye.Combined: 404 | frooxEye.UpdateWithRotation(MathX.Slerp(new floatQ(expressions[FBExpression.LeftRot_x], expressions[FBExpression.LeftRot_y], expressions[FBExpression.LeftRot_z], expressions[FBExpression.LeftRot_w]), new floatQ(expressions[FBExpression.RightRot_x], expressions[FBExpression.RightRot_y], expressions[FBExpression.RightRot_z], expressions[FBExpression.RightRot_w]), 0.5f)); 405 | frooxEye.RawPosition = MathX.Average(new float3(expressions[FBExpression.LeftPos_x], expressions[FBExpression.LeftPos_z], expressions[FBExpression.LeftPos_y]), new float3(expressions[FBExpression.RightPos_x], expressions[FBExpression.RightPos_z], expressions[FBExpression.RightPos_y])); 406 | frooxEye.Openness = MathX.Max(0, expressions[FBExpression.Eyes_Closed_R] + expressions[FBExpression.Eyes_Closed_R]) / 2.0f; 407 | frooxEye.Squeeze = (expressions[FBExpression.Lid_Tightener_R] + expressions[FBExpression.Lid_Tightener_R]) / 2.0f; 408 | frooxEye.Widen = (expressions[FBExpression.Upper_Lid_Raiser_R] + expressions[FBExpression.Upper_Lid_Raiser_R]) / 2.0f; 409 | frooxEye.Frown = (expressions[FBExpression.Lip_Corner_Puller_R] - expressions[FBExpression.Lip_Corner_Depressor_R]) + (expressions[FBExpression.Lip_Corner_Puller_L] - expressions[FBExpression.Lip_Corner_Depressor_L]) / 2.0f; 410 | break; 411 | } 412 | 413 | frooxEye.IsTracking = IsValid(frooxEye.RawPosition); 414 | frooxEye.IsTracking = IsValid(frooxEye.Direction); 415 | frooxEye.IsTracking = IsValid(frooxEye.Openness); 416 | } 417 | 418 | public void CollectDeviceInfos(DataTreeList list) 419 | { 420 | var eyeDataTreeDictionary = new DataTreeDictionary(); 421 | eyeDataTreeDictionary.Add("Name", "Quest Pro Eye Tracking"); 422 | eyeDataTreeDictionary.Add("Type", "Eye Tracking"); 423 | eyeDataTreeDictionary.Add("Model", "Quest Pro"); 424 | list.Add(eyeDataTreeDictionary); 425 | 426 | var mouthDataTreeDictionary = new DataTreeDictionary(); 427 | mouthDataTreeDictionary.Add("Name", "Quest Pro Face Tracking"); 428 | mouthDataTreeDictionary.Add("Type", "Lip Tracking"); 429 | mouthDataTreeDictionary.Add("Model", "Quest Pro"); 430 | list.Add(mouthDataTreeDictionary); 431 | } 432 | 433 | public void RegisterInputs(InputInterface inputInterface) 434 | { 435 | _input = inputInterface; 436 | eyes = new Eyes(_input, "Quest Pro Eye Tracking", false); 437 | mouth = new Mouth(_input, "Quest Pro Face Tracking", new MouthParameterGroup[] { 438 | MouthParameterGroup.JawOpen, 439 | MouthParameterGroup.JawPose, 440 | MouthParameterGroup.TonguePose, 441 | MouthParameterGroup.LipRaise, 442 | MouthParameterGroup.LipHorizontal, 443 | MouthParameterGroup.SmileFrown, 444 | MouthParameterGroup.MouthDimple, 445 | MouthParameterGroup.MouthPout, 446 | MouthParameterGroup.LipOverturn, 447 | MouthParameterGroup.LipOverUnder, 448 | MouthParameterGroup.LipStretchTighten, 449 | MouthParameterGroup.LipsPress, 450 | MouthParameterGroup.CheekPuffSuck, 451 | MouthParameterGroup.CheekRaise, 452 | MouthParameterGroup.ChinRaise, 453 | MouthParameterGroup.NoseWrinkle 454 | }); 455 | } 456 | 457 | public void UpdateInputs(float deltaTime) 458 | { 459 | UpdateMouth(deltaTime); 460 | UpdateEyes(deltaTime); 461 | } 462 | 463 | void UpdateEye(Eye eye, EyeGazeData data) 464 | { 465 | bool _isValid = IsValid(data.open); 466 | _isValid &= IsValid(data.position); 467 | _isValid &= IsValid(data.wide); 468 | _isValid &= IsValid(data.squeeze); 469 | _isValid &= IsValid(data.rotation); 470 | _isValid &= eye.IsTracking; 471 | 472 | eye.IsTracking = _isValid; 473 | 474 | if (eye.IsTracking) 475 | { 476 | eye.UpdateWithRotation(MathX.Slerp(floatQ.Identity, data.rotation, VDFaceTracking.EyeMoveMult)); 477 | eye.Openness = MathX.Pow(MathX.FilterInvalid(data.open, 0.0f), VDFaceTracking.EyeOpenExponent); 478 | eye.Widen = data.wide * VDFaceTracking.EyeWideMult; 479 | } 480 | } 481 | 482 | void UpdateEyes(float deltaTime) 483 | { 484 | eyes.IsEyeTrackingActive = _input.VR_Active; 485 | 486 | eyes.LeftEye.IsTracking = _input.VR_Active; 487 | 488 | var leftEyeData = VDFaceTracking.proxy.GetEyeData(FBEye.Left); 489 | var rightEyeData = VDFaceTracking.proxy.GetEyeData(FBEye.Right); 490 | 491 | eyes.LeftEye.IsTracking = leftEyeData.isValid; 492 | eyes.LeftEye.RawPosition = leftEyeData.position; 493 | eyes.LeftEye.PupilDiameter = 0.004f; 494 | eyes.LeftEye.Squeeze = leftEyeData.squeeze; 495 | eyes.LeftEye.Frown = expressions[FBExpression.Lip_Corner_Puller_L] - expressions[FBExpression.Lip_Corner_Depressor_L] * VDFaceTracking.EyeExpressionMult; 496 | eyes.LeftEye.InnerBrowVertical = expressions[FBExpression.Inner_Brow_Raiser_L]; 497 | eyes.LeftEye.OuterBrowVertical = expressions[FBExpression.Outer_Brow_Raiser_L]; 498 | eyes.LeftEye.Squeeze = expressions[FBExpression.Brow_Lowerer_L]; 499 | 500 | UpdateEye(eyes.LeftEye, leftEyeData); 501 | 502 | eyes.RightEye.IsTracking = rightEyeData.isValid; 503 | eyes.RightEye.RawPosition = rightEyeData.position; 504 | eyes.RightEye.PupilDiameter = 0.004f; 505 | eyes.RightEye.Squeeze = rightEyeData.squeeze; 506 | eyes.RightEye.Frown = expressions[FBExpression.Lip_Corner_Puller_R] - expressions[FBExpression.Lip_Corner_Depressor_R] * VDFaceTracking.EyeExpressionMult; 507 | eyes.RightEye.InnerBrowVertical = expressions[FBExpression.Inner_Brow_Raiser_R]; 508 | eyes.RightEye.OuterBrowVertical = expressions[FBExpression.Outer_Brow_Raiser_R]; 509 | eyes.RightEye.Squeeze = expressions[FBExpression.Brow_Lowerer_R]; 510 | 511 | UpdateEye(eyes.RightEye, rightEyeData); 512 | 513 | if (eyes.LeftEye.IsTracking || eyes.RightEye.IsTracking && (!eyes.LeftEye.IsTracking || !eyes.RightEye.IsTracking)) 514 | { 515 | if (eyes.LeftEye.IsTracking) 516 | { 517 | eyes.CombinedEye.RawPosition = eyes.LeftEye.RawPosition; 518 | eyes.CombinedEye.UpdateWithRotation(eyes.LeftEye.RawRotation); 519 | } 520 | else 521 | { 522 | eyes.CombinedEye.RawPosition = eyes.RightEye.RawPosition; 523 | eyes.CombinedEye.UpdateWithRotation(eyes.RightEye.RawRotation); 524 | } 525 | eyes.CombinedEye.IsTracking = true; 526 | } 527 | else 528 | { 529 | eyes.CombinedEye.IsTracking = false; 530 | } 531 | 532 | eyes.CombinedEye.IsTracking = eyes.LeftEye.IsTracking || eyes.RightEye.IsTracking; 533 | eyes.CombinedEye.RawPosition = (eyes.LeftEye.RawPosition + eyes.RightEye.RawPosition) * 0.5f; 534 | eyes.CombinedEye.UpdateWithRotation(MathX.Slerp(eyes.LeftEye.RawRotation, eyes.RightEye.RawRotation, 0.5f)); 535 | eyes.CombinedEye.PupilDiameter = 0.004f; 536 | 537 | eyes.LeftEye.Openness = MathX.Pow(1.0f - Math.Max(0, Math.Min(1, expressions[(int)Expressions.EyesClosedL] + expressions[(int)Expressions.EyesClosedL] * expressions[(int)Expressions.LidTightenerL])), VDFaceTracking.EyeOpenExponent); 538 | eyes.RightEye.Openness = MathX.Pow(1.0f - (float)Math.Max(0, Math.Min(1, expressions[(int)Expressions.EyesClosedR] + expressions[(int)Expressions.EyesClosedR] * expressions[(int)Expressions.LidTightenerR])), VDFaceTracking.EyeOpenExponent); 539 | 540 | eyes.ComputeCombinedEyeParameters(); 541 | eyes.ConvergenceDistance = 0f; 542 | eyes.Timestamp += deltaTime; 543 | eyes.FinishUpdate(); 544 | } 545 | 546 | void UpdateMouth(float deltaTime) 547 | { 548 | mouth.IsDeviceActive = Engine.Current.InputInterface.VR_Active; 549 | mouth.IsTracking = Engine.Current.InputInterface.VR_Active; 550 | 551 | // Pulled from Resonite: 552 | mouth.IsTracking = true; 553 | mouth.MouthLeftSmileFrown = expressions[FBExpression.Lip_Corner_Puller_L] - expressions[FBExpression.Lip_Corner_Depressor_L]; 554 | mouth.MouthRightSmileFrown = expressions[FBExpression.Lip_Corner_Puller_R] - expressions[FBExpression.Lip_Corner_Depressor_R]; 555 | mouth.MouthLeftDimple = expressions[FBExpression.Dimpler_L]; 556 | mouth.MouthRightDimple = expressions[FBExpression.Dimpler_R]; 557 | mouth.CheekLeftPuffSuck = expressions[FBExpression.Cheek_Puff_L] - expressions[FBExpression.Cheek_Suck_L]; 558 | mouth.CheekRightPuffSuck = expressions[FBExpression.Cheek_Puff_R] - expressions[FBExpression.Cheek_Suck_R]; 559 | mouth.CheekLeftRaise = expressions[FBExpression.Cheek_Raiser_L]; 560 | mouth.CheekRightRaise = expressions[FBExpression.Cheek_Raiser_R]; 561 | mouth.LipUpperLeftRaise = expressions[FBExpression.Upper_Lip_Raiser_L]; 562 | mouth.LipUpperRightRaise = expressions[FBExpression.Upper_Lip_Raiser_R]; 563 | mouth.LipLowerLeftRaise = expressions[FBExpression.Lower_Lip_Depressor_L]; 564 | mouth.LipLowerRightRaise = expressions[FBExpression.Lower_Lip_Depressor_R]; 565 | mouth.MouthPoutLeft = expressions[FBExpression.Lip_Pucker_L]; 566 | mouth.MouthPoutRight = expressions[FBExpression.Lip_Pucker_R]; 567 | mouth.LipUpperHorizontal = expressions[FBExpression.Mouth_Right] - expressions[FBExpression.Mouth_Left]; 568 | mouth.LipLowerHorizontal = mouth.LipUpperHorizontal; 569 | mouth.LipTopLeftOverturn = expressions[FBExpression.Lip_Funneler_LT]; 570 | mouth.LipTopRightOverturn = expressions[FBExpression.Lip_Funneler_RT]; 571 | mouth.LipBottomLeftOverturn = expressions[FBExpression.Lip_Funneler_LB]; 572 | mouth.LipBottomRightOverturn = expressions[FBExpression.Lip_Funneler_RB]; 573 | mouth.LipTopLeftOverUnder = -expressions[FBExpression.Lip_Suck_LT]; 574 | mouth.LipTopRightOverUnder = -expressions[FBExpression.Lip_Suck_RT]; 575 | mouth.LipBottomLeftOverUnder = -expressions[FBExpression.Lip_Suck_LB]; 576 | mouth.LipBottomRightOverUnder = -expressions[FBExpression.Lip_Suck_RB]; 577 | mouth.LipLeftStretchTighten = expressions[FBExpression.Lip_Stretcher_L] - expressions[FBExpression.Lid_Tightener_L]; 578 | mouth.LipRightStretchTighten = expressions[FBExpression.Lip_Stretcher_R] - expressions[FBExpression.Lid_Tightener_R]; 579 | mouth.LipsLeftPress = expressions[FBExpression.Lip_Pressor_L]; 580 | mouth.LipsRightPress = expressions[FBExpression.Lip_Pressor_R]; 581 | mouth.Jaw = new float3(expressions[FBExpression.Jaw_Sideways_Right] - expressions[FBExpression.Jaw_Sideways_Left], -expressions[FBExpression.Lips_Toward], expressions[FBExpression.Jaw_Thrust]); 582 | mouth.JawOpen = MathX.Clamp01(expressions[FBExpression.Jaw_Drop] - expressions[FBExpression.Lips_Toward]); 583 | mouth.Tongue = new float3(0f, 0f, expressions[FBExpression.TongueOut] - expressions[FBExpression.TongueRetreat]); 584 | mouth.NoseWrinkleLeft = expressions[FBExpression.Nose_Wrinkler_L]; 585 | mouth.NoseWrinkleRight = expressions[FBExpression.Nose_Wrinkler_R]; 586 | mouth.ChinRaiseBottom = expressions[FBExpression.Chin_Raiser_B]; 587 | mouth.ChinRaiseTop = expressions[FBExpression.Chin_Raiser_T]; 588 | } 589 | } 590 | 591 | public enum FBEye 592 | { 593 | Left, 594 | Right, 595 | Combined 596 | } 597 | } --------------------------------------------------------------------------------