├── MediaPipeFaceSolver.cs ├── MediaPipePoseSolver.cs ├── README.md └── VectorExtensions.cs /MediaPipeFaceSolver.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using UnityEngine; 5 | using Mediapipe.Unity.CoordinateSystem; 6 | using Mediapipe.Unity.Holistic; 7 | using Google.Protobuf; 8 | using Mediapipe; 9 | using Mediapipe.Unity; 10 | 11 | public class MediaPipeFaceSolver : MonoBehaviour 12 | { 13 | public struct Face 14 | { 15 | public Head Head; 16 | public Eyes Eyes; 17 | // It only has one value for the eye brows and pupil in the original code (Kaleido) 18 | // TODO: investigate getting 2 different brow and pupil values 19 | public float Brow; 20 | public Vector2 Pupils; 21 | public Mouth Mouth; 22 | } 23 | 24 | public struct Head 25 | { 26 | // X, Y and Z represent 3D radian angles. 27 | public float X; 28 | public float Y; 29 | public float Z; 30 | 31 | public float Width; 32 | public float Height; 33 | 34 | /// Center of face detection square. 35 | public Vector3 Position; 36 | // TODO: convert to a Quaternion 37 | // Euler angles normalized between -1 and 1. 38 | public Vector3 NormalizedAngles; 39 | } 40 | 41 | public struct Mouth 42 | { 43 | // Horizontal mouth open 44 | public float X; 45 | // Vertical mouth open 46 | public float Y; 47 | // Mouth vowel shape 48 | public Phoneme Shape; 49 | } 50 | 51 | public struct Phoneme 52 | { 53 | // 'A' shape. 54 | public float A; 55 | // 'E' shape. 56 | public float E; 57 | // 'I' shape. 58 | public float I; 59 | // 'O' shape. 60 | public float O; 61 | // 'U' shape. 62 | public float U; 63 | } 64 | 65 | public struct Eyes 66 | { 67 | // Wideness of left eye 68 | public float Left; 69 | /// Wideness of right eye 70 | public float Right; 71 | } 72 | 73 | public readonly int[] EyeLeftPoints = new int[] { 130, 133, 160, 159, 158, 144, 145, 153 }; 74 | public readonly int[] EyeRightPoints = new int[] { 263, 362, 387, 386, 385, 373, 374, 380 }; 75 | public readonly int[] BrowLeftPoints = new int[] { 35, 244, 63, 105, 66, 229, 230, 231 }; 76 | public readonly int[] BrowRightPoints = new int[] { 265, 464, 293, 334, 296, 449, 450, 451 }; 77 | public readonly int[] PupilLeftPoints = new int[] { 468, 469, 470, 471, 472 }; 78 | public readonly int[] PupilRightPoints = new int[] { 473, 474, 475, 476, 477 }; 79 | 80 | public Vector3 ToVector(Landmark landmark) => new(landmark.X, landmark.Y, landmark.Z); 81 | public Vector2 ToVector2(Vector3 vector) => new(vector.x, vector.y); 82 | 83 | public Vector3 ToVector(NormalizedLandmark landmark) => new(landmark.X, landmark.Y, landmark.Z); 84 | public Vector2 ToVector2(NormalizedLandmark landmark) => new(landmark.X, landmark.Y); 85 | 86 | public Face Solve( 87 | NormalizedLandmarkList list, 88 | bool smoothBlink = false, 89 | float blinkHigh = .35f, 90 | float blinkLow = .5f 91 | ) 92 | { 93 | Head head = CalcHead(list); 94 | Mouth mouth = CalcMouth(list); 95 | 96 | Eyes eyes = CalcEyes(list, blinkHigh, blinkLow); 97 | 98 | if (smoothBlink) 99 | StabilizeBlink(ref eyes, head.Y); 100 | 101 | Vector2 pupils = CalcPupils(list); 102 | float brow = CalcBrow(list); 103 | 104 | return new Face 105 | { 106 | Head = head, 107 | Eyes = eyes, 108 | Brow = brow, 109 | Pupils = pupils, 110 | Mouth = mouth, 111 | }; 112 | } 113 | 114 | public Mouth CalcMouth(NormalizedLandmarkList list) 115 | { 116 | var landmarks = list.Landmark; 117 | 118 | // Eye keypoints 119 | Vector3 eyeInnerCornerL = ToVector(landmarks[133]); 120 | Vector3 eyeInnerCornerR = ToVector(landmarks[362]); 121 | Vector3 eyeOuterCornerL = ToVector(landmarks[130]); 122 | Vector3 eyeOuterCornerR = ToVector(landmarks[263]); 123 | 124 | // Eye keypoint distances 125 | float eyeInnerDistance = Vector3.Distance(eyeInnerCornerL, eyeInnerCornerR); 126 | float eyeOuterDistance = Vector3.Distance(eyeOuterCornerL, eyeOuterCornerR); 127 | 128 | // Mouth keypoints 129 | Vector3 upperInnerLip = ToVector(landmarks[13]); 130 | Vector3 lowerInnerLip = ToVector(landmarks[14]); 131 | Vector3 mouthCornerLeft = ToVector(landmarks[61]); 132 | Vector3 mouthCornerRight = ToVector(landmarks[291]); 133 | 134 | // Mouth keypoint distances 135 | float mouthOpen = Vector3.Distance(upperInnerLip, lowerInnerLip); 136 | float mouthWidth = Vector3.Distance(mouthCornerLeft, mouthCornerRight); 137 | 138 | // Mouth open and mouth shape ratios 139 | float ratioY = mouthOpen / eyeInnerDistance; 140 | float ratioX = mouthWidth / eyeOuterDistance; 141 | 142 | // Normalize and scale mouth open 143 | ratioY = ratioY.Remap(0.15f, 0.7f); 144 | 145 | // Normalize and scale mouth shape 146 | ratioX = ratioX.Remap(0.45f, 0.9f); 147 | ratioX = (ratioX - 0.3f) * 2; 148 | 149 | float mouthX = ratioX; 150 | float mouthY = (mouthOpen / eyeInnerDistance).Remap(0.17f, 0.5f); 151 | 152 | float ratioI = Math.Clamp(mouthX.Remap(0, 1) * 2 * mouthY.Remap(0.2f, 0.7f), 0, 1); 153 | float ratioA = mouthY * 0.4f + mouthY * (1 - ratioI) * 0.6f; 154 | float ratioU = mouthY * (1 - ratioI).Remap(0, 0.3f) * 0.1f; 155 | float ratioE = ratioU.Remap(0.2f, 1) * (1 - ratioI) * 0.3f; 156 | float ratioO = (1 - ratioI) * mouthY.Remap(0.3f, 1) * 0.4f; 157 | 158 | return new Mouth 159 | { 160 | X = ratioX, 161 | Y = ratioY, 162 | Shape = new Phoneme 163 | { 164 | A = ratioA, 165 | E = ratioE, 166 | I = ratioI, 167 | O = ratioO, 168 | U = ratioU, 169 | }, 170 | }; 171 | } 172 | 173 | public Head CalcHead(NormalizedLandmarkList list) 174 | { 175 | // Find 3 vectors that form a plane to represent the head 176 | Vector3[] plane = FaceEulerPlane(list); 177 | Vector3 rotate = VectorExtensions.RollPitchYaw(plane[0], plane[1], plane[2]); 178 | // Find center of face detection box 179 | Vector3 midPoint = Vector3.Lerp(plane[0], plane[1], 0.5f); 180 | // Roughly find the dimensions of the face detection box 181 | float width = Vector3.Distance(plane[0], plane[1]); 182 | float height = Vector3.Distance(midPoint, plane[2]); 183 | 184 | // Flip 185 | rotate.x *= -1; 186 | rotate.y *= -1; 187 | 188 | return new Head 189 | { 190 | X = rotate.x * MathF.PI, 191 | Y = rotate.y * MathF.PI, 192 | Z = rotate.z * MathF.PI, 193 | Width = width, 194 | Height = height, 195 | Position = Vector3.Lerp(midPoint, plane[2], 0.5f), 196 | NormalizedAngles = new Vector3(rotate.x, rotate.y, rotate.z), 197 | }; 198 | } 199 | 200 | public Vector3[] FaceEulerPlane(NormalizedLandmarkList list) 201 | { 202 | // TODO: This vector processing could probably be optimised 203 | var landmarks = list.Landmark; 204 | 205 | // Create face detection square bounds 206 | Vector3 topLeft = ToVector(landmarks[21]); 207 | Vector3 topRight = ToVector(landmarks[251]); 208 | 209 | Vector3 bottomRight = ToVector(landmarks[397]); 210 | Vector3 bottomLeft = ToVector(landmarks[172]); 211 | 212 | Vector3 bottomMidpoint = Vector3.Lerp(bottomRight, bottomLeft, 0.5f); 213 | 214 | return new Vector3[] { topLeft, topRight, bottomMidpoint }; 215 | } 216 | 217 | public float GetEyeOpen(NormalizedLandmarkList list, string side, float high = .85f, float low = .55f) 218 | { 219 | var landmarks = list.Landmark; 220 | 221 | int[] eyePoints = side == "right" ? EyeRightPoints : EyeLeftPoints; 222 | float eyeDistance = EyeLidRatio( 223 | landmarks[eyePoints[0]], 224 | landmarks[eyePoints[1]], 225 | landmarks[eyePoints[2]], 226 | landmarks[eyePoints[3]], 227 | landmarks[eyePoints[4]], 228 | landmarks[eyePoints[5]], 229 | landmarks[eyePoints[6]], 230 | landmarks[eyePoints[7]] 231 | ); 232 | 233 | // Human eye width to height ratio is roughly .3 234 | float maxRatio = 0.285f; 235 | // Compare ratio against max ratio 236 | float ratio = Math.Clamp(eyeDistance / maxRatio, 0, 2); 237 | // Remap eye open and close ratios to increase sensitivity 238 | float eyeOpenRatio = ratio.Remap(low, high); 239 | 240 | return eyeOpenRatio; 241 | } 242 | 243 | public float EyeLidRatio( 244 | NormalizedLandmark outerCorner, 245 | NormalizedLandmark innerCorner, 246 | NormalizedLandmark outerUpperLid, 247 | NormalizedLandmark midUpperLid, 248 | NormalizedLandmark innerUpperLid, 249 | NormalizedLandmark outerLowerLid, 250 | NormalizedLandmark midLowerLid, 251 | NormalizedLandmark innerLowerLid) 252 | { 253 | Vector2 eyeOuterCorner = ToVector2(outerCorner); 254 | Vector2 eyeInnerCorner = ToVector2(innerCorner); 255 | 256 | Vector2 eyeOuterUpperLid = ToVector2(outerUpperLid); 257 | Vector2 eyeMidUpperLid = ToVector2(midUpperLid); 258 | Vector2 eyeInnerUpperLid = ToVector2(innerUpperLid); 259 | 260 | Vector2 eyeOuterLowerLid = ToVector2(outerLowerLid); 261 | Vector2 eyeMidLowerLid = ToVector2(midLowerLid); 262 | Vector2 eyeInnerLowerLid = ToVector2(innerLowerLid); 263 | 264 | // Use 2D Distances instead of 3D for less jitter 265 | float eyeWidth = Vector2.Distance(eyeOuterCorner, eyeInnerCorner); 266 | float eyeOuterLidDistance = Vector2.Distance(eyeOuterUpperLid, eyeOuterLowerLid); 267 | float eyeMidLidDistance = Vector2.Distance(eyeMidUpperLid, eyeMidLowerLid); 268 | float eyeInnerLidDistance = Vector2.Distance(eyeInnerUpperLid, eyeInnerLowerLid); 269 | float eyeLidAvg = (eyeOuterLidDistance + eyeMidLidDistance + eyeInnerLidDistance) / 3; 270 | float ratio = eyeLidAvg / eyeWidth; 271 | 272 | return ratio; 273 | } 274 | 275 | // Calculates pupil position [-1, 1]. 276 | public Vector2 PupilPos(NormalizedLandmarkList list, string side) 277 | { 278 | var landmarks = list.Landmark; 279 | 280 | int[] eyePoints = side == "right" ? EyeRightPoints : EyeLeftPoints; 281 | Vector3 eyeOuterCorner = ToVector(landmarks[eyePoints[0]]); 282 | Vector3 eyeInnerCorner = ToVector(landmarks[eyePoints[1]]); 283 | float eyeWidth = Vector2.Distance(ToVector2(eyeOuterCorner), ToVector2(eyeInnerCorner)); 284 | Vector3 midPoint = Vector3.Lerp(eyeOuterCorner, eyeInnerCorner, .5f); 285 | 286 | int[] pupilPoints = side == "right" ? PupilRightPoints : PupilLeftPoints; 287 | Vector3 pupil = ToVector(landmarks[pupilPoints[0]]); 288 | float dx = midPoint.x - pupil.x; 289 | float dy = midPoint.y - pupil.y - eyeWidth * .075f; 290 | 291 | float ratioX = 4 * dx / (eyeWidth / 2); 292 | float ratioY = 4 * dy / (eyeWidth / 4); 293 | 294 | return new Vector2(ratioX, ratioY); 295 | } 296 | 297 | public void StabilizeBlink(ref Eyes eyes, float headY, bool enableWink = true, float maxRotation = .5f) 298 | { 299 | eyes.Left = Math.Clamp(eyes.Left, 0, 1); 300 | eyes.Right = Math.Clamp(eyes.Right, 0, 1); 301 | 302 | // Difference between each eye 303 | float blinkDiff = MathF.Abs(eyes.Left - eyes.Right); 304 | // Threshold to which difference is considered a wink 305 | float blinkThresh = enableWink ? .8f : 1.2f; 306 | 307 | bool isClosing = eyes.Left < .3f && eyes.Right < .3f; 308 | bool isOpening = eyes.Left > .6f && eyes.Right > .6f; 309 | 310 | // Sets obstructed eye to the opposite eye value 311 | if (headY > maxRotation) 312 | { 313 | eyes.Left = eyes.Right; 314 | return; 315 | } 316 | if (headY < -maxRotation) 317 | { 318 | eyes.Right = eyes.Left; 319 | return; 320 | } 321 | 322 | // Wink of averaged blink values 323 | if (!(blinkDiff >= blinkThresh && !isClosing && !isOpening)) 324 | { 325 | float value = Mathf.Lerp(eyes.Right, eyes.Left, eyes.Right > eyes.Left ? .95f : .05f); 326 | eyes.Left = value; 327 | eyes.Right = value; 328 | } 329 | } 330 | 331 | // Calculate eyes. 332 | public Eyes CalcEyes(NormalizedLandmarkList list, float high = .85f, float low = .55f) 333 | { 334 | var landmarks = list.Landmark; 335 | 336 | // Return early if no iris tracking 337 | if (landmarks.Count != 478) 338 | { 339 | return new Eyes 340 | { 341 | Left = 1, 342 | Right = 1, 343 | }; 344 | } 345 | 346 | // Open [0, 1] 347 | return new Eyes 348 | { 349 | Left = GetEyeOpen(list, "left", high, low), 350 | Right = GetEyeOpen(list, "right", high, low), 351 | }; 352 | } 353 | 354 | // Calculate pupil location normalized to eye bounds 355 | public Vector2 CalcPupils(NormalizedLandmarkList list) 356 | { 357 | var landmarks = list.Landmark; 358 | 359 | // Pupil (x: [-1, 1], y: [-1, 1]) 360 | if (landmarks.Count != 478) 361 | { 362 | return new Vector2(0, 0); 363 | } 364 | 365 | // Track pupils using left eye 366 | Vector2 pupilLeft = PupilPos(list, "left"); 367 | Vector2 pupilRight = PupilPos(list, "right"); 368 | 369 | return (pupilLeft + pupilRight) * .5f; 370 | } 371 | 372 | /// Calculate brow raise 373 | public float GetBrowRaise(NormalizedLandmarkList list, string side) 374 | { 375 | var landmarks = list.Landmark; 376 | 377 | int[] browPoints = side == "right" ? BrowRightPoints : BrowLeftPoints; 378 | float browDistance = EyeLidRatio( 379 | landmarks[browPoints[0]], 380 | landmarks[browPoints[1]], 381 | landmarks[browPoints[2]], 382 | landmarks[browPoints[3]], 383 | landmarks[browPoints[4]], 384 | landmarks[browPoints[5]], 385 | landmarks[browPoints[6]], 386 | landmarks[browPoints[7]] 387 | ); 388 | 389 | float maxBrowRatio = 1.15f; 390 | float browHigh = .125f; 391 | float browLow = .07f; 392 | float browRatio = browDistance / maxBrowRatio - 1; 393 | float browRaiseRatio = (Math.Clamp(browRatio, browLow, browHigh) - browLow) / (browHigh - browLow); 394 | 395 | return browRaiseRatio; 396 | } 397 | 398 | // Take the average of left and right eyebrow raise 399 | public float CalcBrow(NormalizedLandmarkList list) 400 | { 401 | var landmarks = list.Landmark; 402 | 403 | if (landmarks.Count != 478) 404 | return 0; 405 | 406 | float leftBrow = GetBrowRaise(list, "left"); 407 | float rightBrow = GetBrowRaise(list, "right"); 408 | 409 | return (leftBrow + rightBrow) / 2; 410 | } 411 | } 412 | -------------------------------------------------------------------------------- /MediaPipePoseSolver.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using UnityEngine; 5 | using Mediapipe.Unity.CoordinateSystem; 6 | using Mediapipe.Unity.Holistic; 7 | using Google.Protobuf; 8 | using Mediapipe; 9 | using Mediapipe.Unity; 10 | 11 | public class MediaPipePoseSolver : MonoBehaviour 12 | { 13 | public struct Pose 14 | { 15 | public Arm LeftArm; 16 | public Arm RightArm; 17 | public Leg LeftLeg; 18 | public Leg RightLeg; 19 | public Vector3 Spine; 20 | public Hips Hips; 21 | } 22 | 23 | public struct Arm 24 | { 25 | public Vector3 Upper; 26 | public Vector3 Lower; 27 | public Vector3 Hand; 28 | } 29 | 30 | public struct Leg 31 | { 32 | public Vector3 Upper; 33 | public Vector3 Lower; 34 | } 35 | 36 | public struct Hips 37 | { 38 | public Vector3 WorldPosition; 39 | public Vector3 Position; 40 | public Vector3 Rotation; 41 | } 42 | 43 | public Pose Solve(NormalizedLandmarkList normalizedLandmarkList) 44 | { 45 | (Arm leftArm, Arm rightArm) = CalculateArms(normalizedLandmarkList); 46 | (Hips hips, Vector3 spine) = CalculateHips(normalizedLandmarkList); 47 | 48 | var pose = new Pose 49 | { 50 | LeftArm = leftArm, 51 | RightArm = rightArm, 52 | Hips = hips, 53 | Spine = spine 54 | }; 55 | 56 | return pose; 57 | } 58 | 59 | public Vector3 ToVector(NormalizedLandmark landmark) => new(landmark.X, landmark.Y, landmark.Z); 60 | public Vector2 ToVector2(Vector3 vector) => new (vector.x, vector.y ); 61 | 62 | private (Arm, Arm) CalculateArms(NormalizedLandmarkList normalizedLandmarkList) 63 | { 64 | var landmarks = normalizedLandmarkList.Landmark; 65 | 66 | var rightArm = new Arm(); 67 | var leftArm = new Arm(); 68 | 69 | rightArm.Upper = VectorExtensions.FindRotation(ToVector(landmarks[11]), ToVector(landmarks[13]), true); 70 | leftArm.Upper = VectorExtensions.FindRotation(ToVector(landmarks[12]), ToVector(landmarks[14]), true); 71 | rightArm.Upper.y = VectorExtensions.AngleBetween3DCoords(ToVector(landmarks[12]), ToVector(landmarks[11]), ToVector(landmarks[13])); 72 | leftArm.Upper.y = VectorExtensions.AngleBetween3DCoords(ToVector(landmarks[11]), ToVector(landmarks[12]), ToVector(landmarks[14])); 73 | 74 | rightArm.Lower = VectorExtensions.FindRotation(ToVector(landmarks[13]), ToVector(landmarks[15]), true); 75 | leftArm.Lower = VectorExtensions.FindRotation(ToVector(landmarks[14]), ToVector(landmarks[16]), true); 76 | rightArm.Lower.y = VectorExtensions.AngleBetween3DCoords(ToVector(landmarks[11]), ToVector(landmarks[13]), ToVector(landmarks[15])); 77 | leftArm.Lower.y = VectorExtensions.AngleBetween3DCoords(ToVector(landmarks[12]), ToVector(landmarks[14]), ToVector(landmarks[16])); 78 | rightArm.Lower.z = Math.Clamp(rightArm.Lower.z, -2.14f, 0f); 79 | leftArm.Lower.z = Math.Clamp(leftArm.Lower.z, -2.14f, 0f); 80 | 81 | rightArm.Hand = VectorExtensions.FindRotation(ToVector(landmarks[15]), Vector3.Lerp(ToVector(landmarks[17]), ToVector(landmarks[19]), .5f), true); 82 | leftArm.Hand = VectorExtensions.FindRotation(ToVector(landmarks[16]), Vector3.Lerp(ToVector(landmarks[18]), ToVector(landmarks[20]), .5f), true); 83 | 84 | // Modify rotations slightly for more natural movement 85 | RigArm(ref rightArm, "right"); 86 | RigArm(ref leftArm, "left"); 87 | 88 | return (leftArm, rightArm); 89 | } 90 | 91 | private void RigArm(ref Arm arm, string side) 92 | { 93 | float invert = side == "right" ? 1f : -1f; 94 | 95 | arm.Upper.z *= -2.3f * invert; 96 | 97 | arm.Upper.y *= MathF.PI * invert; 98 | arm.Upper.y -= Math.Max(arm.Lower.x, 0); 99 | arm.Upper.y -= -invert * Math.Max(arm.Lower.z, 0); 100 | arm.Upper.x -= 0.3f * invert; 101 | 102 | arm.Lower.z *= -2.14f * invert; 103 | arm.Lower.y *= 2.14f * invert; 104 | arm.Lower.x *= 2.14f * invert; 105 | 106 | // Clamp values to realistic humanoid limits 107 | arm.Upper.x = Math.Clamp(arm.Upper.x, -0.5f, MathF.PI); 108 | arm.Lower.x = Math.Clamp(arm.Lower.x, -0.3f, 0.3f); 109 | 110 | arm.Hand.y = Math.Clamp(arm.Hand.z * 2, -0.6f, 0.6f); // sides 111 | arm.Hand.z = arm.Hand.z * -2.3f * invert; // up and down 112 | } 113 | 114 | private (Hips, Vector3) CalculateHips(NormalizedLandmarkList list) 115 | { 116 | var landmarks = list.Landmark; 117 | 118 | // Find 2D normalized hip and shoulder joint positions / distances 119 | Vector2 hipLeft2d = ToVector2(ToVector(landmarks[23])); 120 | Vector2 hipRight2d = ToVector2(ToVector(landmarks[24])); 121 | Vector2 shoulderLeft2d = ToVector2(ToVector(landmarks[11])); 122 | Vector2 shoulderRight2d = ToVector2(ToVector(landmarks[12])); 123 | 124 | Vector2 hipCenter2d = Vector2.Lerp(hipLeft2d, hipRight2d, 1); 125 | Vector2 shoulderCenter2d = Vector2.Lerp(shoulderLeft2d, shoulderRight2d, 1); 126 | float spineLength = Vector2.Distance(hipCenter2d, shoulderCenter2d); 127 | 128 | var hips = new Hips 129 | { 130 | Position = new Vector3(Math.Clamp(-1 * (hipCenter2d.x - .65f), -1, 1), 0, Math.Clamp(spineLength - 1, -2, 0)), 131 | Rotation = VectorExtensions.RollPitchYaw(ToVector(landmarks[23]), ToVector(landmarks[24])), 132 | }; 133 | 134 | if (hips.Rotation.y > .5f) 135 | hips.Rotation.y -= 2; 136 | 137 | hips.Rotation.y += .5f; 138 | 139 | //Stop jumping between left and right shoulder tilt 140 | if (hips.Rotation.z > 0) 141 | hips.Rotation.z = 1 - hips.Rotation.z; 142 | 143 | if (hips.Rotation.z < 0) 144 | hips.Rotation.z = -1 - hips.Rotation.z; 145 | 146 | float turnAroundAmountHips = Math.Abs(hips.Rotation.y).Remap(.2f, .4f); 147 | hips.Rotation.z *= 1 - turnAroundAmountHips; 148 | hips.Rotation.x = 0; // Temp fix for inaccurate X axis 149 | 150 | Vector3 spine = VectorExtensions.RollPitchYaw(ToVector(landmarks[11]), ToVector(landmarks[12])); 151 | 152 | if (spine.y > .5f) 153 | spine.y -= 2; 154 | 155 | spine.y += .5f; 156 | 157 | // Prevent jumping between left and right shoulder tilt 158 | if (spine.z > 0) 159 | spine.z = 1 - spine.z; 160 | 161 | if (spine.z < 0) 162 | spine.z = -1 - spine.z; 163 | 164 | // Fix weird large numbers when 2 shoulder points get too close 165 | float turnAroundAmount = Math.Abs(spine.y).Remap(.2f, .4f); 166 | spine.z *= 1 - turnAroundAmount; 167 | spine.x = 0; // Temp fix for inaccurate X axis 168 | 169 | RigHips(ref hips, ref spine); 170 | return (hips, spine); 171 | } 172 | 173 | private void RigHips(ref Hips hips, ref Vector3 spine) 174 | { 175 | // Convert normalized values to radians 176 | hips.Rotation *= MathF.PI; 177 | 178 | hips.WorldPosition = new Vector3 179 | ( 180 | hips.Position.x * (.5f + 1.8f * -hips.Position.z), 181 | 0, 182 | hips.Position.z * (.1f + hips.Position.z * -2) 183 | ); 184 | 185 | spine *= MathF.PI; 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MediaPipe-UnitySolver 2 | ### Face, Pose, and Hand Tracking Solver for MediaPipeUnity Plugin 3 | 4 | Library converting [Moetion](https://github.com/vignetteapp/Moetion) (Inspired by [KalidoKit](https://github.com/yeemachine/kalidokit)) types to work within Unity for use with [MediaPipeUnityPlugin](https://github.com/homuler/MediaPipeUnityPlugin) landmark outputs. 5 | 6 | UnitySolver is composed of 3 classes for Face and Pose solving. Hand solving to be added later. 7 | 8 | ## 🦆 Usage 9 | 10 | ### NOTE: Pose solver needs a lot of work, not in a usable state with Mixamo/RPM rigs. 🛠 11 | 12 | ### Pose 13 | ```c# 14 | // Give NormalizedLandmarkList from calculator as parameter for MediaPipePoseSolver solve function 15 | var pose = _poseSolver.Solve(list); 16 | 17 | SetEuler(LeftHand, pose.LeftArm.Hand, 1f, 0.3f); 18 | SetEuler(RightHand, pose.RightArm.Hand, 1f, 0.3f); 19 | 20 | SetEuler(LeftLowerArm, pose.LeftArm.Lower, 1f, 0.3f); 21 | SetEuler(LeftUpperArm, pose.LeftArm.Upper, 1f, 0.3f); 22 | 23 | SetEuler(RightLowerArm, pose.RightArm.Lower, 1f, 0.3f); 24 | SetEuler(RightUpperArm, pose.RightArm.Upper, 1f, 0.3f); 25 | 26 | SetEuler(Hips, pose.Hips.Rotation, 0.4f, 0.1f, true); 27 | SetEuler(Spine, pose.Spine, 0.3f, 0.1f, true); 28 | ``` 29 | 30 | ```c# 31 | private void SetEuler(Transform rigPart, Vector3 target, float dampener = 1, float lerpAmount = 0.3f, bool flipped = false) 32 | { 33 | // Convert radians to degrees 34 | Vector3 toDegrees = new(target.x * Mathf.Rad2Deg, target.y * Mathf.Rad2Deg, target.z * Mathf.Rad2Deg); 35 | Vector3 euler = new(toDegrees.x * dampener, toDegrees.y * dampener, toDegrees.z * dampener); 36 | 37 | if (flipped) 38 | { 39 | Quaternion quat = Quaternion.Euler(-euler); 40 | rigPart.localRotation = Quaternion.Slerp(rigPart.localRotation, quat, lerpAmount); 41 | } 42 | else 43 | { 44 | Quaternion quat = Quaternion.Euler(euler); 45 | rigPart.localRotation = Quaternion.Slerp(rigPart.localRotation, quat, lerpAmount); 46 | } 47 | } 48 | ``` 49 | #### Pose Output 50 | ```c# 51 | public struct Pose 52 | { 53 | public Arm LeftArm; 54 | public Arm RightArm; 55 | public Leg LeftLeg; 56 | public Leg RightLeg; 57 | public Vector3 Spine; 58 | public Hips Hips; 59 | } 60 | 61 | public struct Arm 62 | { 63 | public Vector3 Upper; 64 | public Vector3 Lower; 65 | public Vector3 Hand; 66 | } 67 | 68 | public struct Leg 69 | { 70 | public Vector3 Upper; 71 | public Vector3 Lower; 72 | } 73 | 74 | public struct Hips 75 | { 76 | public Vector3 WorldPosition; 77 | public Vector3 Position; 78 | public Vector3 Rotation; 79 | } 80 | ``` 81 | ### Face 82 | ```c# 83 | // Give NormalizedLandmarkList from calculator as parameter for MediaPipeFaceSolver solve function 84 | var face = _faceSolver.Solve(list); 85 | ``` 86 | #### Face Output 87 | ```c# 88 | public struct Face 89 | { 90 | public Head Head; 91 | public Eyes Eyes; 92 | public float Brow; 93 | public Vector2 Pupils; 94 | public Mouth Mouth; 95 | } 96 | 97 | public struct Head 98 | { 99 | public float X; 100 | public float Y; 101 | public float Z; 102 | 103 | public float Width; 104 | public float Height; 105 | 106 | public Vector3 Position; 107 | public Vector3 NormalizedAngles; 108 | } 109 | 110 | public struct Mouth 111 | { 112 | public float X; 113 | public float Y; 114 | public Phoneme Shape; 115 | } 116 | 117 | public struct Phoneme 118 | { 119 | public float A; 120 | public float E; 121 | public float I; 122 | public float O; 123 | public float U; 124 | } 125 | 126 | public struct Eyes 127 | { 128 | // Wideness of eyes 129 | public float Left; 130 | public float Right; 131 | } 132 | ``` 133 | 134 | ### :ghost: Contribute 135 | This library is a work in progress and contributions to improve it are very welcome. 136 | ```bash 137 | git clone https://github.com/BrandonBartram98/MediaPipe-UnitySolver 138 | ``` 139 | -------------------------------------------------------------------------------- /VectorExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using UnityEngine; 5 | 6 | public static class VectorExtensions 7 | { 8 | public static float Find2DAngle(float cx, float cy, float ex, float ey) 9 | { 10 | var dy = ey - cy; 11 | var dx = ex - cx; 12 | return MathF.Atan2(dy, dx); 13 | } 14 | 15 | public static Vector2 Unit(Vector2 vector) => vector / vector.magnitude; 16 | public static Vector3 Unit(Vector3 vector) => vector / vector.magnitude; 17 | 18 | public static float Remap(this float val, float min, float max) => (Math.Clamp(val, min, max) - min) / (max - min); 19 | 20 | public static float NormalizeRadians(float radians) 21 | { 22 | if (radians >= MathF.PI / 2) 23 | radians -= 2 * MathF.PI; 24 | 25 | if (radians <= -MathF.PI / 2) 26 | { 27 | radians += 2 * MathF.PI; 28 | radians = MathF.PI - radians; 29 | } 30 | 31 | return radians / MathF.PI; 32 | } 33 | 34 | public static float NormalizeAngle(this float radians) 35 | { 36 | var twoPi = MathF.PI * 2; 37 | var angle = radians % twoPi; 38 | angle = angle > MathF.PI ? angle - twoPi : angle < -MathF.PI ? twoPi + angle : angle; 39 | return angle / MathF.PI; 40 | } 41 | 42 | public static Vector3 FindRotation(Vector3 vector, Vector3 other, bool normalize) 43 | { 44 | Vector3 result = new( 45 | Find2DAngle(vector.z, vector.x, other.z, other.x), 46 | Find2DAngle(vector.z, vector.y, other.z, other.y), 47 | Find2DAngle(vector.x, vector.y, other.x, other.y) 48 | ); 49 | 50 | if (normalize) 51 | { 52 | return result.normalized; 53 | } 54 | 55 | return result; 56 | } 57 | 58 | /// Find 2D angle between 3 points in 3D space. Returns a single angle normalized to 0, 1 59 | public static float AngleBetween3DCoords(Vector3 a, Vector3 b, Vector3 c) 60 | { 61 | var vec1 = a - b; 62 | var vec2 = c - b; 63 | 64 | var vec1Norm = Unit(vec1); 65 | var vec2Norm = Unit(vec2); 66 | 67 | var dotProducts = Vector3.Dot(vec1Norm, vec2Norm); 68 | var angle = MathF.Acos(dotProducts); 69 | 70 | return NormalizeRadians(angle); 71 | } 72 | 73 | public static Vector3 RollPitchYaw(Vector3 a, Vector3 b, Vector3? c = null) 74 | { 75 | if (c == null) 76 | { 77 | return new Vector3( 78 | Find2DAngle(a.z, a.y, b.z, b.y).NormalizeAngle(), 79 | Find2DAngle(a.z, a.x, b.z, b.x).NormalizeAngle(), 80 | Find2DAngle(a.x, a.y, b.x, b.y).NormalizeAngle() 81 | ); 82 | } 83 | 84 | var qb = b - a; 85 | var qc = (Vector3)(c - a); 86 | var n = Vector3.Cross(qb, qc); 87 | 88 | var unitZ = Unit(n); 89 | var unitX = Unit(qb); 90 | var unitY = Vector3.Cross(unitZ, unitX); 91 | 92 | var beta = MathF.Asin(unitZ.x); 93 | var alpha = MathF.Atan2(-unitZ.y, unitZ.z); 94 | var gamma = MathF.Atan2(-unitY.x, unitX.x); 95 | 96 | return new Vector3(alpha.NormalizeAngle(), beta.NormalizeAngle(), gamma.NormalizeAngle()); 97 | } 98 | } 99 | --------------------------------------------------------------------------------