using System; using System.Net; using System.Net.Sockets; using System.Text; using System.Collections.Generic; using UnityEngine; /// /// Receives LiveLink Face data from Python bridge and applies to SkinnedMeshRenderer /// Compatible with VMagicMirror and ARKit blend shapes /// public class LiveLinkFaceReceiver : MonoBehaviour { [Header("Network Settings")] [Tooltip("UDP port to listen on (must match Python sender)")] public int listenPort = 5555; [Tooltip("Data format: JSON or Binary")] public DataFormat dataFormat = DataFormat.JSON; [Header("Target")] [Tooltip("SkinnedMeshRenderer with ARKit blend shapes")] public SkinnedMeshRenderer targetRenderer; [Header("Settings")] [Tooltip("Smoothing factor (0=no smooth, 1=max smooth)")] [Range(0f, 1f)] public float smoothing = 0.3f; [Tooltip("Multiplier for all blend shape values")] [Range(0f, 2f)] public float globalMultiplier = 1.0f; [Header("Debug")] public bool showDebugInfo = true; public int framesReceived = 0; public float currentFPS = 0f; // Private private UdpClient udpClient; private Dictionary blendShapeIndexCache; private Dictionary currentValues; private Dictionary targetValues; private float lastFrameTime; private int frameCount; // ARKit blend shape names (52 standard shapes) private static readonly string[] ARKIT_BLENDSHAPES = new string[] { "eyeBlinkLeft", "eyeLookDownLeft", "eyeLookInLeft", "eyeLookOutLeft", "eyeLookUpLeft", "eyeSquintLeft", "eyeWideLeft", "eyeBlinkRight", "eyeLookDownRight", "eyeLookInRight", "eyeLookOutRight", "eyeLookUpRight", "eyeSquintRight", "eyeWideRight", "jawForward", "jawLeft", "jawRight", "jawOpen", "mouthClose", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", "mouthSmileLeft", "mouthSmileRight", "mouthFrownLeft", "mouthFrownRight", "mouthDimpleLeft", "mouthDimpleRight", "mouthStretchLeft", "mouthStretchRight", "mouthRollLower", "mouthRollUpper", "mouthShrugLower", "mouthShrugUpper", "mouthPressLeft", "mouthPressRight", "mouthLowerDownLeft", "mouthLowerDownRight", "mouthUpperUpLeft", "mouthUpperUpRight", "browDownLeft", "browDownRight", "browInnerUp", "browOuterUpLeft", "browOuterUpRight", "cheekPuff", "cheekSquintLeft", "cheekSquintRight", "noseSneerLeft", "noseSneerRight", "tongueOut" }; public enum DataFormat { JSON, Binary } void Start() { // Validate target if (targetRenderer == null) { Debug.LogError("LiveLinkFaceReceiver: Target SkinnedMeshRenderer is not assigned!"); enabled = false; return; } // Initialize blendShapeIndexCache = new Dictionary(); currentValues = new Dictionary(); targetValues = new Dictionary(); // Cache blend shape indices CacheBlendShapeIndices(); // Start UDP receiver try { udpClient = new UdpClient(listenPort); udpClient.Client.ReceiveTimeout = 1000; Debug.Log($"LiveLinkFaceReceiver: Listening on UDP port {listenPort}"); Debug.Log($"Data format: {dataFormat}"); Debug.Log($"Cached {blendShapeIndexCache.Count} blend shapes"); } catch (Exception e) { Debug.LogError($"LiveLinkFaceReceiver: Failed to start UDP receiver: {e.Message}"); enabled = false; } lastFrameTime = Time.time; } void CacheBlendShapeIndices() { Mesh mesh = targetRenderer.sharedMesh; int blendShapeCount = mesh.blendShapeCount; Debug.Log($"Caching blend shapes from mesh (total: {blendShapeCount}):"); for (int i = 0; i < blendShapeCount; i++) { string name = mesh.GetBlendShapeName(i); // Store with original name blendShapeIndexCache[name] = i; // Store with lowercase for case-insensitive lookup string lowerName = name.ToLower(); if (!blendShapeIndexCache.ContainsKey(lowerName)) { blendShapeIndexCache[lowerName] = i; } // Also try without prefix (some meshes use different naming) if (name.StartsWith("blendShape.") || name.StartsWith("blendShape1.")) { string shortName = name.Substring(name.IndexOf('.') + 1); if (!blendShapeIndexCache.ContainsKey(shortName)) { blendShapeIndexCache[shortName] = i; } // Lowercase version of short name string lowerShortName = shortName.ToLower(); if (!blendShapeIndexCache.ContainsKey(lowerShortName)) { blendShapeIndexCache[lowerShortName] = i; } } // Debug: Print first 10 blend shapes if (i < 10) { Debug.Log($" [{i}] {name}"); } } } void Update() { // Receive data if (udpClient != null && udpClient.Available > 0) { try { IPEndPoint remoteEndPoint = new IPEndPoint(IPAddress.Any, 0); byte[] data = udpClient.Receive(ref remoteEndPoint); if (dataFormat == DataFormat.JSON) { ParseJSON(data); } else { ParseBinary(data); } framesReceived++; frameCount++; // Calculate FPS float elapsed = Time.time - lastFrameTime; if (elapsed >= 1.0f) { currentFPS = frameCount / elapsed; frameCount = 0; lastFrameTime = Time.time; } } catch (Exception e) { if (showDebugInfo) { Debug.LogWarning($"LiveLinkFaceReceiver: Error receiving data: {e.Message}"); } } } // Apply blend shapes with smoothing ApplyBlendShapes(); } void ParseJSON(byte[] data) { string json = Encoding.UTF8.GetString(data); var message = JsonUtility.FromJson(json); if (message != null && message.blendShapes != null) { foreach (var kvp in message.blendShapes) { targetValues[kvp.Key] = kvp.Value; } } } void ParseBinary(byte[] data) { if (data.Length < 8) return; // Check magic header "FACE" if (data[0] != 'F' || data[1] != 'A' || data[2] != 'C' || data[3] != 'E') { Debug.LogWarning("LiveLinkFaceReceiver: Invalid binary packet (bad magic)"); return; } // Read count int count = BitConverter.ToInt32(data, 4); if (count != 52) { Debug.LogWarning($"LiveLinkFaceReceiver: Unexpected blend shape count: {count}"); } // Read float values int offset = 8; for (int i = 0; i < Math.Min(count, ARKIT_BLENDSHAPES.Length); i++) { if (offset + 4 <= data.Length) { float value = BitConverter.ToSingle(data, offset); targetValues[ARKIT_BLENDSHAPES[i]] = value; offset += 4; } } } void ApplyBlendShapes() { foreach (var kvp in targetValues) { string name = kvp.Key; float targetValue = kvp.Value * globalMultiplier * 100f; // 0-100 range // Get current value (with smoothing) if (!currentValues.ContainsKey(name)) { currentValues[name] = 0f; } float currentValue = currentValues[name]; float newValue = Mathf.Lerp(targetValue, currentValue, smoothing); currentValues[name] = newValue; // Try to find blend shape index (case-insensitive) int index = -1; if (blendShapeIndexCache.TryGetValue(name, out index)) { // Found with exact name } else if (blendShapeIndexCache.TryGetValue(name.ToLower(), out index)) { // Found with lowercase } else { // Not found - skip continue; } // Apply to mesh targetRenderer.SetBlendShapeWeight(index, newValue); } } void OnDestroy() { if (udpClient != null) { udpClient.Close(); udpClient = null; } } void OnGUI() { if (!showDebugInfo) return; GUILayout.BeginArea(new Rect(10, 10, 300, 150)); GUILayout.Box($"LiveLink Face Receiver\n" + $"Port: {listenPort}\n" + $"Format: {dataFormat}\n" + $"Frames: {framesReceived}\n" + $"FPS: {currentFPS:F1}\n" + $"Active: {CountActiveShapes()}"); GUILayout.EndArea(); } int CountActiveShapes() { int count = 0; foreach (var kvp in currentValues) { if (kvp.Value > 1.0f) count++; } return count; } [Serializable] private class BlendShapeMessage { public Dictionary blendShapes; } }