Streamingle_URP/Assets/Scripts/Streamingle/StreamingleControl/MotionCapture/LiveLinkFaceReceiver.cs

using System;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Collections.Generic;
using UnityEngine;

/// <summary>
/// Receives LiveLink Face data from Python bridge and applies to SkinnedMeshRenderer
/// Compatible with VMagicMirror and ARKit blend shapes
/// </summary>
public class LiveLinkFaceReceiver : MonoBehaviour
{
    [Header("Network Settings")]
    [Tooltip("UDP port to listen on (must match Python sender)")]
    public int listenPort = 5555;

    [Tooltip("Data format: JSON or Binary")]
    public DataFormat dataFormat = DataFormat.JSON;

    [Header("Target")]
    [Tooltip("SkinnedMeshRenderer with ARKit blend shapes")]
    public SkinnedMeshRenderer targetRenderer;

    [Header("Settings")]
    [Tooltip("Smoothing factor (0=no smooth, 1=max smooth)")]
    [Range(0f, 1f)]
    public float smoothing = 0.3f;

    [Tooltip("Multiplier for all blend shape values")]
    [Range(0f, 2f)]
    public float globalMultiplier = 1.0f;

    [Header("Debug")]
    public bool showDebugInfo = true;
    public int framesReceived = 0;
    public float currentFPS = 0f;

    // Private
    private UdpClient udpClient;
    private Dictionary<string, int> blendShapeIndexCache;
    private Dictionary<string, float> currentValues;
    private Dictionary<string, float> targetValues;
    private float lastFrameTime;
    private int frameCount;

    // ARKit blend shape names (52 standard shapes)
    private static readonly string[] ARKIT_BLENDSHAPES = new string[]
    {
        "eyeBlinkLeft", "eyeLookDownLeft", "eyeLookInLeft", "eyeLookOutLeft",
        "eyeLookUpLeft", "eyeSquintLeft", "eyeWideLeft",
        "eyeBlinkRight", "eyeLookDownRight", "eyeLookInRight", "eyeLookOutRight",
        "eyeLookUpRight", "eyeSquintRight", "eyeWideRight",
        "jawForward", "jawLeft", "jawRight", "jawOpen",
        "mouthClose", "mouthFunnel", "mouthPucker",
        "mouthLeft", "mouthRight",
        "mouthSmileLeft", "mouthSmileRight",
        "mouthFrownLeft", "mouthFrownRight",
        "mouthDimpleLeft", "mouthDimpleRight",
        "mouthStretchLeft", "mouthStretchRight",
        "mouthRollLower", "mouthRollUpper",
        "mouthShrugLower", "mouthShrugUpper",
        "mouthPressLeft", "mouthPressRight",
        "mouthLowerDownLeft", "mouthLowerDownRight",
        "mouthUpperUpLeft", "mouthUpperUpRight",
        "browDownLeft", "browDownRight", "browInnerUp",
        "browOuterUpLeft", "browOuterUpRight",
        "cheekPuff", "cheekSquintLeft", "cheekSquintRight",
        "noseSneerLeft", "noseSneerRight",
        "tongueOut"
    };

    public enum DataFormat
    {
        JSON,
        Binary
    }

    void Start()
    {
        // Validate target
        if (targetRenderer == null)
        {
            Debug.LogError("LiveLinkFaceReceiver: Target SkinnedMeshRenderer is not assigned!");
            enabled = false;
            return;
        }

        // Initialize
        blendShapeIndexCache = new Dictionary<string, int>();
        currentValues = new Dictionary<string, float>();
        targetValues = new Dictionary<string, float>();

        // Cache blend shape indices
        CacheBlendShapeIndices();

        // Start UDP receiver
        try
        {
            udpClient = new UdpClient(listenPort);
            udpClient.Client.ReceiveTimeout = 1000;
            Debug.Log($"LiveLinkFaceReceiver: Listening on UDP port {listenPort}");
            Debug.Log($"Data format: {dataFormat}");
            Debug.Log($"Cached {blendShapeIndexCache.Count} blend shapes");
        }
        catch (Exception e)
        {
            Debug.LogError($"LiveLinkFaceReceiver: Failed to start UDP receiver: {e.Message}");
            enabled = false;
        }

        lastFrameTime = Time.time;
    }

    void CacheBlendShapeIndices()
    {
        Mesh mesh = targetRenderer.sharedMesh;
        int blendShapeCount = mesh.blendShapeCount;

        Debug.Log($"Caching blend shapes from mesh (total: {blendShapeCount}):");

        for (int i = 0; i < blendShapeCount; i++)
        {
            string name = mesh.GetBlendShapeName(i);

            // Store with original name
            blendShapeIndexCache[name] = i;

            // Store with lowercase for case-insensitive lookup
            string lowerName = name.ToLower();
            if (!blendShapeIndexCache.ContainsKey(lowerName))
            {
                blendShapeIndexCache[lowerName] = i;
            }

            // Also try without prefix (some meshes use different naming)
            if (name.StartsWith("blendShape.") || name.StartsWith("blendShape1."))
            {
                string shortName = name.Substring(name.IndexOf('.') + 1);
                if (!blendShapeIndexCache.ContainsKey(shortName))
                {
                    blendShapeIndexCache[shortName] = i;
                }

                // Lowercase version of short name
                string lowerShortName = shortName.ToLower();
                if (!blendShapeIndexCache.ContainsKey(lowerShortName))
                {
                    blendShapeIndexCache[lowerShortName] = i;
                }
            }

            // Debug: Print first 10 blend shapes
            if (i < 10)
            {
                Debug.Log($"  [{i}] {name}");
            }
        }
    }

    void Update()
    {
        // Receive data
        if (udpClient != null && udpClient.Available > 0)
        {
            try
            {
                IPEndPoint remoteEndPoint = new IPEndPoint(IPAddress.Any, 0);
                byte[] data = udpClient.Receive(ref remoteEndPoint);

                if (dataFormat == DataFormat.JSON)
                {
                    ParseJSON(data);
                }
                else
                {
                    ParseBinary(data);
                }

                framesReceived++;
                frameCount++;

                // Calculate FPS
                float elapsed = Time.time - lastFrameTime;
                if (elapsed >= 1.0f)
                {
                    currentFPS = frameCount / elapsed;
                    frameCount = 0;
                    lastFrameTime = Time.time;
                }
            }
            catch (Exception e)
            {
                if (showDebugInfo)
                {
                    Debug.LogWarning($"LiveLinkFaceReceiver: Error receiving data: {e.Message}");
                }
            }
        }

        // Apply blend shapes with smoothing
        ApplyBlendShapes();
    }

    void ParseJSON(byte[] data)
    {
        string json = Encoding.UTF8.GetString(data);
        var message = JsonUtility.FromJson<BlendShapeMessage>(json);

        if (message != null && message.blendShapes != null)
        {
            foreach (var kvp in message.blendShapes)
            {
                targetValues[kvp.Key] = kvp.Value;
            }
        }
    }

    void ParseBinary(byte[] data)
    {
        if (data.Length < 8) return;

        // Check magic header "FACE"
        if (data[0] != 'F' || data[1] != 'A' || data[2] != 'C' || data[3] != 'E')
        {
            Debug.LogWarning("LiveLinkFaceReceiver: Invalid binary packet (bad magic)");
            return;
        }

        // Read count
        int count = BitConverter.ToInt32(data, 4);
        if (count != 52)
        {
            Debug.LogWarning($"LiveLinkFaceReceiver: Unexpected blend shape count: {count}");
        }

        // Read float values
        int offset = 8;
        for (int i = 0; i < Math.Min(count, ARKIT_BLENDSHAPES.Length); i++)
        {
            if (offset + 4 <= data.Length)
            {
                float value = BitConverter.ToSingle(data, offset);
                targetValues[ARKIT_BLENDSHAPES[i]] = value;
                offset += 4;
            }
        }
    }

    void ApplyBlendShapes()
    {
        foreach (var kvp in targetValues)
        {
            string name = kvp.Key;
            float targetValue = kvp.Value * globalMultiplier * 100f; // 0-100 range

            // Get current value (with smoothing)
            if (!currentValues.ContainsKey(name))
            {
                currentValues[name] = 0f;
            }

            float currentValue = currentValues[name];
            float newValue = Mathf.Lerp(targetValue, currentValue, smoothing);
            currentValues[name] = newValue;

            // Try to find blend shape index (case-insensitive)
            int index = -1;
            if (blendShapeIndexCache.TryGetValue(name, out index))
            {
                // Found with exact name
            }
            else if (blendShapeIndexCache.TryGetValue(name.ToLower(), out index))
            {
                // Found with lowercase
            }
            else
            {
                // Not found - skip
                continue;
            }

            // Apply to mesh
            targetRenderer.SetBlendShapeWeight(index, newValue);
        }
    }

    void OnDestroy()
    {
        if (udpClient != null)
        {
            udpClient.Close();
            udpClient = null;
        }
    }

    void OnGUI()
    {
        if (!showDebugInfo) return;

        GUILayout.BeginArea(new Rect(10, 10, 300, 150));
        GUILayout.Box($"LiveLink Face Receiver\n" +
                      $"Port: {listenPort}\n" +
                      $"Format: {dataFormat}\n" +
                      $"Frames: {framesReceived}\n" +
                      $"FPS: {currentFPS:F1}\n" +
                      $"Active: {CountActiveShapes()}");
        GUILayout.EndArea();
    }

    int CountActiveShapes()
    {
        int count = 0;
        foreach (var kvp in currentValues)
        {
            if (kvp.Value > 1.0f) count++;
        }
        return count;
    }

    [Serializable]
    private class BlendShapeMessage
    {
        public Dictionary<string, float> blendShapes;
    }
}