using System; using System.Collections.Generic; using System.Runtime.CompilerServices; using UnityEngine; [assembly: InternalsVisibleTo("Streamingle.Gaze.Editor")] [assembly: InternalsVisibleTo("Streamingle.Gaze.Tests.Editor")] namespace Streamingle.Gaze { [ExecuteAlways] [DisallowMultipleComponent] [DefaultExecutionOrder(10000)] public sealed class BlendshapeGazeDriver : MonoBehaviour { private const float MinimumInfluence = 0.0001f; private const float ExternalWriteTolerance = 0.001f; [Header("Required")] [SerializeField] private SkinnedMeshRenderer targetRenderer; [SerializeField] private BlendshapeGazeProfile profile; [SerializeField] [Tooltip("Eye midpoint. Its position is used as the gaze origin.")] private Transform gazeOrigin; [SerializeField] [Tooltip("Optional forward/up fallback when Gaze Origin is not assigned. Gaze Origin rotation is authoritative when present.")] private Transform headReference; [Header("Default Target")] [SerializeField] [Tooltip("Optional output camera override. Timeline clips with no target use this before Camera.main.")] private Camera targetCamera; [SerializeField] [Tooltip("Optional fallback target when Timeline is not driving this component.")] private Transform defaultTarget; [SerializeField] private bool defaultToMainCamera; [SerializeField, Range(0f, 1f)] private float defaultInfluence; [SerializeField] private bool defaultCorrectives; [Header("Camera Cut")] [SerializeField] private GazeCutResponse cutResponse = GazeCutResponse.Snap; [SerializeField, Min(0f)] private float cutTransitionDuration = 0.1f; [SerializeField, Range(0f, 180f)] [Tooltip("A target-direction jump at or above this angle is treated as a camera cut. Set 0 to disable angle detection.")] private float cutDetectionAngle = 15f; [Header("Debug Gizmos")] [SerializeField] private bool drawGizmos = true; [SerializeField, Min(0.01f)] private float gizmoDistance = 0.35f; private readonly Dictionary timelineStates = new Dictionary(); private int[] blendShapeIndices = Array.Empty(); private float[] baselineWeights = Array.Empty(); private float[] evaluatedWeights = Array.Empty(); private float[] lastAppliedWeights = Array.Empty(); private string[] cachedChannelNames = Array.Empty(); private SkinnedMeshRenderer cachedRenderer; private Mesh cachedMesh; private BlendshapeGazeProfile cachedProfile; private bool hasAppliedWeights; private bool hasRuntimeState; private GazeTimelineState runtimeState; private bool hasPreviousDirection; private Vector3 previousDesiredDirection; private UnityEngine.Object previousTargetIdentity; private bool cutTransitionActive; private Vector3 cutTransitionFromDirection; private float cutTransitionStartTime; private int calibrationPreviewOverrideCount; public SkinnedMeshRenderer TargetRenderer { get => targetRenderer; set { if (targetRenderer == value) return; ReleaseWeights(); targetRenderer = value; InvalidateCache(); } } public BlendshapeGazeProfile Profile { get => profile; set { if (profile == value) return; ReleaseWeights(); profile = value; InvalidateCache(); } } public Transform GazeOrigin { get => gazeOrigin; set => gazeOrigin = value; } public Transform HeadReference { get => headReference; set => headReference = value; } public Camera TargetCamera { get => targetCamera; set => targetCamera = value; } public GazeDirectionResult LastDirection { get; private set; } public Vector3 LastOriginPosition { get; private set; } public Vector3 LastTargetPosition { get; private set; } public bool HasActiveRequest { get; private set; } public Vector3 GetGazeOriginPosition() { if (gazeOrigin != null) return gazeOrigin.position; if (targetRenderer != null) return targetRenderer.bounds.center; return transform.position; } public bool RebuildCache() { ReleaseWeights(); InvalidateCache(); return EnsureCache(); } private Transform DirectionReference => gazeOrigin != null ? gazeOrigin : (headReference != null ? headReference : transform); private void LateUpdate() { if (Application.IsPlaying(gameObject)) { EvaluateNow(false); return; } #if UNITY_EDITOR // ExecuteAlways receives the editor player loop requested by the // Timeline mixer. This is a stronger final ordering point than a // one-shot delayCall: Director animation has already updated the Head, // GazeOrigin and facial blendshapes before this high-order LateUpdate. if (!Application.isPlaying && timelineStates.Count > 0) EvaluateNow(true); #endif } public void SetTimelineState(int sourceId, GazeTimelineState state) { state.Influence = SanitizeInfluence(state.Influence); state.CutTransitionDuration = SanitizeNonNegative(state.CutTransitionDuration); timelineStates[sourceId] = state; } public void ClearTimelineState(int sourceId) { timelineStates.Remove(sourceId); if (timelineStates.Count == 0) { HasActiveRequest = false; ReleaseWeights(); ResetDirectionHistory(); } } public void SetRuntimeTarget( Transform target, float influence = 1f, bool enableCorrectives = false) { runtimeState = new GazeTimelineState { Target = target, UseMainCamera = target == null, Influence = SanitizeInfluence(influence), EnableCorrectives = enableCorrectives, ResetSmoothing = !hasRuntimeState }; hasRuntimeState = true; } public void SetRuntimeTargetPosition( Vector3 worldPosition, float influence = 1f, bool enableCorrectives = false) { runtimeState = new GazeTimelineState { TargetPosition = worldPosition, HasTargetPosition = true, Influence = SanitizeInfluence(influence), EnableCorrectives = enableCorrectives, ResetSmoothing = !hasRuntimeState }; hasRuntimeState = true; } public void ClearRuntimeTarget() { hasRuntimeState = false; if (timelineStates.Count == 0) { HasActiveRequest = false; ReleaseWeights(); ResetDirectionHistory(); } } public void EvaluateNow(bool deterministic) { // Calibration owns the configured blendshape channels while its preview // session is active. Timeline/editor evaluation can still update its // request state, but must not overwrite (or restore over) that preview. if (calibrationPreviewOverrideCount > 0) { HasActiveRequest = false; return; } if (!TryGetActiveState(out var state, out var usesRuntimeState)) { HasActiveRequest = false; ReleaseWeights(); ResetDirectionHistory(); return; } state.Deterministic |= deterministic; if (!TryResolveTarget(state, out var targetPosition, out var targetIdentity)) { HasActiveRequest = false; ReleaseWeights(); ResetDirectionHistory(); return; } if (!EnsureCache()) { HasActiveRequest = false; ReleaseWeights(); return; } LastOriginPosition = GetGazeOriginPosition(); LastTargetPosition = targetPosition; var desiredDirection = targetPosition - LastOriginPosition; if (desiredDirection.sqrMagnitude <= Mathf.Epsilon) { HasActiveRequest = false; ReleaseWeights(); return; } desiredDirection.Normalize(); var effectiveDirection = ApplyCutResponse( desiredDirection, targetIdentity, state, state.Deterministic || !Application.isPlaying); var reference = DirectionReference; LastDirection = GazeDirectionSolver.Calculate( reference != null ? reference.rotation : Quaternion.identity, effectiveDirection, profile.MinYaw, profile.MaxYaw, profile.MinPitch, profile.MaxPitch); if (!profile.TryEvaluate(LastDirection, state.EnableCorrectives, evaluatedWeights)) { HasActiveRequest = false; ReleaseWeights(); return; } CaptureExternalBaseline(); ApplyWeights(Mathf.Clamp01(state.Influence)); HasActiveRequest = true; if (usesRuntimeState) runtimeState.ResetSmoothing = false; } internal void BeginCalibrationPreviewOverride() { if (calibrationPreviewOverrideCount == 0) { // If gaze was already driving the renderer, return those channels to // their external facial baseline before the preview captures it. ReleaseWeights(); ResetDirectionHistory(); HasActiveRequest = false; } calibrationPreviewOverrideCount++; } internal void EndCalibrationPreviewOverride() { if (calibrationPreviewOverrideCount <= 0) return; calibrationPreviewOverrideCount--; if (calibrationPreviewOverrideCount == 0) ResetDirectionHistory(); } internal void PrepareForTimelineEvaluation() { if (!isActiveAndEnabled || calibrationPreviewOverrideCount > 0) return; // PlayableGraph PrepareFrame runs before Timeline animation outputs. // Restore the prior facial baseline here, then LateUpdate/editor final // evaluation can capture the newly sampled value even when it happens // to be numerically identical to our previous gaze result. ReleaseWeights(); } private bool TryGetActiveState(out GazeTimelineState state, out bool usesRuntimeState) { var found = false; var bestInfluence = MinimumInfluence; var bestSourceId = int.MaxValue; state = default; usesRuntimeState = false; foreach (var pair in timelineStates) { var candidate = pair.Value; if (candidate.Influence <= MinimumInfluence) continue; if (found && candidate.Influence < bestInfluence) continue; if (found && Mathf.Approximately(candidate.Influence, bestInfluence) && pair.Key >= bestSourceId) continue; state = candidate; bestInfluence = candidate.Influence; bestSourceId = pair.Key; found = true; } if (found) return true; if (hasRuntimeState && runtimeState.Influence > MinimumInfluence) { state = runtimeState; usesRuntimeState = true; return true; } if (defaultInfluence <= MinimumInfluence || (defaultTarget == null && !defaultToMainCamera)) return false; state = new GazeTimelineState { Target = defaultTarget, UseMainCamera = defaultTarget == null && defaultToMainCamera, Influence = defaultInfluence, EnableCorrectives = defaultCorrectives }; return true; } private bool TryResolveTarget( GazeTimelineState state, out Vector3 targetPosition, out UnityEngine.Object identity) { if (state.HasTargetPosition) { targetPosition = state.TargetPosition; identity = null; return IsFinite(targetPosition); } if (state.HasTargetDirection) { var targetDirection = state.TargetDirection; if (!IsFinite(targetDirection) || targetDirection.sqrMagnitude <= Mathf.Epsilon) { targetPosition = default; identity = null; return false; } targetPosition = GetGazeOriginPosition() + targetDirection.normalized; identity = null; return true; } var target = state.Target; if (target == null && state.UseMainCamera) { var mainCamera = targetCamera != null ? targetCamera : Camera.main; target = mainCamera != null ? mainCamera.transform : null; } if (target == null) { targetPosition = default; identity = null; return false; } targetPosition = target.position; identity = target; return IsFinite(targetPosition); } private Vector3 ApplyCutResponse( Vector3 desiredDirection, UnityEngine.Object targetIdentity, GazeTimelineState state, bool deterministic) { var response = state.OverrideCutResponse ? state.CutResponse : cutResponse; var duration = state.OverrideCutResponse ? state.CutTransitionDuration : cutTransitionDuration; if (deterministic || state.ResetSmoothing || !hasPreviousDirection) { previousDesiredDirection = desiredDirection; previousTargetIdentity = targetIdentity; hasPreviousDirection = true; cutTransitionActive = false; return desiredDirection; } var identityChanged = previousTargetIdentity != targetIdentity; // Zero disables pose-jump detection while still allowing an explicit target // identity change to count as a cut. var angleJump = cutDetectionAngle > 0f && Vector3.Angle(previousDesiredDirection, desiredDirection) >= cutDetectionAngle; if (response == GazeCutResponse.Smooth && duration > Mathf.Epsilon && (identityChanged || angleJump && !cutTransitionActive)) { cutTransitionFromDirection = cutTransitionActive ? EvaluateCutTransition(previousDesiredDirection, duration) : previousDesiredDirection; cutTransitionStartTime = Time.unscaledTime; cutTransitionActive = true; } previousDesiredDirection = desiredDirection; previousTargetIdentity = targetIdentity; if (response != GazeCutResponse.Smooth || duration <= Mathf.Epsilon) { cutTransitionActive = false; return desiredDirection; } return EvaluateCutTransition(desiredDirection, duration); } private Vector3 EvaluateCutTransition(Vector3 destination, float duration) { if (!cutTransitionActive) return destination; var t = Mathf.Clamp01((Time.unscaledTime - cutTransitionStartTime) / duration); if (t >= 1f) { cutTransitionActive = false; return destination; } return Vector3.Slerp(cutTransitionFromDirection, destination, t).normalized; } private bool EnsureCache() { var mesh = targetRenderer != null ? targetRenderer.sharedMesh : null; if (targetRenderer == null || mesh == null || profile == null || profile.Channels.Count == 0 || !profile.IsCompatibleWith(mesh)) return false; if (cachedRenderer == targetRenderer && cachedMesh == mesh && cachedProfile == profile && blendShapeIndices.Length == profile.Channels.Count && ChannelMappingMatches()) return true; ReleaseWeights(); cachedRenderer = targetRenderer; cachedMesh = mesh; cachedProfile = profile; var count = profile.Channels.Count; blendShapeIndices = new int[count]; baselineWeights = new float[count]; evaluatedWeights = new float[count]; lastAppliedWeights = new float[count]; cachedChannelNames = new string[count]; var resolvedIndices = new HashSet(); for (var i = 0; i < count; i++) { cachedChannelNames[i] = profile.Channels[i].BlendShapeName; blendShapeIndices[i] = profile.FindBlendShapeIndex(mesh, i); if (blendShapeIndices[i] < 0 || !resolvedIndices.Add(blendShapeIndices[i])) { InvalidateCache(); return false; } } return true; } private void CaptureExternalBaseline() { for (var i = 0; i < blendShapeIndices.Length; i++) { var current = targetRenderer.GetBlendShapeWeight(blendShapeIndices[i]); if (!hasAppliedWeights || Mathf.Abs(current - lastAppliedWeights[i]) > ExternalWriteTolerance) baselineWeights[i] = current; } } private void ApplyWeights(float influence) { for (var i = 0; i < blendShapeIndices.Length; i++) { var targetWeight = evaluatedWeights[i]; var channel = profile.Channels[i]; float result; if (float.IsNaN(targetWeight)) { result = baselineWeights[i]; } else if (channel.Usage == GazeChannelUsage.Corrective && channel.CorrectiveBlendMode == GazeCorrectiveBlendMode.AdditiveFromCenter) { var centerWeight = profile.GetSampleWeight(GazeCalibrationPoint.Center, i); result = baselineWeights[i] + (targetWeight - centerWeight) * influence; } else { result = Mathf.LerpUnclamped(baselineWeights[i], targetWeight, influence); } targetRenderer.SetBlendShapeWeight(blendShapeIndices[i], result); lastAppliedWeights[i] = result; } hasAppliedWeights = true; } private bool ChannelMappingMatches() { if (cachedChannelNames.Length != profile.Channels.Count) return false; for (var i = 0; i < cachedChannelNames.Length; i++) { var channel = profile.Channels[i]; if (channel == null || !string.Equals( cachedChannelNames[i], channel.BlendShapeName, StringComparison.Ordinal)) return false; } return true; } private void ReleaseWeights() { if (!hasAppliedWeights || cachedRenderer == null) return; // The old indices are only meaningful for the mesh they were built from. if (cachedRenderer.sharedMesh != cachedMesh) { hasAppliedWeights = false; return; } // If Animator/Timeline/a live receiver has written since our previous LateUpdate, // that current value is the new base. Otherwise restore the last base so stopping // the gaze track never leaves its final pose stuck on the face. for (var i = 0; i < blendShapeIndices.Length && i < baselineWeights.Length; i++) { var index = blendShapeIndices[i]; if (index < 0) continue; var current = cachedRenderer.GetBlendShapeWeight(index); if (Mathf.Abs(current - lastAppliedWeights[i]) > ExternalWriteTolerance) baselineWeights[i] = current; cachedRenderer.SetBlendShapeWeight(index, baselineWeights[i]); } hasAppliedWeights = false; } private void ResetDirectionHistory() { hasPreviousDirection = false; previousTargetIdentity = null; cutTransitionActive = false; } private void InvalidateCache() { cachedRenderer = null; cachedMesh = null; cachedProfile = null; blendShapeIndices = Array.Empty(); baselineWeights = Array.Empty(); evaluatedWeights = Array.Empty(); lastAppliedWeights = Array.Empty(); cachedChannelNames = Array.Empty(); hasAppliedWeights = false; } private void OnDisable() { #if UNITY_EDITOR BlendshapeGazeMixerBehaviour.CancelEditorFinalEvaluation(this); #endif ReleaseWeights(); timelineStates.Clear(); ResetDirectionHistory(); HasActiveRequest = false; } private void OnValidate() { defaultInfluence = SanitizeInfluence(defaultInfluence); cutTransitionDuration = SanitizeNonNegative(cutTransitionDuration); cutDetectionAngle = IsFinite(cutDetectionAngle) ? Mathf.Clamp(cutDetectionAngle, 0f, 180f) : 15f; gizmoDistance = IsFinite(gizmoDistance) ? Mathf.Max(0.01f, gizmoDistance) : 0.35f; ReleaseWeights(); InvalidateCache(); } private void OnDrawGizmos() { if (!drawGizmos || profile == null) return; var origin = gazeOrigin != null ? gazeOrigin.position : (targetRenderer != null ? targetRenderer.bounds.center : transform.position); var reference = DirectionReference; var rotation = reference != null ? reference.rotation : Quaternion.identity; var distance = Mathf.Max(0.01f, gizmoDistance); var downLeft = origin + rotation * GazeDirectionSolver.DirectionFromAngles( profile.MinYaw, profile.MinPitch) * distance; var downRight = origin + rotation * GazeDirectionSolver.DirectionFromAngles( profile.MaxYaw, profile.MinPitch) * distance; var upLeft = origin + rotation * GazeDirectionSolver.DirectionFromAngles( profile.MinYaw, profile.MaxPitch) * distance; var upRight = origin + rotation * GazeDirectionSolver.DirectionFromAngles( profile.MaxYaw, profile.MaxPitch) * distance; Gizmos.color = new Color(0.2f, 0.9f, 1f, 0.8f); Gizmos.DrawWireSphere(origin, distance * 0.025f); Gizmos.DrawLine(origin, downLeft); Gizmos.DrawLine(origin, downRight); Gizmos.DrawLine(origin, upLeft); Gizmos.DrawLine(origin, upRight); Gizmos.DrawLine(downLeft, downRight); Gizmos.DrawLine(downRight, upRight); Gizmos.DrawLine(upRight, upLeft); Gizmos.DrawLine(upLeft, downLeft); Gizmos.color = Color.green; Gizmos.DrawLine(origin, origin + rotation * Vector3.forward * distance); if (!HasActiveRequest) return; Gizmos.color = LastDirection.WasClamped ? Color.red : Color.cyan; Gizmos.DrawLine(origin, LastTargetPosition); Gizmos.DrawWireSphere(LastTargetPosition, distance * 0.035f); var clampedDirection = rotation * GazeDirectionSolver.DirectionFromAngles( LastDirection.ClampedYaw, LastDirection.ClampedPitch); Gizmos.color = Color.yellow; Gizmos.DrawLine(origin, origin + clampedDirection * distance); } private static bool IsFinite(Vector3 value) { return !float.IsNaN(value.x) && !float.IsInfinity(value.x) && !float.IsNaN(value.y) && !float.IsInfinity(value.y) && !float.IsNaN(value.z) && !float.IsInfinity(value.z); } private static bool IsFinite(float value) { return !float.IsNaN(value) && !float.IsInfinity(value); } private static float SanitizeInfluence(float value) { return IsFinite(value) ? Mathf.Clamp01(value) : 0f; } private static float SanitizeNonNegative(float value) { return IsFinite(value) ? Mathf.Max(0f, value) : 0f; } } }