streamingle-unity-utilities/Runtime/Gaze/BlendshapeGazeDriver.cs

750 lines
26 KiB
C#

using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using UnityEngine;
[assembly: InternalsVisibleTo("Streamingle.Gaze.Editor")]
[assembly: InternalsVisibleTo("Streamingle.Gaze.Tests.Editor")]
namespace Streamingle.Gaze
{
[ExecuteAlways]
[DisallowMultipleComponent]
[DefaultExecutionOrder(10000)]
public sealed class BlendshapeGazeDriver : MonoBehaviour
{
private const float MinimumInfluence = 0.0001f;
private const float ExternalWriteTolerance = 0.001f;
[Header("Required")]
[SerializeField]
private SkinnedMeshRenderer targetRenderer;
[SerializeField]
private BlendshapeGazeProfile profile;
[SerializeField]
[Tooltip("Eye midpoint. Its position is used as the gaze origin.")]
private Transform gazeOrigin;
[SerializeField]
[Tooltip("Optional forward/up fallback when Gaze Origin is not assigned. Gaze Origin rotation is authoritative when present.")]
private Transform headReference;
[Header("Default Target")]
[SerializeField]
[Tooltip("Optional output camera override. Timeline clips with no target use this before Camera.main.")]
private Camera targetCamera;
[SerializeField]
[Tooltip("Optional fallback target when Timeline is not driving this component.")]
private Transform defaultTarget;
[SerializeField]
private bool defaultToMainCamera;
[SerializeField, Range(0f, 1f)]
private float defaultInfluence;
[SerializeField]
private bool defaultCorrectives;
[Header("Camera Cut")]
[SerializeField]
private GazeCutResponse cutResponse = GazeCutResponse.Snap;
[SerializeField, Min(0f)]
private float cutTransitionDuration = 0.1f;
[SerializeField, Range(0f, 180f)]
[Tooltip("A target-direction jump at or above this angle is treated as a camera cut. Set 0 to disable angle detection.")]
private float cutDetectionAngle = 15f;
[Header("Debug Gizmos")]
[SerializeField]
private bool drawGizmos = true;
[SerializeField, Min(0.01f)]
private float gizmoDistance = 0.35f;
private readonly Dictionary<int, GazeTimelineState> timelineStates =
new Dictionary<int, GazeTimelineState>();
private int[] blendShapeIndices = Array.Empty<int>();
private float[] baselineWeights = Array.Empty<float>();
private float[] evaluatedWeights = Array.Empty<float>();
private float[] lastAppliedWeights = Array.Empty<float>();
private string[] cachedChannelNames = Array.Empty<string>();
private SkinnedMeshRenderer cachedRenderer;
private Mesh cachedMesh;
private BlendshapeGazeProfile cachedProfile;
private bool hasAppliedWeights;
private bool hasRuntimeState;
private GazeTimelineState runtimeState;
private bool hasPreviousDirection;
private Vector3 previousDesiredDirection;
private UnityEngine.Object previousTargetIdentity;
private bool cutTransitionActive;
private Vector3 cutTransitionFromDirection;
private float cutTransitionStartTime;
private int calibrationPreviewOverrideCount;
public SkinnedMeshRenderer TargetRenderer
{
get => targetRenderer;
set
{
if (targetRenderer == value)
return;
ReleaseWeights();
targetRenderer = value;
InvalidateCache();
}
}
public BlendshapeGazeProfile Profile
{
get => profile;
set
{
if (profile == value)
return;
ReleaseWeights();
profile = value;
InvalidateCache();
}
}
public Transform GazeOrigin
{
get => gazeOrigin;
set => gazeOrigin = value;
}
public Transform HeadReference
{
get => headReference;
set => headReference = value;
}
public Camera TargetCamera
{
get => targetCamera;
set => targetCamera = value;
}
public GazeDirectionResult LastDirection { get; private set; }
public Vector3 LastOriginPosition { get; private set; }
public Vector3 LastTargetPosition { get; private set; }
public bool HasActiveRequest { get; private set; }
public Vector3 GetGazeOriginPosition()
{
if (gazeOrigin != null)
return gazeOrigin.position;
if (targetRenderer != null)
return targetRenderer.bounds.center;
return transform.position;
}
public bool RebuildCache()
{
ReleaseWeights();
InvalidateCache();
return EnsureCache();
}
private Transform DirectionReference => gazeOrigin != null
? gazeOrigin
: (headReference != null ? headReference : transform);
private void LateUpdate()
{
if (Application.IsPlaying(gameObject))
{
EvaluateNow(false);
return;
}
#if UNITY_EDITOR
// ExecuteAlways receives the editor player loop requested by the
// Timeline mixer. This is a stronger final ordering point than a
// one-shot delayCall: Director animation has already updated the Head,
// GazeOrigin and facial blendshapes before this high-order LateUpdate.
if (!Application.isPlaying && timelineStates.Count > 0)
EvaluateNow(true);
#endif
}
public void SetTimelineState(int sourceId, GazeTimelineState state)
{
state.Influence = SanitizeInfluence(state.Influence);
state.CutTransitionDuration = SanitizeNonNegative(state.CutTransitionDuration);
timelineStates[sourceId] = state;
}
public void ClearTimelineState(int sourceId)
{
timelineStates.Remove(sourceId);
if (timelineStates.Count == 0)
{
HasActiveRequest = false;
ReleaseWeights();
ResetDirectionHistory();
}
}
public void SetRuntimeTarget(
Transform target,
float influence = 1f,
bool enableCorrectives = false)
{
runtimeState = new GazeTimelineState
{
Target = target,
UseMainCamera = target == null,
Influence = SanitizeInfluence(influence),
EnableCorrectives = enableCorrectives,
ResetSmoothing = !hasRuntimeState
};
hasRuntimeState = true;
}
public void SetRuntimeTargetPosition(
Vector3 worldPosition,
float influence = 1f,
bool enableCorrectives = false)
{
runtimeState = new GazeTimelineState
{
TargetPosition = worldPosition,
HasTargetPosition = true,
Influence = SanitizeInfluence(influence),
EnableCorrectives = enableCorrectives,
ResetSmoothing = !hasRuntimeState
};
hasRuntimeState = true;
}
public void ClearRuntimeTarget()
{
hasRuntimeState = false;
if (timelineStates.Count == 0)
{
HasActiveRequest = false;
ReleaseWeights();
ResetDirectionHistory();
}
}
public void EvaluateNow(bool deterministic)
{
// Calibration owns the configured blendshape channels while its preview
// session is active. Timeline/editor evaluation can still update its
// request state, but must not overwrite (or restore over) that preview.
if (calibrationPreviewOverrideCount > 0)
{
HasActiveRequest = false;
return;
}
if (!TryGetActiveState(out var state, out var usesRuntimeState))
{
HasActiveRequest = false;
ReleaseWeights();
ResetDirectionHistory();
return;
}
state.Deterministic |= deterministic;
if (!TryResolveTarget(state, out var targetPosition, out var targetIdentity))
{
HasActiveRequest = false;
ReleaseWeights();
ResetDirectionHistory();
return;
}
if (!EnsureCache())
{
HasActiveRequest = false;
ReleaseWeights();
return;
}
LastOriginPosition = GetGazeOriginPosition();
LastTargetPosition = targetPosition;
var desiredDirection = targetPosition - LastOriginPosition;
if (desiredDirection.sqrMagnitude <= Mathf.Epsilon)
{
HasActiveRequest = false;
ReleaseWeights();
return;
}
desiredDirection.Normalize();
var effectiveDirection = ApplyCutResponse(
desiredDirection,
targetIdentity,
state,
state.Deterministic || !Application.isPlaying);
var reference = DirectionReference;
LastDirection = GazeDirectionSolver.Calculate(
reference != null ? reference.rotation : Quaternion.identity,
effectiveDirection,
profile.MinYaw,
profile.MaxYaw,
profile.MinPitch,
profile.MaxPitch);
if (!profile.TryEvaluate(LastDirection, state.EnableCorrectives, evaluatedWeights))
{
HasActiveRequest = false;
ReleaseWeights();
return;
}
CaptureExternalBaseline();
ApplyWeights(Mathf.Clamp01(state.Influence));
HasActiveRequest = true;
if (usesRuntimeState)
runtimeState.ResetSmoothing = false;
}
internal void BeginCalibrationPreviewOverride()
{
if (calibrationPreviewOverrideCount == 0)
{
// If gaze was already driving the renderer, return those channels to
// their external facial baseline before the preview captures it.
ReleaseWeights();
ResetDirectionHistory();
HasActiveRequest = false;
}
calibrationPreviewOverrideCount++;
}
internal void EndCalibrationPreviewOverride()
{
if (calibrationPreviewOverrideCount <= 0)
return;
calibrationPreviewOverrideCount--;
if (calibrationPreviewOverrideCount == 0)
ResetDirectionHistory();
}
internal void PrepareForTimelineEvaluation()
{
if (!isActiveAndEnabled || calibrationPreviewOverrideCount > 0)
return;
// PlayableGraph PrepareFrame runs before Timeline animation outputs.
// Restore the prior facial baseline here, then LateUpdate/editor final
// evaluation can capture the newly sampled value even when it happens
// to be numerically identical to our previous gaze result.
ReleaseWeights();
}
private bool TryGetActiveState(out GazeTimelineState state, out bool usesRuntimeState)
{
var found = false;
var bestInfluence = MinimumInfluence;
var bestSourceId = int.MaxValue;
state = default;
usesRuntimeState = false;
foreach (var pair in timelineStates)
{
var candidate = pair.Value;
if (candidate.Influence <= MinimumInfluence)
continue;
if (found && candidate.Influence < bestInfluence)
continue;
if (found && Mathf.Approximately(candidate.Influence, bestInfluence)
&& pair.Key >= bestSourceId)
continue;
state = candidate;
bestInfluence = candidate.Influence;
bestSourceId = pair.Key;
found = true;
}
if (found)
return true;
if (hasRuntimeState && runtimeState.Influence > MinimumInfluence)
{
state = runtimeState;
usesRuntimeState = true;
return true;
}
if (defaultInfluence <= MinimumInfluence || (defaultTarget == null && !defaultToMainCamera))
return false;
state = new GazeTimelineState
{
Target = defaultTarget,
UseMainCamera = defaultTarget == null && defaultToMainCamera,
Influence = defaultInfluence,
EnableCorrectives = defaultCorrectives
};
return true;
}
private bool TryResolveTarget(
GazeTimelineState state,
out Vector3 targetPosition,
out UnityEngine.Object identity)
{
if (state.HasTargetPosition)
{
targetPosition = state.TargetPosition;
identity = null;
return IsFinite(targetPosition);
}
if (state.HasTargetDirection)
{
var targetDirection = state.TargetDirection;
if (!IsFinite(targetDirection) || targetDirection.sqrMagnitude <= Mathf.Epsilon)
{
targetPosition = default;
identity = null;
return false;
}
targetPosition = GetGazeOriginPosition() + targetDirection.normalized;
identity = null;
return true;
}
var target = state.Target;
if (target == null && state.UseMainCamera)
{
var mainCamera = targetCamera != null ? targetCamera : Camera.main;
target = mainCamera != null ? mainCamera.transform : null;
}
if (target == null)
{
targetPosition = default;
identity = null;
return false;
}
targetPosition = target.position;
identity = target;
return IsFinite(targetPosition);
}
private Vector3 ApplyCutResponse(
Vector3 desiredDirection,
UnityEngine.Object targetIdentity,
GazeTimelineState state,
bool deterministic)
{
var response = state.OverrideCutResponse ? state.CutResponse : cutResponse;
var duration = state.OverrideCutResponse
? state.CutTransitionDuration
: cutTransitionDuration;
if (deterministic || state.ResetSmoothing || !hasPreviousDirection)
{
previousDesiredDirection = desiredDirection;
previousTargetIdentity = targetIdentity;
hasPreviousDirection = true;
cutTransitionActive = false;
return desiredDirection;
}
var identityChanged = previousTargetIdentity != targetIdentity;
// Zero disables pose-jump detection while still allowing an explicit target
// identity change to count as a cut.
var angleJump = cutDetectionAngle > 0f
&& Vector3.Angle(previousDesiredDirection, desiredDirection) >= cutDetectionAngle;
if (response == GazeCutResponse.Smooth && duration > Mathf.Epsilon
&& (identityChanged || angleJump && !cutTransitionActive))
{
cutTransitionFromDirection = cutTransitionActive
? EvaluateCutTransition(previousDesiredDirection, duration)
: previousDesiredDirection;
cutTransitionStartTime = Time.unscaledTime;
cutTransitionActive = true;
}
previousDesiredDirection = desiredDirection;
previousTargetIdentity = targetIdentity;
if (response != GazeCutResponse.Smooth || duration <= Mathf.Epsilon)
{
cutTransitionActive = false;
return desiredDirection;
}
return EvaluateCutTransition(desiredDirection, duration);
}
private Vector3 EvaluateCutTransition(Vector3 destination, float duration)
{
if (!cutTransitionActive)
return destination;
var t = Mathf.Clamp01((Time.unscaledTime - cutTransitionStartTime) / duration);
if (t >= 1f)
{
cutTransitionActive = false;
return destination;
}
return Vector3.Slerp(cutTransitionFromDirection, destination, t).normalized;
}
private bool EnsureCache()
{
var mesh = targetRenderer != null ? targetRenderer.sharedMesh : null;
if (targetRenderer == null || mesh == null || profile == null || profile.Channels.Count == 0
|| !profile.IsCompatibleWith(mesh))
return false;
if (cachedRenderer == targetRenderer && cachedMesh == mesh && cachedProfile == profile
&& blendShapeIndices.Length == profile.Channels.Count && ChannelMappingMatches())
return true;
ReleaseWeights();
cachedRenderer = targetRenderer;
cachedMesh = mesh;
cachedProfile = profile;
var count = profile.Channels.Count;
blendShapeIndices = new int[count];
baselineWeights = new float[count];
evaluatedWeights = new float[count];
lastAppliedWeights = new float[count];
cachedChannelNames = new string[count];
var resolvedIndices = new HashSet<int>();
for (var i = 0; i < count; i++)
{
cachedChannelNames[i] = profile.Channels[i].BlendShapeName;
blendShapeIndices[i] = profile.FindBlendShapeIndex(mesh, i);
if (blendShapeIndices[i] < 0 || !resolvedIndices.Add(blendShapeIndices[i]))
{
InvalidateCache();
return false;
}
}
return true;
}
private void CaptureExternalBaseline()
{
for (var i = 0; i < blendShapeIndices.Length; i++)
{
var current = targetRenderer.GetBlendShapeWeight(blendShapeIndices[i]);
if (!hasAppliedWeights || Mathf.Abs(current - lastAppliedWeights[i]) > ExternalWriteTolerance)
baselineWeights[i] = current;
}
}
private void ApplyWeights(float influence)
{
for (var i = 0; i < blendShapeIndices.Length; i++)
{
var targetWeight = evaluatedWeights[i];
var channel = profile.Channels[i];
float result;
if (float.IsNaN(targetWeight))
{
result = baselineWeights[i];
}
else if (channel.Usage == GazeChannelUsage.Corrective
&& channel.CorrectiveBlendMode == GazeCorrectiveBlendMode.AdditiveFromCenter)
{
var centerWeight = profile.GetSampleWeight(GazeCalibrationPoint.Center, i);
result = baselineWeights[i] + (targetWeight - centerWeight) * influence;
}
else
{
result = Mathf.LerpUnclamped(baselineWeights[i], targetWeight, influence);
}
targetRenderer.SetBlendShapeWeight(blendShapeIndices[i], result);
lastAppliedWeights[i] = result;
}
hasAppliedWeights = true;
}
private bool ChannelMappingMatches()
{
if (cachedChannelNames.Length != profile.Channels.Count)
return false;
for (var i = 0; i < cachedChannelNames.Length; i++)
{
var channel = profile.Channels[i];
if (channel == null || !string.Equals(
cachedChannelNames[i],
channel.BlendShapeName,
StringComparison.Ordinal))
return false;
}
return true;
}
private void ReleaseWeights()
{
if (!hasAppliedWeights || cachedRenderer == null)
return;
// The old indices are only meaningful for the mesh they were built from.
if (cachedRenderer.sharedMesh != cachedMesh)
{
hasAppliedWeights = false;
return;
}
// If Animator/Timeline/a live receiver has written since our previous LateUpdate,
// that current value is the new base. Otherwise restore the last base so stopping
// the gaze track never leaves its final pose stuck on the face.
for (var i = 0; i < blendShapeIndices.Length && i < baselineWeights.Length; i++)
{
var index = blendShapeIndices[i];
if (index < 0)
continue;
var current = cachedRenderer.GetBlendShapeWeight(index);
if (Mathf.Abs(current - lastAppliedWeights[i]) > ExternalWriteTolerance)
baselineWeights[i] = current;
cachedRenderer.SetBlendShapeWeight(index, baselineWeights[i]);
}
hasAppliedWeights = false;
}
private void ResetDirectionHistory()
{
hasPreviousDirection = false;
previousTargetIdentity = null;
cutTransitionActive = false;
}
private void InvalidateCache()
{
cachedRenderer = null;
cachedMesh = null;
cachedProfile = null;
blendShapeIndices = Array.Empty<int>();
baselineWeights = Array.Empty<float>();
evaluatedWeights = Array.Empty<float>();
lastAppliedWeights = Array.Empty<float>();
cachedChannelNames = Array.Empty<string>();
hasAppliedWeights = false;
}
private void OnDisable()
{
#if UNITY_EDITOR
BlendshapeGazeMixerBehaviour.CancelEditorFinalEvaluation(this);
#endif
ReleaseWeights();
timelineStates.Clear();
ResetDirectionHistory();
HasActiveRequest = false;
}
private void OnValidate()
{
defaultInfluence = SanitizeInfluence(defaultInfluence);
cutTransitionDuration = SanitizeNonNegative(cutTransitionDuration);
cutDetectionAngle = IsFinite(cutDetectionAngle)
? Mathf.Clamp(cutDetectionAngle, 0f, 180f)
: 15f;
gizmoDistance = IsFinite(gizmoDistance)
? Mathf.Max(0.01f, gizmoDistance)
: 0.35f;
ReleaseWeights();
InvalidateCache();
}
private void OnDrawGizmos()
{
if (!drawGizmos || profile == null)
return;
var origin = gazeOrigin != null
? gazeOrigin.position
: (targetRenderer != null ? targetRenderer.bounds.center : transform.position);
var reference = DirectionReference;
var rotation = reference != null ? reference.rotation : Quaternion.identity;
var distance = Mathf.Max(0.01f, gizmoDistance);
var downLeft = origin + rotation * GazeDirectionSolver.DirectionFromAngles(
profile.MinYaw,
profile.MinPitch) * distance;
var downRight = origin + rotation * GazeDirectionSolver.DirectionFromAngles(
profile.MaxYaw,
profile.MinPitch) * distance;
var upLeft = origin + rotation * GazeDirectionSolver.DirectionFromAngles(
profile.MinYaw,
profile.MaxPitch) * distance;
var upRight = origin + rotation * GazeDirectionSolver.DirectionFromAngles(
profile.MaxYaw,
profile.MaxPitch) * distance;
Gizmos.color = new Color(0.2f, 0.9f, 1f, 0.8f);
Gizmos.DrawWireSphere(origin, distance * 0.025f);
Gizmos.DrawLine(origin, downLeft);
Gizmos.DrawLine(origin, downRight);
Gizmos.DrawLine(origin, upLeft);
Gizmos.DrawLine(origin, upRight);
Gizmos.DrawLine(downLeft, downRight);
Gizmos.DrawLine(downRight, upRight);
Gizmos.DrawLine(upRight, upLeft);
Gizmos.DrawLine(upLeft, downLeft);
Gizmos.color = Color.green;
Gizmos.DrawLine(origin, origin + rotation * Vector3.forward * distance);
if (!HasActiveRequest)
return;
Gizmos.color = LastDirection.WasClamped ? Color.red : Color.cyan;
Gizmos.DrawLine(origin, LastTargetPosition);
Gizmos.DrawWireSphere(LastTargetPosition, distance * 0.035f);
var clampedDirection = rotation * GazeDirectionSolver.DirectionFromAngles(
LastDirection.ClampedYaw,
LastDirection.ClampedPitch);
Gizmos.color = Color.yellow;
Gizmos.DrawLine(origin, origin + clampedDirection * distance);
}
private static bool IsFinite(Vector3 value)
{
return !float.IsNaN(value.x) && !float.IsInfinity(value.x)
&& !float.IsNaN(value.y) && !float.IsInfinity(value.y)
&& !float.IsNaN(value.z) && !float.IsInfinity(value.z);
}
private static bool IsFinite(float value)
{
return !float.IsNaN(value) && !float.IsInfinity(value);
}
private static float SanitizeInfluence(float value)
{
return IsFinite(value) ? Mathf.Clamp01(value) : 0f;
}
private static float SanitizeNonNegative(float value)
{
return IsFinite(value) ? Mathf.Max(0f, value) : 0f;
}
}
}