streamingle-unity-utilities/Runtime/Gaze/BlendshapeGazeProfile.cs

635 lines
23 KiB
C#

using System;
using System.Collections.Generic;
using System.Text;
using UnityEngine;
namespace Streamingle.Gaze
{
[CreateAssetMenu(fileName = "BlendshapeGazeProfile", menuName = "Streamingle Utilities/Gaze/Blendshape Gaze Profile")]
public sealed class BlendshapeGazeProfile : ScriptableObject
{
private const float DefaultMinYaw = -35f;
private const float DefaultMaxYaw = 35f;
private const float DefaultMinPitch = -25f;
private const float DefaultMaxPitch = 20f;
private static readonly string[] StandardEyeLookNames =
{
"eyeLookUpLeft",
"eyeLookUpRight",
"eyeLookDownLeft",
"eyeLookDownRight",
"eyeLookInLeft",
"eyeLookInRight",
"eyeLookOutLeft",
"eyeLookOutRight"
};
[Header("Direction Limits")]
[SerializeField, Range(-90f, -0.1f)]
private float minYaw = DefaultMinYaw;
[SerializeField, Range(0.1f, 90f)]
private float maxYaw = DefaultMaxYaw;
[SerializeField, Range(-90f, -0.1f)]
private float minPitch = DefaultMinPitch;
[SerializeField, Range(0.1f, 90f)]
private float maxPitch = DefaultMaxPitch;
[Header("Source Mesh")]
[SerializeField]
[Tooltip("Optional calibration mesh guard. Clear this only when intentionally sharing a profile across compatible meshes.")]
private Mesh sourceMesh;
[Header("Blendshape Mapping")]
[SerializeField]
private List<GazeBlendShapeChannel> channels = new List<GazeBlendShapeChannel>();
[Header("3x3 Calibration")]
[SerializeField]
private List<GazeCalibrationSample> samples = new List<GazeCalibrationSample>(9);
[SerializeField, HideInInspector]
private string channelLayoutSignature;
public float MinYaw => minYaw;
public float MaxYaw => maxYaw;
public float MinPitch => minPitch;
public float MaxPitch => maxPitch;
public Mesh SourceMesh => sourceMesh;
public IReadOnlyList<GazeBlendShapeChannel> Channels => channels;
public IReadOnlyList<GazeCalibrationSample> Samples => samples;
public bool IsFullyCalibrated
{
get
{
EnsureGrid();
if (channels.Count == 0)
return false;
for (var i = 0; i < samples.Count; i++)
{
if (samples[i] == null || !samples[i].Captured
|| samples[i].Weights.Count != channels.Count)
return false;
}
return true;
}
}
public bool IsSampleCaptured(GazeCalibrationPoint point)
{
EnsureGrid();
var sampleIndex = (int)point;
return sampleIndex >= 0 && sampleIndex < samples.Count
&& samples[sampleIndex] != null
&& samples[sampleIndex].Captured
&& samples[sampleIndex].Weights.Count == channels.Count;
}
public float[] GetSampleWeightsCopy(GazeCalibrationPoint point)
{
EnsureGrid();
var sampleIndex = (int)point;
if (sampleIndex < 0 || sampleIndex >= samples.Count)
return Array.Empty<float>();
var copy = new float[channels.Count];
var sample = samples[sampleIndex];
if (sample == null)
return copy;
for (var i = 0; i < copy.Length; i++)
copy[i] = sample.GetWeight(i);
return copy;
}
public bool CanGenerateCorners
{
get
{
return channels.Count > 0
&& IsSampleCaptured(GazeCalibrationPoint.Center)
&& IsSampleCaptured(GazeCalibrationPoint.CenterLeft)
&& IsSampleCaptured(GazeCalibrationPoint.CenterRight)
&& IsSampleCaptured(GazeCalibrationPoint.UpCenter)
&& IsSampleCaptured(GazeCalibrationPoint.DownCenter);
}
}
public bool GenerateCornersFromCardinals()
{
if (!CanGenerateCorners)
return false;
var center = GetSampleWeightsCopy(GazeCalibrationPoint.Center);
var left = GetSampleWeightsCopy(GazeCalibrationPoint.CenterLeft);
var right = GetSampleWeightsCopy(GazeCalibrationPoint.CenterRight);
var up = GetSampleWeightsCopy(GazeCalibrationPoint.UpCenter);
var down = GetSampleWeightsCopy(GazeCalibrationPoint.DownCenter);
SetGeneratedCorner(GazeCalibrationPoint.DownLeft, down, left, center);
SetGeneratedCorner(GazeCalibrationPoint.DownRight, down, right, center);
SetGeneratedCorner(GazeCalibrationPoint.UpLeft, up, left, center);
SetGeneratedCorner(GazeCalibrationPoint.UpRight, up, right, center);
return true;
}
public bool SeedStandardEyeLook(float maximumWeight = 100f)
{
if (channels.Count == 0 || float.IsNaN(maximumWeight) || float.IsInfinity(maximumWeight))
return false;
var standardIndices = new int[StandardEyeLookNames.Length];
for (var i = 0; i < StandardEyeLookNames.Length; i++)
{
standardIndices[i] = FindChannelIndexByNormalizedSuffix(StandardEyeLookNames[i]);
if (standardIndices[i] < 0)
return false;
}
maximumWeight = Mathf.Clamp(maximumWeight, 0f, 100f);
var center = new float[channels.Count];
var left = new float[channels.Count];
var right = new float[channels.Count];
var up = new float[channels.Count];
var down = new float[channels.Count];
// StandardEyeLookNames order: Up L/R, Down L/R, In L/R, Out L/R.
up[standardIndices[0]] = maximumWeight;
up[standardIndices[1]] = maximumWeight;
down[standardIndices[2]] = maximumWeight;
down[standardIndices[3]] = maximumWeight;
right[standardIndices[4]] = maximumWeight;
left[standardIndices[5]] = maximumWeight;
left[standardIndices[6]] = maximumWeight;
right[standardIndices[7]] = maximumWeight;
EnsureGrid();
samples[(int)GazeCalibrationPoint.Center].SetWeights(center, channels.Count);
samples[(int)GazeCalibrationPoint.CenterLeft].SetWeights(left, channels.Count);
samples[(int)GazeCalibrationPoint.CenterRight].SetWeights(right, channels.Count);
samples[(int)GazeCalibrationPoint.UpCenter].SetWeights(up, channels.Count);
samples[(int)GazeCalibrationPoint.DownCenter].SetWeights(down, channels.Count);
return GenerateCornersFromCardinals();
}
public bool CopySample(
GazeCalibrationPoint source,
GazeCalibrationPoint destination,
bool mirrorLeftRight)
{
if (!IsSampleCaptured(source))
return false;
var destinationIndex = (int)destination;
if (destinationIndex < 0 || destinationIndex >= 9)
return false;
var sourceWeights = GetSampleWeightsCopy(source);
if (!mirrorLeftRight)
{
samples[destinationIndex].SetWeights(sourceWeights, channels.Count);
return true;
}
var mirrored = new float[channels.Count];
for (var i = 0; i < mirrored.Length; i++)
{
var counterpart = FindMirroredChannelIndex(i);
mirrored[i] = sourceWeights[counterpart];
}
samples[destinationIndex].SetWeights(mirrored, channels.Count);
return true;
}
public void SetDirectionLimits(float left, float right, float down, float up)
{
minYaw = SanitizeNegativeDirectionLimit(left, DefaultMinYaw);
maxYaw = SanitizePositiveDirectionLimit(right, DefaultMaxYaw);
minPitch = SanitizeNegativeDirectionLimit(down, DefaultMinPitch);
maxPitch = SanitizePositiveDirectionLimit(up, DefaultMaxPitch);
}
public void SetSourceMesh(Mesh mesh)
{
sourceMesh = mesh;
}
public bool IsCompatibleWith(Mesh mesh)
{
return mesh != null && (sourceMesh == null || sourceMesh == mesh);
}
public bool SyncEyeLookChannels(SkinnedMeshRenderer renderer)
{
var mesh = renderer != null ? renderer.sharedMesh : null;
if (mesh == null)
return false;
var previousChannels = new List<GazeBlendShapeChannel>(channels);
var synchronized = new List<GazeBlendShapeChannel>();
var matchedEyeChannelCount = 0;
for (var standardIndex = 0; standardIndex < StandardEyeLookNames.Length; standardIndex++)
{
var standardName = StandardEyeLookNames[standardIndex];
var meshName = FindBlendShapeName(mesh, standardName);
if (string.IsNullOrEmpty(meshName))
continue;
var existing = FindChannel(previousChannels, meshName);
synchronized.Add(existing ?? new GazeBlendShapeChannel(meshName));
matchedEyeChannelCount++;
}
if (matchedEyeChannelCount != StandardEyeLookNames.Length)
return false;
// Preserve explicitly configured correctives when resyncing the eight eye channels.
for (var i = 0; i < previousChannels.Count; i++)
{
var channel = previousChannels[i];
if (channel != null && channel.Usage == GazeChannelUsage.Corrective
&& !ContainsChannel(synchronized, channel.BlendShapeName))
synchronized.Add(channel);
}
if (synchronized.Count == 0)
return false;
channels = synchronized;
EnsureGrid();
InvalidateSamples();
ResizeSamples();
UpdateChannelLayoutSignature();
sourceMesh = mesh;
return true;
}
public bool CaptureSample(GazeCalibrationPoint point, SkinnedMeshRenderer renderer)
{
if (renderer == null || renderer.sharedMesh == null || channels.Count == 0
|| !IsCompatibleWith(renderer.sharedMesh))
return false;
if (sourceMesh == null)
sourceMesh = renderer.sharedMesh;
var sampleIndex = (int)point;
if (sampleIndex < 0 || sampleIndex >= 9)
return false;
EnsureGrid();
var values = new float[channels.Count];
for (var i = 0; i < channels.Count; i++)
{
var channel = channels[i];
var index = channel != null
? FindBlendShapeIndex(renderer.sharedMesh, channel.BlendShapeName)
: -1;
if (index < 0)
return false;
values[i] = renderer.GetBlendShapeWeight(index);
}
samples[sampleIndex].SetWeights(values, channels.Count);
return true;
}
public void SetSample(GazeCalibrationPoint point, IReadOnlyList<float> weights)
{
EnsureGrid();
var sampleIndex = (int)point;
if (sampleIndex < 0 || sampleIndex >= samples.Count)
throw new ArgumentOutOfRangeException(nameof(point), point, "Calibration point must be in the 3x3 grid.");
if (weights == null || weights.Count != channels.Count)
throw new ArgumentException("A sample must contain exactly one weight per configured channel.", nameof(weights));
for (var channelIndex = 0; channelIndex < weights.Count; channelIndex++)
{
if (!IsFinite(weights[channelIndex]))
throw new ArgumentException("Calibration sample weights must be finite.", nameof(weights));
}
samples[sampleIndex].SetWeights(weights, channels.Count);
}
public float GetSampleWeight(GazeCalibrationPoint point, int channelIndex)
{
EnsureGrid();
var sampleIndex = (int)point;
return sampleIndex >= 0 && sampleIndex < samples.Count
&& channelIndex >= 0 && channelIndex < channels.Count
? samples[sampleIndex].GetWeight(channelIndex)
: 0f;
}
public void ReplaceChannels(IReadOnlyList<GazeBlendShapeChannel> replacements)
{
if (channels == null)
channels = new List<GazeBlendShapeChannel>();
channels.Clear();
if (replacements != null)
{
for (var i = 0; i < replacements.Count; i++)
{
var channel = replacements[i];
if (channel != null)
channels.Add(new GazeBlendShapeChannel(
channel.BlendShapeName,
channel.Usage,
channel.CorrectiveBlendMode));
}
}
EnsureGrid();
InvalidateSamples();
ResizeSamples();
UpdateChannelLayoutSignature();
}
public bool TryEvaluate(
GazeDirectionResult direction,
bool enableCorrectives,
float[] output)
{
if (output == null || output.Length < channels.Count || !IsFullyCalibrated)
return false;
EnsureGrid();
GazeDirectionSolver.Evaluate3x3(
samples,
channels.Count,
direction.NormalizedYaw,
direction.NormalizedPitch,
output);
for (var i = 0; i < channels.Count; i++)
{
if (!IsFinite(output[i]))
return false;
}
if (!enableCorrectives)
{
for (var i = 0; i < channels.Count; i++)
{
if (channels[i] != null && channels[i].Usage == GazeChannelUsage.Corrective)
output[i] = float.NaN;
}
}
return true;
}
public int FindBlendShapeIndex(Mesh mesh, int channelIndex)
{
return mesh != null && channelIndex >= 0 && channelIndex < channels.Count
? FindBlendShapeIndex(mesh, channels[channelIndex].BlendShapeName)
: -1;
}
public static int FindBlendShapeIndex(Mesh mesh, string name)
{
if (mesh == null || string.IsNullOrWhiteSpace(name))
return -1;
var exact = mesh.GetBlendShapeIndex(name);
if (exact >= 0)
return exact;
for (var i = 0; i < mesh.blendShapeCount; i++)
{
var meshName = mesh.GetBlendShapeName(i);
if (meshName.EndsWith(name, StringComparison.OrdinalIgnoreCase))
return i;
}
return -1;
}
private static string FindBlendShapeName(Mesh mesh, string standardName)
{
var index = FindBlendShapeIndex(mesh, standardName);
return index >= 0 ? mesh.GetBlendShapeName(index) : null;
}
private static GazeBlendShapeChannel FindChannel(
IReadOnlyList<GazeBlendShapeChannel> source,
string name)
{
for (var i = 0; i < source.Count; i++)
{
var channel = source[i];
if (channel != null && string.Equals(
channel.BlendShapeName,
name,
StringComparison.OrdinalIgnoreCase))
return channel;
}
return null;
}
private static bool ContainsChannel(
IReadOnlyList<GazeBlendShapeChannel> source,
string name)
{
return FindChannel(source, name) != null;
}
private void SetGeneratedCorner(
GazeCalibrationPoint point,
IReadOnlyList<float> vertical,
IReadOnlyList<float> horizontal,
IReadOnlyList<float> center)
{
var weights = new float[channels.Count];
for (var i = 0; i < weights.Length; i++)
{
var value = vertical[i] + horizontal[i] - center[i];
weights[i] = ClampGeneratedWeight(i, value);
}
samples[(int)point].SetWeights(weights, channels.Count);
}
private float ClampGeneratedWeight(int channelIndex, float value)
{
return channels[channelIndex] != null
&& channels[channelIndex].Usage == GazeChannelUsage.Corrective
? Mathf.Clamp(value, -100f, 100f)
: Mathf.Clamp(value, 0f, 100f);
}
private int FindChannelIndexByNormalizedSuffix(string standardName)
{
var normalizedSuffix = NormalizeChannelName(standardName);
for (var i = 0; i < channels.Count; i++)
{
var channel = channels[i];
var normalizedName = NormalizeChannelName(channel != null ? channel.BlendShapeName : null);
if (normalizedName.EndsWith(normalizedSuffix, StringComparison.Ordinal))
return i;
}
return -1;
}
private int FindMirroredChannelIndex(int channelIndex)
{
if (channelIndex < 0 || channelIndex >= channels.Count || channels[channelIndex] == null)
return channelIndex;
const string leftSuffix = "left";
const string rightSuffix = "right";
var normalizedName = NormalizeChannelName(channels[channelIndex].BlendShapeName);
string counterpartName;
if (normalizedName.EndsWith(leftSuffix, StringComparison.Ordinal))
{
counterpartName = normalizedName.Substring(0, normalizedName.Length - leftSuffix.Length)
+ rightSuffix;
}
else if (normalizedName.EndsWith(rightSuffix, StringComparison.Ordinal))
{
counterpartName = normalizedName.Substring(0, normalizedName.Length - rightSuffix.Length)
+ leftSuffix;
}
else
{
return channelIndex;
}
for (var i = 0; i < channels.Count; i++)
{
var channel = channels[i];
if (channel != null && string.Equals(
NormalizeChannelName(channel.BlendShapeName),
counterpartName,
StringComparison.Ordinal))
return i;
}
return channelIndex;
}
private static string NormalizeChannelName(string value)
{
if (string.IsNullOrEmpty(value))
return string.Empty;
var builder = new StringBuilder(value.Length);
for (var i = 0; i < value.Length; i++)
{
if (char.IsLetterOrDigit(value[i]))
builder.Append(char.ToLowerInvariant(value[i]));
}
return builder.ToString();
}
private void OnEnable()
{
if (channels == null)
channels = new List<GazeBlendShapeChannel>();
EnsureGrid();
ResizeSamples();
if (string.IsNullOrEmpty(channelLayoutSignature))
UpdateChannelLayoutSignature();
}
private void OnValidate()
{
minYaw = SanitizeNegativeDirectionLimit(minYaw, DefaultMinYaw);
maxYaw = SanitizePositiveDirectionLimit(maxYaw, DefaultMaxYaw);
minPitch = SanitizeNegativeDirectionLimit(minPitch, DefaultMinPitch);
maxPitch = SanitizePositiveDirectionLimit(maxPitch, DefaultMaxPitch);
if (channels == null)
channels = new List<GazeBlendShapeChannel>();
else
channels.RemoveAll(channel => channel == null);
EnsureGrid();
var currentSignature = BuildChannelLayoutSignature();
if (!string.IsNullOrEmpty(channelLayoutSignature)
&& !string.Equals(channelLayoutSignature, currentSignature, StringComparison.Ordinal))
InvalidateSamples();
ResizeSamples();
channelLayoutSignature = currentSignature;
}
private static float SanitizeNegativeDirectionLimit(float value, float fallback)
{
if (float.IsNaN(value) || float.IsInfinity(value))
value = fallback;
return Mathf.Clamp(value, -90f, -0.1f);
}
private static float SanitizePositiveDirectionLimit(float value, float fallback)
{
if (float.IsNaN(value) || float.IsInfinity(value))
value = fallback;
return Mathf.Clamp(value, 0.1f, 90f);
}
private static bool IsFinite(float value)
{
return !float.IsNaN(value) && !float.IsInfinity(value);
}
private void EnsureGrid()
{
if (samples == null)
samples = new List<GazeCalibrationSample>(9);
while (samples.Count < 9)
samples.Add(new GazeCalibrationSample((GazeCalibrationPoint)samples.Count));
if (samples.Count > 9)
samples.RemoveRange(9, samples.Count - 9);
for (var i = 0; i < 9; i++)
{
if (samples[i] == null)
samples[i] = new GazeCalibrationSample((GazeCalibrationPoint)i);
else
samples[i].SetPoint((GazeCalibrationPoint)i);
}
}
private void ResizeSamples()
{
for (var i = 0; i < samples.Count; i++)
samples[i].Resize(channels.Count);
}
private void InvalidateSamples()
{
for (var i = 0; i < samples.Count; i++)
samples[i]?.Invalidate();
}
private void UpdateChannelLayoutSignature()
{
channelLayoutSignature = BuildChannelLayoutSignature();
}
private string BuildChannelLayoutSignature()
{
var builder = new StringBuilder(channels.Count * 24);
for (var i = 0; i < channels.Count; i++)
{
var channel = channels[i];
builder.Append(channel != null ? channel.BlendShapeName : "<null>");
builder.Append('|');
builder.Append(channel != null ? (int)channel.Usage : -1);
builder.Append('|');
builder.Append(channel != null ? (int)channel.CorrectiveBlendMode : -1);
builder.Append(';');
}
return builder.ToString();
}
}
}