427 lines
15 KiB
C#
427 lines
15 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using UnityEngine;
|
|
|
|
namespace Streamingle.Gaze.Editor
|
|
{
|
|
/// <summary>
|
|
/// Result of an editor-only gaze-origin estimation pass.
|
|
/// </summary>
|
|
internal readonly struct GazeOriginEstimate
|
|
{
|
|
internal GazeOriginEstimate(
|
|
Vector3 worldPosition,
|
|
Vector3 leftEyeWorldPosition,
|
|
Vector3 rightEyeWorldPosition,
|
|
bool usedEyeBlendShapes,
|
|
int leftVertexCount,
|
|
int rightVertexCount,
|
|
string message)
|
|
{
|
|
WorldPosition = worldPosition;
|
|
LeftEyeWorldPosition = leftEyeWorldPosition;
|
|
RightEyeWorldPosition = rightEyeWorldPosition;
|
|
UsedEyeBlendShapes = usedEyeBlendShapes;
|
|
LeftVertexCount = leftVertexCount;
|
|
RightVertexCount = rightVertexCount;
|
|
Message = message;
|
|
}
|
|
|
|
internal Vector3 WorldPosition { get; }
|
|
internal Vector3 LeftEyeWorldPosition { get; }
|
|
internal Vector3 RightEyeWorldPosition { get; }
|
|
internal bool UsedEyeBlendShapes { get; }
|
|
internal int LeftVertexCount { get; }
|
|
internal int RightVertexCount { get; }
|
|
internal string Message { get; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Estimates the midpoint between the eyes from the vertices influenced by the configured
|
|
/// ARKit eyeLook blend shapes. This is deliberately editor-only: BakeMesh is never used by
|
|
/// the runtime gaze driver.
|
|
/// </summary>
|
|
internal static class GazeOriginEstimator
|
|
{
|
|
private const float MinimumDelta = 1e-7f;
|
|
private const int MinimumVerticesPerEye = 3;
|
|
|
|
internal static GazeOriginEstimate Estimate(
|
|
SkinnedMeshRenderer renderer,
|
|
IReadOnlyList<string> configuredChannelNames)
|
|
{
|
|
if (renderer == null)
|
|
{
|
|
return new GazeOriginEstimate(
|
|
Vector3.zero,
|
|
Vector3.zero,
|
|
Vector3.zero,
|
|
false,
|
|
0,
|
|
0,
|
|
"Target Renderer is not assigned.");
|
|
}
|
|
|
|
Mesh sourceMesh = renderer.sharedMesh;
|
|
if (sourceMesh == null || sourceMesh.vertexCount == 0)
|
|
{
|
|
return Fallback(renderer, "The renderer has no readable shared mesh.");
|
|
}
|
|
|
|
var leftShapeIndices = new HashSet<int>();
|
|
var rightShapeIndices = new HashSet<int>();
|
|
CollectShapeIndices(sourceMesh, configuredChannelNames, leftShapeIndices, rightShapeIndices);
|
|
|
|
if (leftShapeIndices.Count == 0 || rightShapeIndices.Count == 0)
|
|
{
|
|
return Fallback(
|
|
renderer,
|
|
"Could not find configured eyeLook blend shapes for both eyes. Used Renderer bounds center.");
|
|
}
|
|
|
|
int vertexCount = sourceMesh.vertexCount;
|
|
var leftMaximumDeltas = new float[vertexCount];
|
|
var rightMaximumDeltas = new float[vertexCount];
|
|
var deltaVertices = new Vector3[vertexCount];
|
|
var deltaNormals = new Vector3[vertexCount];
|
|
var deltaTangents = new Vector3[vertexCount];
|
|
|
|
AccumulateMaximumDeltas(
|
|
sourceMesh,
|
|
leftShapeIndices,
|
|
leftMaximumDeltas,
|
|
deltaVertices,
|
|
deltaNormals,
|
|
deltaTangents);
|
|
AccumulateMaximumDeltas(
|
|
sourceMesh,
|
|
rightShapeIndices,
|
|
rightMaximumDeltas,
|
|
deltaVertices,
|
|
deltaNormals,
|
|
deltaTangents);
|
|
|
|
Mesh bakedMesh = null;
|
|
try
|
|
{
|
|
bakedMesh = new Mesh { name = "GazeOriginEstimator_TemporaryMesh" };
|
|
renderer.BakeMesh(bakedMesh);
|
|
Vector3[] neutralVertices = bakedMesh.vertices;
|
|
|
|
if (neutralVertices == null || neutralVertices.Length != vertexCount)
|
|
{
|
|
return Fallback(renderer, "BakeMesh vertex count did not match the source mesh.");
|
|
}
|
|
|
|
bool hasLeft = TryCalculateWeightedCentroid(
|
|
renderer.transform,
|
|
neutralVertices,
|
|
leftMaximumDeltas,
|
|
out Vector3 leftCenter,
|
|
out int leftVertexCount);
|
|
bool hasRight = TryCalculateWeightedCentroid(
|
|
renderer.transform,
|
|
neutralVertices,
|
|
rightMaximumDeltas,
|
|
out Vector3 rightCenter,
|
|
out int rightVertexCount);
|
|
|
|
if (!hasLeft || !hasRight)
|
|
{
|
|
return Fallback(
|
|
renderer,
|
|
"Eye blend shapes did not contain enough significant vertices. Used Renderer bounds center.",
|
|
leftVertexCount,
|
|
rightVertexCount);
|
|
}
|
|
|
|
// Deliberately give both eyes equal influence. A larger or stronger blend shape on one
|
|
// side must not pull the gaze origin away from the center of the face.
|
|
Vector3 midpoint = (leftCenter + rightCenter) * 0.5f;
|
|
return new GazeOriginEstimate(
|
|
midpoint,
|
|
leftCenter,
|
|
rightCenter,
|
|
true,
|
|
leftVertexCount,
|
|
rightVertexCount,
|
|
$"Estimated from {leftVertexCount} left-eye and {rightVertexCount} right-eye vertices.");
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return Fallback(renderer, $"Eye vertex estimation failed: {exception.Message}");
|
|
}
|
|
finally
|
|
{
|
|
if (bakedMesh != null)
|
|
{
|
|
UnityEngine.Object.DestroyImmediate(bakedMesh);
|
|
}
|
|
}
|
|
}
|
|
|
|
internal static int FindBlendShapeIndex(Mesh mesh, string configuredName)
|
|
{
|
|
if (mesh == null || string.IsNullOrWhiteSpace(configuredName))
|
|
{
|
|
return -1;
|
|
}
|
|
|
|
string normalizedConfiguredName = Normalize(configuredName);
|
|
int suffixMatch = -1;
|
|
for (int index = 0; index < mesh.blendShapeCount; index++)
|
|
{
|
|
string meshName = mesh.GetBlendShapeName(index);
|
|
if (string.Equals(meshName, configuredName, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
return index;
|
|
}
|
|
|
|
if (meshName.EndsWith(configuredName, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
suffixMatch = index;
|
|
continue;
|
|
}
|
|
|
|
string normalizedMeshName = Normalize(meshName);
|
|
if (normalizedMeshName == normalizedConfiguredName)
|
|
{
|
|
return index;
|
|
}
|
|
|
|
if (normalizedMeshName.EndsWith(normalizedConfiguredName, StringComparison.Ordinal))
|
|
{
|
|
suffixMatch = index;
|
|
}
|
|
}
|
|
|
|
return suffixMatch;
|
|
}
|
|
|
|
private static void CollectShapeIndices(
|
|
Mesh mesh,
|
|
IReadOnlyList<string> configuredChannelNames,
|
|
ISet<int> leftShapeIndices,
|
|
ISet<int> rightShapeIndices)
|
|
{
|
|
if (configuredChannelNames != null)
|
|
{
|
|
for (int index = 0; index < configuredChannelNames.Count; index++)
|
|
{
|
|
string channelName = configuredChannelNames[index];
|
|
int blendShapeIndex = FindBlendShapeIndex(mesh, channelName);
|
|
if (blendShapeIndex < 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
string sideName = string.IsNullOrWhiteSpace(channelName)
|
|
? mesh.GetBlendShapeName(blendShapeIndex)
|
|
: channelName;
|
|
if (IsLeftChannel(sideName))
|
|
{
|
|
leftShapeIndices.Add(blendShapeIndex);
|
|
}
|
|
else if (IsRightChannel(sideName))
|
|
{
|
|
rightShapeIndices.Add(blendShapeIndex);
|
|
}
|
|
}
|
|
}
|
|
|
|
// A newly created profile has no bindings yet. Scanning the mesh for the standard suffixes
|
|
// still gives Auto Create a useful result before the user presses Sync Channels.
|
|
if (leftShapeIndices.Count > 0 && rightShapeIndices.Count > 0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
for (int index = 0; index < mesh.blendShapeCount; index++)
|
|
{
|
|
string name = mesh.GetBlendShapeName(index);
|
|
string normalizedName = Normalize(name);
|
|
if (!normalizedName.Contains("eyelook"))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (IsLeftChannel(name))
|
|
{
|
|
leftShapeIndices.Add(index);
|
|
}
|
|
else if (IsRightChannel(name))
|
|
{
|
|
rightShapeIndices.Add(index);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void AccumulateMaximumDeltas(
|
|
Mesh mesh,
|
|
IEnumerable<int> shapeIndices,
|
|
float[] maximumDeltas,
|
|
Vector3[] deltaVertices,
|
|
Vector3[] deltaNormals,
|
|
Vector3[] deltaTangents)
|
|
{
|
|
foreach (int shapeIndex in shapeIndices)
|
|
{
|
|
int frameCount = mesh.GetBlendShapeFrameCount(shapeIndex);
|
|
for (int frameIndex = 0; frameIndex < frameCount; frameIndex++)
|
|
{
|
|
mesh.GetBlendShapeFrameVertices(
|
|
shapeIndex,
|
|
frameIndex,
|
|
deltaVertices,
|
|
deltaNormals,
|
|
deltaTangents);
|
|
|
|
for (int vertexIndex = 0; vertexIndex < deltaVertices.Length; vertexIndex++)
|
|
{
|
|
float magnitude = deltaVertices[vertexIndex].magnitude;
|
|
if (magnitude > maximumDeltas[vertexIndex])
|
|
{
|
|
maximumDeltas[vertexIndex] = magnitude;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static bool TryCalculateWeightedCentroid(
|
|
Transform rendererTransform,
|
|
IReadOnlyList<Vector3> neutralVertices,
|
|
IReadOnlyList<float> maximumDeltas,
|
|
out Vector3 worldCentroid,
|
|
out int includedVertexCount)
|
|
{
|
|
var positiveDeltas = new List<float>();
|
|
for (int index = 0; index < maximumDeltas.Count; index++)
|
|
{
|
|
float delta = maximumDeltas[index];
|
|
if (delta > MinimumDelta && IsFinite(delta))
|
|
{
|
|
positiveDeltas.Add(delta);
|
|
}
|
|
}
|
|
|
|
if (positiveDeltas.Count < MinimumVerticesPerEye)
|
|
{
|
|
worldCentroid = Vector3.zero;
|
|
includedVertexCount = 0;
|
|
return false;
|
|
}
|
|
|
|
positiveDeltas.Sort();
|
|
float median = Percentile(positiveDeltas, 0.5f);
|
|
float percentile95 = Percentile(positiveDeltas, 0.95f);
|
|
float percentile99 = Percentile(positiveDeltas, 0.99f);
|
|
|
|
// Remove tiny cross-face noise and reject pathological spikes. Valid high-motion eye
|
|
// vertices are retained and winsorized so one vertex cannot dominate the centroid.
|
|
float lowerThreshold = Mathf.Max(MinimumDelta, Mathf.Max(median * 0.05f, percentile95 * 0.015f));
|
|
float upperThreshold = Mathf.Max(percentile99 * 4f, lowerThreshold);
|
|
float maximumWeight = Mathf.Max(percentile95 * 2f, lowerThreshold);
|
|
|
|
Vector3 weightedPosition = Vector3.zero;
|
|
float weightSum = 0f;
|
|
includedVertexCount = 0;
|
|
for (int index = 0; index < maximumDeltas.Count; index++)
|
|
{
|
|
float delta = maximumDeltas[index];
|
|
if (!IsFinite(delta) || delta < lowerThreshold || delta > upperThreshold)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
float weight = Mathf.Min(delta, maximumWeight);
|
|
weightedPosition += rendererTransform.TransformPoint(neutralVertices[index]) * weight;
|
|
weightSum += weight;
|
|
includedVertexCount++;
|
|
}
|
|
|
|
if (includedVertexCount < MinimumVerticesPerEye || weightSum <= Mathf.Epsilon)
|
|
{
|
|
worldCentroid = Vector3.zero;
|
|
return false;
|
|
}
|
|
|
|
worldCentroid = weightedPosition / weightSum;
|
|
return IsFinite(worldCentroid);
|
|
}
|
|
|
|
private static float Percentile(IReadOnlyList<float> sortedValues, float percentile)
|
|
{
|
|
if (sortedValues.Count == 0)
|
|
{
|
|
return 0f;
|
|
}
|
|
|
|
float position = Mathf.Clamp01(percentile) * (sortedValues.Count - 1);
|
|
int lowerIndex = Mathf.FloorToInt(position);
|
|
int upperIndex = Mathf.CeilToInt(position);
|
|
return Mathf.Lerp(sortedValues[lowerIndex], sortedValues[upperIndex], position - lowerIndex);
|
|
}
|
|
|
|
private static GazeOriginEstimate Fallback(
|
|
SkinnedMeshRenderer renderer,
|
|
string message,
|
|
int leftVertexCount = 0,
|
|
int rightVertexCount = 0)
|
|
{
|
|
Vector3 center = renderer != null ? renderer.bounds.center : Vector3.zero;
|
|
return new GazeOriginEstimate(
|
|
center,
|
|
center,
|
|
center,
|
|
false,
|
|
leftVertexCount,
|
|
rightVertexCount,
|
|
message);
|
|
}
|
|
|
|
private static bool IsLeftChannel(string name)
|
|
{
|
|
return Normalize(name).EndsWith("left", StringComparison.Ordinal);
|
|
}
|
|
|
|
private static bool IsRightChannel(string name)
|
|
{
|
|
return Normalize(name).EndsWith("right", StringComparison.Ordinal);
|
|
}
|
|
|
|
private static string Normalize(string value)
|
|
{
|
|
if (string.IsNullOrEmpty(value))
|
|
{
|
|
return string.Empty;
|
|
}
|
|
|
|
var characters = new char[value.Length];
|
|
int count = 0;
|
|
for (int index = 0; index < value.Length; index++)
|
|
{
|
|
char character = value[index];
|
|
if (!char.IsLetterOrDigit(character))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
characters[count++] = char.ToLowerInvariant(character);
|
|
}
|
|
|
|
return new string(characters, 0, count);
|
|
}
|
|
|
|
private static bool IsFinite(Vector3 value)
|
|
{
|
|
return IsFinite(value.x) && IsFinite(value.y) && IsFinite(value.z);
|
|
}
|
|
|
|
private static bool IsFinite(float value)
|
|
{
|
|
return !float.IsNaN(value) && !float.IsInfinity(value);
|
|
}
|
|
}
|
|
}
|