using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; using Unity.Cinemachine; using UnityEditor; using UnityEngine; using UnityEngine.Playables; using UnityEngine.Timeline; namespace Streamingle.Editor { /// /// Explicitly snapshots the difference between generated cut boundaries and /// the currently edited AI preview Timeline. It never mutates or saves Unity /// assets, scenes, or Timeline clips. /// public static class AICameraCutCorrectionRecorder { internal const string CurrentSchemaVersion = "ai-camera-cut-correction-snapshot-v1"; internal const int RequiredSampleRate = 60; internal const int MoveMatchWindowFrames = 180; private const double FrameAlignmentTolerance = 0.25d; private const double CostEpsilon = 0.000000001d; private const string EditorStyleHashPrefix = "editor-style-sha256:"; /// /// Captures one correction snapshot for the active AI preview belonging /// to . The returned value is the full /// path of the newly written JSON file. /// public static string CaptureForCli( PlayableDirector sourceDirector, string generatedDirectory, string editorStyleId, string outputDirectory) { if (EditorApplication.isPlayingOrWillChangePlaymode) { throw new InvalidOperationException( "Cut correction capture is available only in Edit Mode."); } if (sourceDirector == null) { throw new ArgumentNullException(nameof(sourceDirector)); } if (!AICameraTimelinePreviewImporter .TryGetPreviewTimelineForEditor( sourceDirector, out var timeline)) { throw new InvalidOperationException( "The selected source Timeline has no AI camera preview."); } if (!AICameraTimelinePreviewImporter .TryGetPreviewGenerationProvenanceForEditor( sourceDirector, out var provenance)) { throw new InvalidOperationException( "The AI preview has no generation provenance. Re-import " + "the generated camera before recording corrections."); } var fullGeneratedDirectory = RequireDirectoryPath( generatedDirectory, nameof(generatedDirectory)); // Fail closed if any generated payload no longer matches metadata. AICameraTimelinePreviewImporter.ValidateGeneratedDirectoryForCli( fullGeneratedDirectory); return CaptureTimeline( timeline, fullGeneratedDirectory, editorStyleId, outputDirectory, DateTime.UtcNow, provenance.MetadataSha256); } internal static string CaptureTimelineForTests( TimelineAsset timeline, string generatedDirectory, string editorStyleId, string outputDirectory, DateTime capturedUtc, string expectedMetadataSha256 = null) { return CaptureTimeline( timeline, RequireDirectoryPath( generatedDirectory, nameof(generatedDirectory)), editorStyleId, outputDirectory, capturedUtc, expectedMetadataSha256); } private static string CaptureTimeline( TimelineAsset timeline, string generatedDirectory, string editorStyleId, string outputDirectory, DateTime capturedUtc, string expectedMetadataSha256) { if (timeline == null) { throw new ArgumentNullException(nameof(timeline)); } var input = ReadGenerationInput(generatedDirectory); if (!string.IsNullOrWhiteSpace(expectedMetadataSha256)) { var expected = NormalizeSha256( expectedMetadataSha256, "preview provenance metadata SHA-256"); if (!string.Equals( expected, input.MetadataSha256, StringComparison.Ordinal)) { throw new InvalidDataException( "The selected AI preview belongs to a different " + "generation. Its metadata SHA-256 does not match."); } } var finalBoundaries = ExtractFinalBoundaryFrames( timeline, input.Metadata.frameCount); var corrections = BuildCorrections( input.OriginalBoundaryFrames, finalBoundaries); var anonymizedEditorStyleId = AnonymizeEditorStyleId( editorStyleId); var timelineAssetPath = AssetDatabase.GetAssetPath(timeline); var snapshot = new CorrectionSnapshot { schemaVersion = CurrentSchemaVersion, capturedUtc = capturedUtc .ToUniversalTime() .ToString( "yyyy-MM-dd'T'HH:mm:ss.fff'Z'", CultureInfo.InvariantCulture), sampleRate = RequiredSampleRate, songName = input.Metadata.songName, songId = input.Metadata.songId, datasetId = input.Metadata.datasetId, generationHashSha256 = input.MetadataSha256, generationOutputSha256 = new OutputSha256 { worldCamera = NormalizeSha256( input.Metadata.outputSha256.worldCamera, "metadata outputSha256.worldCamera"), time = NormalizeSha256( input.Metadata.outputSha256.time, "metadata outputSha256.time"), shots = NormalizeSha256( input.Metadata.outputSha256.shots, "metadata outputSha256.shots") }, editorStyleId = anonymizedEditorStyleId, previewTimelineGuid = string.IsNullOrWhiteSpace( timelineAssetPath) ? string.Empty : AssetDatabase.AssetPathToGUID(timelineAssetPath), matchingPolicy = "ordered-dp-v1", moveMatchWindowFrames = MoveMatchWindowFrames, ordinalBase = 0, originalBoundaryFrames = input.OriginalBoundaryFrames, finalBoundaryFrames = finalBoundaries, originalBoundarySha256 = ComputeBoundarySha256( input.OriginalBoundaryFrames), finalBoundarySha256 = ComputeBoundarySha256( finalBoundaries), originalBoundaryCount = input.OriginalBoundaryFrames.Length, finalBoundaryCount = finalBoundaries.Length, corrections = corrections }; foreach (var correction in corrections) { switch (correction.action) { case "keep": snapshot.keepCount++; break; case "move": snapshot.moveCount++; break; case "add": snapshot.addCount++; break; case "delete": snapshot.deleteCount++; break; } } var json = JsonUtility.ToJson(snapshot, true) + Environment.NewLine; return WriteSnapshotAtomically( outputDirectory, snapshot, json); } private static GenerationInput ReadGenerationInput( string generatedDirectory) { var metadataPath = Path.Combine( generatedDirectory, "metadata.json"); if (!File.Exists(metadataPath)) { throw new FileNotFoundException( "Generated camera metadata was not found.", metadataPath); } var metadataBytes = File.ReadAllBytes(metadataPath); var metadataJson = Encoding.UTF8.GetString(metadataBytes); if (metadataJson.Length > 0 && metadataJson[0] == '\uFEFF') { metadataJson = metadataJson.Substring(1); } var metadata = JsonUtility.FromJson( metadataJson); if (metadata == null) { throw new InvalidDataException( "Generated camera metadata is invalid JSON."); } ValidateMetadata(metadata); var shotsPath = ResolveSafePayloadPath( generatedDirectory, metadata.shotsFile, "shotsFile"); if (!File.Exists(shotsPath)) { throw new FileNotFoundException( "Generated camera shot definitions were not found.", shotsPath); } var shotFile = JsonUtility.FromJson( File.ReadAllText(shotsPath)); if (shotFile?.shots == null || shotFile.shots.Count == 0) { throw new InvalidDataException( "Generated shot definitions contain no shots."); } var cursor = 0; for (var index = 0; index < shotFile.shots.Count; index++) { var shot = shotFile.shots[index]; if (shot == null || shot.startFrame != cursor || shot.endFrameExclusive <= shot.startFrame || shot.endFrameExclusive > metadata.frameCount) { throw new InvalidDataException( $"Generated shot {index} is not a contiguous, " + "positive frame range."); } cursor = shot.endFrameExclusive; } if (cursor != metadata.frameCount) { throw new InvalidDataException( "Generated shots do not cover the complete song."); } var boundaries = shotFile.shots .Take(shotFile.shots.Count - 1) .Select(shot => shot.endFrameExclusive) .ToArray(); ValidateBoundarySequence( boundaries, metadata.frameCount, "generated"); return new GenerationInput { Metadata = metadata, MetadataSha256 = ComputeSha256(metadataBytes), OriginalBoundaryFrames = boundaries }; } private static void ValidateMetadata(GeneratedMetadata metadata) { if (string.IsNullOrWhiteSpace(metadata.songName)) { throw new InvalidDataException( "metadata.songName is required."); } if (string.IsNullOrWhiteSpace(metadata.songId)) { throw new InvalidDataException( "metadata.songId is required."); } if (string.IsNullOrWhiteSpace(metadata.datasetId)) { throw new InvalidDataException( "metadata.datasetId is required."); } if (metadata.sampleRate != RequiredSampleRate) { throw new InvalidDataException( $"Cut correction capture requires exactly " + $"{RequiredSampleRate} FPS, but metadata declares " + $"{metadata.sampleRate} FPS."); } if (metadata.frameCount <= 0) { throw new InvalidDataException( "metadata.frameCount must be positive."); } if (metadata.outputSha256 == null) { throw new InvalidDataException( "metadata.outputSha256 is required."); } NormalizeSha256( metadata.outputSha256.worldCamera, "metadata outputSha256.worldCamera"); NormalizeSha256( metadata.outputSha256.time, "metadata outputSha256.time"); NormalizeSha256( metadata.outputSha256.shots, "metadata outputSha256.shots"); } private static int[] ExtractFinalBoundaryFrames( TimelineAsset timeline, int frameCount) { var matchingTracks = timeline.GetOutputTracks() .OfType() .Where(track => track.name == AICameraTimelinePreviewImporter .GeneratedCinemachineTrackName) .ToArray(); if (matchingTracks.Length != 1) { throw new InvalidDataException( $"The preview Timeline must contain exactly one " + $"'{AICameraTimelinePreviewImporter.GeneratedCinemachineTrackName}' " + "track."); } var clips = matchingTracks[0] .GetClips() .OrderBy(clip => clip.start) .ThenBy(clip => clip.duration) .ToArray(); if (clips.Length == 0) { throw new InvalidDataException( "The generated Cinemachine track contains no clips."); } var starts = new int[clips.Length]; var ends = new int[clips.Length]; for (var index = 0; index < clips.Length; index++) { var clip = clips[index]; if (clip.easeInDuration > 0.0000001d || clip.easeOutDuration > 0.0000001d) { throw new InvalidDataException( $"Cinemachine clip {index} has an ease/blend. " + "Cut learning accepts hard cuts only."); } starts[index] = ToAlignedFrame( clip.start, $"Cinemachine clip {index} start"); ends[index] = ToAlignedFrame( clip.end, $"Cinemachine clip {index} end"); if (ends[index] <= starts[index]) { throw new InvalidDataException( $"Cinemachine clip {index} has no positive duration."); } } if (starts[0] != 0) { throw new InvalidDataException( "The generated Cinemachine track must begin at frame 0."); } for (var index = 1; index < clips.Length; index++) { if (starts[index] != ends[index - 1]) { throw new InvalidDataException( $"Cut boundary before clip {index} has a gap or " + "overlap. Align both clip edges to the same frame " + "before recording corrections."); } } if (ends[ends.Length - 1] != frameCount) { throw new InvalidDataException( $"The generated Cinemachine track must end at frame " + $"{frameCount}."); } var boundaries = starts.Skip(1).ToArray(); ValidateBoundarySequence(boundaries, frameCount, "edited"); return boundaries; } private static int ToAlignedFrame(double seconds, string label) { if (double.IsNaN(seconds) || double.IsInfinity(seconds)) { throw new InvalidDataException( $"{label} is not a finite Timeline time."); } var frameValue = seconds * RequiredSampleRate; var rounded = Math.Round( frameValue, MidpointRounding.AwayFromZero); if (Math.Abs(frameValue - rounded) > FrameAlignmentTolerance) { throw new InvalidDataException( $"{label} is not aligned to a {RequiredSampleRate} FPS " + "frame boundary."); } if (rounded < int.MinValue || rounded > int.MaxValue) { throw new InvalidDataException( $"{label} is outside the supported frame range."); } return (int)rounded; } private static CorrectionRecord[] BuildCorrections( int[] originalBoundaries, int[] finalBoundaries) { var originalCount = originalBoundaries.Length; var finalCount = finalBoundaries.Length; var costs = new double[originalCount + 1, finalCount + 1]; var steps = new DiffStep[originalCount + 1, finalCount + 1]; for (var originalIndex = 0; originalIndex <= originalCount; originalIndex++) { for (var finalIndex = 0; finalIndex <= finalCount; finalIndex++) { costs[originalIndex, finalIndex] = double.PositiveInfinity; } } costs[0, 0] = 0d; for (var originalIndex = 0; originalIndex <= originalCount; originalIndex++) { for (var finalIndex = 0; finalIndex <= finalCount; finalIndex++) { if (originalIndex == 0 && finalIndex == 0) { continue; } var bestCost = double.PositiveInfinity; var bestPriority = int.MaxValue; var bestStep = DiffStep.None; if (originalIndex > 0 && finalIndex > 0) { var delta = Math.Abs( originalBoundaries[originalIndex - 1] - finalBoundaries[finalIndex - 1]); if (delta <= MoveMatchWindowFrames) { var matchCost = delta == 0 ? 0d : 0.25d + 1.5d * delta / MoveMatchWindowFrames; SelectCandidate( costs[originalIndex - 1, finalIndex - 1] + matchCost, 0, DiffStep.Match, ref bestCost, ref bestPriority, ref bestStep); } } if (originalIndex > 0) { SelectCandidate( costs[originalIndex - 1, finalIndex] + 1d, 1, DiffStep.Delete, ref bestCost, ref bestPriority, ref bestStep); } if (finalIndex > 0) { SelectCandidate( costs[originalIndex, finalIndex - 1] + 1d, 2, DiffStep.Add, ref bestCost, ref bestPriority, ref bestStep); } costs[originalIndex, finalIndex] = bestCost; steps[originalIndex, finalIndex] = bestStep; } } var reverse = new List( originalCount + finalCount); var originalCursor = originalCount; var finalCursor = finalCount; while (originalCursor > 0 || finalCursor > 0) { switch (steps[originalCursor, finalCursor]) { case DiffStep.Match: { var originalFrame = originalBoundaries[originalCursor - 1]; var finalFrame = finalBoundaries[finalCursor - 1]; reverse.Add(new CorrectionRecord { action = originalFrame == finalFrame ? "keep" : "move", hasOriginal = true, originalFrame = originalFrame, originalOrdinal = originalCursor - 1, hasFinal = true, finalFrame = finalFrame, finalOrdinal = finalCursor - 1, deltaFrames = finalFrame - originalFrame }); originalCursor--; finalCursor--; break; } case DiffStep.Delete: reverse.Add(new CorrectionRecord { action = "delete", hasOriginal = true, originalFrame = originalBoundaries[originalCursor - 1], originalOrdinal = originalCursor - 1, hasFinal = false, finalFrame = -1, finalOrdinal = -1, deltaFrames = 0 }); originalCursor--; break; case DiffStep.Add: reverse.Add(new CorrectionRecord { action = "add", hasOriginal = false, originalFrame = -1, originalOrdinal = -1, hasFinal = true, finalFrame = finalBoundaries[finalCursor - 1], finalOrdinal = finalCursor - 1, deltaFrames = 0 }); finalCursor--; break; default: throw new InvalidOperationException( "Unable to construct a deterministic cut diff."); } } reverse.Reverse(); return reverse.ToArray(); } private static void SelectCandidate( double candidateCost, int candidatePriority, DiffStep candidateStep, ref double bestCost, ref int bestPriority, ref DiffStep bestStep) { if (double.IsPositiveInfinity(candidateCost)) { return; } if (candidateCost < bestCost - CostEpsilon || (Math.Abs(candidateCost - bestCost) <= CostEpsilon && candidatePriority < bestPriority)) { bestCost = candidateCost; bestPriority = candidatePriority; bestStep = candidateStep; } } private static void ValidateBoundarySequence( int[] boundaries, int frameCount, string label) { var previous = 0; for (var index = 0; index < boundaries.Length; index++) { var frame = boundaries[index]; if (frame <= previous || frame >= frameCount) { throw new InvalidDataException( $"The {label} cut boundary at index {index} is " + "outside the song or not strictly increasing."); } previous = frame; } } private static string AnonymizeEditorStyleId(string editorStyleId) { if (string.IsNullOrWhiteSpace(editorStyleId)) { throw new ArgumentException( "An editor style ID is required. Use a stable nickname " + "that does not contain personal information.", nameof(editorStyleId)); } var normalized = editorStyleId.Trim(); if (normalized.StartsWith( EditorStyleHashPrefix, StringComparison.OrdinalIgnoreCase)) { var existingHash = NormalizeSha256( normalized.Substring(EditorStyleHashPrefix.Length), "editor style ID"); return EditorStyleHashPrefix + existingHash; } return EditorStyleHashPrefix + ComputeSha256( Encoding.UTF8.GetBytes(normalized)); } private static string ComputeBoundarySha256(int[] boundaries) { var canonical = string.Join( ",", boundaries.Select(frame => frame.ToString(CultureInfo.InvariantCulture))); return ComputeSha256(Encoding.UTF8.GetBytes(canonical)); } private static string ComputeSha256(byte[] bytes) { using var algorithm = SHA256.Create(); var hash = algorithm.ComputeHash(bytes); var builder = new StringBuilder(hash.Length * 2); foreach (var value in hash) { builder.Append(value.ToString("x2", CultureInfo.InvariantCulture)); } return builder.ToString(); } private static string NormalizeSha256(string value, string label) { var normalized = value?.Trim().ToLowerInvariant(); if (normalized == null || normalized.Length != 64 || normalized.Any(character => !(character >= '0' && character <= '9') && !(character >= 'a' && character <= 'f'))) { throw new InvalidDataException( $"{label} must be a 64-character SHA-256 value."); } return normalized; } private static string ResolveSafePayloadPath( string rootDirectory, string relativePath, string label) { if (string.IsNullOrWhiteSpace(relativePath) || Path.IsPathRooted(relativePath)) { throw new InvalidDataException( $"metadata.{label} must be a relative path."); } var segments = relativePath.Split( new[] { '/', '\\' }, StringSplitOptions.None); if (segments.Any(segment => string.IsNullOrWhiteSpace(segment) || segment == "." || segment == "..")) { throw new InvalidDataException( $"metadata.{label} contains an unsafe path segment."); } var root = Path.GetFullPath(rootDirectory); var candidate = Path.GetFullPath( Path.Combine(root, relativePath)); var rootPrefix = root.TrimEnd( Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; if (!candidate.StartsWith( rootPrefix, StringComparison.OrdinalIgnoreCase)) { throw new InvalidDataException( $"metadata.{label} escapes the generated directory."); } return candidate; } private static string RequireDirectoryPath( string value, string parameterName) { if (string.IsNullOrWhiteSpace(value)) { throw new ArgumentException( "A directory path is required.", parameterName); } var path = Path.GetFullPath(value); if (!Directory.Exists(path)) { throw new DirectoryNotFoundException( $"Directory was not found: {path}"); } return path; } private static string WriteSnapshotAtomically( string outputDirectory, CorrectionSnapshot snapshot, string json) { if (string.IsNullOrWhiteSpace(outputDirectory)) { throw new ArgumentException( "A correction log output directory is required.", nameof(outputDirectory)); } var outputRoot = Path.GetFullPath(outputDirectory); Directory.CreateDirectory(outputRoot); var songToken = SanitizeFileToken(snapshot.songId); var timestampToken = DateTime.Parse( snapshot.capturedUtc, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal) .ToString( "yyyyMMdd'T'HHmmssfff'Z'", CultureInfo.InvariantCulture); var baseName = $"cut_correction_{songToken}_{timestampToken}_" + $"{snapshot.generationHashSha256.Substring(0, 8)}_" + $"{snapshot.finalBoundarySha256.Substring(0, 8)}"; var outputPath = Path.Combine(outputRoot, baseName + ".json"); for (var suffix = 2; File.Exists(outputPath); suffix++) { outputPath = Path.Combine( outputRoot, $"{baseName}_{suffix:D2}.json"); } var temporaryPath = outputPath + $".tmp-{Guid.NewGuid():N}"; try { File.WriteAllText( temporaryPath, json, new UTF8Encoding(false)); File.Move(temporaryPath, outputPath); } finally { if (File.Exists(temporaryPath)) { File.Delete(temporaryPath); } } return outputPath; } private static string SanitizeFileToken(string value) { var builder = new StringBuilder(); foreach (var character in value ?? string.Empty) { builder.Append( char.IsLetterOrDigit(character) || character == '-' || character == '_' ? character : '_'); if (builder.Length >= 48) { break; } } return builder.Length == 0 ? "song" : builder.ToString(); } private enum DiffStep { None, Match, Delete, Add } private sealed class GenerationInput { internal GeneratedMetadata Metadata; internal string MetadataSha256; internal int[] OriginalBoundaryFrames; } [Serializable] private sealed class GeneratedMetadata { public string songName; public string songId; public string datasetId; public int sampleRate; public int frameCount; public string shotsFile; public OutputSha256 outputSha256; } [Serializable] private sealed class GeneratedShotFile { public List shots; } [Serializable] private sealed class GeneratedShot { public int startFrame; public int endFrameExclusive; } [Serializable] private sealed class CorrectionSnapshot { public string schemaVersion; public string capturedUtc; public int sampleRate; public string songName; public string songId; public string datasetId; public string generationHashSha256; public OutputSha256 generationOutputSha256; public string editorStyleId; public string previewTimelineGuid; public string matchingPolicy; public int moveMatchWindowFrames; public int ordinalBase; public int[] originalBoundaryFrames; public int[] finalBoundaryFrames; public string originalBoundarySha256; public string finalBoundarySha256; public int originalBoundaryCount; public int finalBoundaryCount; public int keepCount; public int moveCount; public int addCount; public int deleteCount; public CorrectionRecord[] corrections; } [Serializable] private sealed class OutputSha256 { public string worldCamera; public string time; public string shots; } [Serializable] private sealed class CorrectionRecord { public string action; public bool hasOriginal; public int originalFrame; public int originalOrdinal; public bool hasFinal; public int finalFrame; public int finalOrdinal; public int deltaFrames; } } }