using System; using System.Diagnostics; using System.Globalization; using System.Runtime.InteropServices; using System.Threading.Tasks; using OpenCvSharp; namespace splitter; public class TrackingSplitter : LoggingBase, ISegmentProcessor, IDisposable { private readonly int _segmentNo; private readonly int _cropWidth; private readonly int _cropHeight; private readonly bool _debugOverlay; private readonly bool _plainText; private readonly IObjectDetector _detector; private readonly CommandLine _cmd; public TrackingSplitter(int segmentNo, int cropWidth, int cropHeight, bool debugOverlay, bool plainText, IObjectDetector detector, CommandLine cmd) : base(segmentNo) { _segmentNo = segmentNo; _cropWidth = cropWidth; _cropHeight = cropHeight; _debugOverlay = debugOverlay; _plainText = plainText; _detector = detector; _cmd = cmd; } public void Dispose() { if (_detector is IDisposable d) d.Dispose(); } public async Task ProcessSegment(string inputFile, string outputFile, double start, double length, string[] ffmpegPassthroughParameters) { using var capture = new VideoCapture(inputFile); if (!capture.IsOpened()) throw new Exception("Cannot open video"); var name = Path.GetFileNameWithoutExtension(outputFile); var skip = TimeSpan.FromSeconds(start); var duration = TimeSpan.FromSeconds(length); capture.Set(VideoCaptureProperties.PosMsec, start); var videoWidth = (int)capture.Get(VideoCaptureProperties.FrameWidth); var videoHeight = (int)capture.Get(VideoCaptureProperties.FrameHeight); var fps = capture.Get(VideoCaptureProperties.Fps); var totalFrames = (int)(length * fps); var originalCropWidth = _cropWidth; var originalCropHeight = _cropHeight; Console.WriteLine($"[TrackingSplitter] skip={skip}, duration={duration}, fps={fps}, totalFrames={totalFrames}"); var encWidth = _debugOverlay ? videoWidth : originalCropWidth; var encHeight = _debugOverlay ? videoHeight : originalCropHeight; var ffmpeg = StartFfmpegNvenc( inputFile, outputFile, encWidth, encHeight, fps, skip, ffmpegPassthroughParameters); using var stdin = ffmpeg.StandardInput.BaseStream; using var frame = new Mat(); using var outputBgr = new Mat(encHeight, encWidth, MatType.CV_8UC3); var frameBytes = encWidth * encHeight * 3; var videoBuffer = new byte[frameBytes]; var kalman = new KalmanTracker(); // initial reset is now done inside CameraController var camera = new CameraController( videoWidth, videoHeight, originalCropWidth, originalCropHeight, kalman, _cmd ); var startTime = DateTime.UtcNow; for (var i = 0; i < totalFrames; i++) { if (!capture.Read(frame) || frame.Empty()) break; Rect? objectBox = null; Point2f? objectCenter = null; var objects = _detector.DetectAll(frame, videoWidth, videoHeight); var primary = SelectTrackedObject(objects, kalman.LastMeasurement); camera.Update(primary); objectBox = camera.ObjectBox; objectCenter = camera.ObjectCenter; var smoothedCenter = camera.SmoothedCenter; var cameraCenter = camera.CameraCenter; var state = camera.State; var lostFrames = camera.LostFrames; var roi = camera.Roi; if (_debugOverlay) { if (objectBox.HasValue) { var fb = objectBox.Value; Cv2.Rectangle(frame, new Rect(fb.X, fb.Y, fb.Width, fb.Height), Scalar.LimeGreen, 2); } Cv2.Circle(frame, new Point((int)smoothedCenter.X, (int)smoothedCenter.Y), 6, Scalar.LimeGreen, -1); Cv2.Rectangle(frame, roi, objectCenter.HasValue ? Scalar.Yellow : Scalar.Red, 3); DrawText(frame, $"Faces: {objects.Count}", 20, 40, Scalar.White); DrawText(frame, $"LostFrames: {lostFrames}", 20, 70, Scalar.White); DrawText(frame, $"Noise: {kalman.CurrentNoise:F3}", 20, 130, Scalar.White); DrawText(frame, $"Camera: {cameraCenter.X:F1},{cameraCenter.Y:F1}", 20, 160, Scalar.White); } if (_debugOverlay) { frame.CopyTo(outputBgr); Marshal.Copy(outputBgr.Data, videoBuffer, 0, frameBytes); stdin.Write(videoBuffer, 0, frameBytes); } else { using var cropped = new Mat(frame, roi); cropped.CopyTo(outputBgr); Marshal.Copy(outputBgr.Data, videoBuffer, 0, frameBytes); stdin.Write(videoBuffer, 0, frameBytes); } var elapsed = DateTime.UtcNow - startTime; var progress = (double)i / totalFrames; var speed = i > 0 ? (i / elapsed.TotalSeconds)/fps : 0.0; var remainingFrames = totalFrames - i; var etaSeconds = speed > 0 ? remainingFrames / speed : 0; var eta = TimeSpan.FromSeconds(etaSeconds); DrawProgress(name, progress, eta, speed); } stdin.Flush(); stdin.Close(); await ffmpeg.WaitForExitAsync(); ClearProgress(); if (ffmpeg.ExitCode != 0) LogError($"Segment {name} FFmpeg encoding failed"); else LogInfo($"Segment {name} processing completed"); } private (Rect box, Point2f center)? SelectTrackedObject( List<(Rect box, Point2f center)> foundObjects, Point2f? previousCenter) { if (foundObjects == null || foundObjects.Count == 0) return null; if (!previousCenter.HasValue) { // Largest area var bestIndex = 0; var bestArea = float.MinValue; for (int i = 0; i < foundObjects.Count; i++) { var f = foundObjects[i]; var area = f.box.Width * f.box.Height; if (area > bestArea) { bestArea = area; bestIndex = i; } } return foundObjects[bestIndex]; } else { // Closest to previous center var prev = previousCenter.Value; var bestIndex = 0; var bestDist2 = float.MaxValue; for (int i = 0; i < foundObjects.Count; i++) { var f = foundObjects[i]; var dx = f.center.X - prev.X; var dy = f.center.Y - prev.Y; var d2 = dx * dx + dy * dy; if (d2 < bestDist2) { bestDist2 = d2; bestIndex = i; } } return foundObjects[bestIndex]; } } private Process StartFfmpegNvenc( string srcFileName, string destFileName, int width, int height, double fps, TimeSpan skip, string[] passthrough) { var pass = passthrough.Length > 0 ? string.Join(" ", passthrough) : ""; var skipSeconds = skip.TotalSeconds.ToString("0.###", CultureInfo.InvariantCulture); var fpsStr = fps.ToString("0.###", CultureInfo.InvariantCulture); var args = "-y " + $"-f rawvideo -pix_fmt bgr24 -s {width}x{height} -r {fpsStr} -i - " + $"-ss {skipSeconds} -i \"{srcFileName}\" " + "-map 0:v:0 -map 1:a:0? -shortest " + "-c:v h264_nvenc -preset p4 -b:v 8M -pix_fmt yuv420p " + "-c:a aac -b:a 192k " + pass + $" \"{destFileName}\""; var psi = new ProcessStartInfo { FileName = "ffmpeg", Arguments = args, RedirectStandardInput = true, RedirectStandardError = true, RedirectStandardOutput = true, UseShellExecute = false, CreateNoWindow = true }; var process = new Process { StartInfo = psi }; process.Start(); _ = Task.Run(() => { try { string? line; while ((line = process.StandardError.ReadLine()) != null) { if (_plainText) Console.WriteLine($"[ffmpeg] {line}"); } } catch { } }); return process; } private static void DrawText(Mat img, string text, int x, int y, Scalar color) { Cv2.PutText(img, text, new Point(x, y), HersheyFonts.HersheySimplex, 0.6, color, 2); } }