From 7d2ccad070ba7fc013c6b3cd96be0109922e97ef Mon Sep 17 00:00:00 2001 From: unclshura Date: Mon, 11 May 2026 17:41:26 +0100 Subject: [PATCH] Separated CameraController from the main loop --- CameraController.cs | 155 ++++++++++++++++++++++++++++++++++++++++++++ KalmanTracker.cs | 2 +- TrackingSplitter.cs | 153 ++++++++----------------------------------- 3 files changed, 182 insertions(+), 128 deletions(-) create mode 100644 CameraController.cs diff --git a/CameraController.cs b/CameraController.cs new file mode 100644 index 0000000..a8c82fe --- /dev/null +++ b/CameraController.cs @@ -0,0 +1,155 @@ +using System; +using OpenCvSharp; + +namespace splitter; + +public enum TrackState +{ + Tracking, + LostFreeze, + LostDrift +} + +public sealed class CameraController +{ + private readonly int _videoWidth; + private readonly int _videoHeight; + private readonly int _cropWidth; + private readonly int _cropHeight; + private readonly KalmanTracker _kalman; + private readonly int _lostFreezeFrames; + private readonly float _cameraEasing; + + private int _lostFrames; + private Point2f _cameraCenter; + private TrackState _state; + private Point2f _smoothedCenter; + private Rect? _objectBox; + private Point2f? _objectCenter; + private Rect _roi; + + public CameraController( + int videoWidth, + int videoHeight, + int cropWidth, + int cropHeight, + KalmanTracker kalman, + int lostFreezeFrames, + float cameraEasing) + { + _videoWidth = videoWidth; + _videoHeight = videoHeight; + _cropWidth = cropWidth; + _cropHeight = cropHeight; + _kalman = kalman; + _lostFreezeFrames = lostFreezeFrames; + _cameraEasing = cameraEasing; + + _cameraCenter = new Point2f(videoWidth / 2f, videoHeight / 2f); + _state = TrackState.Tracking; + + _kalman.Reset(_cameraCenter); + } + + public int LostFrames => _lostFrames; + public Point2f CameraCenter => _cameraCenter; + public TrackState State => _state; + public Point2f SmoothedCenter => _smoothedCenter; + public Rect? ObjectBox => _objectBox; + public Point2f? ObjectCenter => _objectCenter; + public Rect Roi => _roi; + + public void Update((Rect box, Point2f center)? primary) + { + Rect? objectBox = null; + Point2f? objectCenter = null; + + if (primary.HasValue) + { + objectCenter = primary.Value.center; + objectBox = primary.Value.box; + } + + bool isLost = !objectCenter.HasValue; + + // LOST / REACQUIRE STATE MACHINE + if (isLost) + { + _lostFrames++; + + if (_lostFrames <= _lostFreezeFrames) + { + // LOST_FREEZE: freeze camera + _state = TrackState.LostFreeze; + objectCenter = null; // Kalman predicts but camera won't move + } + else + { + // LOST_DRIFT: drift camera to center + _state = TrackState.LostDrift; + objectCenter = new Point2f(_videoWidth / 2f, _videoHeight / 2f); + } + } + else + { + // Object reacquired + _state = TrackState.Tracking; + _lostFrames = 0; + } + + // KALMAN + CAMERA UPDATE + Point2f smoothedCenter; + + if (_state == TrackState.Tracking) + { + smoothedCenter = _kalman.Update(objectCenter); + + // first, faster internal easing (as in your original code) + float fastEasing = 0.015f; + _cameraCenter = new Point2f( + _cameraCenter.X + (smoothedCenter.X - _cameraCenter.X) * fastEasing, + _cameraCenter.Y + (smoothedCenter.Y - _cameraCenter.Y) * fastEasing); + + // then, external configurable easing + _cameraCenter = new Point2f( + _cameraCenter.X + (smoothedCenter.X - _cameraCenter.X) * _cameraEasing, + _cameraCenter.Y + (smoothedCenter.Y - _cameraCenter.Y) * _cameraEasing); + } + else if (_state == TrackState.LostFreeze) + { + // Freeze camera — do nothing + smoothedCenter = _kalman.LastMeasurement ?? _cameraCenter; + } + else // LOST_DRIFT + { + smoothedCenter = _kalman.Update(objectCenter); + + float driftEasing = 0.01f; + var fallbackCenter = new Point2f(_videoWidth / 2f, _videoHeight / 2f); + + _cameraCenter = new Point2f( + _cameraCenter.X + (fallbackCenter.X - _cameraCenter.X) * driftEasing, + _cameraCenter.Y + (fallbackCenter.Y - _cameraCenter.Y) * driftEasing); + } + + var halfW = _cropWidth / 2f; + var halfH = _cropHeight / 2f; + + smoothedCenter.X = Math.Clamp(smoothedCenter.X, halfW, _videoWidth - halfW); + smoothedCenter.Y = Math.Clamp(smoothedCenter.Y, halfH, _videoHeight - halfH); + + _cameraCenter.X = Math.Clamp(_cameraCenter.X, halfW, _videoWidth - halfW); + _cameraCenter.Y = Math.Clamp(_cameraCenter.Y, halfH, _videoHeight - halfH); + + var x = (int)Math.Round(_cameraCenter.X - halfW); + var y = (int)Math.Round(_cameraCenter.Y - halfH); + + x = Math.Clamp(x, 0, _videoWidth - _cropWidth); + y = Math.Clamp(y, 0, _videoHeight - _cropHeight); + + _roi = new Rect(x, y, _cropWidth, _cropHeight); + _smoothedCenter = smoothedCenter; + _objectBox = objectBox; + _objectCenter = objectCenter; + } +} diff --git a/KalmanTracker.cs b/KalmanTracker.cs index 07f5a79..9e6f04e 100644 --- a/KalmanTracker.cs +++ b/KalmanTracker.cs @@ -1,6 +1,6 @@ namespace splitter; -internal sealed class KalmanTracker +public sealed class KalmanTracker { // State vector: [x, y, vx, vy] private float[] _state = new float[4]; diff --git a/TrackingSplitter.cs b/TrackingSplitter.cs index 2c47bd9..0d91d9f 100644 --- a/TrackingSplitter.cs +++ b/TrackingSplitter.cs @@ -15,13 +15,6 @@ public class TrackingSplitter( private const int LostFreezeFrames = 60; // 2 seconds at 30 FPS private const float CameraEasing = 0.03f; - private enum TrackState - { - Tracking, - LostFreeze, - LostDrift - } - public async Task TrackAndExtract( string srcFileName, string destFileName, @@ -49,38 +42,39 @@ public class TrackingSplitter( Console.WriteLine($"[TrackingSplitter] skip={skip}, duration={duration}, fps={fps}, totalFrames={totalFrames}"); - // encoder size depends on mode var encWidth = debugOverlay ? videoWidth : originalCropWidth; var encHeight = debugOverlay ? videoHeight : originalCropHeight; var ffmpeg = StartFfmpegNvenc( - srcFileName, - destFileName, - encWidth, - encHeight, - fps, - skip, - passthrough); + srcFileName, + destFileName, + encWidth, + encHeight, + fps, + skip, + passthrough); using var stdin = ffmpeg.StandardInput.BaseStream; - // Reusable frame and output mat using var frame = new Mat(); using var outputBgr = new Mat(encHeight, encWidth, MatType.CV_8UC3); - // Reusable raw video buffer - var frameBytes = encWidth * encHeight * 3; + var frameBytes = encWidth * encHeight * 3; var videoBuffer = new byte[frameBytes]; var kalman = new KalmanTracker(); - kalman.Reset(new Point2f(videoWidth / 2f, videoHeight / 2f)); + // initial reset is now done inside CameraController - var lostFrames = 0; - var reacquireCounter = 0; // kept for overlay display + var camera = new CameraController( + videoWidth, + videoHeight, + originalCropWidth, + originalCropHeight, + kalman, + LostFreezeFrames, + CameraEasing); - var cameraCenter = new Point2f(videoWidth / 2f, videoHeight / 2f); - var startTime = DateTime.UtcNow; - var state = TrackState.Tracking; + var startTime = DateTime.UtcNow; for (var i = 0; i < totalFrames; i++) { @@ -93,109 +87,19 @@ public class TrackingSplitter( var objects = detector.DetectAll(frame, videoWidth, videoHeight); var primary = SelectTrackedObject(objects, kalman.LastMeasurement); - if (primary.HasValue) - { - objectCenter = primary.Value.center; - objectBox = primary.Value.box; - } + camera.Update(primary); - bool isLost = !objectCenter.HasValue; + objectBox = camera.ObjectBox; + objectCenter = camera.ObjectCenter; - // LOST / REACQUIRE STATE MACHINE - if (isLost) - { - lostFrames++; - - if (lostFrames <= LostFreezeFrames) - { - // LOST_FREEZE: freeze camera - state = TrackState.LostFreeze; - objectCenter = null; // Kalman predicts but camera won't move - } - else - { - // LOST_DRIFT: drift camera to center - state = TrackState.LostDrift; - objectCenter = new Point2f(videoWidth / 2f, videoHeight / 2f); - } - } - else - { - // Object reacquired - state = TrackState.Tracking; - lostFrames = 0; - } - - // KALMAN + CAMERA UPDATE - Point2f smoothedCenter; - - if (state == TrackState.Tracking) - { - smoothedCenter = kalman.Update(objectCenter); - - float easing = 0.015f; // faster tracking - cameraCenter = new Point2f( - cameraCenter.X + (smoothedCenter.X - cameraCenter.X) * easing, - cameraCenter.Y + (smoothedCenter.Y - cameraCenter.Y) * easing); - } - else if (state == TrackState.LostFreeze) - { - // Freeze camera — do nothing - smoothedCenter = kalman.LastMeasurement ?? new Point2f(0, 0); - } - else // LOST_DRIFT - { - smoothedCenter = kalman.Update(objectCenter); - - float driftEasing = 0.01f; - var fallbackCenter = new Point2f(videoWidth / 2f, videoHeight / 2f); - - cameraCenter = new Point2f( - cameraCenter.X + (fallbackCenter.X - cameraCenter.X) * driftEasing, - cameraCenter.Y + (fallbackCenter.Y - cameraCenter.Y) * driftEasing); - } - - var halfW = originalCropWidth / 2f; - var halfH = originalCropHeight / 2f; - - smoothedCenter.X = Math.Clamp(smoothedCenter.X, halfW, videoWidth - halfW); - smoothedCenter.Y = Math.Clamp(smoothedCenter.Y, halfH, videoHeight - halfH); - - if (state == TrackState.Tracking) - { - smoothedCenter = kalman.Update(objectCenter); - - cameraCenter = new Point2f( - cameraCenter.X + (smoothedCenter.X - cameraCenter.X) * CameraEasing, - cameraCenter.Y + (smoothedCenter.Y - cameraCenter.Y) * CameraEasing); - } - else if (state == TrackState.LostFreeze) - { - // Freeze camera — do nothing - } - else if (state == TrackState.LostDrift) - { - var fallbackCenter = new Point2f(videoWidth / 2f, videoHeight / 2f); - - cameraCenter = new Point2f( - cameraCenter.X + (fallbackCenter.X - cameraCenter.X) * 0.01f, - cameraCenter.Y + (fallbackCenter.Y - cameraCenter.Y) * 0.01f); - } - - cameraCenter.X = Math.Clamp(cameraCenter.X, halfW, videoWidth - halfW); - cameraCenter.Y = Math.Clamp(cameraCenter.Y, halfH, videoHeight - halfH); - - var x = (int)Math.Round(cameraCenter.X - halfW); - var y = (int)Math.Round(cameraCenter.Y - halfH); - - x = Math.Clamp(x, 0, videoWidth - originalCropWidth); - y = Math.Clamp(y, 0, videoHeight - originalCropHeight); - - var roi = new Rect(x, y, originalCropWidth, originalCropHeight); + var smoothedCenter = camera.SmoothedCenter; + var cameraCenter = camera.CameraCenter; + var state = camera.State; + var lostFrames = camera.LostFrames; + var roi = camera.Roi; if (debugOverlay) { - // overlays always drawn on frame if (objectBox.HasValue) { var fb = objectBox.Value; @@ -213,23 +117,18 @@ public class TrackingSplitter( DrawText(frame, $"Faces: {objects.Count}", 20, 40, Scalar.White); DrawText(frame, $"LostFrames: {lostFrames}", 20, 70, Scalar.White); - DrawText(frame, $"Reacquire: {reacquireCounter}", 20, 100, Scalar.White); DrawText(frame, $"Noise: {kalman.CurrentNoise:F3}", 20, 130, Scalar.White); DrawText(frame, $"Camera: {cameraCenter.X:F1},{cameraCenter.Y:F1}", 20, 160, Scalar.White); } if (debugOverlay) { - // DEBUG MODE: write FULL FRAME with overlays - // Ensure contiguous buffer by copying into preallocated outputBgr frame.CopyTo(outputBgr); - Marshal.Copy(outputBgr.Data, videoBuffer, 0, frameBytes); stdin.Write(videoBuffer, 0, frameBytes); } else { - // PRODUCTION MODE: actual crop using var cropped = new Mat(frame, roi); cropped.CopyTo(outputBgr);