Preview interface added, but not used yet

This commit is contained in:
Alexander Shabarshov 2026-06-21 11:15:27 +01:00
parent 2058ae0f7e
commit fec13d0d07
14 changed files with 584 additions and 367 deletions

View File

@ -8,8 +8,6 @@ using CommunityToolkit.Mvvm.Input;
namespace Splitter_UI.ViewModels; namespace Splitter_UI.ViewModels;
public record Segment(double Start, double End);
public partial class JobViewModel : ObservableObject public partial class JobViewModel : ObservableObject
{ {
private SingleJob Job { get; } private SingleJob Job { get; }

View File

@ -68,7 +68,7 @@ public partial class MainViewModel : ViewModelBase
foreach (var file in files) foreach (var file in files)
{ {
var fileJobs = await _processor.GenerateJobs(file.GetJob(), false, _cancellationTokenSource.Token); var fileJobs = await _processor.GenerateJobs(file.GetJob(), false, file.Segments, _cancellationTokenSource.Token);
jobs.AddRange(fileJobs); jobs.AddRange(fileJobs);
} }

View File

@ -36,20 +36,7 @@
VerticalAlignment="Center" VerticalAlignment="Center"
HorizontalAlignment="Center" /> HorizontalAlignment="Center" />
</Button> </Button>
<!--
<Slider Grid.Column="1"
Minimum="0"
Maximum="{Binding Selected.DurationSeconds}"
Value="{Binding Selected.SliderLiveValue, Mode=TwoWay}"
Margin="5,0,5,0" />
<controls:PreviewSlider Grid.Column="1"
Minimum="0"
Maximum="{Binding Selected.DurationSeconds}"
Value="{Binding Selected.SliderLiveValue, Mode=TwoWay}"
SegmentDuration="{Binding Selected.SegmentDuration}"
Margin="5,0,5,0" />
-->
<controls:TimelinePreviewSlider Grid.Column="1" <controls:TimelinePreviewSlider Grid.Column="1"
ViewModel="{Binding Selected}" ViewModel="{Binding Selected}"
Margin="5,0,5,0" /> Margin="5,0,5,0" />

View File

@ -0,0 +1,39 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace splitter;
public static class DebugOverlay
{
public static void DrawDebug(
Mat frame,
List<DetectedPerson> objects,
CameraController camera,
KalmanTracker kalman)
{
if (camera.ObjectBox.HasValue)
{
var fb = camera.ObjectBox.Value;
Cv2.Rectangle(frame, fb, Scalar.LimeGreen, 2);
}
Cv2.Circle(frame,
new Point((int)camera.SmoothedCenter.X, (int)camera.SmoothedCenter.Y),
6, Scalar.LimeGreen, -1);
Cv2.Rectangle(frame, camera.Roi,
camera.ObjectCenter.HasValue ? Scalar.Yellow : Scalar.Red, 3);
DrawText(frame, $"Faces: {objects.Count}", 20, 40, Scalar.White);
DrawText(frame, $"LostFrames: {camera.LostFrames}", 20, 70, Scalar.White);
DrawText(frame, $"Noise: {kalman.CurrentNoise:F3}", 20, 130, Scalar.White);
DrawText(frame, $"Camera: {camera.CameraCenter.X:F1},{camera.CameraCenter.Y:F1}", 20, 160, Scalar.White);
}
public static void DrawText(Mat img, string text, int x, int y, Scalar color)
{
Cv2.PutText(img, text, new Point(x, y),
HersheyFonts.HersheySimplex, 0.6, color, 2);
}
}

View File

@ -2,6 +2,6 @@
public interface IJobProcessor public interface IJobProcessor
{ {
Task<List<SingleTask>> GenerateJobs(SingleJob job, bool estimateOnly, CancellationToken token); Task<List<SingleTask>> GenerateJobs(SingleJob job, bool estimateOnly, IReadOnlyCollection<Segment> predefinedSegments, CancellationToken token);
Task<bool> ProcessJobs(List<SingleTask> tasks, bool singleThreaded, CancellationToken token); Task<bool> ProcessJobs(List<SingleTask> tasks, bool singleThreaded, CancellationToken token);
} }

View File

@ -5,7 +5,7 @@ namespace splitter;
public class JobProcessor(ILogger logger) : LoggingBase(logger, 0), IJobProcessor public class JobProcessor(ILogger logger) : LoggingBase(logger, 0), IJobProcessor
{ {
public async Task<List<SingleTask>> GenerateJobs(SingleJob job, bool estimateOnly, CancellationToken token) public async Task<List<SingleTask>> GenerateJobs(SingleJob job, bool estimateOnly, IReadOnlyCollection<Segment> predefinedSegments, CancellationToken token)
{ {
var baseName = Path.GetFileNameWithoutExtension(job.InputFile); var baseName = Path.GetFileNameWithoutExtension(job.InputFile);
@ -78,24 +78,31 @@ public class JobProcessor(ILogger logger) : LoggingBase(logger, 0), IJobProcesso
processorFactory = i => new SimpleSplitter(i, _logger); processorFactory = i => new SimpleSplitter(i, _logger);
} }
var jobs = Enumerable.Range(0, segments) var segmentsToUse = predefinedSegments;
.Select(i => new SingleTask
if (predefinedSegments.Count == 0)
{
segmentsToUse = Enumerable.Range(0, segments).Select(i => new Segment
(
Start: i * segmentLength,
End : (i == segments - 1)
? Math.Max(0.1, info.Duration)
: (i + 1) * segmentLength
)).ToList();
}
return segmentsToUse.Select((s, i) => new SingleTask
( (
Job : job, Job : job,
Info: info, Info : info,
OutputFileName : BuildOutputFileName(job, i), OutputFileName : BuildOutputFileName(job, i),
SegmentIndex : i, SegmentIndex : i,
TotalSegments : segments, TotalSegments : predefinedSegments.Count,
SegmentStart : i * segmentLength, SegmentStart : s.Start,
SegmentLength : (i == segments - 1) SegmentLength : s.End - s.Start,
? Math.Max(0.1, info.Duration - i * segmentLength) ProcessorFactory: processorFactory
: segmentLength,
ProcessorFactory : processorFactory
) )
) ).ToList();
.ToList();
return jobs;
} }
public async Task<bool> ProcessJobs(List<SingleTask> tasks, bool singleThreaded, CancellationToken token) public async Task<bool> ProcessJobs(List<SingleTask> tasks, bool singleThreaded, CancellationToken token)

View File

@ -3,9 +3,189 @@ using System.Globalization;
namespace splitter; namespace splitter;
public class SimpleSplitter(int segmentNo, ILogger logger) : LoggingBase(logger, segmentNo), ISegmentProcessor public sealed class SimpleSplitter : LoggingBase, ISegmentProcessor
{ {
// ------------------------------------------------------------
// Internal state (opaque to caller)
// ------------------------------------------------------------
private sealed class State : IFrameProcessingState
{
public Process? DecodeProcess { get; set; }
public Stream? DecodeStdout { get; set; }
public string InputFile { get; }
public double Start { get; }
public double Length { get; }
public int? Rotate { get; }
public string[] Passthrough { get; }
public VideoInfo Info { get; }
public bool PlainText { get; }
public State(SingleTask job)
{
InputFile = job.Job.InputFile;
Start = job.SegmentStart;
Length = job.SegmentLength;
Rotate = job.Job.Rotate;
Passthrough = job.Job.Passthrough;
Info = job.Info;
PlainText = job.Job.PlainText;
}
}
public SimpleSplitter(int segmentNo, ILogger logger)
: base(logger, segmentNo)
{
}
// ============================================================
// InitSegment
// ============================================================
public IFrameProcessingState InitSegment(SingleTask job, CancellationToken token)
{
var state = new State(job);
var decode = StartDecode(job, token);
state.DecodeProcess = decode;
state.DecodeStdout = decode.StandardOutput.BaseStream;
return state;
}
// ============================================================
// GetNextProcessedFrame
// ============================================================
public Mat? GetNextProcessedFrame(IFrameProcessingState processorState, CancellationToken token)
{
var state = (State)processorState;
if (state.DecodeStdout == null)
return null;
// SimpleSplitter does not modify frames; it only copies or rotates.
// For preview, we decode raw frames and return them as-is.
// Determine expected frame size
var w = state.Info.Width;
var h = state.Info.Height;
var bytes = w * h * 3;
var buffer = new byte[bytes];
var read = state.DecodeStdout.Read(buffer, 0, bytes);
if (read != bytes)
return null;
var mat = new Mat(h, w, MatType.CV_8UC3);
System.Runtime.InteropServices.Marshal.Copy(buffer, 0, mat.Data, bytes);
return mat;
}
// ============================================================
// FinishSegment
// ============================================================
public void FinishSegment(IFrameProcessingState processorState)
{
var state = (State)processorState;
try
{
if (state.DecodeProcess != null && !state.DecodeProcess.HasExited)
state.DecodeProcess.Kill(entireProcessTree: true);
}
catch { }
try
{
if (state.DecodeProcess != null && !state.DecodeProcess.HasExited)
state.DecodeProcess.WaitForExit();
}
catch { }
}
// ============================================================
// ProcessSegment (now uses preview API)
// ============================================================
public async Task ProcessSegment(SingleTask job, CancellationToken token) public async Task ProcessSegment(SingleTask job, CancellationToken token)
{
var state = (State)InitSegment(job, token);
var encode = StartEncode(job);
using var encodeStdin = encode.StandardInput.BaseStream;
var name = Path.GetFileNameWithoutExtension(job.OutputFileName);
var sw = Stopwatch.StartNew();
while (true)
{
token.ThrowIfCancellationRequested();
var frame = GetNextProcessedFrame(state, token);
if (frame == null)
break;
// Write raw frame to encoder
var bytes = frame.Width * frame.Height * 3;
var buffer = new byte[bytes];
System.Runtime.InteropServices.Marshal.Copy(frame.Data, buffer, 0, bytes);
encodeStdin.Write(buffer, 0, bytes);
frame.Dispose();
}
encodeStdin.Flush();
encodeStdin.Close();
await encode.WaitForExitAsync(token);
FinishSegment(state);
ClearProgress(name);
if (encode.ExitCode != 0)
LogError($"Segment {name} FFmpeg encoding failed");
else
LogInfo($"Segment {name} processing completed");
}
// ============================================================
// FFmpeg helpers
// ============================================================
private Process StartDecode(SingleTask job, CancellationToken token)
{
var ss = job.SegmentStart.ToString("0.###", CultureInfo.InvariantCulture);
var t = job.SegmentLength.ToString("0.###", CultureInfo.InvariantCulture);
var rotate = GetRotationFilter(job.Job.Rotate);
var vf = rotate != null ? $"-vf format=bgr24,{rotate}" : "-vf format=bgr24";
var args =
$"-i \"{job.Job.InputFile}\" -ss {ss} -t {t} " +
"-an -sn " +
$"{vf} " +
"-f rawvideo -";
var psi = new ProcessStartInfo
{
FileName = "ffmpeg",
Arguments = args,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
var p = Process.Start(psi) ?? throw new Exception("Failed to start ffmpeg decode.");
return p;
}
private Process StartEncode(SingleTask job)
{ {
var inputFile = job.Job.InputFile; var inputFile = job.Job.InputFile;
var outputFile = job.OutputFileName; var outputFile = job.OutputFileName;
@ -18,7 +198,6 @@ public class SimpleSplitter(int segmentNo, ILogger logger) : LoggingBase(logger,
if (rotation == null) if (rotation == null)
{ {
// Copy path: keep original SAR/DAR exactly as in source
args = args =
$"-ss {start.ToString(CultureInfo.InvariantCulture)} " + $"-ss {start.ToString(CultureInfo.InvariantCulture)} " +
$"-i \"{inputFile}\" " + $"-i \"{inputFile}\" " +
@ -31,33 +210,27 @@ public class SimpleSplitter(int segmentNo, ILogger logger) : LoggingBase(logger,
var sarArg = ""; var sarArg = "";
var darArg = ""; var darArg = "";
var sar = job.Info.SampleAspectRatio; // e.g. "4:3" var sar = job.Info.SampleAspectRatio;
if (sar != null) if (sar != null)
{ {
// Rotation path: must re-encode and recompute DAR
var sarNum = Convert.ToInt64(job.Info.Sar.X); var sarNum = Convert.ToInt64(job.Info.Sar.X);
var sarDen = Convert.ToInt64(job.Info.Sar.Y); var sarDen = Convert.ToInt64(job.Info.Sar.Y);
// After rotation, width/height swap
var w = job.Info.Width; var w = job.Info.Width;
var h = job.Info.Height; var h = job.Info.Height;
if (job.Job.Rotate == 90 || job.Job.Rotate == 270) if (job.Job.Rotate == 90 || job.Job.Rotate == 270)
{
(w, h) = (h, w); (w, h) = (h, w);
}
// Compute DAR = (w * sarNum) : (h * sarDen)
var darNum = w * sarNum; var darNum = w * sarNum;
var darDen = h * sarDen; var darDen = h * sarDen;
// Reduce fraction
long Gcd(long a, long b) long Gcd(long a, long b)
{ {
while (b != 0) (a, b) = (b, a % b); while (b != 0) (a, b) = (b, a % b);
return a; return a;
} }
var g = Gcd(darNum, darDen); var g = Gcd(darNum, darDen);
darNum /= g; darNum /= g;
darDen /= g; darDen /= g;
@ -78,32 +251,21 @@ public class SimpleSplitter(int segmentNo, ILogger logger) : LoggingBase(logger,
$"{string.Join(" ", job.Job.Passthrough)} " + $"{string.Join(" ", job.Job.Passthrough)} " +
$"\"{outputFile}\" -y"; $"\"{outputFile}\" -y";
} }
var psi = new ProcessStartInfo var psi = new ProcessStartInfo
{ {
FileName = "ffmpeg", FileName = "ffmpeg",
Arguments = args, Arguments = args,
RedirectStandardInput = true,
RedirectStandardError = true, RedirectStandardError = true,
UseShellExecute = false, UseShellExecute = false,
CreateNoWindow = true CreateNoWindow = true
}; };
using var proc = Process.Start(psi) ?? throw new Exception("Failed to start ffmpeg."); return Process.Start(psi) ?? throw new Exception("Failed to start ffmpeg encode.");
var name = Path.GetFileNameWithoutExtension(outputFile);
await ShowFFMpegProgress(length, proc, name, token);
await proc.WaitForExitAsync(token);
ClearProgress(name);
if (proc.ExitCode != 0)
LogError($"Segment {name} FFmpeg encoding failed");
else
LogInfo($"Segment {name} processing completed");
} }
private string? GetRotationFilter(int? degrees) =>
string? GetRotationFilter(int? degrees) =>
degrees switch degrees switch
{ {
90 => "transpose=1", 90 => "transpose=1",
@ -111,80 +273,4 @@ public class SimpleSplitter(int segmentNo, ILogger logger) : LoggingBase(logger,
270 => "transpose=2", 270 => "transpose=2",
_ => null _ => null
}; };
private static long Gcd(long a, long b)
{
a = Math.Abs(a);
b = Math.Abs(b);
while (b != 0)
{
var t = b;
b = a % b;
a = t;
}
return a;
}
private async Task ShowFFMpegProgress(double length, Process proc, string name, CancellationToken token)
{
var sw = Stopwatch.StartNew();
string? line;
while ((line = await proc.StandardError.ReadLineAsync(token)) != null)
{
// Look for "time=00:00:03.52"
var idx = line.IndexOf("time=", StringComparison.OrdinalIgnoreCase);
if (idx < 0)
continue;
var timeStr = ExtractTimestamp(line, idx + 5);
if (timeStr == null)
continue;
if (!TryParseFfmpegTime(timeStr, out var current))
continue;
var progress = current.TotalSeconds / length;
if (progress < 0) progress = 0;
if (progress > 1) progress = 1;
var elapsed = sw.Elapsed;
var speed = current.TotalSeconds > 0
? current.TotalSeconds / elapsed.TotalSeconds
: 0;
var remaining = length - current.TotalSeconds;
var etaSeconds = speed > 0 ? remaining / speed : remaining;
var eta = TimeSpan.FromSeconds(etaSeconds);
DrawProgress(name, progress, eta, speed);
}
}
private static string? ExtractTimestamp(string line, int startIndex)
{
// FFmpeg formats: HH:MM:SS.xx
// We read until whitespace
var end = startIndex;
while (end < line.Length && !char.IsWhiteSpace(line[end]))
end++;
if (end <= startIndex)
return null;
return line[startIndex..end];
}
private static bool TryParseFfmpegTime(string s, out TimeSpan ts)
{
// FFmpeg uses "00:00:03.52"
return TimeSpan.TryParseExact(
s,
@"hh\:mm\:ss\.ff",
CultureInfo.InvariantCulture,
out ts);
}
} }

View File

@ -2,6 +2,8 @@
namespace splitter; namespace splitter;
public record Segment(double Start, double End);
public class SingleJob public class SingleJob
{ {
/// <summary> /// <summary>

View File

@ -4,10 +4,58 @@ using System.Runtime.InteropServices;
namespace splitter; namespace splitter;
public class TrackingSplitter : LoggingBase, ISegmentProcessor public sealed class TrackingSplitter : LoggingBase, ISegmentProcessor
{ {
private readonly IObjectTracker _tracker; private readonly IObjectTracker _tracker;
// ------------------------------------------------------------
// Internal state (never exposed)
// ------------------------------------------------------------
private sealed class FrameProcessingState : IFrameProcessingState
{
public SingleTask Job { get; }
public KalmanTracker Kalman { get; }
public CameraController Camera { get; }
public Mat FrameMat { get; }
public Mat OutMat { get; }
public byte[] InBuffer { get; }
public byte[] OutBuffer { get; }
public IVideoEnhancer? Enhancer { get; }
public int InBytes { get; }
public int OutBytes { get; }
public Process? DecodeProcess { get; set; }
public Stream? DecodeStdout { get; set; }
public FrameProcessingState(
SingleTask job,
KalmanTracker kalman,
CameraController camera,
Mat frameMat,
Mat outMat,
byte[] inBuffer,
byte[] outBuffer,
IVideoEnhancer? enhancer,
int inBytes,
int outBytes)
{
Job = job;
Kalman = kalman;
Camera = camera;
FrameMat = frameMat;
OutMat = outMat;
InBuffer = inBuffer;
OutBuffer = outBuffer;
Enhancer = enhancer;
InBytes = inBytes;
OutBytes = outBytes;
}
}
public TrackingSplitter( public TrackingSplitter(
int progressLine, int progressLine,
IObjectTracker tracker, IObjectTracker tracker,
@ -18,144 +66,131 @@ public class TrackingSplitter : LoggingBase, ISegmentProcessor
_tracker = tracker; _tracker = tracker;
} }
// ============================================================
// PUBLIC PREVIEW API
// ============================================================
// ------------------------------------------------------------
// InitSegment
// ------------------------------------------------------------
public IFrameProcessingState InitSegment(SingleTask job, CancellationToken token)
{
var state = (FrameProcessingState)CreateFrameState(job);
if (state.Enhancer != null)
state.Enhancer.InitializeAsync(
state.OutMat.Width,
state.OutMat.Height,
5,
token).Wait(token);
var decode = StartFfmpegDecode(
job.Job.InputFile,
job.SegmentStart,
job.SegmentLength,
job.Job.Rotate,
job.Job.PlainText,
token).Result;
state.DecodeProcess = decode;
state.DecodeStdout = decode.StandardOutput.BaseStream;
return state;
}
// ------------------------------------------------------------
// GetNextProcessedFrame
// ------------------------------------------------------------
public Mat? GetNextProcessedFrame(
IFrameProcessingState processorState,
CancellationToken token)
{
var state = (FrameProcessingState)processorState;
if (state.DecodeStdout == null)
return null;
if (!TryReadNextFrame(state.DecodeStdout, state, token))
return null;
return ProcessFrame(state.FrameMat, state, state.Job, token);
}
// ------------------------------------------------------------
// FinishSegment
// ------------------------------------------------------------
public void FinishSegment(IFrameProcessingState processorState)
{
var state = (FrameProcessingState)processorState;
try
{
if (state.DecodeProcess != null && !state.DecodeProcess.HasExited)
state.DecodeProcess.Kill(entireProcessTree: true);
}
catch { }
try
{
if (state.DecodeProcess != null && !state.DecodeProcess.HasExited)
state.DecodeProcess.WaitForExit();
}
catch { }
if (state.Enhancer is IAsyncDisposable ad)
ad.DisposeAsync().AsTask().Wait();
else if (state.Enhancer is IDisposable d)
d.Dispose();
}
// ============================================================
// PROCESSSEGMENT (full pipeline)
// ============================================================
public async Task ProcessSegment(SingleTask job, CancellationToken token) public async Task ProcessSegment(SingleTask job, CancellationToken token)
{ {
var inputFile = job.Job.InputFile; var name = Path.GetFileNameWithoutExtension(job.OutputFileName);
var outputFile = job.OutputFileName;
var start = job.SegmentStart;
var length = job.SegmentLength;
var videoWidth = job.Info.Width;
var videoHeight = job.Info.Height;
var fps = job.Info.Fps; var fps = job.Info.Fps;
var bitrate = job.Info.Bitrate;
var ffmpegPassthroughParameters = job.Job.Passthrough;
var name = Path.GetFileNameWithoutExtension(outputFile); var state = (FrameProcessingState)InitSegment(job, token);
if (videoWidth <= 0 || videoHeight <= 0 || fps <= 0)
{
LogError($"{name}: ffprobe failed to get metadata");
return;
}
if (job.Job.Crop == null)
{
LogError($"{name}: Crop parameters are required");
return;
}
// Processing size (what you crop / feed into enhancer)
var procWidth = job.Job.Debug ? videoWidth : job.Job.Crop.Value.width;
var procHeight = job.Job.Debug ? videoHeight : job.Job.Crop.Value.height;
IVideoEnhancer? enhancer = null;
const int window = 5;
if (job.Job.Enhance)
{
enhancer = new RealBasicVsr2xDmlEnhancer();
await enhancer.InitializeAsync(procWidth, procHeight, window, token);
}
// Encoding size (what FFmpeg encoder expects)
var encWidth = enhancer != null ? procWidth * enhancer.ResolutionMultiplier : procWidth;
var encHeight = enhancer != null ? procHeight * enhancer.ResolutionMultiplier : procHeight;
LogInfo($"{name}: src={videoWidth}x{videoHeight} @ {fps:F3}fps, seg=[{start:F3},{length:F3}] proc={procWidth}x{procHeight} enc={encWidth}x{encHeight}");
var decode = await StartFfmpegDecode(inputFile, start, length, job.Job.Rotate, job.Job.PlainText, token);
using var decodeStdout = decode.StandardOutput.BaseStream;
var encode = await StartFfmpegEncode( var encode = await StartFfmpegEncode(
inputFile, job.Job.InputFile,
outputFile, job.OutputFileName,
start, job.SegmentStart,
length, job.SegmentLength,
encWidth, state.OutMat.Width,
encHeight, state.OutMat.Height,
job.Info, job.Info,
ffmpegPassthroughParameters, job.Job.Passthrough,
job.Job.PlainText, job.Job.PlainText,
token); token);
using var encodeStdin = encode.StandardInput.BaseStream; using var encodeStdin = encode.StandardInput.BaseStream;
// Input: always full frame var totalFrames = (int)Math.Round(job.SegmentLength * fps);
var inBytes = videoWidth * videoHeight * 3;
// Output: encoded frame size (may be 4x if enhancement enabled)
var outBytes = encWidth * encHeight * 3;
var inBuffer = new byte[inBytes];
var outBuffer = new byte[outBytes];
using var frameMat = new Mat(videoHeight, videoWidth, MatType.CV_8UC3);
// outMat is processing size (crop), not necessarily encoding size
using var outMat = new Mat(procHeight, procWidth, MatType.CV_8UC3);
var kalman = new KalmanTracker();
var camera = new CameraController(
videoWidth,
videoHeight,
job.Job.Crop.Value.width,
job.Job.Crop.Value.height,
kalman,
job.Job);
try
{
var startTime = DateTime.UtcNow;
var totalFrames = (int)Math.Round(length * fps);
var frameIndex = 0; var frameIndex = 0;
var startTime = DateTime.UtcNow;
var enhancedOutput = new Mat[window];
//totalFrames = 10;
while (frameIndex < totalFrames) while (frameIndex < totalFrames)
{ {
token.ThrowIfCancellationRequested(); token.ThrowIfCancellationRequested();
frameIndex++; var frame = GetNextProcessedFrame(state, token);
if (frame == null)
var read = await ReadExact(decodeStdout, inBuffer, 0, inBytes, token);
if (read != inBytes)
break; break;
Marshal.Copy(inBuffer, 0, frameMat.Data, inBytes); frameIndex++;
var (objects, primary) = _tracker.SelectTrackedObject(job, frameMat, kalman.LastMeasurement); EncodeFrame(frame, state, encodeStdin);
camera.Update(primary);
var roi = camera.Roi;
if (job.Job.Debug)
{
DrawDebug(frameMat, objects, camera, kalman);
frameMat.CopyTo(outMat); // outMat: procWidth x procHeight == full frame in debug
}
else
{
using var cropped = new Mat(frameMat, roi);
cropped.CopyTo(outMat); // outMat: procWidth x procHeight == crop
}
Mat frameToWrite = outMat;
if (enhancer != null)
{
if (enhancer.TryProcessFrame(outMat, out var enhanced, token))
frameToWrite = enhanced; // enhanced: encWidth x encHeight
else
continue;
}
Marshal.Copy(frameToWrite.Data, outBuffer, 0, outBytes);
encodeStdin.Write(outBuffer, 0, outBytes);
var elapsed = DateTime.UtcNow - startTime; var elapsed = DateTime.UtcNow - startTime;
var progress = totalFrames > 0 ? (double)frameIndex / totalFrames : 0.0; var progress = totalFrames > 0 ? (double)frameIndex / totalFrames : 0.0;
var speed = elapsed.TotalSeconds > 0 ? (frameIndex / elapsed.TotalSeconds) / fps : 0.0; var speed = elapsed.TotalSeconds > 0 ? (frameIndex / elapsed.TotalSeconds) / fps : 0.0;
var remainingFrames = Math.Max(totalFrames - frameIndex, 0); var remainingFrames = Math.Max(totalFrames - frameIndex, 0);
var etaSeconds = speed > 0 ? remainingFrames / speed : 0.0; var etaSeconds = speed > 0 ? remainingFrames / speed : 0.0;
var eta = TimeSpan.FromSeconds(etaSeconds); var eta = TimeSpan.FromSeconds(etaSeconds);
@ -163,32 +198,10 @@ public class TrackingSplitter : LoggingBase, ISegmentProcessor
DrawProgress(name, progress, eta, speed); DrawProgress(name, progress, eta, speed);
} }
if (enhancer != null)
{
int count = enhancer.Flush(enhancedOutput, token);
for (int i = 0; i < count; i++)
{
var mat = enhancedOutput[i]; // encWidth x encHeight
Marshal.Copy(mat.Data, outBuffer, 0, outBytes);
encodeStdin.Write(outBuffer, 0, outBytes);
}
}
encodeStdin.Flush(); encodeStdin.Flush();
encodeStdin.Close(); encodeStdin.Close();
await encode.WaitForExitAsync(); await encode.WaitForExitAsync();
}
finally
{
if (enhancer is IAsyncDisposable asyncDisp)
await asyncDisp.DisposeAsync();
else if (enhancer is IDisposable disp)
disp?.Dispose();
}
try { if (!decode.HasExited) decode.Kill(entireProcessTree: true); } catch { }
try { if (!decode.HasExited) await decode.WaitForExitAsync(); } catch { }
ClearProgress(name); ClearProgress(name);
@ -196,12 +209,123 @@ public class TrackingSplitter : LoggingBase, ISegmentProcessor
LogError($"{name}: FFmpeg encoding failed"); LogError($"{name}: FFmpeg encoding failed");
else else
LogInfo($"{name}: Segment processing completed"); LogInfo($"{name}: Segment processing completed");
FinishSegment(state);
} }
// ============================================================
// INTERNAL HELPERS
// ============================================================
// ---------- FFmpeg decode / encode ---------- private object CreateFrameState(SingleTask job)
{
var w = job.Info.Width;
var h = job.Info.Height;
var cw = job.Job.Debug ? w : job.Job.Crop!.Value.width;
var ch = job.Job.Debug ? h : job.Job.Crop!.Value.height;
private async Task<Process> StartFfmpegDecode(string inputFile, double start, double length, int? rotate, bool plainText, CancellationToken token) var kalman = new KalmanTracker();
var camera = new CameraController(w, h, cw, ch, kalman, job.Job);
var frameMat = new Mat(h, w, MatType.CV_8UC3);
var outMat = new Mat(ch, cw, MatType.CV_8UC3);
var inBytes = w * h * 3;
var outBytes = cw * ch * 3;
var inBuffer = new byte[inBytes];
var outBuffer = new byte[outBytes];
IVideoEnhancer? enhancer = job.Job.Enhance
? new RealBasicVsr2xDmlEnhancer()
: null;
return new FrameProcessingState(
job,
kalman,
camera,
frameMat,
outMat,
inBuffer,
outBuffer,
enhancer,
inBytes,
outBytes);
}
private bool TryReadNextFrame(
Stream decodeStdout,
FrameProcessingState state,
CancellationToken token)
{
var read = ReadExact(
decodeStdout,
state.InBuffer,
0,
state.InBytes,
token).Result;
if (read != state.InBytes)
return false;
Marshal.Copy(state.InBuffer, 0, state.FrameMat.Data, state.InBytes);
return true;
}
private Mat ProcessFrame(
Mat inputFrame,
FrameProcessingState state,
SingleTask job,
CancellationToken token)
{
var (objects, primary) =
_tracker.SelectTrackedObject(job, inputFrame, state.Kalman.LastMeasurement);
state.Camera.Update(primary);
var roi = state.Camera.Roi;
if (job.Job.Debug)
{
DebugOverlay.DrawDebug(inputFrame, objects, state.Camera, state.Kalman);
inputFrame.CopyTo(state.OutMat);
}
else
{
using var cropped = new Mat(inputFrame, roi);
cropped.CopyTo(state.OutMat);
}
if (state.Enhancer != null)
{
if (state.Enhancer.TryProcessFrame(state.OutMat, out var enhanced, token))
return enhanced;
return state.OutMat;
}
return state.OutMat;
}
private void EncodeFrame(
Mat frame,
FrameProcessingState state,
Stream encodeStdin)
{
Marshal.Copy(frame.Data, state.OutBuffer, 0, state.OutBytes);
encodeStdin.Write(state.OutBuffer, 0, state.OutBytes);
}
// ------------------------------------------------------------
// FFmpeg helpers
// ------------------------------------------------------------
private async Task<Process> StartFfmpegDecode(
string inputFile,
double start,
double length,
int? rotate,
bool plainText,
CancellationToken token)
{ {
var ss = start .ToString("0.###", CultureInfo.InvariantCulture); var ss = start .ToString("0.###", CultureInfo.InvariantCulture);
var t = length.ToString("0.###", CultureInfo.InvariantCulture); var t = length.ToString("0.###", CultureInfo.InvariantCulture);
@ -256,7 +380,6 @@ public class TrackingSplitter : LoggingBase, ISegmentProcessor
case 270: rotateStr = ",transpose=2"; break; case 270: rotateStr = ",transpose=2"; break;
} }
} }
return rotateStr; return rotateStr;
} }
@ -265,7 +388,8 @@ public class TrackingSplitter : LoggingBase, ISegmentProcessor
string outputFile, string outputFile,
double start, double start,
double length, double length,
int width, int height, int width,
int height,
VideoInfo info, VideoInfo info,
string[] passthrough, string[] passthrough,
bool plainText, bool plainText,
@ -275,19 +399,17 @@ public class TrackingSplitter : LoggingBase, ISegmentProcessor
var fpsStr = info.Fps.ToString("0.###", CultureInfo.InvariantCulture); var fpsStr = info.Fps.ToString("0.###", CultureInfo.InvariantCulture);
var ss = start.ToString("0.###", CultureInfo.InvariantCulture); var ss = start.ToString("0.###", CultureInfo.InvariantCulture);
var t = length.ToString("0.###", CultureInfo.InvariantCulture); var t = length.ToString("0.###", CultureInfo.InvariantCulture);
var sarArg = !string.IsNullOrWhiteSpace(info.SampleAspectRatio) var sarArg = !string.IsNullOrWhiteSpace(info.SampleAspectRatio)
? $"-vf setsar={info.SampleAspectRatio} " ? $"-vf setsar={info.SampleAspectRatio} "
: ""; : "";
var darArg = ""; var darArg = "";
if (info.Sar is { } s) if (info.Sar is { } s)
{ {
// compute DAR from output size and SAR
var darNum = width * s.X; var darNum = width * s.X;
var darDen = height * s.Y; var darDen = height * s.Y;
// clamp to int and reduce
var dn = (int)Math.Min(int.MaxValue, Math.Max(int.MinValue, darNum)); var dn = (int)Math.Min(int.MaxValue, Math.Max(int.MinValue, darNum));
var dd = (int)Math.Min(int.MaxValue, Math.Max(int.MinValue, darDen)); var dd = (int)Math.Min(int.MaxValue, Math.Max(int.MinValue, darDen));
ReduceFraction(ref dn, ref dd); ReduceFraction(ref dn, ref dd);
@ -306,9 +428,6 @@ public class TrackingSplitter : LoggingBase, ISegmentProcessor
"-c:a copy " + "-c:a copy " +
pass + $" \"{outputFile}\""; pass + $" \"{outputFile}\"";
// "-c:a aac -b:a 192k " +
var psi = new ProcessStartInfo var psi = new ProcessStartInfo
{ {
FileName = "ffmpeg", FileName = "ffmpeg",
@ -330,19 +449,15 @@ public class TrackingSplitter : LoggingBase, ISegmentProcessor
{ {
string? line; string? line;
while ((line = await p.StandardError.ReadLineAsync(token)) != null) while ((line = await p.StandardError.ReadLineAsync(token)) != null)
{
if (plainText) if (plainText)
LogInfo($"[ffmpeg-encode] {fileName}: {line}"); LogInfo($"[ffmpeg-encode] {fileName}: {line}");
} }
}
catch { } catch { }
}); });
return p; return p;
} }
// ---------- helpers ----------
private static void ReduceFraction(ref int num, ref int den) private static void ReduceFraction(ref int num, ref int den)
{ {
int Gcd(int a, int b) int Gcd(int a, int b)
@ -363,7 +478,13 @@ public class TrackingSplitter : LoggingBase, ISegmentProcessor
den /= g; den /= g;
} }
} }
private static async Task<int> ReadExact(Stream s, byte[] buffer, int offset, int count, CancellationToken token)
private static async Task<int> ReadExact(
Stream s,
byte[] buffer,
int offset,
int count,
CancellationToken token)
{ {
var total = 0; var total = 0;
while (total < count) while (total < count)
@ -376,35 +497,5 @@ public class TrackingSplitter : LoggingBase, ISegmentProcessor
return total; return total;
} }
private void DrawDebug(
Mat frame,
List<DetectedPerson> objects,
CameraController camera,
KalmanTracker kalman)
{
if (camera.ObjectBox.HasValue)
{
var fb = camera.ObjectBox.Value;
Cv2.Rectangle(frame, fb, Scalar.LimeGreen, 2);
}
Cv2.Circle(frame,
new Point((int)camera.SmoothedCenter.X, (int)camera.SmoothedCenter.Y),
6, Scalar.LimeGreen, -1);
Cv2.Rectangle(frame, camera.Roi,
camera.ObjectCenter.HasValue ? Scalar.Yellow : Scalar.Red, 3);
DrawText(frame, $"Faces: {objects.Count}", 20, 40, Scalar.White);
DrawText(frame, $"LostFrames: {camera.LostFrames}", 20, 70, Scalar.White);
DrawText(frame, $"Noise: {kalman.CurrentNoise:F3}", 20, 130, Scalar.White);
DrawText(frame, $"Camera: {camera.CameraCenter.X:F1},{camera.CameraCenter.Y:F1}", 20, 160, Scalar.White);
}
private static void DrawText(Mat img, string text, int x, int y, Scalar color)
{
Cv2.PutText(img, text, new Point(x, y),
HersheyFonts.HersheySimplex, 0.6, color, 2);
}
} }

View File

@ -1,6 +1,14 @@
namespace splitter.algo; namespace splitter.algo;
public interface IFrameProcessingState
{
}
public interface ISegmentProcessor public interface ISegmentProcessor
{ {
IFrameProcessingState InitSegment(SingleTask job, CancellationToken token);
Mat? GetNextProcessedFrame( IFrameProcessingState processorState, CancellationToken token);
void FinishSegment(IFrameProcessingState processorState);
Task ProcessSegment( SingleTask job, CancellationToken token); Task ProcessSegment( SingleTask job, CancellationToken token);
} }

View File

@ -12,7 +12,6 @@ public sealed class YoloV10ObjectDetector : LoggingBase, IObjectDetector, IDispo
private const int _inputWidth = 640; private const int _inputWidth = 640;
private const int _inputHeight = 640; private const int _inputHeight = 640;
private const float _scoreThreshold = 0.35f;
private const float _nmsThreshold = 0.45f; private const float _nmsThreshold = 0.45f;
private const int _personClassIndex = 0; private const int _personClassIndex = 0;

View File

@ -38,7 +38,7 @@ static partial class Program
var allJobs = new List<SingleTask>(); var allJobs = new List<SingleTask>();
foreach ( var job in cmd.Jobs ) foreach ( var job in cmd.Jobs )
{ {
var jobs = await processor.GenerateJobs(job, cmd.Master.EstimateOnly, CancellationToken.None); var jobs = await processor.GenerateJobs(job, cmd.Master.EstimateOnly, [], CancellationToken.None);
allJobs.AddRange(jobs); allJobs.AddRange(jobs);
} }

View File

@ -375,7 +375,7 @@ public sealed class SpectreConsoleLogger : ILogger, IDisposable
return new Measurement(width, width); return new Measurement(width, width);
} }
public IEnumerable<Segment> Render(RenderOptions options, int maxWidth) public IEnumerable<Spectre.Console.Rendering.Segment> Render(RenderOptions options, int maxWidth)
{ {
var width = Math.Max(1, maxWidth); var width = Math.Max(1, maxWidth);