Correct(-ish) position displayed when sppeding up or slowing dowm audio

This commit is contained in:
Alexander Shabarshov 2026-08-15 10:02:46 +01:00
parent 4c8d4d2ed9
commit 46ed183e3f
4 changed files with 134 additions and 64 deletions

View File

@ -47,7 +47,7 @@ public partial class WaveformBandViewModel : ObservableObject
return; return;
} }
double ratio = current.TotalSeconds / total.TotalSeconds; double ratio = current.TotalMicroseconds / total.TotalMicroseconds;
PlaybackX = ratio * CanvasWidth; PlaybackX = ratio * CanvasWidth;
} }

View File

@ -3,9 +3,7 @@
<PropertyGroup> <PropertyGroup>
<IsPackable>true</IsPackable> <IsPackable>true</IsPackable>
<Authors>Alexander Shabarshov</Authors> <Authors>Alexander Shabarshov</Authors>
<Description> <Description>Audio engine core for ABStemPlayer. Audio pipeline. </Description>
Audio engine core for ABStemPlayer. Audio pipeline.
</Description>
<PackageProjectUrl>https://github.com/unclshura/ABStemPlayer</PackageProjectUrl> <PackageProjectUrl>https://github.com/unclshura/ABStemPlayer</PackageProjectUrl>
<PackageTags>audio, player, ab, guitar, drums, piano, stem, mixer, c#</PackageTags> <PackageTags>audio, player, ab, guitar, drums, piano, stem, mixer, c#</PackageTags>
<PackageReadmeFile>README.md</PackageReadmeFile> <PackageReadmeFile>README.md</PackageReadmeFile>
@ -34,8 +32,8 @@
<PropertyGroup> <PropertyGroup>
<Version>1.0.0</Version> <Version>1.0.0</Version>
<BuildNumber>0</BuildNumber> <BuildNumber>0</BuildNumber>
<SourceRevisionId>0000</SourceRevisionId> <SourceRevisionId></SourceRevisionId>
<InformationalVersion>$(Version).$(BuildNumber)+$(SourceRevisionId)</InformationalVersion> <InformationalVersion>$(Version).$(BuildNumber)$(SourceRevisionId)</InformationalVersion>
<AssemblyVersion>$(Version)</AssemblyVersion> <AssemblyVersion>$(Version)</AssemblyVersion>
<FileVersion>$(Version).$(BuildNumber)</FileVersion> <FileVersion>$(Version).$(BuildNumber)</FileVersion>
</PropertyGroup> </PropertyGroup>

View File

@ -10,6 +10,15 @@ public sealed class RubberBandTimeStretchEngine : ITimeStretchEngine, IAsyncDisp
private readonly int _sampleRate; private readonly int _sampleRate;
private long[] _sourcePositions; private long[] _sourcePositions;
private sealed class SourceChunkInfo
{
public long SourcePosition; // starting frame index in source stream
public int SourceFrames; // number of frames in this chunk
}
private readonly Queue<SourceChunkInfo>[] _pendingInput;
private double[] _fractionalInput;
// One RubberBand/ffmpeg process per stem (each is stereo: 2 channels) // One RubberBand/ffmpeg process per stem (each is stereo: 2 channels)
private sealed class StemProcess : IDisposable private sealed class StemProcess : IDisposable
{ {
@ -51,6 +60,12 @@ public sealed class RubberBandTimeStretchEngine : ITimeStretchEngine, IAsyncDisp
_stemProcesses.Capacity = stemCount; _stemProcesses.Capacity = stemCount;
_stemCount = stemCount; _stemCount = stemCount;
_sourcePositions = new long[_stemCount]; _sourcePositions = new long[_stemCount];
_fractionalInput = new double[_stemCount];
_pendingInput = Enumerable.Range(0, _stemCount)
.Select(_ => new Queue<SourceChunkInfo>())
.ToArray();
} }
public async Task Configure(PlaybackSpeedSettings settings, CancellationToken token) public async Task Configure(PlaybackSpeedSettings settings, CancellationToken token)
@ -75,32 +90,48 @@ public sealed class RubberBandTimeStretchEngine : ITimeStretchEngine, IAsyncDisp
public Task SubmitStems(IReadOnlyList<AudioBlock> stemBlocks, CancellationToken token) public Task SubmitStems(IReadOnlyList<AudioBlock> stemBlocks, CancellationToken token)
{ {
if ( stemBlocks.Count != _stemCount) if (stemBlocks.Count != _stemCount)
throw new ArgumentException($"Expected {_stemCount} stems, but got {stemBlocks.Count}."); throw new ArgumentException($"Expected {_stemCount} stems, but got {stemBlocks.Count}.");
EnsureStemProcesses(stemBlocks.Count);
// No-stretch path: just enqueue into per-stem rings // No-stretch path: just enqueue into per-stem rings
if (Math.Abs(_speed - 1.0f) < 0.01f) if (Math.Abs(_speed - 1.0f) < 0.01f)
{ {
EnsureStemProcesses(stemBlocks.Count);
for (int i = 0; i < stemBlocks.Count; i++) for (int i = 0; i < stemBlocks.Count; i++)
{ {
var proc = _stemProcesses[i]; var block = stemBlocks[i];
var bytes = MemoryMarshal.AsBytes(stemBlocks[i].Buffer.Span);
proc.Ring.Write(bytes, bytes.Length, token); // Track source position for passthrough mode
_pendingInput[i].Enqueue(new SourceChunkInfo
{
SourcePosition = block.Position,
SourceFrames = block.Frames
});
var bytes = MemoryMarshal.AsBytes(block.Buffer.Span);
_stemProcesses[i].Ring.Write(bytes, bytes.Length, token);
} }
return Task.CompletedTask; return Task.CompletedTask;
} }
// Stretch path: one ffmpeg+rubberband per stem // Stretch path: one ffmpeg+rubberband per stem
EnsureStemProcesses(stemBlocks.Count);
StartProcessesIfNeeded(stemBlocks.Count); StartProcessesIfNeeded(stemBlocks.Count);
for (int i = 0; i < stemBlocks.Count; i++) for (int i = 0; i < stemBlocks.Count; i++)
{ {
var block = stemBlocks[i];
// Track source position for stretched mode
_pendingInput[i].Enqueue(new SourceChunkInfo
{
SourcePosition = block.Position,
SourceFrames = block.Frames
});
var proc = _stemProcesses[i]; var proc = _stemProcesses[i];
var span = stemBlocks[i].Buffer.Span; var span = block.Buffer.Span;
var bytes = MemoryMarshal.AsBytes(span); var bytes = MemoryMarshal.AsBytes(span);
try try
@ -108,29 +139,31 @@ public sealed class RubberBandTimeStretchEngine : ITimeStretchEngine, IAsyncDisp
if (token.IsCancellationRequested) if (token.IsCancellationRequested)
return Task.CompletedTask; return Task.CompletedTask;
// Only write if ffmpeg is alive
if (proc.Stdin != null && !(proc.Ff?.Proc?.HasExited ?? true)) if (proc.Stdin != null && !(proc.Ff?.Proc?.HasExited ?? true))
{
proc.Stdin.Write(bytes); proc.Stdin.Write(bytes);
if (token.IsCancellationRequested) try
return Task.CompletedTask; {
try proc.Stdin.Flush();
{ }
proc.Stdin?.Flush(); catch (ObjectDisposedException)
} {
catch (System.ObjectDisposedException) // ffmpeg exited early
{ }
// process has exited
} }
} }
catch catch
{ {
// ignore // Ignore write errors (ffmpeg may exit early)
} }
} }
return Task.CompletedTask; return Task.CompletedTask;
} }
public async Task<TimeStretchedAudioBlock[]> ReceiveStems(CancellationToken token) public async Task<TimeStretchedAudioBlock[]> ReceiveStems(CancellationToken token)
{ {
EnsureStemProcesses(_stemCount); EnsureStemProcesses(_stemCount);
@ -160,9 +193,9 @@ public sealed class RubberBandTimeStretchEngine : ITimeStretchEngine, IAsyncDisp
return Array.Empty<TimeStretchedAudioBlock>(); return Array.Empty<TimeStretchedAudioBlock>();
// Determine block size (final block may be smaller) // Determine block size (final block may be smaller)
int bytesToRead = Math.Min(bytesPerBlock, available); int bytesToRead = Math.Min(bytesPerBlock, available);
int samplesToRead = bytesToRead / sizeof(float); int samplesToRead = bytesToRead / sizeof(float);
int framesToRead = samplesToRead / 2; int framesToRead = samplesToRead / 2;
// Allocate a temporary byte[] buffer (safe across await) // Allocate a temporary byte[] buffer (safe across await)
byte[] temp = new byte[bytesToRead]; byte[] temp = new byte[bytesToRead];
@ -195,10 +228,10 @@ public sealed class RubberBandTimeStretchEngine : ITimeStretchEngine, IAsyncDisp
var outBytes = MemoryMarshal.AsBytes(outBuf.Span); var outBytes = MemoryMarshal.AsBytes(outBuf.Span);
temp.AsSpan().CopyTo(outBytes); temp.AsSpan().CopyTo(outBytes);
// Compute source position //
long sourceFrames = (long)(framesToRead * _speed); // *** Correct source-position mapping ***
long sourcePos = _sourcePositions[i]; //
_sourcePositions[i] += sourceFrames; long sourcePos = ComputeSourcePosition(i, framesToRead);
result[i] = new TimeStretchedAudioBlock( result[i] = new TimeStretchedAudioBlock(
outBuf, outBuf,
@ -211,6 +244,54 @@ public sealed class RubberBandTimeStretchEngine : ITimeStretchEngine, IAsyncDisp
return result; return result;
} }
private long ComputeSourcePosition(int stemIndex, int outputFrames)
{
// inputFrames = outputFrames / speed
double neededInputFrames = outputFrames / _speed;
// Add fractional requirement
_fractionalInput[stemIndex] += neededInputFrames;
long sourcePos = 0;
bool first = true;
var queue = _pendingInput[stemIndex];
// If no input chunks exist (warm-up), return last known position
if (queue.Count == 0)
return _sourcePositions[stemIndex];
while (_fractionalInput[stemIndex] >= 1 && queue.Count > 0)
{
var chunk = queue.Peek();
int take = (int)Math.Min(chunk.SourceFrames, Math.Floor(_fractionalInput[stemIndex]));
if (take <= 0)
break;
if (first)
{
sourcePos = chunk.SourcePosition;
first = false;
}
chunk.SourceFrames -= take;
_fractionalInput[stemIndex] -= take;
if (chunk.SourceFrames == 0)
queue.Dequeue();
}
// If we consumed nothing (fraction < 1), use last known position
if (first)
sourcePos = _sourcePositions[stemIndex];
_sourcePositions[stemIndex] = sourcePos;
return sourcePos;
}
private void EnsureStemProcesses(int stemCount) private void EnsureStemProcesses(int stemCount)
{ {

View File

@ -8,14 +8,14 @@ public sealed class StemPlaybackEngine : IStemPlaybackEngine, IDisposable
{ {
private sealed class PipelineState : IDisposable private sealed class PipelineState : IDisposable
{ {
public IStemDecoder[] Decoders = Array.Empty<IStemDecoder>(); public IStemDecoder[] Decoders = Array.Empty<IStemDecoder>();
public bool OutputStarted; public bool OutputStarted;
public CancellationTokenSource? Cts; public CancellationTokenSource? Cts;
public Task? RenderTask; public Task? RenderTask;
public void Dispose() public void Dispose()
{ {
try { Cts?.Cancel(); } catch { } try { Cts?.Cancel(); } catch { }
try { Cts?.Dispose(); } catch { } try { Cts?.Dispose(); } catch { }
foreach (var d in Decoders) foreach (var d in Decoders)
@ -39,7 +39,6 @@ public sealed class StemPlaybackEngine : IStemPlaybackEngine, IDisposable
private LoopRegion _loopRegion = new(); private LoopRegion _loopRegion = new();
private long _decodedFramePosition;
private long _loopStartFrames; private long _loopStartFrames;
private long _loopEndFrames; private long _loopEndFrames;
@ -96,7 +95,6 @@ public sealed class StemPlaybackEngine : IStemPlaybackEngine, IDisposable
} }
_pendingSeekFrames = 0; _pendingSeekFrames = 0;
_decodedFramePosition = 0;
} }
} }
@ -126,7 +124,6 @@ public sealed class StemPlaybackEngine : IStemPlaybackEngine, IDisposable
d.Seek(_pendingSeekFrames); d.Seek(_pendingSeekFrames);
} }
_decodedFramePosition = _pendingSeekFrames;
} }
_pipeline.RenderTask = Task.Run(() => RenderLoopAsync(_pipeline, _pipeline.Cts.Token)); _pipeline.RenderTask = Task.Run(() => RenderLoopAsync(_pipeline, _pipeline.Cts.Token));
@ -163,7 +160,6 @@ public sealed class StemPlaybackEngine : IStemPlaybackEngine, IDisposable
if (!IsPlaying && _pipeline is null) if (!IsPlaying && _pipeline is null)
return; return;
_decodedFramePosition = 0;
_pendingSeekFrames = 0; _pendingSeekFrames = 0;
pipelineToDispose = _pipeline; pipelineToDispose = _pipeline;
@ -201,12 +197,6 @@ public sealed class StemPlaybackEngine : IStemPlaybackEngine, IDisposable
{ {
foreach (var d in _pipeline.Decoders) foreach (var d in _pipeline.Decoders)
d.Seek(frameIndex); d.Seek(frameIndex);
_decodedFramePosition = frameIndex;
}
else
{
_decodedFramePosition = frameIndex;
} }
} }
@ -349,15 +339,9 @@ public sealed class StemPlaybackEngine : IStemPlaybackEngine, IDisposable
try { d.Seek(loopStart); } catch { } try { d.Seek(loopStart); } catch { }
} }
lock (_stateLock)
_decodedFramePosition = loopStart;
// continue decoding from the loop start // continue decoding from the loop start
continue; continue;
} }
lock (_stateLock)
_decodedFramePosition = nextPosition;
} }
} }
catch { } catch { }
@ -430,19 +414,7 @@ public sealed class StemPlaybackEngine : IStemPlaybackEngine, IDisposable
_outputDevice.Write(mixed.Buffer.Span); _outputDevice.Write(mixed.Buffer.Span);
gotFirstBlock = true; gotFirstBlock = true;
try await ReportProgress(mixed).ConfigureAwait(false);
{
double progress;
lock (_stateLock)
{
var total = _session?.StemSet.TotalFrames ?? 1L;
progress = (double)mixed.Position / Math.Max(total, 1L);
}
if (_progressReporter != null)
await _progressReporter.ReportProgress(progress).ConfigureAwait(false);
}
catch { }
try { mixed.Dispose(); } catch { } try { mixed.Dispose(); } catch { }
foreach (var b in stretchedBlocks) foreach (var b in stretchedBlocks)
@ -459,6 +431,25 @@ public sealed class StemPlaybackEngine : IStemPlaybackEngine, IDisposable
} }
} }
private async Task ReportProgress(MixedAudioBlock mixed)
{
if (_progressReporter == null)
return;
try
{
double progress;
lock (_stateLock)
{
var total = _session?.StemSet.TotalFrames ?? 1L;
progress = (double)mixed.Position / Math.Max(total, 1L);
}
await _progressReporter.ReportProgress(progress).ConfigureAwait(false);
}
catch { }
}
private long TimeToFrames(TimeSpan time) private long TimeToFrames(TimeSpan time)
{ {
return (long)(time.TotalSeconds * _outputDevice.SampleRate); return (long)(time.TotalSeconds * _outputDevice.SampleRate);