DEEP-40 Corellation matrix added

DeepTrace/Controls/CorrelationChart.razor

@@ -0,0 +1,204 @@
+@using DeepTrace.Data;
+@using DeepTrace.ML;
+@using PrometheusAPI;
+
+@if (_matrix.Count > 0)
+{
+    <ApexChart @ref="_chart"
+        Title="Correlation"
+        Options="@_options"
+        >
+            @for (var i = 0; i < _matrix!.Count; i++)
+            {
+                <ApexPointSeries TItem="HeatMapData"
+                        Name="@_matrix[i].Name"
+                        Items="@_matrix[i].Series"
+                        SeriesType="SeriesType.Heatmap"
+                        XValue="@(e => e.Name)"
+                        YAggregate="@(e => (decimal)e.Sum(x => float.IsNaN(x.Value) || !float.IsFinite(x.Value) ? 0F : x.Value))"
+                        ShowDataLabels="false" />
+            }
+    </ApexChart>
+}
+
+@code {
+    [CascadingParameter]
+    protected bool IsDarkMode { get; set; }
+
+    [Parameter] public TimeSeriesData? Data { get; set; }
+
+    private ApexChart<HeatMapData>?        _chart;
+    private ApexChartOptions<HeatMapData>? _options;
+    private List<HeatMapDataSeries>        _matrix = new();
+    private TimeSeriesData                 _currentData = new() { Series = { new() } };
+
+    private record HeatMapData( string Name, float Value)
+    {
+        public override string ToString() => $"{Value:N4} {Name}";
+    }    
+    private record HeatMapDataSeries(string Name, List<HeatMapData> Series)
+    {
+        public override string ToString() => $"{Name} {Series.Count}";
+    }
+
+    protected override async Task OnInitializedAsync()
+    {
+        await UpdateChart();
+        await base.OnInitializedAsync();
+    }
+
+    protected override async Task OnParametersSetAsync()
+    {
+        await UpdateChart();
+        await base.OnParametersSetAsync();
+    }
+
+    private decimal GetValue(object o)
+    {
+        if (o is float f && !float.IsNaN(f) )
+            return (decimal)f;
+        return 0m;
+    }
+
+    private async Task UpdateChart()
+    {
+        if (Data == _currentData)
+            return;
+
+        _currentData = Data?.Series.Count > 0 && Data.Series.All( x => x.Data.Count > 0 )
+            ? Data
+            : new()
+                {
+                    Series =
+                    {
+                        new()
+                        {
+                            Name = "??",
+                            Data = new List<TimeSeries>
+                            {
+                                new TimeSeries
+                                {
+                                    TimeStamp = DateTime.Now,
+                                    Value = 0.0F
+                                }
+                            }
+                        }
+                    }
+                };
+
+        var matrix = Correlation.Matrix(DataSourceDefinition.Normalize(_currentData.Series) ?? _currentData.Series);
+        if (matrix.GetLength(0) == 0 )
+        {
+            matrix = new float[_currentData.Series.Count, _currentData.Series.Count];
+        }
+
+        _matrix.Clear();
+        for( var i = 0; i < matrix.GetLength(0); i++ )
+        {
+            _matrix.Add(new(
+                TimeSeriesDataSet.MakeLabel(_currentData.Series[i].Name), 
+                Enumerable.Range(0, matrix.GetLength(1))
+                    .Select(x => new HeatMapData(
+                        TimeSeriesDataSet.MakeLabel(_currentData.Series[x].Name),
+                        matrix[x, i]
+                    ))
+                    .ToList()
+                )
+            );
+        }
+
+        _options = CreateOptions();
+
+        if (_chart == null)
+            return;
+
+        //await InvokeAsync(StateHasChanged);
+        await _chart.UpdateSeriesAsync();
+        await _chart.UpdateOptionsAsync(true, true, true);
+        await InvokeAsync(StateHasChanged);
+    }
+
+    private ApexChartOptions<HeatMapData> CreateOptions()
+    {
+        var backgroundColor = IsDarkMode ? "var(--mud-palette-surface)" : "#f3f3f3";
+        var gridColor       = IsDarkMode ? "var(--mud-palette-drawer-background)" : "#f3f3f3";
+        var borderColor     = IsDarkMode ? "var(--mud-palette-text-primary)" : "#e7e7e7";
+        var lineColors      = _currentData.Series.Select(x => x.Color).ToList();
+        var mode            = IsDarkMode
+            ? Mode.Dark
+            : Mode.Light
+            ;
+
+        var options = new ApexChartOptions<HeatMapData>
+            {
+                Chart = new()
+                {
+                    Background = backgroundColor,
+                    Toolbar = new()
+                    {
+                        Show = true
+                    },
+                    DropShadow = new()
+                    {
+                        Enabled = false,
+                        Color   = "",
+                        Top     = 18,
+                        Left    = 7,
+                        Blur    = 10,
+                        Opacity = 0.2d
+                    }
+                },
+                DataLabels = new()
+                {
+                    Enabled = false
+                },
+                //Tooltip = new ApexCharts.Tooltip
+                //{
+                //    Y = new()
+                //    {
+                //        Formatter = @"function(value, opts) {
+                //                    if (value === undefined) {return '';}
+                //                    return Number(value).toLocaleString();}",
+                //    },
+                //    X = new()
+                //    {
+                //        Formatter = @"function(value, opts) {
+                //                    if (value === undefined) {return '';}
+                //                    return (new Date(value)).toISOString();}",
+                //    }
+
+                //},
+                //Xaxis = new()
+                //{
+                //    Type = XAxisType.Category
+                //},
+                //Grid = new()
+                //{
+                //    BorderColor = borderColor,
+                //    Row = new()
+                //    {
+                //        Colors = new List<string> { gridColor, "transparent" },
+                //        Opacity = 0.5d
+                //    }
+                //},
+                Colors = new List<string> { "#008FFB" },
+                //Stroke = new() { Curve = Curve.Straight, Width = 2 },
+                //Legend = new()
+                //{
+                //    Position = LegendPosition.Top,
+                //    HorizontalAlign = ApexCharts.Align.Right,
+                //    Floating = true,
+                //    OffsetX = -5,
+                //    OffsetY = -25
+                //},
+                Theme = new()
+                {
+                    Mode = mode,
+                    //Palette = PaletteType.Palette8,
+                }
+            };
+
+        return options;
+    }
+
+}

DeepTrace/Controls/TimeSeriesChart.razor

@@ -1,7 +1,7 @@
 @using DeepTrace.Data;
 @using DeepTrace.Services;
 @using PrometheusAPI;
-    using DeepTrace.Data;
+
 <ApexChart @ref="_chart"
     TItem="TimeSeries"
     Title="Data view"
@@ -11,7 +11,7 @@
     @foreach (var ts in _currentData.Series)
     {
         <ApexPointSeries TItem="TimeSeries"
-            Name="@ts.Name"
+            Name="@ts.Label"
             Items="@ts.Data"
             SeriesType="SeriesType.Line"
             XValue="@(e => e.TimeStamp)"
@@ -21,6 +21,7 @@
     }
 </ApexChart>
 
+
 @code {
 
     [CascadingParameter]
@@ -35,39 +36,70 @@ protected bool IsDarkMode { get; set; }
 
     private ApexChart<TimeSeries>?        _chart;
     private ApexChartOptions<TimeSeries>? _options;
-private TimeSeriesData                _currentData = new() { Series = { new () } };
+    private TimeSeriesData                _currentData = CreateEmpty();
 
-protected override void OnInitialized()
+    protected override async Task OnInitializedAsync()
     {
-    _options = CreateOptions();
+        await UpdateChart();
-    base.OnInitialized();
+        await base.OnInitializedAsync();
     }
 
+    //protected override async Task OnAfterRenderAsync(bool firstRender)
+    //{
+    //    if (firstRender)
+    //        await UpdateChart();
+    //    base.OnAfterRender(firstRender);
+    //}
+
     protected override async Task OnParametersSetAsync()
     {
-    Console.WriteLine("OnParametersSet");
-
         await UpdateChart();
         await base.OnParametersSetAsync();
     }
 
+
     private async Task UpdateChart()
     {
         if (Data == _currentData)
             return;
 
-    _currentData = Data ?? new() { Series = { new() } }; ;
+        _currentData = Data?.Series.Count > 0 && Data.Series.All(x => x.Data.Count > 0)
+            ? Data
+            : CreateEmpty();
+
         _options = CreateOptions();
 
         if (_chart == null)
             return;
 
         //await InvokeAsync(StateHasChanged);
-    await _chart!.UpdateSeriesAsync();
+        if (_currentData.Series.Count > 0)
-    await _chart!.UpdateOptionsAsync(true, true, true);
+        {
-    await InvokeAsync(StateHasChanged);
+            await _chart.UpdateSeriesAsync();
-
+            await _chart.UpdateOptionsAsync(true, true, true);
         }
+        await InvokeAsync(StateHasChanged);
+    }
+
+    private static TimeSeriesData CreateEmpty() => new()
+        {
+            Series =
+                {
+                new()
+                {
+                    Name = "??",
+                    Data = new List<TimeSeries>
+                    {
+                        new TimeSeries
+                        {
+                            TimeStamp = DateTime.Now,
+                            Value = 0.0F
+                        }
+                    }
+                }
+            }
+        };
+
 
 private ApexChartOptions<TimeSeries> CreateOptions()
 {

DeepTrace/Data/DataSourceDefinition.cs

@@ -1,4 +1,7 @@
-namespace DeepTrace.Data;
+using PrometheusAPI;
+using System.Diagnostics;
+
+namespace DeepTrace.Data;
 
 public class DataSourceQuery
 {
@@ -80,4 +83,109 @@ public class DataSourceDefinition
 
         return data;
     }
+
+    /// <summary>
+    /// Make time series coherent. Timestamps made the same across all series. Values interpolated using linear interpolation.
+    /// </summary>
+    public static List<TimeSeriesDataSet>? Normalize(List<TimeSeriesDataSet> source, int nIntervals = 50)
+    {
+        if ( source.Count == 0 || source.Any( x => x.Data.Count == 0 ) )
+            return null;
+
+        var minTime = source.SelectMany( x => x.Data.Select( y => y.TimeStamp)).Where( x => x != DateTime.MinValue).Min();
+        var maxTime = source.SelectMany( x => x.Data.Select( y => y.TimeStamp)).Where( x => x != DateTime.MinValue).Max();
+
+        if (minTime ==default || maxTime == default)
+            return null;
+
+        var res          = new List<TimeSeriesDataSet>();
+        var timeInterval = TimeSpan.FromMilliseconds( (double)(maxTime - minTime).TotalMilliseconds / (double)(nIntervals-1) );
+
+        foreach( var data in source )
+        {
+            static float GetValue(float v) => !float.IsNaN(v) && !float.IsInfinity(v) ? v : 0f;
+
+            if (data.Data.Count == 0)
+                return null;
+
+            var d     = new List<TimeSeries>(nIntervals);
+            var dest  = new TimeSeriesDataSet
+            {
+                Name  = data.Name,
+                Color = data.Color,
+                Data  = d
+            };
+            res.Add(dest);
+
+            var prev = data.Data[0]; // point in time prior to current
+            if (prev == null)
+                return null;
+            var nextIndex = 0;
+            var prevIndex = 0;
+            var next = prev; // point in time next to current
+
+            for (var i = 0; i < nIntervals; i++)
+            {
+                var ts = minTime + TimeSpan.FromMilliseconds(timeInterval.TotalMilliseconds * i);
+
+                float v;
+
+                if (next.TimeStamp < ts)
+                {
+                    // if next point timetamp become less than current - move the point forward
+                    for (var idx = nextIndex+1; idx < data.Data.Count; idx++)
+                    {
+                        // skip points if timestamp is in the past
+                        if (data.Data[idx].TimeStamp < ts)
+                            continue;
+
+                        // now we sure that point in in future comparing to the current "ts"
+                        nextIndex = idx;
+                        next = data.Data[idx];
+                    }
+
+                    // now try to adjust prev point as there can be point in time closee to current
+                    for (var idx = nextIndex-1; idx >= prevIndex; idx--)
+                    {
+                        // skip points if timestamp is in the past
+                        if (data.Data[idx].TimeStamp > ts)
+                            continue;
+
+                        // now we sure that point in in future comparing to the current "ts"
+                        prevIndex = idx;
+                        prev = data.Data[idx];
+                    }
+                }
+
+                if (next == prev || next.TimeStamp == ts)
+                {
+                    v = GetValue(next.Value);
+                }
+                else if (prev.TimeStamp == ts)
+                {
+                    v = GetValue(prev.Value);
+                }
+                else
+                {
+                    //Debug.Assert(ts >= prev.TimeStamp);
+                    //Debug.Assert(ts <= next.TimeStamp);
+
+                    // https://stackoverflow.com/questions/8672998/resample-aggregate-and-interpolate-of-timeseries-trend-data
+                    var dt = next.TimeStamp.Subtract(prev.TimeStamp).TotalMilliseconds;
+                    var dv = (double)GetValue(next.Value) - GetValue(prev.Value);
+                    v      = (float)(GetValue(prev.Value) + dv * ts.Subtract(prev.TimeStamp).TotalMilliseconds / dt);
+                }
+                
+
+                var curr = new TimeSeries(
+                    timeStamp: ts,
+                    value: v
+                );
+                d.Add(curr);
+            }
+        }
+
+        return res;
+    }
+
 }

DeepTrace/Data/TimeSeriesData.cs

@@ -1,4 +1,5 @@
 using PrometheusAPI;
+using System.Reflection.Emit;
 
 namespace DeepTrace.Data;
 
@@ -12,4 +13,24 @@ public class TimeSeriesDataSet
     public string Name { get; init; } = "Value";
     public string Color { get; init; } = "";
     public List<TimeSeries> Data { get; init; } = new List<TimeSeries>();
+
+    public string Label => MakeLabel(Name);
+
+    public static string MakeLabel(string s)
+    {
+        var pos = s.IndexOf("{");
+        if (pos > 0)
+            s = s[..pos];
+        pos = s.LastIndexOf("(");
+        if (pos > 0)
+            s = s[(pos + 1)..];
+        pos = s.LastIndexOf("[");
+        if (pos > 0)
+            s = s[..pos];
+        pos = s.LastIndexOf(")");
+        if (pos > 0)
+            s = s[..pos];
+
+        return s;
+    }
 }

DeepTrace/ML/Correlation.cs

@@ -0,0 +1,94 @@
+using DeepTrace.Data;
+
+namespace DeepTrace.ML;
+
+/// <summary>
+/// https://xamlbrewer.wordpress.com/2019/03/04/machine-learning-with-ml-net-in-uwp-feature-correlation-analysis/
+/// </summary>
+public static class Correlation
+{
+    /// <summary>
+    /// Computes the Pearson Product-Moment Correlation coefficient.
+    /// </summary>
+    /// <param name="dataA">Sample data A.</param>
+    /// <param name="dataB">Sample data B.</param>
+    /// <returns>The Pearson product-moment correlation coefficient.</returns>
+    /// <remarks>Original Source: https://github.com/mathnet/mathnet-numerics/blob/master/src/Numerics/Statistics/Correlation.cs </remarks>
+    public static float Pearson(IEnumerable<float> dataA, IEnumerable<float> dataB)
+    {
+        var n = 0;
+        var r = 0.0;
+
+        var meanA = 0d;
+        var meanB = 0d;
+        var varA = 0d;
+        var varB = 0d;
+
+        using (IEnumerator<float> ieA = dataA.GetEnumerator())
+        using (IEnumerator<float> ieB = dataB.GetEnumerator())
+        {
+            while (ieA.MoveNext())
+            {
+                if (!ieB.MoveNext())
+                {
+                    throw new ArgumentOutOfRangeException(nameof(dataB), "Array too short.");
+                }
+
+                var currentA    = ieA.Current;
+                var currentB    = ieB.Current;
+
+                var deltaA      = currentA - meanA;
+                var scaleDeltaA = deltaA / ++n;
+
+                var deltaB      = currentB - meanB;
+                var scaleDeltaB = deltaB / n;
+
+                meanA += scaleDeltaA;
+                meanB += scaleDeltaB;
+
+                varA  += scaleDeltaA * deltaA * (n - 1);
+                varB  += scaleDeltaB * deltaB * (n - 1);
+
+                r     += (deltaA * deltaB * (n - 1)) / n;
+            }
+
+            if (ieB.MoveNext())
+            {
+                throw new ArgumentOutOfRangeException(nameof(dataA), "Array too short.");
+            }
+        }
+
+        return (float)(r / Math.Sqrt(varA * varB));
+    }
+
+    public static float[,] Matrix(List<TimeSeriesDataSet> src)
+    {
+        var data   = src?.Select(x=> x.Data).ToList();
+        var len = data?.Count ?? 0;
+
+        if (data == null || len < 2)
+            return new float[0,0];
+
+        var matrix = new float[len, len];
+
+
+        // Populate diagram
+        for (int x = 0; x < len; ++x)
+        {
+            for (int y = 0; y < len - 1 - x; ++y)
+            {
+                var seriesA = data[x];
+                var seriesB = data[len - 1 - y];
+
+                var value = Pearson(seriesA.Select(x => x.Value), seriesB.Select(x => x.Value));
+
+                matrix[x,         y        ] = value;
+                matrix[len-1 - y, len-1 - x] = value;
+            }
+
+            matrix[x, x] = 1;
+        }
+
+        return matrix;
+    }
+}

DeepTrace/Pages/DataSources.razor

@@ -99,10 +99,23 @@
     <MudItem xs="12" sm="6" md="6" lg="9">
         <MudCard>
             <MudCardContent>
-                <div hidden="@IsChartShown"><MudProgressCircular Color="MudBlazor.Color.Default" /></div>
+                @*
-                <div hidden="@IsChartHidden">
+                <MudTabs Elevation="2" Rounded="true" ApplyEffectsToContainer="true" PanelClass="pa-6">
-                    <TimeSeriesChart Data="@DisplayData" @bind-MinDate=MinDate @bind-MaxDate=MaxDate />
+                    <MudTabPanel Text="Graph">
+                </MudTabPanel>
+                <MudTabPanel Text="Correlation">
+                </MudTabPanel>
+                </MudTabs>
+                *@
+                <div hidden="@IsChartShown">
+                    <MudProgressCircular Color="MudBlazor.Color.Default" />
                 </div>
+                <MudItem hidden="@IsChartHidden" xs="12" sm="6" md="6" lg="6">
+                    <TimeSeriesChart Data="@DisplayData" @bind-MinDate=MinDate @bind-MaxDate=MaxDate />
+                </MudItem>
+                <MudItem hidden="@IsChartHidden" xs="12" sm="6" md="6" lg="6">
+                    <CorrelationChart Data="@DisplayData" />
+                </MudItem>
             </MudCardContent>
         </MudCard>
     </MudItem>
@@ -111,6 +124,7 @@
 
 
 @code {
+
     [CascadingParameter]
     protected bool IsDarkMode { get; set; }
 
@@ -145,7 +159,7 @@
     };
     private MudDateRangePicker? _picker;
 
-    private bool IsChartHidden => DisplayData == null;
+    private bool IsChartHidden => DisplayData == null || DisplayData.Series.Count == 0;
     private bool IsChartShown => !IsChartHidden;
 
     private async Task<IEnumerable<string>> SearchForQuery(string value)

PrometheusAPI/TimeSeries.cs

@@ -1,6 +1,4 @@
 using Microsoft.ML.Data;
-using Newtonsoft.Json.Converters;
-using System.Text.Json.Nodes;
 using System.Text.Json.Serialization;
 
 namespace PrometheusAPI;

UnitTests/TimeSeriesTest.cs

@@ -1,3 +1,4 @@
+using DeepTrace.Data;
 using PrometheusAPI;
 using System.Text.Json;
 using System.Text.Json.Nodes;
@@ -17,6 +18,101 @@ public class TimeSeriesTest
         Assert.IsNotNull(ts);
 
         Assert.AreEqual(123.45, ts.Value, 0.0001);
-        Assert.AreEqual(DateTime.Parse("15/06/2023 12:00:06"), ts.TimeStamp);
+        Assert.AreEqual(DateTime.Parse("15/06/2023 12:00:06").ToUniversalTime(), ts.TimeStamp);
+    }
+
+    [TestMethod]
+    public void Normalize_Attributes()
+    {
+        List<TimeSeriesDataSet> src = CreateSimpleDataSet();
+
+        var res = DataSourceDefinition.Normalize(src, 5);
+        Assert.IsNotNull(res);
+        Assert.AreEqual(2, res.Count);
+        Assert.AreEqual("one", res[0].Name);
+        Assert.AreEqual("two", res[1].Name);
+
+
+        Assert.AreEqual(5, res[0].Data.Count);
+        Assert.AreEqual(5, res[1].Data.Count);
+    }
+
+    [TestMethod]
+    public void Normalize_Counts()
+    {
+        List<TimeSeriesDataSet> src = CreateSimpleDataSet();
+
+        var res = DataSourceDefinition.Normalize(src, 5);
+        Assert.IsNotNull(res);
+        Assert.AreEqual(2, res.Count);
+        Assert.AreEqual(5, res[0].Data.Count);
+        Assert.AreEqual(5, res[1].Data.Count);
+    }
+
+    [TestMethod]
+    public void Normalize_TimeStamps()
+    {
+        List<TimeSeriesDataSet> src = CreateSimpleDataSet();
+
+        var res = DataSourceDefinition.Normalize(src, 5);
+        Assert.IsNotNull(res);
+        Assert.AreEqual(2, res.Count);
+
+        Assert.AreEqual(res[0].Data.Count, res[1].Data.Count);
+
+        for (var i = 0; i < res[0].Data.Count; i++)
+            Assert.AreEqual(res[0].Data[i].TimeStamp, res[1].Data[i].TimeStamp);
+
+        Assert.AreEqual(src[0].Data[0].TimeStamp, res[0].Data[0].TimeStamp);
+        Assert.AreEqual(src[0].Data[src[0].Data.Count-1].TimeStamp, res[0].Data[res[0].Data.Count-1].TimeStamp);
+    }
+
+    [TestMethod]
+    public void Normalize_Values()
+    {
+        List<TimeSeriesDataSet> src = CreateSimpleDataSet();
+
+        var res = DataSourceDefinition.Normalize(src, 5);
+        Assert.IsNotNull(res);
+        Assert.AreEqual(2, res.Count);
+
+        Assert.AreEqual(res[0].Data.Count, res[1].Data.Count);
+
+        Assert.AreEqual(src[0].Data[0].Value                    , res[0].Data[0].Value                    , 0.001F, "start one");
+        Assert.AreEqual(src[0].Data[src[0].Data.Count - 1].Value, res[0].Data[res[0].Data.Count - 1].Value, 0.001F, "end one");
+
+        Assert.AreEqual(src[0].Data[0].Value,                     res[0].Data[0].Value,                     0.001F, "start two");
+        Assert.AreEqual(src[0].Data[src[0].Data.Count - 1].Value, res[0].Data[res[0].Data.Count - 1].Value, 0.001F, "end two");
+
+
+        Assert.AreEqual(1.5F                                    , res[0].Data[2].Value                    , 0.001F, "middle one");
+        Assert.AreEqual(1.5F                                    , res[1].Data[2].Value                    , 0.001F, "middle two");
+
+        for (var i = 0; i < res[0].Data.Count; i++)
+            Assert.AreEqual(res[0].Data[i].Value, res[1].Data[i].Value, 0.001F, $"Point {i}");
+    }
+
+    private static List<TimeSeriesDataSet> CreateSimpleDataSet()
+    {
+        return new List<TimeSeriesDataSet>
+        {
+            new TimeSeriesDataSet
+            {
+                Name = "one",
+                Data = new() {
+                    new TimeSeries(DateTime.Parse("2023-01-01 00:00:01"), 1.0F),
+                    new TimeSeries(DateTime.Parse("2023-01-01 00:00:10"), 2.0F),
+                }
+            },
+            new TimeSeriesDataSet
+            {
+                Name = "two",
+                Data = new() {
+                    new TimeSeries(DateTime.Parse("2023-01-01 00:00:01"), 1.0F),
+                    new TimeSeries(DateTime.Parse("2023-01-01 00:00:05"), 1.5F),
+                    new TimeSeries(DateTime.Parse("2023-01-01 00:00:10"), 2.0F),
+                }
+            }
+        };
     }
 }

UnitTests/UnitTests.csproj

@@ -17,6 +17,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\DeepTrace\DeepTrace.csproj" />
     <ProjectReference Include="..\PrometheusAPI\PrometheusAPI.csproj" />
   </ItemGroup>