DEEP-38 Initial work for configurable Measures

DeepTrace/Controllers/DownloadController.cs

@@ -8,9 +8,9 @@ namespace DeepTrace.Controllers
     [Route("api/[controller]")]
     public class DownloadController : Controller
     {
-        private readonly IModelDefinitionService _modelService;
+        private readonly IModelStorageService _modelService;
 
-        public DownloadController(IModelDefinitionService modelService)
+        public DownloadController(IModelStorageService modelService)
         {
             _modelService = modelService;
         }

DeepTrace/Data/ModelDefinition.cs

@@ -29,14 +29,14 @@ public class ModelDefinition
         {
             columnNames.AddRange(measureNames.Select(x => $"{item.Query}_{x}"));
         }
-
+        columnNames.Add("Name");
         return columnNames;
     }
 
     public string ToCsv()
     {
         var current = IntervalDefinitionList.First();
-        var headers = string.Join(",", GetColumnNames().Select(x=>$"\"{x}\"")) + ",Name";
+        var headers = string.Join(",", GetColumnNames().Select(x=>$"\"{x}\""));
 
 
         var writer = new StringBuilder();

DeepTrace/Data/TrainedModelDefinition.cs

@@ -0,0 +1,13 @@
+using MongoDB.Bson.Serialization.Attributes;
+using MongoDB.Bson;
+
+namespace DeepTrace.Data
+{
+    public class TrainedModelDefinition
+    {
+        [BsonId]
+        public ObjectId? Id { get; set; }
+        public string Name { get; set; }
+        public byte[] Value { get; set; } //base64
+    }
+}

DeepTrace/ML/IMeasure.cs

@@ -0,0 +1,11 @@
+using PrometheusAPI;
+
+namespace DeepTrace.ML
+{
+    public interface IMeasure
+    {
+        public string Name { get; }
+        void Reset();
+        float Calculate(IEnumerable<TimeSeries> data);
+    }
+}

DeepTrace/ML/MLHelpers.cs

@@ -34,7 +34,7 @@ public static class MLHelpers
 
         var columnNames = model.GetColumnNames();
         var columns     = columnNames
-            .Select((x,i) => new TextLoader.Column(x, DataKind.Double, i))
+            .Select((x,i) => new TextLoader.Column(x, DataKind.String, i))
             .ToArray()
             ;
 

DeepTrace/ML/MLProcessor.cs

@@ -13,12 +13,12 @@ namespace DeepTrace.ML
         private DataViewSchema? _schema;
         private ITransformer? _transformer;
 
-        private string Name { get; set; }
+        private string Name { get; set; } = "TestModel";
 
         public async Task Train(ModelDefinition modelDef)
         {
             var pipeline = _estimatorBuilder.BuildPipeline(_mlContext, modelDef);
-            var (data, filename) = await MLHelpers.Convert(_mlContext,modelDef);
+            var (data, filename) = await MLHelpers.Convert(_mlContext, modelDef);
             try
             {
                 _schema = data.Schema;

DeepTrace/ML/Measures.cs

@@ -0,0 +1,89 @@
+using PrometheusAPI;
+
+namespace DeepTrace.ML
+{
+    public class MeasureMin : IMeasure
+    {
+        public string Name => "Min";
+        public float Calculate(IEnumerable<TimeSeries> data) => 
+            data
+            .Where(x => x.Value != 0.0f)
+            .Min( x => x.Value )
+            ;
+
+        public void Reset() { }
+    }
+
+    public class MeasureMax : IMeasure
+    {
+        public string Name => "Max";
+        public float Calculate(IEnumerable<TimeSeries> data) => data.Max(x => x.Value);
+        public void Reset() { }
+    }
+
+    public class MeasureAvg : IMeasure
+    {
+        public string Name => "Avg";
+        public float Calculate(IEnumerable<TimeSeries> data) => data.Average(x => x.Value);
+        public void Reset() { }
+    }
+
+    /// <summary>
+    /// WARNING: Only works with fixed length interval
+    /// </summary>
+    public class MeasureSum : IMeasure
+    {
+        public string Name => "Sum";
+        public float Calculate(IEnumerable<TimeSeries> data) => data.Sum(x => x.Value);
+        public void Reset() { }
+    }
+
+    public class MeasureMedian : IMeasure
+    {
+        public string Name => "Median";
+
+        public float Calculate(IEnumerable<TimeSeries> data)
+            => MedianHelper.Median(data, x => x.Value);
+
+        public void Reset() { }
+
+    }
+
+    public class MeasureDiff<T> : IMeasure where T : IMeasure, new()
+    {
+        private T _measure = new();
+        public string Name => "Diff_"+_measure.Name;
+        
+        private float _prev = float.NaN;
+
+        public float Calculate(IEnumerable<TimeSeries> data)
+        {
+            var val = _measure.Calculate(data);
+            if (float.IsNaN(_prev))
+            {
+                _prev = val;
+                return 0.0f;
+            }
+
+            val = val - _prev;
+            _prev = val;
+            return val;
+        }
+
+        public void Reset() 
+        {
+            _measure.Reset(); 
+            _prev = float.NaN; 
+        }
+    }
+
+    public class MeasureDiffMin : MeasureDiff<MeasureMin> { }
+    public class MeasureDiffMax : MeasureDiff<MeasureMax> { }
+    public class MeasureDiffAvg : MeasureDiff<MeasureAvg> { }
+    /// <summary>
+    /// WARNING: Only works with fixed length interval
+    /// </summary>
+    public class MeasureDiffSum : MeasureDiff<MeasureSum> { }
+    public class MeasureDiffMedian : MeasureDiff<MeasureMedian> { }
+
+}

DeepTrace/ML/MedianHelper.cs

@@ -0,0 +1,80 @@
+namespace DeepTrace.ML;
+
+/// <summary>
+/// Calculate median.
+/// https://stackoverflow.com/questions/4140719/calculate-median-in-c-sharp
+/// </summary>
+public static class MedianHelper
+{
+    /// <summary>
+    /// Partitions the given list around a pivot element such that all elements on left of pivot are <= pivot
+    /// and the ones at thr right are > pivot. This method can be used for sorting, N-order statistics such as
+    /// as median finding algorithms.
+    /// Pivot is selected ranodmly if random number generator is supplied else its selected as last element in the list.
+    /// Reference: Introduction to Algorithms 3rd Edition, Corman et al, pp 171
+    /// </summary>
+    private static int Partition<T>(this IList<T> list, int start, int end, Random rnd = null) where T : IComparable<T>
+    {
+        if (rnd != null)
+            list.Swap(end, rnd.Next(start, end + 1));
+
+        var pivot = list[end];
+        var lastLow = start - 1;
+        for (var i = start; i < end; i++)
+        {
+            if (list[i].CompareTo(pivot) <= 0)
+                list.Swap(i, ++lastLow);
+        }
+        list.Swap(end, ++lastLow);
+        return lastLow;
+    }
+
+    /// <summary>
+    /// Returns Nth smallest element from the list. Here n starts from 0 so that n=0 returns minimum, n=1 returns 2nd smallest element etc.
+    /// Note: specified list would be mutated in the process.
+    /// Reference: Introduction to Algorithms 3rd Edition, Corman et al, pp 216
+    /// </summary>
+    public static T NthOrderStatistic<T>(this IList<T> list, int n, Random rnd = null) where T : IComparable<T>
+    {
+        return NthOrderStatistic(list, n, 0, list.Count - 1, rnd);
+    }
+    private static T NthOrderStatistic<T>(this IList<T> list, int n, int start, int end, Random rnd) where T : IComparable<T>
+    {
+        while (true)
+        {
+            var pivotIndex = list.Partition(start, end, rnd);
+            if (pivotIndex == n)
+                return list[pivotIndex];
+
+            if (n < pivotIndex)
+                end = pivotIndex - 1;
+            else
+                start = pivotIndex + 1;
+        }
+    }
+
+    public static void Swap<T>(this IList<T> list, int i, int j)
+    {
+        if (i == j)   //This check is not required but Partition function may make many calls so its for perf reason
+            return;
+        var temp = list[i];
+        list[i] = list[j];
+        list[j] = temp;
+    }
+
+    /// <summary>
+    /// Note: specified list would be mutated in the process.
+    /// </summary>
+    public static T Median<T>(IList<T> list) where T : IComparable<T>
+    {
+        return list.NthOrderStatistic((list.Count - 1) / 2);
+    }
+
+    public static TValue Median<T,TValue>(IEnumerable<T> sequence, Func<T, TValue> getValue)
+        where TValue : IComparable<TValue>
+    {
+        var list = sequence.Select(getValue).ToList();
+        var mid = (list.Count - 1) / 2;
+        return list.NthOrderStatistic(mid);
+    }
+}

DeepTrace/Pages/Training.razor

@@ -6,11 +6,13 @@
 @using DeepTrace.Controls;
 @using Microsoft.ML;
 @using PrometheusAPI;
+@using System.Text;
 
 @inject PrometheusClient Prometheus
 @inject IDialogService DialogService
 @inject IDataSourceStorageService StorageService
-@inject IModelDefinitionService ModelService
+@inject IModelStorageService ModelService
+@inject ITrainedModelStorageService TrainedModelService
 @inject IEstimatorBuilder EstimatorBuilder
 @inject NavigationManager NavManager
 @inject IJSRuntime Js
@@ -64,12 +66,24 @@
             <MudCardActions>
                 <MudButton ButtonType="ButtonType.Submit" Variant="Variant.Filled" Color="MudBlazor.Color.Primary" Class="ml-3" OnClick="@HandleAddTableContent" Disabled="@IsAddDisabled">Add</MudButton>
                 <MudButton ButtonType="ButtonType.Submit" Variant="Variant.Filled" Color="MudBlazor.Color.Primary" Class="ml-3" OnClick="@HandleRefresh" Disabled="@IsAddDisabled">Refresh</MudButton>
+                <MudFileUpload T="IBrowserFile" Accept=".csv" FilesChanged="@HandleImport" MaximumFileCount="1" Class="ml-3">
+                    <ButtonTemplate>
+                        <MudButton HtmlTag="label"
+                                   Variant="Variant.Filled"
+                                   Color="MudBlazor.Color.Primary"
+                                   StartIcon="@Icons.Material.Filled.CloudUpload"
+                                   for="@context">
+                            Import
+                        </MudButton>
+                    </ButtonTemplate>
+                </MudFileUpload>
                 <MudButton ButtonType="ButtonType.Submit" Variant="Variant.Filled" Color="MudBlazor.Color.Primary" Class="ml-3" OnClick="@HandleExport" Disabled="@IsAddDisabled">Export</MudButton>
                 <MudSpacer/>
 
                 <MudButton ButtonType="ButtonType.Submit" Variant="Variant.Filled" Color="MudBlazor.Color.Primary" Class="ml-3" OnClick="@HandleTrain">Train</MudButton>
             </MudCardActions>
             
+
             <MudTable 
                 Items="@_modelForm!.CurrentModel.IntervalDefinitionList" 
                 Hover="true" 
@@ -233,6 +247,9 @@
 
         var sources = await StorageService.Load();
         var models = await ModelService.Load();
+        var trainedModels = await TrainedModelService.Load();
+        IList<IBrowserFile> files = new List<IBrowserFile>();
+
         if (sources.Count > 0)
             _dataSources = sources;
         if (models.Count > 0)
@@ -398,6 +415,18 @@
         // await InvokeAsync(StateHasChanged);
     }
 
+    //Doesn't work
+    private async Task HandleImport(IBrowserFile file)
+    {
+        var result = new StringBuilder();
+        var reader = new StreamReader(file.OpenReadStream(file.Size));
+
+        while (reader.Peek() >= 0)
+            result.AppendLine(await reader.ReadLineAsync());
+        
+        result.ToString();
+    }
+
     private async Task HandleExport()
     {
         await Js.InvokeVoidAsync("open", $"{NavManager.BaseUri}api/download/mldata/{Uri.EscapeDataString(_modelForm.CurrentModel.Name)}", "_blank");
@@ -430,6 +459,12 @@
         var bytes = mlProcessor.Export();
         
         //save to Mongo
+        var trainedModel = new TrainedModelDefinition
+        {
+            Name = "TrainedModel",
+            Value = bytes
+        };
+        await TrainedModelService.Store(trainedModel);
     }
 
     

DeepTrace/Program.cs

@@ -16,7 +16,8 @@ builder.Services.AddHttpClient<PrometheusClient>(c => c.BaseAddress = new UriBui
 builder.Services
     .AddSingleton<IMongoClient>( s => new MongoClient(builder.Configuration.GetValue<string>("Connections:MongoDb") ))
     .AddSingleton<IDataSourceStorageService, DataSourceStorageService>()
-    .AddSingleton<IModelDefinitionService, ModelDefinitionService>()
+    .AddSingleton<IModelStorageService, ModelStorageService>()
+    .AddSingleton<ITrainedModelStorageService, TrainedModelStorageService>()
     .AddSingleton<IEstimatorBuilder, EstimatorBuilder>()
     ;
 

DeepTrace/Services/IModelStorageService.cs

@@ -6,7 +6,7 @@ using System.Text;
 namespace DeepTrace.Services
 {
 
-    public interface IModelDefinitionService
+    public interface IModelStorageService
     {
         Task Delete(ModelDefinition source, bool ignoreNotStored = false);
         Task<List<ModelDefinition>> Load();

DeepTrace/Services/ITrainedModelStorageService.cs

@@ -0,0 +1,11 @@
+using DeepTrace.Data;
+
+namespace DeepTrace.Services
+{
+    public interface ITrainedModelStorageService
+    {
+        Task Delete(TrainedModelDefinition source, bool ignoreNotStored = false);
+        Task<List<TrainedModelDefinition>> Load();
+        Task Store(TrainedModelDefinition source);
+    }
+}

DeepTrace/Services/ModelStorageService.cs

@@ -4,7 +4,7 @@ using MongoDB.Driver;
 
 namespace DeepTrace.Services
 {
-    public class ModelDefinitionService : IModelDefinitionService
+    public class ModelStorageService : IModelStorageService
     {
 
         private const string MongoDBDatabaseName = "DeepTrace";
@@ -12,7 +12,7 @@ namespace DeepTrace.Services
 
         private readonly IMongoClient _client;
 
-        public ModelDefinitionService(IMongoClient client)
+        public ModelStorageService(IMongoClient client)
         {
             _client = client;
         }
@@ -51,7 +51,7 @@ namespace DeepTrace.Services
             }
 
             var db = _client.GetDatabase(MongoDBDatabaseName);
-            var collection = db.GetCollection<DataSourceStorage>(MongoDBCollection);
+            var collection = db.GetCollection<ModelDefinition>(MongoDBCollection);
 
             await collection.DeleteOneAsync(filter: new BsonDocument("_id", source.Id));
         }

DeepTrace/Services/TrainedModelStorageService.cs

@@ -0,0 +1,58 @@
+using DeepTrace.Data;
+using MongoDB.Bson;
+using MongoDB.Driver;
+
+namespace DeepTrace.Services
+{
+    public class TrainedModelStorageService: ITrainedModelStorageService
+    {
+        private const string MongoDBDatabaseName = "DeepTrace";
+        private const string MongoDBCollection = "TrainedModels";
+
+        private readonly IMongoClient _client;
+
+        public TrainedModelStorageService(IMongoClient client)
+        {
+            _client = client;
+        }
+
+        public async Task<List<TrainedModelDefinition>> Load()
+        {
+            var db = _client.GetDatabase(MongoDBDatabaseName);
+            var collection = db.GetCollection<TrainedModelDefinition>(MongoDBCollection);
+
+            var res = await (await collection.FindAsync("{}")).ToListAsync();
+            return res;
+        }
+        public async Task Store(TrainedModelDefinition source)
+        {
+            var db = _client.GetDatabase(MongoDBDatabaseName);
+            var collection = db.GetCollection<TrainedModelDefinition>(MongoDBCollection);
+
+            if (source.Id == null)
+                source.Id = ObjectId.GenerateNewId();
+
+            // use upsert (insert or update) to automatically handle subsequent updates
+            await collection.ReplaceOneAsync(
+                filter: new BsonDocument("_id", source.Id),
+                options: new ReplaceOptions { IsUpsert = true },
+                replacement: source
+                );
+        }
+
+        public async Task Delete(TrainedModelDefinition source, bool ignoreNotStored = false)
+        {
+            if (source.Id == null)
+            {
+                if (!ignoreNotStored)
+                    throw new InvalidDataException("Source was not stored yet. There is nothing to delete");
+                return;
+            }
+
+            var db = _client.GetDatabase(MongoDBDatabaseName);
+            var collection = db.GetCollection<TrainedModelDefinition>(MongoDBCollection);
+
+            await collection.DeleteOneAsync(filter: new BsonDocument("_id", source.Id));
+        }
+    }
+}