From 50fe5b03cac2a97570c895672a0265683fa2504d Mon Sep 17 00:00:00 2001 From: Andrey Shabarshov Date: Tue, 11 Jul 2023 16:32:55 +0100 Subject: [PATCH] DEEP-30 Export to CSV added --- DeepTrace/Controllers/DownloadController.cs | 62 +++++++++++++++++++++ DeepTrace/DeepTrace.csproj | 4 ++ DeepTrace/ML/MLHelpers.cs | 24 ++++++++ DeepTrace/ML/SpikeDetector.cs | 24 +++++--- DeepTrace/Pages/Training.razor | 11 +++- DeepTrace/Program.cs | 2 +- 6 files changed, 116 insertions(+), 11 deletions(-) create mode 100644 DeepTrace/Controllers/DownloadController.cs create mode 100644 DeepTrace/ML/MLHelpers.cs diff --git a/DeepTrace/Controllers/DownloadController.cs b/DeepTrace/Controllers/DownloadController.cs new file mode 100644 index 0000000..bd36fa4 --- /dev/null +++ b/DeepTrace/Controllers/DownloadController.cs @@ -0,0 +1,62 @@ +using DeepTrace.Services; +using Microsoft.AspNetCore.Mvc; +using System.Text; + +namespace DeepTrace.Controllers +{ + [ApiController] + [Route("api/[controller]")] + public class DownloadController : Controller + { + private readonly IModelStorageService _modelService; + + public DownloadController(IModelStorageService modelService) + { + _modelService = modelService; + } + + [HttpGet("mldata/{modelName}")] + public async Task GetMLDataCsv([FromRoute] string modelName) + { + var modelStorage = await _modelService.Load(); + var model = modelStorage.FirstOrDefault(x=>x.Name==modelName) ?? throw new ApplicationException($"Model {modelName} not found"); + var previousIntervals = model.IntervalDefinitionList; + + var current = previousIntervals.First(); + var headers = string.Join(",", current.Data.Select((x, i) => $"Q{i + 1}min,Q{i + 1}max,Q{i + 1}avg,Q{i + 1}mean")); + + + var writer = new StringBuilder(); + writer.AppendLine(headers); + + foreach (var currentInterval in previousIntervals) + { + var data = ""; + for (var i = 0; i < currentInterval.Data.Count; i++) + { + + var queryData = currentInterval.Data[i]; + var min = queryData.Data.Min(x => x.Value); + var max = queryData.Data.Max(x => x.Value); + var avg = queryData.Data.Average(x => x.Value); + var mean = queryData.Data.Sum(x => x.Value) / queryData.Data.Count; + + if (i == currentInterval.Data.Count - 1) + { + data += min + "," + max + "," + avg + "," + mean; + } + else + { + data += min + "," + max + "," + avg + "," + mean + ","; + } + + } + writer.AppendLine(data); + } + return new(Encoding.UTF8.GetBytes(writer.ToString()),"text/csv") + { + FileDownloadName = modelName+".csv" + }; + } + } +} diff --git a/DeepTrace/DeepTrace.csproj b/DeepTrace/DeepTrace.csproj index eec8e4b..4c9c1ba 100644 --- a/DeepTrace/DeepTrace.csproj +++ b/DeepTrace/DeepTrace.csproj @@ -18,4 +18,8 @@ + + + + diff --git a/DeepTrace/ML/MLHelpers.cs b/DeepTrace/ML/MLHelpers.cs new file mode 100644 index 0000000..a22af46 --- /dev/null +++ b/DeepTrace/ML/MLHelpers.cs @@ -0,0 +1,24 @@ +using Microsoft.ML; + +namespace DeepTrace.ML; + +public record ModelRecord(MLContext Context, DataViewSchema Schema, ITransformer Transformer); + +public static class MLHelpers +{ + public static byte[] ExportSingleModel( ModelRecord model) + { + using var mem = new MemoryStream(); + model.Context.Model.Save(model.Transformer, model.Schema, mem); + return mem.ToArray(); + } + + public static ModelRecord ImportSingleModel(byte[] data) + { + var mem = new MemoryStream(data); + var mlContext = new MLContext(); + var transformer = mlContext.Model.Load(mem, out var schema); + + return new (mlContext, schema, transformer); + } +} diff --git a/DeepTrace/ML/SpikeDetector.cs b/DeepTrace/ML/SpikeDetector.cs index 25314c4..743e886 100644 --- a/DeepTrace/ML/SpikeDetector.cs +++ b/DeepTrace/ML/SpikeDetector.cs @@ -4,12 +4,13 @@ using Microsoft.ML.Data; using PrometheusAPI; using System.Data; using System.Linq; +using System.Xml.Linq; namespace DeepTrace.ML { public class SpikeDetector : IMLProcessor { - private readonly Dictionary _model = new(); + private readonly Dictionary _model = new(); public void Fit(ModelDefinition modelDef, DataSourceDefinition dataSourceDef) { @@ -44,7 +45,11 @@ namespace DeepTrace.ML foreach ( var (name, model) in _model) { mem.WriteString(name); - model.Context.Model.Save(model.Transformer, model.Schema, mem); + + var bytes = MLHelpers.ExportSingleModel(model); + + mem.WriteInt(bytes.Length); + mem.Write(bytes); } return mem.ToArray(); @@ -62,11 +67,14 @@ namespace DeepTrace.ML for ( var i = 0; i < count; i++ ) { var name = mem.ReadString(); + var size = mem.ReadInt(); + var bytes = new byte[size]; - var mlContext = new MLContext(); - var transformer = mlContext.Model.Load(mem, out var schema); + mem.Read(bytes, 0, bytes.Length); + + var model = MLHelpers.ImportSingleModel(bytes); - _model[name] = (mlContext, schema, transformer); + _model[name] = model; } } @@ -77,15 +85,13 @@ namespace DeepTrace.ML // -------------------------- internals - - class SpikePrediction { [VectorType(3)] public double[] Prediction { get; set; } = new double[3]; } - private static (MLContext Context, DataViewSchema Schema, ITransformer Transformer) FitOne(List dataSet) + private static ModelRecord FitOne(List dataSet) { var mlContext = new MLContext(); var dataView = mlContext.Data.LoadFromEnumerable(dataSet); @@ -96,7 +102,7 @@ namespace DeepTrace.ML var iidSpikeEstimator = mlContext.Transforms.DetectIidSpike(outputColumnName,inputColumnName, 95.0d, dataSet.Count); var transformer = iidSpikeEstimator.Fit(dataView); - return (mlContext, dataView.Schema, transformer); + return new (mlContext, dataView.Schema, transformer); } } } diff --git a/DeepTrace/Pages/Training.razor b/DeepTrace/Pages/Training.razor index e103ca2..29991ec 100644 --- a/DeepTrace/Pages/Training.razor +++ b/DeepTrace/Pages/Training.razor @@ -9,6 +9,8 @@ @inject IDialogService DialogService @inject IDataSourceStorageService StorageService @inject IModelStorageService ModelService +@inject NavigationManager NavManager +@inject IJSRuntime Js Training @@ -59,6 +61,7 @@ Add Refresh + Export Train @@ -143,7 +146,7 @@ _currentModel = value; _self.InvokeAsync(_self.HandleShowQuery); _self.StateHasChanged(); - + } } @@ -348,6 +351,12 @@ // await InvokeAsync(StateHasChanged); } + private async Task HandleExport() + { + await Js.InvokeVoidAsync("open", $"{NavManager.BaseUri}api/download/mldata/{Uri.EscapeDataString(_modelForm.CurrentModel.Name)}", "_blank"); + + } + private async Task HandleDeleteTableContent(IntervalDefinition interval) { _modelForm!.CurrentModel.IntervalDefinitionList.Remove(interval); diff --git a/DeepTrace/Program.cs b/DeepTrace/Program.cs index 5bb352f..9a92acd 100644 --- a/DeepTrace/Program.cs +++ b/DeepTrace/Program.cs @@ -29,7 +29,7 @@ if (!app.Environment.IsDevelopment()) app.UseStaticFiles(); app.UseRouting(); - +app.MapControllerRoute("default","{controller}/{action}"); app.MapBlazorHub(); app.MapFallbackToPage("/_Host");