From 2c49f89f26bf5a3cf301af22d6861f706ec8ddb7 Mon Sep 17 00:00:00 2001 From: Andrey Shabarshov Date: Fri, 21 Jul 2023 17:17:42 +0100 Subject: [PATCH] DEEP-36 Estimator builder created. Minor UI changes --- DeepTrace/DeepTrace.csproj | 19 +++++++++---- DeepTrace/ML/EstimatorBuilder.cs | 46 +++++++++++++++++++++++++++++++ DeepTrace/ML/IEstimatorBuilder.cs | 10 +++++++ DeepTrace/Pages/DataSources.razor | 11 +++++++- DeepTrace/Pages/Training.razor | 11 +++++++- DeepTrace/Program.cs | 2 ++ 6 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 DeepTrace/ML/EstimatorBuilder.cs create mode 100644 DeepTrace/ML/IEstimatorBuilder.cs diff --git a/DeepTrace/DeepTrace.csproj b/DeepTrace/DeepTrace.csproj index eec8e4b..c313b24 100644 --- a/DeepTrace/DeepTrace.csproj +++ b/DeepTrace/DeepTrace.csproj @@ -1,21 +1,30 @@  - net7.0 enable enable - + - - - + + + MLModel1.mbconfig + + + MLModel1.mbconfig + + + MLModel1.mbconfig + PreserveNewest + + + \ No newline at end of file diff --git a/DeepTrace/ML/EstimatorBuilder.cs b/DeepTrace/ML/EstimatorBuilder.cs new file mode 100644 index 0000000..87c09af --- /dev/null +++ b/DeepTrace/ML/EstimatorBuilder.cs @@ -0,0 +1,46 @@ +using DeepTrace.Data; +using Microsoft.ML; +using Microsoft.ML.Trainers; + +namespace DeepTrace.ML +{ + public class EstimatorBuilder : IEstimatorBuilder + { + public IEstimator BuildPipeline(MLContext mlContext, ModelDefinition model) + { + IEstimator? pipeline = null; + var ds = model.DataSource; + + var measureNames = new[] { "min", "max", "avg", "mean" }; + var columnNames = new List(); + foreach (var item in ds.Queries) + { + var estimators = measureNames.Select(x => mlContext.Transforms.Text.FeaturizeText(inputColumnName: $"{item.Query}_{x}", outputColumnName: $"{item.Query}_{x}")); + columnNames.AddRange(measureNames.Select(x => $"{item.Query}_{x}")); + + foreach (var e in estimators) + { + if (pipeline == null) + { + pipeline = e; + } + else + { + pipeline = pipeline.Append(e); + } + } + + } + + pipeline = pipeline! + .Append(mlContext.Transforms.Concatenate(@"Features", columnNames.ToArray())) + .Append(mlContext.Transforms.Conversion.MapValueToKey(outputColumnName: @"Name", inputColumnName: @"Name")) + .Append(mlContext.Transforms.NormalizeMinMax(@"Features", @"Features")) + .Append(mlContext.MulticlassClassification.Trainers.OneVersusAll(binaryEstimator: mlContext.BinaryClassification.Trainers.LbfgsLogisticRegression(new LbfgsLogisticRegressionBinaryTrainer.Options() { L1Regularization = 1F, L2Regularization = 1F, LabelColumnName = @"Name", FeatureColumnName = @"Features" }), labelColumnName: @"Name")) + .Append(mlContext.Transforms.Conversion.MapKeyToValue(outputColumnName: @"PredictedLabel", inputColumnName: @"PredictedLabel")); + + return pipeline; + + } + } +} diff --git a/DeepTrace/ML/IEstimatorBuilder.cs b/DeepTrace/ML/IEstimatorBuilder.cs new file mode 100644 index 0000000..5e5c960 --- /dev/null +++ b/DeepTrace/ML/IEstimatorBuilder.cs @@ -0,0 +1,10 @@ +using DeepTrace.Data; +using Microsoft.ML; + +namespace DeepTrace.ML +{ + public interface IEstimatorBuilder + { + IEstimator BuildPipeline(MLContext mlContext, ModelDefinition model); + } +} \ No newline at end of file diff --git a/DeepTrace/Pages/DataSources.razor b/DeepTrace/Pages/DataSources.razor index 812e4b6..f30cebe 100644 --- a/DeepTrace/Pages/DataSources.razor +++ b/DeepTrace/Pages/DataSources.razor @@ -58,7 +58,8 @@ int pos = i; - + @**@ + @@ -147,6 +148,14 @@ private bool IsChartHidden => DisplayData == null; private bool IsChartShown => !IsChartHidden; + private async Task> SearchForQuery(string value) + { + var metricsNames = await Prometheus.GetMetricsNames(); + if (string.IsNullOrEmpty(value)) + return new string[0]; + return metricsNames.Where(x => x.Contains(value, StringComparison.InvariantCultureIgnoreCase)); + } + private DateTime? MinDate { get => _queryForm.Dates.Start?.Date + _queryForm.TimeStart; diff --git a/DeepTrace/Pages/Training.razor b/DeepTrace/Pages/Training.razor index 762c372..c6d43c6 100644 --- a/DeepTrace/Pages/Training.razor +++ b/DeepTrace/Pages/Training.razor @@ -1,14 +1,17 @@ @page "/training" @using DeepTrace.Data; +@using DeepTrace.ML; @using DeepTrace.Services; @using System.ComponentModel.DataAnnotations; @using DeepTrace.Controls; +@using Microsoft.ML; @using PrometheusAPI; @inject PrometheusClient Prometheus @inject IDialogService DialogService @inject IDataSourceStorageService StorageService @inject IModelStorageService ModelService +@inject IEstimatorBuilder EstimatorBuilder @inject NavigationManager NavManager @inject IJSRuntime Js @@ -71,6 +74,7 @@ Items="@_modelForm!.CurrentModel.IntervalDefinitionList" Hover="true" FixedHeader="@fixed_header" + Height="400px" CanCancelEdit="@canCancelEdit" RowEditPreview="@BackupInterval" RowEditCancel="ResetItemToOriginalValues" @@ -415,12 +419,17 @@ }; var parameters = new DialogParameters(); parameters.Add("Text", text); - + DialogService.Show("Error", parameters, options); } private void HandleTrain() { + var mlContext = new MLContext(); + var pipeline = EstimatorBuilder.BuildPipeline(mlContext, _modelForm!.CurrentModel); + } + + } diff --git a/DeepTrace/Program.cs b/DeepTrace/Program.cs index 9a92acd..74a11ae 100644 --- a/DeepTrace/Program.cs +++ b/DeepTrace/Program.cs @@ -2,6 +2,7 @@ using MudBlazor.Services; using PrometheusAPI; using MongoDB.Driver; using DeepTrace.Services; +using DeepTrace.ML; var builder = WebApplication.CreateBuilder(args); @@ -16,6 +17,7 @@ builder.Services .AddSingleton( s => new MongoClient(builder.Configuration.GetValue("Connections:MongoDb") )) .AddSingleton() .AddSingleton() + .AddSingleton() ; var app = builder.Build();