DEEP-36 Estimator builder created. Minor UI changes

DeepTrace/DeepTrace.csproj

@@ -1,21 +1,30 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
-
   <PropertyGroup>
     <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
-
   <ItemGroup>
     <PackageReference Include="Blazor-ApexCharts" Version="0.9.21-beta" />
     <PackageReference Include="Microsoft.ML" Version="2.0.1" />
+    <PackageReference Include="Microsoft.ML.FastTree" Version="1.7.1" />
     <PackageReference Include="Microsoft.ML.TimeSeries" Version="2.0.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.20.0" />
     <PackageReference Include="MudBlazor" Version="6.4.1" />
   </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\PrometheusAPI\PrometheusAPI.csproj" />
   </ItemGroup>
-
+  <ItemGroup Label="MLModel1">
+    <None Include="MLModel1.consumption.cs">
+      <DependentUpon>MLModel1.mbconfig</DependentUpon>
+    </None>
+    <None Include="MLModel1.training.cs">
+      <DependentUpon>MLModel1.mbconfig</DependentUpon>
+    </None>
+    <None Include="MLModel1.zip">
+      <DependentUpon>MLModel1.mbconfig</DependentUpon>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
 </Project>

DeepTrace/ML/EstimatorBuilder.cs

@@ -0,0 +1,46 @@
+using DeepTrace.Data;
+using Microsoft.ML;
+using Microsoft.ML.Trainers;
+
+namespace DeepTrace.ML
+{
+    public class EstimatorBuilder : IEstimatorBuilder
+    {
+        public IEstimator<ITransformer> BuildPipeline(MLContext mlContext, ModelDefinition model)
+        {
+            IEstimator<ITransformer>? pipeline = null;
+            var ds = model.DataSource;
+
+            var measureNames = new[] { "min", "max", "avg", "mean" };
+            var columnNames = new List<string>();
+            foreach (var item in ds.Queries)
+            {
+                var estimators = measureNames.Select(x => mlContext.Transforms.Text.FeaturizeText(inputColumnName: $"{item.Query}_{x}", outputColumnName: $"{item.Query}_{x}"));
+                columnNames.AddRange(measureNames.Select(x => $"{item.Query}_{x}"));
+
+                foreach (var e in estimators)
+                {
+                    if (pipeline == null)
+                    {
+                        pipeline = e;
+                    }
+                    else
+                    {
+                        pipeline = pipeline.Append(e);
+                    }
+                }
+
+            }
+
+            pipeline = pipeline!
+                .Append(mlContext.Transforms.Concatenate(@"Features", columnNames.ToArray()))
+                .Append(mlContext.Transforms.Conversion.MapValueToKey(outputColumnName: @"Name", inputColumnName: @"Name"))
+                .Append(mlContext.Transforms.NormalizeMinMax(@"Features", @"Features"))
+                .Append(mlContext.MulticlassClassification.Trainers.OneVersusAll(binaryEstimator: mlContext.BinaryClassification.Trainers.LbfgsLogisticRegression(new LbfgsLogisticRegressionBinaryTrainer.Options() { L1Regularization = 1F, L2Regularization = 1F, LabelColumnName = @"Name", FeatureColumnName = @"Features" }), labelColumnName: @"Name"))
+                .Append(mlContext.Transforms.Conversion.MapKeyToValue(outputColumnName: @"PredictedLabel", inputColumnName: @"PredictedLabel"));
+
+            return pipeline;
+
+        }
+    }
+}

DeepTrace/ML/IEstimatorBuilder.cs

@@ -0,0 +1,10 @@
+using DeepTrace.Data;
+using Microsoft.ML;
+
+namespace DeepTrace.ML
+{
+    public interface IEstimatorBuilder
+    {
+        IEstimator<ITransformer> BuildPipeline(MLContext mlContext, ModelDefinition model);
+    }
+}

DeepTrace/Pages/DataSources.razor

@@ -58,7 +58,8 @@
                             int pos = i;
 
                             <MudItem xs="10">
-                                <MudTextField Label="Query" @bind-Value="_queryForm.Source.Queries[pos].Query" Variant="Variant.Text" InputType="InputType.Search" Lines="2" />
+                                @*<MudTextField Label="Query" @bind-Value="_queryForm.Source.Queries[pos].Query" Variant="Variant.Text" InputType="InputType.Search" Lines="2" />*@
+                                <MudAutocomplete Label="Query" @bind-Value="_queryForm.Source.Queries[pos].Query" Lines="1" Variant="Variant.Text" SearchFunc="@SearchForQuery"></MudAutocomplete>
                             </MudItem>
                             <MudItem xs="1">
                                 <MudIconButton Icon="@Icons.Material.Outlined.Add" Variant="Variant.Outlined" aria-label="add" OnClick="@(() => AddQuery(pos))" />
@@ -147,6 +148,14 @@
     private bool IsChartHidden => DisplayData == null;
     private bool IsChartShown => !IsChartHidden;
 
+    private async Task<IEnumerable<string>> SearchForQuery(string value)
+    {
+        var metricsNames = await Prometheus.GetMetricsNames();
+        if (string.IsNullOrEmpty(value))
+            return new string[0];
+        return metricsNames.Where(x => x.Contains(value, StringComparison.InvariantCultureIgnoreCase));
+    }
+
     private DateTime? MinDate
     {
         get => _queryForm.Dates.Start?.Date + _queryForm.TimeStart;

DeepTrace/Pages/Training.razor

@@ -1,14 +1,17 @@
 @page "/training"
 @using DeepTrace.Data;
+@using DeepTrace.ML;
 @using DeepTrace.Services;
 @using System.ComponentModel.DataAnnotations;
 @using DeepTrace.Controls;
+@using Microsoft.ML;
 @using PrometheusAPI;
 
 @inject PrometheusClient Prometheus
 @inject IDialogService DialogService
 @inject IDataSourceStorageService StorageService
 @inject IModelStorageService ModelService
+@inject IEstimatorBuilder EstimatorBuilder
 @inject NavigationManager NavManager
 @inject IJSRuntime Js
 
@@ -71,6 +74,7 @@
                 Items="@_modelForm!.CurrentModel.IntervalDefinitionList" 
                 Hover="true" 
                 FixedHeader="@fixed_header"
+                Height="400px"
                 CanCancelEdit="@canCancelEdit" 
                 RowEditPreview="@BackupInterval"
                 RowEditCancel="ResetItemToOriginalValues"
@@ -421,6 +425,11 @@
 
     private void HandleTrain()
     {
+        var mlContext = new MLContext();
+        var pipeline = EstimatorBuilder.BuildPipeline(mlContext, _modelForm!.CurrentModel);
+
         
     }
+
+    
 }

DeepTrace/Program.cs

@@ -2,6 +2,7 @@ using MudBlazor.Services;
 using PrometheusAPI;
 using MongoDB.Driver;
 using DeepTrace.Services;
+using DeepTrace.ML;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -16,6 +17,7 @@ builder.Services
     .AddSingleton<IMongoClient>( s => new MongoClient(builder.Configuration.GetValue<string>("Connections:MongoDb") ))
     .AddSingleton<IDataSourceStorageService, DataSourceStorageService>()
     .AddSingleton<IModelStorageService, ModelStorageService>()
+    .AddSingleton<IEstimatorBuilder, EstimatorBuilder>()
     ;
 
 var app = builder.Build();