mirror of
https://github.com/NecroticBamboo/DeepTrace.git
synced 2025-12-21 11:21:51 +00:00
DEEP-14 Interval import implemented. Training dialog added. Logging added
This commit is contained in:
parent
0b26c25620
commit
facecd5ed7
@ -3,18 +3,32 @@
|
|||||||
@Text
|
@Text
|
||||||
</DialogContent>
|
</DialogContent>
|
||||||
<DialogActions>
|
<DialogActions>
|
||||||
@if (AllowCancel)
|
@if( IsYesNoCancel )
|
||||||
|
{
|
||||||
|
<MudButton Color="MudBlazor.Color.Primary" OnClick="Yes">Yes</MudButton>
|
||||||
|
<MudButton OnClick="No">No</MudButton>
|
||||||
|
<MudButton OnClick="Cancel">Cancel</MudButton>
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (AllowCancel)
|
||||||
{
|
{
|
||||||
<MudButton OnClick="Cancel">Cancel</MudButton>
|
<MudButton OnClick="Cancel">Cancel</MudButton>
|
||||||
}
|
}
|
||||||
<MudButton Color="MudBlazor.Color.Primary" OnClick="Submit">Ok</MudButton>
|
<MudButton Color="MudBlazor.Color.Primary" OnClick="Submit">Ok</MudButton>
|
||||||
|
}
|
||||||
</DialogActions>
|
</DialogActions>
|
||||||
</MudDialog>
|
</MudDialog>
|
||||||
@code {
|
@code {
|
||||||
[CascadingParameter] MudDialogInstance? MudDialog { get; set; }
|
[CascadingParameter] MudDialogInstance? MudDialog { get; set; }
|
||||||
[Parameter] public bool AllowCancel { get; set; }
|
[Parameter] public bool AllowCancel { get; set; }
|
||||||
[Parameter] public string Text { get; set; } = "";
|
[Parameter] public string Text { get; set; } = "";
|
||||||
|
[Parameter] public bool IsYesNoCancel { get; set; } = false;
|
||||||
|
|
||||||
void Submit() => MudDialog?.Close(DialogResult.Ok(true));
|
void Submit() => MudDialog?.Close(DialogResult.Ok(true));
|
||||||
|
|
||||||
void Cancel() => MudDialog?.Cancel();
|
void Cancel() => MudDialog?.Cancel();
|
||||||
|
|
||||||
|
void Yes() => MudDialog?.Close(DialogResult.Ok(true));
|
||||||
|
void No() => MudDialog?.Close(DialogResult.Ok(false));
|
||||||
}
|
}
|
||||||
51
DeepTrace/Controls/TrainingDialog.razor
Normal file
51
DeepTrace/Controls/TrainingDialog.razor
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
@using DeepTrace.Data;
|
||||||
|
@using DeepTrace.ML;
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.dialog-content{
|
||||||
|
min-width: 1000px;
|
||||||
|
width: fit-content;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
|
||||||
|
<MudDialog Class="dialog-content">
|
||||||
|
<DialogContent>
|
||||||
|
|
||||||
|
<h4>@Text</h4>
|
||||||
|
<MudTextField T="string" ReadOnly="true" Text="@_progressText"></MudTextField>
|
||||||
|
|
||||||
|
|
||||||
|
</DialogContent>
|
||||||
|
<DialogActions>
|
||||||
|
<MudButton Color="MudBlazor.Color.Primary" OnClick="Submit" Disabled="@_isTraining">Ok</MudButton>
|
||||||
|
</DialogActions>
|
||||||
|
</MudDialog>
|
||||||
|
@code {
|
||||||
|
[CascadingParameter] MudDialogInstance? MudDialog { get; set; }
|
||||||
|
[Parameter] public MLProcessor? Processor { get; set; }
|
||||||
|
[Parameter] public ModelDefinition? Model { get; set; }
|
||||||
|
[Parameter] public string Text { get; set; } = "";
|
||||||
|
|
||||||
|
private string _progressText = "";
|
||||||
|
private bool _isTraining = true;
|
||||||
|
|
||||||
|
void Submit() => MudDialog?.Close(DialogResult.Ok(true));
|
||||||
|
|
||||||
|
protected override async Task OnAfterRenderAsync(bool firstRender)
|
||||||
|
{
|
||||||
|
if (!firstRender || Processor==null || Model==null)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await Processor.Train(Model, UpdateProgress);
|
||||||
|
_isTraining = false;
|
||||||
|
await InvokeAsync(StateHasChanged);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async void UpdateProgress(string message)
|
||||||
|
{
|
||||||
|
_progressText = message;
|
||||||
|
await InvokeAsync(StateHasChanged);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -16,7 +16,7 @@
|
|||||||
|
|
||||||
public string Name { get; set; } = string.Empty;
|
public string Name { get; set; } = string.Empty;
|
||||||
|
|
||||||
public List<TimeSeriesDataSet> Data { get; set; }
|
public List<TimeSeriesDataSet> Data { get; set; } = new();
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -191,50 +191,50 @@
|
|||||||
"Version": 0,
|
"Version": 0,
|
||||||
"Type": "Trial",
|
"Type": "Trial",
|
||||||
"TrainerName": "SdcaMaximumEntropyMulti",
|
"TrainerName": "SdcaMaximumEntropyMulti",
|
||||||
"Score": 0.87460317460317472,
|
"Score": 0.57954545454545447,
|
||||||
"RuntimeInSeconds": 5.47599983215332
|
"RuntimeInSeconds": 5.3579998016357422
|
||||||
},
|
|
||||||
{
|
|
||||||
"Version": 0,
|
|
||||||
"Type": "Trial",
|
|
||||||
"TrainerName": "FastForestOva",
|
|
||||||
"Score": 0.91460317460317475,
|
|
||||||
"RuntimeInSeconds": 3.6610000133514404
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"Version": 0,
|
"Version": 0,
|
||||||
"Type": "Trial",
|
"Type": "Trial",
|
||||||
"TrainerName": "FastTreeOva",
|
"TrainerName": "FastTreeOva",
|
||||||
"Score": 0.91460317460317475,
|
"Score": 0.91090909090909089,
|
||||||
"RuntimeInSeconds": 3.2239999771118164
|
"RuntimeInSeconds": 3.3050000667572021
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"Version": 0,
|
"Version": 0,
|
||||||
"Type": "Trial",
|
"Type": "Trial",
|
||||||
"TrainerName": "LbfgsLogisticRegressionOva",
|
"TrainerName": "FastTreeOva",
|
||||||
"Score": 0.96035353535353529,
|
"Score": 0.96460317460317468,
|
||||||
"RuntimeInSeconds": 2.812000036239624
|
"RuntimeInSeconds": 3.1909999847412109
|
||||||
},
|
|
||||||
{
|
|
||||||
"Version": 0,
|
|
||||||
"Type": "Trial",
|
|
||||||
"TrainerName": "SdcaLogisticRegressionOva",
|
|
||||||
"Score": 0.87460317460317472,
|
|
||||||
"RuntimeInSeconds": 7.8530001640319824
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"Version": 0,
|
"Version": 0,
|
||||||
"Type": "Trial",
|
"Type": "Trial",
|
||||||
"TrainerName": "LbfgsMaximumEntropyMulti",
|
"TrainerName": "LbfgsMaximumEntropyMulti",
|
||||||
"Score": 0.96035353535353529,
|
"Score": 0.95031746031746034,
|
||||||
"RuntimeInSeconds": 2.3250000476837158
|
"RuntimeInSeconds": 2.2969999313354492
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"Version": 0,
|
"Version": 0,
|
||||||
"Type": "Trial",
|
"Type": "Trial",
|
||||||
"TrainerName": "LightGbmMulti",
|
"TrainerName": "FastForestOva",
|
||||||
"Score": 0.91460317460317475,
|
"Score": 0.91090909090909089,
|
||||||
"RuntimeInSeconds": 2.875
|
"RuntimeInSeconds": 3.9630000591278076
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Version": 0,
|
||||||
|
"Type": "Trial",
|
||||||
|
"TrainerName": "SdcaLogisticRegressionOva",
|
||||||
|
"Score": 0.57954545454545447,
|
||||||
|
"RuntimeInSeconds": 7.7239999771118164
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Version": 0,
|
||||||
|
"Type": "Trial",
|
||||||
|
"TrainerName": "LbfgsLogisticRegressionOva",
|
||||||
|
"Score": 0.95031746031746034,
|
||||||
|
"RuntimeInSeconds": 3.937000036239624
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"Pipeline": {
|
"Pipeline": {
|
||||||
@ -311,20 +311,16 @@
|
|||||||
"OutputColumnName": "Q1mean"
|
"OutputColumnName": "Q1mean"
|
||||||
},
|
},
|
||||||
"20": {
|
"20": {
|
||||||
"OutputColumnNames": [
|
"NumberOfLeaves": 33,
|
||||||
"Features"
|
"MinimumExampleCountPerLeaf": 14,
|
||||||
],
|
"NumberOfTrees": 4,
|
||||||
"InputColumnNames": [
|
"MaximumBinCountPerFeature": 1022,
|
||||||
"Features"
|
"FeatureFraction": 0.99999999,
|
||||||
]
|
"LearningRate": 0.75792684413443268,
|
||||||
},
|
|
||||||
"21": {
|
|
||||||
"L1Regularization": 1.0,
|
|
||||||
"L2Regularization": 1.0,
|
|
||||||
"LabelColumnName": "Name",
|
"LabelColumnName": "Name",
|
||||||
"FeatureColumnName": "Features"
|
"FeatureColumnName": "Features"
|
||||||
},
|
},
|
||||||
"22": {
|
"21": {
|
||||||
"OutputColumnName": "PredictedLabel",
|
"OutputColumnName": "PredictedLabel",
|
||||||
"InputColumnName": "PredictedLabel"
|
"InputColumnName": "PredictedLabel"
|
||||||
},
|
},
|
||||||
@ -378,8 +374,7 @@
|
|||||||
"FeaturizeText",
|
"FeaturizeText",
|
||||||
"Concatenate",
|
"Concatenate",
|
||||||
"MapValueToKey",
|
"MapValueToKey",
|
||||||
"NormalizeMinMax",
|
"FastTreeOva",
|
||||||
"LbfgsLogisticRegressionOva",
|
|
||||||
"MapKeyToValue"
|
"MapKeyToValue"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@ -5,6 +5,7 @@ using System.Linq;
|
|||||||
using System.Text;
|
using System.Text;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
using Microsoft.ML.Data;
|
using Microsoft.ML.Data;
|
||||||
|
using Microsoft.ML.Trainers.FastTree;
|
||||||
using Microsoft.ML.Trainers;
|
using Microsoft.ML.Trainers;
|
||||||
using Microsoft.ML;
|
using Microsoft.ML;
|
||||||
|
|
||||||
@ -54,8 +55,7 @@ namespace DeepTrace
|
|||||||
.Append(mlContext.Transforms.Text.FeaturizeText(inputColumnName:@"Q5mean",outputColumnName:@"Q5mean"))
|
.Append(mlContext.Transforms.Text.FeaturizeText(inputColumnName:@"Q5mean",outputColumnName:@"Q5mean"))
|
||||||
.Append(mlContext.Transforms.Concatenate(@"Features", new []{@"Q1max",@"Q1avg",@"Q1mean",@"Q2min",@"Q2max",@"Q2avg",@"Q2mean",@"Q3min",@"Q3max",@"Q3avg",@"Q3mean",@"Q4max",@"Q4avg",@"Q4mean",@"Q5min",@"Q5max",@"Q5avg",@"Q5mean"}))
|
.Append(mlContext.Transforms.Concatenate(@"Features", new []{@"Q1max",@"Q1avg",@"Q1mean",@"Q2min",@"Q2max",@"Q2avg",@"Q2mean",@"Q3min",@"Q3max",@"Q3avg",@"Q3mean",@"Q4max",@"Q4avg",@"Q4mean",@"Q5min",@"Q5max",@"Q5avg",@"Q5mean"}))
|
||||||
.Append(mlContext.Transforms.Conversion.MapValueToKey(outputColumnName:@"Name",inputColumnName:@"Name"))
|
.Append(mlContext.Transforms.Conversion.MapValueToKey(outputColumnName:@"Name",inputColumnName:@"Name"))
|
||||||
.Append(mlContext.Transforms.NormalizeMinMax(@"Features", @"Features"))
|
.Append(mlContext.MulticlassClassification.Trainers.OneVersusAll(binaryEstimator:mlContext.BinaryClassification.Trainers.FastTree(new FastTreeBinaryTrainer.Options(){NumberOfLeaves=33,MinimumExampleCountPerLeaf=14,NumberOfTrees=4,MaximumBinCountPerFeature=1022,FeatureFraction=0.99999999,LearningRate=0.757926844134433,LabelColumnName=@"Name",FeatureColumnName=@"Features"}),labelColumnName: @"Name"))
|
||||||
.Append(mlContext.MulticlassClassification.Trainers.OneVersusAll(binaryEstimator: mlContext.BinaryClassification.Trainers.LbfgsLogisticRegression(new LbfgsLogisticRegressionBinaryTrainer.Options(){L1Regularization=1F,L2Regularization=1F,LabelColumnName=@"Name",FeatureColumnName=@"Features"}), labelColumnName:@"Name"))
|
|
||||||
.Append(mlContext.Transforms.Conversion.MapKeyToValue(outputColumnName:@"PredictedLabel",inputColumnName:@"PredictedLabel"));
|
.Append(mlContext.Transforms.Conversion.MapKeyToValue(outputColumnName:@"PredictedLabel",inputColumnName:@"PredictedLabel"));
|
||||||
|
|
||||||
return pipeline;
|
return pipeline;
|
||||||
|
|||||||
@ -7,7 +7,7 @@ namespace DeepTrace.Data
|
|||||||
{
|
{
|
||||||
[BsonId]
|
[BsonId]
|
||||||
public ObjectId? Id { get; set; }
|
public ObjectId? Id { get; set; }
|
||||||
public string Name { get; set; }
|
public string Name { get; set; } = string.Empty;
|
||||||
public byte[] Value { get; set; } //base64
|
public byte[] Value { get; set; } = Array.Empty<byte>(); //base64
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,6 +6,8 @@
|
|||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Blazor-ApexCharts" Version="0.9.21-beta" />
|
<PackageReference Include="Blazor-ApexCharts" Version="0.9.21-beta" />
|
||||||
|
<PackageReference Include="CsvHelper" Version="30.0.1" />
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Logging.Log4Net.AspNetCore" Version="6.1.0" />
|
||||||
<PackageReference Include="Microsoft.ML" Version="2.0.1" />
|
<PackageReference Include="Microsoft.ML" Version="2.0.1" />
|
||||||
<PackageReference Include="Microsoft.ML.FastTree" Version="1.7.1" />
|
<PackageReference Include="Microsoft.ML.FastTree" Version="1.7.1" />
|
||||||
<PackageReference Include="Microsoft.ML.TimeSeries" Version="2.0.1" />
|
<PackageReference Include="Microsoft.ML.TimeSeries" Version="2.0.1" />
|
||||||
|
|||||||
@ -5,7 +5,7 @@ namespace DeepTrace.ML;
|
|||||||
|
|
||||||
public interface IMLProcessor
|
public interface IMLProcessor
|
||||||
{
|
{
|
||||||
Task Train(ModelDefinition modelDef);
|
Task Train(ModelDefinition modelDef, Action<string> log);
|
||||||
byte[] Export();
|
byte[] Export();
|
||||||
void Import(byte[] data);
|
void Import(byte[] data);
|
||||||
string Predict(DataSourceDefinition dataSource);
|
string Predict(DataSourceDefinition dataSource);
|
||||||
|
|||||||
@ -12,13 +12,21 @@ namespace DeepTrace.ML
|
|||||||
private EstimatorBuilder _estimatorBuilder = new EstimatorBuilder();
|
private EstimatorBuilder _estimatorBuilder = new EstimatorBuilder();
|
||||||
private DataViewSchema? _schema;
|
private DataViewSchema? _schema;
|
||||||
private ITransformer? _transformer;
|
private ITransformer? _transformer;
|
||||||
|
private static string _signature = "DeepTrace-Model-v1-" + typeof(MLProcessor).Name;
|
||||||
|
private readonly ILogger<MLProcessor> _logger;
|
||||||
|
|
||||||
|
public MLProcessor(ILogger<MLProcessor> logger)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
private string Name { get; set; } = "TestModel";
|
private string Name { get; set; } = "TestModel";
|
||||||
|
|
||||||
public async Task Train(ModelDefinition modelDef)
|
public async Task Train(ModelDefinition modelDef, Action<string> log)
|
||||||
{
|
{
|
||||||
var pipeline = _estimatorBuilder.BuildPipeline(_mlContext, modelDef);
|
var pipeline = _estimatorBuilder.BuildPipeline(_mlContext, modelDef);
|
||||||
var (data, filename) = await MLHelpers.Convert(_mlContext, modelDef);
|
var (data, filename) = await MLHelpers.Convert(_mlContext, modelDef);
|
||||||
|
_mlContext.Log += (_,e) => LogEvents(log, e);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_schema = data.Schema;
|
_schema = data.Schema;
|
||||||
@ -31,7 +39,15 @@ namespace DeepTrace.ML
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string _signature = "DeepTrace-Model-v1-"+typeof(MLProcessor).Name;
|
private void LogEvents(Action<string> log, LoggingEventArgs e)
|
||||||
|
{
|
||||||
|
if(e.Kind.ToString() != "Trace")
|
||||||
|
{
|
||||||
|
_logger.LogDebug(e.Message);
|
||||||
|
log(e.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
public byte[] Export()
|
public byte[] Export()
|
||||||
{
|
{
|
||||||
|
|||||||
@ -13,7 +13,7 @@ public static class MedianHelper
|
|||||||
/// Pivot is selected ranodmly if random number generator is supplied else its selected as last element in the list.
|
/// Pivot is selected ranodmly if random number generator is supplied else its selected as last element in the list.
|
||||||
/// Reference: Introduction to Algorithms 3rd Edition, Corman et al, pp 171
|
/// Reference: Introduction to Algorithms 3rd Edition, Corman et al, pp 171
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private static int Partition<T>(this IList<T> list, int start, int end, Random rnd = null) where T : IComparable<T>
|
private static int Partition<T>(this IList<T> list, int start, int end, Random? rnd = null) where T : IComparable<T>
|
||||||
{
|
{
|
||||||
if (rnd != null)
|
if (rnd != null)
|
||||||
list.Swap(end, rnd.Next(start, end + 1));
|
list.Swap(end, rnd.Next(start, end + 1));
|
||||||
@ -34,11 +34,11 @@ public static class MedianHelper
|
|||||||
/// Note: specified list would be mutated in the process.
|
/// Note: specified list would be mutated in the process.
|
||||||
/// Reference: Introduction to Algorithms 3rd Edition, Corman et al, pp 216
|
/// Reference: Introduction to Algorithms 3rd Edition, Corman et al, pp 216
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static T NthOrderStatistic<T>(this IList<T> list, int n, Random rnd = null) where T : IComparable<T>
|
public static T NthOrderStatistic<T>(this IList<T> list, int n, Random? rnd = null) where T : IComparable<T>
|
||||||
{
|
{
|
||||||
return NthOrderStatistic(list, n, 0, list.Count - 1, rnd);
|
return NthOrderStatistic(list, n, 0, list.Count - 1, rnd);
|
||||||
}
|
}
|
||||||
private static T NthOrderStatistic<T>(this IList<T> list, int n, int start, int end, Random rnd) where T : IComparable<T>
|
private static T NthOrderStatistic<T>(this IList<T> list, int n, int start, int end, Random? rnd) where T : IComparable<T>
|
||||||
{
|
{
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
|
|||||||
@ -372,6 +372,9 @@
|
|||||||
|
|
||||||
private void HandleTrain()
|
private void HandleTrain()
|
||||||
{
|
{
|
||||||
|
if ( DisplayData == null )
|
||||||
|
return;
|
||||||
|
|
||||||
var mlContext = new MLContext();
|
var mlContext = new MLContext();
|
||||||
|
|
||||||
var dataView = mlContext.Data.LoadFromEnumerable<MyTimeSeries>(DisplayData.Series[0].Data.Select(x => new MyTimeSeries(Time: x.TimeStamp, Value: x.Value)));
|
var dataView = mlContext.Data.LoadFromEnumerable<MyTimeSeries>(DisplayData.Series[0].Data.Select(x => new MyTimeSeries(Time: x.TimeStamp, Value: x.Value)));
|
||||||
@ -476,7 +479,7 @@
|
|||||||
|
|
||||||
// -------- Spike detection tutorial ---------
|
// -------- Spike detection tutorial ---------
|
||||||
|
|
||||||
private static void DetectSpike(MLContext mLContext, IDataView dataView, TimeSeries[]? data)
|
private static void DetectSpike(MLContext mLContext, IDataView dataView, TimeSeries[] data)
|
||||||
{
|
{
|
||||||
string outputColumnName = nameof(IidSpikePrediction.Prediction);
|
string outputColumnName = nameof(IidSpikePrediction.Prediction);
|
||||||
string inputColumnName = nameof(TimeSeriesDataTutorial.Value);
|
string inputColumnName = nameof(TimeSeriesDataTutorial.Value);
|
||||||
@ -531,7 +534,7 @@
|
|||||||
class IidSpikePrediction
|
class IidSpikePrediction
|
||||||
{
|
{
|
||||||
[VectorType(3)]
|
[VectorType(3)]
|
||||||
public double[] Prediction { get; set; }
|
public double[] Prediction { get; set; } = Array.Empty<double>();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
@page "/training"
|
@page "/training"
|
||||||
|
@using CsvHelper;
|
||||||
@using DeepTrace.Data;
|
@using DeepTrace.Data;
|
||||||
@using DeepTrace.ML;
|
@using DeepTrace.ML;
|
||||||
@using DeepTrace.Services;
|
@using DeepTrace.Services;
|
||||||
@ -7,6 +8,7 @@
|
|||||||
@using Microsoft.ML;
|
@using Microsoft.ML;
|
||||||
@using PrometheusAPI;
|
@using PrometheusAPI;
|
||||||
@using System.Text;
|
@using System.Text;
|
||||||
|
@using System.Globalization;
|
||||||
|
|
||||||
@inject PrometheusClient Prometheus
|
@inject PrometheusClient Prometheus
|
||||||
@inject IDialogService DialogService
|
@inject IDialogService DialogService
|
||||||
@ -16,6 +18,7 @@
|
|||||||
@inject IEstimatorBuilder EstimatorBuilder
|
@inject IEstimatorBuilder EstimatorBuilder
|
||||||
@inject NavigationManager NavManager
|
@inject NavigationManager NavManager
|
||||||
@inject IJSRuntime Js
|
@inject IJSRuntime Js
|
||||||
|
@inject ILogger<MLProcessor> MLProcessorLogger
|
||||||
|
|
||||||
|
|
||||||
<PageTitle>Training</PageTitle>
|
<PageTitle>Training</PageTitle>
|
||||||
@ -203,12 +206,12 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ModelForm? _modelForm;
|
private ModelForm? _modelForm;
|
||||||
private TimeSeriesData? DisplayData { get; set; }
|
|
||||||
private List<DataSourceStorage> _dataSources = new();
|
private List<DataSourceStorage> _dataSources = new();
|
||||||
private List<ModelDefinition> _modelDefinitions = new() {new()};
|
private List<ModelDefinition> _modelDefinitions = new() {new()};
|
||||||
|
|
||||||
private DateTime? _minDate;
|
private DateTime? _minDate;
|
||||||
private DateTime? _maxDate;
|
private DateTime? _maxDate;
|
||||||
|
|
||||||
|
private TimeSeriesData? DisplayData { get; set; }
|
||||||
private bool IsAddDisabled => DisplayData == null;
|
private bool IsAddDisabled => DisplayData == null;
|
||||||
|
|
||||||
private DateTime? MinDate
|
private DateTime? MinDate
|
||||||
@ -293,7 +296,7 @@
|
|||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
ShowError(e.Message);
|
await ShowError(e.Message);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -303,20 +306,20 @@
|
|||||||
{
|
{
|
||||||
if (res.Status != StatusType.Success)
|
if (res.Status != StatusType.Success)
|
||||||
{
|
{
|
||||||
ShowError(res.Error ?? "Error");
|
await ShowError(res.Error ?? "Error");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (res.ResultType != ResultTypeType.Matrix)
|
if (res.ResultType != ResultTypeType.Matrix)
|
||||||
{
|
{
|
||||||
ShowError($"Got {res.ResultType}, but Matrix expected for {def.Query}");
|
await ShowError($"Got {res.ResultType}, but Matrix expected for {def.Query}");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var m = res.AsMatrix().Result;
|
var m = res.AsMatrix().Result;
|
||||||
if (m == null || m.Length != 1)
|
if (m == null || m.Length != 1)
|
||||||
{
|
{
|
||||||
ShowError($"No data returned for {def.Query}");
|
await ShowError($"No data returned for {def.Query}");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -335,12 +338,15 @@
|
|||||||
|
|
||||||
private void HandleAddModel()
|
private void HandleAddModel()
|
||||||
{
|
{
|
||||||
|
if (_modelForm == null)
|
||||||
|
return;
|
||||||
|
|
||||||
_modelDefinitions.Add(new());
|
_modelDefinitions.Add(new());
|
||||||
_modelForm.CurrentModel = _modelDefinitions[^1];
|
_modelForm.CurrentModel = _modelDefinitions[^1];
|
||||||
StateHasChanged();
|
StateHasChanged();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void HandleDeleteModel()
|
private async Task HandleDeleteModel()
|
||||||
{
|
{
|
||||||
if (_modelDefinitions.Count < 2)
|
if (_modelDefinitions.Count < 2)
|
||||||
{
|
{
|
||||||
@ -350,7 +356,7 @@
|
|||||||
var pos = _modelDefinitions.IndexOf(_modelForm!.CurrentModel);
|
var pos = _modelDefinitions.IndexOf(_modelForm!.CurrentModel);
|
||||||
if (pos < 0)
|
if (pos < 0)
|
||||||
{
|
{
|
||||||
ShowError("Not found");
|
await ShowError("Not found");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -360,10 +366,10 @@
|
|||||||
|
|
||||||
if (toDelete.Id != null)
|
if (toDelete.Id != null)
|
||||||
{
|
{
|
||||||
ModelService.Delete(toDelete);
|
await ModelService.Delete(toDelete);
|
||||||
}
|
}
|
||||||
|
|
||||||
StateHasChanged();
|
await InvokeAsync(StateHasChanged);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task HandleAddTableContent()
|
private async Task HandleAddTableContent()
|
||||||
@ -399,11 +405,14 @@
|
|||||||
|
|
||||||
private void ItemHasBeenCommitted(object element)
|
private void ItemHasBeenCommitted(object element)
|
||||||
{
|
{
|
||||||
Task.Run(async ()=>ModelService.Store(_modelForm!.CurrentModel));
|
Task.Run(() => ModelService.Store(_modelForm!.CurrentModel));
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task HandleRefresh()
|
private async Task HandleRefresh()
|
||||||
{
|
{
|
||||||
|
if (DisplayData == null)
|
||||||
|
return;
|
||||||
|
|
||||||
var previousIntervals = _modelForm!.CurrentModel.IntervalDefinitionList;
|
var previousIntervals = _modelForm!.CurrentModel.IntervalDefinitionList;
|
||||||
|
|
||||||
foreach(var currentInterval in previousIntervals)
|
foreach(var currentInterval in previousIntervals)
|
||||||
@ -415,20 +424,72 @@
|
|||||||
// await InvokeAsync(StateHasChanged);
|
// await InvokeAsync(StateHasChanged);
|
||||||
}
|
}
|
||||||
|
|
||||||
//Doesn't work
|
private class ImportedCsv
|
||||||
|
{
|
||||||
|
public DateTime Start { get; set; }
|
||||||
|
public DateTime End { get; set; }
|
||||||
|
public string Label { get; set; } = "";
|
||||||
|
}
|
||||||
|
|
||||||
private async Task HandleImport(IBrowserFile file)
|
private async Task HandleImport(IBrowserFile file)
|
||||||
{
|
{
|
||||||
var result = new StringBuilder();
|
try
|
||||||
var reader = new StreamReader(file.OpenReadStream(file.Size));
|
{
|
||||||
|
await HandleImportInternal(file);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
await ShowError($"Can't import: {e.Message}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
while (reader.Peek() >= 0)
|
private async Task HandleImportInternal(IBrowserFile file)
|
||||||
result.AppendLine(await reader.ReadLineAsync());
|
{
|
||||||
|
if ( _modelForm == null )
|
||||||
|
return;
|
||||||
|
|
||||||
result.ToString();
|
using var mem = new MemoryStream();
|
||||||
|
// https://stackoverflow.com/questions/67066860/blazorinputfile-synchronous-reads-are-not-supported
|
||||||
|
await file.OpenReadStream(file.Size).CopyToAsync(mem);
|
||||||
|
mem.Position = 0;
|
||||||
|
|
||||||
|
// var text = Encoding.UTF8.GetString(mem.ToArray());
|
||||||
|
|
||||||
|
using (var reader = new StreamReader(mem))
|
||||||
|
using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture))
|
||||||
|
{
|
||||||
|
var records = csv.GetRecords<ImportedCsv>();
|
||||||
|
|
||||||
|
var yes = await YesNo("If you want to replace all intervals click Yes. Click No to append.");
|
||||||
|
if (yes == null)
|
||||||
|
return;
|
||||||
|
|
||||||
|
if (yes == true)
|
||||||
|
_modelForm.CurrentModel.IntervalDefinitionList.Clear();
|
||||||
|
|
||||||
|
var normalWorkStart = DateTime.MinValue;
|
||||||
|
foreach( var rec in records)
|
||||||
|
{
|
||||||
|
if(normalWorkStart != DateTime.MinValue)
|
||||||
|
{
|
||||||
|
var normalWorkEnd = rec.Start - TimeSpan.FromSeconds(1);
|
||||||
|
if(normalWorkStart<normalWorkEnd)
|
||||||
|
_modelForm.CurrentModel.IntervalDefinitionList.Add(new(normalWorkStart, normalWorkEnd, "Normal work"));
|
||||||
|
}
|
||||||
|
normalWorkStart = rec.End+TimeSpan.FromSeconds(1);
|
||||||
|
_modelForm.CurrentModel.IntervalDefinitionList.Add(new(rec.Start, rec.End, rec.Label));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await HandleRefresh();
|
||||||
|
await InvokeAsync(StateHasChanged);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task HandleExport()
|
private async Task HandleExport()
|
||||||
{
|
{
|
||||||
|
if (_modelForm == null)
|
||||||
|
return;
|
||||||
|
|
||||||
await Js.InvokeVoidAsync("open", $"{NavManager.BaseUri}api/download/mldata/{Uri.EscapeDataString(_modelForm.CurrentModel.Name)}", "_blank");
|
await Js.InvokeVoidAsync("open", $"{NavManager.BaseUri}api/download/mldata/{Uri.EscapeDataString(_modelForm.CurrentModel.Name)}", "_blank");
|
||||||
|
|
||||||
}
|
}
|
||||||
@ -440,7 +501,7 @@
|
|||||||
await InvokeAsync(StateHasChanged);
|
await InvokeAsync(StateHasChanged);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void ShowError(string text)
|
private async Task ShowError(string text)
|
||||||
{
|
{
|
||||||
var options = new DialogOptions
|
var options = new DialogOptions
|
||||||
{
|
{
|
||||||
@ -449,19 +510,50 @@
|
|||||||
var parameters = new DialogParameters();
|
var parameters = new DialogParameters();
|
||||||
parameters.Add("Text", text);
|
parameters.Add("Text", text);
|
||||||
|
|
||||||
DialogService.Show<Controls.Dialog>("Error", parameters, options);
|
var d = DialogService.Show<Controls.Dialog>("Error", parameters, options);
|
||||||
|
await d.Result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<bool?> YesNo(string text)
|
||||||
|
{
|
||||||
|
var options = new DialogOptions
|
||||||
|
{
|
||||||
|
CloseOnEscapeKey = true
|
||||||
|
};
|
||||||
|
var parameters = new DialogParameters();
|
||||||
|
parameters.Add(nameof(Controls.Dialog.Text), text);
|
||||||
|
parameters.Add(nameof(Controls.Dialog.IsYesNoCancel), true);
|
||||||
|
|
||||||
|
var d = DialogService.Show<Controls.Dialog>("Query", parameters, options);
|
||||||
|
var res = await d.Result;
|
||||||
|
return res?.Data as bool?;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task HandleTrain()
|
private async Task HandleTrain()
|
||||||
{
|
{
|
||||||
var mlProcessor = new MLProcessor();
|
var mlProcessor = new MLProcessor(MLProcessorLogger);
|
||||||
await mlProcessor.Train(_modelForm!.CurrentModel);
|
MLProcessorLogger.LogInformation("Training started");
|
||||||
|
|
||||||
|
var options = new DialogOptions
|
||||||
|
{
|
||||||
|
CloseOnEscapeKey = true
|
||||||
|
};
|
||||||
|
var parameters = new DialogParameters();
|
||||||
|
parameters.Add(nameof(Controls.TrainingDialog.Text), _modelForm!.CurrentModel.Name);
|
||||||
|
parameters.Add(nameof(Controls.TrainingDialog.Processor), mlProcessor);
|
||||||
|
parameters.Add(nameof(Controls.TrainingDialog.Model), _modelForm.CurrentModel);
|
||||||
|
|
||||||
|
var d = DialogService.Show<Controls.TrainingDialog>("Training", parameters, options);
|
||||||
|
var res = await d.Result;
|
||||||
|
|
||||||
|
MLProcessorLogger.LogInformation("Training finished");
|
||||||
var bytes = mlProcessor.Export();
|
var bytes = mlProcessor.Export();
|
||||||
|
|
||||||
//save to Mongo
|
//save to Mongo
|
||||||
var trainedModel = new TrainedModelDefinition
|
var trainedModel = new TrainedModelDefinition
|
||||||
{
|
{
|
||||||
Name = "TrainedModel",
|
Id = _modelForm!.CurrentModel.Id,
|
||||||
|
Name = _modelForm!.CurrentModel.Name,
|
||||||
Value = bytes
|
Value = bytes
|
||||||
};
|
};
|
||||||
await TrainedModelService.Store(trainedModel);
|
await TrainedModelService.Store(trainedModel);
|
||||||
|
|||||||
@ -3,9 +3,19 @@ using PrometheusAPI;
|
|||||||
using MongoDB.Driver;
|
using MongoDB.Driver;
|
||||||
using DeepTrace.Services;
|
using DeepTrace.Services;
|
||||||
using DeepTrace.ML;
|
using DeepTrace.ML;
|
||||||
|
using ApexCharts;
|
||||||
|
using log4net;
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
|
builder.Services.AddLogging(logging =>
|
||||||
|
{
|
||||||
|
logging.ClearProviders();
|
||||||
|
GlobalContext.Properties["LOGS_ROOT"] = Environment.GetEnvironmentVariable("LOGS_ROOT") ?? "";
|
||||||
|
System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
|
||||||
|
logging.AddLog4Net("log4net.config");
|
||||||
|
});
|
||||||
|
|
||||||
// Add services to the container.
|
// Add services to the container.
|
||||||
builder.Services.AddRazorPages();
|
builder.Services.AddRazorPages();
|
||||||
builder.Services.AddServerSideBlazor();
|
builder.Services.AddServerSideBlazor();
|
||||||
|
|||||||
@ -2,8 +2,8 @@
|
|||||||
"DetailedErrors": true,
|
"DetailedErrors": true,
|
||||||
"Logging": {
|
"Logging": {
|
||||||
"LogLevel": {
|
"LogLevel": {
|
||||||
"Default": "Information",
|
"Default": "Trace",
|
||||||
"Microsoft.AspNetCore": "Warning"
|
"Microsoft.AspNetCore": "Information"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,8 +1,8 @@
|
|||||||
{
|
{
|
||||||
"Logging": {
|
"Logging": {
|
||||||
"LogLevel": {
|
"LogLevel": {
|
||||||
"Default": "Information",
|
"Default": "Trace",
|
||||||
"Microsoft.AspNetCore": "Warning"
|
"Microsoft.AspNetCore": "Information"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"AllowedHosts": "*",
|
"AllowedHosts": "*",
|
||||||
|
|||||||
63
DeepTrace/log4net.config
Normal file
63
DeepTrace/log4net.config
Normal file
@ -0,0 +1,63 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8" ?>
|
||||||
|
<log4net>
|
||||||
|
<appender name="DebugAppender" type="log4net.Appender.DebugAppender" >
|
||||||
|
<layout type="log4net.Layout.PatternLayout">
|
||||||
|
<conversionPattern value="%date [%-2thread] %-5level %logger - %message%newline" />
|
||||||
|
</layout>
|
||||||
|
</appender>
|
||||||
|
|
||||||
|
<!--
|
||||||
|
<appender name="Console" type="log4net.Appender.ConsoleAppender">
|
||||||
|
<layout type="log4net.Layout.PatternLayout">
|
||||||
|
<conversionPattern value="%date [%-2thread] %-5level %-20logger - %message%newline" />
|
||||||
|
</layout>
|
||||||
|
</appender>
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!--Console appender-->
|
||||||
|
<appender name="Console" type="log4net.Appender.ManagedColoredConsoleAppender">
|
||||||
|
<mapping>
|
||||||
|
<level value="INFO" />
|
||||||
|
<forecolor value="Green" />
|
||||||
|
</mapping>
|
||||||
|
<mapping>
|
||||||
|
<level value="WARN" />
|
||||||
|
<forecolor value="Yellow" />
|
||||||
|
</mapping>
|
||||||
|
<mapping>
|
||||||
|
<level value="ERROR" />
|
||||||
|
<forecolor value="Red" />
|
||||||
|
</mapping>
|
||||||
|
<mapping>
|
||||||
|
<level value="DEBUG" />
|
||||||
|
<forecolor value="Blue" />
|
||||||
|
</mapping>
|
||||||
|
<layout type="log4net.Layout.PatternLayout">
|
||||||
|
<conversionPattern value="%date{HH:mm:ss,fff} [%3thread] %-5level %-15logger{1} %message%newline" />
|
||||||
|
</layout>
|
||||||
|
</appender>
|
||||||
|
|
||||||
|
<appender name="RollingFile" type="log4net.Appender.RollingFileAppender">
|
||||||
|
<file value="log-file.txt" />
|
||||||
|
<appendToFile value="false" />
|
||||||
|
<layout type="log4net.Layout.PatternLayout">
|
||||||
|
<conversionPattern value="%date [%3thread] %-5level %logger [%property{NDC}] - %message%newline" />
|
||||||
|
</layout>
|
||||||
|
</appender>
|
||||||
|
|
||||||
|
<appender name="BufferingForwardingAppender" type="log4net.Appender.BufferingForwardingAppender" >
|
||||||
|
<bufferSize value="1"/>
|
||||||
|
<appender-ref ref="DebugAppender" />
|
||||||
|
<appender-ref ref="Console" />
|
||||||
|
<appender-ref ref="RollingFile" />
|
||||||
|
</appender>
|
||||||
|
|
||||||
|
<logger name="System.Net.Http">
|
||||||
|
<level value="ERROR"/>
|
||||||
|
</logger>
|
||||||
|
|
||||||
|
<root>
|
||||||
|
<level value="ALL"/>
|
||||||
|
<appender-ref ref="BufferingForwardingAppender" />
|
||||||
|
</root>
|
||||||
|
</log4net>
|
||||||
Loading…
x
Reference in New Issue
Block a user