diff --git a/DeepTrace/Controls/TimeSeriesChart.razor b/DeepTrace/Controls/TimeSeriesChart.razor index 8e7e6a5..d285361 100644 --- a/DeepTrace/Controls/TimeSeriesChart.razor +++ b/DeepTrace/Controls/TimeSeriesChart.razor @@ -1,4 +1,7 @@ -@using PrometheusAPI; +@using DeepTrace.Data; +@using DeepTrace.Services; +@using PrometheusAPI; + using DeepTrace.Data; Data { get; init; } = Array.Empty(); - } - - public class TimeSeriesData - { - public List Series { get; init; } = new List(); - } - - [CascadingParameter] protected bool IsDarkMode { get; set; } diff --git a/DeepTrace/Data/IntervalDefinition.cs b/DeepTrace/Data/IntervalDefinition.cs new file mode 100644 index 0000000..f13788b --- /dev/null +++ b/DeepTrace/Data/IntervalDefinition.cs @@ -0,0 +1,22 @@ +namespace DeepTrace.Data +{ + public class IntervalDefinition + { + public IntervalDefinition() { } + public IntervalDefinition(DateTime from, DateTime to, string name) + { + From = from; + To = to; + Name = name; + } + + public DateTime From { get; set; } = DateTime.MinValue; + + public DateTime To { get; set; } = DateTime.MaxValue; + + public string Name { get; set; } = string.Empty; + + public List Data { get; set; } + + } +} diff --git a/DeepTrace/Data/ModelDefinition.cs b/DeepTrace/Data/ModelDefinition.cs new file mode 100644 index 0000000..6062a76 --- /dev/null +++ b/DeepTrace/Data/ModelDefinition.cs @@ -0,0 +1,18 @@ +namespace DeepTrace.Data +{ + + public class ModelDefinition + { + private static int _instanceId; + public ModelDefinition() + { + var id = Interlocked.Increment(ref _instanceId); + Name = $"Model #{id}"; + } + + public string Name { get; set; } + public string DataSourceName { get; set; } = string.Empty; + public string AIparameters { get; set; } = string.Empty; + public List IntervalDefinitionList { get; set; } = new(); + } +} diff --git a/DeepTrace/Data/TimeSeriesData.cs b/DeepTrace/Data/TimeSeriesData.cs new file mode 100644 index 0000000..486d0c5 --- /dev/null +++ b/DeepTrace/Data/TimeSeriesData.cs @@ -0,0 +1,15 @@ +using PrometheusAPI; + +namespace DeepTrace.Data; + +public class TimeSeriesData +{ + public List Series { get; init; } = new List(); +} + +public class TimeSeriesDataSet +{ + public string Name { get; init; } = "Value"; + public string Color { get; init; } = ""; + public IReadOnlyCollection Data { get; init; } = Array.Empty(); +} \ No newline at end of file diff --git a/DeepTrace/Pages/DataSources.razor b/DeepTrace/Pages/DataSources.razor index 9a3926b..a7a9a19 100644 --- a/DeepTrace/Pages/DataSources.razor +++ b/DeepTrace/Pages/DataSources.razor @@ -182,7 +182,7 @@ } } - private TimeSeriesChart.TimeSeriesData? DisplayData { get; set; } + private TimeSeriesData? DisplayData { get; set; } private string[] _palette = new[] { @@ -312,7 +312,7 @@ return; } - var data = new List(); + var data = new List(); foreach ( var (res, def) in tasks.Select((x,i) => (x.Result, _queryForm.Source.Queries[i]) )) { @@ -414,7 +414,7 @@ private int DetectPeriod(MLContext mlContext, IDataView dataView) { - string inputColumnName = nameof(TimeSeriesData.Value); + string inputColumnName = nameof(TimeSeriesDataTutorial.Value); int period = mlContext.AnomalyDetection.DetectSeasonality(dataView, inputColumnName); Console.WriteLine("Period of the series is: {0}.", period); return period; @@ -423,7 +423,7 @@ private void DetectAnomaly(MLContext mlContext, IDataView dataView, int period) { string outputColumnName = nameof(IidSpikePrediction.Prediction); - string inputColumnName = nameof(TimeSeriesData.Value); + string inputColumnName = nameof(TimeSeriesDataTutorial.Value); var options = new SrCnnEntireAnomalyDetectorOptions() { Threshold = 0.3, @@ -469,7 +469,7 @@ private static void DetectSpike(MLContext mLContext, IDataView dataView, TimeSeries[]? data) { string outputColumnName = nameof(IidSpikePrediction.Prediction); - string inputColumnName = nameof(TimeSeriesData.Value); + string inputColumnName = nameof(TimeSeriesDataTutorial.Value); var iidSpikeEstimator = mLContext.Transforms.DetectIidSpike(outputColumnName, inputColumnName, 95.0d, data.Length / 4); @@ -508,11 +508,11 @@ } - class TimeSeriesData + class TimeSeriesDataTutorial { public float Value; - - public TimeSeriesData(float value) + + public TimeSeriesDataTutorial(float value) { Value = value; } diff --git a/DeepTrace/Pages/Training.razor b/DeepTrace/Pages/Training.razor index 2814f5f..76f3682 100644 --- a/DeepTrace/Pages/Training.razor +++ b/DeepTrace/Pages/Training.razor @@ -1,6 +1,15 @@ @page "/training" -@using DeepTrace.Controls; @using DeepTrace.Data; +@using DeepTrace.Services; +@using System.ComponentModel.DataAnnotations; +@using DeepTrace.Controls; +@using PrometheusAPI; + +@inject PrometheusClient Prometheus +@inject IDialogService DialogService +@inject IDataSourceStorageService StorageService +@inject IModelStorageService ModelService + Training @@ -18,41 +27,56 @@ - - + + @foreach (var model in _modelDefinitions) + { + @model.Name + } + - Add - Delete + Add + @if (_modelDefinitions.Count > 1) + { + Delete + } + + + + @foreach (var source in _dataSources) + { + @source.Name + } + - - - + + + - Add - Delete - + Add Train - + From To Name + @context.From @context.To @context.Name + @@ -63,7 +87,7 @@ @@ -72,18 +96,175 @@ @code { - public TimeSpan? TimeStart { get; set; } - public TimeSpan? TimeEnd { get; set; } - public String ModelName { get; set; } = String.Empty; - private bool IsChartHidden => DisplayData == null; private bool IsChartShown => !IsChartHidden; - private TimeSeriesChart.TimeSeriesData? DisplayData { get; set; } + private class ModelForm + { + public ModelForm(Training self) + { + _self = self; + } + private DataSourceStorage _current = new(); + private readonly Training _self; + [Required] + public DataSourceStorage DataSourceStorageSource + { + get + { + return _current; + } + set + { + if(_current == value) + { + return; + } + _current = value; + CurrentModel.DataSourceName = _current.Name; + _self.InvokeAsync(_self.HandleShowQuery); + } + } + [Required] + public ModelStorage CurrentModel { get; set; } = new(); - private void HandleAdd() + public DateRange Dates { get; set; } = new DateRange(DateTime.UtcNow.Date - TimeSpan.FromDays(14), DateTime.UtcNow.Date); + + public TimeSpan? TimeStart { get; set; } + public TimeSpan? TimeEnd { get; set; } + + public TimeSpan Step { get; set; } = TimeSpan.FromSeconds(20); + public double StepSec + { + get => Step.TotalSeconds; + set => Step = TimeSpan.FromSeconds(value); + } + } + + private ModelForm? _modelForm; + private TimeSeriesData? DisplayData { get; set; } + private List _dataSources = new(); + private List _modelDefinitions = new() {new()}; + + private DateTime? _minDate; + private DateTime? _maxDate; + private bool IsAddDisabled => DisplayData==null; + + private DateTime? MinDate + { + get + { + return _minDate ?? DisplayData?.Series.FirstOrDefault()?.Data.MinBy(x=>x.TimeStamp)?.TimeStamp; + } + set + { + if (_minDate == value) return; + _minDate = value; + InvokeAsync(HandleShowQuery); + } + } + + private DateTime? MaxDate + { + get + { + return _maxDate ?? DisplayData?.Series.FirstOrDefault()?.Data.MaxBy(x => x.TimeStamp)?.TimeStamp; + } + set + { + if (_maxDate == value) return; + _maxDate = value; + InvokeAsync(HandleShowQuery); + } + } + + protected override async Task OnInitializedAsync() + { + _modelForm = new(this); + + base.OnInitialized(); + + var sources = await StorageService.Load(); + var models = await ModelService.Load(); + if (sources.Count > 0) + _dataSources = sources; + if (models.Count > 0) + _modelDefinitions = models; + + _modelForm.CurrentModel = _modelDefinitions[0]; + var source = _dataSources.FirstOrDefault(x => x.Name == _modelDefinitions[0].DataSourceName); + _modelForm.DataSourceStorageSource = source ?? _dataSources[0]; + + } + + private async Task HandleShowQuery() + { + if (_modelForm!.DataSourceStorageSource.Queries.Count < 1 || string.IsNullOrWhiteSpace(_modelForm.DataSourceStorageSource.Queries[0].Query) || _modelForm.Dates.End == null || _modelForm.Dates.Start == null) + return; + + var startDate = MinDate ?? (DateTime.UtcNow - TimeSpan.FromDays(30)); + var endDate = MaxDate ?? DateTime.UtcNow; + + // use automatic step value to always request 500 elements + var seconds = (endDate - startDate).TotalSeconds / 500.0; + if (seconds < 1.0) + seconds = 1.0; + var step = TimeSpan.FromSeconds(seconds); + + var tasks = _modelForm.DataSourceStorageSource.Queries + .Select(x => Prometheus.RangeQuery(x.Query, startDate, endDate, step, TimeSpan.FromSeconds(2))) + .ToArray(); + + try + { + await Task.WhenAll(tasks); + } + catch (Exception e) + { + ShowError(e.Message); + return; + } + + var data = new List(); + + foreach (var (res, def) in tasks.Select((x, i) => (x.Result, _modelForm.DataSourceStorageSource.Queries[i]))) + { + if (res.Status != StatusType.Success) + { + ShowError(res.Error ?? "Error"); + return; + } + + if (res.ResultType != ResultTypeType.Matrix) + { + ShowError($"Got {res.ResultType}, but Matrix expected for {def.Query}"); + return; + } + + var m = res.AsMatrix().Result; + if (m == null || m.Length != 1) + { + ShowError($"No data returned for {def.Query}"); + return; + } + + data.Add( + new() + { + Name = def.Query, + Color = def.Color, + Data = m[0].Values!.ToList() + } + ); + } + + DisplayData = new() { Series = data }; + await InvokeAsync(StateHasChanged); + } + + private void HandleAddModel() { } @@ -93,9 +274,18 @@ } - private void HandleAddTableContent() + private async Task HandleAddTableContent() { - + var interval = new IntervalDefinition + { + From = MinDate!.Value, + To = MaxDate!.Value, + Name = "Unknown", + Data = DisplayData!.Series + }; + _modelForm!.CurrentModel.IntervalDefinitionList.Add(interval); + await ModelService.Store(_modelForm!.CurrentModel); + await InvokeAsync(StateHasChanged); } private void HandleDeleteTableContent() @@ -108,27 +298,15 @@ } - - private static List tableElements = new List + private void ShowError(string text) { - new TableElement(new TimeSpan(1),new TimeSpan(1),"cc"), - new TableElement(new TimeSpan(1),new TimeSpan(1),"qq") - }; - - public class TableElement - { - public TableElement(TimeSpan from, TimeSpan to, string name) - { - From = from; - To = to; - Name = name; - } - - public TimeSpan From { get; set; } - - public TimeSpan To { get; set; } - - public string Name { get; set; } - + var options = new DialogOptions + { + CloseOnEscapeKey = true + }; + var parameters = new DialogParameters(); + parameters.Add("Text", text); + + DialogService.Show("Error", parameters, options); } } diff --git a/DeepTrace/Program.cs b/DeepTrace/Program.cs index fd27dba..5bb352f 100644 --- a/DeepTrace/Program.cs +++ b/DeepTrace/Program.cs @@ -15,6 +15,7 @@ builder.Services.AddHttpClient(c => c.BaseAddress = new UriBui builder.Services .AddSingleton( s => new MongoClient(builder.Configuration.GetValue("Connections:MongoDb") )) .AddSingleton() + .AddSingleton() ; var app = builder.Build(); diff --git a/DeepTrace/Services/IModelStorageService.cs b/DeepTrace/Services/IModelStorageService.cs new file mode 100644 index 0000000..59947c7 --- /dev/null +++ b/DeepTrace/Services/IModelStorageService.cs @@ -0,0 +1,19 @@ +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Bson; +using DeepTrace.Data; + +namespace DeepTrace.Services +{ + public class ModelStorage : ModelDefinition + { + [BsonId] + public ObjectId? Id { get; set; } + } + + public interface IModelStorageService + { + Task Delete(ModelStorage source, bool ignoreNotStored = false); + Task> Load(); + Task Store(ModelStorage source); + } +} diff --git a/DeepTrace/Services/ModelStorageService.cs b/DeepTrace/Services/ModelStorageService.cs new file mode 100644 index 0000000..3e28974 --- /dev/null +++ b/DeepTrace/Services/ModelStorageService.cs @@ -0,0 +1,58 @@ +using MongoDB.Bson; +using MongoDB.Driver; + +namespace DeepTrace.Services +{ + public class ModelStorageService : IModelStorageService + { + + private const string MongoDBDatabaseName = "DeepTrace"; + private const string MongoDBCollection = "Models"; + + private readonly IMongoClient _client; + + public ModelStorageService(IMongoClient client) + { + _client = client; + } + + public async Task> Load() + { + var db = _client.GetDatabase(MongoDBDatabaseName); + var collection = db.GetCollection(MongoDBCollection); + + var res = await (await collection.FindAsync("{}")).ToListAsync(); + return res; + } + public async Task Store(ModelStorage source) + { + var db = _client.GetDatabase(MongoDBDatabaseName); + var collection = db.GetCollection(MongoDBCollection); + + if (source.Id == null) + source.Id = ObjectId.GenerateNewId(); + + // use upsert (insert or update) to automatically handle subsequent updates + await collection.ReplaceOneAsync( + filter: new BsonDocument("_id", source.Id), + options: new ReplaceOptions { IsUpsert = true }, + replacement: source + ); + } + + public async Task Delete(ModelStorage source, bool ignoreNotStored = false) + { + if (source.Id == null) + { + if (!ignoreNotStored) + throw new InvalidDataException("Source was not stored yet. There is nothing to delete"); + return; + } + + var db = _client.GetDatabase(MongoDBDatabaseName); + var collection = db.GetCollection(MongoDBCollection); + + await collection.DeleteOneAsync($"_id = {source.Id}"); + } + } +}