mirror of
https://github.com/NecroticBamboo/DeepTrace.git
synced 2025-12-21 11:21:51 +00:00
528 lines
18 KiB
Plaintext
528 lines
18 KiB
Plaintext
@page "/datasources"
|
|
@using DeepTrace.Controls
|
|
@using DeepTrace.Data;
|
|
@using DeepTrace.Services;
|
|
@using Microsoft.ML;
|
|
@using System.Data;
|
|
@using MudBlazor;
|
|
@using PrometheusAPI;
|
|
@using System.ComponentModel.DataAnnotations;
|
|
@using Microsoft.ML.Data;
|
|
@using Microsoft.ML.TimeSeries;
|
|
|
|
@inject PrometheusClient Prometheus
|
|
@inject IDialogService DialogService
|
|
@inject IDataSourceStorageService StorageService
|
|
|
|
<PageTitle>DataSources</PageTitle>
|
|
|
|
<style>
|
|
.graph {
|
|
max-width: 800px;
|
|
max-height: 600px;
|
|
}
|
|
</style>
|
|
|
|
<h1>DataSources</h1>
|
|
|
|
<MudGrid Justify="Justify.FlexStart">
|
|
<MudItem xs="12" sm="6" md="6" lg="3">
|
|
|
|
<MudCard Class="mb-3">
|
|
<MudCardActions>
|
|
<MudSelect T="DataSourceStorage" Label="Query name" AnchorOrigin="Origin.BottomCenter" @bind-Value="_queryForm.Source">
|
|
@foreach (var source in _dataSources)
|
|
{
|
|
<MudSelectItem Value="@source">@source.Name</MudSelectItem>
|
|
}
|
|
</MudSelect>
|
|
<MudButton ButtonType="ButtonType.Submit" Variant="Variant.Filled" Color="MudBlazor.Color.Primary" Class="ml-3" OnClick="@HandleAddSource">Add</MudButton>
|
|
@if (_dataSources.Count > 1)
|
|
{
|
|
<MudButton ButtonType="ButtonType.Submit" Variant="Variant.Filled" Color="MudBlazor.Color.Primary" Class="ml-3" OnClick="@HandleDeleteSource">Delete</MudButton>
|
|
}
|
|
</MudCardActions>
|
|
</MudCard>
|
|
|
|
<MudCard>
|
|
<EditForm Model="@_queryForm" OnSubmit="@HandleSubmit" class="form-group" Context="editContext">
|
|
<DataAnnotationsValidator />
|
|
<ValidationSummary />
|
|
<MudCardContent>
|
|
|
|
<MudTextField Label="Name" @bind-Value="_queryForm.Source.Name" Variant="Variant.Text" InputType="InputType.Search" />
|
|
|
|
<MudGrid>
|
|
@for ( var i = 0; i < _queryForm.Source.Queries.Count; i++ )
|
|
{
|
|
int pos = i;
|
|
var def = _queryForm.Source.Queries[pos];
|
|
|
|
<MudItem xs="10">
|
|
<MudTextField Label="Query" @bind-Value="def.Query" Variant="Variant.Text" InputType="InputType.Search" Lines="2" />
|
|
</MudItem>
|
|
<MudItem xs="1">
|
|
<MudIconButton Icon="@Icons.Material.Outlined.Add" Variant="Variant.Outlined" aria-label="add" OnClick="@(() => AddQuery(pos))"/>
|
|
@if (_queryForm.Source.Queries.Count > 1)
|
|
{
|
|
<MudIconButton Icon="@Icons.Material.Outlined.Delete" Variant="Variant.Outlined" aria-label="delete" OnClick="@(() => DeleteQuery(pos))"/>
|
|
}
|
|
</MudItem>
|
|
<MudItem xs="12">
|
|
<MudColorPicker DisableToolbar="false" Label="Color" @bind-Text="def.Color" Style="@($"color: {def.Color};")" PickerVariant="PickerVariant.Inline" DisableAlpha=true />
|
|
</MudItem>
|
|
}
|
|
</MudGrid>
|
|
|
|
<MudDateRangePicker @ref="_picker" Label="Date range" @bind-DateRange="_queryForm.Dates" AutoClose="true">
|
|
<PickerActions>
|
|
<MudButton Class="mr-auto align-self-start" OnClick="@(() => _picker!.Clear())">Clear</MudButton>
|
|
<MudButton OnClick="@(() => _picker!.Close(false))">Cancel</MudButton>
|
|
<MudButton Color="MudBlazor.Color.Primary" OnClick="@(() => _picker!.Close())">Ok</MudButton>
|
|
</PickerActions>
|
|
</MudDateRangePicker>
|
|
|
|
<MudTimePicker Label="Start time" @bind-Time="_queryForm.TimeStart" />
|
|
<MudTimePicker Label="End time" @bind-Time="_queryForm.TimeEnd" />
|
|
@*<MudNumericField @bind-Value="_queryForm.StepSec" Label="Step (sec)" Variant="Variant.Text" Min="0" />*@
|
|
<MudTextField Label="Description" @bind-Value="_queryForm.Source.Description" Variant="Variant.Text" InputType="InputType.Search" Lines="3" />
|
|
</MudCardContent>
|
|
<MudCardActions>
|
|
<MudButton ButtonType="ButtonType.Submit" Variant="Variant.Filled" Color="MudBlazor.Color.Primary" Class="ml-auto" OnClick="@HandleTrain">Train</MudButton>
|
|
<MudButton ButtonType="ButtonType.Submit" Variant="Variant.Filled" Color="MudBlazor.Color.Primary" Class="ml-auto">Submit</MudButton>
|
|
</MudCardActions>
|
|
</EditForm>
|
|
</MudCard>
|
|
</MudItem>
|
|
<MudItem xs="12" sm="6" md="6" lg="9">
|
|
<MudCard>
|
|
<MudCardContent>
|
|
<div hidden="@IsChartShown"><MudProgressCircular Color="MudBlazor.Color.Default" /></div>
|
|
<div hidden="@IsChartHidden">
|
|
<TimeSeriesChart Data="@DisplayData" @bind-MinDate=MinDate @bind-MaxDate=MaxDate />
|
|
</div>
|
|
</MudCardContent>
|
|
</MudCard>
|
|
</MudItem>
|
|
|
|
</MudGrid>
|
|
|
|
|
|
@code {
|
|
[CascadingParameter]
|
|
protected bool IsDarkMode { get; set; }
|
|
|
|
private class PrometheusForm
|
|
{
|
|
[Required]
|
|
public DataSourceStorage Source { get; set; } = new();
|
|
|
|
public DateRange Dates { get; set; } = new DateRange(DateTime.UtcNow.Date - TimeSpan.FromDays(14), DateTime.UtcNow.Date);
|
|
|
|
public TimeSpan? TimeStart { get; set; }
|
|
public TimeSpan? TimeEnd { get; set; }
|
|
|
|
public TimeSpan Step { get; set; } = TimeSpan.FromSeconds(20);
|
|
public double StepSec
|
|
{
|
|
get => Step.TotalSeconds;
|
|
set => Step = TimeSpan.FromSeconds(value);
|
|
}
|
|
}
|
|
|
|
private PrometheusForm _queryForm = new();
|
|
private List<DataSourceStorage> _dataSources = new()
|
|
{
|
|
new()
|
|
{
|
|
Queries =
|
|
{
|
|
new("""ec2_cpu_utilization_24ae8d""", "#F57F17")
|
|
}
|
|
}
|
|
};
|
|
private MudDateRangePicker? _picker;
|
|
|
|
private bool IsChartHidden => DisplayData == null;
|
|
private bool IsChartShown => !IsChartHidden;
|
|
|
|
private DateTime? MinDate
|
|
{
|
|
get => _queryForm.Dates.Start?.Date + _queryForm.TimeStart;
|
|
set
|
|
{
|
|
if (value == null)
|
|
{
|
|
_queryForm.Dates = new DateRange(DateTime.UtcNow.Date - TimeSpan.FromDays(1), DateTime.UtcNow.Date);
|
|
return;
|
|
}
|
|
|
|
_queryForm.Dates = new DateRange(value.Value.Date, _queryForm.Dates.End);
|
|
_queryForm.TimeStart = value.Value.TimeOfDay;
|
|
|
|
InvokeAsync(StateHasChanged);
|
|
}
|
|
}
|
|
|
|
private DateTime? MaxDate
|
|
{
|
|
get => _queryForm.Dates.End?.Date + _queryForm.TimeEnd;
|
|
set
|
|
{
|
|
if (value == null)
|
|
{
|
|
_queryForm.Dates = new DateRange(DateTime.UtcNow.Date - TimeSpan.FromDays(1), DateTime.UtcNow.Date);
|
|
return;
|
|
}
|
|
|
|
_queryForm.Dates = new DateRange(_queryForm.Dates.Start, value.Value.Date);
|
|
_queryForm.TimeEnd = value.Value.TimeOfDay;
|
|
|
|
InvokeAsync(StateHasChanged);
|
|
}
|
|
}
|
|
|
|
private TimeSeriesData? DisplayData { get; set; }
|
|
|
|
private string[] _palette = new[]
|
|
{
|
|
"#F44336", "#E91E63", "#9C27B0", "#673AB7", "#3F51B5",
|
|
"#FFEBEE", "#FCE4EC", "#F3E5F5", "#EDE7F6", "#E8EAF6",
|
|
"#FFCDD2", "#F8BBD0", "#E1BEE7", "#D1C4E9", "#C5CAE9",
|
|
"#EF9A9A", "#F48FB1", "#CE93D8", "#B39DDB", "#9FA8DA",
|
|
"#E57373", "#F06292", "#BA68C8", "#9575CD", "#7986CB",
|
|
"#EF5350", "#EC407A", "#AB47BC", "#7E57C2", "#5C6BC0",
|
|
"#E53935", "#D81B60", "#8E24AA", "#5E35B1", "#3949AB"
|
|
};
|
|
|
|
|
|
protected override async Task OnInitializedAsync()
|
|
{
|
|
base.OnInitialized();
|
|
var res = await StorageService.Load();
|
|
if (res.Count > 0)
|
|
_dataSources = res;
|
|
_queryForm.Source = _dataSources[0];
|
|
}
|
|
|
|
private void AddQuery(int pos)
|
|
{
|
|
pos += 1;
|
|
var color = pos < _palette.Length
|
|
? _palette[pos]
|
|
: "#F44336"
|
|
;
|
|
_queryForm.Source.Queries.Insert(pos, new("",color));
|
|
StateHasChanged();
|
|
}
|
|
|
|
private void DeleteQuery(int pos)
|
|
{
|
|
_queryForm.Source.Queries.RemoveAt(pos);
|
|
StateHasChanged();
|
|
}
|
|
|
|
|
|
private void HandleAddSource()
|
|
{
|
|
_dataSources.Add(new() { Queries = { new("", _palette[0]) } });
|
|
_queryForm.Source = _dataSources[^1];
|
|
StateHasChanged();
|
|
}
|
|
|
|
private async Task HandleDeleteSource()
|
|
{
|
|
if (_dataSources.Count < 2)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var pos = _dataSources.IndexOf(_queryForm.Source);
|
|
if (pos < 0)
|
|
{
|
|
ShowError("Not found");
|
|
return;
|
|
}
|
|
|
|
var toDelete = _dataSources[pos];
|
|
_dataSources.RemoveAt(pos);
|
|
_queryForm.Source = _dataSources[pos<_dataSources.Count ? pos: _dataSources.Count-1];
|
|
|
|
if ( toDelete.Id == null )
|
|
{
|
|
await StorageService.Delete(toDelete);
|
|
}
|
|
|
|
StateHasChanged();
|
|
}
|
|
|
|
private async Task HandleSubmit()
|
|
{
|
|
if (_queryForm.Source.Queries.Count < 1 || string.IsNullOrWhiteSpace(_queryForm.Source.Queries[0].Query) || _queryForm.Dates.End == null || _queryForm.Dates.Start == null)
|
|
return;
|
|
|
|
var startTime = _queryForm.TimeStart
|
|
?? (
|
|
_queryForm.Dates.End!.Value.Date == DateTime.Now.Date
|
|
? DateTime.UtcNow.TimeOfDay
|
|
: new TimeSpan(0, 0, 0, 0)
|
|
)
|
|
;
|
|
|
|
var endTime = _queryForm.TimeEnd
|
|
?? (
|
|
_queryForm.Dates.End!.Value.Date == DateTime.Now.Date
|
|
? DateTime.UtcNow.TimeOfDay
|
|
: new TimeSpan(0, 23, 59, 29, 999)
|
|
)
|
|
;
|
|
|
|
var startDate = _queryForm.Dates.Start.Value.Date + startTime;
|
|
var endDate = _queryForm.Dates.End.Value.Date + endTime;
|
|
|
|
//step = _queryForm.Step;
|
|
//var n = (endDate - startDate).TotalSeconds / step.TotalSeconds;
|
|
//if (n > 1000)
|
|
|
|
// use automatic step value to always request 500 elements
|
|
var seconds = (endDate - startDate).TotalSeconds / 500.0;
|
|
if (seconds < 1.0)
|
|
seconds = 1.0;
|
|
var step = TimeSpan.FromSeconds(seconds);
|
|
|
|
// -------------------------------------------------- check syntax and format all queries
|
|
if (!await FormatQueries())
|
|
return;
|
|
|
|
// -------------------------------------------------- persist data source!
|
|
await StorageService.Store(_queryForm.Source);
|
|
|
|
// -------------------------------------------------- execute queries
|
|
var tasks = _queryForm.Source.Queries
|
|
.Select(x => Prometheus.RangeQuery(x.Query, startDate, endDate, step, TimeSpan.FromSeconds(2)))
|
|
.ToArray();
|
|
|
|
try
|
|
{
|
|
await Task.WhenAll(tasks);
|
|
}
|
|
catch ( Exception e )
|
|
{
|
|
ShowError(e.Message);
|
|
return;
|
|
}
|
|
|
|
var data = new List<TimeSeriesDataSet>();
|
|
|
|
foreach ( var (res, def) in tasks.Select((x,i) => (x.Result, _queryForm.Source.Queries[i]) ))
|
|
{
|
|
if (res.Status != StatusType.Success)
|
|
{
|
|
ShowError(res.Error ?? "Error");
|
|
return;
|
|
}
|
|
|
|
if (res.ResultType != ResultTypeType.Matrix)
|
|
{
|
|
ShowError($"Got {res.ResultType}, but Matrix expected for {def.Query}");
|
|
return;
|
|
}
|
|
|
|
var m = res.AsMatrix().Result;
|
|
if (m == null || m.Length != 1)
|
|
{
|
|
ShowError($"No data returned for {def.Query}");
|
|
return;
|
|
}
|
|
|
|
data.Add(
|
|
new()
|
|
{
|
|
Name = def.Query,
|
|
Color = def.Color,
|
|
Data = m[0].Values!.ToList()
|
|
}
|
|
);
|
|
}
|
|
|
|
DisplayData = new() { Series = data };
|
|
await InvokeAsync(StateHasChanged);
|
|
|
|
// --------------------- playground -----------------------------
|
|
|
|
// var mlContext = new MLContext();
|
|
//
|
|
// var dataView = mlContext.Data.LoadFromEnumerable<MyTimeSeries>(DisplayData.Series[0].Data.Select(x => new MyTimeSeries(Time: x.TimeStamp, Value: x.Value)));
|
|
//
|
|
// //DetectSpike(mlContext, dataView, data);
|
|
// int period = DetectPeriod(mlContext, dataView);
|
|
// DetectAnomaly(mlContext, dataView, period);
|
|
|
|
}
|
|
|
|
private void HandleTrain()
|
|
{
|
|
var mlContext = new MLContext();
|
|
|
|
var dataView = mlContext.Data.LoadFromEnumerable<MyTimeSeries>(DisplayData.Series[0].Data.Select(x => new MyTimeSeries(Time: x.TimeStamp, Value: x.Value)));
|
|
|
|
//DetectSpike(mlContext, dataView, data);
|
|
int period = DetectPeriod(mlContext, dataView);
|
|
DetectAnomaly(mlContext, dataView, period);
|
|
}
|
|
|
|
private async Task<bool> FormatQueries()
|
|
{
|
|
try
|
|
{
|
|
var formatTasks = _queryForm.Source.Queries
|
|
.Select(x => Prometheus.FormatQuery(x.Query))
|
|
.ToArray();
|
|
|
|
await Task.WhenAll(formatTasks);
|
|
|
|
for ( var i = 0; i < formatTasks.Length; i++ )
|
|
{
|
|
_queryForm.Source.Queries[i].Query = formatTasks[i].Result;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
ShowError(e.Message);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private record MyTimeSeries(DateTime Time, double Value);
|
|
|
|
private void ShowError(string text)
|
|
{
|
|
var options = new DialogOptions
|
|
{
|
|
CloseOnEscapeKey = true
|
|
};
|
|
var parameters = new DialogParameters();
|
|
parameters.Add("Text", text);
|
|
|
|
DialogService.Show<Controls.Dialog>("Error", parameters, options);
|
|
}
|
|
|
|
|
|
// -------- Tutorial: Detect anomalies in time series with ML.NET --------
|
|
|
|
private int DetectPeriod(MLContext mlContext, IDataView dataView)
|
|
{
|
|
string inputColumnName = nameof(TimeSeriesDataTutorial.Value);
|
|
int period = mlContext.AnomalyDetection.DetectSeasonality(dataView, inputColumnName);
|
|
Console.WriteLine("Period of the series is: {0}.", period);
|
|
return period;
|
|
}
|
|
|
|
private void DetectAnomaly(MLContext mlContext, IDataView dataView, int period)
|
|
{
|
|
string outputColumnName = nameof(IidSpikePrediction.Prediction);
|
|
string inputColumnName = nameof(TimeSeriesDataTutorial.Value);
|
|
var options = new SrCnnEntireAnomalyDetectorOptions()
|
|
{
|
|
Threshold = 0.3,
|
|
Sensitivity = 64.0,
|
|
DetectMode = SrCnnDetectMode.AnomalyAndMargin,
|
|
Period = period,
|
|
};
|
|
|
|
var outputDataView = mlContext.AnomalyDetection.DetectEntireAnomalyBySrCnn(dataView, outputColumnName, inputColumnName, options);
|
|
var predictions = mlContext.Data.CreateEnumerable<PredictionClass>(outputDataView, reuseRowObject: false);
|
|
Console.WriteLine("Index,Data,Anomaly,AnomalyScore,Mag,ExpectedValue,BoundaryUnit,UpperBoundary,LowerBoundary");
|
|
var index = 0;
|
|
|
|
foreach (var p in predictions)
|
|
{
|
|
if (p.Prediction is not null)
|
|
{
|
|
string output;
|
|
if (p.Prediction[0] == 1)
|
|
output = "{0},{1},{2},{3},{4}, <-- alert is on! detected anomaly";
|
|
else
|
|
output = "{0},{1},{2},{3},{4}";
|
|
|
|
Console.WriteLine(output, index, p.Prediction[0], p.Prediction[3], p.Prediction[5], p.Prediction[6]);
|
|
}
|
|
++index;
|
|
}
|
|
|
|
Console.WriteLine("");
|
|
}
|
|
|
|
public class PredictionClass
|
|
{
|
|
// Vector to hold anomaly detection results, including isAnomaly, anomalyScore,
|
|
// magnitude, expectedValue, boundaryUnits, upperBoundary and lowerBoundary.
|
|
[VectorType(7)]
|
|
public double[]? Prediction { get; set; }
|
|
}
|
|
|
|
|
|
// -------- Spike detection tutorial ---------
|
|
|
|
private static void DetectSpike(MLContext mLContext, IDataView dataView, TimeSeries[]? data)
|
|
{
|
|
string outputColumnName = nameof(IidSpikePrediction.Prediction);
|
|
string inputColumnName = nameof(TimeSeriesDataTutorial.Value);
|
|
|
|
var iidSpikeEstimator = mLContext.Transforms.DetectIidSpike(outputColumnName,
|
|
inputColumnName, 95.0d, data.Length / 4);
|
|
ITransformer iidSpikeTransform = iidSpikeEstimator.Fit(dataView);
|
|
IDataView transformedData = iidSpikeTransform.Transform(dataView);
|
|
|
|
var predictionColumn = mLContext.Data.CreateEnumerable<IidSpikePrediction>(
|
|
transformedData, reuseRowObject: false);
|
|
|
|
Console.WriteLine($"{outputColumnName} column obtained " +
|
|
$"post-transformation.");
|
|
|
|
Console.WriteLine("Number\tData\tAlert\tScore\tP-Value");
|
|
|
|
int k = 0;
|
|
foreach (var prediction in predictionColumn)
|
|
PrintPrediction(k, data[k++].Value, prediction);
|
|
}
|
|
|
|
private static void PrintPrediction(int k, float value, IidSpikePrediction prediction)
|
|
{
|
|
var anomaly = prediction.Prediction[0];
|
|
if (anomaly == 1)
|
|
{
|
|
Console.WriteLine("{0}\t{1}\t{2}\t{3:0.00}\t{4:0.00} <--- anomaly", k, value,
|
|
prediction.Prediction[0], prediction.Prediction[1],
|
|
prediction.Prediction[2]);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("{0}\t{1}\t{2}\t{3:0.00}\t{4:0.00}", k, value,
|
|
prediction.Prediction[0], prediction.Prediction[1],
|
|
prediction.Prediction[2]);
|
|
}
|
|
|
|
}
|
|
|
|
|
|
class TimeSeriesDataTutorial
|
|
{
|
|
public float Value;
|
|
|
|
public TimeSeriesDataTutorial(float value)
|
|
{
|
|
Value = value;
|
|
}
|
|
}
|
|
|
|
class IidSpikePrediction
|
|
{
|
|
[VectorType(3)]
|
|
public double[] Prediction { get; set; }
|
|
}
|
|
|
|
}
|