DEEP-18 Persistence added for data sources

DeepTrace/Data/DataSourceDefinition.cs

@@ -10,7 +10,7 @@ public class DataSourceQuery
     }
     
     public string Query { get; set; }
-    public MudBlazor.Utilities.MudColor Color { get; set; }
+    public string Color { get; set; }
     
 }
 

DeepTrace/DeepTrace.csproj

@@ -10,13 +10,10 @@
     <PackageReference Include="Blazor-ApexCharts" Version="0.9.21-beta" />
     <PackageReference Include="Microsoft.ML" Version="2.0.1" />
     <PackageReference Include="Microsoft.ML.TimeSeries" Version="2.0.1" />
+    <PackageReference Include="MongoDB.Driver" Version="2.20.0" />
     <PackageReference Include="MudBlazor" Version="6.4.1" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="Services\" />
-  </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\PrometheusAPI\PrometheusAPI.csproj" />
   </ItemGroup>

DeepTrace/Pages/DataSources.razor

@@ -1,6 +1,7 @@
 @page "/datasources"
 @using DeepTrace.Controls
 @using DeepTrace.Data;
+@using DeepTrace.Services;
 @using Microsoft.ML;
 @using System.Data;
 @using MudBlazor;
@@ -11,6 +12,7 @@
 
 @inject PrometheusClient Prometheus
 @inject IDialogService DialogService
+@inject IDataSourceStorageService StorageService
 
 <PageTitle>DataSources</PageTitle>
 
@@ -28,7 +30,7 @@
         
         <MudCard Class="mb-3">
             <MudCardActions>
-                <MudSelect T="DataSourceDefinition" Label="Query name" AnchorOrigin="Origin.BottomCenter" @bind-Value="_queryForm.Source">
+                <MudSelect T="DataSourceStorage" Label="Query name" AnchorOrigin="Origin.BottomCenter" @bind-Value="_queryForm.Source">
                     @foreach (var source in _dataSources)
                     {
                         <MudSelectItem Value="@source">@source.Name</MudSelectItem>
@@ -67,7 +69,7 @@
                                 }
                             </MudItem>
                             <MudItem xs="12">
-                                <MudColorPicker DisableToolbar="false" Label="Color" @bind-Value="def.Color" PickerVariant="PickerVariant.Inline" DisableAlpha=true />
+                                <MudColorPicker DisableToolbar="false" Label="Color" @bind-Text="def.Color" Style="@($"color: {def.Color};")" PickerVariant="PickerVariant.Inline" DisableAlpha=true />
                             </MudItem>
                         }
                     </MudGrid>
@@ -113,7 +115,7 @@
     private class PrometheusForm
     {
         [Required]
-        public DataSourceDefinition Source { get; set; } = new();
+        public DataSourceStorage Source { get; set; } = new();
 
         public DateRange Dates { get; set; } = new DateRange(DateTime.UtcNow.Date - TimeSpan.FromDays(14), DateTime.UtcNow.Date);
 
@@ -129,7 +131,7 @@
     }
 
     private PrometheusForm _queryForm = new();
-    private List<DataSourceDefinition> _dataSources = new() 
+    private List<DataSourceStorage> _dataSources = new()
     { 
         new() 
         { 
@@ -194,9 +196,12 @@
     };
 
 
-    protected override void OnInitialized()
+    protected override async Task OnInitializedAsync()
     {
         base.OnInitialized();
+        var res = await StorageService.Load();
+        if (res.Count > 0)
+            _dataSources = res;
         _queryForm.Source = _dataSources[0];
     }
 
@@ -225,7 +230,7 @@
         StateHasChanged();
     }
 
-    private void HandleDeleteSource()
+    private async Task HandleDeleteSource()
     {
         if (_dataSources.Count < 2)
         {
@@ -238,8 +243,16 @@
             ShowError("Not found");
             return;
         }
+
+        var toDelete = _dataSources[pos];
         _dataSources.RemoveAt(pos);
         _queryForm.Source = _dataSources[pos<_dataSources.Count ? pos: _dataSources.Count-1];
+
+        if ( toDelete.Id == null )
+        {
+            await StorageService.Delete(toDelete);
+        }
+
         StateHasChanged();
     }
 
@@ -267,19 +280,24 @@
         var startDate = _queryForm.Dates.Start.Value.Date + startTime;
         var endDate = _queryForm.Dates.End.Value.Date + endTime;
 
-        if (!await FormatQueries())
-            return;
-
-        TimeSpan step;
-
         //step = _queryForm.Step;
         //var n = (endDate - startDate).TotalSeconds / step.TotalSeconds;
         //if (n > 1000)
+
+        // use automatic step value to always request 500 elements
         var seconds = (endDate - startDate).TotalSeconds / 500.0;
         if (seconds < 1.0)
             seconds = 1.0;
-        step = TimeSpan.FromSeconds(seconds);
+        var step = TimeSpan.FromSeconds(seconds);
 
+        // -------------------------------------------------- check syntax and format all queries
+        if (!await FormatQueries())
+            return;
+
+        // -------------------------------------------------- persist data source!
+        await StorageService.Store(_queryForm.Source);
+
+        // -------------------------------------------------- execute queries
         var tasks = _queryForm.Source.Queries
             .Select(x => Prometheus.RangeQuery(x.Query, startDate, endDate, step, TimeSpan.FromSeconds(2)))
             .ToArray();
@@ -321,7 +339,7 @@
                 new()
                 {
                     Name  = def.Query, 
-                    Color = def.Color.Value,
+                    Color = def.Color,
                     Data  = m[0].Values!.ToList()
                 }
             );

DeepTrace/Program.cs

@@ -1,8 +1,7 @@
-using Microsoft.AspNetCore.Components;
-using Microsoft.AspNetCore.Components.Web;
-using DeepTrace.Data;
 using MudBlazor.Services;
 using PrometheusAPI;
+using MongoDB.Driver;
+using DeepTrace.Services;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -11,7 +10,12 @@ builder.Services.AddRazorPages();
 builder.Services.AddServerSideBlazor();
 builder.Services.AddMudServices();
 
-builder.Services.AddHttpClient<PrometheusClient>(c => c.BaseAddress = new UriBuilder("http://localhost:9090").Uri);
+builder.Services.AddHttpClient<PrometheusClient>(c => c.BaseAddress = new UriBuilder(builder.Configuration.GetValue<string>("Connections:Prometheus")!).Uri);
+
+builder.Services
+    .AddSingleton<IMongoClient>( s => new MongoClient(builder.Configuration.GetValue<string>("Connections:MongoDb") ))
+    .AddSingleton<IDataSourceStorageService, DataSourceStorageService>()
+    ;
 
 var app = builder.Build();
 
@@ -21,7 +25,6 @@ if (!app.Environment.IsDevelopment())
     app.UseExceptionHandler("/Error");
 }
 
-
 app.UseStaticFiles();
 
 app.UseRouting();

DeepTrace/Services/DataSourceStorageService.cs

@@ -0,0 +1,58 @@
+using MongoDB.Bson;
+using MongoDB.Driver;
+
+namespace DeepTrace.Services
+{
+    public class DataSourceStorageService : IDataSourceStorageService
+    {
+
+        private const string MongoDBDatabaseName = "DeepTrace";
+        private const string MongoDBCollection = "Sources";
+
+        private readonly IMongoClient _client;
+
+        public DataSourceStorageService(IMongoClient client)
+        {
+            _client = client;
+        }
+
+        public async Task<List<DataSourceStorage>> Load()
+        {
+            var db = _client.GetDatabase(MongoDBDatabaseName);
+            var collection = db.GetCollection<DataSourceStorage>(MongoDBCollection);
+
+            var res = await (await collection.FindAsync("{}")).ToListAsync();
+            return res;
+        }
+        public async Task Store(DataSourceStorage source)
+        {
+            var db = _client.GetDatabase(MongoDBDatabaseName);
+            var collection = db.GetCollection<DataSourceStorage>(MongoDBCollection);
+
+            if ( source.Id == null ) 
+                source.Id = ObjectId.GenerateNewId();
+
+            // use upsert (insert or update) to automatically handle subsequent updates
+            await collection.ReplaceOneAsync(
+                filter: new BsonDocument("_id", source.Id),
+                options: new ReplaceOptions { IsUpsert = true },
+                replacement: source
+                );
+        }
+
+        public async Task Delete(DataSourceStorage source, bool ignoreNotStored = false)
+        {
+            if ( source.Id == null )
+            {
+                if (!ignoreNotStored)
+                    throw new InvalidDataException("Source was not stored yet. There is nothing to delete");
+                return;
+            }
+
+            var db = _client.GetDatabase(MongoDBDatabaseName);
+            var collection = db.GetCollection<DataSourceStorage>(MongoDBCollection);
+
+            await collection.DeleteOneAsync($"_id = {source.Id}");
+        }
+    }
+}

DeepTrace/Services/IDataSourceStorageService.cs

@@ -0,0 +1,19 @@
+using DeepTrace.Data;
+using MongoDB.Bson.Serialization.Attributes;
+using MongoDB.Bson;
+
+namespace DeepTrace.Services
+{
+    public class DataSourceStorage : DataSourceDefinition
+    {
+        [BsonId]
+        public ObjectId? Id { get; set; }
+    }
+
+    public interface IDataSourceStorageService
+    {
+        Task Delete(DataSourceStorage source, bool ignoreNotStored = false);
+        Task<List<DataSourceStorage>> Load();
+        Task Store(DataSourceStorage source);
+    }
+}

DeepTrace/appsettings.json

@@ -5,5 +5,10 @@
       "Microsoft.AspNetCore": "Warning"
     }
   },
-  "AllowedHosts": "*"
+  "AllowedHosts": "*",
+  
+  "Connections": {
+    "Prometheus": "http://localhost:9090",
+    "MongoDb": "mongodb://localhost:27017"
+  }
 }