DEEP-26 LoadSpike shows spike end

QRBee.Load.Generator/GeneratorSettings.cs

@@ -8,6 +8,8 @@ internal class Anomaly
 
 internal class GeneratorSettings
 {
+    public string QRBeeURL              { get; set; } = "http://localhost:5000/";
+    public int DefaultConnectionLimit   { get; set; } = 100;
     public int NumberOfClients          { get; set; } = 100;
     public int NumberOfMerchants        { get; set; } = 10;
     public int NumberOfThreads          { get; set; } = 20;

QRBee.Load.Generator/LoadGenerator.cs

@@ -36,6 +36,8 @@ internal class LoadGenerator : IHostedService
         _loadSpike               = loadSpike;
         _logger                  = logger;
         _settings                = settings;
+
+        _logger.LogInformation($"Connected to QRBee on {_client.BaseUrl}");
     }
     public async Task StartAsync(CancellationToken cancellationToken)
     {
@@ -70,12 +72,28 @@ internal class LoadGenerator : IHostedService
     {
         DateTime nextReport = DateTime.MinValue;
 
+        var lastPaymentsGenerated = 0L;
+        var lastPaymentsProcessed = 0L;
+        var lastPaymentsConfirmed = 0L;
+        var lastPaymentsFailed    = 0L;
+
         while (true)
         {
             if (DateTime.Now > nextReport)
             {
-                _logger.LogInformation($"S: {_paymentsGenerated,10:N0} R: {_paymentsProcessed,10:N0} C: {_paymentsConfirmed,10:N0} F: {_paymentsFailed,10:N0}");
+                _logger.LogInformation(
+                    $"S: {_paymentsGenerated-lastPaymentsGenerated,10:N0} " +
+                    $"R: {_paymentsProcessed-lastPaymentsProcessed,10:N0} " +
+                    $"C: {_paymentsConfirmed-lastPaymentsConfirmed,10:N0} " +
+                    $"F: {_paymentsFailed-lastPaymentsFailed,10:N0}"
+                    );
+
                 nextReport = DateTime.Now + TimeSpan.FromSeconds(1);
+
+                lastPaymentsGenerated = _paymentsGenerated;
+                lastPaymentsProcessed = _paymentsProcessed;
+                lastPaymentsConfirmed = _paymentsConfirmed;
+                lastPaymentsFailed    = _paymentsFailed;
             }
             await Task.Delay(1000);
         }
@@ -157,7 +175,9 @@ internal class LoadGenerator : IHostedService
                 var newQueue = new List<Task<PaymentResponse>>();
 
                 lock (_lock)
-                    newQueue = Interlocked.Exchange(ref _responseQueue, newQueue);
+                {
+                    (newQueue, _responseQueue) = (_responseQueue, newQueue);
+                }
 
                 if (newQueue.Count == 0)
                 {
@@ -165,7 +185,7 @@ internal class LoadGenerator : IHostedService
                     continue;
                 }
 
-                var tasks = newQueue.ToList();
+                var tasks = newQueue.Where(x => x != null).ToList();
 
                 while ( tasks.Any() )
                 {
@@ -198,6 +218,19 @@ internal class LoadGenerator : IHostedService
                         else
                             Interlocked.Increment(ref _paymentsFailed);
                     }
+                    catch (TaskCanceledException )
+                    {
+                        Interlocked.Increment(ref _paymentsFailed);
+                        // no message - too noisy
+                    }
+                    catch (HttpRequestException httpEx)
+                    {
+                        if ( httpEx.InnerException is IOException )
+                        {
+                            Interlocked.Increment(ref _paymentsFailed);
+                            // no message - too noisy
+                        }
+                    }
                     catch (Exception ex)
                     {
                         Interlocked.Increment(ref _paymentsFailed);
@@ -232,6 +265,7 @@ internal class LoadGenerator : IHostedService
 
                 var resp = _client.PayAsync(req);
 
+                lock (_lock)
                     _responseQueue.Add(resp);
                 Interlocked.Increment(ref _paymentsGenerated);
             }

QRBee.Load.Generator/LoadSpike.cs

@@ -17,6 +17,7 @@ namespace QRBee.Load.Generator
         private TimeSpan         _spikeDuration;
         private TimeSpan         _spikeDelay;
         private double           _spikeProbability;
+        private bool             _spikeActive;
 
         private ThreadSafeRandom _rng = new();
         private DateTime         _spikeEnd = DateTime.MinValue;
@@ -55,12 +56,21 @@ namespace QRBee.Load.Generator
         {
             if (DateTime.Now > _spikeEnd)
             {
+                if (_spikeActive)
+                {
+                    _spikeActive = false;
+                    _logger.LogWarning($"Anomaly: Load spike ended");
+                }
+
                 var dice = _rng.NextDouble();
                 if (dice < _spikeProbability)
                 {
                     // start load spike
                     _spikeEnd = DateTime.Now + _spikeDuration;
+                    _spikeActive = true;
+
                     _logger.LogWarning($"Anomaly: Load spike until {_spikeEnd} Dice={dice}");
+
                     await Task.Delay(_spikeDelay);
                 }
                 else

QRBee.Load.Generator/Program.cs

@@ -1,5 +1,4 @@
-// See https://aka.ms/new-console-template for more information
+using log4net;
-using log4net;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
@@ -8,6 +7,7 @@ using QRBee.Droid.Services;
 using QRBee.Load.Generator;
 using Microsoft.Extensions.Configuration;
 using System.Net;
+using Microsoft.Extensions.Options;
 
 Console.WriteLine("=== QRBee artificaial load generator ===");
 
@@ -24,7 +24,12 @@ builder.ConfigureServices((context, services) =>
     });
 
     services
-        .AddHttpClient<QRBee.Core.Client.Client, QRBee.Core.Client.Client>(httpClient => new QRBee.Core.Client.Client("https://localhost:7000/", httpClient))
+        .AddHttpClient<QRBee.Core.Client.Client, QRBee.Core.Client.Client>((httpClient,x) => 
+        {
+            var settings = x.GetRequiredService<IOptions<GeneratorSettings>>();
+            var url = settings.Value.QRBeeURL;
+            return new QRBee.Core.Client.Client(url, httpClient); 
+        })
         .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler() { ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator });
         ;
     services
@@ -40,7 +45,9 @@ builder.ConfigureServices((context, services) =>
         ;
 });
 
-ServicePointManager.DefaultConnectionLimit = 500;
+ServicePointManager.DefaultConnectionLimit = 10;
+ServicePointManager.ReusePort = true;
+ServicePointManager.CheckCertificateRevocationList = false;
 
 var host = builder.Build();
 host.Run();

QRBee.Load.Generator/appsettings.json

@@ -12,6 +12,7 @@
   },
 
   "GeneratorSettings": {
+    "QRBeeURL": "http://localhost:5000/",
     "NumberOfClients": 100,
     "NumberOfMerchants": 10,
     "NumberOfThreads": 10,
@@ -22,7 +23,7 @@
 
     //0.004
     "LoadSpike": {
-      "Probability": 0.005,
+      "Probability": 0.002,
       "Parameters": {
         "Duration": "00:00:15",
         "Delay": "00:00:00.0100000"

QRBee.Load.Generator/log4net.config

@@ -33,7 +33,7 @@
       <forecolor value="Blue" />
     </mapping>
     <layout type="log4net.Layout.PatternLayout">
-      <conversionPattern value="%date{HH:mm:ss,fff} [%-2thread] %-5level %-15logger{1} %message%newline" />
+      <conversionPattern value="%date{HH:mm:ss,fff} [%3thread] %-5level %-15logger{1} %message%newline" />
     </layout>
   </appender>
 	
@@ -41,7 +41,7 @@
 		<file value="log-file.txt" />
 		<appendToFile value="false" />
 		<layout type="log4net.Layout.PatternLayout">
-			<conversionPattern value="%date [%thread] %-5level %logger [%property{NDC}] - %message%newline" />
+			<conversionPattern value="%date [%3thread] %-5level %logger [%property{NDC}] - %message%newline" />
 		</layout>
 	</appender>
 	

QRBeeApi/Program.cs

@@ -6,6 +6,7 @@ using QRBee.Api;
 using QRBee.Api.Services;
 using QRBee.Api.Services.Database;
 using QRBee.Core.Security;
+using System.Net;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -56,17 +57,18 @@ builder.Services
     .AddSingleton<CustomMetrics>()
     ;
 
+ServicePointManager.DefaultConnectionLimit = 10;
+ServicePointManager.ReusePort = true;
+ServicePointManager.CheckCertificateRevocationList = false;
+
 var app = builder.Build();
 
 // Configure the HTTP request pipeline.
-//if (app.Environment.IsDevelopment())
+app.UseOpenTelemetryPrometheusScrapingEndpoint();
-{
 app.UseSwagger();
 app.UseSwaggerUI();
-}
 
-app.UseOpenTelemetryPrometheusScrapingEndpoint();
+//app.UseHttpsRedirection();
-app.UseHttpsRedirection();
 app.UseAuthorization();
 app.MapControllers();
 

QRBeeApi/Properties/launchSettings.json

@@ -1,13 +1,4 @@
 {
-  "$schema": "https://json.schemastore.org/launchsettings.json",
-  "iisSettings": {
-    "windowsAuthentication": false,
-    "anonymousAuthentication": true,
-    "iisExpress": {
-      "applicationUrl": "http://localhost:1103",
-      "sslPort": 44300
-    }
-  },
   "profiles": {
     "QRBeeApi": {
       "commandName": "Project",
@@ -16,7 +7,7 @@
         "ASPNETCORE_ENVIRONMENT": "Development",
         "LOGS_ROOT": "C:\\Users\\andre\\logs"
       },
-      "applicationUrl": "https://localhost:7000;http://localhost:5000",
+      "applicationUrl": "http://localhost:5000",
       "dotnetRunMessages": true
     },
     "Docker": {
@@ -25,5 +16,14 @@
       "publishAllPorts": true,
       "useSSL": true
     }
+  },
+  "$schema": "https://json.schemastore.org/launchsettings.json",
+  "iisSettings": {
+    "windowsAuthentication": false,
+    "anonymousAuthentication": true,
+    "iisExpress": {
+      "applicationUrl": "http://localhost:1103",
+      "sslPort": 44300
+    }
   }
 }

QRBeeApi/QRBee.Api.csproj

@@ -18,16 +18,16 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Logging.Log4Net.AspNetCore" Version="6.1.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.15.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.18.1" />
-    <PackageReference Include="MongoDB.Driver" Version="2.15.0" />
+    <PackageReference Include="MongoDB.Driver" Version="2.20.0" />
-    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.5.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.6.0-alpha.1" />
-    <PackageReference Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" Version="1.5.0-rc.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" Version="1.6.0-alpha.1" />
     <PackageReference Include="OpenTelemetry.Instrumentation.Process" Version="0.5.0-beta.3" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.3.0" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
-    <PackageReference Include="OpenTelemetry" Version="1.5.1" />
+    <PackageReference Include="OpenTelemetry" Version="1.6.0-alpha.1" />
-    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.5.1" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.6.0-alpha.1" />
-    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.5.0-beta.1" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.5.1-beta.1" />
-    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.5.0-beta.1" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.5.1-beta.1" />
     <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.5.0" />
   </ItemGroup>
 

QRBeeApi/log4net.config

@@ -25,7 +25,7 @@
       <forecolor value="Blue" />
     </mapping>
     <layout type="log4net.Layout.PatternLayout">
-      <conversionPattern value="%date{HH:mm:ss,fff} [%-2thread] %-5level %-15logger{1} %message%newline" />
+      <conversionPattern value="%date{HH:mm:ss,fff} [%3thread] %-5level %-15logger{1} %message%newline" />
     </layout>
   </appender>