DEEP-26 Fixes for anomalies

QRBee.Load.Generator/LoadGenerator.cs

@@ -14,6 +14,10 @@ internal class LoadGenerator : IHostedService
     private readonly ILogger<LoadGenerator> _logger;
     private readonly IOptions<GeneratorSettings> _settings;
 
+    private TimeSpan _spikeDuration;
+    private TimeSpan _spikeDelay;
+    private double _spikeProbability;
+
     public LoadGenerator( 
         QRBee.Core.Client.Client client, 
         ClientPool clientPool, 
@@ -27,6 +31,28 @@ internal class LoadGenerator : IHostedService
         _paymentRequestGenerator = paymentRequestGenerator;
         _logger                  = logger;
         _settings                = settings;
+
+        var loadSpike     = _settings.Value.LoadSpike;
+        _spikeDuration    = TimeSpan.Zero;
+        _spikeDelay       = TimeSpan.Zero;
+        _spikeProbability = loadSpike?.Probability ?? 0.0;
+
+        if (loadSpike != null && loadSpike.Probability > 0.0)
+        {
+            if (!loadSpike.Parameters.TryGetValue("Duration", out var duration)
+                || !TimeSpan.TryParse(duration, out _spikeDuration))
+            {
+                _spikeProbability = 0.0;
+            }
+            else
+            {
+                if (!loadSpike.Parameters.TryGetValue("Delay", out duration)
+                    || !TimeSpan.TryParse(duration, out _spikeDelay))
+                {
+                    _spikeDelay = TimeSpan.FromMilliseconds(10);
+                }
+            }
+        }
     }
     public async Task StartAsync(CancellationToken cancellationToken)
     {
@@ -114,7 +140,7 @@ internal class LoadGenerator : IHostedService
 
                 var tasks = newQueue.ToList();
 
-                while (tasks.Any(x => !x.IsCompleted))
+                while (tasks.Any())
                 {
                     try
                     {
@@ -127,6 +153,7 @@ internal class LoadGenerator : IHostedService
                     {
                         Interlocked.Increment(ref _paymentsFailed);
                         _logger.LogError(ex, "Confirmation thread");
+                        tasks = tasks.Where(x => x != null).ToList();
                     }
                 }
             }
@@ -205,27 +232,6 @@ internal class LoadGenerator : IHostedService
         await Task.Delay(500 + _rng.Next() % 124);
 
         var spikeEnd         = DateTime.MinValue;
-        var loadSpike        = _settings.Value.LoadSpike;
-        var spikeDuration    = TimeSpan.Zero;
-        var spikeDelay       = TimeSpan.Zero;
-        var spikeProbability = loadSpike?.Probability ?? 0.0;
-
-        if ( loadSpike != null && loadSpike.Probability > 0.0 )
-        {
-            if ( !loadSpike.Parameters.TryGetValue("Duration", out var duration)
-                || !TimeSpan.TryParse( duration, out spikeDuration ) )
-            {
-                spikeProbability = 0.0;
-            }
-            else
-            {
-                if (!loadSpike.Parameters.TryGetValue("Delay", out duration)
-                    || !TimeSpan.TryParse(duration, out spikeDelay))
-                {
-                    spikeDelay = TimeSpan.FromMilliseconds(10);
-                }
-            }
-        }
 
         while (true)
         {
@@ -249,12 +255,13 @@ internal class LoadGenerator : IHostedService
 
             if (DateTime.Now > spikeEnd)
             {
-                if (loadSpike != null && _rng.NextDouble() < spikeProbability)
+                var dice = _rng.NextDouble();
+                if (dice < _spikeProbability)
                 {
                     // start load spike
-                    spikeEnd = DateTime.Now + spikeDuration;
+                    spikeEnd = DateTime.Now + _spikeDuration;
-                    _logger.LogWarning($"Anomaly: Load spike until {spikeEnd}");
+                    _logger.LogWarning($"Anomaly: Load spike until {spikeEnd} Dice={dice}");
-                    await Task.Delay(spikeDuration);
+                    await Task.Delay(_spikeDelay);
                 }
                 else
                 {
@@ -266,7 +273,7 @@ internal class LoadGenerator : IHostedService
             }
             else
             {
-                await Task.Delay(spikeDuration);
+                await Task.Delay(_spikeDelay);
             }
         }
     }

QRBee.Load.Generator/PaymentRequestGenerator.cs

@@ -32,6 +32,8 @@ internal class PaymentRequestGenerator
 
         if (_largeAmountValue <= 0.0)
             _largeAmountProbability = 0.0;
+        else
+            _logger.LogDebug($"Large amount spike configured: Probability={_largeAmountProbability} Value=\"{_largeAmountValue}\"");
     }
 
     public async Task<PaymentRequest> GeneratePaymentRequest(int clientId, int merchantId)
@@ -79,9 +81,10 @@ internal class PaymentRequestGenerator
 
     private decimal GetAmount()
     {
-        if (_rng.NextDouble() < _largeAmountProbability)
+        var dice = _rng.NextDouble();
+        if (dice < _largeAmountProbability)
         {
-            _logger.LogWarning($"Anomaly: Large amount");
+            _logger.LogWarning($"Anomaly: Large amount Dice={dice}");
             return Convert.ToDecimal(_rng.NextDoubleInRange(_largeAmountValue, _largeAmountValue* 1.10));
         }
         return Convert.ToDecimal(_rng.NextDoubleInRange(_minAmount, _maxAmount));

QRBee.Load.Generator/appsettings.json

@@ -14,8 +14,8 @@
   "GeneratorSettings": {
     "NumberOfClients": 100,
     "NumberOfMerchants": 10,
-    "NumberOfThreads": 20,
+    "NumberOfThreads": 10,
-    "DelayBetweenMessagesMSec": 100,
+    "DelayBetweenMessagesMSec": 300,
     "DelayJitterMSec": 50,
     "MinAmount": 10,
     "MaxAmount": 100,
@@ -23,7 +23,7 @@
     "LoadSpike": {
       "Probability": 0.001,
       "Parameters": {
-        "Duration": "00:00:05",
+        "Duration": "00:00:15",
         "Delay": "00:00:00.0100000"
       }
     },
@@ -31,7 +31,7 @@
     "LargeAmount": {
       "Probability": 0.03,
       "Parameters": {
-        "Valie": "1_000"
+        "Value": "1000"
       }
     }
   }