DEEP-33, DEEP-34, DEEP-35 Anomaly generators separated, Connections set to 500, Custom metrics added

QRBee.Load.Generator/GeneratorSettings.cs

@@ -19,5 +19,5 @@ internal class GeneratorSettings
     public Anomaly LoadSpike   { get; set; } = new();
     public Anomaly LargeAmount { get; set; } = new();
     public Anomaly TransactionCorruption { get; set;} = new();
-    public Anomaly CoherentTransactionCorruption { get; set; } = new();
+    public Anomaly UnconfirmedTransaction { get; set; } = new();
 }

QRBee.Load.Generator/LoadGenerator.cs

@@ -8,22 +8,22 @@ using QRBee.Load.Generator;
 
 internal class LoadGenerator : IHostedService
 {
-    private readonly Client _client;
+    private readonly Client                      _client;
-    private readonly ClientPool _clientPool;
+    private readonly ClientPool                  _clientPool;
-    private readonly PaymentRequestGenerator _paymentRequestGenerator;
+    private readonly PaymentRequestGenerator     _paymentRequestGenerator;
-    private readonly TransactionDefiler _transactionDefiler;
+    private readonly TransactionDefiler          _transactionDefiler;
-    private readonly ILogger<LoadGenerator> _logger;
+    private readonly UnconfirmedTransactions     _unconfirmedTransactions;
+    private readonly LoadSpike                   _loadSpike;
+    private readonly ILogger<LoadGenerator>      _logger;
     private readonly IOptions<GeneratorSettings> _settings;
 
-    private TimeSpan _spikeDuration;
-    private TimeSpan _spikeDelay;
-    private double _spikeProbability;
-
     public LoadGenerator( 
         QRBee.Core.Client.Client client, 
         ClientPool clientPool, 
         PaymentRequestGenerator paymentRequestGenerator,
         TransactionDefiler transactionDefiler,
+        UnconfirmedTransactions unconfirmedTransactions,
+        LoadSpike loadSpike,
         ILogger<LoadGenerator> logger,
         IOptions<GeneratorSettings> settings
         ) 
@@ -32,30 +32,10 @@ internal class LoadGenerator : IHostedService
         _clientPool              = clientPool;
         _paymentRequestGenerator = paymentRequestGenerator;
         _transactionDefiler      = transactionDefiler;
+        _unconfirmedTransactions = unconfirmedTransactions;
+        _loadSpike               = loadSpike;
         _logger                  = logger;
         _settings                = settings;
-
-        var loadSpike     = _settings.Value.LoadSpike;
-        _spikeDuration    = TimeSpan.Zero;
-        _spikeDelay       = TimeSpan.Zero;
-        _spikeProbability = loadSpike?.Probability ?? 0.0;
-
-        if (loadSpike != null && loadSpike.Probability > 0.0)
-        {
-            if (!loadSpike.Parameters.TryGetValue("Duration", out var duration)
-                || !TimeSpan.TryParse(duration, out _spikeDuration))
-            {
-                _spikeProbability = 0.0;
-            }
-            else
-            {
-                if (!loadSpike.Parameters.TryGetValue("Delay", out duration)
-                    || !TimeSpan.TryParse(duration, out _spikeDelay))
-                {
-                    _spikeDelay = TimeSpan.FromMilliseconds(10);
-                }
-            }
-        }
     }
     public async Task StartAsync(CancellationToken cancellationToken)
     {
@@ -200,17 +180,20 @@ internal class LoadGenerator : IHostedService
 
                         if (res?.Success ?? false)
                         {
-                            var paymentConfirmation = new PaymentConfirmation
+                            if (_unconfirmedTransactions.ShouldConfirm())
                             {
-                                MerchantId            = res.PaymentRequest.ClientResponse.MerchantRequest.MerchantId,
+                                var paymentConfirmation = new PaymentConfirmation
-                                MerchantTransactionId = res.PaymentRequest.ClientResponse.MerchantRequest.MerchantTransactionId,
+                                {
-                                GatewayTransactionId  = res.GatewayTransactionId
+                                    MerchantId            = res.PaymentRequest.ClientResponse.MerchantRequest.MerchantId,
-                            };
+                                    MerchantTransactionId = res.PaymentRequest.ClientResponse.MerchantRequest.MerchantTransactionId,
+                                    GatewayTransactionId  = res.GatewayTransactionId
+                                };
 
-                            _transactionDefiler.CorruptPaymentConfirmation(paymentConfirmation);
+                                _transactionDefiler.CorruptPaymentConfirmation(paymentConfirmation);
 
-                            var confirmationTask = _client.ConfirmPayAsync(paymentConfirmation);
+                                var confirmationTask = _client.ConfirmPayAsync(paymentConfirmation);
-                            _confirmationQueue.Add(confirmationTask);
+                                _confirmationQueue.Add(confirmationTask);
+                            }
                         }
                         else
                             Interlocked.Increment(ref _paymentsFailed);
@@ -236,8 +219,6 @@ internal class LoadGenerator : IHostedService
         // initial delay
         await Task.Delay(500 + _rng.Next() % 124);
 
-        var spikeEnd         = DateTime.MinValue;
-
         while (true)
         {
             try
@@ -260,28 +241,7 @@ internal class LoadGenerator : IHostedService
                 _logger.LogError(ex, "Generation thread");
             }
 
-            if (DateTime.Now > spikeEnd)
+            await _loadSpike.Delay();
-            {
-                var dice = _rng.NextDouble();
-                if (dice < _spikeProbability)
-                {
-                    // start load spike
-                    spikeEnd = DateTime.Now + _spikeDuration;
-                    _logger.LogWarning($"Anomaly: Load spike until {spikeEnd} Dice={dice}");
-                    await Task.Delay(_spikeDelay);
-                }
-                else
-                {
-                    await Task.Delay(_rng.NextInRange(
-                        _settings.Value.DelayBetweenMessagesMSec,
-                        _settings.Value.DelayBetweenMessagesMSec + _settings.Value.DelayJitterMSec
-                        ));
-                }
-            }
-            else
-            {
-                await Task.Delay(_spikeDelay);
-            }
         }
     }
 }

QRBee.Load.Generator/LoadSpike.cs

@@ -0,0 +1,81 @@
+using log4net.Core;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace QRBee.Load.Generator
+{
+    internal class LoadSpike
+    {
+        private readonly IOptions<GeneratorSettings> _settings;
+        private readonly ILogger<LoadSpike>          _logger;
+
+        private TimeSpan         _spikeDuration;
+        private TimeSpan         _spikeDelay;
+        private double           _spikeProbability;
+
+        private ThreadSafeRandom _rng = new();
+        private DateTime         _spikeEnd = DateTime.MinValue;
+
+        public LoadSpike( IOptions<GeneratorSettings> settings, ILogger<LoadSpike> logger )
+        {
+            _settings         = settings;
+            _logger           = logger;
+
+            var loadSpike     = settings.Value.LoadSpike;
+            _spikeDuration    = TimeSpan.Zero;
+            _spikeDelay       = TimeSpan.Zero;
+            _spikeProbability = loadSpike?.Probability ?? 0.0;
+
+            if (loadSpike != null && loadSpike.Probability > 0.0)
+            {
+                if (!loadSpike.Parameters.TryGetValue("Duration", out var duration)
+                    || !TimeSpan.TryParse(duration, out _spikeDuration))
+                {
+                    _spikeProbability = 0.0;
+                }
+                else
+                {
+                    if (loadSpike.Parameters.TryGetValue("Delay", out duration)
+                        && TimeSpan.TryParse(duration, out _spikeDelay))
+                    {
+                        _spikeDelay = TimeSpan.FromMilliseconds(10);
+                        _logger.LogDebug($"Load spike configured. Probablility={_spikeProbability} Duration=\"{_spikeDuration:g}\" Delay=\"{_spikeDelay:g}\"");
+                    }
+                }
+            }
+
+        }
+
+        public async Task Delay()
+        {
+            if (DateTime.Now > _spikeEnd)
+            {
+                var dice = _rng.NextDouble();
+                if (dice < _spikeProbability)
+                {
+                    // start load spike
+                    _spikeEnd = DateTime.Now + _spikeDuration;
+                    _logger.LogWarning($"Anomaly: Load spike until {_spikeEnd} Dice={dice}");
+                    await Task.Delay(_spikeDelay);
+                }
+                else
+                {
+                    await Task.Delay(_rng.NextInRange(
+                        _settings.Value.DelayBetweenMessagesMSec,
+                        _settings.Value.DelayBetweenMessagesMSec + _settings.Value.DelayJitterMSec
+                        ));
+                }
+            }
+            else
+            {
+                await Task.Delay(_spikeDelay);
+            }
+        }
+
+    }
+}

QRBee.Load.Generator/Program.cs

@@ -7,6 +7,7 @@ using QRBee.Api.Services;
 using QRBee.Droid.Services;
 using QRBee.Load.Generator;
 using Microsoft.Extensions.Configuration;
+using System.Net;
 
 Console.WriteLine("=== QRBee artificaial load generator ===");
 
@@ -31,13 +32,15 @@ builder.ConfigureServices((context, services) =>
         .AddSingleton<ClientPool>()
         .AddSingleton<PaymentRequestGenerator>()
         .AddSingleton<TransactionDefiler>()
+        .AddSingleton<UnconfirmedTransactions>()
+        .AddSingleton<LoadSpike>()
         .AddSingleton<PrivateKeyHandlerFactory>(x => no => new PrivateKeyHandler(x.GetRequiredService<ILogger<ServerPrivateKeyHandler>>(), x.GetRequiredService<IConfiguration>(), no))
         .AddSingleton<SecurityServiceFactory>(x => no => new AndroidSecurityService(x.GetRequiredService<PrivateKeyHandlerFactory>()(no)))
         .AddHostedService<LoadGenerator>()
         ;
 });
 
-
+ServicePointManager.DefaultConnectionLimit = 500;
 
 var host = builder.Build();
 host.Run();

QRBee.Load.Generator/TransactionDefiler.cs

@@ -7,14 +7,13 @@ namespace QRBee.Load.Generator
     internal class TransactionDefiler
     {
         private readonly ILogger<TransactionDefiler> _logger;
-        private ThreadSafeRandom _rng = new ThreadSafeRandom();
+
+        private ThreadSafeRandom _rng = new();
+
         private double _corruptionProbability;
-        private double _coherentCorruptionProbability;
+        private int    _sequenceLengthMin;
-        private bool _multipleCorruption = false;
+        private int    _sequenceLengthMax;
-        private int _sequenceLengthMin;
+        private int    _corruptionCounter;
-        private int _sequenceLengthMax;
-        private int _sequenceLength;
-        private int _corruptionCounter = 0;
 
         public TransactionDefiler(IOptions<GeneratorSettings> settings, ILogger<TransactionDefiler> logger) 
         {
@@ -22,63 +21,40 @@ namespace QRBee.Load.Generator
 
             var transactionCorruption = settings.Value.TransactionCorruption;
             _corruptionProbability = transactionCorruption.Probability;
-
+            if (_corruptionProbability > 0)
-            var coherentTransactionCorruption = settings.Value.CoherentTransactionCorruption;
-            _coherentCorruptionProbability = coherentTransactionCorruption.Probability;
-            if (_coherentCorruptionProbability > 0)
             {
-                if (coherentTransactionCorruption.Parameters.TryGetValue("SequenceLengthMin", out var s))
+                if (transactionCorruption.Parameters.TryGetValue("SequenceLengthMin", out var s))
                     _sequenceLengthMin = int.Parse(s);
-                if (coherentTransactionCorruption.Parameters.TryGetValue("SequenceLengthMax", out s))
+                if (transactionCorruption.Parameters.TryGetValue("SequenceLengthMax", out s))
                     _sequenceLengthMax = int.Parse(s);
+                _logger.LogDebug($"Transaction corruption configured: Probability={_corruptionProbability}, SequenceLengthMin={_sequenceLengthMin}, SequenceLengthMax={_sequenceLengthMax}");
             }
-
-
-            _logger.LogDebug($"Transaction corruption configured: Probability={_corruptionProbability}");
-            _logger.LogDebug($"Coherent transaction corruption configured: Probability={_coherentCorruptionProbability}, SequenceLengthMin={_sequenceLengthMin}, SequenceLengthMax={_sequenceLengthMax}");
         }
 
         public void CorruptPaymentRequest(PaymentRequest paymentRequest)
         {
-            var dice = _rng.NextDouble();
+            if(Interlocked.Decrement(ref _corruptionCounter) > 0 )
-            if(_multipleCorruption)
             {
-                _corruptionCounter++;
-                _logger.LogWarning($"Anomaly: Coherent corrupted transaction Dice={dice}, Corruption counter = {_corruptionCounter}, Sequence length = {_sequenceLength}");
                 paymentRequest.ClientResponse.MerchantRequest.Amount += 0.01M;
-
-                if (_corruptionCounter >= _sequenceLength)
-                {
-                    _corruptionCounter = 0;
-                    _multipleCorruption = false;
-                }
                 return;
             }
 
+            var dice = _rng.NextDouble();
             if (dice < _corruptionProbability)
             {
-                _logger.LogWarning($"Anomaly: Corrupted transaction Dice={dice}");
                 paymentRequest.ClientResponse.MerchantRequest.Amount += 10M;
-
+                var cc = _rng.NextInRange(_sequenceLengthMin, _sequenceLengthMax);
-                if(dice < _coherentCorruptionProbability && _multipleCorruption==false)
+                _corruptionCounter = cc;
-                {
+                _logger.LogWarning($"Anomaly: Corrupted transaction. Dice={dice} SequenceLength={cc}");
-                    _sequenceLength = _rng.NextInRange(_sequenceLengthMin,_sequenceLengthMax);
-                    _multipleCorruption = true;
-                }
-                
             }
-            
         }
 
         public void CorruptPaymentConfirmation(PaymentConfirmation paymentConfirmation)
         {
-            var dice = _rng.NextDouble();
+            if (Interlocked.Decrement(ref _corruptionCounter) > 0)
-            if (dice < _corruptionProbability)
             {
-                _logger.LogWarning($"Anomaly: Corrupted transaction confirmation Dice={dice}");
                 paymentConfirmation.GatewayTransactionId = "BadGatewayTransactionId";
             }
-
         }
     }
 }

QRBee.Load.Generator/UnconfirmedTransactions.cs

@@ -0,0 +1,52 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace QRBee.Load.Generator;
+
+internal class UnconfirmedTransactions
+{
+    private readonly ILogger<UnconfirmedTransactions> _logger;
+
+    private double           _unconfirmedProbability;
+    private TimeSpan         _unconfirmedDuration;
+    private DateTime         _anomalyEnd  = DateTime.MinValue;
+    private ThreadSafeRandom _rng = new();
+
+    public UnconfirmedTransactions(IOptions<GeneratorSettings> settings, ILogger<UnconfirmedTransactions> logger)
+    {
+        _logger                 = logger;
+        _unconfirmedProbability = settings.Value.UnconfirmedTransaction.Probability;
+        _unconfirmedDuration    = TimeSpan.Zero;
+
+        if (_unconfirmedProbability > 0.0)
+        {
+            if (settings.Value.UnconfirmedTransaction.Parameters.TryGetValue("Duration", out var duration)
+                || TimeSpan.TryParse(duration, out _unconfirmedDuration))
+            {
+                _logger.LogInformation($"Unconfirmed transactions configured. Probablility={_unconfirmedProbability}");
+            }
+            else
+            {
+                _unconfirmedProbability = 0.0;
+            }
+        }
+    }
+
+    public bool ShouldConfirm()
+    {
+        if ( DateTime.UtcNow < _anomalyEnd )
+        {
+            return false;
+        }
+
+        var dice = _rng.NextDouble();
+        if (dice < _unconfirmedProbability)
+        {
+            _anomalyEnd = DateTime.UtcNow + _unconfirmedDuration;
+            _logger.LogWarning($"Anomaly: Unconfirmed transaction. Dice={dice} Ends=\"{_anomalyEnd.ToLocalTime():s}\"");
+            return false;
+        }
+
+        return true;
+    }
+}

QRBee.Load.Generator/appsettings.json

@@ -14,40 +14,43 @@
   "GeneratorSettings": {
     "NumberOfClients": 100,
     "NumberOfMerchants": 10,
-    "NumberOfThreads": 5,
+    "NumberOfThreads": 10,
     "DelayBetweenMessagesMSec": 500,
     "DelayJitterMSec": 50,
     "MinAmount": 10,
     "MaxAmount": 100,
 
+    //0.004
     "LoadSpike": {
-      "Probability": 0.004,
+      "Probability": 0.005,
       "Parameters": {
         "Duration": "00:00:15",
         "Delay": "00:00:00.0100000"
       }
     },
 
+    //0.002
     "TransactionCorruption": {
-      "Probability": 0.004,
+      "Probability": 0,
-      "Parameters": {
-
-      }
-    },
-
-    "CoherentTransactionCorruption": {
-      "Probability": 0.002,
       "Parameters": {
         "SequenceLengthMin": 2,
         "SequenceLengthMax": 10
       }
     },
 
+    //0.003
     "LargeAmount": {
-      "Probability": 0.003,
+      "Probability": 0,
       "Parameters": {
         "Value": "1000"
       }
+    },
+
+    //0.003
+    "UnconfirmedTransaction": {
+      "Probability": 0,
+      "Parameters": {
+      }
     }
   }
 }

QRBeeApi/Services/CustomMetrics.cs

@@ -4,47 +4,58 @@ namespace QRBee.Api.Services
 {
     public class CustomMetrics
     {
-        private Counter<int> MerchantRequestCounter { get; }
+        private Counter<int>  MerchantRequestCounter               { get; }
-        private Counter<int> MerchantResponseCounter { get; }
+        private Counter<int>  MerchantResponseCounter              { get; }
-        private Counter<int> SucceededTransactionsCounter { get; }
+        private Counter<int>  SucceededTransactionsCounter         { get; }
-        private Counter<int> FailedTransactionsCounter { get; }
+        private Counter<int>  FailedTransactionsCounter            { get; }
-        private Counter<int> CorruptTransactionsCounter { get; }
+        private Counter<int>  CorruptTransactionsCounter           { get; }
-        private Counter<int> SucceededPaymentConfirmationsCounter { get; }
+        private Counter<int>  SucceededPaymentConfirmationsCounter { get; }
-        private Counter<int> FailedPaymentConfirmationsCounter { get; }
+        private Counter<int>  FailedPaymentConfirmationsCounter    { get; }
-        private Counter<long> TotalCreditCardCheckTime { get; }
+        private Counter<long> TotalCreditCardCheckTime             { get; }
-        private Counter<long> TotalPaymentTime { get; }
+        private Counter<long> TotalPaymentTime                     { get; }
 
+        private UpDownCounter<int> ConcurrentPayments { get; }
+        private UpDownCounter<int> ConcurrentConfirmations { get; }
 
         public string MetricName { get; }
 
         public CustomMetrics(string meterName = "QRBeeMetrics") {
-            var meter = new Meter(meterName);
+            var meter  = new Meter(meterName);
             MetricName = meterName;
 
-            MerchantRequestCounter = meter.CreateCounter<int>("merchant-requests", description: "Merchant has sent a request");
+            MerchantRequestCounter               = meter.CreateCounter<int>("merchant-requests", description: "Merchant has sent a request");
-            MerchantResponseCounter = meter.CreateCounter<int>("merchant-responses");
+            MerchantResponseCounter              = meter.CreateCounter<int>("merchant-responses");
 
-            SucceededTransactionsCounter = meter.CreateCounter<int>("transaction-succeeded", description: "Transaction succeeded");
+            SucceededTransactionsCounter         = meter.CreateCounter<int>("transaction-succeeded", description: "Transaction succeeded");
-            FailedTransactionsCounter = meter.CreateCounter<int>("transaction-failed", description: "Transaction failed");
+            FailedTransactionsCounter            = meter.CreateCounter<int>("transaction-failed", description: "Transaction failed");
-            CorruptTransactionsCounter = meter.CreateCounter<int>("transaction-corrupt", description: "Transaction was corrupted");
+            CorruptTransactionsCounter           = meter.CreateCounter<int>("transaction-corrupt", description: "Transaction was corrupted");
 
             SucceededPaymentConfirmationsCounter = meter.CreateCounter<int>("payment-confirmation-succeeded", description: "Payment confirmation succeeded");
-            FailedPaymentConfirmationsCounter = meter.CreateCounter<int>("payment-confirmation-failed", description: "Payment confirmation failed");
+            FailedPaymentConfirmationsCounter    = meter.CreateCounter<int>("payment-confirmation-failed", description: "Payment confirmation failed");
+
+            TotalCreditCardCheckTime             = meter.CreateCounter<long>("total-credit-card-check-time","msec");
+            TotalPaymentTime                     = meter.CreateCounter<long>("total-payment-time","msec");
+
+            ConcurrentPayments                   = meter.CreateUpDownCounter<int>("concurrent-payments");
+            ConcurrentConfirmations              = meter.CreateUpDownCounter<int>("concurrent-confirmations");
 
-            TotalCreditCardCheckTime = meter.CreateCounter<long>("total-credit-card-check-time","msec");
-            TotalPaymentTime = meter.CreateCounter<long>("total-payment-time","msec");
         }
 
 
-        public void AddMerchantRequest() => MerchantRequestCounter.Add(1);
+        public void AddMerchantRequest()                           => MerchantRequestCounter.Add(1);
-        public void AddMerchantResponse() => MerchantResponseCounter.Add(1);
+        public void AddMerchantResponse()                          => MerchantResponseCounter.Add(1);
-        public void AddSucceededTransaction() => SucceededTransactionsCounter.Add(1);
+        public void AddSucceededTransaction()                      => SucceededTransactionsCounter.Add(1);
-        public void AddFailedTransaction() => FailedTransactionsCounter.Add(1);
+        public void AddFailedTransaction()                         => FailedTransactionsCounter.Add(1);
-        public void AddCorruptTransaction() => CorruptTransactionsCounter.Add(1);
+        public void AddCorruptTransaction()                        => CorruptTransactionsCounter.Add(1);
-        public void AddSucceededPaymentConfirmation() => SucceededPaymentConfirmationsCounter.Add(1);
+        public void AddSucceededPaymentConfirmation()              => SucceededPaymentConfirmationsCounter.Add(1);
-        public void AddFailedPaymentConfirmation() => FailedPaymentConfirmationsCounter.Add(1);
+        public void AddFailedPaymentConfirmation()                 => FailedPaymentConfirmationsCounter.Add(1);
         public void AddTotalCreditCardCheckTime(long milliseconds) => TotalCreditCardCheckTime.Add(milliseconds);
-        public void AddTotalPaymentTime(long milliseconds) => TotalPaymentTime.Add(milliseconds);
+        public void AddTotalPaymentTime(long milliseconds)         => TotalPaymentTime.Add(milliseconds);
+
+        public void IncreaseConcurrentPayments()                   => ConcurrentPayments.Add(1);
+        public void DecreaseConcurrentPayments()                   => ConcurrentPayments.Add(-1);
+        public void IncreaseConcurrentConfirmations()              => ConcurrentConfirmations.Add(1);
+        public void DecreaseConcurrentConfirmation()               => ConcurrentConfirmations.Add(-1);
 
     }
 }

QRBeeApi/Services/QRBeeAPIService.cs

@@ -15,20 +15,28 @@ namespace QRBee.Api.Services
     /// </summary>
     public class QRBeeAPIService: IQRBeeAPI
     {
-        private readonly IStorage           _storage;
+        private readonly IStorage                 _storage;
-        private readonly ISecurityService   _securityService;
+        private readonly ISecurityService         _securityService;
-        private readonly IPrivateKeyHandler _privateKeyHandler;
+        private readonly IPrivateKeyHandler       _privateKeyHandler;
-        private readonly IPaymentGateway    _paymentGateway;
+        private readonly IPaymentGateway          _paymentGateway;
         private readonly ILogger<QRBeeAPIService> _logger;
-        private readonly TransactionMonitoring _transactionMonitoring;
+        private readonly TransactionMonitoring    _transactionMonitoring;
-        private static readonly object      _lock = new ();
+        private static readonly object            _lock = new ();
 
-        private readonly CustomMetrics _customMetrics;
+        private readonly CustomMetrics            _customMetrics;
 
         private const int MaxNameLength = 512;
         private const int MaxEmailLength = 512;
 
-        public QRBeeAPIService(IStorage storage, ISecurityService securityService, IPrivateKeyHandler privateKeyHandler, IPaymentGateway paymentGateway, ILogger<QRBeeAPIService> logger, TransactionMonitoring transactionMonitoring, CustomMetrics metrics)
+        public QRBeeAPIService(
+            IStorage storage,
+            ISecurityService securityService,
+            IPrivateKeyHandler privateKeyHandler,
+            IPaymentGateway paymentGateway,
+            ILogger<QRBeeAPIService> logger,
+            TransactionMonitoring transactionMonitoring,
+            CustomMetrics metrics
+            )
         {
             _storage           = storage;
             _securityService   = securityService;
@@ -132,8 +140,19 @@ namespace QRBee.Api.Services
                 throw new ApplicationException($"Digital signature is not valid.");
             }
         }
-
         public async Task<PaymentResponse> Pay(PaymentRequest value)
+        {
+            _customMetrics.IncreaseConcurrentPayments();
+            try
+            {
+                return await PayInternal(value);
+            }
+            finally
+            {
+                _customMetrics.DecreaseConcurrentPayments();
+            }
+        }
+        public async Task<PaymentResponse> PayInternal(PaymentRequest value)
         {
 
             // --------------------------------- RECEIVE PAYMENT REQUEST --------------------------------------
@@ -233,6 +252,7 @@ namespace QRBee.Api.Services
                 //10. Make response for merchant
                 var response = MakePaymentResponse(value, info.TransactionId ?? "", gatewayResponse.GatewayTransactionId ?? "", info.Status==TransactionInfo.TransactionStatus.Succeeded, info.RejectReason);
                 
+
                 _customMetrics.AddMerchantResponse();
                 return response;
             }
@@ -404,6 +424,18 @@ namespace QRBee.Api.Services
         }
 
         public async Task ConfirmPay(PaymentConfirmation value)
+        {
+            _customMetrics.IncreaseConcurrentConfirmations();
+            try
+            {
+                await ConfirmPayInternal(value);
+            }
+            finally
+            {
+                _customMetrics.DecreaseConcurrentConfirmation();
+            }
+        }
+        public async Task ConfirmPayInternal(PaymentConfirmation value)
         {
             var id = $"{value.MerchantId}-{value.MerchantTransactionId}";
             var trans = await _storage.GetTransactionInfoByTransactionId(id);