[PM-17562] Add strict delay support for RabbitMQ; Refactor implementation (#5899)

* [PM-17562] Add strict delay support for RabbitMQ

* fix lint error

* Added more robust FailureReason handling and some additional tests

* Fix two issues noted by SonarQube

* Fix typo; Add alternate handling if MessageId is null or empty

* Set MessageId on all message publishers

test/Core.Test/AdminConsole/Models/Data/Integrations/IntegrationMessageTests.cs

@@ -7,12 +7,17 @@ namespace Bit.Core.Test.Models.Data.Integrations;
 
 public class IntegrationMessageTests
 {
+    private const string _messageId = "TestMessageId";
+
     [Fact]
     public void ApplyRetry_IncrementsRetryCountAndSetsDelayUntilDate()
     {
         var message = new IntegrationMessage<WebhookIntegrationConfigurationDetails>
         {
+            Configuration = new WebhookIntegrationConfigurationDetails("https://localhost"),
+            MessageId = _messageId,
             RetryCount = 2,
+            RenderedTemplate = string.Empty,
             DelayUntilDate = null
         };
 
@@ -30,19 +35,22 @@ public class IntegrationMessageTests
         var message = new IntegrationMessage<WebhookIntegrationConfigurationDetails>
         {
             Configuration = new WebhookIntegrationConfigurationDetails("https://localhost"),
+            MessageId = _messageId,
             RenderedTemplate = "This is the message",
             IntegrationType = IntegrationType.Webhook,
             RetryCount = 2,
-            DelayUntilDate = null
+            DelayUntilDate = DateTime.UtcNow
         };
 
         var json = message.ToJson();
         var result = IntegrationMessage<WebhookIntegrationConfigurationDetails>.FromJson(json);
 
         Assert.Equal(message.Configuration, result.Configuration);
+        Assert.Equal(message.MessageId, result.MessageId);
         Assert.Equal(message.RenderedTemplate, result.RenderedTemplate);
         Assert.Equal(message.IntegrationType, result.IntegrationType);
         Assert.Equal(message.RetryCount, result.RetryCount);
+        Assert.Equal(message.DelayUntilDate, result.DelayUntilDate);
     }
 
     [Fact]
@@ -51,4 +59,26 @@ public class IntegrationMessageTests
         var json = "{ Invalid JSON";
         Assert.Throws<JsonException>(() => IntegrationMessage<WebhookIntegrationConfigurationDetails>.FromJson(json));
     }
+
+    [Fact]
+    public void ToJson_BaseIntegrationMessage_DeserializesCorrectly()
+    {
+        var message = new IntegrationMessage
+        {
+            MessageId = _messageId,
+            RenderedTemplate = "This is the message",
+            IntegrationType = IntegrationType.Webhook,
+            RetryCount = 2,
+            DelayUntilDate = DateTime.UtcNow
+        };
+
+        var json = message.ToJson();
+        var result = JsonSerializer.Deserialize<IntegrationMessage>(json);
+
+        Assert.Equal(message.MessageId, result.MessageId);
+        Assert.Equal(message.RenderedTemplate, result.RenderedTemplate);
+        Assert.Equal(message.IntegrationType, result.IntegrationType);
+        Assert.Equal(message.RetryCount, result.RetryCount);
+        Assert.Equal(message.DelayUntilDate, result.DelayUntilDate);
+    }
 }

test/Core.Test/AdminConsole/Services/AzureServiceBusEventListenerServiceTests.cs

@@ -0,0 +1,133 @@
+using System.Text.Json;
+using Azure.Messaging.ServiceBus;
+using Bit.Core.Models.Data;
+using Bit.Core.Services;
+using Bit.Test.Common.AutoFixture;
+using Bit.Test.Common.AutoFixture.Attributes;
+using Bit.Test.Common.Helpers;
+using Microsoft.Extensions.Logging;
+using NSubstitute;
+using Xunit;
+
+namespace Bit.Core.Test.Services;
+
+[SutProviderCustomize]
+public class AzureServiceBusEventListenerServiceTests
+{
+    private readonly IEventMessageHandler _handler = Substitute.For<IEventMessageHandler>();
+    private readonly ILogger<AzureServiceBusEventListenerService> _logger =
+        Substitute.For<ILogger<AzureServiceBusEventListenerService>>();
+    private const string _messageId = "messageId";
+
+    private SutProvider<AzureServiceBusEventListenerService> GetSutProvider()
+    {
+        return new SutProvider<AzureServiceBusEventListenerService>()
+            .SetDependency(_handler)
+            .SetDependency(_logger)
+            .SetDependency("test-subscription", "subscriptionName")
+            .Create();
+    }
+
+    [Theory, BitAutoData]
+    public async Task ProcessErrorAsync_LogsError(ProcessErrorEventArgs args)
+    {
+        var sutProvider = GetSutProvider();
+
+        await sutProvider.Sut.ProcessErrorAsync(args);
+
+        _logger.Received(1).Log(
+            LogLevel.Error,
+            Arg.Any<EventId>(),
+            Arg.Any<object>(),
+            Arg.Any<Exception>(),
+            Arg.Any<Func<object, Exception, string>>());
+    }
+
+    [Fact]
+    public async Task ProcessReceivedMessageAsync_EmptyJson_LogsError()
+    {
+        var sutProvider = GetSutProvider();
+        await sutProvider.Sut.ProcessReceivedMessageAsync(string.Empty, _messageId);
+
+        _logger.Received(1).Log(
+            LogLevel.Error,
+            Arg.Any<EventId>(),
+            Arg.Any<object>(),
+            Arg.Any<JsonException>(),
+            Arg.Any<Func<object, Exception, string>>());
+    }
+
+    [Fact]
+    public async Task ProcessReceivedMessageAsync_InvalidJson_LogsError()
+    {
+        var sutProvider = GetSutProvider();
+        await sutProvider.Sut.ProcessReceivedMessageAsync("{ Inavlid JSON }", _messageId);
+
+        _logger.Received(1).Log(
+            LogLevel.Error,
+            Arg.Any<EventId>(),
+            Arg.Is<object>(o => o.ToString().Contains("Invalid JSON")),
+            Arg.Any<Exception>(),
+            Arg.Any<Func<object, Exception, string>>());
+    }
+
+    [Fact]
+    public async Task ProcessReceivedMessageAsync_InvalidJsonArray_LogsError()
+    {
+        var sutProvider = GetSutProvider();
+        await sutProvider.Sut.ProcessReceivedMessageAsync(
+            "{ \"not a valid\", \"list of event messages\" }",
+            _messageId
+        );
+
+        _logger.Received(1).Log(
+            LogLevel.Error,
+            Arg.Any<EventId>(),
+            Arg.Any<object>(),
+            Arg.Any<JsonException>(),
+            Arg.Any<Func<object, Exception, string>>());
+    }
+
+    [Fact]
+    public async Task ProcessReceivedMessageAsync_InvalidJsonObject_LogsError()
+    {
+        var sutProvider = GetSutProvider();
+        await sutProvider.Sut.ProcessReceivedMessageAsync(
+            JsonSerializer.Serialize(DateTime.UtcNow), // wrong object - not EventMessage
+            _messageId
+        );
+
+        _logger.Received(1).Log(
+            LogLevel.Error,
+            Arg.Any<EventId>(),
+            Arg.Any<object>(),
+            Arg.Any<JsonException>(),
+            Arg.Any<Func<object, Exception, string>>());
+    }
+
+    [Theory, BitAutoData]
+    public async Task ProcessReceivedMessageAsync_SingleEvent_DelegatesToHandler(EventMessage message)
+    {
+        var sutProvider = GetSutProvider();
+        await sutProvider.Sut.ProcessReceivedMessageAsync(
+            JsonSerializer.Serialize(message),
+            _messageId
+        );
+
+        await sutProvider.GetDependency<IEventMessageHandler>().Received(1).HandleEventAsync(
+            Arg.Is(AssertHelper.AssertPropertyEqual(message, new[] { "IdempotencyId" })));
+    }
+
+    [Theory, BitAutoData]
+    public async Task ProcessReceivedMessageAsync_ManyEvents_DelegatesToHandler(IEnumerable<EventMessage> messages)
+    {
+        var sutProvider = GetSutProvider();
+        await sutProvider.Sut.ProcessReceivedMessageAsync(
+            JsonSerializer.Serialize(messages),
+            _messageId
+        );
+
+        await sutProvider.GetDependency<IEventMessageHandler>().Received(1).HandleManyEventsAsync(
+            Arg.Is(AssertHelper.AssertPropertyEqual(messages, new[] { "IdempotencyId" })));
+    }
+}

test/Core.Test/AdminConsole/Services/AzureServiceBusIntegrationListenerServiceTests.cs

@@ -0,0 +1,124 @@
+#nullable enable
+
+using Azure.Messaging.ServiceBus;
+using Bit.Core.AdminConsole.Models.Data.Integrations;
+using Bit.Core.Services;
+using Bit.Test.Common.AutoFixture;
+using Bit.Test.Common.AutoFixture.Attributes;
+using Microsoft.Extensions.Logging;
+using NSubstitute;
+using Xunit;
+
+namespace Bit.Core.Test.Services;
+
+[SutProviderCustomize]
+public class AzureServiceBusIntegrationListenerServiceTests
+{
+    private const int _maxRetries = 3;
+    private const string _topicName = "test_topic";
+    private const string _subscriptionName = "test_subscription";
+    private readonly IIntegrationHandler _handler = Substitute.For<IIntegrationHandler>();
+    private readonly IAzureServiceBusService _serviceBusService = Substitute.For<IAzureServiceBusService>();
+    private readonly ILogger<AzureServiceBusIntegrationListenerService> _logger =
+        Substitute.For<ILogger<AzureServiceBusIntegrationListenerService>>();
+
+    private SutProvider<AzureServiceBusIntegrationListenerService> GetSutProvider()
+    {
+        return new SutProvider<AzureServiceBusIntegrationListenerService>()
+            .SetDependency(_handler)
+            .SetDependency(_serviceBusService)
+            .SetDependency(_topicName, "topicName")
+            .SetDependency(_subscriptionName, "subscriptionName")
+            .SetDependency(_maxRetries, "maxRetries")
+            .SetDependency(_logger)
+            .Create();
+    }
+
+    [Theory, BitAutoData]
+    public async Task ProcessErrorAsync_LogsError(ProcessErrorEventArgs args)
+    {
+        var sutProvider = GetSutProvider();
+        await sutProvider.Sut.ProcessErrorAsync(args);
+
+        _logger.Received(1).Log(
+            LogLevel.Error,
+            Arg.Any<EventId>(),
+            Arg.Any<object>(),
+            Arg.Any<Exception>(),
+            Arg.Any<Func<object, Exception?, string>>());
+    }
+
+    [Theory, BitAutoData]
+    public async Task HandleMessageAsync_FailureNotRetryable_PublishesToDeadLetterQueue(IntegrationMessage<WebhookIntegrationConfiguration> message)
+    {
+        var sutProvider = GetSutProvider();
+        message.DelayUntilDate = DateTime.UtcNow.AddMinutes(-1);
+        message.RetryCount = 0;
+
+        var result = new IntegrationHandlerResult(false, message);
+        result.Retryable = false;
+        _handler.HandleAsync(Arg.Any<string>()).Returns(result);
+
+        var expected = (IntegrationMessage<WebhookIntegrationConfiguration>)IntegrationMessage<WebhookIntegrationConfiguration>.FromJson(message.ToJson())!;
+
+        Assert.False(await sutProvider.Sut.HandleMessageAsync(message.ToJson()));
+
+        await _handler.Received(1).HandleAsync(Arg.Is(expected.ToJson()));
+        await _serviceBusService.DidNotReceiveWithAnyArgs().PublishToRetryAsync(default);
+    }
+
+    [Theory, BitAutoData]
+    public async Task HandleMessageAsync_FailureRetryableButTooManyRetries_PublishesToDeadLetterQueue(IntegrationMessage<WebhookIntegrationConfiguration> message)
+    {
+        var sutProvider = GetSutProvider();
+        message.DelayUntilDate = DateTime.UtcNow.AddMinutes(-1);
+        message.RetryCount = _maxRetries;
+        var result = new IntegrationHandlerResult(false, message);
+        result.Retryable = true;
+
+        _handler.HandleAsync(Arg.Any<string>()).Returns(result);
+
+        var expected = (IntegrationMessage<WebhookIntegrationConfiguration>)IntegrationMessage<WebhookIntegrationConfiguration>.FromJson(message.ToJson())!;
+
+        Assert.False(await sutProvider.Sut.HandleMessageAsync(message.ToJson()));
+
+        await _handler.Received(1).HandleAsync(Arg.Is(expected.ToJson()));
+        await _serviceBusService.DidNotReceiveWithAnyArgs().PublishToRetryAsync(default);
+    }
+
+    [Theory, BitAutoData]
+    public async Task HandleMessageAsync_FailureRetryable_PublishesToRetryQueue(IntegrationMessage<WebhookIntegrationConfiguration> message)
+    {
+        var sutProvider = GetSutProvider();
+        message.DelayUntilDate = DateTime.UtcNow.AddMinutes(-1);
+        message.RetryCount = 0;
+
+        var result = new IntegrationHandlerResult(false, message);
+        result.Retryable = true;
+        result.DelayUntilDate = DateTime.UtcNow.AddMinutes(1);
+        _handler.HandleAsync(Arg.Any<string>()).Returns(result);
+
+        var expected = (IntegrationMessage<WebhookIntegrationConfiguration>)IntegrationMessage<WebhookIntegrationConfiguration>.FromJson(message.ToJson())!;
+
+        Assert.True(await sutProvider.Sut.HandleMessageAsync(message.ToJson()));
+
+        await _handler.Received(1).HandleAsync(Arg.Is(expected.ToJson()));
+        await _serviceBusService.Received(1).PublishToRetryAsync(message);
+    }
+
+    [Theory, BitAutoData]
+    public async Task HandleMessageAsync_SuccessfulResult_Succeeds(IntegrationMessage<WebhookIntegrationConfiguration> message)
+    {
+        var sutProvider = GetSutProvider();
+        message.DelayUntilDate = DateTime.UtcNow.AddMinutes(-1);
+        var result = new IntegrationHandlerResult(true, message);
+        _handler.HandleAsync(Arg.Any<string>()).Returns(result);
+
+        var expected = (IntegrationMessage<WebhookIntegrationConfiguration>)IntegrationMessage<WebhookIntegrationConfiguration>.FromJson(message.ToJson())!;
+
+        Assert.True(await sutProvider.Sut.HandleMessageAsync(message.ToJson()));
+
+        await _handler.Received(1).HandleAsync(Arg.Is(expected.ToJson()));
+        await _serviceBusService.DidNotReceiveWithAnyArgs().PublishToRetryAsync(default);
+    }
+}

test/Core.Test/AdminConsole/Services/EventIntegrationEventWriteServiceTests.cs

@@ -0,0 +1,57 @@
+using System.Text.Json;
+using Bit.Core.Models.Data;
+using Bit.Core.Services;
+using Bit.Test.Common.AutoFixture.Attributes;
+using Bit.Test.Common.Helpers;
+using NSubstitute;
+using Xunit;
+
+namespace Bit.Core.Test.Services;
+
+[SutProviderCustomize]
+public class EventIntegrationEventWriteServiceTests
+{
+    private readonly IEventIntegrationPublisher _eventIntegrationPublisher = Substitute.For<IEventIntegrationPublisher>();
+    private readonly EventIntegrationEventWriteService Subject;
+
+    public EventIntegrationEventWriteServiceTests()
+    {
+        Subject = new EventIntegrationEventWriteService(_eventIntegrationPublisher);
+    }
+
+    [Theory, BitAutoData]
+    public async Task CreateAsync_EventPublishedToEventQueue(EventMessage eventMessage)
+    {
+        var expected = JsonSerializer.Serialize(eventMessage);
+        await Subject.CreateAsync(eventMessage);
+        await _eventIntegrationPublisher.Received(1).PublishEventAsync(
+            Arg.Is<string>(body => AssertJsonStringsMatch(eventMessage, body)));
+    }
+
+    [Theory, BitAutoData]
+    public async Task CreateManyAsync_EventsPublishedToEventQueue(IEnumerable<EventMessage> eventMessages)
+    {
+        await Subject.CreateManyAsync(eventMessages);
+        await _eventIntegrationPublisher.Received(1).PublishEventAsync(
+            Arg.Is<string>(body => AssertJsonStringsMatch(eventMessages, body)));
+    }
+
+    private static bool AssertJsonStringsMatch(EventMessage expected, string body)
+    {
+        var actual = JsonSerializer.Deserialize<EventMessage>(body);
+        AssertHelper.AssertPropertyEqual(expected, actual, new[] { "IdempotencyId" });
+        return true;
+    }
+
+    private static bool AssertJsonStringsMatch(IEnumerable<EventMessage> expected, string body)
+    {
+        using var actual = JsonSerializer.Deserialize<IEnumerable<EventMessage>>(body).GetEnumerator();
+
+        foreach (var expectedMessage in expected)
+        {
+            actual.MoveNext();
+            AssertHelper.AssertPropertyEqual(expectedMessage, actual.Current, new[] { "IdempotencyId" });
+        }
+        return true;
+    }
+}

test/Core.Test/AdminConsole/Services/EventIntegrationHandlerTests.cs

@@ -24,7 +24,7 @@ public class EventIntegrationHandlerTests
     private const string _templateWithActingUser = "#ActingUserName#, #ActingUserEmail#";
     private const string _url = "https://localhost";
     private const string _url2 = "https://example.com";
-    private readonly IIntegrationPublisher _integrationPublisher = Substitute.For<IIntegrationPublisher>();
+    private readonly IEventIntegrationPublisher _eventIntegrationPublisher = Substitute.For<IEventIntegrationPublisher>();
 
     private SutProvider<EventIntegrationHandler<WebhookIntegrationConfigurationDetails>> GetSutProvider(
         List<OrganizationIntegrationConfigurationDetails> configurations)
@@ -35,7 +35,7 @@ public class EventIntegrationHandlerTests
 
         return new SutProvider<EventIntegrationHandler<WebhookIntegrationConfigurationDetails>>()
             .SetDependency(configurationRepository)
-            .SetDependency(_integrationPublisher)
+            .SetDependency(_eventIntegrationPublisher)
             .SetDependency(IntegrationType.Webhook)
             .Create();
     }
@@ -45,6 +45,7 @@ public class EventIntegrationHandlerTests
         return new IntegrationMessage<WebhookIntegrationConfigurationDetails>()
         {
             IntegrationType = IntegrationType.Webhook,
+            MessageId = "TestMessageId",
             Configuration = new WebhookIntegrationConfigurationDetails(_url),
             RenderedTemplate = template,
             RetryCount = 0,
@@ -87,7 +88,7 @@ public class EventIntegrationHandlerTests
         var sutProvider = GetSutProvider(NoConfigurations());
 
         await sutProvider.Sut.HandleEventAsync(eventMessage);
-        Assert.Empty(_integrationPublisher.ReceivedCalls());
+        Assert.Empty(_eventIntegrationPublisher.ReceivedCalls());
     }
 
     [Theory, BitAutoData]
@@ -101,8 +102,9 @@ public class EventIntegrationHandlerTests
             $"Date: {eventMessage.Date}, Type: {eventMessage.Type}, UserId: {eventMessage.UserId}"
         );
 
-        Assert.Single(_integrationPublisher.ReceivedCalls());
-        await _integrationPublisher.Received(1).PublishAsync(Arg.Is(AssertHelper.AssertPropertyEqual(expectedMessage)));
+        Assert.Single(_eventIntegrationPublisher.ReceivedCalls());
+        await _eventIntegrationPublisher.Received(1).PublishAsync(Arg.Is(
+            AssertHelper.AssertPropertyEqual(expectedMessage, new[] { "MessageId" })));
         await sutProvider.GetDependency<IOrganizationRepository>().DidNotReceiveWithAnyArgs().GetByIdAsync(Arg.Any<Guid>());
         await sutProvider.GetDependency<IUserRepository>().DidNotReceiveWithAnyArgs().GetByIdAsync(Arg.Any<Guid>());
     }
@@ -120,8 +122,9 @@ public class EventIntegrationHandlerTests
 
         var expectedMessage = EventIntegrationHandlerTests.expectedMessage($"{user.Name}, {user.Email}");
 
-        Assert.Single(_integrationPublisher.ReceivedCalls());
-        await _integrationPublisher.Received(1).PublishAsync(Arg.Is(AssertHelper.AssertPropertyEqual(expectedMessage)));
+        Assert.Single(_eventIntegrationPublisher.ReceivedCalls());
+        await _eventIntegrationPublisher.Received(1).PublishAsync(Arg.Is(
+            AssertHelper.AssertPropertyEqual(expectedMessage, new[] { "MessageId" })));
         await sutProvider.GetDependency<IOrganizationRepository>().DidNotReceiveWithAnyArgs().GetByIdAsync(Arg.Any<Guid>());
         await sutProvider.GetDependency<IUserRepository>().Received(1).GetByIdAsync(eventMessage.ActingUserId ?? Guid.Empty);
     }
@@ -136,12 +139,13 @@ public class EventIntegrationHandlerTests
         sutProvider.GetDependency<IOrganizationRepository>().GetByIdAsync(Arg.Any<Guid>()).Returns(organization);
         await sutProvider.Sut.HandleEventAsync(eventMessage);
 
-        Assert.Single(_integrationPublisher.ReceivedCalls());
+        Assert.Single(_eventIntegrationPublisher.ReceivedCalls());
 
         var expectedMessage = EventIntegrationHandlerTests.expectedMessage($"Org: {organization.Name}");
 
-        Assert.Single(_integrationPublisher.ReceivedCalls());
-        await _integrationPublisher.Received(1).PublishAsync(Arg.Is(AssertHelper.AssertPropertyEqual(expectedMessage)));
+        Assert.Single(_eventIntegrationPublisher.ReceivedCalls());
+        await _eventIntegrationPublisher.Received(1).PublishAsync(Arg.Is(
+            AssertHelper.AssertPropertyEqual(expectedMessage, new[] { "MessageId" })));
         await sutProvider.GetDependency<IOrganizationRepository>().Received(1).GetByIdAsync(eventMessage.OrganizationId ?? Guid.Empty);
         await sutProvider.GetDependency<IUserRepository>().DidNotReceiveWithAnyArgs().GetByIdAsync(Arg.Any<Guid>());
     }
@@ -159,8 +163,9 @@ public class EventIntegrationHandlerTests
 
         var expectedMessage = EventIntegrationHandlerTests.expectedMessage($"{user.Name}, {user.Email}");
 
-        Assert.Single(_integrationPublisher.ReceivedCalls());
-        await _integrationPublisher.Received(1).PublishAsync(Arg.Is(AssertHelper.AssertPropertyEqual(expectedMessage)));
+        Assert.Single(_eventIntegrationPublisher.ReceivedCalls());
+        await _eventIntegrationPublisher.Received(1).PublishAsync(Arg.Is(
+            AssertHelper.AssertPropertyEqual(expectedMessage, new[] { "MessageId" })));
         await sutProvider.GetDependency<IOrganizationRepository>().DidNotReceiveWithAnyArgs().GetByIdAsync(Arg.Any<Guid>());
         await sutProvider.GetDependency<IUserRepository>().Received(1).GetByIdAsync(eventMessage.UserId ?? Guid.Empty);
     }
@@ -171,7 +176,7 @@ public class EventIntegrationHandlerTests
         var sutProvider = GetSutProvider(NoConfigurations());
 
         await sutProvider.Sut.HandleManyEventsAsync(eventMessages);
-        Assert.Empty(_integrationPublisher.ReceivedCalls());
+        Assert.Empty(_eventIntegrationPublisher.ReceivedCalls());
     }
 
     [Theory, BitAutoData]
@@ -186,7 +191,8 @@ public class EventIntegrationHandlerTests
             var expectedMessage = EventIntegrationHandlerTests.expectedMessage(
                 $"Date: {eventMessage.Date}, Type: {eventMessage.Type}, UserId: {eventMessage.UserId}"
             );
-            await _integrationPublisher.Received(1).PublishAsync(Arg.Is(AssertHelper.AssertPropertyEqual(expectedMessage)));
+            await _eventIntegrationPublisher.Received(1).PublishAsync(Arg.Is(
+                AssertHelper.AssertPropertyEqual(expectedMessage, new[] { "MessageId" })));
         }
     }
 
@@ -203,10 +209,12 @@ public class EventIntegrationHandlerTests
             var expectedMessage = EventIntegrationHandlerTests.expectedMessage(
                 $"Date: {eventMessage.Date}, Type: {eventMessage.Type}, UserId: {eventMessage.UserId}"
             );
-            await _integrationPublisher.Received(1).PublishAsync(Arg.Is(AssertHelper.AssertPropertyEqual(expectedMessage)));
+            await _eventIntegrationPublisher.Received(1).PublishAsync(Arg.Is(
+                AssertHelper.AssertPropertyEqual(expectedMessage, new[] { "MessageId" })));
 
             expectedMessage.Configuration = new WebhookIntegrationConfigurationDetails(_url2);
-            await _integrationPublisher.Received(1).PublishAsync(Arg.Is(AssertHelper.AssertPropertyEqual(expectedMessage)));
+            await _eventIntegrationPublisher.Received(1).PublishAsync(Arg.Is(
+                AssertHelper.AssertPropertyEqual(expectedMessage, new[] { "MessageId" })));
         }
     }
 }

test/Core.Test/AdminConsole/Services/IntegrationHandlerTests.cs

@@ -15,6 +15,7 @@ public class IntegrationHandlerTests
         var expected = new IntegrationMessage<WebhookIntegrationConfigurationDetails>()
         {
             Configuration = new WebhookIntegrationConfigurationDetails("https://localhost"),
+            MessageId = "TestMessageId",
             IntegrationType = IntegrationType.Webhook,
             RenderedTemplate = "Template",
             DelayUntilDate = null,

test/Core.Test/AdminConsole/Services/RabbitMqEventListenerServiceTests.cs

@@ -0,0 +1,173 @@
+using System.Text.Json;
+using Bit.Core.Models.Data;
+using Bit.Core.Services;
+using Bit.Test.Common.AutoFixture;
+using Bit.Test.Common.AutoFixture.Attributes;
+using Bit.Test.Common.Helpers;
+using Microsoft.Extensions.Logging;
+using NSubstitute;
+using RabbitMQ.Client;
+using RabbitMQ.Client.Events;
+using Xunit;
+
+namespace Bit.Core.Test.Services;
+
+[SutProviderCustomize]
+public class RabbitMqEventListenerServiceTests
+{
+    private const string _queueName = "test_queue";
+    private readonly IRabbitMqService _rabbitMqService = Substitute.For<IRabbitMqService>();
+    private readonly ILogger<RabbitMqEventListenerService> _logger = Substitute.For<ILogger<RabbitMqEventListenerService>>();
+
+    private SutProvider<RabbitMqEventListenerService> GetSutProvider()
+    {
+        return new SutProvider<RabbitMqEventListenerService>()
+            .SetDependency(_rabbitMqService)
+            .SetDependency(_logger)
+            .SetDependency(_queueName, "queueName")
+            .Create();
+    }
+
+    [Fact]
+    public async Task StartAsync_CreatesQueue()
+    {
+        var sutProvider = GetSutProvider();
+        var cancellationToken = CancellationToken.None;
+        await sutProvider.Sut.StartAsync(cancellationToken);
+
+        await _rabbitMqService.Received(1).CreateEventQueueAsync(
+            Arg.Is(_queueName),
+            Arg.Is(cancellationToken)
+        );
+    }
+
+    [Fact]
+    public async Task ProcessReceivedMessageAsync_EmptyJson_LogsError()
+    {
+        var sutProvider = GetSutProvider();
+        var eventArgs = new BasicDeliverEventArgs(
+            consumerTag: string.Empty,
+            deliveryTag: 0,
+            redelivered: true,
+            exchange: string.Empty,
+            routingKey: string.Empty,
+            new BasicProperties(),
+            body: new byte[0]);
+
+        await sutProvider.Sut.ProcessReceivedMessageAsync(eventArgs);
+
+        _logger.Received(1).Log(
+            LogLevel.Error,
+            Arg.Any<EventId>(),
+            Arg.Any<object>(),
+            Arg.Any<JsonException>(),
+            Arg.Any<Func<object, Exception, string>>());
+    }
+
+    [Fact]
+    public async Task ProcessReceivedMessageAsync_InvalidJson_LogsError()
+    {
+        var sutProvider = GetSutProvider();
+        var eventArgs = new BasicDeliverEventArgs(
+            consumerTag: string.Empty,
+            deliveryTag: 0,
+            redelivered: true,
+            exchange: string.Empty,
+            routingKey: string.Empty,
+            new BasicProperties(),
+            body: JsonSerializer.SerializeToUtf8Bytes("{ Inavlid JSON"));
+
+        await sutProvider.Sut.ProcessReceivedMessageAsync(eventArgs);
+
+        _logger.Received(1).Log(
+            LogLevel.Error,
+            Arg.Any<EventId>(),
+            Arg.Is<object>(o => o.ToString().Contains("Invalid JSON")),
+            Arg.Any<Exception>(),
+            Arg.Any<Func<object, Exception, string>>());
+    }
+
+    [Fact]
+    public async Task ProcessReceivedMessageAsync_InvalidJsonArray_LogsError()
+    {
+        var sutProvider = GetSutProvider();
+        var eventArgs = new BasicDeliverEventArgs(
+            consumerTag: string.Empty,
+            deliveryTag: 0,
+            redelivered: true,
+            exchange: string.Empty,
+            routingKey: string.Empty,
+            new BasicProperties(),
+            body: JsonSerializer.SerializeToUtf8Bytes(new[] { "not a valid", "list of event messages" }));
+
+        await sutProvider.Sut.ProcessReceivedMessageAsync(eventArgs);
+
+        _logger.Received(1).Log(
+            LogLevel.Error,
+            Arg.Any<EventId>(),
+            Arg.Any<object>(),
+            Arg.Any<JsonException>(),
+            Arg.Any<Func<object, Exception, string>>());
+    }
+
+    [Fact]
+    public async Task ProcessReceivedMessageAsync_InvalidJsonObject_LogsError()
+    {
+        var sutProvider = GetSutProvider();
+        var eventArgs = new BasicDeliverEventArgs(
+            consumerTag: string.Empty,
+            deliveryTag: 0,
+            redelivered: true,
+            exchange: string.Empty,
+            routingKey: string.Empty,
+            new BasicProperties(),
+            body: JsonSerializer.SerializeToUtf8Bytes(DateTime.UtcNow));  // wrong object - not EventMessage
+
+        await sutProvider.Sut.ProcessReceivedMessageAsync(eventArgs);
+
+        _logger.Received(1).Log(
+            LogLevel.Error,
+            Arg.Any<EventId>(),
+            Arg.Any<object>(),
+            Arg.Any<JsonException>(),
+            Arg.Any<Func<object, Exception, string>>());
+    }
+
+    [Theory, BitAutoData]
+    public async Task ProcessReceivedMessageAsync_SingleEvent_DelegatesToHandler(EventMessage message)
+    {
+        var sutProvider = GetSutProvider();
+        var eventArgs = new BasicDeliverEventArgs(
+            consumerTag: string.Empty,
+            deliveryTag: 0,
+            redelivered: true,
+            exchange: string.Empty,
+            routingKey: string.Empty,
+            new BasicProperties(),
+            body: JsonSerializer.SerializeToUtf8Bytes(message));
+
+        await sutProvider.Sut.ProcessReceivedMessageAsync(eventArgs);
+
+        await sutProvider.GetDependency<IEventMessageHandler>().Received(1).HandleEventAsync(
+            Arg.Is(AssertHelper.AssertPropertyEqual(message, new[] { "IdempotencyId" })));
+    }
+
+    [Theory, BitAutoData]
+    public async Task ProcessReceivedMessageAsync_ManyEvents_DelegatesToHandler(IEnumerable<EventMessage> messages)
+    {
+        var sutProvider = GetSutProvider();
+        var eventArgs = new BasicDeliverEventArgs(
+            consumerTag: string.Empty,
+            deliveryTag: 0,
+            redelivered: true,
+            exchange: string.Empty,
+            routingKey: string.Empty,
+            new BasicProperties(),
+            body: JsonSerializer.SerializeToUtf8Bytes(messages));
+
+        await sutProvider.Sut.ProcessReceivedMessageAsync(eventArgs);
+
+        await sutProvider.GetDependency<IEventMessageHandler>().Received(1).HandleManyEventsAsync(
+            Arg.Is(AssertHelper.AssertPropertyEqual(messages, new[] { "IdempotencyId" })));
+    }
+}

test/Core.Test/AdminConsole/Services/RabbitMqIntegrationListenerServiceTests.cs

@@ -0,0 +1,230 @@
+using System.Text;
+using Bit.Core.AdminConsole.Models.Data.Integrations;
+using Bit.Core.Services;
+using Bit.Test.Common.AutoFixture;
+using Bit.Test.Common.AutoFixture.Attributes;
+using Bit.Test.Common.Helpers;
+using NSubstitute;
+using RabbitMQ.Client;
+using RabbitMQ.Client.Events;
+using Xunit;
+
+namespace Bit.Core.Test.Services;
+
+[SutProviderCustomize]
+public class RabbitMqIntegrationListenerServiceTests
+{
+    private const int _maxRetries = 3;
+    private const string _queueName = "test_queue";
+    private const string _retryQueueName = "test_queue_retry";
+    private const string _routingKey = "test_routing_key";
+    private readonly IIntegrationHandler _handler = Substitute.For<IIntegrationHandler>();
+    private readonly IRabbitMqService _rabbitMqService = Substitute.For<IRabbitMqService>();
+
+    private SutProvider<RabbitMqIntegrationListenerService> GetSutProvider()
+    {
+        return new SutProvider<RabbitMqIntegrationListenerService>()
+            .SetDependency(_handler)
+            .SetDependency(_rabbitMqService)
+            .SetDependency(_queueName, "queueName")
+            .SetDependency(_retryQueueName, "retryQueueName")
+            .SetDependency(_routingKey, "routingKey")
+            .SetDependency(_maxRetries, "maxRetries")
+            .Create();
+    }
+
+    [Fact]
+    public async Task StartAsync_CreatesQueues()
+    {
+        var sutProvider = GetSutProvider();
+        var cancellationToken = CancellationToken.None;
+        await sutProvider.Sut.StartAsync(cancellationToken);
+
+        await _rabbitMqService.Received(1).CreateIntegrationQueuesAsync(
+            Arg.Is(_queueName),
+            Arg.Is(_retryQueueName),
+            Arg.Is(_routingKey),
+            Arg.Is(cancellationToken)
+        );
+    }
+
+    [Theory, BitAutoData]
+    public async Task ProcessReceivedMessageAsync_FailureNotRetryable_PublishesToDeadLetterQueue(IntegrationMessage<WebhookIntegrationConfiguration> message)
+    {
+        var sutProvider = GetSutProvider();
+        var cancellationToken = CancellationToken.None;
+        await sutProvider.Sut.StartAsync(cancellationToken);
+
+        message.DelayUntilDate = DateTime.UtcNow.AddMinutes(-1);
+        message.RetryCount = 0;
+        var eventArgs = new BasicDeliverEventArgs(
+            consumerTag: string.Empty,
+            deliveryTag: 0,
+            redelivered: true,
+            exchange: string.Empty,
+            routingKey: string.Empty,
+            new BasicProperties(),
+            body: Encoding.UTF8.GetBytes(message.ToJson())
+        );
+        var result = new IntegrationHandlerResult(false, message);
+        result.Retryable = false;
+        _handler.HandleAsync(Arg.Any<string>()).Returns(result);
+
+        var expected = IntegrationMessage<WebhookIntegrationConfiguration>.FromJson(message.ToJson());
+
+        await sutProvider.Sut.ProcessReceivedMessageAsync(eventArgs, cancellationToken);
+
+        await _handler.Received(1).HandleAsync(Arg.Is(expected.ToJson()));
+
+        await _rabbitMqService.Received(1).PublishToDeadLetterAsync(
+            Arg.Any<IChannel>(),
+            Arg.Is(AssertHelper.AssertPropertyEqual(expected, new[] { "DelayUntilDate" })),
+            Arg.Any<CancellationToken>());
+
+        await _rabbitMqService.DidNotReceiveWithAnyArgs()
+            .RepublishToRetryQueueAsync(default, default);
+        await _rabbitMqService.DidNotReceiveWithAnyArgs()
+            .PublishToRetryAsync(default, default, default);
+    }
+
+    [Theory, BitAutoData]
+    public async Task ProcessReceivedMessageAsync_FailureRetryableButTooManyRetries_PublishesToDeadLetterQueue(IntegrationMessage<WebhookIntegrationConfiguration> message)
+    {
+        var sutProvider = GetSutProvider();
+        var cancellationToken = CancellationToken.None;
+        await sutProvider.Sut.StartAsync(cancellationToken);
+
+        message.DelayUntilDate = DateTime.UtcNow.AddMinutes(-1);
+        message.RetryCount = _maxRetries;
+        var eventArgs = new BasicDeliverEventArgs(
+            consumerTag: string.Empty,
+            deliveryTag: 0,
+            redelivered: true,
+            exchange: string.Empty,
+            routingKey: string.Empty,
+            new BasicProperties(),
+            body: Encoding.UTF8.GetBytes(message.ToJson())
+        );
+        var result = new IntegrationHandlerResult(false, message);
+        result.Retryable = true;
+        _handler.HandleAsync(Arg.Any<string>()).Returns(result);
+
+        var expected = IntegrationMessage<WebhookIntegrationConfiguration>.FromJson(message.ToJson());
+
+        await sutProvider.Sut.ProcessReceivedMessageAsync(eventArgs, cancellationToken);
+
+        expected.ApplyRetry(result.DelayUntilDate);
+        await _rabbitMqService.Received(1).PublishToDeadLetterAsync(
+            Arg.Any<IChannel>(),
+            Arg.Is(AssertHelper.AssertPropertyEqual(expected, new[] { "DelayUntilDate" })),
+            Arg.Any<CancellationToken>());
+
+        await _rabbitMqService.DidNotReceiveWithAnyArgs()
+            .RepublishToRetryQueueAsync(default, default);
+        await _rabbitMqService.DidNotReceiveWithAnyArgs()
+            .PublishToRetryAsync(default, default, default);
+    }
+
+    [Theory, BitAutoData]
+    public async Task ProcessReceivedMessageAsync_FailureRetryable_PublishesToRetryQueue(IntegrationMessage<WebhookIntegrationConfiguration> message)
+    {
+        var sutProvider = GetSutProvider();
+        var cancellationToken = CancellationToken.None;
+        await sutProvider.Sut.StartAsync(cancellationToken);
+
+        message.DelayUntilDate = DateTime.UtcNow.AddMinutes(-1);
+        message.RetryCount = 0;
+        var eventArgs = new BasicDeliverEventArgs(
+            consumerTag: string.Empty,
+            deliveryTag: 0,
+            redelivered: true,
+            exchange: string.Empty,
+            routingKey: string.Empty,
+            new BasicProperties(),
+            body: Encoding.UTF8.GetBytes(message.ToJson())
+        );
+        var result = new IntegrationHandlerResult(false, message);
+        result.Retryable = true;
+        result.DelayUntilDate = DateTime.UtcNow.AddMinutes(1);
+        _handler.HandleAsync(Arg.Any<string>()).Returns(result);
+
+        var expected = IntegrationMessage<WebhookIntegrationConfiguration>.FromJson(message.ToJson());
+
+        await sutProvider.Sut.ProcessReceivedMessageAsync(eventArgs, cancellationToken);
+
+        await _handler.Received(1).HandleAsync(Arg.Is(expected.ToJson()));
+
+        expected.ApplyRetry(result.DelayUntilDate);
+        await _rabbitMqService.Received(1).PublishToRetryAsync(
+            Arg.Any<IChannel>(),
+            Arg.Is(AssertHelper.AssertPropertyEqual(expected, new[] { "DelayUntilDate" })),
+            Arg.Any<CancellationToken>());
+
+        await _rabbitMqService.DidNotReceiveWithAnyArgs()
+            .RepublishToRetryQueueAsync(default, default);
+        await _rabbitMqService.DidNotReceiveWithAnyArgs()
+            .PublishToDeadLetterAsync(default, default, default);
+    }
+
+    [Theory, BitAutoData]
+    public async Task ProcessReceivedMessageAsync_SuccessfulResult_Succeeds(IntegrationMessage<WebhookIntegrationConfiguration> message)
+    {
+        var sutProvider = GetSutProvider();
+        var cancellationToken = CancellationToken.None;
+        await sutProvider.Sut.StartAsync(cancellationToken);
+
+        message.DelayUntilDate = DateTime.UtcNow.AddMinutes(-1);
+        var eventArgs = new BasicDeliverEventArgs(
+            consumerTag: string.Empty,
+            deliveryTag: 0,
+            redelivered: true,
+            exchange: string.Empty,
+            routingKey: string.Empty,
+            new BasicProperties(),
+            body: Encoding.UTF8.GetBytes(message.ToJson())
+        );
+        var result = new IntegrationHandlerResult(true, message);
+        _handler.HandleAsync(Arg.Any<string>()).Returns(result);
+
+        await sutProvider.Sut.ProcessReceivedMessageAsync(eventArgs, cancellationToken);
+
+        await _handler.Received(1).HandleAsync(Arg.Is(message.ToJson()));
+
+        await _rabbitMqService.DidNotReceiveWithAnyArgs()
+            .RepublishToRetryQueueAsync(default, default);
+        await _rabbitMqService.DidNotReceiveWithAnyArgs()
+            .PublishToRetryAsync(default, default, default);
+        await _rabbitMqService.DidNotReceiveWithAnyArgs()
+            .PublishToDeadLetterAsync(default, default, default);
+    }
+
+    [Theory, BitAutoData]
+    public async Task ProcessReceivedMessageAsync_TooEarlyRetry_RepublishesToRetryQueue(IntegrationMessage<WebhookIntegrationConfiguration> message)
+    {
+        var sutProvider = GetSutProvider();
+        var cancellationToken = CancellationToken.None;
+        await sutProvider.Sut.StartAsync(cancellationToken);
+
+        message.DelayUntilDate = DateTime.UtcNow.AddMinutes(1);
+        var eventArgs = new BasicDeliverEventArgs(
+            consumerTag: string.Empty,
+            deliveryTag: 0,
+            redelivered: true,
+            exchange: string.Empty,
+            routingKey: string.Empty,
+            new BasicProperties(),
+            body: Encoding.UTF8.GetBytes(message.ToJson())
+        );
+
+        await sutProvider.Sut.ProcessReceivedMessageAsync(eventArgs, cancellationToken);
+
+        await _rabbitMqService.Received(1)
+            .RepublishToRetryQueueAsync(Arg.Any<IChannel>(), Arg.Any<BasicDeliverEventArgs>());
+
+        await _handler.DidNotReceiveWithAnyArgs().HandleAsync(default);
+        await _rabbitMqService.DidNotReceiveWithAnyArgs()
+            .PublishToRetryAsync(default, default, default);
+        await _rabbitMqService.DidNotReceiveWithAnyArgs()
+            .PublishToDeadLetterAsync(default, default, default);
+    }
+}

test/Core.Test/AdminConsole/Services/SlackServiceTests.cs

@@ -1,4 +1,6 @@
-using System.Net;
+#nullable enable
+
+using System.Net;
 using System.Text.Json;
 using Bit.Core.Services;
 using Bit.Test.Common.AutoFixture;
@@ -257,10 +259,10 @@ public class SlackServiceTests
     public void GetRedirectUrl_ReturnsCorrectUrl()
     {
         var sutProvider = GetSutProvider();
-        var ClientId = sutProvider.GetDependency<GlobalSettings>().Slack.ClientId;
-        var Scopes = sutProvider.GetDependency<GlobalSettings>().Slack.Scopes;
+        var clientId = sutProvider.GetDependency<GlobalSettings>().Slack.ClientId;
+        var scopes = sutProvider.GetDependency<GlobalSettings>().Slack.Scopes;
         var redirectUrl = "https://example.com/callback";
-        var expectedUrl = $"https://slack.com/oauth/v2/authorize?client_id={ClientId}&scope={Scopes}&redirect_uri={redirectUrl}";
+        var expectedUrl = $"https://slack.com/oauth/v2/authorize?client_id={clientId}&scope={scopes}&redirect_uri={redirectUrl}";
         var result = sutProvider.Sut.GetRedirectUrl(redirectUrl);
         Assert.Equal(expectedUrl, result);
     }

test/Core.Test/AdminConsole/Services/WebhookIntegrationHandlerTests.cs

@@ -79,6 +79,7 @@ public class WebhookIntegrationHandlerTests
         Assert.Equal(result.Message, message);
         Assert.True(result.DelayUntilDate.HasValue);
         Assert.InRange(result.DelayUntilDate.Value, DateTime.UtcNow.AddSeconds(59), DateTime.UtcNow.AddSeconds(61));
+        Assert.Equal("Too Many Requests", result.FailureReason);
     }
 
     [Theory, BitAutoData]
@@ -99,6 +100,7 @@ public class WebhookIntegrationHandlerTests
         Assert.Equal(result.Message, message);
         Assert.True(result.DelayUntilDate.HasValue);
         Assert.InRange(result.DelayUntilDate.Value, DateTime.UtcNow.AddSeconds(59), DateTime.UtcNow.AddSeconds(61));
+        Assert.Equal("Too Many Requests", result.FailureReason);
     }
 
     [Theory, BitAutoData]
@@ -117,6 +119,7 @@ public class WebhookIntegrationHandlerTests
         Assert.True(result.Retryable);
         Assert.Equal(result.Message, message);
         Assert.False(result.DelayUntilDate.HasValue);
+        Assert.Equal("Internal Server Error", result.FailureReason);
     }
 
     [Theory, BitAutoData]
@@ -135,5 +138,6 @@ public class WebhookIntegrationHandlerTests
         Assert.False(result.Retryable);
         Assert.Equal(result.Message, message);
         Assert.Null(result.DelayUntilDate);
+        Assert.Equal("Temporary Redirect", result.FailureReason);
     }
 }