[PM-17562] Add integration filter support (#5971)

* [PM-17562] Add integration filter support

* Repond to PR feedback; Remove Date-related filters

* Use tables to format the filter class descriptions

* [PM-17562] Add database support for integration filters (#5988)

* [PM-17562] Add database support for integration filters

* Respond to PR review - fix database scripts

* Further database updates; fix Filters to be last in views, stored procs, etc

* Fix for missing nulls in stored procedures in main migration script

* Reorder Filters to the bottom of OrganizationIntegrationConfiguration

* Separate out the creation of filters from the IntegrationFilterService to IntegrationFIlterFactory

* Move properties to static readonly field

* Fix unit tests failing from merge

---------

Co-authored-by: Matt Bishop <mbishop@bitwarden.com>

test/Core.Test/AdminConsole/Services/EventIntegrationHandlerTests.cs

@@ -10,6 +10,7 @@ using Bit.Core.Services;
 using Bit.Test.Common.AutoFixture;
 using Bit.Test.Common.AutoFixture.Attributes;
 using Bit.Test.Common.Helpers;
+using Microsoft.Extensions.Logging;
 using NSubstitute;
 using Xunit;
 
@@ -25,6 +26,8 @@ public class EventIntegrationHandlerTests
     private const string _url = "https://localhost";
     private const string _url2 = "https://example.com";
     private readonly IEventIntegrationPublisher _eventIntegrationPublisher = Substitute.For<IEventIntegrationPublisher>();
+    private readonly ILogger<EventIntegrationHandler<WebhookIntegrationConfigurationDetails>> _logger =
+        Substitute.For<ILogger<EventIntegrationHandler<WebhookIntegrationConfigurationDetails>>>();
 
     private SutProvider<EventIntegrationHandler<WebhookIntegrationConfigurationDetails>> GetSutProvider(
         List<OrganizationIntegrationConfigurationDetails> configurations)
@@ -37,6 +40,7 @@ public class EventIntegrationHandlerTests
             .SetDependency(configurationRepository)
             .SetDependency(_eventIntegrationPublisher)
             .SetDependency(IntegrationType.Webhook)
+            .SetDependency(_logger)
             .Create();
     }
 
@@ -82,6 +86,29 @@ public class EventIntegrationHandlerTests
         return [config, config2];
     }
 
+    private static List<OrganizationIntegrationConfigurationDetails> InvalidFilterConfiguration()
+    {
+        var config = Substitute.For<OrganizationIntegrationConfigurationDetails>();
+        config.Configuration = null;
+        config.IntegrationConfiguration = JsonSerializer.Serialize(new { Url = _url });
+        config.Template = _templateBase;
+        config.Filters = "Invalid Configuration!";
+
+        return [config];
+    }
+
+    private static List<OrganizationIntegrationConfigurationDetails> ValidFilterConfiguration()
+    {
+        var config = Substitute.For<OrganizationIntegrationConfigurationDetails>();
+        config.Configuration = null;
+        config.IntegrationConfiguration = JsonSerializer.Serialize(new { Url = _url });
+        config.Template = _templateBase;
+        config.Filters = JsonSerializer.Serialize(new IntegrationFilterGroup() { });
+
+        return [config];
+    }
+
+
     [Theory, BitAutoData]
     public async Task HandleEventAsync_BaseTemplateNoConfigurations_DoesNothing(EventMessage eventMessage)
     {
@@ -92,7 +119,7 @@ public class EventIntegrationHandlerTests
     }
 
     [Theory, BitAutoData]
-    public async Task HandleEventAsync_BaseTemplateOneConfiguration_CallsProcessEventIntegrationAsync(EventMessage eventMessage)
+    public async Task HandleEventAsync_BaseTemplateOneConfiguration_PublishesIntegrationMessage(EventMessage eventMessage)
     {
         var sutProvider = GetSutProvider(OneConfiguration(_templateBase));
 
@@ -109,6 +136,27 @@ public class EventIntegrationHandlerTests
         await sutProvider.GetDependency<IUserRepository>().DidNotReceiveWithAnyArgs().GetByIdAsync(Arg.Any<Guid>());
     }
 
+    [Theory, BitAutoData]
+    public async Task HandleEventAsync_BaseTemplateTwoConfigurations_PublishesIntegrationMessages(EventMessage eventMessage)
+    {
+        var sutProvider = GetSutProvider(TwoConfigurations(_templateBase));
+
+        await sutProvider.Sut.HandleEventAsync(eventMessage);
+
+        var expectedMessage = EventIntegrationHandlerTests.expectedMessage(
+            $"Date: {eventMessage.Date}, Type: {eventMessage.Type}, UserId: {eventMessage.UserId}"
+        );
+        await _eventIntegrationPublisher.Received(1).PublishAsync(Arg.Is(
+            AssertHelper.AssertPropertyEqual(expectedMessage, new[] { "MessageId" })));
+
+        expectedMessage.Configuration = new WebhookIntegrationConfigurationDetails(_url2);
+        await _eventIntegrationPublisher.Received(1).PublishAsync(Arg.Is(
+            AssertHelper.AssertPropertyEqual(expectedMessage, new[] { "MessageId" })));
+
+        await sutProvider.GetDependency<IOrganizationRepository>().DidNotReceiveWithAnyArgs().GetByIdAsync(Arg.Any<Guid>());
+        await sutProvider.GetDependency<IUserRepository>().DidNotReceiveWithAnyArgs().GetByIdAsync(Arg.Any<Guid>());
+    }
+
     [Theory, BitAutoData]
     public async Task HandleEventAsync_ActingUserTemplate_LoadsUserFromRepository(EventMessage eventMessage)
     {
@@ -170,6 +218,50 @@ public class EventIntegrationHandlerTests
         await sutProvider.GetDependency<IUserRepository>().Received(1).GetByIdAsync(eventMessage.UserId ?? Guid.Empty);
     }
 
+    [Theory, BitAutoData]
+    public async Task HandleEventAsync_FilterReturnsFalse_DoesNothing(EventMessage eventMessage)
+    {
+        var sutProvider = GetSutProvider(ValidFilterConfiguration());
+        sutProvider.GetDependency<IIntegrationFilterService>().EvaluateFilterGroup(
+            Arg.Any<IntegrationFilterGroup>(), Arg.Any<EventMessage>()).Returns(false);
+
+        await sutProvider.Sut.HandleEventAsync(eventMessage);
+        Assert.Empty(_eventIntegrationPublisher.ReceivedCalls());
+    }
+
+    [Theory, BitAutoData]
+    public async Task HandleEventAsync_FilterReturnsTrue_PublishesIntegrationMessage(EventMessage eventMessage)
+    {
+        var sutProvider = GetSutProvider(ValidFilterConfiguration());
+        sutProvider.GetDependency<IIntegrationFilterService>().EvaluateFilterGroup(
+            Arg.Any<IntegrationFilterGroup>(), Arg.Any<EventMessage>()).Returns(true);
+
+        await sutProvider.Sut.HandleEventAsync(eventMessage);
+
+        var expectedMessage = EventIntegrationHandlerTests.expectedMessage(
+            $"Date: {eventMessage.Date}, Type: {eventMessage.Type}, UserId: {eventMessage.UserId}"
+        );
+
+        Assert.Single(_eventIntegrationPublisher.ReceivedCalls());
+        await _eventIntegrationPublisher.Received(1).PublishAsync(Arg.Is(
+            AssertHelper.AssertPropertyEqual(expectedMessage, new[] { "MessageId" })));
+    }
+
+    [Theory, BitAutoData]
+    public async Task HandleEventAsync_InvalidFilter_LogsErrorDoesNothing(EventMessage eventMessage)
+    {
+        var sutProvider = GetSutProvider(InvalidFilterConfiguration());
+
+        await sutProvider.Sut.HandleEventAsync(eventMessage);
+        Assert.Empty(_eventIntegrationPublisher.ReceivedCalls());
+        _logger.Received(1).Log(
+            LogLevel.Error,
+            Arg.Any<EventId>(),
+            Arg.Any<object>(),
+            Arg.Any<JsonException>(),
+            Arg.Any<Func<object, Exception, string>>());
+    }
+
     [Theory, BitAutoData]
     public async Task HandleManyEventsAsync_BaseTemplateNoConfigurations_DoesNothing(List<EventMessage> eventMessages)
     {
@@ -180,7 +272,7 @@ public class EventIntegrationHandlerTests
     }
 
     [Theory, BitAutoData]
-    public async Task HandleManyEventsAsync_BaseTemplateOneConfiguration_CallsProcessEventIntegrationAsync(List<EventMessage> eventMessages)
+    public async Task HandleManyEventsAsync_BaseTemplateOneConfiguration_PublishesIntegrationMessages(List<EventMessage> eventMessages)
     {
         var sutProvider = GetSutProvider(OneConfiguration(_templateBase));
 
@@ -197,7 +289,7 @@ public class EventIntegrationHandlerTests
     }
 
     [Theory, BitAutoData]
-    public async Task HandleManyEventsAsync_BaseTemplateTwoConfigurations_CallsProcessEventIntegrationAsyncMultipleTimes(
+    public async Task HandleManyEventsAsync_BaseTemplateTwoConfigurations_PublishesIntegrationMessages(
         List<EventMessage> eventMessages)
     {
         var sutProvider = GetSutProvider(TwoConfigurations(_templateBase));

test/Core.Test/AdminConsole/Services/IntegrationFilterFactoryTests.cs

@@ -0,0 +1,46 @@
+using Bit.Core.Models.Data;
+using Bit.Core.Services;
+using Bit.Test.Common.AutoFixture.Attributes;
+using Xunit;
+
+namespace Bit.Core.Test.Services;
+
+public class IntegrationFilterFactoryTests
+{
+    [Theory, BitAutoData]
+    public void BuildEqualityFilter_ReturnsCorrectMatch(EventMessage message)
+    {
+        var different = Guid.NewGuid();
+        var expected = Guid.NewGuid();
+        message.UserId = expected;
+
+        var filter = IntegrationFilterFactory.BuildEqualityFilter<Guid?>("UserId");
+
+        Assert.True(filter(message, expected));
+        Assert.False(filter(message, different));
+    }
+
+    [Theory, BitAutoData]
+    public void BuildEqualityFilter_UserIdIsNull_ReturnsFalse(EventMessage message)
+    {
+        message.UserId = null;
+
+        var filter = IntegrationFilterFactory.BuildEqualityFilter<Guid?>("UserId");
+
+        Assert.False(filter(message, Guid.NewGuid()));
+    }
+
+    [Theory, BitAutoData]
+    public void BuildInFilter_ReturnsCorrectMatch(EventMessage message)
+    {
+        var match = Guid.NewGuid();
+        message.UserId = match;
+        var inList = new List<Guid?> { Guid.NewGuid(), match, Guid.NewGuid() };
+        var outList = new List<Guid?> { Guid.NewGuid(), Guid.NewGuid() };
+
+        var filter = IntegrationFilterFactory.BuildInFilter<Guid?>("UserId");
+
+        Assert.True(filter(message, inList));
+        Assert.False(filter(message, outList));
+    }
+}

test/Core.Test/AdminConsole/Services/IntegrationFilterServiceTests.cs

@@ -0,0 +1,399 @@
+#nullable enable
+
+using System.Text.Json;
+using Bit.Core.AdminConsole.Models.Data.EventIntegrations;
+using Bit.Core.Models.Data;
+using Bit.Core.Services;
+using Bit.Test.Common.AutoFixture.Attributes;
+using Xunit;
+
+namespace Bit.Core.Test.Services;
+
+public class IntegrationFilterServiceTests
+{
+    private readonly IntegrationFilterService _service = new();
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_EqualsUserId_Matches(EventMessage eventMessage)
+    {
+        var userId = Guid.NewGuid();
+        eventMessage.UserId = userId;
+
+        var group = new IntegrationFilterGroup
+        {
+            AndOperator = true,
+            Rules =
+            [
+                new()
+                {
+                    Property = "UserId",
+                    Operation = IntegrationFilterOperation.Equals,
+                    Value = userId
+                }
+            ]
+        };
+
+        var result = _service.EvaluateFilterGroup(group, eventMessage);
+        Assert.True(result);
+
+        var jsonGroup = JsonSerializer.Serialize(group);
+        var roundtrippedGroup = JsonSerializer.Deserialize<IntegrationFilterGroup>(jsonGroup);
+        Assert.NotNull(roundtrippedGroup);
+        Assert.True(_service.EvaluateFilterGroup(roundtrippedGroup, eventMessage));
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_EqualsUserId_DoesNotMatch(EventMessage eventMessage)
+    {
+        eventMessage.UserId = Guid.NewGuid();
+        var otherUserId = Guid.NewGuid();
+
+        var group = new IntegrationFilterGroup
+        {
+            AndOperator = true,
+            Rules =
+            [
+                new()
+                {
+                    Property = "UserId",
+                    Operation = IntegrationFilterOperation.Equals,
+                    Value = otherUserId
+                }
+            ]
+        };
+
+        var result = _service.EvaluateFilterGroup(group, eventMessage);
+        Assert.False(result);
+
+        var jsonGroup = JsonSerializer.Serialize(group);
+        var roundtrippedGroup = JsonSerializer.Deserialize<IntegrationFilterGroup>(jsonGroup);
+        Assert.NotNull(roundtrippedGroup);
+        Assert.False(_service.EvaluateFilterGroup(roundtrippedGroup, eventMessage));
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_NotEqualsUniqueUserId_ReturnsTrue(EventMessage eventMessage)
+    {
+        var otherId = Guid.NewGuid();
+        eventMessage.UserId = otherId;
+
+        var group = new IntegrationFilterGroup
+        {
+            AndOperator = true,
+            Rules =
+            [
+                new()
+                {
+                    Property = "UserId",
+                    Operation = IntegrationFilterOperation.NotEquals,
+                    Value = Guid.NewGuid()
+                }
+            ]
+        };
+
+        var result = _service.EvaluateFilterGroup(group, eventMessage);
+        Assert.True(result);
+
+        var jsonGroup = JsonSerializer.Serialize(group);
+        var roundtrippedGroup = JsonSerializer.Deserialize<IntegrationFilterGroup>(jsonGroup);
+        Assert.NotNull(roundtrippedGroup);
+        Assert.True(_service.EvaluateFilterGroup(roundtrippedGroup, eventMessage));
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_NotEqualsMatchingUserId_ReturnsFalse(EventMessage eventMessage)
+    {
+        var id = Guid.NewGuid();
+        eventMessage.UserId = id;
+
+        var group = new IntegrationFilterGroup
+        {
+            AndOperator = true,
+            Rules =
+            [
+                new()
+                {
+                    Property = "UserId",
+                    Operation = IntegrationFilterOperation.NotEquals,
+                    Value = id
+                }
+            ]
+        };
+
+        var result = _service.EvaluateFilterGroup(group, eventMessage);
+        Assert.False(result);
+
+        var jsonGroup = JsonSerializer.Serialize(group);
+        var roundtrippedGroup = JsonSerializer.Deserialize<IntegrationFilterGroup>(jsonGroup);
+        Assert.NotNull(roundtrippedGroup);
+        Assert.False(_service.EvaluateFilterGroup(roundtrippedGroup, eventMessage));
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_InCollectionId_Matches(EventMessage eventMessage)
+    {
+        var id = Guid.NewGuid();
+        eventMessage.CollectionId = id;
+
+        var group = new IntegrationFilterGroup
+        {
+            AndOperator = true,
+            Rules =
+            [
+                new()
+                {
+                    Property = "CollectionId",
+                    Operation = IntegrationFilterOperation.In,
+                    Value = new Guid?[] { Guid.NewGuid(), id }
+                }
+            ]
+        };
+
+        var result = _service.EvaluateFilterGroup(group, eventMessage);
+        Assert.True(result);
+
+        var jsonGroup = JsonSerializer.Serialize(group);
+        var roundtrippedGroup = JsonSerializer.Deserialize<IntegrationFilterGroup>(jsonGroup);
+        Assert.NotNull(roundtrippedGroup);
+        Assert.True(_service.EvaluateFilterGroup(roundtrippedGroup, eventMessage));
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_InCollectionId_DoesNotMatch(EventMessage eventMessage)
+    {
+        eventMessage.CollectionId = Guid.NewGuid();
+
+        var group = new IntegrationFilterGroup
+        {
+            AndOperator = true,
+            Rules =
+            [
+                new()
+                {
+                    Property = "CollectionId",
+                    Operation = IntegrationFilterOperation.In,
+                    Value = new Guid?[] { Guid.NewGuid(), Guid.NewGuid() }
+                }
+            ]
+        };
+
+        var result = _service.EvaluateFilterGroup(group, eventMessage);
+        Assert.False(result);
+
+        var jsonGroup = JsonSerializer.Serialize(group);
+        var roundtrippedGroup = JsonSerializer.Deserialize<IntegrationFilterGroup>(jsonGroup);
+        Assert.NotNull(roundtrippedGroup);
+        Assert.False(_service.EvaluateFilterGroup(roundtrippedGroup, eventMessage));
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_NotInCollectionIdUniqueId_ReturnsTrue(EventMessage eventMessage)
+    {
+        eventMessage.CollectionId = Guid.NewGuid();
+
+        var group = new IntegrationFilterGroup
+        {
+            AndOperator = true,
+            Rules =
+            [
+                new()
+                {
+                    Property = "CollectionId",
+                    Operation = IntegrationFilterOperation.NotIn,
+                    Value = new Guid?[] { Guid.NewGuid(), Guid.NewGuid() }
+                }
+            ]
+        };
+
+        var result = _service.EvaluateFilterGroup(group, eventMessage);
+        Assert.True(result);
+
+        var jsonGroup = JsonSerializer.Serialize(group);
+        var roundtrippedGroup = JsonSerializer.Deserialize<IntegrationFilterGroup>(jsonGroup);
+        Assert.NotNull(roundtrippedGroup);
+        Assert.True(_service.EvaluateFilterGroup(roundtrippedGroup, eventMessage));
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_NotInCollectionIdPresent_ReturnsFalse(EventMessage eventMessage)
+    {
+        var matchId = Guid.NewGuid();
+        eventMessage.CollectionId = matchId;
+
+        var group = new IntegrationFilterGroup
+        {
+            AndOperator = true,
+            Rules =
+            [
+                new()
+                {
+                    Property = "CollectionId",
+                    Operation = IntegrationFilterOperation.NotIn,
+                    Value = new Guid?[] { Guid.NewGuid(), matchId }
+                }
+            ]
+        };
+
+        var result = _service.EvaluateFilterGroup(group, eventMessage);
+        Assert.False(result);
+
+        var jsonGroup = JsonSerializer.Serialize(group);
+        var roundtrippedGroup = JsonSerializer.Deserialize<IntegrationFilterGroup>(jsonGroup);
+        Assert.NotNull(roundtrippedGroup);
+        Assert.False(_service.EvaluateFilterGroup(roundtrippedGroup, eventMessage));
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_NestedGroups_AllMatch(EventMessage eventMessage)
+    {
+        var id = Guid.NewGuid();
+        var collectionId = Guid.NewGuid();
+        eventMessage.UserId = id;
+        eventMessage.CollectionId = collectionId;
+
+        var nestedGroup = new IntegrationFilterGroup
+        {
+            AndOperator = true,
+            Rules =
+            [
+                new() { Property = "UserId", Operation = IntegrationFilterOperation.Equals, Value = id },
+                new()
+                {
+                    Property = "CollectionId",
+                    Operation = IntegrationFilterOperation.In,
+                    Value = new Guid?[] { collectionId, Guid.NewGuid() }
+                }
+            ]
+        };
+
+        var topGroup = new IntegrationFilterGroup
+        {
+            AndOperator = true,
+            Groups = [nestedGroup]
+        };
+
+        var result = _service.EvaluateFilterGroup(topGroup, eventMessage);
+        Assert.True(result);
+
+        var jsonGroup = JsonSerializer.Serialize(topGroup);
+        var roundtrippedGroup = JsonSerializer.Deserialize<IntegrationFilterGroup>(jsonGroup);
+        Assert.NotNull(roundtrippedGroup);
+        Assert.True(_service.EvaluateFilterGroup(roundtrippedGroup, eventMessage));
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_UnknownProperty_ReturnsFalse(EventMessage eventMessage)
+    {
+        var group = new IntegrationFilterGroup
+        {
+            Rules =
+            [
+                new() { Property = "NotARealProperty", Operation = IntegrationFilterOperation.Equals, Value = "test" }
+            ]
+        };
+
+        var result = _service.EvaluateFilterGroup(group, eventMessage);
+        Assert.False(result);
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_UnsupportedOperation_ReturnsFalse(EventMessage eventMessage)
+    {
+        var group = new IntegrationFilterGroup
+        {
+            Rules =
+            [
+                new()
+                {
+                    Property = "UserId",
+                    Operation = (IntegrationFilterOperation)999, // Unknown operation
+                    Value = eventMessage.UserId
+                }
+            ]
+        };
+
+        var result = _service.EvaluateFilterGroup(group, eventMessage);
+        Assert.False(result);
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_WrongTypeForInList_ThrowsException(EventMessage eventMessage)
+    {
+        var group = new IntegrationFilterGroup
+        {
+            Rules =
+            [
+                new()
+                {
+                    Property = "CollectionId",
+                    Operation = IntegrationFilterOperation.In,
+                    Value = "not an array" // Should be Guid[]
+                }
+            ]
+        };
+
+        Assert.Throws<InvalidCastException>(() =>
+            _service.EvaluateFilterGroup(group, eventMessage));
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_NullValue_ThrowsException(EventMessage eventMessage)
+    {
+        var group = new IntegrationFilterGroup
+        {
+            Rules =
+            [
+                new()
+                {
+                    Property = "UserId",
+                    Operation = IntegrationFilterOperation.Equals,
+                    Value = null
+                }
+            ]
+        };
+
+        Assert.Throws<InvalidCastException>(() =>
+            _service.EvaluateFilterGroup(group, eventMessage));
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_EmptyRuleList_ReturnsTrue(EventMessage eventMessage)
+    {
+        var group = new IntegrationFilterGroup
+        {
+            Rules = [],
+            Groups = [],
+            AndOperator = true
+        };
+
+        var result = _service.EvaluateFilterGroup(group, eventMessage);
+        Assert.True(result); // Nothing to fail, returns true by design
+    }
+
+    [Theory, BitAutoData]
+    public void EvaluateFilterGroup_InvalidNestedGroup_ReturnsFalse(EventMessage eventMessage)
+    {
+        var group = new IntegrationFilterGroup
+        {
+            Groups =
+            [
+                new()
+                {
+                    Rules =
+                    [
+                        new()
+                        {
+                            Property = "Nope",
+                            Operation = IntegrationFilterOperation.Equals,
+                            Value = "bad"
+                        }
+                    ]
+                }
+            ],
+            AndOperator = true
+        };
+
+        var result = _service.EvaluateFilterGroup(group, eventMessage);
+        Assert.False(result);
+    }
+}