[Innovation Sprint] Phishing Detection (#5516)

* Initial stubbing out of the phishing service

* Add the phishing domain controller

* Add changes for the phishing domain get

* Add distributed cache to the phishing domain

Signed-off-by: Cy Okeke <cokeke@bitwarden.com>

* Rename the variable name

Signed-off-by: Cy Okeke <cokeke@bitwarden.com>

* Removed IPhishingDomainService

* Feature/phishing detection cronjob (#5512)

* Added caching to EF implementation. Added error handling and logging

* Refactored update method to use sqlbulkcopy instead of performing a round trip for each new insert

* Initial implementation for quartz job to get list of phishing domains

* Updated phishing domain settings to be its own interface

* Add phishing domain detection with checksum-based updates

* Updated auth for phishing domain endpoints to either require api, or licensing claims to support both web and browser clients, and selfhost api clients

* [Innovation Sprint] Updated Phishing domains to rely on blob storage (#5517)

* Updated phishing detection data layer to rely on azure blob storage instead of sql server

* dotnet format

* Took rider refactors

* Ensuring phishing.testcategory.com exists to test against

* Added redis to dev's docker-compose

* Removed redis from cloud profile

* Remove the Authorize attribute

* error whitespace fix whitespace formatting

* error WHITESPACE: Fix whitespace formatting

* Wrapped phishing detection feature behind feature flag (#5532)

* Increased timeout for fetching source list a bunch

* Removed PhishingDomains policy

---------

Signed-off-by: Cy Okeke <cokeke@bitwarden.com>
Co-authored-by: Cy Okeke <cokeke@bitwarden.com>

src/Api/Controllers/PhishingDomainsController.cs

@@ -0,0 +1,34 @@
+using Bit.Core;
+using Bit.Core.Repositories;
+using Bit.Core.Services;
+using Microsoft.AspNetCore.Mvc;
+
+namespace Bit.Api.Controllers;
+
+[Route("phishing-domains")]
+public class PhishingDomainsController(IPhishingDomainRepository phishingDomainRepository, IFeatureService featureService) : Controller
+{
+    [HttpGet]
+    public async Task<ActionResult<ICollection<string>>> GetPhishingDomainsAsync()
+    {
+        if (!featureService.IsEnabled(FeatureFlagKeys.PhishingDetection))
+        {
+            return NotFound();
+        }
+
+        var domains = await phishingDomainRepository.GetActivePhishingDomainsAsync();
+        return Ok(domains);
+    }
+
+    [HttpGet("checksum")]
+    public async Task<ActionResult<string>> GetChecksumAsync()
+    {
+        if (!featureService.IsEnabled(FeatureFlagKeys.PhishingDetection))
+        {
+            return NotFound();
+        }
+
+        var checksum = await phishingDomainRepository.GetCurrentChecksumAsync();
+        return Ok(checksum);
+    }
+}

src/Api/Jobs/JobsHostedService.cs

@@ -58,6 +58,13 @@ public class JobsHostedService : BaseJobsHostedService
             .StartNow()
             .WithCronSchedule("0 0 * * * ?")
             .Build();
+        var updatePhishingDomainsTrigger = TriggerBuilder.Create()
+            .WithIdentity("UpdatePhishingDomainsTrigger")
+            .StartNow()
+            .WithSimpleSchedule(x => x
+                .WithIntervalInHours(24)
+                .RepeatForever())
+            .Build();
 
 
         var jobs = new List<Tuple<Type, ITrigger>>
@@ -68,6 +75,7 @@ public class JobsHostedService : BaseJobsHostedService
             new Tuple<Type, ITrigger>(typeof(ValidateUsersJob), everyTopOfTheSixthHourTrigger),
             new Tuple<Type, ITrigger>(typeof(ValidateOrganizationsJob), everyTwelfthHourAndThirtyMinutesTrigger),
             new Tuple<Type, ITrigger>(typeof(ValidateOrganizationDomainJob), validateOrganizationDomainTrigger),
+            new Tuple<Type, ITrigger>(typeof(UpdatePhishingDomainsJob), updatePhishingDomainsTrigger),
         };
 
         if (_globalSettings.SelfHosted && _globalSettings.EnableCloudCommunication)
@@ -96,6 +104,7 @@ public class JobsHostedService : BaseJobsHostedService
         services.AddTransient<ValidateUsersJob>();
         services.AddTransient<ValidateOrganizationsJob>();
         services.AddTransient<ValidateOrganizationDomainJob>();
+        services.AddTransient<UpdatePhishingDomainsJob>();
     }
 
     public static void AddCommercialSecretsManagerJobServices(IServiceCollection services)

src/Api/Jobs/UpdatePhishingDomainsJob.cs

@@ -0,0 +1,97 @@
+using Bit.Core;
+using Bit.Core.Jobs;
+using Bit.Core.PhishingDomainFeatures.Interfaces;
+using Bit.Core.Repositories;
+using Bit.Core.Services;
+using Bit.Core.Settings;
+using Quartz;
+
+namespace Bit.Api.Jobs;
+
+public class UpdatePhishingDomainsJob : BaseJob
+{
+    private readonly GlobalSettings _globalSettings;
+    private readonly IPhishingDomainRepository _phishingDomainRepository;
+    private readonly ICloudPhishingDomainQuery _cloudPhishingDomainQuery;
+    private readonly IFeatureService _featureService;
+    public UpdatePhishingDomainsJob(
+        GlobalSettings globalSettings,
+        IPhishingDomainRepository phishingDomainRepository,
+        ICloudPhishingDomainQuery cloudPhishingDomainQuery,
+        IFeatureService featureService,
+        ILogger<UpdatePhishingDomainsJob> logger)
+        : base(logger)
+    {
+        _globalSettings = globalSettings;
+        _phishingDomainRepository = phishingDomainRepository;
+        _cloudPhishingDomainQuery = cloudPhishingDomainQuery;
+        _featureService = featureService;
+    }
+
+    protected override async Task ExecuteJobAsync(IJobExecutionContext context)
+    {
+        if (!_featureService.IsEnabled(FeatureFlagKeys.PhishingDetection))
+        {
+            _logger.LogInformation(Constants.BypassFiltersEventId, "Skipping phishing domain update. Feature flag is disabled.");
+            return;
+        }
+
+        if (string.IsNullOrWhiteSpace(_globalSettings.PhishingDomain?.UpdateUrl))
+        {
+            _logger.LogInformation(Constants.BypassFiltersEventId, "Skipping phishing domain update. No URL configured.");
+            return;
+        }
+
+        if (_globalSettings.SelfHosted && !_globalSettings.EnableCloudCommunication)
+        {
+            _logger.LogInformation(Constants.BypassFiltersEventId, "Skipping phishing domain update. Cloud communication is disabled in global settings.");
+            return;
+        }
+
+        var remoteChecksum = await _cloudPhishingDomainQuery.GetRemoteChecksumAsync();
+        if (string.IsNullOrWhiteSpace(remoteChecksum))
+        {
+            _logger.LogWarning(Constants.BypassFiltersEventId, "Could not retrieve remote checksum. Skipping update.");
+            return;
+        }
+
+        var currentChecksum = await _phishingDomainRepository.GetCurrentChecksumAsync();
+
+        if (string.Equals(currentChecksum, remoteChecksum, StringComparison.OrdinalIgnoreCase))
+        {
+            _logger.LogInformation(Constants.BypassFiltersEventId,
+                "Phishing domains list is up to date (checksum: {Checksum}). Skipping update.",
+                currentChecksum);
+            return;
+        }
+
+        _logger.LogInformation(Constants.BypassFiltersEventId,
+            "Checksums differ (current: {CurrentChecksum}, remote: {RemoteChecksum}). Fetching updated domains from {Source}.",
+            currentChecksum, remoteChecksum, _globalSettings.SelfHosted ? "Bitwarden cloud API" : "external source");
+
+        try
+        {
+            var domains = await _cloudPhishingDomainQuery.GetPhishingDomainsAsync();
+            if (!domains.Contains("phishing.testcategory.com", StringComparer.OrdinalIgnoreCase))
+            {
+                domains.Add("phishing.testcategory.com");
+            }
+
+            if (domains.Count > 0)
+            {
+                _logger.LogInformation(Constants.BypassFiltersEventId, "Updating {Count} phishing domains with checksum {Checksum}.",
+                    domains.Count, remoteChecksum);
+                await _phishingDomainRepository.UpdatePhishingDomainsAsync(domains, remoteChecksum);
+                _logger.LogInformation(Constants.BypassFiltersEventId, "Successfully updated phishing domains.");
+            }
+            else
+            {
+                _logger.LogWarning(Constants.BypassFiltersEventId, "No valid domains found in the response. Skipping update.");
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(Constants.BypassFiltersEventId, ex, "Error updating phishing domains.");
+        }
+    }
+}

src/Api/Startup.cs

@@ -182,6 +182,7 @@ public class Startup
         services.AddBillingOperations();
         services.AddReportingServices();
         services.AddImportServices();
+        services.AddPhishingDomainServices(globalSettings);
 
         // Authorization Handlers
         services.AddAuthorizationHandlers();

src/Api/Utilities/ServiceCollectionExtensions.cs

@@ -3,6 +3,10 @@ using Bit.Api.Tools.Authorization;
 using Bit.Api.Vault.AuthorizationHandlers.Collections;
 using Bit.Core.AdminConsole.OrganizationFeatures.Groups.Authorization;
 using Bit.Core.IdentityServer;
+using Bit.Core.PhishingDomainFeatures;
+using Bit.Core.PhishingDomainFeatures.Interfaces;
+using Bit.Core.Repositories;
+using Bit.Core.Repositories.Implementations;
 using Bit.Core.Settings;
 using Bit.Core.Utilities;
 using Bit.Core.Vault.Authorization.SecurityTasks;
@@ -109,4 +113,25 @@ public static class ServiceCollectionExtensions
 
         services.AddScoped<IAuthorizationHandler, OrganizationRequirementHandler>();
     }
+
+    public static void AddPhishingDomainServices(this IServiceCollection services, GlobalSettings globalSettings)
+    {
+        services.AddHttpClient("PhishingDomains", client =>
+        {
+            client.DefaultRequestHeaders.Add("User-Agent", globalSettings.SelfHosted ? "Bitwarden Self-Hosted" : "Bitwarden");
+            client.Timeout = TimeSpan.FromSeconds(1000); // the source list is very slow
+        });
+
+        services.AddSingleton<AzurePhishingDomainStorageService>();
+        services.AddSingleton<IPhishingDomainRepository, AzurePhishingDomainRepository>();
+
+        if (globalSettings.SelfHosted)
+        {
+            services.AddScoped<ICloudPhishingDomainQuery, CloudPhishingDomainRelayQuery>();
+        }
+        else
+        {
+            services.AddScoped<ICloudPhishingDomainQuery, CloudPhishingDomainDirectQuery>();
+        }
+    }
 }

src/Api/appsettings.Development.json

@@ -37,6 +37,10 @@
     },
     "storage": {
       "connectionString": "UseDevelopmentStorage=true"
+    },
+    "phishingDomain": {
+      "updateUrl": "https://phish.co.za/latest/phishing-domains-ACTIVE.txt",
+      "checksumUrl": "https://raw.githubusercontent.com/Phishing-Database/checksums/refs/heads/master/phishing-domains-ACTIVE.txt.sha256"
     }
   }
 }

src/Api/appsettings.json

@@ -71,6 +71,9 @@
       "accessKeySecret": "SECRET",
       "region": "SECRET"
     },
+    "phishingDomain": {
+      "updateUrl": "SECRET"
+    },
     "distributedIpRateLimiting": {
       "enabled": true,
       "maxRedisTimeoutsThreshold": 10,