[Innovation Sprint] Phishing Detection (#5516)

* Initial stubbing out of the phishing service

* Add the phishing domain controller

* Add changes for the phishing domain get

* Add distributed cache to the phishing domain

Signed-off-by: Cy Okeke <cokeke@bitwarden.com>

* Rename the variable name

Signed-off-by: Cy Okeke <cokeke@bitwarden.com>

* Removed IPhishingDomainService

* Feature/phishing detection cronjob (#5512)

* Added caching to EF implementation. Added error handling and logging

* Refactored update method to use sqlbulkcopy instead of performing a round trip for each new insert

* Initial implementation for quartz job to get list of phishing domains

* Updated phishing domain settings to be its own interface

* Add phishing domain detection with checksum-based updates

* Updated auth for phishing domain endpoints to either require api, or licensing claims to support both web and browser clients, and selfhost api clients

* [Innovation Sprint] Updated Phishing domains to rely on blob storage (#5517)

* Updated phishing detection data layer to rely on azure blob storage instead of sql server

* dotnet format

* Took rider refactors

* Ensuring phishing.testcategory.com exists to test against

* Added redis to dev's docker-compose

* Removed redis from cloud profile

* Remove the Authorize attribute

* error whitespace fix whitespace formatting

* error WHITESPACE: Fix whitespace formatting

* Wrapped phishing detection feature behind feature flag (#5532)

* Increased timeout for fetching source list a bunch

* Removed PhishingDomains policy

---------

Signed-off-by: Cy Okeke <cokeke@bitwarden.com>
Co-authored-by: Cy Okeke <cokeke@bitwarden.com>

src/Core/Constants.cs

@@ -203,6 +203,7 @@ public static class FeatureFlagKeys
     public const string DesktopCipherForms = "pm-18520-desktop-cipher-forms";
     public const string PM19941MigrateCipherDomainToSdk = "pm-19941-migrate-cipher-domain-to-sdk";
     public const string EndUserNotifications = "pm-10609-end-user-notifications";
+    public const string PhishingDetection = "phishing-detection";
 
     public static List<string> GetAllKeys()
     {

src/Core/PhishingDomainFeatures/AzurePhishingDomainStorageService.cs

@@ -0,0 +1,92 @@
+using System.Text;
+using Azure.Storage.Blobs;
+using Azure.Storage.Blobs.Models;
+using Bit.Core.Settings;
+using Microsoft.Extensions.Logging;
+
+namespace Bit.Core.PhishingDomainFeatures;
+
+public class AzurePhishingDomainStorageService
+{
+    private const string _containerName = "phishingdomains";
+    private const string _domainsFileName = "domains.txt";
+    private const string _checksumFileName = "checksum.txt";
+
+    private readonly BlobServiceClient _blobServiceClient;
+    private readonly ILogger<AzurePhishingDomainStorageService> _logger;
+    private BlobContainerClient _containerClient;
+
+    public AzurePhishingDomainStorageService(
+        GlobalSettings globalSettings,
+        ILogger<AzurePhishingDomainStorageService> logger)
+    {
+        _blobServiceClient = new BlobServiceClient(globalSettings.Storage.ConnectionString);
+        _logger = logger;
+    }
+
+    public async Task<ICollection<string>> GetDomainsAsync()
+    {
+        await InitAsync();
+
+        var blobClient = _containerClient.GetBlobClient(_domainsFileName);
+        if (!await blobClient.ExistsAsync())
+        {
+            return [];
+        }
+
+        var response = await blobClient.DownloadAsync();
+        using var streamReader = new StreamReader(response.Value.Content);
+        var content = await streamReader.ReadToEndAsync();
+
+        return [.. content
+            .Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries)
+            .Select(line => line.Trim())
+            .Where(line => !string.IsNullOrWhiteSpace(line) && !line.StartsWith('#'))];
+    }
+
+    public async Task<string> GetChecksumAsync()
+    {
+        await InitAsync();
+
+        var blobClient = _containerClient.GetBlobClient(_checksumFileName);
+        if (!await blobClient.ExistsAsync())
+        {
+            return string.Empty;
+        }
+
+        var response = await blobClient.DownloadAsync();
+        using var streamReader = new StreamReader(response.Value.Content);
+        return (await streamReader.ReadToEndAsync()).Trim();
+    }
+
+    public async Task UpdateDomainsAsync(IEnumerable<string> domains, string checksum)
+    {
+        await InitAsync();
+
+        var domainsContent = string.Join(Environment.NewLine, domains);
+        var domainsStream = new MemoryStream(Encoding.UTF8.GetBytes(domainsContent));
+        var domainsBlobClient = _containerClient.GetBlobClient(_domainsFileName);
+
+        await domainsBlobClient.UploadAsync(domainsStream, new BlobUploadOptions
+        {
+            HttpHeaders = new BlobHttpHeaders { ContentType = "text/plain" }
+        }, CancellationToken.None);
+
+        var checksumStream = new MemoryStream(Encoding.UTF8.GetBytes(checksum));
+        var checksumBlobClient = _containerClient.GetBlobClient(_checksumFileName);
+
+        await checksumBlobClient.UploadAsync(checksumStream, new BlobUploadOptions
+        {
+            HttpHeaders = new BlobHttpHeaders { ContentType = "text/plain" }
+        }, CancellationToken.None);
+    }
+
+    private async Task InitAsync()
+    {
+        if (_containerClient is null)
+        {
+            _containerClient = _blobServiceClient.GetBlobContainerClient(_containerName);
+            await _containerClient.CreateIfNotExistsAsync();
+        }
+    }
+}

src/Core/PhishingDomainFeatures/CloudPhishingDomainDirectQuery.cs

@@ -0,0 +1,100 @@
+using Bit.Core.PhishingDomainFeatures.Interfaces;
+using Bit.Core.Settings;
+using Microsoft.Extensions.Logging;
+
+namespace Bit.Core.PhishingDomainFeatures;
+
+/// <summary>
+/// Implementation of ICloudPhishingDomainQuery for cloud environments
+/// that directly calls the external phishing domain source
+/// </summary>
+public class CloudPhishingDomainDirectQuery : ICloudPhishingDomainQuery
+{
+    private readonly IGlobalSettings _globalSettings;
+    private readonly IHttpClientFactory _httpClientFactory;
+    private readonly ILogger<CloudPhishingDomainDirectQuery> _logger;
+
+    public CloudPhishingDomainDirectQuery(
+        IGlobalSettings globalSettings,
+        IHttpClientFactory httpClientFactory,
+        ILogger<CloudPhishingDomainDirectQuery> logger)
+    {
+        _globalSettings = globalSettings;
+        _httpClientFactory = httpClientFactory;
+        _logger = logger;
+    }
+
+    public async Task<List<string>> GetPhishingDomainsAsync()
+    {
+        if (string.IsNullOrWhiteSpace(_globalSettings.PhishingDomain?.UpdateUrl))
+        {
+            throw new InvalidOperationException("Phishing domain update URL is not configured.");
+        }
+
+        var httpClient = _httpClientFactory.CreateClient("PhishingDomains");
+        var response = await httpClient.GetAsync(_globalSettings.PhishingDomain.UpdateUrl);
+        response.EnsureSuccessStatusCode();
+
+        var content = await response.Content.ReadAsStringAsync();
+        return ParseDomains(content);
+    }
+
+    /// <summary>
+    /// Gets the SHA256 checksum of the remote phishing domains list
+    /// </summary>
+    /// <returns>The SHA256 checksum as a lowercase hex string</returns>
+    public async Task<string> GetRemoteChecksumAsync()
+    {
+        if (string.IsNullOrWhiteSpace(_globalSettings.PhishingDomain?.ChecksumUrl))
+        {
+            _logger.LogWarning("Phishing domain checksum URL is not configured.");
+            return string.Empty;
+        }
+
+        try
+        {
+            var httpClient = _httpClientFactory.CreateClient("PhishingDomains");
+            var response = await httpClient.GetAsync(_globalSettings.PhishingDomain.ChecksumUrl);
+            response.EnsureSuccessStatusCode();
+
+            var content = await response.Content.ReadAsStringAsync();
+            return ParseChecksumResponse(content);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error retrieving phishing domain checksum from {Url}",
+                _globalSettings.PhishingDomain.ChecksumUrl);
+            return string.Empty;
+        }
+    }
+
+    /// <summary>
+    /// Parses a checksum response in the format "hash *filename"
+    /// </summary>
+    private static string ParseChecksumResponse(string checksumContent)
+    {
+        if (string.IsNullOrWhiteSpace(checksumContent))
+        {
+            return string.Empty;
+        }
+
+        // Format is typically "hash *filename"
+        var parts = checksumContent.Split(' ', 2);
+
+        return parts.Length > 0 ? parts[0].Trim() : string.Empty;
+    }
+
+    private static List<string> ParseDomains(string content)
+    {
+        if (string.IsNullOrWhiteSpace(content))
+        {
+            return [];
+        }
+
+        return content
+            .Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries)
+            .Select(line => line.Trim())
+            .Where(line => !string.IsNullOrWhiteSpace(line) && !line.StartsWith("#"))
+            .ToList();
+    }
+}

src/Core/PhishingDomainFeatures/CloudPhishingDomainRelayQuery.cs

@@ -0,0 +1,66 @@
+using Bit.Core.PhishingDomainFeatures.Interfaces;
+using Bit.Core.Services;
+using Bit.Core.Settings;
+using Microsoft.Extensions.Logging;
+
+namespace Bit.Core.PhishingDomainFeatures;
+
+/// <summary>
+/// Implementation of ICloudPhishingDomainQuery for self-hosted environments
+/// that relays the request to the Bitwarden cloud API
+/// </summary>
+public class CloudPhishingDomainRelayQuery : BaseIdentityClientService, ICloudPhishingDomainQuery
+{
+    private readonly IGlobalSettings _globalSettings;
+
+    public CloudPhishingDomainRelayQuery(
+        IHttpClientFactory httpFactory,
+        IGlobalSettings globalSettings,
+        ILogger<CloudPhishingDomainRelayQuery> logger)
+        : base(
+            httpFactory,
+            globalSettings.Installation.ApiUri,
+            globalSettings.Installation.IdentityUri,
+            "api.licensing",
+            $"installation.{globalSettings.Installation.Id}",
+            globalSettings.Installation.Key,
+            logger)
+    {
+        _globalSettings = globalSettings;
+    }
+
+    public async Task<List<string>> GetPhishingDomainsAsync()
+    {
+        if (!_globalSettings.SelfHosted || !_globalSettings.EnableCloudCommunication)
+        {
+            throw new InvalidOperationException("This query is only for self-hosted installations with cloud communication enabled.");
+        }
+
+        var result = await SendAsync<object, string[]>(HttpMethod.Get, "phishing-domains", null, true);
+        return result?.ToList() ?? new List<string>();
+    }
+
+    /// <summary>
+    /// Gets the SHA256 checksum of the remote phishing domains list
+    /// </summary>
+    /// <returns>The SHA256 checksum as a lowercase hex string</returns>
+    public async Task<string> GetRemoteChecksumAsync()
+    {
+        if (!_globalSettings.SelfHosted || !_globalSettings.EnableCloudCommunication)
+        {
+            throw new InvalidOperationException("This query is only for self-hosted installations with cloud communication enabled.");
+        }
+
+        try
+        {
+            // For self-hosted environments, we get the checksum from the Bitwarden cloud API
+            var result = await SendAsync<object, string>(HttpMethod.Get, "phishing-domains/checksum", null, true);
+            return result ?? string.Empty;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error retrieving phishing domain checksum from Bitwarden cloud API");
+            return string.Empty;
+        }
+    }
+}

src/Core/PhishingDomainFeatures/Interfaces/ICloudPhishingDomainQuery.cs

@@ -0,0 +1,7 @@
+namespace Bit.Core.PhishingDomainFeatures.Interfaces;
+
+public interface ICloudPhishingDomainQuery
+{
+    Task<List<string>> GetPhishingDomainsAsync();
+    Task<string> GetRemoteChecksumAsync();
+}

src/Core/Repositories/IPhishingDomainRepository.cs

@@ -0,0 +1,8 @@
+namespace Bit.Core.Repositories;
+
+public interface IPhishingDomainRepository
+{
+    Task<ICollection<string>> GetActivePhishingDomainsAsync();
+    Task UpdatePhishingDomainsAsync(IEnumerable<string> domains, string checksum);
+    Task<string> GetCurrentChecksumAsync();
+}

src/Core/Repositories/Implementations/AzurePhishingDomainRepository.cs

@@ -0,0 +1,126 @@
+using System.Text.Json;
+using Bit.Core.PhishingDomainFeatures;
+using Microsoft.Extensions.Caching.Distributed;
+using Microsoft.Extensions.Logging;
+
+namespace Bit.Core.Repositories.Implementations;
+
+public class AzurePhishingDomainRepository : IPhishingDomainRepository
+{
+    private readonly AzurePhishingDomainStorageService _storageService;
+    private readonly IDistributedCache _cache;
+    private readonly ILogger<AzurePhishingDomainRepository> _logger;
+    private const string _domainsCacheKey = "PhishingDomains_v1";
+    private const string _checksumCacheKey = "PhishingDomains_Checksum_v1";
+    private static readonly DistributedCacheEntryOptions _cacheOptions = new()
+    {
+        AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(24),
+        SlidingExpiration = TimeSpan.FromHours(1)
+    };
+
+    public AzurePhishingDomainRepository(
+        AzurePhishingDomainStorageService storageService,
+        IDistributedCache cache,
+        ILogger<AzurePhishingDomainRepository> logger)
+    {
+        _storageService = storageService;
+        _cache = cache;
+        _logger = logger;
+    }
+
+    public async Task<ICollection<string>> GetActivePhishingDomainsAsync()
+    {
+        try
+        {
+            var cachedDomains = await _cache.GetStringAsync(_domainsCacheKey);
+            if (!string.IsNullOrEmpty(cachedDomains))
+            {
+                _logger.LogDebug("Retrieved phishing domains from cache");
+                return JsonSerializer.Deserialize<ICollection<string>>(cachedDomains) ?? [];
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to retrieve phishing domains from cache");
+        }
+
+        var domains = await _storageService.GetDomainsAsync();
+
+        try
+        {
+            await _cache.SetStringAsync(
+                _domainsCacheKey,
+                JsonSerializer.Serialize(domains),
+                _cacheOptions);
+            _logger.LogDebug("Stored {Count} phishing domains in cache", domains.Count);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to store phishing domains in cache");
+        }
+
+        return domains;
+    }
+
+    public async Task<string> GetCurrentChecksumAsync()
+    {
+        try
+        {
+            var cachedChecksum = await _cache.GetStringAsync(_checksumCacheKey);
+            if (!string.IsNullOrEmpty(cachedChecksum))
+            {
+                _logger.LogDebug("Retrieved phishing domain checksum from cache");
+                return cachedChecksum;
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to retrieve phishing domain checksum from cache");
+        }
+
+        var checksum = await _storageService.GetChecksumAsync();
+
+        try
+        {
+            if (!string.IsNullOrEmpty(checksum))
+            {
+                await _cache.SetStringAsync(
+                    _checksumCacheKey,
+                    checksum,
+                    _cacheOptions);
+                _logger.LogDebug("Stored phishing domain checksum in cache");
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to store phishing domain checksum in cache");
+        }
+
+        return checksum;
+    }
+
+    public async Task UpdatePhishingDomainsAsync(IEnumerable<string> domains, string checksum)
+    {
+        var domainsList = domains.ToList();
+        await _storageService.UpdateDomainsAsync(domainsList, checksum);
+
+        try
+        {
+            await _cache.SetStringAsync(
+                _domainsCacheKey,
+                JsonSerializer.Serialize(domainsList),
+                _cacheOptions);
+
+            await _cache.SetStringAsync(
+                _checksumCacheKey,
+                checksum,
+                _cacheOptions);
+
+            _logger.LogDebug("Updated phishing domains cache after update operation");
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to update phishing domains in cache");
+        }
+    }
+}

src/Core/Settings/GlobalSettings.cs

@@ -85,6 +85,7 @@ public class GlobalSettings : IGlobalSettings
     public virtual ILaunchDarklySettings LaunchDarkly { get; set; } = new LaunchDarklySettings();
     public virtual string DevelopmentDirectory { get; set; }
     public virtual IWebPushSettings WebPush { get; set; } = new WebPushSettings();
+    public virtual IPhishingDomainSettings PhishingDomain { get; set; } = new PhishingDomainSettings();
 
     public virtual bool EnableEmailVerification { get; set; }
     public virtual string KdfDefaultHashKey { get; set; }
@@ -644,6 +645,12 @@ public class GlobalSettings : IGlobalSettings
         public int MaxNetworkRetries { get; set; } = 2;
     }
 
+    public class PhishingDomainSettings : IPhishingDomainSettings
+    {
+        public string UpdateUrl { get; set; }
+        public string ChecksumUrl { get; set; }
+    }
+
     public class DistributedIpRateLimitingSettings
     {
         public string RedisConnectionString { get; set; }

src/Core/Settings/IGlobalSettings.cs

@@ -29,4 +29,5 @@ public interface IGlobalSettings
     string DevelopmentDirectory { get; set; }
     IWebPushSettings WebPush { get; set; }
     GlobalSettings.EventLoggingSettings EventLogging { get; set; }
+    IPhishingDomainSettings PhishingDomain { get; set; }
 }

src/Core/Settings/IPhishingDomainSettings.cs

@@ -0,0 +1,7 @@
+namespace Bit.Core.Settings;
+
+public interface IPhishingDomainSettings
+{
+    string UpdateUrl { get; set; }
+    string ChecksumUrl { get; set; }
+}