Shard notification hub (#4450)

* Allow for binning of comb IDs by date and value

* Introduce notification hub pool

* Replace device type sharding with comb + range sharding

* Fix proxy interface

* Use enumerable services for multiServiceNotificationHub

* Fix push interface usage

* Fix push notification service dependencies

* Fix push notification keys

* Fixup documentation

* Remove deprecated settings

* Fix tests

* PascalCase method names

* Remove unused request model properties

* Remove unused setting

* Improve DateFromComb precision

* Prefer readonly service enumerable

* Pascal case template holes

* Name TryParse methods TryParse

* Apply suggestions from code review

Co-authored-by: Justin Baur <19896123+justindbaur@users.noreply.github.com>

* AllClients is a set of clients and must be deduplicated

* Fix registration start time

* Add logging to initialization of a notification hub

* more logging

* Add lower level logging for hub settings

* Log when connection is resolved

* Improve log message

* Log pushes to notification hub

* temporarily elevate log messages for visibility

* Log in multi-service when relaying to another push service

* Revert to more reasonable logging free of user information

* Fixup merge

Deleting user was extracted to a command in #4803, this updates that work to use just the device ids as I did elsewhere in abd67e8ec

* Do not use bouncy castle exception types

* Add required services for logging

---------

Co-authored-by: Justin Baur <19896123+justindbaur@users.noreply.github.com>
Co-authored-by: bnagawiecki <107435978+bnagawiecki@users.noreply.github.com>

src/Core/AdminConsole/OrganizationFeatures/OrganizationUsers/RemoveOrganizationUserCommand.cs

@@ -162,12 +162,12 @@ public class RemoveOrganizationUserCommand : IRemoveOrganizationUserCommand
         }
     }
 
-    private async Task<IEnumerable<KeyValuePair<string, DeviceType>>> GetUserDeviceIdsAsync(Guid userId)
+    private async Task<IEnumerable<string>> GetUserDeviceIdsAsync(Guid userId)
     {
         var devices = await _deviceRepository.GetManyByUserIdAsync(userId);
         return devices
             .Where(d => !string.IsNullOrWhiteSpace(d.PushToken))
-            .Select(d => new KeyValuePair<string, DeviceType>(d.Id.ToString(), d.Type));
+            .Select(d => d.Id.ToString());
     }
 
     private async Task DeleteAndPushUserRegistrationAsync(Guid organizationId, Guid userId)

src/Core/AdminConsole/Services/Implementations/OrganizationService.cs

@@ -1838,12 +1838,12 @@ public class OrganizationService : IOrganizationService
     }
 
 
-    private async Task<IEnumerable<KeyValuePair<string, DeviceType>>> GetUserDeviceIdsAsync(Guid userId)
+    private async Task<IEnumerable<string>> GetUserDeviceIdsAsync(Guid userId)
     {
         var devices = await _deviceRepository.GetManyByUserIdAsync(userId);
         return devices
             .Where(d => !string.IsNullOrWhiteSpace(d.PushToken))
-            .Select(d => new KeyValuePair<string, DeviceType>(d.Id.ToString(), d.Type));
+            .Select(d => d.Id.ToString());
     }
 
     public async Task ReplaceAndUpdateCacheAsync(Organization org, EventType? orgEvent = null)

src/Core/Models/Api/Request/PushDeviceRequestModel.cs

@@ -1,5 +1,4 @@
 using System.ComponentModel.DataAnnotations;
-using Bit.Core.Enums;
 
 namespace Bit.Core.Models.Api;
 
@@ -7,6 +6,4 @@ public class PushDeviceRequestModel
 {
     [Required]
     public string Id { get; set; }
-    [Required]
-    public DeviceType Type { get; set; }
 }

src/Core/Models/Api/Request/PushUpdateRequestModel.cs

@@ -1,5 +1,4 @@
 using System.ComponentModel.DataAnnotations;
-using Bit.Core.Enums;
 
 namespace Bit.Core.Models.Api;
 
@@ -8,9 +7,9 @@ public class PushUpdateRequestModel
     public PushUpdateRequestModel()
     { }
 
-    public PushUpdateRequestModel(IEnumerable<KeyValuePair<string, DeviceType>> devices, string organizationId)
+    public PushUpdateRequestModel(IEnumerable<string> deviceIds, string organizationId)
     {
-        Devices = devices.Select(d => new PushDeviceRequestModel { Id = d.Key, Type = d.Value });
+        Devices = deviceIds.Select(d => new PushDeviceRequestModel { Id = d });
         OrganizationId = organizationId;
     }
 

src/Core/Models/Data/InstallationDeviceEntity.cs

@@ -37,4 +37,25 @@ public class InstallationDeviceEntity : ITableEntity
     {
         return deviceId != null && deviceId.Length == 73 && deviceId[36] == '_';
     }
+    public static bool TryParse(string deviceId, out InstallationDeviceEntity installationDeviceEntity)
+    {
+        installationDeviceEntity = null;
+        var installationId = Guid.Empty;
+        var deviceIdGuid = Guid.Empty;
+        if (!IsInstallationDeviceId(deviceId))
+        {
+            return false;
+        }
+        var parts = deviceId.Split("_");
+        if (parts.Length < 2)
+        {
+            return false;
+        }
+        if (!Guid.TryParse(parts[0], out installationId) || !Guid.TryParse(parts[1], out deviceIdGuid))
+        {
+            return false;
+        }
+        installationDeviceEntity = new InstallationDeviceEntity(installationId, deviceIdGuid);
+        return true;
+    }
 }

src/Core/NotificationHub/INotificationHubClientProxy.cs

@@ -0,0 +1,8 @@
+using Microsoft.Azure.NotificationHubs;
+
+namespace Bit.Core.NotificationHub;
+
+public interface INotificationHubProxy
+{
+    Task<(INotificationHubClient Client, NotificationOutcome Outcome)[]> SendTemplateNotificationAsync(IDictionary<string, string> properties, string tagExpression);
+}

src/Core/NotificationHub/INotificationHubPool.cs

@@ -0,0 +1,9 @@
+using Microsoft.Azure.NotificationHubs;
+
+namespace Bit.Core.NotificationHub;
+
+public interface INotificationHubPool
+{
+    NotificationHubClient ClientFor(Guid comb);
+    INotificationHubProxy AllClients { get; }
+}

src/Core/NotificationHub/NotificationHubClientProxy.cs

@@ -0,0 +1,26 @@
+using Microsoft.Azure.NotificationHubs;
+
+namespace Bit.Core.NotificationHub;
+
+public class NotificationHubClientProxy : INotificationHubProxy
+{
+    private readonly IEnumerable<INotificationHubClient> _clients;
+
+    public NotificationHubClientProxy(IEnumerable<INotificationHubClient> clients)
+    {
+        _clients = clients;
+    }
+
+    private async Task<(INotificationHubClient, T)[]> ApplyToAllClientsAsync<T>(Func<INotificationHubClient, Task<T>> action)
+    {
+        var tasks = _clients.Select(async c => (c, await action(c)));
+        return await Task.WhenAll(tasks);
+    }
+
+    // partial proxy of INotificationHubClient implementation
+    // Note: Any other methods that are needed can simply be delegated as done here.
+    public async Task<(INotificationHubClient Client, NotificationOutcome Outcome)[]> SendTemplateNotificationAsync(IDictionary<string, string> properties, string tagExpression)
+    {
+        return await ApplyToAllClientsAsync(async c => await c.SendTemplateNotificationAsync(properties, tagExpression));
+    }
+}

src/Core/NotificationHub/NotificationHubConnection.cs

@@ -0,0 +1,128 @@
+using Bit.Core.Settings;
+using Bit.Core.Utilities;
+using Microsoft.Azure.NotificationHubs;
+
+class NotificationHubConnection
+{
+    public string HubName { get; init; }
+    public string ConnectionString { get; init; }
+    public bool EnableSendTracing { get; init; }
+    private NotificationHubClient _hubClient;
+    /// <summary>
+    /// Gets the NotificationHubClient for this connection.
+    /// 
+    /// If the client is null, it will be initialized.
+    /// 
+    /// <throws>Exception</throws> if the connection is invalid.
+    /// </summary>
+    public NotificationHubClient HubClient
+    {
+        get
+        {
+            if (_hubClient == null)
+            {
+                if (!IsValid)
+                {
+                    throw new Exception("Invalid notification hub settings");
+                }
+                Init();
+            }
+            return _hubClient;
+        }
+        private set
+        {
+            _hubClient = value;
+        }
+    }
+    /// <summary>
+    /// Gets the start date for registration.
+    /// 
+    /// If null, registration is always disabled.
+    /// </summary>
+    public DateTime? RegistrationStartDate { get; init; }
+    /// <summary>
+    /// Gets the end date for registration.
+    /// 
+    /// If null, registration has no end date.
+    /// </summary>
+    public DateTime? RegistrationEndDate { get; init; }
+    /// <summary>
+    /// Gets whether all data needed to generate a connection to Notification Hub is present.
+    /// </summary>
+    public bool IsValid
+    {
+        get
+        {
+            {
+                var invalid = string.IsNullOrWhiteSpace(HubName) || string.IsNullOrWhiteSpace(ConnectionString);
+                return !invalid;
+            }
+        }
+    }
+
+    public string LogString
+    {
+        get
+        {
+            return $"HubName: {HubName}, EnableSendTracing: {EnableSendTracing}, RegistrationStartDate: {RegistrationStartDate}, RegistrationEndDate: {RegistrationEndDate}";
+        }
+    }
+
+    /// <summary>
+    /// Gets whether registration is enabled for the given comb ID.
+    /// This is based off of the generation time encoded in the comb ID.
+    /// </summary>
+    /// <param name="comb"></param>
+    /// <returns></returns>
+    public bool RegistrationEnabled(Guid comb)
+    {
+        var combTime = CoreHelpers.DateFromComb(comb);
+        return RegistrationEnabled(combTime);
+    }
+
+    /// <summary>
+    /// Gets whether registration is enabled for the given time.
+    /// </summary>
+    /// <param name="queryTime">The time to check</param>
+    /// <returns></returns>
+    public bool RegistrationEnabled(DateTime queryTime)
+    {
+        if (queryTime >= RegistrationEndDate || RegistrationStartDate == null)
+        {
+            return false;
+        }
+
+        return RegistrationStartDate < queryTime;
+    }
+
+    private NotificationHubConnection() { }
+
+    /// <summary>
+    /// Creates a new NotificationHubConnection from the given settings.
+    /// </summary>
+    /// <param name="settings"></param>
+    /// <returns></returns>
+    public static NotificationHubConnection From(GlobalSettings.NotificationHubSettings settings)
+    {
+        return new()
+        {
+            HubName = settings.HubName,
+            ConnectionString = settings.ConnectionString,
+            EnableSendTracing = settings.EnableSendTracing,
+            // Comb time is not precise enough for millisecond accuracy
+            RegistrationStartDate = settings.RegistrationStartDate.HasValue ? Truncate(settings.RegistrationStartDate.Value, TimeSpan.FromMilliseconds(10)) : null,
+            RegistrationEndDate = settings.RegistrationEndDate
+        };
+    }
+
+    private NotificationHubConnection Init()
+    {
+        HubClient = NotificationHubClient.CreateClientFromConnectionString(ConnectionString, HubName, EnableSendTracing);
+        return this;
+    }
+
+    private static DateTime Truncate(DateTime dateTime, TimeSpan resolution)
+    {
+        return dateTime.AddTicks(-(dateTime.Ticks % resolution.Ticks));
+    }
+}

src/Core/NotificationHub/NotificationHubPool.cs

@@ -0,0 +1,62 @@
+using Bit.Core.Settings;
+using Bit.Core.Utilities;
+using Microsoft.Azure.NotificationHubs;
+using Microsoft.Extensions.Logging;
+
+namespace Bit.Core.NotificationHub;
+
+public class NotificationHubPool : INotificationHubPool
+{
+    private List<NotificationHubConnection> _connections { get; }
+    private readonly IEnumerable<INotificationHubClient> _clients;
+    private readonly ILogger<NotificationHubPool> _logger;
+    public NotificationHubPool(ILogger<NotificationHubPool> logger, GlobalSettings globalSettings)
+    {
+        _logger = logger;
+        _connections = FilterInvalidHubs(globalSettings.NotificationHubPool.NotificationHubs);
+        _clients = _connections.GroupBy(c => c.ConnectionString).Select(g => g.First().HubClient);
+    }
+
+    private List<NotificationHubConnection> FilterInvalidHubs(IEnumerable<GlobalSettings.NotificationHubSettings> hubs)
+    {
+        List<NotificationHubConnection> result = new();
+        _logger.LogDebug("Filtering {HubCount} notification hubs", hubs.Count());
+        foreach (var hub in hubs)
+        {
+            var connection = NotificationHubConnection.From(hub);
+            if (!connection.IsValid)
+            {
+                _logger.LogWarning("Invalid notification hub settings: {HubName}", hub.HubName ?? "hub name missing");
+                continue;
+            }
+            _logger.LogDebug("Adding notification hub: {ConnectionLogString}", connection.LogString);
+            result.Add(connection);
+        }
+
+        return result;
+    }
+
+
+    /// <summary>
+    /// Gets the NotificationHubClient for the given comb ID.
+    /// </summary>
+    /// <param name="comb"></param>
+    /// <returns></returns>
+    /// <exception cref="InvalidOperationException">Thrown when no notification hub is found for a given comb.</exception>
+    public NotificationHubClient ClientFor(Guid comb)
+    {
+        var possibleConnections = _connections.Where(c => c.RegistrationEnabled(comb)).ToArray();
+        if (possibleConnections.Length == 0)
+        {
+            throw new InvalidOperationException($"No valid notification hubs are available for the given comb ({comb}).\n" +
+                $"The comb's datetime is {CoreHelpers.DateFromComb(comb)}." +
+                $"Hub start and end times are configured as follows:\n" +
+                string.Join("\n", _connections.Select(c => $"Hub {c.HubName} - Start: {c.RegistrationStartDate}, End: {c.RegistrationEndDate}")));
+        }
+        var resolvedConnection = possibleConnections[CoreHelpers.BinForComb(comb, possibleConnections.Length)];
+        _logger.LogTrace("Resolved notification hub for comb {Comb} out of {HubCount} hubs.\n{ConnectionInfo}", comb, possibleConnections.Length, resolvedConnection.LogString);
+        return resolvedConnection.HubClient;
+    }
+
+    public INotificationHubProxy AllClients { get { return new NotificationHubClientProxy(_clients); } }
+}

src/Core/NotificationHub/NotificationHubPushNotificationService.cs

@@ -6,45 +6,31 @@ using Bit.Core.Enums;
 using Bit.Core.Models;
 using Bit.Core.Models.Data;
 using Bit.Core.Repositories;
-using Bit.Core.Settings;
+using Bit.Core.Services;
 using Bit.Core.Tools.Entities;
 using Bit.Core.Vault.Entities;
 using Microsoft.AspNetCore.Http;
-using Microsoft.Azure.NotificationHubs;
 using Microsoft.Extensions.Logging;
 
-namespace Bit.Core.Services;
+namespace Bit.Core.NotificationHub;
 
 public class NotificationHubPushNotificationService : IPushNotificationService
 {
     private readonly IInstallationDeviceRepository _installationDeviceRepository;
-    private readonly GlobalSettings _globalSettings;
     private readonly IHttpContextAccessor _httpContextAccessor;
-    private readonly List<NotificationHubClient> _clients = [];
     private readonly bool _enableTracing = false;
+    private readonly INotificationHubPool _notificationHubPool;
     private readonly ILogger _logger;
 
     public NotificationHubPushNotificationService(
         IInstallationDeviceRepository installationDeviceRepository,
-        GlobalSettings globalSettings,
+        INotificationHubPool notificationHubPool,
         IHttpContextAccessor httpContextAccessor,
         ILogger<NotificationsApiPushNotificationService> logger)
     {
         _installationDeviceRepository = installationDeviceRepository;
-        _globalSettings = globalSettings;
         _httpContextAccessor = httpContextAccessor;
-
-        foreach (var hub in globalSettings.NotificationHubs)
-        {
-            var client = NotificationHubClient.CreateClientFromConnectionString(
-                hub.ConnectionString,
-                hub.HubName,
-                hub.EnableSendTracing);
-            _clients.Add(client);
-
-            _enableTracing = _enableTracing || hub.EnableSendTracing;
-        }
-
+        _notificationHubPool = notificationHubPool;
         _logger = logger;
     }
 
@@ -264,30 +250,23 @@ public class NotificationHubPushNotificationService : IPushNotificationService
 
     private async Task SendPayloadAsync(string tag, PushType type, object payload)
     {
-        var tasks = new List<Task<NotificationOutcome>>();
-        foreach (var client in _clients)
-        {
-            var task = client.SendTemplateNotificationAsync(
-                new Dictionary<string, string>
-                {
-                    { "type",  ((byte)type).ToString() },
-                    { "payload", JsonSerializer.Serialize(payload) }
-                }, tag);
-            tasks.Add(task);
-        }
-
-        await Task.WhenAll(tasks);
+        var results = await _notificationHubPool.AllClients.SendTemplateNotificationAsync(
+            new Dictionary<string, string>
+            {
+                { "type",  ((byte)type).ToString() },
+                { "payload", JsonSerializer.Serialize(payload) }
+            }, tag);
 
         if (_enableTracing)
         {
-            for (var i = 0; i < tasks.Count; i++)
+            foreach (var (client, outcome) in results)
             {
-                if (_clients[i].EnableTestSend)
+                if (!client.EnableTestSend)
                 {
-                    var outcome = await tasks[i];
-                    _logger.LogInformation("Azure Notification Hub Tracking ID: {id} | {type} push notification with {success} successes and {failure} failures with a payload of {@payload} and result of {@results}",
-                        outcome.TrackingId, type, outcome.Success, outcome.Failure, payload, outcome.Results);
+                    continue;
                 }
+                _logger.LogInformation("Azure Notification Hub Tracking ID: {Id} | {Type} push notification with {Success} successes and {Failure} failures with a payload of {@Payload} and result of {@Results}",
+                    outcome.TrackingId, type, outcome.Success, outcome.Failure, payload, outcome.Results);
             }
         }
     }

src/Core/NotificationHub/NotificationHubPushRegistrationService.cs

@@ -1,50 +1,34 @@
 using Bit.Core.Enums;
 using Bit.Core.Models.Data;
 using Bit.Core.Repositories;
+using Bit.Core.Services;
 using Bit.Core.Settings;
 using Microsoft.Azure.NotificationHubs;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 
-namespace Bit.Core.Services;
+namespace Bit.Core.NotificationHub;
 
 public class NotificationHubPushRegistrationService : IPushRegistrationService
 {
     private readonly IInstallationDeviceRepository _installationDeviceRepository;
     private readonly GlobalSettings _globalSettings;
+    private readonly INotificationHubPool _notificationHubPool;
     private readonly IServiceProvider _serviceProvider;
     private readonly ILogger<NotificationHubPushRegistrationService> _logger;
-    private Dictionary<NotificationHubType, NotificationHubClient> _clients = [];
 
     public NotificationHubPushRegistrationService(
         IInstallationDeviceRepository installationDeviceRepository,
         GlobalSettings globalSettings,
+        INotificationHubPool notificationHubPool,
         IServiceProvider serviceProvider,
         ILogger<NotificationHubPushRegistrationService> logger)
     {
         _installationDeviceRepository = installationDeviceRepository;
         _globalSettings = globalSettings;
+        _notificationHubPool = notificationHubPool;
         _serviceProvider = serviceProvider;
         _logger = logger;
-
-        // Is this dirty to do in the ctor?
-        void addHub(NotificationHubType type)
-        {
-            var hubRegistration = globalSettings.NotificationHubs.FirstOrDefault(
-                h => h.HubType == type && h.EnableRegistration);
-            if (hubRegistration != null)
-            {
-                var client = NotificationHubClient.CreateClientFromConnectionString(
-                    hubRegistration.ConnectionString,
-                    hubRegistration.HubName,
-                    hubRegistration.EnableSendTracing);
-                _clients.Add(type, client);
-            }
-        }
-
-        addHub(NotificationHubType.General);
-        addHub(NotificationHubType.iOS);
-        addHub(NotificationHubType.Android);
     }
 
     public async Task CreateOrUpdateRegistrationAsync(string pushToken, string deviceId, string userId,
@@ -117,7 +101,7 @@ public class NotificationHubPushRegistrationService : IPushRegistrationService
         BuildInstallationTemplate(installation, "badgeMessage", badgeMessageTemplate ?? messageTemplate,
             userId, identifier);
 
-        await GetClient(type).CreateOrUpdateInstallationAsync(installation);
+        await ClientFor(GetComb(deviceId)).CreateOrUpdateInstallationAsync(installation);
         if (InstallationDeviceEntity.IsInstallationDeviceId(deviceId))
         {
             await _installationDeviceRepository.UpsertAsync(new InstallationDeviceEntity(deviceId));
@@ -152,11 +136,11 @@ public class NotificationHubPushRegistrationService : IPushRegistrationService
         installation.Templates.Add(fullTemplateId, template);
     }
 
-    public async Task DeleteRegistrationAsync(string deviceId, DeviceType deviceType)
+    public async Task DeleteRegistrationAsync(string deviceId)
     {
         try
         {
-            await GetClient(deviceType).DeleteInstallationAsync(deviceId);
+            await ClientFor(GetComb(deviceId)).DeleteInstallationAsync(deviceId);
             if (InstallationDeviceEntity.IsInstallationDeviceId(deviceId))
             {
                 await _installationDeviceRepository.DeleteAsync(new InstallationDeviceEntity(deviceId));
@@ -168,31 +152,31 @@ public class NotificationHubPushRegistrationService : IPushRegistrationService
         }
     }
 
-    public async Task AddUserRegistrationOrganizationAsync(IEnumerable<KeyValuePair<string, DeviceType>> devices, string organizationId)
+    public async Task AddUserRegistrationOrganizationAsync(IEnumerable<string> deviceIds, string organizationId)
     {
-        await PatchTagsForUserDevicesAsync(devices, UpdateOperationType.Add, $"organizationId:{organizationId}");
-        if (devices.Any() && InstallationDeviceEntity.IsInstallationDeviceId(devices.First().Key))
+        await PatchTagsForUserDevicesAsync(deviceIds, UpdateOperationType.Add, $"organizationId:{organizationId}");
+        if (deviceIds.Any() && InstallationDeviceEntity.IsInstallationDeviceId(deviceIds.First()))
         {
-            var entities = devices.Select(e => new InstallationDeviceEntity(e.Key));
+            var entities = deviceIds.Select(e => new InstallationDeviceEntity(e));
             await _installationDeviceRepository.UpsertManyAsync(entities.ToList());
         }
     }
 
-    public async Task DeleteUserRegistrationOrganizationAsync(IEnumerable<KeyValuePair<string, DeviceType>> devices, string organizationId)
+    public async Task DeleteUserRegistrationOrganizationAsync(IEnumerable<string> deviceIds, string organizationId)
     {
-        await PatchTagsForUserDevicesAsync(devices, UpdateOperationType.Remove,
+        await PatchTagsForUserDevicesAsync(deviceIds, UpdateOperationType.Remove,
             $"organizationId:{organizationId}");
-        if (devices.Any() && InstallationDeviceEntity.IsInstallationDeviceId(devices.First().Key))
+        if (deviceIds.Any() && InstallationDeviceEntity.IsInstallationDeviceId(deviceIds.First()))
         {
-            var entities = devices.Select(e => new InstallationDeviceEntity(e.Key));
+            var entities = deviceIds.Select(e => new InstallationDeviceEntity(e));
             await _installationDeviceRepository.UpsertManyAsync(entities.ToList());
         }
     }
 
-    private async Task PatchTagsForUserDevicesAsync(IEnumerable<KeyValuePair<string, DeviceType>> devices, UpdateOperationType op,
+    private async Task PatchTagsForUserDevicesAsync(IEnumerable<string> deviceIds, UpdateOperationType op,
         string tag)
     {
-        if (!devices.Any())
+        if (!deviceIds.Any())
         {
             return;
         }
@@ -212,11 +196,11 @@ public class NotificationHubPushRegistrationService : IPushRegistrationService
             operation.Path += $"/{tag}";
         }
 
-        foreach (var device in devices)
+        foreach (var deviceId in deviceIds)
         {
             try
             {
-                await GetClient(device.Value).PatchInstallationAsync(device.Key, new List<PartialUpdateOperation> { operation });
+                await ClientFor(GetComb(deviceId)).PatchInstallationAsync(deviceId, new List<PartialUpdateOperation> { operation });
             }
             catch (Exception e) when (e.InnerException == null || !e.InnerException.Message.Contains("(404) Not Found"))
             {
@@ -225,53 +209,29 @@ public class NotificationHubPushRegistrationService : IPushRegistrationService
         }
     }
 
-    private NotificationHubClient GetClient(DeviceType deviceType)
+    private NotificationHubClient ClientFor(Guid deviceId)
     {
-        var hubType = NotificationHubType.General;
-        switch (deviceType)
+        return _notificationHubPool.ClientFor(deviceId);
+    }
+
+    private Guid GetComb(string deviceId)
+    {
+        var deviceIdString = deviceId;
+        InstallationDeviceEntity installationDeviceEntity;
+        Guid deviceIdGuid;
+        if (InstallationDeviceEntity.TryParse(deviceIdString, out installationDeviceEntity))
         {
-            case DeviceType.Android:
-                hubType = NotificationHubType.Android;
-                break;
-            case DeviceType.iOS:
-                hubType = NotificationHubType.iOS;
-                break;
-            case DeviceType.ChromeExtension:
-            case DeviceType.FirefoxExtension:
-            case DeviceType.OperaExtension:
-            case DeviceType.EdgeExtension:
-            case DeviceType.VivaldiExtension:
-            case DeviceType.SafariExtension:
-                hubType = NotificationHubType.GeneralBrowserExtension;
-                break;
-            case DeviceType.WindowsDesktop:
-            case DeviceType.MacOsDesktop:
-            case DeviceType.LinuxDesktop:
-                hubType = NotificationHubType.GeneralDesktop;
-                break;
-            case DeviceType.ChromeBrowser:
-            case DeviceType.FirefoxBrowser:
-            case DeviceType.OperaBrowser:
-            case DeviceType.EdgeBrowser:
-            case DeviceType.IEBrowser:
-            case DeviceType.UnknownBrowser:
-            case DeviceType.SafariBrowser:
-            case DeviceType.VivaldiBrowser:
-                hubType = NotificationHubType.GeneralWeb;
-                break;
-            default:
-                break;
+            // Strip off the installation id (PartitionId). RowKey is the ID in the Installation's table.
+            deviceIdString = installationDeviceEntity.RowKey;
         }
 
-        if (!_clients.ContainsKey(hubType))
+        if (Guid.TryParse(deviceIdString, out deviceIdGuid))
         {
-            _logger.LogWarning("No hub client for '{0}'. Using general hub instead.", hubType);
-            hubType = NotificationHubType.General;
-            if (!_clients.ContainsKey(hubType))
-            {
-                throw new Exception("No general hub client found.");
-            }
         }
-        return _clients[hubType];
+        else
+        {
+            throw new Exception($"Invalid device id {deviceId}.");
+        }
+        return deviceIdGuid;
     }
 }

src/Core/Services/IPushRegistrationService.cs

@@ -6,7 +6,7 @@ public interface IPushRegistrationService
 {
     Task CreateOrUpdateRegistrationAsync(string pushToken, string deviceId, string userId,
         string identifier, DeviceType type);
-    Task DeleteRegistrationAsync(string deviceId, DeviceType type);
-    Task AddUserRegistrationOrganizationAsync(IEnumerable<KeyValuePair<string, DeviceType>> devices, string organizationId);
-    Task DeleteUserRegistrationOrganizationAsync(IEnumerable<KeyValuePair<string, DeviceType>> devices, string organizationId);
+    Task DeleteRegistrationAsync(string deviceId);
+    Task AddUserRegistrationOrganizationAsync(IEnumerable<string> deviceIds, string organizationId);
+    Task DeleteUserRegistrationOrganizationAsync(IEnumerable<string> deviceIds, string organizationId);
 }

src/Core/Services/Implementations/DeviceService.cs

@@ -38,13 +38,13 @@ public class DeviceService : IDeviceService
     public async Task ClearTokenAsync(Device device)
     {
         await _deviceRepository.ClearPushTokenAsync(device.Id);
-        await _pushRegistrationService.DeleteRegistrationAsync(device.Id.ToString(), device.Type);
+        await _pushRegistrationService.DeleteRegistrationAsync(device.Id.ToString());
     }
 
     public async Task DeleteAsync(Device device)
     {
         await _deviceRepository.DeleteAsync(device);
-        await _pushRegistrationService.DeleteRegistrationAsync(device.Id.ToString(), device.Type);
+        await _pushRegistrationService.DeleteRegistrationAsync(device.Id.ToString());
     }
 
     public async Task UpdateDevicesTrustAsync(string currentDeviceIdentifier,

src/Core/Services/Implementations/MultiServicePushNotificationService.cs

@@ -1,61 +1,31 @@
 using Bit.Core.Auth.Entities;
 using Bit.Core.Enums;
-using Bit.Core.Repositories;
 using Bit.Core.Settings;
 using Bit.Core.Tools.Entities;
-using Bit.Core.Utilities;
 using Bit.Core.Vault.Entities;
-using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 
 namespace Bit.Core.Services;
 
 public class MultiServicePushNotificationService : IPushNotificationService
 {
-    private readonly List<IPushNotificationService> _services = new List<IPushNotificationService>();
+    private readonly IEnumerable<IPushNotificationService> _services;
     private readonly ILogger<MultiServicePushNotificationService> _logger;
 
     public MultiServicePushNotificationService(
-        IHttpClientFactory httpFactory,
-        IDeviceRepository deviceRepository,
-        IInstallationDeviceRepository installationDeviceRepository,
-        GlobalSettings globalSettings,
-        IHttpContextAccessor httpContextAccessor,
+        [FromKeyedServices("implementation")] IEnumerable<IPushNotificationService> services,
         ILogger<MultiServicePushNotificationService> logger,
-        ILogger<RelayPushNotificationService> relayLogger,
-        ILogger<NotificationsApiPushNotificationService> hubLogger)
+        GlobalSettings globalSettings)
     {
-        if (globalSettings.SelfHosted)
-        {
-            if (CoreHelpers.SettingHasValue(globalSettings.PushRelayBaseUri) &&
-                globalSettings.Installation?.Id != null &&
-                CoreHelpers.SettingHasValue(globalSettings.Installation?.Key))
-            {
-                _services.Add(new RelayPushNotificationService(httpFactory, deviceRepository, globalSettings,
-                    httpContextAccessor, relayLogger));
-            }
-            if (CoreHelpers.SettingHasValue(globalSettings.InternalIdentityKey) &&
-                CoreHelpers.SettingHasValue(globalSettings.BaseServiceUri.InternalNotifications))
-            {
-                _services.Add(new NotificationsApiPushNotificationService(
-                    httpFactory, globalSettings, httpContextAccessor, hubLogger));
-            }
-        }
-        else
-        {
-            var generalHub = globalSettings.NotificationHubs?.FirstOrDefault(h => h.HubType == NotificationHubType.General);
-            if (CoreHelpers.SettingHasValue(generalHub?.ConnectionString))
-            {
-                _services.Add(new NotificationHubPushNotificationService(installationDeviceRepository,
-                    globalSettings, httpContextAccessor, hubLogger));
-            }
-            if (CoreHelpers.SettingHasValue(globalSettings.Notifications?.ConnectionString))
-            {
-                _services.Add(new AzureQueuePushNotificationService(globalSettings, httpContextAccessor));
-            }
-        }
+        _services = services;
 
         _logger = logger;
+        _logger.LogInformation("Hub services: {Services}", _services.Count());
+        globalSettings?.NotificationHubPool?.NotificationHubs?.ForEach(hub =>
+        {
+            _logger.LogInformation("HubName: {HubName}, EnableSendTracing: {EnableSendTracing}, RegistrationStartDate: {RegistrationStartDate}, RegistrationEndDate: {RegistrationEndDate}", hub.HubName, hub.EnableSendTracing, hub.RegistrationStartDate, hub.RegistrationEndDate);
+        });
     }
 
     public Task PushSyncCipherCreateAsync(Cipher cipher, IEnumerable<Guid> collectionIds)

src/Core/Services/Implementations/RelayPushRegistrationService.cs

@@ -38,37 +38,36 @@ public class RelayPushRegistrationService : BaseIdentityClientService, IPushRegi
         await SendAsync(HttpMethod.Post, "push/register", requestModel);
     }
 
-    public async Task DeleteRegistrationAsync(string deviceId, DeviceType type)
+    public async Task DeleteRegistrationAsync(string deviceId)
     {
         var requestModel = new PushDeviceRequestModel
         {
             Id = deviceId,
-            Type = type,
         };
         await SendAsync(HttpMethod.Post, "push/delete", requestModel);
     }
 
     public async Task AddUserRegistrationOrganizationAsync(
-        IEnumerable<KeyValuePair<string, DeviceType>> devices, string organizationId)
+        IEnumerable<string> deviceIds, string organizationId)
     {
-        if (!devices.Any())
+        if (!deviceIds.Any())
         {
             return;
         }
 
-        var requestModel = new PushUpdateRequestModel(devices, organizationId);
+        var requestModel = new PushUpdateRequestModel(deviceIds, organizationId);
         await SendAsync(HttpMethod.Put, "push/add-organization", requestModel);
     }
 
     public async Task DeleteUserRegistrationOrganizationAsync(
-        IEnumerable<KeyValuePair<string, DeviceType>> devices, string organizationId)
+        IEnumerable<string> deviceIds, string organizationId)
     {
-        if (!devices.Any())
+        if (!deviceIds.Any())
         {
             return;
         }
 
-        var requestModel = new PushUpdateRequestModel(devices, organizationId);
+        var requestModel = new PushUpdateRequestModel(deviceIds, organizationId);
         await SendAsync(HttpMethod.Put, "push/delete-organization", requestModel);
     }
 }

src/Core/Services/NoopImplementations/NoopPushRegistrationService.cs

@@ -4,7 +4,7 @@ namespace Bit.Core.Services;
 
 public class NoopPushRegistrationService : IPushRegistrationService
 {
-    public Task AddUserRegistrationOrganizationAsync(IEnumerable<KeyValuePair<string, DeviceType>> devices, string organizationId)
+    public Task AddUserRegistrationOrganizationAsync(IEnumerable<string> deviceIds, string organizationId)
     {
         return Task.FromResult(0);
     }
@@ -15,12 +15,12 @@ public class NoopPushRegistrationService : IPushRegistrationService
         return Task.FromResult(0);
     }
 
-    public Task DeleteRegistrationAsync(string deviceId, DeviceType deviceType)
+    public Task DeleteRegistrationAsync(string deviceId)
     {
         return Task.FromResult(0);
     }
 
-    public Task DeleteUserRegistrationOrganizationAsync(IEnumerable<KeyValuePair<string, DeviceType>> devices, string organizationId)
+    public Task DeleteUserRegistrationOrganizationAsync(IEnumerable<string> deviceIds, string organizationId)
     {
         return Task.FromResult(0);
     }

src/Core/Settings/GlobalSettings.cs

@@ -1,5 +1,4 @@
 using Bit.Core.Auth.Settings;
-using Bit.Core.Enums;
 using Bit.Core.Settings.LoggingSettings;
 
 namespace Bit.Core.Settings;
@@ -65,7 +64,7 @@ public class GlobalSettings : IGlobalSettings
     public virtual SentrySettings Sentry { get; set; } = new SentrySettings();
     public virtual SyslogSettings Syslog { get; set; } = new SyslogSettings();
     public virtual ILogLevelSettings MinLogLevel { get; set; } = new LogLevelSettings();
-    public virtual List<NotificationHubSettings> NotificationHubs { get; set; } = new();
+    public virtual NotificationHubPoolSettings NotificationHubPool { get; set; } = new();
     public virtual YubicoSettings Yubico { get; set; } = new YubicoSettings();
     public virtual DuoSettings Duo { get; set; } = new DuoSettings();
     public virtual BraintreeSettings Braintree { get; set; } = new BraintreeSettings();
@@ -424,7 +423,7 @@ public class GlobalSettings : IGlobalSettings
         public string ConnectionString
         {
             get => _connectionString;
-            set => _connectionString = value.Trim('"');
+            set => _connectionString = value?.Trim('"');
         }
         public string HubName { get; set; }
         /// <summary>
@@ -433,10 +432,32 @@ public class GlobalSettings : IGlobalSettings
         /// </summary>
         public bool EnableSendTracing { get; set; } = false;
         /// <summary>
-        /// At least one hub configuration should have registration enabled, preferably the General hub as a safety net.
+        /// The date and time at which registration will be enabled.
+        /// 
+        /// **This value should not be updated once set, as it is used to determine installation location of devices.**
+        /// 
+        /// If null, registration is disabled.
+        /// 
         /// </summary>
-        public bool EnableRegistration { get; set; }
-        public NotificationHubType HubType { get; set; }
+        public DateTime? RegistrationStartDate { get; set; }
+        /// <summary>
+        /// The date and time at which registration will be disabled.
+        /// 
+        /// **This value should not be updated once set, as it is used to determine installation location of devices.**
+        /// 
+        /// If null, hub registration has no yet known expiry.
+        /// </summary>
+        public DateTime? RegistrationEndDate { get; set; }
+    }
+
+    public class NotificationHubPoolSettings
+    {
+        /// <summary>
+        /// List of Notification Hub settings to use for sending push notifications.
+        /// 
+        /// Note that hubs on the same namespace share active device limits, so multiple namespaces should be used to increase capacity.
+        /// </summary>
+        public List<NotificationHubSettings> NotificationHubs { get; set; } = new();
     }
 
     public class YubicoSettings

src/Core/Utilities/CoreHelpers.cs

@@ -76,6 +76,39 @@ public static class CoreHelpers
         return new Guid(guidArray);
     }
 
+    internal static DateTime DateFromComb(Guid combGuid)
+    {
+        var guidArray = combGuid.ToByteArray();
+        var daysArray = new byte[4];
+        var msecsArray = new byte[4];
+
+        Array.Copy(guidArray, guidArray.Length - 6, daysArray, 2, 2);
+        Array.Copy(guidArray, guidArray.Length - 4, msecsArray, 0, 4);
+
+        Array.Reverse(daysArray);
+        Array.Reverse(msecsArray);
+
+        var days = BitConverter.ToInt32(daysArray, 0);
+        var msecs = BitConverter.ToInt32(msecsArray, 0);
+
+        var time = TimeSpan.FromDays(days) + TimeSpan.FromMilliseconds(msecs * 3.333333);
+        return new DateTime(_baseDateTicks + time.Ticks, DateTimeKind.Utc);
+    }
+
+    internal static long BinForComb(Guid combGuid, int binCount)
+    {
+        // From System.Web.Util.HashCodeCombiner
+        uint CombineHashCodes(uint h1, byte h2)
+        {
+            return (uint)(((h1 << 5) + h1) ^ h2);
+        }
+        var guidArray = combGuid.ToByteArray();
+        var randomArray = new byte[10];
+        Array.Copy(guidArray, 0, randomArray, 0, 10);
+        var hash = randomArray.Aggregate((uint)randomArray.Length, CombineHashCodes);
+        return hash % binCount;
+    }
+
     public static string CleanCertificateThumbprint(string thumbprint)
     {
         // Clean possible garbage characters from thumbprint copy/paste