diff --git a/src/Admin/appsettings.json b/src/Admin/appsettings.json index f889b30e3e..2880c73076 100644 --- a/src/Admin/appsettings.json +++ b/src/Admin/appsettings.json @@ -36,6 +36,10 @@ "events": { "connectionString": "SECRET" }, + "serviceBus": { + "connectionString": "SECRET", + "applicationCacheTopicName": "SECRET" + }, "documentDb": { "uri": "SECRET", "key": "SECRET" diff --git a/src/Api/Startup.cs b/src/Api/Startup.cs index 93f0a187c9..91ff3064a4 100644 --- a/src/Api/Startup.cs +++ b/src/Api/Startup.cs @@ -129,6 +129,11 @@ namespace Bit.Api Jobs.JobsHostedService.AddJobsServices(services); services.AddHostedService(); } + if(CoreHelpers.SettingHasValue(globalSettings.ServiceBus.ConnectionString) && + CoreHelpers.SettingHasValue(globalSettings.ServiceBus.ApplicationCacheTopicName)) + { + services.AddHostedService(); + } } public void Configure( diff --git a/src/Api/appsettings.json b/src/Api/appsettings.json index 3b3c0fc638..e26337a310 100644 --- a/src/Api/appsettings.json +++ b/src/Api/appsettings.json @@ -51,6 +51,10 @@ "connectionString": "SECRET", "hubName": "SECRET" }, + "serviceBus": { + "connectionString": "SECRET", + "applicationCacheTopicName": "SECRET" + }, "yubico": { "clientid": "SECRET", "key": "SECRET" diff --git a/src/Billing/appsettings.json b/src/Billing/appsettings.json index 1945612186..12c20d659f 100644 --- a/src/Billing/appsettings.json +++ b/src/Billing/appsettings.json @@ -36,6 +36,10 @@ "events": { "connectionString": "SECRET" }, + "serviceBus": { + "connectionString": "SECRET", + "applicationCacheTopicName": "SECRET" + }, "documentDb": { "uri": "SECRET", "key": "SECRET" diff --git a/src/Core/Core.csproj b/src/Core/Core.csproj index c3870ac0e6..f1abb50b98 100644 --- a/src/Core/Core.csproj +++ b/src/Core/Core.csproj @@ -33,6 +33,7 @@ + diff --git a/src/Core/Enums/ApplicationCacheMessageType.cs b/src/Core/Enums/ApplicationCacheMessageType.cs new file mode 100644 index 0000000000..b91b079953 --- /dev/null +++ b/src/Core/Enums/ApplicationCacheMessageType.cs @@ -0,0 +1,8 @@ +namespace Bit.Core.Enums +{ + public enum ApplicationCacheMessageType : byte + { + UpsertOrganizationAbility = 0, + DeleteOrganizationAbility = 1 + } +} diff --git a/src/Core/GlobalSettings.cs b/src/Core/GlobalSettings.cs index fc73a73dd0..21f3446505 100644 --- a/src/Core/GlobalSettings.cs +++ b/src/Core/GlobalSettings.cs @@ -22,8 +22,8 @@ namespace Bit.Core public virtual SqlSettings SqlServer { get; set; } = new SqlSettings(); public virtual SqlSettings PostgreSql { get; set; } = new SqlSettings(); public virtual MailSettings Mail { get; set; } = new MailSettings(); - public virtual StorageSettings Storage { get; set; } = new StorageSettings(); - public virtual StorageSettings Events { get; set; } = new StorageSettings(); + public virtual ConnectionStringSettings Storage { get; set; } = new ConnectionStringSettings(); + public virtual ConnectionStringSettings Events { get; set; } = new ConnectionStringSettings(); public virtual NotificationsSettings Notifications { get; set; } = new NotificationsSettings(); public virtual AttachmentSettings Attachment { get; set; } = new AttachmentSettings(); public virtual IdentityServerSettings IdentityServer { get; set; } = new IdentityServerSettings(); @@ -36,6 +36,7 @@ namespace Bit.Core public virtual BraintreeSettings Braintree { get; set; } = new BraintreeSettings(); public virtual BitPaySettings BitPay { get; set; } = new BitPaySettings(); public virtual AmazonSettings Amazon { get; set; } = new AmazonSettings(); + public virtual ServiceBusSettings ServiceBus { get; set; } = new ServiceBusSettings(); public class BaseServiceUriSettings { @@ -77,7 +78,7 @@ namespace Bit.Core } } - public class StorageSettings + public class ConnectionStringSettings { private string _connectionString; @@ -154,7 +155,7 @@ namespace Bit.Core public string Dsn { get; set; } } - public class NotificationsSettings : StorageSettings + public class NotificationsSettings : ConnectionStringSettings { public string AzureSignalRConnectionString { get; set; } } @@ -213,5 +214,11 @@ namespace Bit.Core public string AccessKeySecret { get; set; } public string Region { get; set; } } + + public class ServiceBusSettings : ConnectionStringSettings + { + public string ApplicationCacheTopicName { get; set; } + public string ApplicationCacheSubscriptionName { get; set; } + } } } diff --git a/src/Core/HostedServices/ApplicationCacheHostedService.cs b/src/Core/HostedServices/ApplicationCacheHostedService.cs new file mode 100644 index 0000000000..38136aa345 --- /dev/null +++ b/src/Core/HostedServices/ApplicationCacheHostedService.cs @@ -0,0 +1,109 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Bit.Core.Enums; +using Bit.Core.Repositories; +using Bit.Core.Services; +using Bit.Core.Utilities; +using Microsoft.Azure.ServiceBus; +using Microsoft.Azure.ServiceBus.Management; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Bit.Core.HostedServices +{ + public class ApplicationCacheHostedService : IHostedService, IDisposable + { + private readonly InMemoryServiceBusApplicationCacheService _applicationCacheService; + private readonly IOrganizationRepository _organizationRepository; + protected readonly ILogger _logger; + private readonly SubscriptionClient _subscriptionClient; + private readonly ManagementClient _managementClient; + private readonly string _subName; + private readonly string _topicName; + + public ApplicationCacheHostedService( + IApplicationCacheService applicationCacheService, + IOrganizationRepository organizationRepository, + ILogger logger, + GlobalSettings globalSettings) + { + _topicName = globalSettings.ServiceBus.ApplicationCacheTopicName; + _subName = CoreHelpers.GetApplicationCacheServiceBusSubcriptionName(globalSettings); + _applicationCacheService = applicationCacheService as InMemoryServiceBusApplicationCacheService; + _organizationRepository = organizationRepository; + _logger = logger; + _managementClient = new ManagementClient(globalSettings.ServiceBus.ConnectionString); + _subscriptionClient = new SubscriptionClient(globalSettings.ServiceBus.ConnectionString, + _topicName, _subName); + } + + public virtual async Task StartAsync(CancellationToken cancellationToken) + { + try + { + await _managementClient.CreateSubscriptionAsync(new SubscriptionDescription(_topicName, _subName) + { + DefaultMessageTimeToLive = TimeSpan.FromDays(14), + LockDuration = TimeSpan.FromSeconds(30), + EnableDeadLetteringOnFilterEvaluationExceptions = true, + EnableDeadLetteringOnMessageExpiration = true, + }, new RuleDescription("default", new SqlFilter($"sys.Label != '{_subName}'"))); + } + catch(MessagingEntityAlreadyExistsException) { } + _subscriptionClient.RegisterMessageHandler(ProcessMessageAsync, + new MessageHandlerOptions(ExceptionReceivedHandlerAsync) + { + MaxConcurrentCalls = 2, + AutoComplete = false, + }); + } + + public virtual async Task StopAsync(CancellationToken cancellationToken) + { + await _subscriptionClient.CloseAsync(); + try + { + await _managementClient.DeleteSubscriptionAsync(_topicName, _subName, cancellationToken); + } + catch { } + } + + public virtual void Dispose() + { } + + private async Task ProcessMessageAsync(Message message, CancellationToken cancellationToken) + { + if(message.Label != _subName && _applicationCacheService != null) + { + switch((ApplicationCacheMessageType)message.UserProperties["type"]) + { + case ApplicationCacheMessageType.UpsertOrganizationAbility: + var upsertedOrgId = (Guid)message.UserProperties["id"]; + var upsertedOrg = await _organizationRepository.GetByIdAsync(upsertedOrgId); + if(upsertedOrg != null) + { + await _applicationCacheService.BaseUpsertOrganizationAbilityAsync(upsertedOrg); + } + break; + case ApplicationCacheMessageType.DeleteOrganizationAbility: + await _applicationCacheService.BaseDeleteOrganizationAbilityAsync( + (Guid)message.UserProperties["id"]); + break; + default: + break; + } + } + if(!cancellationToken.IsCancellationRequested) + { + await _subscriptionClient.CompleteAsync(message.SystemProperties.LockToken); + } + } + + private Task ExceptionReceivedHandlerAsync(ExceptionReceivedEventArgs args) + { + _logger.LogError(args.Exception, "Message handler encountered an exception."); + return Task.FromResult(0); + } + } +} diff --git a/src/Core/Services/Implementations/InMemoryApplicationCacheService.cs b/src/Core/Services/Implementations/InMemoryApplicationCacheService.cs index f61246d9fb..7190866423 100644 --- a/src/Core/Services/Implementations/InMemoryApplicationCacheService.cs +++ b/src/Core/Services/Implementations/InMemoryApplicationCacheService.cs @@ -21,13 +21,13 @@ namespace Bit.Core.Services _organizationRepository = organizationRepository; } - public async Task> GetOrganizationAbilitiesAsync() + public virtual async Task> GetOrganizationAbilitiesAsync() { await InitOrganizationAbilitiesAsync(); return _orgAbilities; } - public async Task UpsertOrganizationAbilityAsync(Organization organization) + public virtual async Task UpsertOrganizationAbilityAsync(Organization organization) { await InitOrganizationAbilitiesAsync(); var newAbility = new OrganizationAbility(organization); @@ -42,7 +42,7 @@ namespace Bit.Core.Services } } - public Task DeleteOrganizationAbilityAsync(Guid organizationId) + public virtual Task DeleteOrganizationAbilityAsync(Guid organizationId) { if(_orgAbilities != null && _orgAbilities.ContainsKey(organizationId)) { diff --git a/src/Core/Services/Implementations/InMemoryServiceBusApplicationCacheService.cs b/src/Core/Services/Implementations/InMemoryServiceBusApplicationCacheService.cs new file mode 100644 index 0000000000..ebf1e2f158 --- /dev/null +++ b/src/Core/Services/Implementations/InMemoryServiceBusApplicationCacheService.cs @@ -0,0 +1,66 @@ +using System; +using System.Threading.Tasks; +using Bit.Core.Enums; +using Bit.Core.Models.Table; +using Bit.Core.Repositories; +using Bit.Core.Utilities; +using Microsoft.Azure.ServiceBus; + +namespace Bit.Core.Services +{ + public class InMemoryServiceBusApplicationCacheService : InMemoryApplicationCacheService, IApplicationCacheService + { + private readonly TopicClient _topicClient; + private readonly string _subName; + + public InMemoryServiceBusApplicationCacheService( + IOrganizationRepository organizationRepository, + GlobalSettings globalSettings) + : base(organizationRepository) + { + _subName = CoreHelpers.GetApplicationCacheServiceBusSubcriptionName(globalSettings); + _topicClient = new TopicClient(globalSettings.ServiceBus.ConnectionString, + globalSettings.ServiceBus.ApplicationCacheTopicName); + } + + public override async Task UpsertOrganizationAbilityAsync(Organization organization) + { + await base.UpsertOrganizationAbilityAsync(organization); + var message = new Message + { + Label = _subName, + UserProperties = + { + { "type", (byte)ApplicationCacheMessageType.UpsertOrganizationAbility }, + { "id", organization.Id }, + } + }; + var task = _topicClient.SendAsync(message); + } + + public override async Task DeleteOrganizationAbilityAsync(Guid organizationId) + { + await base.DeleteOrganizationAbilityAsync(organizationId); + var message = new Message + { + Label = _subName, + UserProperties = + { + { "type", (byte)ApplicationCacheMessageType.DeleteOrganizationAbility }, + { "id", organizationId }, + } + }; + var task = _topicClient.SendAsync(message); + } + + public async Task BaseUpsertOrganizationAbilityAsync(Organization organization) + { + await base.UpsertOrganizationAbilityAsync(organization); + } + + public async Task BaseDeleteOrganizationAbilityAsync(Guid organizationId) + { + await base.DeleteOrganizationAbilityAsync(organizationId); + } + } +} diff --git a/src/Core/Utilities/CoreHelpers.cs b/src/Core/Utilities/CoreHelpers.cs index 41791158df..cf05134015 100644 --- a/src/Core/Utilities/CoreHelpers.cs +++ b/src/Core/Utilities/CoreHelpers.cs @@ -501,5 +501,27 @@ namespace Bit.Core.Utilities return !invalid; } + + public static string GetApplicationCacheServiceBusSubcriptionName(GlobalSettings globalSettings) + { + var subName = globalSettings.ServiceBus.ApplicationCacheSubscriptionName; + if(string.IsNullOrWhiteSpace(subName)) + { + var websiteInstanceId = Environment.GetEnvironmentVariable("WEBSITE_INSTANCE_ID"); + if(string.IsNullOrWhiteSpace(websiteInstanceId)) + { + throw new Exception("No service bus subscription name available."); + } + else + { + subName = $"{globalSettings.ProjectName.ToLower()}_{websiteInstanceId}"; + if(subName.Length > 50) + { + subName = subName.Substring(0, 50); + } + } + } + return subName; + } } } diff --git a/src/Core/Utilities/ServiceCollectionExtensions.cs b/src/Core/Utilities/ServiceCollectionExtensions.cs index 0705df6d97..ff5689c073 100644 --- a/src/Core/Utilities/ServiceCollectionExtensions.cs +++ b/src/Core/Utilities/ServiceCollectionExtensions.cs @@ -86,7 +86,16 @@ namespace Bit.Core.Utilities services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + + if(CoreHelpers.SettingHasValue(globalSettings.ServiceBus.ConnectionString) && + CoreHelpers.SettingHasValue(globalSettings.ServiceBus.ApplicationCacheTopicName)) + { + services.AddSingleton(); + } + else + { + services.AddSingleton(); + } if(CoreHelpers.SettingHasValue(globalSettings.Mail.SendGridApiKey)) { diff --git a/src/Events/Startup.cs b/src/Events/Startup.cs index cc4f8f9e20..610a53f895 100644 --- a/src/Events/Startup.cs +++ b/src/Events/Startup.cs @@ -48,7 +48,16 @@ namespace Bit.Events }); // Services - services.AddSingleton(); + var usingServiceBusAppCache = CoreHelpers.SettingHasValue(globalSettings.ServiceBus.ConnectionString) && + CoreHelpers.SettingHasValue(globalSettings.ServiceBus.ApplicationCacheTopicName); + if(usingServiceBusAppCache) + { + services.AddSingleton(); + } + else + { + services.AddSingleton(); + } services.AddScoped(); if(!globalSettings.SelfHosted && CoreHelpers.SettingHasValue(globalSettings.Events.ConnectionString)) { @@ -61,6 +70,11 @@ namespace Bit.Events // Mvc services.AddMvc(); + + if(usingServiceBusAppCache) + { + services.AddHostedService(); + } } public void Configure( diff --git a/src/Identity/Startup.cs b/src/Identity/Startup.cs index d649c52861..c97156eef0 100644 --- a/src/Identity/Startup.cs +++ b/src/Identity/Startup.cs @@ -63,6 +63,12 @@ namespace Bit.Identity // Services services.AddBaseServices(); services.AddDefaultServices(globalSettings); + + if(CoreHelpers.SettingHasValue(globalSettings.ServiceBus.ConnectionString) && + CoreHelpers.SettingHasValue(globalSettings.ServiceBus.ApplicationCacheTopicName)) + { + services.AddHostedService(); + } } public void Configure( diff --git a/src/Identity/appsettings.json b/src/Identity/appsettings.json index 62c887701a..6b856de89a 100644 --- a/src/Identity/appsettings.json +++ b/src/Identity/appsettings.json @@ -47,6 +47,10 @@ "connectionString": "SECRET", "hubName": "SECRET" }, + "serviceBus": { + "connectionString": "SECRET", + "applicationCacheTopicName": "SECRET" + }, "yubico": { "clientid": "SECRET", "key": "SECRET" diff --git a/src/Notifications/appsettings.json b/src/Notifications/appsettings.json index 6a6e6578d4..d8448c69a9 100644 --- a/src/Notifications/appsettings.json +++ b/src/Notifications/appsettings.json @@ -26,6 +26,10 @@ "events": { "connectionString": "SECRET" }, + "serviceBus": { + "connectionString": "SECRET", + "applicationCacheTopicName": "SECRET" + }, "documentDb": { "uri": "SECRET", "key": "SECRET"