From 597fa01344acd1658f772aa0f574c8d594d6f78a Mon Sep 17 00:00:00 2001 From: Kyle Spearrin Date: Fri, 2 Apr 2021 11:14:21 -0400 Subject: [PATCH] job to delete trashed ciphers nightly (#1243) * job to delete trashed items nightly * remove script from migration project file * admin setting for controlling trash deleting dates --- src/Admin/AdminSettings.cs | 1 + src/Admin/Jobs/DeleteCiphersJob.cs | 40 +++++++++++++++++++ src/Admin/Jobs/JobsHostedService.cs | 9 ++++- src/Core/Repositories/ICipherRepository.cs | 1 + .../SqlServer/CipherRepository.cs | 12 ++++++ .../Cipher_DeleteDeleted.sql | 19 +++++++++ src/Sql/dbo/Tables/Cipher.sql | 5 +++ .../2021-03-26_00_CipherDeletedIndex.sql | 36 +++++++++++++++++ 8 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 src/Admin/Jobs/DeleteCiphersJob.cs create mode 100644 src/Sql/dbo/Stored Procedures/Cipher_DeleteDeleted.sql create mode 100644 util/Migrator/DbScripts/2021-03-26_00_CipherDeletedIndex.sql diff --git a/src/Admin/AdminSettings.cs b/src/Admin/AdminSettings.cs index 2eecc3a490..64de4f0837 100644 --- a/src/Admin/AdminSettings.cs +++ b/src/Admin/AdminSettings.cs @@ -4,6 +4,7 @@ { public virtual string Admins { get; set; } public virtual CloudflareSettings Cloudflare { get; set; } + public int? DeleteTrashDaysAgo { get; set; } public class CloudflareSettings { diff --git a/src/Admin/Jobs/DeleteCiphersJob.cs b/src/Admin/Jobs/DeleteCiphersJob.cs new file mode 100644 index 0000000000..62b1f13052 --- /dev/null +++ b/src/Admin/Jobs/DeleteCiphersJob.cs @@ -0,0 +1,40 @@ +using System; +using System.Threading.Tasks; +using Bit.Core; +using Bit.Core.Jobs; +using Bit.Core.Repositories; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Quartz; + +namespace Bit.Admin.Jobs +{ + public class DeleteCiphersJob : BaseJob + { + private readonly ICipherRepository _cipherRepository; + private readonly AdminSettings _adminSettings; + + public DeleteCiphersJob( + ICipherRepository cipherRepository, + IOptions adminSettings, + ILogger logger) + : base(logger) + { + _cipherRepository = cipherRepository; + _adminSettings = adminSettings?.Value; + } + + protected async override Task ExecuteJobAsync(IJobExecutionContext context) + { + _logger.LogInformation(Constants.BypassFiltersEventId, "Execute job task: DeleteDeletedAsync"); + var deleteDate = DateTime.UtcNow.AddDays(-30); + var daysAgoSetting = (_adminSettings?.DeleteTrashDaysAgo).GetValueOrDefault(); + if (daysAgoSetting > 0) + { + deleteDate = DateTime.UtcNow.AddDays(-1 * daysAgoSetting); + } + await _cipherRepository.DeleteDeletedAsync(deleteDate); + _logger.LogInformation(Constants.BypassFiltersEventId, "Finished job task: DeleteDeletedAsync"); + } + } +} diff --git a/src/Admin/Jobs/JobsHostedService.cs b/src/Admin/Jobs/JobsHostedService.cs index 06842eaca1..8bef67ff48 100644 --- a/src/Admin/Jobs/JobsHostedService.cs +++ b/src/Admin/Jobs/JobsHostedService.cs @@ -55,13 +55,19 @@ namespace Bit.Admin.Jobs .StartNow() .WithCronSchedule("0 0 0 ? * SUN", x => x.InTimeZone(timeZone)) .Build(); + var everyDayAtMidnightUtc = TriggerBuilder.Create() + .WithIdentity("EveryDayAtMidnightUtc") + .StartNow() + .WithCronSchedule("0 0 0 * * ?") + .Build(); var jobs = new List> { new Tuple(typeof(DeleteSendsJob), everyFiveMinutesTrigger), new Tuple(typeof(DatabaseExpiredGrantsJob), everyFridayAt10pmTrigger), new Tuple(typeof(DatabaseUpdateStatisticsJob), everySaturdayAtMidnightTrigger), - new Tuple(typeof(DatabaseRebuildlIndexesJob), everySundayAtMidnightTrigger) + new Tuple(typeof(DatabaseRebuildlIndexesJob), everySundayAtMidnightTrigger), + new Tuple(typeof(DeleteCiphersJob), everyDayAtMidnightUtc) }; if (!_globalSettings.SelfHosted) @@ -83,6 +89,7 @@ namespace Bit.Admin.Jobs services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); } } } diff --git a/src/Core/Repositories/ICipherRepository.cs b/src/Core/Repositories/ICipherRepository.cs index 3f84ae6cfb..aac2cae0dc 100644 --- a/src/Core/Repositories/ICipherRepository.cs +++ b/src/Core/Repositories/ICipherRepository.cs @@ -36,5 +36,6 @@ namespace Bit.Core.Repositories Task SoftDeleteAsync(IEnumerable ids, Guid userId); Task SoftDeleteByIdsOrganizationIdAsync(IEnumerable ids, Guid organizationId); Task RestoreAsync(IEnumerable ids, Guid userId); + Task DeleteDeletedAsync(DateTime deletedDateBefore); } } diff --git a/src/Core/Repositories/SqlServer/CipherRepository.cs b/src/Core/Repositories/SqlServer/CipherRepository.cs index f7123a8363..772d4d898a 100644 --- a/src/Core/Repositories/SqlServer/CipherRepository.cs +++ b/src/Core/Repositories/SqlServer/CipherRepository.cs @@ -624,6 +624,18 @@ namespace Bit.Core.Repositories.SqlServer } } + public async Task DeleteDeletedAsync(DateTime deletedDateBefore) + { + using (var connection = new SqlConnection(ConnectionString)) + { + await connection.ExecuteAsync( + $"[{Schema}].[Cipher_DeleteDeleted]", + new { DeletedDateBefore = deletedDateBefore }, + commandType: CommandType.StoredProcedure, + commandTimeout: 43200); + } + } + private DataTable BuildCiphersTable(SqlBulkCopy bulkCopy, IEnumerable ciphers) { var c = ciphers.FirstOrDefault(); diff --git a/src/Sql/dbo/Stored Procedures/Cipher_DeleteDeleted.sql b/src/Sql/dbo/Stored Procedures/Cipher_DeleteDeleted.sql new file mode 100644 index 0000000000..7cae97f1ba --- /dev/null +++ b/src/Sql/dbo/Stored Procedures/Cipher_DeleteDeleted.sql @@ -0,0 +1,19 @@ +CREATE PROCEDURE [dbo].[Cipher_DeleteDeleted] + @DeletedDateBefore DATETIME2 (7) +AS +BEGIN + SET NOCOUNT ON + + DECLARE @BatchSize INT = 100 + + WHILE @BatchSize > 0 + BEGIN + DELETE TOP(@BatchSize) + FROM + [dbo].[Cipher] + WHERE + [DeletedDate] < @DeletedDateBefore + + SET @BatchSize = @@ROWCOUNT + END +END \ No newline at end of file diff --git a/src/Sql/dbo/Tables/Cipher.sql b/src/Sql/dbo/Tables/Cipher.sql index 81c3c8adbd..f10be7bb7b 100644 --- a/src/Sql/dbo/Tables/Cipher.sql +++ b/src/Sql/dbo/Tables/Cipher.sql @@ -26,3 +26,8 @@ GO CREATE NONCLUSTERED INDEX [IX_Cipher_OrganizationId] ON [dbo].[Cipher]([OrganizationId] ASC); + +GO +CREATE NONCLUSTERED INDEX [IX_Cipher_DeletedDate] + ON [dbo].[Cipher]([DeletedDate] ASC); + diff --git a/util/Migrator/DbScripts/2021-03-26_00_CipherDeletedIndex.sql b/util/Migrator/DbScripts/2021-03-26_00_CipherDeletedIndex.sql new file mode 100644 index 0000000000..e0d65cd93c --- /dev/null +++ b/util/Migrator/DbScripts/2021-03-26_00_CipherDeletedIndex.sql @@ -0,0 +1,36 @@ +IF OBJECT_ID('[dbo].[Cipher_DeleteDeleted]') IS NOT NULL +BEGIN + DROP PROCEDURE [dbo].[Cipher_DeleteDeleted] +END +GO + +CREATE PROCEDURE [dbo].[Cipher_DeleteDeleted] + @DeletedDateBefore DATETIME2 (7) +AS +BEGIN + SET NOCOUNT ON + + DECLARE @BatchSize INT = 100 + + WHILE @BatchSize > 0 + BEGIN + DELETE TOP(@BatchSize) + FROM + [dbo].[Cipher] + WHERE + [DeletedDate] < @DeletedDateBefore + + SET @BatchSize = @@ROWCOUNT + END +END +GO + +IF NOT EXISTS ( + SELECT * FROM sys.indexes WHERE [Name]='IX_Cipher_DeletedDate' + AND object_id = OBJECT_ID('[dbo].[Cipher]') +) +BEGIN + CREATE NONCLUSTERED INDEX [IX_Cipher_DeletedDate] + ON [dbo].[Cipher]([DeletedDate] ASC) +END +GO