job to delete trashed ciphers nightly (#1243)

* job to delete trashed items nightly

* remove script from migration project file

* admin setting for controlling trash deleting dates

src/Admin/AdminSettings.cs

@@ -4,6 +4,7 @@
     {
         public virtual string Admins { get; set; }
         public virtual CloudflareSettings Cloudflare { get; set; }
+        public int? DeleteTrashDaysAgo { get; set; }
 
         public class CloudflareSettings
         {

src/Admin/Jobs/DeleteCiphersJob.cs

@@ -0,0 +1,40 @@
+using System;
+using System.Threading.Tasks;
+using Bit.Core;
+using Bit.Core.Jobs;
+using Bit.Core.Repositories;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Quartz;
+
+namespace Bit.Admin.Jobs
+{
+    public class DeleteCiphersJob : BaseJob
+    {
+        private readonly ICipherRepository _cipherRepository;
+        private readonly AdminSettings _adminSettings;
+
+        public DeleteCiphersJob(
+            ICipherRepository cipherRepository,
+            IOptions<AdminSettings> adminSettings,
+            ILogger<DeleteCiphersJob> logger)
+            : base(logger)
+        {
+            _cipherRepository = cipherRepository;
+            _adminSettings = adminSettings?.Value;
+        }
+
+        protected async override Task ExecuteJobAsync(IJobExecutionContext context)
+        {
+            _logger.LogInformation(Constants.BypassFiltersEventId, "Execute job task: DeleteDeletedAsync");
+            var deleteDate = DateTime.UtcNow.AddDays(-30);
+            var daysAgoSetting = (_adminSettings?.DeleteTrashDaysAgo).GetValueOrDefault();
+            if (daysAgoSetting > 0)
+            {
+                deleteDate = DateTime.UtcNow.AddDays(-1 * daysAgoSetting);
+            }
+            await _cipherRepository.DeleteDeletedAsync(deleteDate);
+            _logger.LogInformation(Constants.BypassFiltersEventId, "Finished job task: DeleteDeletedAsync");
+        }
+    }
+}

src/Admin/Jobs/JobsHostedService.cs

@@ -55,13 +55,19 @@ namespace Bit.Admin.Jobs
                 .StartNow()
                 .WithCronSchedule("0 0 0 ? * SUN", x => x.InTimeZone(timeZone))
                 .Build();
+            var everyDayAtMidnightUtc = TriggerBuilder.Create()
+                .WithIdentity("EveryDayAtMidnightUtc")
+                .StartNow()
+                .WithCronSchedule("0 0 0 * * ?")
+                .Build();
 
             var jobs = new List<Tuple<Type, ITrigger>>
             {
                 new Tuple<Type, ITrigger>(typeof(DeleteSendsJob), everyFiveMinutesTrigger),
                 new Tuple<Type, ITrigger>(typeof(DatabaseExpiredGrantsJob), everyFridayAt10pmTrigger),
                 new Tuple<Type, ITrigger>(typeof(DatabaseUpdateStatisticsJob), everySaturdayAtMidnightTrigger),
-                new Tuple<Type, ITrigger>(typeof(DatabaseRebuildlIndexesJob), everySundayAtMidnightTrigger)
+                new Tuple<Type, ITrigger>(typeof(DatabaseRebuildlIndexesJob), everySundayAtMidnightTrigger),
+                new Tuple<Type, ITrigger>(typeof(DeleteCiphersJob), everyDayAtMidnightUtc)
             };
 
             if (!_globalSettings.SelfHosted)
@@ -83,6 +89,7 @@ namespace Bit.Admin.Jobs
             services.AddTransient<DatabaseRebuildlIndexesJob>();
             services.AddTransient<DatabaseExpiredGrantsJob>();
             services.AddTransient<DeleteSendsJob>();
+            services.AddTransient<DeleteCiphersJob>();
         }
     }
 }

src/Core/Repositories/ICipherRepository.cs

@@ -36,5 +36,6 @@ namespace Bit.Core.Repositories
         Task SoftDeleteAsync(IEnumerable<Guid> ids, Guid userId);
         Task SoftDeleteByIdsOrganizationIdAsync(IEnumerable<Guid> ids, Guid organizationId);
         Task<DateTime> RestoreAsync(IEnumerable<Guid> ids, Guid userId);
+        Task DeleteDeletedAsync(DateTime deletedDateBefore);
     }
 }

src/Core/Repositories/SqlServer/CipherRepository.cs

@@ -624,6 +624,18 @@ namespace Bit.Core.Repositories.SqlServer
             }
         }
 
+        public async Task DeleteDeletedAsync(DateTime deletedDateBefore)
+        {
+            using (var connection = new SqlConnection(ConnectionString))
+            {
+                await connection.ExecuteAsync(
+                    $"[{Schema}].[Cipher_DeleteDeleted]",
+                    new { DeletedDateBefore = deletedDateBefore },
+                    commandType: CommandType.StoredProcedure,
+                    commandTimeout: 43200);
+            }
+        }
+
         private DataTable BuildCiphersTable(SqlBulkCopy bulkCopy, IEnumerable<Cipher> ciphers)
         {
             var c = ciphers.FirstOrDefault();

src/Sql/dbo/Stored Procedures/Cipher_DeleteDeleted.sql

@@ -0,0 +1,19 @@
+CREATE PROCEDURE [dbo].[Cipher_DeleteDeleted]
+    @DeletedDateBefore DATETIME2 (7)
+AS
+BEGIN
+    SET NOCOUNT ON
+
+    DECLARE @BatchSize INT = 100
+
+    WHILE @BatchSize > 0
+    BEGIN
+        DELETE TOP(@BatchSize)
+        FROM
+            [dbo].[Cipher]
+        WHERE
+            [DeletedDate] < @DeletedDateBefore
+
+        SET @BatchSize = @@ROWCOUNT
+    END
+END

src/Sql/dbo/Tables/Cipher.sql

@@ -26,3 +26,8 @@ GO
 CREATE NONCLUSTERED INDEX [IX_Cipher_OrganizationId]
     ON [dbo].[Cipher]([OrganizationId] ASC);
 
+
+GO
+CREATE NONCLUSTERED INDEX [IX_Cipher_DeletedDate]
+    ON [dbo].[Cipher]([DeletedDate] ASC);
+

util/Migrator/DbScripts/2021-03-26_00_CipherDeletedIndex.sql

@@ -0,0 +1,36 @@
+IF OBJECT_ID('[dbo].[Cipher_DeleteDeleted]') IS NOT NULL
+BEGIN
+    DROP PROCEDURE [dbo].[Cipher_DeleteDeleted]
+END
+GO
+
+CREATE PROCEDURE [dbo].[Cipher_DeleteDeleted]
+    @DeletedDateBefore DATETIME2 (7)
+AS
+BEGIN
+    SET NOCOUNT ON
+
+    DECLARE @BatchSize INT = 100
+
+    WHILE @BatchSize > 0
+    BEGIN
+        DELETE TOP(@BatchSize)
+        FROM
+            [dbo].[Cipher]
+        WHERE
+            [DeletedDate] < @DeletedDateBefore
+
+        SET @BatchSize = @@ROWCOUNT
+    END
+END
+GO
+
+IF NOT EXISTS (
+    SELECT * FROM sys.indexes  WHERE [Name]='IX_Cipher_DeletedDate'
+    AND object_id = OBJECT_ID('[dbo].[Cipher]')
+)
+BEGIN
+    CREATE NONCLUSTERED INDEX [IX_Cipher_DeletedDate]
+        ON [dbo].[Cipher]([DeletedDate] ASC)
+END
+GO