[SM-755] Add EF Secrets Manager cleanup on deletes (#3228)

* Add Delete Cascade for EF AccessPolicy table

* Add AP removal on EF user deletion

* Remove SM entities on EF org delete

src/Infrastructure.EntityFramework/Repositories/OrganizationRepository.cs

@@ -144,27 +144,42 @@ public class OrganizationRepository : Repository<Core.Entities.Organization, Org
             var dbContext = GetDatabaseContext(scope);
             await dbContext.UserBumpAccountRevisionDateByOrganizationIdAsync(organization.Id);
             var deleteCiphersTransaction = await dbContext.Database.BeginTransactionAsync();
-            dbContext.Ciphers.RemoveRange(
+            await dbContext.Ciphers.Where(c => c.UserId == null && c.OrganizationId == organization.Id)
-                dbContext.Ciphers.Where(c => c.UserId == null && c.OrganizationId == organization.Id));
+                .ExecuteDeleteAsync();
             await deleteCiphersTransaction.CommitAsync();
+
             var organizationDeleteTransaction = await dbContext.Database.BeginTransactionAsync();
-            dbContext.SsoUsers.RemoveRange(dbContext.SsoUsers.Where(su => su.OrganizationId == organization.Id));
+            await dbContext.SsoUsers.Where(su => su.OrganizationId == organization.Id)
-            dbContext.SsoConfigs.RemoveRange(dbContext.SsoConfigs.Where(sc => sc.OrganizationId == organization.Id));
+                .ExecuteDeleteAsync();
-            var collectionUsers = from cu in dbContext.CollectionUsers
+            await dbContext.SsoConfigs.Where(sc => sc.OrganizationId == organization.Id)
-                                  join ou in dbContext.OrganizationUsers on cu.OrganizationUserId equals ou.Id
+                .ExecuteDeleteAsync();
-                                  where ou.OrganizationId == organization.Id
+            await dbContext.CollectionUsers.Where(cu => cu.OrganizationUser.OrganizationId == organization.Id)
-                                  select cu;
+                .ExecuteDeleteAsync();
-            dbContext.CollectionUsers.RemoveRange(collectionUsers);
+            await dbContext.UserProjectAccessPolicy.Where(ap => ap.OrganizationUser.OrganizationId == organization.Id)
-            dbContext.OrganizationUsers.RemoveRange(
+                .ExecuteDeleteAsync();
-                dbContext.OrganizationUsers.Where(ou => ou.OrganizationId == organization.Id));
+            await dbContext.UserServiceAccountAccessPolicy.Where(ap => ap.OrganizationUser.OrganizationId == organization.Id)
-            dbContext.ProviderOrganizations.RemoveRange(
+                .ExecuteDeleteAsync();
-                dbContext.ProviderOrganizations.Where(po => po.OrganizationId == organization.Id));
+            await dbContext.OrganizationUsers.Where(ou => ou.OrganizationId == organization.Id)
+                .ExecuteDeleteAsync();
+            await dbContext.ProviderOrganizations.Where(po => po.OrganizationId == organization.Id)
+                .ExecuteDeleteAsync();
+
+            await dbContext.GroupServiceAccountAccessPolicy.Where(ap => ap.GrantedServiceAccount.OrganizationId == organization.Id)
+                .ExecuteDeleteAsync();
+            await dbContext.Project.Where(p => p.OrganizationId == organization.Id)
+                .ExecuteDeleteAsync();
+            await dbContext.Secret.Where(s => s.OrganizationId == organization.Id)
+                .ExecuteDeleteAsync();
+            await dbContext.ApiKeys.Where(ak => ak.ServiceAccount.OrganizationId == organization.Id)
+                .ExecuteDeleteAsync();
+            await dbContext.ServiceAccount.Where(sa => sa.OrganizationId == organization.Id)
+                .ExecuteDeleteAsync();
 
             // The below section are 3 SPROCS in SQL Server but are only called by here
-            dbContext.OrganizationApiKeys.RemoveRange(
+            await dbContext.OrganizationApiKeys.Where(oa => oa.OrganizationId == organization.Id)
-                dbContext.OrganizationApiKeys.Where(oa => oa.OrganizationId == organization.Id));
+                .ExecuteDeleteAsync();
-            dbContext.OrganizationConnections.RemoveRange(
+            await dbContext.OrganizationConnections.Where(oc => oc.OrganizationId == organization.Id)
-                dbContext.OrganizationConnections.Where(oc => oc.OrganizationId == organization.Id));
+                .ExecuteDeleteAsync();
             var sponsoringOrgs = await dbContext.OrganizationSponsorships
                 .Where(os => os.SponsoringOrganizationId == organization.Id)
                 .ToListAsync();

src/Infrastructure.EntityFramework/Repositories/OrganizationUserRepository.cs

@@ -93,6 +93,11 @@ public class OrganizationUserRepository : Repository<Core.Entities.OrganizationU
                 .Where(gu => gu.OrganizationUserId == organizationUserId);
             dbContext.GroupUsers.RemoveRange(groupUsers);
 
+            dbContext.UserProjectAccessPolicy.RemoveRange(
+                dbContext.UserProjectAccessPolicy.Where(ap => ap.OrganizationUserId == organizationUserId));
+            dbContext.UserServiceAccountAccessPolicy.RemoveRange(
+                dbContext.UserServiceAccountAccessPolicy.Where(ap => ap.OrganizationUserId == organizationUserId));
+
             var orgSponsorships = await dbContext.OrganizationSponsorships
                 .Where(os => os.SponsoringOrganizationUserId == organizationUserId)
                 .ToListAsync();

src/Infrastructure.EntityFramework/Repositories/UserRepository.cs

@@ -167,6 +167,10 @@ public class UserRepository : Repository<Core.Entities.User, User, Guid>, IUserR
                              where ou.UserId == user.Id
                              select gu;
             dbContext.GroupUsers.RemoveRange(groupUsers);
+            dbContext.UserProjectAccessPolicy.RemoveRange(
+                dbContext.UserProjectAccessPolicy.Where(ap => ap.OrganizationUser.UserId == user.Id));
+            dbContext.UserServiceAccountAccessPolicy.RemoveRange(
+                dbContext.UserServiceAccountAccessPolicy.Where(ap => ap.OrganizationUser.UserId == user.Id));
             dbContext.OrganizationUsers.RemoveRange(dbContext.OrganizationUsers.Where(ou => ou.UserId == user.Id));
             dbContext.ProviderUsers.RemoveRange(dbContext.ProviderUsers.Where(pu => pu.UserId == user.Id));
             dbContext.SsoUsers.RemoveRange(dbContext.SsoUsers.Where(su => su.UserId == user.Id));

src/Infrastructure.EntityFramework/SecretsManager/Configurations/AccessPolicyEntityTypeConfiguration.cs

@@ -39,6 +39,12 @@ public class UserProjectAccessPolicyEntityTypeConfiguration : IEntityTypeConfigu
         builder
             .Property(e => e.GrantedProjectId)
             .HasColumnName(nameof(UserProjectAccessPolicy.GrantedProjectId));
+
+        builder
+            .HasOne(e => e.GrantedProject)
+            .WithMany(e => e.UserAccessPolicies)
+            .HasForeignKey(nameof(UserProjectAccessPolicy.GrantedProjectId))
+            .OnDelete(DeleteBehavior.Cascade);
     }
 }
 
@@ -67,6 +73,18 @@ public class GroupProjectAccessPolicyEntityTypeConfiguration : IEntityTypeConfig
         builder
             .Property(e => e.GrantedProjectId)
             .HasColumnName(nameof(GroupProjectAccessPolicy.GrantedProjectId));
+
+        builder
+            .HasOne(e => e.GrantedProject)
+            .WithMany(e => e.GroupAccessPolicies)
+            .HasForeignKey(nameof(GroupProjectAccessPolicy.GrantedProjectId))
+            .OnDelete(DeleteBehavior.Cascade);
+
+        builder
+            .HasOne(e => e.Group)
+            .WithMany()
+            .HasForeignKey(nameof(GroupProjectAccessPolicy.GroupId))
+            .OnDelete(DeleteBehavior.Cascade);
     }
 }
 
@@ -81,6 +99,12 @@ public class GroupServiceAccountAccessPolicyEntityTypeConfiguration : IEntityTyp
         builder
             .Property(e => e.GrantedServiceAccountId)
             .HasColumnName(nameof(GroupServiceAccountAccessPolicy.GrantedServiceAccountId));
+
+        builder
+            .HasOne(e => e.Group)
+            .WithMany()
+            .HasForeignKey(nameof(GroupProjectAccessPolicy.GroupId))
+            .OnDelete(DeleteBehavior.Cascade);
     }
 }
 
@@ -95,5 +119,11 @@ public class ServiceAccountProjectAccessPolicyEntityTypeConfiguration : IEntityT
         builder
             .Property(e => e.GrantedProjectId)
             .HasColumnName(nameof(ServiceAccountProjectAccessPolicy.GrantedProjectId));
+
+        builder
+            .HasOne(e => e.GrantedProject)
+            .WithMany(e => e.ServiceAccountAccessPolicies)
+            .HasForeignKey(nameof(ServiceAccountProjectAccessPolicy.GrantedProjectId))
+            .OnDelete(DeleteBehavior.Cascade);
     }
 }

util/MySqlMigrations/Migrations/20230824202452_ACCESSPOLICY_ADD_DELETE_CASCADE.cs

@@ -0,0 +1,63 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Bit.MySqlMigrations.Migrations;
+
+/// <inheritdoc />
+public partial class ACCESSPOLICY_ADD_DELETE_CASCADE : Migration
+{
+    /// <inheritdoc />
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.DropForeignKey(
+            name: "FK_AccessPolicy_Group_GroupId",
+            table: "AccessPolicy");
+
+        migrationBuilder.DropForeignKey(
+            name: "FK_AccessPolicy_Project_GrantedProjectId",
+            table: "AccessPolicy");
+
+        migrationBuilder.AddForeignKey(
+            name: "FK_AccessPolicy_Group_GroupId",
+            table: "AccessPolicy",
+            column: "GroupId",
+            principalTable: "Group",
+            principalColumn: "Id",
+            onDelete: ReferentialAction.Cascade);
+
+        migrationBuilder.AddForeignKey(
+            name: "FK_AccessPolicy_Project_GrantedProjectId",
+            table: "AccessPolicy",
+            column: "GrantedProjectId",
+            principalTable: "Project",
+            principalColumn: "Id",
+            onDelete: ReferentialAction.Cascade);
+    }
+
+    /// <inheritdoc />
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.DropForeignKey(
+            name: "FK_AccessPolicy_Group_GroupId",
+            table: "AccessPolicy");
+
+        migrationBuilder.DropForeignKey(
+            name: "FK_AccessPolicy_Project_GrantedProjectId",
+            table: "AccessPolicy");
+
+        migrationBuilder.AddForeignKey(
+            name: "FK_AccessPolicy_Group_GroupId",
+            table: "AccessPolicy",
+            column: "GroupId",
+            principalTable: "Group",
+            principalColumn: "Id");
+
+        migrationBuilder.AddForeignKey(
+            name: "FK_AccessPolicy_Project_GrantedProjectId",
+            table: "AccessPolicy",
+            column: "GrantedProjectId",
+            principalTable: "Project",
+            principalColumn: "Id");
+    }
+}

util/MySqlMigrations/Migrations/DatabaseContextModelSnapshot.cs

@@ -2072,11 +2072,13 @@ namespace Bit.MySqlMigrations.Migrations
                 {
                     b.HasOne("Bit.Infrastructure.EntityFramework.SecretsManager.Models.Project", "GrantedProject")
                         .WithMany("GroupAccessPolicies")
-                        .HasForeignKey("GrantedProjectId");
+                        .HasForeignKey("GrantedProjectId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.HasOne("Bit.Infrastructure.EntityFramework.Models.Group", "Group")
                         .WithMany()
-                        .HasForeignKey("GroupId");
+                        .HasForeignKey("GroupId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.Navigation("GrantedProject");
 
@@ -2091,7 +2093,8 @@ namespace Bit.MySqlMigrations.Migrations
 
                     b.HasOne("Bit.Infrastructure.EntityFramework.Models.Group", "Group")
                         .WithMany()
-                        .HasForeignKey("GroupId");
+                        .HasForeignKey("GroupId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.Navigation("GrantedServiceAccount");
 
@@ -2102,7 +2105,8 @@ namespace Bit.MySqlMigrations.Migrations
                 {
                     b.HasOne("Bit.Infrastructure.EntityFramework.SecretsManager.Models.Project", "GrantedProject")
                         .WithMany("ServiceAccountAccessPolicies")
-                        .HasForeignKey("GrantedProjectId");
+                        .HasForeignKey("GrantedProjectId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.HasOne("Bit.Infrastructure.EntityFramework.SecretsManager.Models.ServiceAccount", "ServiceAccount")
                         .WithMany()
@@ -2117,7 +2121,8 @@ namespace Bit.MySqlMigrations.Migrations
                 {
                     b.HasOne("Bit.Infrastructure.EntityFramework.SecretsManager.Models.Project", "GrantedProject")
                         .WithMany("UserAccessPolicies")
-                        .HasForeignKey("GrantedProjectId");
+                        .HasForeignKey("GrantedProjectId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.HasOne("Bit.Infrastructure.EntityFramework.Models.OrganizationUser", "OrganizationUser")
                         .WithMany()

util/PostgresMigrations/Migrations/20230824202447_ACCESSPOLICY_ADD_DELETE_CASCADE.cs

@@ -0,0 +1,63 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Bit.PostgresMigrations.Migrations;
+
+/// <inheritdoc />
+public partial class ACCESSPOLICY_ADD_DELETE_CASCADE : Migration
+{
+    /// <inheritdoc />
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.DropForeignKey(
+            name: "FK_AccessPolicy_Group_GroupId",
+            table: "AccessPolicy");
+
+        migrationBuilder.DropForeignKey(
+            name: "FK_AccessPolicy_Project_GrantedProjectId",
+            table: "AccessPolicy");
+
+        migrationBuilder.AddForeignKey(
+            name: "FK_AccessPolicy_Group_GroupId",
+            table: "AccessPolicy",
+            column: "GroupId",
+            principalTable: "Group",
+            principalColumn: "Id",
+            onDelete: ReferentialAction.Cascade);
+
+        migrationBuilder.AddForeignKey(
+            name: "FK_AccessPolicy_Project_GrantedProjectId",
+            table: "AccessPolicy",
+            column: "GrantedProjectId",
+            principalTable: "Project",
+            principalColumn: "Id",
+            onDelete: ReferentialAction.Cascade);
+    }
+
+    /// <inheritdoc />
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.DropForeignKey(
+            name: "FK_AccessPolicy_Group_GroupId",
+            table: "AccessPolicy");
+
+        migrationBuilder.DropForeignKey(
+            name: "FK_AccessPolicy_Project_GrantedProjectId",
+            table: "AccessPolicy");
+
+        migrationBuilder.AddForeignKey(
+            name: "FK_AccessPolicy_Group_GroupId",
+            table: "AccessPolicy",
+            column: "GroupId",
+            principalTable: "Group",
+            principalColumn: "Id");
+
+        migrationBuilder.AddForeignKey(
+            name: "FK_AccessPolicy_Project_GrantedProjectId",
+            table: "AccessPolicy",
+            column: "GrantedProjectId",
+            principalTable: "Project",
+            principalColumn: "Id");
+    }
+}

util/PostgresMigrations/Migrations/DatabaseContextModelSnapshot.cs

@@ -2083,11 +2083,13 @@ namespace Bit.PostgresMigrations.Migrations
                 {
                     b.HasOne("Bit.Infrastructure.EntityFramework.SecretsManager.Models.Project", "GrantedProject")
                         .WithMany("GroupAccessPolicies")
-                        .HasForeignKey("GrantedProjectId");
+                        .HasForeignKey("GrantedProjectId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.HasOne("Bit.Infrastructure.EntityFramework.Models.Group", "Group")
                         .WithMany()
-                        .HasForeignKey("GroupId");
+                        .HasForeignKey("GroupId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.Navigation("GrantedProject");
 
@@ -2102,7 +2104,8 @@ namespace Bit.PostgresMigrations.Migrations
 
                     b.HasOne("Bit.Infrastructure.EntityFramework.Models.Group", "Group")
                         .WithMany()
-                        .HasForeignKey("GroupId");
+                        .HasForeignKey("GroupId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.Navigation("GrantedServiceAccount");
 
@@ -2113,7 +2116,8 @@ namespace Bit.PostgresMigrations.Migrations
                 {
                     b.HasOne("Bit.Infrastructure.EntityFramework.SecretsManager.Models.Project", "GrantedProject")
                         .WithMany("ServiceAccountAccessPolicies")
-                        .HasForeignKey("GrantedProjectId");
+                        .HasForeignKey("GrantedProjectId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.HasOne("Bit.Infrastructure.EntityFramework.SecretsManager.Models.ServiceAccount", "ServiceAccount")
                         .WithMany()
@@ -2128,7 +2132,8 @@ namespace Bit.PostgresMigrations.Migrations
                 {
                     b.HasOne("Bit.Infrastructure.EntityFramework.SecretsManager.Models.Project", "GrantedProject")
                         .WithMany("UserAccessPolicies")
-                        .HasForeignKey("GrantedProjectId");
+                        .HasForeignKey("GrantedProjectId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.HasOne("Bit.Infrastructure.EntityFramework.Models.OrganizationUser", "OrganizationUser")
                         .WithMany()

util/SqliteMigrations/Migrations/20230824202443_ACCESSPOLICY_ADD_DELETE_CASCADE.cs

@@ -0,0 +1,63 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Bit.SqliteMigrations.Migrations;
+
+/// <inheritdoc />
+public partial class ACCESSPOLICY_ADD_DELETE_CASCADE : Migration
+{
+    /// <inheritdoc />
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.DropForeignKey(
+            name: "FK_AccessPolicy_Group_GroupId",
+            table: "AccessPolicy");
+
+        migrationBuilder.DropForeignKey(
+            name: "FK_AccessPolicy_Project_GrantedProjectId",
+            table: "AccessPolicy");
+
+        migrationBuilder.AddForeignKey(
+            name: "FK_AccessPolicy_Group_GroupId",
+            table: "AccessPolicy",
+            column: "GroupId",
+            principalTable: "Group",
+            principalColumn: "Id",
+            onDelete: ReferentialAction.Cascade);
+
+        migrationBuilder.AddForeignKey(
+            name: "FK_AccessPolicy_Project_GrantedProjectId",
+            table: "AccessPolicy",
+            column: "GrantedProjectId",
+            principalTable: "Project",
+            principalColumn: "Id",
+            onDelete: ReferentialAction.Cascade);
+    }
+
+    /// <inheritdoc />
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.DropForeignKey(
+            name: "FK_AccessPolicy_Group_GroupId",
+            table: "AccessPolicy");
+
+        migrationBuilder.DropForeignKey(
+            name: "FK_AccessPolicy_Project_GrantedProjectId",
+            table: "AccessPolicy");
+
+        migrationBuilder.AddForeignKey(
+            name: "FK_AccessPolicy_Group_GroupId",
+            table: "AccessPolicy",
+            column: "GroupId",
+            principalTable: "Group",
+            principalColumn: "Id");
+
+        migrationBuilder.AddForeignKey(
+            name: "FK_AccessPolicy_Project_GrantedProjectId",
+            table: "AccessPolicy",
+            column: "GrantedProjectId",
+            principalTable: "Project",
+            principalColumn: "Id");
+    }
+}

util/SqliteMigrations/Migrations/DatabaseContextModelSnapshot.cs

@@ -2070,11 +2070,13 @@ namespace Bit.SqliteMigrations.Migrations
                 {
                     b.HasOne("Bit.Infrastructure.EntityFramework.SecretsManager.Models.Project", "GrantedProject")
                         .WithMany("GroupAccessPolicies")
-                        .HasForeignKey("GrantedProjectId");
+                        .HasForeignKey("GrantedProjectId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.HasOne("Bit.Infrastructure.EntityFramework.Models.Group", "Group")
                         .WithMany()
-                        .HasForeignKey("GroupId");
+                        .HasForeignKey("GroupId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.Navigation("GrantedProject");
 
@@ -2089,7 +2091,8 @@ namespace Bit.SqliteMigrations.Migrations
 
                     b.HasOne("Bit.Infrastructure.EntityFramework.Models.Group", "Group")
                         .WithMany()
-                        .HasForeignKey("GroupId");
+                        .HasForeignKey("GroupId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.Navigation("GrantedServiceAccount");
 
@@ -2100,7 +2103,8 @@ namespace Bit.SqliteMigrations.Migrations
                 {
                     b.HasOne("Bit.Infrastructure.EntityFramework.SecretsManager.Models.Project", "GrantedProject")
                         .WithMany("ServiceAccountAccessPolicies")
-                        .HasForeignKey("GrantedProjectId");
+                        .HasForeignKey("GrantedProjectId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.HasOne("Bit.Infrastructure.EntityFramework.SecretsManager.Models.ServiceAccount", "ServiceAccount")
                         .WithMany()
@@ -2115,7 +2119,8 @@ namespace Bit.SqliteMigrations.Migrations
                 {
                     b.HasOne("Bit.Infrastructure.EntityFramework.SecretsManager.Models.Project", "GrantedProject")
                         .WithMany("UserAccessPolicies")
-                        .HasForeignKey("GrantedProjectId");
+                        .HasForeignKey("GrantedProjectId")
+                        .OnDelete(DeleteBehavior.Cascade);
 
                     b.HasOne("Bit.Infrastructure.EntityFramework.Models.OrganizationUser", "OrganizationUser")
                         .WithMany()