SM-365: Add Export & Import Functionality for SM (#2591)

* SM-365: Add Export endpoint

* SM-365: Add SM Import/Export support

* SM-365: Fix DI and add temp NoAccessCheck

* SM-365: Add access checks to import / export

* SM-365: dotnet format

* SM-365: Fix import bugs

* SM-365: Fix import bug with EF & refactor based on PR comments

* SM-365: Update access permissions in export

* SM-365: Address PR comments

* SM-365: Refactor for readability and PR comments

bitwarden_license/src/Commercial.Core/SecretsManager/Commands/Porting/ImportCommand.cs

@@ -0,0 +1,101 @@
+using Bit.Core.SecretsManager.Commands.Porting;
+using Bit.Core.SecretsManager.Commands.Porting.Interfaces;
+using Bit.Core.SecretsManager.Entities;
+using Bit.Core.SecretsManager.Repositories;
+
+namespace Bit.Commercial.Core.SecretsManager.Commands.Porting;
+
+public class ImportCommand : IImportCommand
+{
+    private readonly IProjectRepository _projectRepository;
+    private readonly ISecretRepository _secretRepository;
+
+    public ImportCommand(IProjectRepository projectRepository, ISecretRepository secretRepository)
+    {
+        _projectRepository = projectRepository;
+        _secretRepository = secretRepository;
+    }
+
+    public async Task ImportAsync(Guid organizationId, SMImport import)
+    {
+        var importedProjects = new List<Guid>();
+        var importedSecrets = new List<Guid>();
+
+        try
+        {
+            import = AssignNewIds(import);
+
+            if (import.Projects.Any())
+            {
+                importedProjects = (await _projectRepository.ImportAsync(import.Projects.Select(p => new Project
+                {
+                    Id = p.Id,
+                    OrganizationId = organizationId,
+                    Name = p.Name,
+                }))).Select(p => p.Id).ToList();
+            }
+
+            if (import.Secrets != null && import.Secrets.Any())
+            {
+                importedSecrets = (await _secretRepository.ImportAsync(import.Secrets.Select(s => new Secret
+                {
+                    Id = s.Id,
+                    OrganizationId = organizationId,
+                    Key = s.Key,
+                    Value = s.Value,
+                    Note = s.Note,
+                    Projects = s.ProjectIds?.Select(id => new Project { Id = id }).ToList(),
+                }))).Select(s => s.Id).ToList();
+            }
+        }
+        catch (Exception)
+        {
+            if (importedProjects.Any())
+            {
+                await _projectRepository.DeleteManyByIdAsync(importedProjects);
+            }
+
+            if (importedSecrets.Any())
+            {
+                await _secretRepository.HardDeleteManyByIdAsync(importedSecrets);
+            }
+
+            throw new Exception("Error attempting import");
+        }
+    }
+
+    public SMImport AssignNewIds(SMImport import)
+    {
+        var projects = new Dictionary<Guid, SMImport.InnerProject>();
+        var secrets = new List<SMImport.InnerSecret>();
+
+        if (import.Projects != null && import.Projects.Any())
+        {
+            projects = import.Projects.ToDictionary(
+                p => p.Id,
+                p => new SMImport.InnerProject { Id = Guid.NewGuid(), Name = p.Name }
+            );
+        }
+
+        if (import.Secrets != null && import.Secrets.Any())
+        {
+            foreach (var secret in import.Secrets)
+            {
+                secrets.Add(new SMImport.InnerSecret
+                {
+                    Id = Guid.NewGuid(),
+                    Key = secret.Key,
+                    Value = secret.Value,
+                    Note = secret.Note,
+                    ProjectIds = secret.ProjectIds?.Select(id => projects[id].Id),
+                });
+            }
+        }
+
+        return new SMImport
+        {
+            Projects = projects.Values,
+            Secrets = secrets,
+        };
+    }
+}

bitwarden_license/src/Commercial.Core/SecretsManager/SecretsManagerCollectionExtensions.cs

@@ -1,10 +1,12 @@
 using Bit.Commercial.Core.SecretsManager.Commands.AccessPolicies;
 using Bit.Commercial.Core.SecretsManager.Commands.AccessTokens;
+using Bit.Commercial.Core.SecretsManager.Commands.Porting;
 using Bit.Commercial.Core.SecretsManager.Commands.Projects;
 using Bit.Commercial.Core.SecretsManager.Commands.Secrets;
 using Bit.Commercial.Core.SecretsManager.Commands.ServiceAccounts;
 using Bit.Core.SecretsManager.Commands.AccessPolicies.Interfaces;
 using Bit.Core.SecretsManager.Commands.AccessTokens.Interfaces;
+using Bit.Core.SecretsManager.Commands.Porting.Interfaces;
 using Bit.Core.SecretsManager.Commands.Projects.Interfaces;
 using Bit.Core.SecretsManager.Commands.Secrets.Interfaces;
 using Bit.Core.SecretsManager.Commands.ServiceAccounts.Interfaces;
@@ -28,5 +30,6 @@ public static class SecretsManagerCollectionExtensions
         services.AddScoped<ICreateAccessPoliciesCommand, CreateAccessPoliciesCommand>();
         services.AddScoped<IUpdateAccessPolicyCommand, UpdateAccessPolicyCommand>();
         services.AddScoped<IDeleteAccessPolicyCommand, DeleteAccessPolicyCommand>();
+        services.AddScoped<IImportCommand, ImportCommand>();
     }
 }

bitwarden_license/src/Commercial.Infrastructure.EntityFramework/SecretsManager/Repositories/ProjectRepository.cs

@@ -103,4 +103,14 @@ public class ProjectRepository : Repository<Core.SecretsManager.Entities.Project
 
         return await query.AnyAsync();
     }
+
+    public async Task<IEnumerable<Core.SecretsManager.Entities.Project>> ImportAsync(IEnumerable<Core.SecretsManager.Entities.Project> projects)
+    {
+        using var scope = ServiceScopeFactory.CreateScope();
+        var entities = projects.Select(p => Mapper.Map<Project>(p));
+        var dbContext = GetDatabaseContext(scope);
+        await GetDbSet(dbContext).AddRangeAsync(entities);
+        await dbContext.SaveChangesAsync();
+        return projects;
+    }
 }

bitwarden_license/src/Commercial.Infrastructure.EntityFramework/SecretsManager/Repositories/SecretRepository.cs

@@ -92,7 +92,6 @@ public class SecretRepository : Repository<Core.SecretsManager.Entities.Secret,
 
     public async Task<Core.SecretsManager.Entities.Secret> UpdateAsync(Core.SecretsManager.Entities.Secret secret)
     {
-
         using (var scope = ServiceScopeFactory.CreateScope())
         {
             var dbContext = GetDatabaseContext(scope);
@@ -136,4 +135,60 @@ public class SecretRepository : Repository<Core.SecretsManager.Entities.Secret,
             await dbContext.SaveChangesAsync();
         }
     }
+
+    public async Task HardDeleteManyByIdAsync(IEnumerable<Guid> ids)
+    {
+        using (var scope = ServiceScopeFactory.CreateScope())
+        {
+            var dbContext = GetDatabaseContext(scope);
+            var utcNow = DateTime.UtcNow;
+            var secrets = dbContext.Secret.Where(c => ids.Contains(c.Id));
+            await secrets.ForEachAsync(secret =>
+            {
+                dbContext.Attach(secret);
+                dbContext.Remove(secret);
+            });
+            await dbContext.SaveChangesAsync();
+        }
+    }
+
+    public async Task<IEnumerable<Core.SecretsManager.Entities.Secret>> ImportAsync(IEnumerable<Core.SecretsManager.Entities.Secret> secrets)
+    {
+        try
+        {
+            using (var scope = ServiceScopeFactory.CreateScope())
+            {
+                var dbContext = GetDatabaseContext(scope);
+                var entities = new List<Secret>();
+                var projects = secrets
+                    .SelectMany(s => s.Projects ?? Enumerable.Empty<Core.SecretsManager.Entities.Project>())
+                    .DistinctBy(p => p.Id)
+                    .Select(p => Mapper.Map<Project>(p))
+                    .ToDictionary(p => p.Id, p => p);
+
+                dbContext.AttachRange(projects);
+
+                foreach (var s in secrets)
+                {
+                    var entity = Mapper.Map<Secret>(s);
+
+                    if (s.Projects?.Count > 0)
+                    {
+                        entity.Projects = s.Projects.Select(p => projects[p.Id]).ToList();
+                    }
+
+                    entities.Add(entity);
+                }
+                await GetDbSet(dbContext).AddRangeAsync(entities);
+                await dbContext.SaveChangesAsync();
+            }
+            return secrets;
+        }
+        catch (Exception e)
+        {
+            Console.WriteLine(e);
+        }
+
+        return secrets;
+    }
 }

src/Api/SecretsManager/Controllers/SecretsManagerPortingController.cs

@@ -0,0 +1,66 @@
+using Bit.Api.SecretsManager.Models.Request;
+using Bit.Api.SecretsManager.Models.Response;
+using Bit.Core.Context;
+using Bit.Core.Enums;
+using Bit.Core.Exceptions;
+using Bit.Core.SecretsManager.Commands.Porting.Interfaces;
+using Bit.Core.SecretsManager.Repositories;
+using Bit.Core.Services;
+using Microsoft.AspNetCore.Mvc;
+
+namespace Bit.Api.SecretsManager.Controllers;
+
+[SecretsManager]
+public class SecretsManagerPortingController : Controller
+{
+    private readonly ISecretRepository _secretRepository;
+    private readonly IProjectRepository _projectRepository;
+    private readonly IUserService _userService;
+    private readonly IImportCommand _importCommand;
+    private readonly ICurrentContext _currentContext;
+
+    public SecretsManagerPortingController(ISecretRepository secretRepository, IProjectRepository projectRepository, IUserService userService, IImportCommand importCommand, ICurrentContext currentContext)
+    {
+        _secretRepository = secretRepository;
+        _projectRepository = projectRepository;
+        _userService = userService;
+        _importCommand = importCommand;
+        _currentContext = currentContext;
+    }
+
+    [HttpGet("sm/{organizationId}/export")]
+    public async Task<SMExportResponseModel> Export([FromRoute] Guid organizationId, [FromRoute] string format = "json")
+    {
+        if (!await _currentContext.OrganizationAdmin(organizationId))
+        {
+            throw new UnauthorizedAccessException();
+        }
+
+        var userId = _userService.GetProperUserId(User).Value;
+        var projects = await _projectRepository.GetManyByOrganizationIdAsync(organizationId, userId, AccessClientType.NoAccessCheck);
+        var secrets = await _secretRepository.GetManyByOrganizationIdAsync(organizationId);
+
+        if (projects == null && secrets == null)
+        {
+            throw new NotFoundException();
+        }
+
+        return new SMExportResponseModel(projects, secrets);
+    }
+
+    [HttpPost("sm/{organizationId}/import")]
+    public async Task Import([FromRoute] Guid organizationId, [FromBody] SMImportRequestModel importRequest)
+    {
+        if (!await _currentContext.OrganizationAdmin(organizationId))
+        {
+            throw new UnauthorizedAccessException();
+        }
+
+        if (importRequest.Projects?.Count() > 1000 || importRequest.Secrets?.Count() > 6000)
+        {
+            throw new BadRequestException("You cannot import this much data at once, the limit is 1000 projects and 6000 secrets.");
+        }
+
+        await _importCommand.ImportAsync(organizationId, importRequest.ToSMImport());
+    }
+}

src/Api/SecretsManager/Models/Request/SMImportRequestModel.cs

@@ -0,0 +1,70 @@
+using System.ComponentModel.DataAnnotations;
+using Bit.Core.SecretsManager.Commands.Porting;
+using Bit.Core.Utilities;
+
+namespace Bit.Api.SecretsManager.Models.Request;
+
+public class SMImportRequestModel
+{
+    public IEnumerable<InnerProjectImportRequestModel> Projects { get; set; }
+    public IEnumerable<InnerSecretImportRequestModel> Secrets { get; set; }
+
+    public class InnerProjectImportRequestModel
+    {
+        public InnerProjectImportRequestModel() { }
+
+        [Required]
+        public Guid Id { get; set; }
+
+        [Required]
+        [EncryptedString]
+        [EncryptedStringLength(1000)]
+        public string Name { get; set; }
+    }
+
+    public class InnerSecretImportRequestModel
+    {
+        public InnerSecretImportRequestModel() { }
+
+        [Required]
+        public Guid Id { get; set; }
+
+        [Required]
+        [EncryptedString]
+        [EncryptedStringLength(1000)]
+        public string Key { get; set; }
+
+        [Required]
+        [EncryptedString]
+        [EncryptedStringLength(1000)]
+        public string Value { get; set; }
+
+        [Required]
+        [EncryptedString]
+        [EncryptedStringLength(1000)]
+        public string Note { get; set; }
+
+        [Required]
+        public IEnumerable<Guid> ProjectIds { get; set; }
+    }
+
+    public SMImport ToSMImport()
+    {
+        return new SMImport
+        {
+            Projects = Projects?.Select(p => new SMImport.InnerProject
+            {
+                Id = p.Id,
+                Name = p.Name,
+            }),
+            Secrets = Secrets?.Select(s => new SMImport.InnerSecret
+            {
+                Id = s.Id,
+                Key = s.Key,
+                Value = s.Value,
+                Note = s.Note,
+                ProjectIds = s.ProjectIds,
+            }),
+        };
+    }
+}

src/Api/SecretsManager/Models/Response/SMExportResponseModel.cs

@@ -0,0 +1,46 @@
+using Bit.Core.Models.Api;
+using Bit.Core.SecretsManager.Entities;
+
+namespace Bit.Api.SecretsManager.Models.Response;
+
+public class SMExportResponseModel : ResponseModel
+{
+    public SMExportResponseModel(IEnumerable<Project> projects, IEnumerable<Secret> secrets, string obj = "SecretsManagerExportResponseModel") : base(obj)
+    {
+        Secrets = secrets?.Select(s => new InnerSecretExportResponseModel(s));
+        Projects = projects?.Select(p => new InnerProjectExportResponseModel(p));
+    }
+
+    public IEnumerable<InnerProjectExportResponseModel> Projects { get; set; }
+    public IEnumerable<InnerSecretExportResponseModel> Secrets { get; set; }
+
+    public class InnerProjectExportResponseModel
+    {
+        public InnerProjectExportResponseModel(Project project)
+        {
+            Id = project.Id;
+            Name = project.Name;
+        }
+
+        public Guid Id { get; set; }
+        public string Name { get; set; }
+    }
+
+    public class InnerSecretExportResponseModel
+    {
+        public InnerSecretExportResponseModel(Secret secret)
+        {
+            Id = secret.Id;
+            Key = secret.Key;
+            Value = secret.Value;
+            Note = secret.Note;
+            ProjectIds = secret.Projects?.Select(p => p.Id);
+        }
+
+        public Guid Id { get; set; }
+        public string Key { get; set; }
+        public string Value { get; set; }
+        public string Note { get; set; }
+        public IEnumerable<Guid> ProjectIds { get; set; }
+    }
+}

src/Api/SecretsManager/Models/Response/SMImportResponseModel.cs

@@ -0,0 +1,50 @@
+using Bit.Core.Models.Api;
+using Bit.Core.SecretsManager.Commands.Porting;
+
+namespace Bit.Api.SecretsManager.Models.Response;
+
+public class SMImportResponseModel : ResponseModel
+{
+    public SMImportResponseModel(SMImport import, string obj = "SecretsManagerImportResponseModel") : base(obj)
+    {
+        Projects = import.Projects?.Select(p => new InnerProjectImportResponseModel(p));
+        Secrets = import.Secrets?.Select(s => new InnerSecretImportResponseModel(s));
+    }
+
+    public IEnumerable<InnerProjectImportResponseModel> Projects { get; set; }
+    public IEnumerable<InnerSecretImportResponseModel> Secrets { get; set; }
+
+    public class InnerProjectImportResponseModel
+    {
+        public InnerProjectImportResponseModel() { }
+
+        public InnerProjectImportResponseModel(SMImport.InnerProject project)
+        {
+            Id = project.Id;
+            Name = project.Name;
+        }
+
+        public Guid Id { get; set; }
+        public string Name { get; set; }
+    }
+
+    public class InnerSecretImportResponseModel
+    {
+        public InnerSecretImportResponseModel() { }
+
+        public InnerSecretImportResponseModel(SMImport.InnerSecret secret)
+        {
+            Id = secret.Id;
+            Key = secret.Key;
+            Value = secret.Value;
+            Note = secret.Note;
+            ProjectIds = secret.ProjectIds;
+        }
+
+        public Guid Id { get; set; }
+        public string Key { get; set; }
+        public string Value { get; set; }
+        public string Note { get; set; }
+        public IEnumerable<Guid> ProjectIds { get; set; }
+    }
+}

src/Core/SecretsManager/Commands/Porting/Interfaces/IImportCommand.cs

@@ -0,0 +1,7 @@
+namespace Bit.Core.SecretsManager.Commands.Porting.Interfaces;
+
+public interface IImportCommand
+{
+    Task ImportAsync(Guid organizationId, SMImport import);
+    SMImport AssignNewIds(SMImport import);
+}

src/Core/SecretsManager/Commands/Porting/SMImport.cs

@@ -0,0 +1,41 @@
+namespace Bit.Core.SecretsManager.Commands.Porting;
+
+public class SMImport
+{
+    public IEnumerable<InnerProject> Projects { get; set; }
+    public IEnumerable<InnerSecret> Secrets { get; set; }
+
+    public class InnerProject
+    {
+        public InnerProject() { }
+
+        public InnerProject(Core.SecretsManager.Entities.Project project)
+        {
+            Id = project.Id;
+            Name = project.Name;
+        }
+
+        public Guid Id { get; set; }
+        public string Name { get; set; }
+    }
+
+    public class InnerSecret
+    {
+        public InnerSecret() { }
+
+        public InnerSecret(Core.SecretsManager.Entities.Secret secret)
+        {
+            Id = secret.Id;
+            Key = secret.Key;
+            Value = secret.Value;
+            Note = secret.Note;
+            ProjectIds = secret.Projects != null && secret.Projects.Any() ? secret.Projects.Select(p => p.Id) : null;
+        }
+
+        public Guid Id { get; set; }
+        public string Key { get; set; }
+        public string Value { get; set; }
+        public string Note { get; set; }
+        public IEnumerable<Guid> ProjectIds { get; set; }
+    }
+}

src/Core/SecretsManager/Repositories/IProjectRepository.cs

@@ -11,6 +11,7 @@ public interface IProjectRepository
     Task<Project> CreateAsync(Project project);
     Task ReplaceAsync(Project project);
     Task DeleteManyByIdAsync(IEnumerable<Guid> ids);
+    Task<IEnumerable<Project>> ImportAsync(IEnumerable<Project> projects);
     Task<bool> UserHasReadAccessToProject(Guid id, Guid userId);
     Task<bool> UserHasWriteAccessToProject(Guid id, Guid userId);
 }

src/Core/SecretsManager/Repositories/ISecretRepository.cs

@@ -11,4 +11,6 @@ public interface ISecretRepository
     Task<Secret> CreateAsync(Secret secret);
     Task<Secret> UpdateAsync(Secret secret);
     Task SoftDeleteManyByIdAsync(IEnumerable<Guid> ids);
+    Task HardDeleteManyByIdAsync(IEnumerable<Guid> ids);
+    Task<IEnumerable<Secret>> ImportAsync(IEnumerable<Secret> secrets);
 }