diff --git a/src/Api/Controllers/AccountsController.cs b/src/Api/Controllers/AccountsController.cs index 7c27e5a12d..2a3dfbd045 100644 --- a/src/Api/Controllers/AccountsController.cs +++ b/src/Api/Controllers/AccountsController.cs @@ -11,6 +11,7 @@ using Bit.Core.Domains; using Bit.Core.Enums; using Bit.Core; using System.Security.Claims; +using System.Linq; namespace Bit.Api.Controllers { @@ -19,16 +20,18 @@ namespace Bit.Api.Controllers public class AccountsController : Controller { private readonly IUserService _userService; + private readonly ICipherService _cipherService; private readonly UserManager _userManager; private readonly CurrentContext _currentContext; public AccountsController( - IDataProtectionProvider dataProtectionProvider, IUserService userService, + ICipherService cipherService, UserManager userManager, CurrentContext currentContext) { _userService = userService; + _cipherService = cipherService; _userManager = userManager; _currentContext = currentContext; } @@ -206,5 +209,14 @@ namespace Bit.Api.Controllers var response = new TwoFactorResponseModel(user); return response; } + + [HttpPost("import")] + public async Task PostImport([FromBody]ImportRequestModel model) + { + await _cipherService.ImportCiphersAsync( + model.Folders.Select(f => f.ToFolder(User.GetUserId())).ToList(), + model.Sites.Select(s => s.ToSite(User.GetUserId())).ToList(), + model.SiteRelationships); + } } } diff --git a/src/Api/Controllers/SitesController.cs b/src/Api/Controllers/SitesController.cs index 0dd98eaf6a..f24b5c49bd 100644 --- a/src/Api/Controllers/SitesController.cs +++ b/src/Api/Controllers/SitesController.cs @@ -144,5 +144,4 @@ namespace Bit.Api.Controllers } } } - } diff --git a/src/Api/Models/Request/Accounts/ImportRequestModel.cs b/src/Api/Models/Request/Accounts/ImportRequestModel.cs new file mode 100644 index 0000000000..01c4e81a5d --- /dev/null +++ b/src/Api/Models/Request/Accounts/ImportRequestModel.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; + +namespace Bit.Api.Models +{ + public class ImportRequestModel + { + public FolderRequestModel[] Folders { get; set; } + public SiteRequestModel[] Sites { get; set; } + public KeyValuePair[] SiteRelationships { get; set; } + } +} diff --git a/src/Api/Models/Request/Ciphers/CipherRequestModel.cs b/src/Api/Models/Request/Ciphers/CipherRequestModel.cs index d3c391a75c..0f0d0d2959 100644 --- a/src/Api/Models/Request/Ciphers/CipherRequestModel.cs +++ b/src/Api/Models/Request/Ciphers/CipherRequestModel.cs @@ -71,10 +71,6 @@ namespace Bit.Api.Models { yield return new ValidationResult("Uri is required for a site cypher.", new[] { "Uri" }); } - if(string.IsNullOrWhiteSpace(Username)) - { - yield return new ValidationResult("Username is required for a site cypher.", new[] { "Username" }); - } if(string.IsNullOrWhiteSpace(Password)) { yield return new ValidationResult("Password is required for a site cypher.", new[] { "Password" }); diff --git a/src/Api/Models/Request/Sites/SiteRequestModel.cs b/src/Api/Models/Request/Sites/SiteRequestModel.cs index 84d5994f6b..7bfc2a93e9 100644 --- a/src/Api/Models/Request/Sites/SiteRequestModel.cs +++ b/src/Api/Models/Request/Sites/SiteRequestModel.cs @@ -14,7 +14,6 @@ namespace Bit.Api.Models [Required] [EncryptedString] public string Uri { get; set; } - [Required] [EncryptedString] public string Username { get; set; } [Required] @@ -31,7 +30,7 @@ namespace Bit.Api.Models FolderId = string.IsNullOrWhiteSpace(FolderId) ? null : FolderId, Name = Name, Uri = Uri, - Username = Username, + Username = string.IsNullOrWhiteSpace(Username) ? null : Username, Password = Password, Notes = string.IsNullOrWhiteSpace(Notes) ? null : Notes }; @@ -42,7 +41,7 @@ namespace Bit.Api.Models existingSite.FolderId = string.IsNullOrWhiteSpace(FolderId) ? null : FolderId; existingSite.Name = Name; existingSite.Uri = Uri; - existingSite.Username = Username; + existingSite.Username = string.IsNullOrWhiteSpace(Username) ? null : Username; existingSite.Password = Password; existingSite.Notes = string.IsNullOrWhiteSpace(Notes) ? null : Notes; diff --git a/src/Api/Startup.cs b/src/Api/Startup.cs index 0ab6b08587..e1d68cfa69 100644 --- a/src/Api/Startup.cs +++ b/src/Api/Startup.cs @@ -53,7 +53,7 @@ namespace Bit.Api services.AddSingleton(s => globalSettings); // Repositories - var documentDBClient = DocumentClientHelpers.InitClient(globalSettings.DocumentDB); + var documentDBClient = DocumentDBHelpers.InitClient(globalSettings.DocumentDB); services.AddSingleton(s => new Repos.UserRepository(documentDBClient, globalSettings.DocumentDB.DatabaseId)); services.AddSingleton(s => new Repos.SiteRepository(documentDBClient, globalSettings.DocumentDB.DatabaseId)); services.AddSingleton(s => new Repos.FolderRepository(documentDBClient, globalSettings.DocumentDB.DatabaseId)); @@ -116,6 +116,7 @@ namespace Bit.Api // Services services.AddSingleton(); + services.AddSingleton(); services.AddScoped(); // Cors diff --git a/src/Core/Repositories/DocumentDB/CipherRepository.cs b/src/Core/Repositories/DocumentDB/CipherRepository.cs index 698e8c899f..3b127cac2a 100644 --- a/src/Core/Repositories/DocumentDB/CipherRepository.cs +++ b/src/Core/Repositories/DocumentDB/CipherRepository.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Azure.Documents.Client; using Bit.Core.Domains; +using Bit.Core.Repositories.DocumentDB.Utilities; namespace Bit.Core.Repositories.DocumentDB { @@ -15,26 +16,53 @@ namespace Bit.Core.Repositories.DocumentDB public async Task UpdateDirtyCiphersAsync(IEnumerable ciphers) { - // Make sure we are dealing with cipher types since we accept any via dynamic. - var cleanedCiphers = ciphers.Where(c => c is Cipher); - if(cleanedCiphers.Count() == 0) + await DocumentDBHelpers.QueryWithRetryAsync(async () => { - return; - } + // Make sure we are dealing with cipher types since we accept any via dynamic. + var cleanedCiphers = ciphers.Where(c => c is Cipher); + if(cleanedCiphers.Count() == 0) + { + return; + } - var userId = ((Cipher)cleanedCiphers.First()).UserId; - StoredProcedureResponse sprocResponse = await Client.ExecuteStoredProcedureAsync( - ResolveSprocIdLink(userId, "bulkUpdateDirtyCiphers"), - // Do sets of 50. Recursion will handle the rest below. - cleanedCiphers.Take(50), - userId, - Cipher.TypeValue); + var userId = ((Cipher)cleanedCiphers.First()).UserId; + StoredProcedureResponse sprocResponse = await Client.ExecuteStoredProcedureAsync( + ResolveSprocIdLink(userId, "bulkUpdateDirtyCiphers"), + // Do sets of 50. Recursion will handle the rest below. + cleanedCiphers.Take(50), + userId); - var replacedCount = sprocResponse.Response; - if(replacedCount != cleanedCiphers.Count()) + var replacedCount = sprocResponse.Response; + if(replacedCount != cleanedCiphers.Count()) + { + await UpdateDirtyCiphersAsync(cleanedCiphers.Skip(replacedCount)); + } + }); + } + + public async Task CreateAsync(IEnumerable ciphers) + { + await DocumentDBHelpers.QueryWithRetryAsync(async () => { - await UpdateDirtyCiphersAsync(cleanedCiphers.Skip(replacedCount)); - } + // Make sure we are dealing with cipher types since we accept any via dynamic. + var cleanedCiphers = ciphers.Where(c => c is Cipher); + if(cleanedCiphers.Count() == 0) + { + return; + } + + var userId = ((Cipher)cleanedCiphers.First()).UserId; + StoredProcedureResponse sprocResponse = await Client.ExecuteStoredProcedureAsync( + ResolveSprocIdLink(userId, "bulkCreate"), + // Do sets of 50. Recursion will handle the rest below. + cleanedCiphers.Take(50)); + + var createdCount = sprocResponse.Response; + if(createdCount != cleanedCiphers.Count()) + { + await CreateAsync(cleanedCiphers.Skip(createdCount)); + } + }); } } } diff --git a/src/Core/Repositories/DocumentDB/Stored Procedures/bulkCreate.js b/src/Core/Repositories/DocumentDB/Stored Procedures/bulkCreate.js new file mode 100644 index 0000000000..d947a154d8 --- /dev/null +++ b/src/Core/Repositories/DocumentDB/Stored Procedures/bulkCreate.js @@ -0,0 +1,60 @@ +/** +* This script called as stored procedure to import lots of documents in one batch. +* The script sets response body to the number of docs imported and is called multiple times +* by the client until total number of docs desired by the client is imported. +* @param {Object[]} docs - Array of documents to import. +*/ + +function bulkCreate(docs) { + var collection = getContext().getCollection(); + var collectionLink = collection.getSelfLink(); + + // The count of imported docs, also used as current doc index. + var count = 0; + + // Validate input. + if (!docs) throw new Error('The array is undefined or null.'); + + var docsLength = docs.length; + if (docsLength == 0) { + getContext().getResponse().setBody(0); + return; + } + + // Call the CRUD API to create a document. + tryCreate(docs[count], callback); + + // Note that there are 2 exit conditions: + // 1) The createDocument request was not accepted. + // In this case the callback will not be called, we just call setBody and we are done. + // 2) The callback was called docs.length times. + // In this case all documents were created and we don't need to call tryCreate anymore. Just call setBody and we are done. + function tryCreate(doc, callback) { + var isAccepted = collection.createDocument(collectionLink, doc, callback); + + // If the request was accepted, callback will be called. + // Otherwise report current count back to the client, + // which will call the script again with remaining set of docs. + // This condition will happen when this stored procedure has been running too long + // and is about to get cancelled by the server. This will allow the calling client + // to resume this batch from the point we got to before isAccepted was set to false + if (!isAccepted) getContext().getResponse().setBody(count); + } + + // This is called when collection.createDocument is done and the document has been persisted. + function callback(err, doc, options) { + if (err) throw err; + + // One more document has been inserted, increment the count. + count++; + + if (count >= docsLength) { + // If we have created all documents, we are done. Just set the response. + getContext().getResponse().setBody(count); + } + else { + // Create next document. + tryCreate(docs[count], callback); + } + } +} diff --git a/src/Core/Repositories/DocumentDB/Stored Procedures/bulkUpdateDirtyCiphers.js b/src/Core/Repositories/DocumentDB/Stored Procedures/bulkUpdateDirtyCiphers.js index dd75296e8d..f688ed20bc 100644 --- a/src/Core/Repositories/DocumentDB/Stored Procedures/bulkUpdateDirtyCiphers.js +++ b/src/Core/Repositories/DocumentDB/Stored Procedures/bulkUpdateDirtyCiphers.js @@ -10,7 +10,7 @@ function bulkUpdateDirtyCiphers(ciphers, userId) { // Validate input. if (!ciphers) { - throw new Error("The ciphers array is undefined or null."); + throw new Error('The ciphers array is undefined or null.'); } var ciphersLength = ciphers.length; diff --git a/src/Core/Repositories/DocumentDB/UserRepository.cs b/src/Core/Repositories/DocumentDB/UserRepository.cs index 86c4ade2d0..418c12a107 100644 --- a/src/Core/Repositories/DocumentDB/UserRepository.cs +++ b/src/Core/Repositories/DocumentDB/UserRepository.cs @@ -1,6 +1,8 @@ using System; using System.Linq; using System.Threading.Tasks; +using Bit.Core.Repositories.DocumentDB.Utilities; +using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; namespace Bit.Core.Repositories.DocumentDB @@ -26,7 +28,10 @@ namespace Bit.Core.Repositories.DocumentDB public async Task ReplaceAndDirtyCiphersAsync(Domains.User user) { - await Client.ExecuteStoredProcedureAsync(ResolveSprocIdLink(user, "replaceUserAndDirtyCiphers"), user); + await DocumentDBHelpers.QueryWithRetryAsync(async () => + { + await Client.ExecuteStoredProcedureAsync(ResolveSprocIdLink(user, "replaceUserAndDirtyCiphers"), user); + }); } public override async Task DeleteByIdAsync(string id) diff --git a/src/Core/Repositories/DocumentDB/Utilities/DocumentClientHelpers.cs b/src/Core/Repositories/DocumentDB/Utilities/DocumentDBHelpers.cs similarity index 55% rename from src/Core/Repositories/DocumentDB/Utilities/DocumentClientHelpers.cs rename to src/Core/Repositories/DocumentDB/Utilities/DocumentDBHelpers.cs index 250ef139e0..0158137625 100644 --- a/src/Core/Repositories/DocumentDB/Utilities/DocumentClientHelpers.cs +++ b/src/Core/Repositories/DocumentDB/Utilities/DocumentDBHelpers.cs @@ -1,9 +1,11 @@ using System; +using System.Threading.Tasks; +using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; namespace Bit.Core.Repositories.DocumentDB.Utilities { - public class DocumentClientHelpers + public class DocumentDBHelpers { public static DocumentClient InitClient(GlobalSettings.DocumentDBSettings settings) { @@ -29,6 +31,43 @@ namespace Bit.Core.Repositories.DocumentDB.Utilities return client; } + public static async Task QueryWithRetryAsync(Func func) + { + var queryComplete = false; + while(!queryComplete) + { + try + { + await func(); + queryComplete = true; + } + catch(DocumentClientException e) + { + await HandleDocumentClientExceptionAsync(e); + } + catch(AggregateException e) + { + var docEx = e.InnerException as DocumentClientException; + if(docEx != null) + { + await HandleDocumentClientExceptionAsync(docEx); + } + } + } + } + + private static async Task HandleDocumentClientExceptionAsync(DocumentClientException e) + { + var statusCode = (int)e.StatusCode; + if(statusCode == 429 || statusCode == 503) + { + await Task.Delay(e.RetryAfter); + } + else { + throw e; + } + } + private static Func GetPartitionKeyExtractor() { return doc => diff --git a/src/Core/Repositories/ICipherRepository.cs b/src/Core/Repositories/ICipherRepository.cs index 36246177a9..b924199f7a 100644 --- a/src/Core/Repositories/ICipherRepository.cs +++ b/src/Core/Repositories/ICipherRepository.cs @@ -6,5 +6,6 @@ namespace Bit.Core.Repositories public interface ICipherRepository { Task UpdateDirtyCiphersAsync(IEnumerable ciphers); + Task CreateAsync(IEnumerable ciphers); } } diff --git a/src/Core/Services/CipherService.cs b/src/Core/Services/CipherService.cs new file mode 100644 index 0000000000..954692bfb9 --- /dev/null +++ b/src/Core/Services/CipherService.cs @@ -0,0 +1,54 @@ +using System; +using System.Linq; +using System.Collections.Generic; +using System.Threading.Tasks; +using Bit.Core.Domains; +using Bit.Core.Repositories; + +namespace Bit.Core.Services +{ + public class CipherService : ICipherService + { + private readonly IFolderRepository _folderRepository; + private readonly ICipherRepository _cipherRepository; + + public CipherService( + IFolderRepository folderRepository, + ICipherRepository cipherRepository) + { + _folderRepository = folderRepository; + _cipherRepository = cipherRepository; + } + + public async Task ImportCiphersAsync( + List folders, + List sites, + IEnumerable> siteRelationships) + { + // create all the folders + var folderTasks = new List(); + foreach(var folder in folders) + { + folderTasks.Add(_folderRepository.CreateAsync(folder)); + } + await Task.WhenAll(folderTasks); + + // associate the newly created folders to the sites + foreach(var relationship in siteRelationships) + { + var site = sites.ElementAtOrDefault(relationship.Key); + var folder = folders.ElementAtOrDefault(relationship.Value); + + if(site == null || folder == null) + { + continue; + } + + site.FolderId = folder.Id; + } + + // create all the sites + await _cipherRepository.CreateAsync(sites); + } + } +} diff --git a/src/Core/Services/ICipherService.cs b/src/Core/Services/ICipherService.cs new file mode 100644 index 0000000000..32d7f0c48a --- /dev/null +++ b/src/Core/Services/ICipherService.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Bit.Core.Domains; + +namespace Bit.Core.Services +{ + public interface ICipherService + { + Task ImportCiphersAsync(List folders, List sites, IEnumerable> siteRelationships); + } +}