nhyatt/bitwarden
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2025-05-20 19:14:32 -05:00
Code

Added GetTakeCount helper to break down large arrays into manageable sizes for documentdb requests (current max limit of 512kb per request).

Browse Source
This commit is contained in:
Kyle Spearrin 2015-12-30 22:38:00 -05:00
parent 4c4f803c1f
commit 79f9f60a78
2 changed files with 42 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/Core/Repositories/DocumentDB/CipherRepository.cs
View File

@ -33,8 +33,6 @@ namespace Bit.Core.Repositories.DocumentDB
} }
public async Task UpdateDirtyCiphersAsync(IEnumerable<dynamic> ciphers) public async Task UpdateDirtyCiphersAsync(IEnumerable<dynamic> ciphers)
{
await DocumentDBHelpers.ExecuteWithRetryAsync(async () =>
{ {
// Make sure we are dealing with cipher types since we accept any via dynamic. // Make sure we are dealing with cipher types since we accept any via dynamic.
var cleanedCiphers = ciphers.Where(c => c is Cipher); var cleanedCiphers = ciphers.Where(c => c is Cipher);
@ -43,12 +41,14 @@ namespace Bit.Core.Repositories.DocumentDB
return; return;
} }
var takeCount = DocumentDBHelpers.GetTakeCount(ciphers, 500);
await DocumentDBHelpers.ExecuteWithRetryAsync(async () =>
{
var userId = ((Cipher)cleanedCiphers.First()).UserId; var userId = ((Cipher)cleanedCiphers.First()).UserId;
StoredProcedureResponse<int> sprocResponse = await Client.ExecuteStoredProcedureAsync<int>( StoredProcedureResponse<int> sprocResponse = await Client.ExecuteStoredProcedureAsync<int>(
ResolveSprocIdLink(userId, "updateDirtyCiphers"), ResolveSprocIdLink(userId, "updateDirtyCiphers"),
// TODO: Figure out how to better determine the max number of document to send without cleanedCiphers.Take(takeCount),
// going over 512kb limit for DocumentDB. 50 could still be too large in some cases.
cleanedCiphers.Take(50),
userId); userId);
var replacedCount = sprocResponse.Response; var replacedCount = sprocResponse.Response;
@ -60,8 +60,6 @@ namespace Bit.Core.Repositories.DocumentDB
} }
public async Task CreateAsync(IEnumerable<dynamic> ciphers) public async Task CreateAsync(IEnumerable<dynamic> ciphers)
{
await DocumentDBHelpers.ExecuteWithRetryAsync(async () =>
{ {
// Make sure we are dealing with cipher types since we accept any via dynamic. // Make sure we are dealing with cipher types since we accept any via dynamic.
var cleanedCiphers = ciphers.Where(c => c is Cipher); var cleanedCiphers = ciphers.Where(c => c is Cipher);
@ -70,12 +68,14 @@ namespace Bit.Core.Repositories.DocumentDB
return; return;
} }
var takeCount = DocumentDBHelpers.GetTakeCount(ciphers, 500);
await DocumentDBHelpers.ExecuteWithRetryAsync(async () =>
{
var userId = ((Cipher)cleanedCiphers.First()).UserId; var userId = ((Cipher)cleanedCiphers.First()).UserId;
StoredProcedureResponse<int> sprocResponse = await Client.ExecuteStoredProcedureAsync<int>( StoredProcedureResponse<int> sprocResponse = await Client.ExecuteStoredProcedureAsync<int>(
ResolveSprocIdLink(userId, "bulkCreate"), ResolveSprocIdLink(userId, "bulkCreate"),
// TODO: Figure out how to better determine the max number of document to send without cleanedCiphers.Take(takeCount));
// going over 512kb limit for DocumentDB. 50 could still be too large in some cases.
cleanedCiphers.Take(50));
var createdCount = sprocResponse.Response; var createdCount = sprocResponse.Response;
if(createdCount != cleanedCiphers.Count()) if(createdCount != cleanedCiphers.Count())

22
src/Core/Repositories/DocumentDB/Utilities/DocumentDBHelpers.cs
View File

@ -1,7 +1,11 @@
using System; using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Azure.Documents; using Microsoft.Azure.Documents;
using Microsoft.Azure.Documents.Client; using Microsoft.Azure.Documents.Client;
using Newtonsoft.Json;
namespace Bit.Core.Repositories.DocumentDB.Utilities namespace Bit.Core.Repositories.DocumentDB.Utilities
{ {
@ -58,6 +62,24 @@ namespace Bit.Core.Repositories.DocumentDB.Utilities
} }
} }
public static int GetTakeCount(IEnumerable<object> docs, int maxSizeKb = 500)
{
var takeCount = docs.Count();
while(takeCount > 1)
{
var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(docs.Take(takeCount)));
if((bytes.Length / 1000) <= maxSizeKb)
{
// array is is small enough
break;
}
takeCount = Convert.ToInt32(Math.Ceiling((double)takeCount / 2));
}
return takeCount;
}
private static async Task HandleDocumentClientExceptionAsync(DocumentClientException e, int retryCount, int? retryMax) private static async Task HandleDocumentClientExceptionAsync(DocumentClientException e, int retryCount, int? retryMax)
{ {
if(retryMax.HasValue && retryCount >= retryMax.Value) if(retryMax.HasValue && retryCount >= retryMax.Value)