nhyatt/bitwarden
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2025-05-20 11:04:31 -05:00
Code

Added GetTakeCount helper to break down large arrays into manageable sizes for documentdb requests (current max limit of 512kb per request).

Browse Source
This commit is contained in:
Kyle Spearrin 2015-12-30 22:38:00 -05:00
parent 4c4f803c1f
commit 79f9f60a78
2 changed files with 42 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/Core/Repositories/DocumentDB/CipherRepository.cs
View File

@ -34,21 +34,21 @@ namespace Bit.Core.Repositories.DocumentDB
public async Task UpdateDirtyCiphersAsync(IEnumerable<dynamic> ciphers)
{
// Make sure we are dealing with cipher types since we accept any via dynamic.
var cleanedCiphers = ciphers.Where(c => c is Cipher);
if(cleanedCiphers.Count() == 0)
{
return;
}
var takeCount = DocumentDBHelpers.GetTakeCount(ciphers, 500);
await DocumentDBHelpers.ExecuteWithRetryAsync(async () =>
{
// Make sure we are dealing with cipher types since we accept any via dynamic.
var cleanedCiphers = ciphers.Where(c => c is Cipher);
if(cleanedCiphers.Count() == 0)
{
return;
}
var userId = ((Cipher)cleanedCiphers.First()).UserId;
StoredProcedureResponse<int> sprocResponse = await Client.ExecuteStoredProcedureAsync<int>(
ResolveSprocIdLink(userId, "updateDirtyCiphers"),
// TODO: Figure out how to better determine the max number of document to send without
// going over 512kb limit for DocumentDB. 50 could still be too large in some cases.
cleanedCiphers.Take(50),
cleanedCiphers.Take(takeCount),
userId);
var replacedCount = sprocResponse.Response;
@ -61,21 +61,21 @@ namespace Bit.Core.Repositories.DocumentDB
public async Task CreateAsync(IEnumerable<dynamic> ciphers)
{
// Make sure we are dealing with cipher types since we accept any via dynamic.
var cleanedCiphers = ciphers.Where(c => c is Cipher);
if(cleanedCiphers.Count() == 0)
{
return;
}
var takeCount = DocumentDBHelpers.GetTakeCount(ciphers, 500);
await DocumentDBHelpers.ExecuteWithRetryAsync(async () =>
{
// Make sure we are dealing with cipher types since we accept any via dynamic.
var cleanedCiphers = ciphers.Where(c => c is Cipher);
if(cleanedCiphers.Count() == 0)
{
return;
}
var userId = ((Cipher)cleanedCiphers.First()).UserId;
StoredProcedureResponse<int> sprocResponse = await Client.ExecuteStoredProcedureAsync<int>(
ResolveSprocIdLink(userId, "bulkCreate"),
// TODO: Figure out how to better determine the max number of document to send without
// going over 512kb limit for DocumentDB. 50 could still be too large in some cases.
cleanedCiphers.Take(50));
cleanedCiphers.Take(takeCount));
var createdCount = sprocResponse.Response;
if(createdCount != cleanedCiphers.Count())

22
src/Core/Repositories/DocumentDB/Utilities/DocumentDBHelpers.cs
View File

@ -1,7 +1,11 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Azure.Documents;
using Microsoft.Azure.Documents.Client;
using Newtonsoft.Json;
namespace Bit.Core.Repositories.DocumentDB.Utilities
{
@ -58,6 +62,24 @@ namespace Bit.Core.Repositories.DocumentDB.Utilities
}
}
public static int GetTakeCount(IEnumerable<object> docs, int maxSizeKb = 500)
{
var takeCount = docs.Count();
while(takeCount > 1)
{
var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(docs.Take(takeCount)));
if((bytes.Length / 1000) <= maxSizeKb)
{
// array is is small enough
break;
}
takeCount = Convert.ToInt32(Math.Ceiling((double)takeCount / 2));
}
return takeCount;
}
private static async Task HandleDocumentClientExceptionAsync(DocumentClientException e, int retryCount, int? retryMax)
{
if(retryMax.HasValue && retryCount >= retryMax.Value)