nhyatt/bitwarden
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2025-07-01 08:02:49 -05:00
Code Activity

Fix queue message encoding for Azure (UTF-16 in XML) (#1439)

Browse Source 
* Revert "Encode into b64 to avoid illegal xml encoding when sending to Azure (#1425)"

This reverts commit 2c9a5bb4ab.

* Azure queue to use base64 encoding universally

* Ensure byte size calc is using encoded byte count

* Remove message text extension from blockIP svc

* Remove unused using on blockIp hosted service
This commit is contained in:
Chad Scharf
2021-07-07 10:49:59 -04:00
committed by GitHub
parent 908e1504af
commit 898c7baf89
6 changed files with 78 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
src/Core/Services/Implementations/AzureQueueService.cs
View File

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Azure.Storage.Queues;
using Bit.Core.Utilities;
@ -17,13 +18,14 @@ namespace Bit.Core.Services
{
_queueClient = queueClient;
_jsonSettings = jsonSettings;
if (!_jsonSettings.Converters.Any(c => c.GetType() == typeof(EncodedStringConverter)))
{
_jsonSettings.Converters.Add(new EncodedStringConverter());
}
}
public async Task CreateAsync(T message) => await CreateManyAsync(new[] { message });
public async Task CreateAsync(T message)
{
var json = JsonConvert.SerializeObject(message, _jsonSettings);
var base64 = CoreHelpers.Base64EncodeString(json);
await _queueClient.SendMessageAsync(base64);
}
public async Task CreateManyAsync(IEnumerable<T> messages)
{
@ -32,36 +34,62 @@ namespace Bit.Core.Services
return;
}
foreach (var json in SerializeMany(messages))
if (!messages.Skip(1).Any())
{
await CreateAsync(messages.First());
return;
}
foreach (var json in SerializeMany(messages, _jsonSettings))
{
await _queueClient.SendMessageAsync(json);
}
}
private IEnumerable<string> SerializeMany(IEnumerable<T> messages)
protected IEnumerable<string> SerializeMany(IEnumerable<T> messages, JsonSerializerSettings jsonSettings)
{
string SerializeMessage(T message) => JsonConvert.SerializeObject(message, _jsonSettings);
// Calculate Base-64 encoded text with padding
int getBase64Size(int byteCount) => ((4 * byteCount / 3) + 3) & ~3;
var messagesLists = new List<List<T>> { new List<T>() };
var strings = new List<string>();
var ListMessageLength = 2; // to account for json array brackets "[]"
foreach (var (message, jsonEvent) in messages.Select(m => (m, SerializeMessage(m))))
var messagesList = new List<string>();
var messagesListSize = 0;
int calculateByteSize(int totalSize, int toAdd) =>
// Calculate the total length this would be w/ "[]" and commas
getBase64Size(totalSize + toAdd + messagesList.Count + 2);
// Format the final array string, i.e. [{...},{...}]
string getArrayString()
{
var messageLength = jsonEvent.Length + 1; // To account for json array comma
if (ListMessageLength + messageLength > _queueClient.MessageMaxBytes)
if (messagesList.Count == 1)
{
messagesLists.Add(new List<T> { message });
ListMessageLength = 2 + messageLength;
}
else
{
messagesLists.Last().Add(message);
ListMessageLength += messageLength;
return CoreHelpers.Base64EncodeString(messagesList[0]);
}
return CoreHelpers.Base64EncodeString(
string.Concat("[", string.Join(',', messagesList), "]"));
}
var serializedMessages = messages.Select(message =>
JsonConvert.SerializeObject(message, jsonSettings));
foreach (var message in serializedMessages)
{
var messageSize = Encoding.UTF8.GetByteCount(message);
if (calculateByteSize(messagesListSize, messageSize) > _queueClient.MessageMaxBytes)
{
yield return getArrayString();
messagesListSize = 0;
messagesList.Clear();
}
messagesList.Add(message);
messagesListSize += messageSize;
}
if (messagesList.Any())
{
yield return getArrayString();
}
return messagesLists.Select(l => JsonConvert.SerializeObject(l, _jsonSettings));
}
}
}

20
src/Core/Utilities/CoreHelpers.cs
View File

@ -24,6 +24,9 @@ using Bit.Core.Models.Table;
using IdentityModel;
using System.Text.Json;
using Bit.Core.Enums.Provider;
using Azure.Storage.Queues;
using Azure.Storage.Queues.Models;
using System.Threading;
namespace Bit.Core.Utilities
{
@ -901,5 +904,22 @@ namespace Bit.Core.Utilities
list.Add(item);
return list;
}
public static string DecodeMessageText(this QueueMessage message)
{
var text = message?.MessageText;
if (string.IsNullOrWhiteSpace(text))
{
return text;
}
try
{
return Base64DecodeString(text);
}
catch
{
return text;
}
}
}
}

35
src/Core/Utilities/EncodedStringConverter.cs
View File

@ -1,35 +0,0 @@
using System;
using Newtonsoft.Json;
namespace Bit.Core.Utilities
{
public class EncodedStringConverter : JsonConverter
{
public override bool CanConvert(Type objectType) => objectType == typeof(string);
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
if (reader.TokenType == JsonToken.Null)
{
return existingValue;
}
var value = reader.Value as string;
return System.Net.WebUtility.HtmlDecode(value);
}
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
if (value == null)
{
if (serializer.NullValueHandling == NullValueHandling.Include)
{
writer.WriteNull();
}
return;
}
writer.WriteValue(System.Net.WebUtility.HtmlEncode((string)value));
}
}
}