From 2c9a5bb4ab58ab68192b2fb94ff459b7326a99b1 Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Fri, 2 Jul 2021 17:11:33 -0400 Subject: [PATCH] Encode into b64 to avoid illegal xml encoding when sending to Azure (#1425) * Encode into b64 to avoid illegal xml encoding when sending to Azure * Revert "Encode into b64 to avoid illegal xml encoding when sending to Azure" This reverts commit d50de941da656ac7b1622da5e68a0e98169d690a. * HtmlEncode strings if they use multi-byte characters * Add serializer to event processor * Rename to used class * Formatting * PR feedback --- .../AzureQueueMailHostedService.cs | 9 ++++- .../Implementations/AzureQueueService.cs | 29 +++++++-------- src/Core/Utilities/EncodedStringConverter.cs | 35 +++++++++++++++++++ .../AzureQueueHostedService.cs | 9 ++++- 4 files changed, 63 insertions(+), 19 deletions(-) create mode 100644 src/Core/Utilities/EncodedStringConverter.cs diff --git a/src/Admin/HostedServices/AzureQueueMailHostedService.cs b/src/Admin/HostedServices/AzureQueueMailHostedService.cs index 790643f3cc..67e8623d73 100644 --- a/src/Admin/HostedServices/AzureQueueMailHostedService.cs +++ b/src/Admin/HostedServices/AzureQueueMailHostedService.cs @@ -12,11 +12,13 @@ using Azure.Storage.Queues.Models; using System.Linq; using System.Collections.Generic; using Newtonsoft.Json.Linq; +using Bit.Core.Utilities; namespace Bit.Admin.HostedServices { public class AzureQueueMailHostedService : IHostedService { + private readonly JsonSerializer _jsonSerializer; private readonly ILogger _logger; private readonly GlobalSettings _globalSettings; private readonly IMailService _mailService; @@ -33,6 +35,11 @@ namespace Bit.Admin.HostedServices _logger = logger; _mailService = mailService; _globalSettings = globalSettings; + + _jsonSerializer = JsonSerializer.Create(new JsonSerializerSettings + { + Converters = new[] { new EncodedStringConverter() }, + }); } public Task StartAsync(CancellationToken cancellationToken) @@ -72,7 +79,7 @@ namespace Bit.Admin.HostedServices var token = JToken.Parse(message.MessageText); if (token is JArray) { - foreach (var mailQueueMessage in token.ToObject>()) + foreach (var mailQueueMessage in token.ToObject>(_jsonSerializer)) { await _mailService.SendEnqueuedMailMessageAsync(mailQueueMessage); } diff --git a/src/Core/Services/Implementations/AzureQueueService.cs b/src/Core/Services/Implementations/AzureQueueService.cs index faccc123ae..bb5d89595e 100644 --- a/src/Core/Services/Implementations/AzureQueueService.cs +++ b/src/Core/Services/Implementations/AzureQueueService.cs @@ -1,9 +1,8 @@ using System.Collections.Generic; -using System.ComponentModel.DataAnnotations.Schema; using System.Linq; using System.Threading.Tasks; using Azure.Storage.Queues; -using IdentityServer4.Extensions; +using Bit.Core.Utilities; using Microsoft.EntityFrameworkCore.Internal; using Newtonsoft.Json; @@ -18,13 +17,13 @@ namespace Bit.Core.Services { _queueClient = queueClient; _jsonSettings = jsonSettings; + if (!_jsonSettings.Converters.Any(c => c.GetType() == typeof(EncodedStringConverter))) + { + _jsonSettings.Converters.Add(new EncodedStringConverter()); + } } - public async Task CreateAsync(T message) - { - var json = JsonConvert.SerializeObject(message, _jsonSettings); - await _queueClient.SendMessageAsync(json); - } + public async Task CreateAsync(T message) => await CreateManyAsync(new[] { message }); public async Task CreateManyAsync(IEnumerable messages) { @@ -33,25 +32,21 @@ namespace Bit.Core.Services return; } - if (!messages.Skip(1).Any()) - { - await CreateAsync(messages.First()); - return; - } - - foreach (var json in SerializeMany(messages, _jsonSettings)) + foreach (var json in SerializeMany(messages)) { await _queueClient.SendMessageAsync(json); } } - protected IEnumerable SerializeMany(IEnumerable messages, JsonSerializerSettings jsonSettings) + private IEnumerable SerializeMany(IEnumerable messages) { + string SerializeMessage(T message) => JsonConvert.SerializeObject(message, _jsonSettings); + var messagesLists = new List> { new List() }; var strings = new List(); var ListMessageLength = 2; // to account for json array brackets "[]" - foreach (var (message, jsonEvent) in messages.Select(e => (e, JsonConvert.SerializeObject(e, jsonSettings)))) + foreach (var (message, jsonEvent) in messages.Select(m => (m, SerializeMessage(m)))) { var messageLength = jsonEvent.Length + 1; // To account for json array comma @@ -66,7 +61,7 @@ namespace Bit.Core.Services ListMessageLength += messageLength; } } - return messagesLists.Select(l => JsonConvert.SerializeObject(l, jsonSettings)); + return messagesLists.Select(l => JsonConvert.SerializeObject(l, _jsonSettings)); } } } diff --git a/src/Core/Utilities/EncodedStringConverter.cs b/src/Core/Utilities/EncodedStringConverter.cs new file mode 100644 index 0000000000..5bb033d5fd --- /dev/null +++ b/src/Core/Utilities/EncodedStringConverter.cs @@ -0,0 +1,35 @@ +using System; +using Newtonsoft.Json; + +namespace Bit.Core.Utilities +{ + public class EncodedStringConverter : JsonConverter + { + public override bool CanConvert(Type objectType) => objectType == typeof(string); + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.Null) + { + return existingValue; + } + + var value = reader.Value as string; + return System.Net.WebUtility.HtmlDecode(value); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + if (value == null) + { + if (serializer.NullValueHandling == NullValueHandling.Include) + { + writer.WriteNull(); + } + return; + } + + writer.WriteValue(System.Net.WebUtility.HtmlEncode((string)value)); + } + } +} diff --git a/src/EventsProcessor/AzureQueueHostedService.cs b/src/EventsProcessor/AzureQueueHostedService.cs index 9964e67a25..470f646d11 100644 --- a/src/EventsProcessor/AzureQueueHostedService.cs +++ b/src/EventsProcessor/AzureQueueHostedService.cs @@ -12,11 +12,13 @@ using Microsoft.Extensions.Logging; using Azure.Storage.Queues; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using Bit.Core.Utilities; namespace Bit.EventsProcessor { public class AzureQueueHostedService : IHostedService, IDisposable { + private readonly JsonSerializer _jsonSerializer; private readonly ILogger _logger; private readonly IConfiguration _configuration; @@ -31,6 +33,11 @@ namespace Bit.EventsProcessor { _logger = logger; _configuration = configuration; + + _jsonSerializer = JsonSerializer.Create(new JsonSerializerSettings + { + Converters = new[] { new EncodedStringConverter() }, + }); } public Task StartAsync(CancellationToken cancellationToken) @@ -111,7 +118,7 @@ namespace Bit.EventsProcessor var token = JToken.Parse(message); if (token is JArray) { - var indexedEntities = token.ToObject>() + var indexedEntities = token.ToObject>(_jsonSerializer) .SelectMany(e => EventTableEntity.IndexEvent(e)); events.AddRange(indexedEntities); }