Start Migration from Newtonsoft.Json to System.Text.Json (#1803)

* Start switch to System.Text.Json

* Work on switching to System.Text.Json

* Main work on STJ refactor

* Fix build errors

* Run formatting

* Delete unused file

* Use legacy for two factor providers

* Run formatter

* Add TokenProviderTests

* Run formatting

* Fix merge issues

* Switch to use JsonSerializer

* Address PR feedback

* Fix formatting

* Ran formatter

* Switch to async

* Ensure Enums are serialized as strings

* Fix formatting

* Enqueue single items as arrays

* Remove CreateAsync method on AzureQueueService

src/Core/Utilities/CoreHelpers.cs

@@ -23,7 +23,6 @@ using Bit.Core.Settings;
 using IdentityModel;
 using Microsoft.AspNetCore.DataProtection;
 using MimeKit;
-using Newtonsoft.Json;
 
 namespace Bit.Core.Utilities
 {
@@ -332,12 +331,12 @@ namespace Bit.Core.Utilities
 
         /// <summary>
         /// Creates a clone of the given object through serializing to json and deserializing.
-        /// This method is subject to the limitations of Newstonsoft. For example, properties with
+        /// This method is subject to the limitations of System.Text.Json. For example, properties with
         /// inaccessible setters will not be set.
         /// </summary>
         public static T CloneObject<T>(T obj)
         {
-            return JsonConvert.DeserializeObject<T>(JsonConvert.SerializeObject(obj));
+            return JsonSerializer.Deserialize<T>(JsonSerializer.Serialize(obj));
         }
 
         public static bool SettingHasValue(string setting)

src/Core/Utilities/CustomIpRateLimitMiddleware.cs

@@ -7,7 +7,6 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
-using Newtonsoft.Json;
 
 namespace Bit.Core.Utilities
 {
@@ -41,10 +40,8 @@ namespace Bit.Core.Utilities
                 $"Slow down! Too many requests. Try again in {rule.Period}." : _options.QuotaExceededMessage;
             httpContext.Response.Headers["Retry-After"] = retryAfter;
             httpContext.Response.StatusCode = _options.HttpStatusCode;
-
-            httpContext.Response.ContentType = "application/json";
             var errorModel = new ErrorResponseModel { Message = message };
-            return httpContext.Response.WriteAsync(JsonConvert.SerializeObject(errorModel));
+            return httpContext.Response.WriteAsJsonAsync(errorModel, cancellationToken: httpContext.RequestAborted);
         }
 
         public override void LogBlockedRequest(HttpContext httpContext, ClientRequestIdentity identity,

src/Core/Utilities/DuoApi.cs

@@ -15,9 +15,9 @@ using System.IO;
 using System.Net;
 using System.Security.Cryptography;
 using System.Text;
+using System.Text.Json;
 using System.Text.RegularExpressions;
 using System.Web;
-using Newtonsoft.Json;
 
 namespace Bit.Core.Utilities.Duo
 {
@@ -175,7 +175,7 @@ namespace Bit.Core.Utilities.Duo
             var res = ApiCall(method, path, parameters, timeout, out var statusCode);
             try
             {
-                var dict = JsonConvert.DeserializeObject<Dictionary<string, object>>(res);
+                var dict = JsonSerializer.Deserialize<Dictionary<string, object>>(res);
                 if (dict["stat"] as string == "OK")
                 {
                     return dict["response"] as T;

src/Core/Utilities/EnumKeyResolver.cs

@@ -1,21 +0,0 @@
-using System;
-using Newtonsoft.Json.Serialization;
-
-namespace Bit.Core.Utilities
-{
-    public class EnumKeyResolver<T> : DefaultContractResolver where T : struct
-    {
-        protected override JsonDictionaryContract CreateDictionaryContract(Type objectType)
-        {
-            var contract = base.CreateDictionaryContract(objectType);
-            var keyType = contract.DictionaryKeyType;
-
-            if (keyType.BaseType == typeof(Enum))
-            {
-                contract.DictionaryKeyResolver = propName => ((T)Enum.Parse(keyType, propName)).ToString();
-            }
-
-            return contract;
-        }
-    }
-}

src/Core/Utilities/ExpandoObjectJsonConverter.cs

@@ -1,19 +0,0 @@
-using System;
-using System.Dynamic;
-using Newtonsoft.Json;
-
-namespace Bit.Core.Utilities
-{
-    public class ExpandoObjectJsonConverter : JsonConverter
-    {
-        public override bool CanWrite => false;
-        public override bool CanConvert(Type objectType) => true;
-
-        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
-        {
-            return serializer.Deserialize<ExpandoObject>(reader);
-        }
-
-        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) => throw new NotImplementedException();
-    }
-}

src/Core/Utilities/HandlebarsObjectJsonConverter.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bit.Core.Utilities
+{
+    public class HandlebarsObjectJsonConverter : JsonConverter<object>
+    {
+        public override bool CanConvert(Type typeToConvert) => true;
+        public override object Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+        {
+            return JsonSerializer.Deserialize<Dictionary<string, object>>(ref reader, options);
+        }
+        public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options)
+        {
+            JsonSerializer.Serialize(writer, value, options);
+        }
+    }
+}

src/Core/Utilities/JsonHelpers.cs

@@ -0,0 +1,106 @@
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using NS = Newtonsoft.Json;
+
+namespace Bit.Core.Utilities
+{
+    public static class JsonHelpers
+    {
+        public static JsonSerializerOptions Default { get; }
+        public static JsonSerializerOptions Indented { get; }
+        public static JsonSerializerOptions IgnoreWritingNull { get; }
+        public static JsonSerializerOptions CamelCase { get; }
+        public static JsonSerializerOptions IgnoreWritingNullAndCamelCase { get; }
+
+        static JsonHelpers()
+        {
+            Default = new JsonSerializerOptions();
+
+            Indented = new JsonSerializerOptions
+            {
+                WriteIndented = true,
+            };
+
+            IgnoreWritingNull = new JsonSerializerOptions
+            {
+                DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+            };
+
+            CamelCase = new JsonSerializerOptions
+            {
+                PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+            };
+
+            IgnoreWritingNullAndCamelCase = new JsonSerializerOptions
+            {
+                PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+                DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+            };
+        }
+
+        // NOTE: This is built into .NET 6, it SHOULD be removed when we upgrade
+        public static T ToObject<T>(this JsonElement element, JsonSerializerOptions options = null)
+        {
+            return JsonSerializer.Deserialize<T>(element.GetRawText(), options ?? Default);
+        }
+
+        public static T DeserializeOrNew<T>(string json, JsonSerializerOptions options = null)
+            where T : new()
+        {
+            if (string.IsNullOrWhiteSpace(json))
+            {
+                return new T();
+            }
+
+            return JsonSerializer.Deserialize<T>(json, options);
+        }
+
+        #region Legacy Newtonsoft.Json usage
+        private const string LegacyMessage = "Usage of Newtonsoft.Json should be kept to a minimum and will further be removed when we move to .NET 6";
+
+        [Obsolete(LegacyMessage)]
+        public static NS.JsonSerializerSettings LegacyDefault { get; } = new NS.JsonSerializerSettings();
+
+        [Obsolete(LegacyMessage)]
+        public static string LegacySerialize(object value, NS.JsonSerializerSettings settings = null)
+        {
+            return NS.JsonConvert.SerializeObject(value, settings ?? LegacyDefault);
+        }
+
+        [Obsolete(LegacyMessage)]
+        public static T LegacyDeserialize<T>(string value, NS.JsonSerializerSettings settings = null)
+        {
+            return NS.JsonConvert.DeserializeObject<T>(value, settings ?? LegacyDefault);
+        }
+        #endregion
+    }
+
+    public class MsEpochConverter : JsonConverter<DateTime?>
+    {
+        public override DateTime? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+        {
+            if (reader.TokenType == JsonTokenType.Null)
+            {
+                return null;
+            }
+
+            if (!long.TryParse(reader.GetString(), out var milliseconds))
+            {
+                return null;
+            }
+
+            return CoreHelpers.FromEpocMilliseconds(milliseconds);
+        }
+
+        public override void Write(Utf8JsonWriter writer, DateTime? value, JsonSerializerOptions options)
+        {
+            if (!value.HasValue)
+            {
+                writer.WriteNullValue();
+            }
+
+            writer.WriteStringValue(CoreHelpers.ToEpocMilliseconds(value.Value).ToString());
+        }
+    }
+}