1
0
mirror of https://github.com/bitwarden/server.git synced 2025-06-30 07:36:14 -05:00

Start Migration from Newtonsoft.Json to System.Text.Json (#1803)

* Start switch to System.Text.Json

* Work on switching to System.Text.Json

* Main work on STJ refactor

* Fix build errors

* Run formatting

* Delete unused file

* Use legacy for two factor providers

* Run formatter

* Add TokenProviderTests

* Run formatting

* Fix merge issues

* Switch to use JsonSerializer

* Address PR feedback

* Fix formatting

* Ran formatter

* Switch to async

* Ensure Enums are serialized as strings

* Fix formatting

* Enqueue single items as arrays

* Remove CreateAsync method on AzureQueueService
This commit is contained in:
Justin Baur
2022-01-21 09:36:25 -05:00
committed by GitHub
parent 897a76ff48
commit 5268f2781e
91 changed files with 974 additions and 698 deletions

View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Threading.Tasks;
using AutoFixture.Xunit2;
using Bit.Api.Controllers;
@ -14,7 +15,6 @@ using Bit.Core.Settings;
using Bit.Core.Utilities;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using NSubstitute;
using Xunit;
@ -66,7 +66,7 @@ namespace Bit.Api.Test.Controllers
send.Id = default;
send.Type = SendType.Text;
send.Data = JsonConvert.SerializeObject(new Dictionary<string, string>());
send.Data = JsonSerializer.Serialize(new Dictionary<string, string>());
send.HideEmail = true;
_sendService.AccessAsync(id, null).Returns((send, false, false));
@ -81,4 +81,3 @@ namespace Bit.Api.Test.Controllers
}
}
}

View File

@ -1,9 +1,9 @@
using System;
using System.IO;
using System.Linq;
using System.Reflection;
using Newtonsoft.Json;
using System.Text.Json;
using Bit.Core.Utilities;
using Xunit;
using Xunit.Sdk;
namespace Bit.Test.Common.Helpers
{
@ -29,10 +29,9 @@ namespace Bit.Test.Common.Helpers
if (actualPropInfo == null)
{
var settings = new JsonSerializerSettings { Formatting = Formatting.Indented };
throw new Exception(string.Concat($"Expected actual object to contain a property named {expectedPropInfo.Name}, but it does not\n",
$"Expected:\n{JsonConvert.SerializeObject(expected, settings)}\n",
$"Actual:\n{JsonConvert.SerializeObject(actual, new JsonSerializerSettings { Formatting = Formatting.Indented })}"));
$"Expected:\n{JsonSerializer.Serialize(expected, JsonHelpers.Indented)}\n",
$"Actual:\n{JsonSerializer.Serialize(actual, JsonHelpers.Indented)}"));
}
if (expectedPropInfo.PropertyType == typeof(string) || expectedPropInfo.PropertyType.IsValueType)
@ -54,5 +53,16 @@ namespace Bit.Test.Common.Helpers
Assert.Equal(expected, actual);
return true;
};
public static JsonElement AssertJsonProperty(JsonElement element, string propertyName, JsonValueKind jsonValueKind)
{
if (!element.TryGetProperty(propertyName, out var subElement))
{
throw new XunitException($"Could not find property by name '{propertyName}'");
}
Assert.Equal(jsonValueKind, subElement.ValueKind);
return subElement;
}
}
}

View File

@ -9,7 +9,7 @@ namespace Bit.Core.Test.AutoFixture.CipherAttachmentMetaData
protected virtual IPostprocessComposer<CipherAttachment.MetaData> ComposerAction(IFixture fixture,
ICustomizationComposer<CipherAttachment.MetaData> composer)
{
return composer.With(d => d.Size, fixture.Create<long>()).Without(d => d.SizeString);
return composer.With(d => d.Size, fixture.Create<long>());
}
public void Customize(IFixture fixture)
{

View File

@ -0,0 +1,41 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Bit.Core.Entities;
using Bit.Core.Enums;
using Bit.Core.Identity;
using Bit.Test.Common.AutoFixture;
using Bit.Test.Common.AutoFixture.Attributes;
using Xunit;
namespace Bit.Core.Test.Identity
{
public class AuthenticationTokenProviderTests : BaseTokenProviderTests<AuthenticatorTokenProvider>
{
public override TwoFactorProviderType TwoFactorProviderType => TwoFactorProviderType.Authenticator;
public static IEnumerable<object[]> CanGenerateTwoFactorTokenAsyncData
=> SetupCanGenerateData(
(
new Dictionary<string, object>
{
["Key"] = "stuff",
},
true
),
(
new Dictionary<string, object>
{
["Key"] = ""
},
false
)
);
[Theory, BitMemberAutoData(nameof(CanGenerateTwoFactorTokenAsyncData))]
public override async Task RunCanGenerateTwoFactorTokenAsync(Dictionary<string, object> metaData, bool expectedResponse,
User user, SutProvider<AuthenticatorTokenProvider> sutProvider)
{
await base.RunCanGenerateTwoFactorTokenAsync(metaData, expectedResponse, user, sutProvider);
}
}
}

View File

@ -0,0 +1,97 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Bit.Core.Entities;
using Bit.Core.Enums;
using Bit.Core.Models;
using Bit.Core.Services;
using Bit.Core.Utilities;
using Bit.Test.Common.AutoFixture;
using Bit.Test.Common.AutoFixture.Attributes;
using Microsoft.AspNetCore.Identity;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using NSubstitute;
using Xunit;
namespace Bit.Core.Test.Identity
{
[SutProviderCustomize]
public abstract class BaseTokenProviderTests<T>
where T : IUserTwoFactorTokenProvider<User>
{
public abstract TwoFactorProviderType TwoFactorProviderType { get; }
#region Helpers
protected static IEnumerable<object[]> SetupCanGenerateData(params (Dictionary<string, object> MetaData, bool ExpectedResponse)[] data)
{
return data.Select(d =>
new object[]
{
d.MetaData,
d.ExpectedResponse,
});
}
protected virtual IUserService AdditionalSetup(SutProvider<T> sutProvider, User user)
{
var userService = Substitute.For<IUserService>();
sutProvider.GetDependency<IServiceProvider>()
.GetService(typeof(IUserService))
.Returns(userService);
SetupUserService(userService, user);
return userService;
}
protected virtual void SetupUserService(IUserService userService, User user)
{
userService
.TwoFactorProviderIsEnabledAsync(TwoFactorProviderType, user)
.Returns(true);
}
protected static UserManager<User> SubstituteUserManager()
{
return new UserManager<User>(Substitute.For<IUserStore<User>>(),
Substitute.For<IOptions<IdentityOptions>>(),
Substitute.For<IPasswordHasher<User>>(),
Enumerable.Empty<IUserValidator<User>>(),
Enumerable.Empty<IPasswordValidator<User>>(),
Substitute.For<ILookupNormalizer>(),
Substitute.For<IdentityErrorDescriber>(),
Substitute.For<IServiceProvider>(),
Substitute.For<ILogger<UserManager<User>>>());
}
protected void MockDatabase(User user, Dictionary<string, object> metaData)
{
var providers = new Dictionary<TwoFactorProviderType, TwoFactorProvider>
{
[TwoFactorProviderType] = new TwoFactorProvider
{
Enabled = true,
MetaData = metaData,
},
};
user.TwoFactorProviders = JsonHelpers.LegacySerialize(providers);
}
#endregion
public virtual async Task RunCanGenerateTwoFactorTokenAsync(Dictionary<string, object> metaData, bool expectedResponse,
User user, SutProvider<T> sutProvider)
{
var userManager = SubstituteUserManager();
MockDatabase(user, metaData);
AdditionalSetup(sutProvider, user);
var response = await sutProvider.Sut.CanGenerateTwoFactorTokenAsync(userManager, user);
Assert.Equal(expectedResponse, response);
}
}
}

View File

@ -0,0 +1,48 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Bit.Core.Entities;
using Bit.Core.Enums;
using Bit.Core.Identity;
using Bit.Test.Common.AutoFixture;
using Bit.Test.Common.AutoFixture.Attributes;
using Xunit;
namespace Bit.Core.Test.Identity
{
public class EmailTokenProviderTests : BaseTokenProviderTests<EmailTokenProvider>
{
public override TwoFactorProviderType TwoFactorProviderType => TwoFactorProviderType.Email;
public static IEnumerable<object[]> CanGenerateTwoFactorTokenAsyncData
=> SetupCanGenerateData(
(
new Dictionary<string, object>
{
["Email"] = "test@email.com",
},
true
),
(
new Dictionary<string, object>
{
["NotEmail"] = "value",
},
false
),
(
new Dictionary<string, object>
{
["Email"] = "",
},
false
)
);
[Theory, BitMemberAutoData(nameof(CanGenerateTwoFactorTokenAsyncData))]
public override async Task RunCanGenerateTwoFactorTokenAsync(Dictionary<string, object> metaData, bool expectedResponse,
User user, SutProvider<EmailTokenProvider> sutProvider)
{
await base.RunCanGenerateTwoFactorTokenAsync(metaData, expectedResponse, user, sutProvider);
}
}
}

View File

@ -0,0 +1,62 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Bit.Core.Entities;
using Bit.Core.Enums;
using Bit.Core.Identity;
using Bit.Test.Common.AutoFixture;
using Bit.Test.Common.AutoFixture.Attributes;
using NSubstitute;
using Xunit;
namespace Bit.Core.Test.Identity
{
public class U2fTokenProviderTests : BaseTokenProviderTests<U2fTokenProvider>
{
public override TwoFactorProviderType TwoFactorProviderType => TwoFactorProviderType.U2f;
public static IEnumerable<object[]> CanGenerateTwoFactorTokenAsyncData()
{
return new[]
{
new object[]
{
new Dictionary<string, object>
{
["Something"] = "Hello"
},
true, // canAccessPremium
true, // expectedResponse
},
new object[]
{
new Dictionary<string, object>(),
true, // canAccessPremium
false, // expectedResponse
},
new object[]
{
new Dictionary<string, object>
{
["Key"] = "Value"
},
false, // canAccessPremium
false, // expectedResponse
},
};
}
[Theory, BitMemberAutoData(nameof(CanGenerateTwoFactorTokenAsyncData))]
public async Task CanGenerateTwoFactorTokenAsync_Success(Dictionary<string, object> metaData, bool canAccessPremium,
bool expectedResponse, User user, SutProvider<U2fTokenProvider> sutProvider)
{
var userManager = SubstituteUserManager();
MockDatabase(user, metaData);
AdditionalSetup(sutProvider, user)
.CanAccessPremium(user)
.Returns(canAccessPremium);
var response = await sutProvider.Sut.CanGenerateTwoFactorTokenAsync(userManager, user);
Assert.Equal(expectedResponse, response);
}
}
}

View File

@ -1,6 +1,6 @@
using Bit.Core.Entities;
using System.Text.Json;
using Bit.Core.Entities;
using Bit.Core.Test.AutoFixture.CipherFixtures;
using Newtonsoft.Json;
using Xunit;
namespace Bit.Core.Test.Models
@ -12,7 +12,7 @@ namespace Bit.Core.Test.Models
[InlineOrganizationCipherAutoData]
public void Clone_CreatesExactCopy(Cipher cipher)
{
Assert.Equal(JsonConvert.SerializeObject(cipher), JsonConvert.SerializeObject(cipher.Clone()));
Assert.Equal(JsonSerializer.Serialize(cipher), JsonSerializer.Serialize(cipher.Clone()));
}
}
}

View File

@ -0,0 +1,28 @@
using System.Text.Json;
using Bit.Core.Models.Data;
using Bit.Test.Common.Helpers;
using Xunit;
namespace Bit.Core.Test.Models.Data
{
public class SendFileDataTests
{
[Fact]
public void Serialize_Success()
{
var sut = new SendFileData
{
Id = "test",
Size = 100,
FileName = "thing.pdf",
Validated = true,
};
var json = JsonSerializer.Serialize(sut);
var document = JsonDocument.Parse(json);
var root = document.RootElement;
AssertHelper.AssertJsonProperty(root, "Size", JsonValueKind.String);
Assert.False(root.TryGetProperty("SizeString", out _));
}
}
}

View File

@ -1,10 +1,6 @@
using System;
using System.Text.Json;
using AutoFixture.Xunit2;
using System.Text.Json;
using Bit.Core.Models.Data;
using Bit.Core.Utilities;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
using Xunit;
namespace Bit.Core.Test.Models
@ -33,21 +29,30 @@ namespace Bit.Core.Test.Models
[Fact]
public void Serialization_Success()
{
// minify expected json
var expected = JsonConvert.SerializeObject(JsonConvert.DeserializeObject(_exampleSerializedPermissions));
DefaultContractResolver contractResolver = new DefaultContractResolver
var permissions = new Permissions
{
NamingStrategy = new CamelCaseNamingStrategy()
AccessEventLogs = false,
AccessImportExport = false,
AccessReports = false,
CreateNewCollections = true,
EditAnyCollection = true,
DeleteAnyCollection = true,
EditAssignedCollections = false,
DeleteAssignedCollections = false,
ManageGroups = false,
ManagePolicies = false,
ManageSso = false,
ManageUsers = false,
ManageResetPassword = false,
};
var actual = JsonConvert.SerializeObject(
CoreHelpers.LoadClassFromJsonData<Permissions>(_exampleSerializedPermissions), new JsonSerializerSettings
{
ContractResolver = contractResolver,
});
// minify expected json
var expected = JsonSerializer.Serialize(permissions, JsonHelpers.CamelCase);
var actual = JsonSerializer.Serialize(
JsonHelpers.DeserializeOrNew<Permissions>(_exampleSerializedPermissions, JsonHelpers.CamelCase),
JsonHelpers.CamelCase);
Console.WriteLine(actual);
Assert.Equal(expected, actual);
}
}

View File

@ -1,4 +1,9 @@
using Bit.Core.Entities;
using System.Collections.Generic;
using System.Text.Json;
using Bit.Core.Entities;
using Bit.Core.Enums;
using Bit.Core.Models;
using Bit.Test.Common.Helpers;
using Xunit;
namespace Bit.Core.Test.Models.Tables
@ -39,5 +44,43 @@ namespace Bit.Core.Test.Models.Tables
Assert.Equal(expectedRemainingBytes, bytesRemaining);
}
[Fact]
public void SetTwoFactorProviders()
{
var user = new User();
user.SetTwoFactorProviders(new Dictionary<TwoFactorProviderType, TwoFactorProvider>
{
[TwoFactorProviderType.WebAuthn] = new TwoFactorProvider
{
Enabled = true,
MetaData = new Dictionary<string, object>
{
["Item"] = "thing",
},
},
[TwoFactorProviderType.Email] = new TwoFactorProvider
{
Enabled = false,
MetaData = new Dictionary<string, object>
{
["Email"] = "test@email.com",
},
},
});
using var jsonDocument = JsonDocument.Parse(user.TwoFactorProviders);
var root = jsonDocument.RootElement;
var webAuthn = AssertHelper.AssertJsonProperty(root, "WebAuthn", JsonValueKind.Object);
AssertHelper.AssertJsonProperty(webAuthn, "Enabled", JsonValueKind.True);
var webMetaData = AssertHelper.AssertJsonProperty(webAuthn, "MetaData", JsonValueKind.Object);
AssertHelper.AssertJsonProperty(webMetaData, "Item", JsonValueKind.String);
var email = AssertHelper.AssertJsonProperty(root, "Email", JsonValueKind.Object);
AssertHelper.AssertJsonProperty(email, "Enabled", JsonValueKind.False);
var emailMetaData = AssertHelper.AssertJsonProperty(email, "MetaData", JsonValueKind.Object);
AssertHelper.AssertJsonProperty(emailMetaData, "Email", JsonValueKind.String);
}
}
}

View File

@ -1,10 +1,17 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text.Json;
using System.Threading.Tasks;
using Bit.Core.Context;
using Bit.Core.Entities;
using Bit.Core.Models.Business;
using Bit.Core.Repositories;
using Bit.Core.Services;
using Bit.Core.Settings;
using Bit.Test.Common.AutoFixture;
using Bit.Test.Common.AutoFixture.Attributes;
using Bit.Test.Common.Helpers;
using Fido2NetLib;
using Microsoft.AspNetCore.DataProtection;
using Microsoft.AspNetCore.Identity;
@ -17,103 +24,41 @@ namespace Bit.Core.Test.Services
{
public class UserServiceTests
{
private readonly UserService _sut;
private readonly IUserRepository _userRepository;
private readonly ICipherRepository _cipherRepository;
private readonly IOrganizationUserRepository _organizationUserRepository;
private readonly IOrganizationRepository _organizationRepository;
private readonly IMailService _mailService;
private readonly IPushNotificationService _pushService;
private readonly IUserStore<User> _userStore;
private readonly IOptions<IdentityOptions> _optionsAccessor;
private readonly IPasswordHasher<User> _passwordHasher;
private readonly IEnumerable<IUserValidator<User>> _userValidators;
private readonly IEnumerable<IPasswordValidator<User>> _passwordValidators;
private readonly ILookupNormalizer _keyNormalizer;
private readonly IdentityErrorDescriber _errors;
private readonly IServiceProvider _services;
private readonly ILogger<UserManager<User>> _logger;
private readonly ILicensingService _licenseService;
private readonly IEventService _eventService;
private readonly IApplicationCacheService _applicationCacheService;
private readonly IDataProtectionProvider _dataProtectionProvider;
private readonly IPaymentService _paymentService;
private readonly IPolicyRepository _policyRepository;
private readonly IReferenceEventService _referenceEventService;
private readonly IFido2 _fido2;
private readonly CurrentContext _currentContext;
private readonly GlobalSettings _globalSettings;
private readonly IOrganizationService _organizationService;
private readonly IProviderUserRepository _providerUserRepository;
public UserServiceTests()
[Theory, CustomAutoData(typeof(SutProviderCustomization))]
public async Task UpdateLicenseAsync_Success(SutProvider<UserService> sutProvider,
User user, UserLicense userLicense)
{
_userRepository = Substitute.For<IUserRepository>();
_cipherRepository = Substitute.For<ICipherRepository>();
_organizationUserRepository = Substitute.For<IOrganizationUserRepository>();
_organizationRepository = Substitute.For<IOrganizationRepository>();
_mailService = Substitute.For<IMailService>();
_pushService = Substitute.For<IPushNotificationService>();
_userStore = Substitute.For<IUserStore<User>>();
_optionsAccessor = Substitute.For<IOptions<IdentityOptions>>();
_passwordHasher = Substitute.For<IPasswordHasher<User>>();
_userValidators = new List<IUserValidator<User>>();
_passwordValidators = new List<IPasswordValidator<User>>();
_keyNormalizer = Substitute.For<ILookupNormalizer>();
_errors = new IdentityErrorDescriber();
_services = Substitute.For<IServiceProvider>();
_logger = Substitute.For<ILogger<UserManager<User>>>();
_licenseService = Substitute.For<ILicensingService>();
_eventService = Substitute.For<IEventService>();
_applicationCacheService = Substitute.For<IApplicationCacheService>();
_dataProtectionProvider = Substitute.For<IDataProtectionProvider>();
_paymentService = Substitute.For<IPaymentService>();
_policyRepository = Substitute.For<IPolicyRepository>();
_referenceEventService = Substitute.For<IReferenceEventService>();
_fido2 = Substitute.For<IFido2>();
_currentContext = new CurrentContext(null);
_globalSettings = new GlobalSettings();
_organizationService = Substitute.For<IOrganizationService>();
_providerUserRepository = Substitute.For<IProviderUserRepository>();
using var tempDir = new TempDirectory();
_sut = new UserService(
_userRepository,
_cipherRepository,
_organizationUserRepository,
_organizationRepository,
_mailService,
_pushService,
_userStore,
_optionsAccessor,
_passwordHasher,
_userValidators,
_passwordValidators,
_keyNormalizer,
_errors,
_services,
_logger,
_licenseService,
_eventService,
_applicationCacheService,
_dataProtectionProvider,
_paymentService,
_policyRepository,
_referenceEventService,
_fido2,
_currentContext,
_globalSettings,
_organizationService,
_providerUserRepository
);
}
var now = DateTime.UtcNow;
userLicense.Issued = now.AddDays(-10);
userLicense.Expires = now.AddDays(10);
userLicense.Version = 1;
userLicense.Premium = true;
// Remove this test when we add actual tests. It only proves that
// we've properly constructed the system under test.
[Fact]
public void ServiceExists()
{
Assert.NotNull(_sut);
user.EmailVerified = true;
user.Email = userLicense.Email;
sutProvider.GetDependency<Settings.GlobalSettings>().SelfHosted = true;
sutProvider.GetDependency<Settings.GlobalSettings>().LicenseDirectory = tempDir.Directory;
sutProvider.GetDependency<ILicensingService>()
.VerifyLicense(userLicense)
.Returns(true);
await sutProvider.Sut.UpdateLicenseAsync(user, userLicense);
var filePath = Path.Combine(tempDir.Directory, "user", $"{user.Id}.json");
Assert.True(File.Exists(filePath));
var document = JsonDocument.Parse(File.OpenRead(filePath));
var root = document.RootElement;
Assert.Equal(JsonValueKind.Object, root.ValueKind);
// Sort of a lazy way to test that it is indented but not sure of a better way
Assert.Contains('\n', root.GetRawText());
AssertHelper.AssertJsonProperty(root, "LicenseKey", JsonValueKind.String);
AssertHelper.AssertJsonProperty(root, "Id", JsonValueKind.String);
AssertHelper.AssertJsonProperty(root, "Premium", JsonValueKind.True);
var versionProp = AssertHelper.AssertJsonProperty(root, "Version", JsonValueKind.Number);
Assert.Equal(1, versionProp.GetInt32());
}
}
}

View File

@ -0,0 +1,66 @@
using System.Text.Json;
using Bit.Core.Utilities;
using Xunit;
namespace Bit.Core.Test.Helpers
{
public class JsonHelpersTests
{
private static void CompareJson<T>(T value, JsonSerializerOptions options, Newtonsoft.Json.JsonSerializerSettings settings)
{
var stgJson = JsonSerializer.Serialize(value, options);
var nsJson = Newtonsoft.Json.JsonConvert.SerializeObject(value, settings);
Assert.Equal(stgJson, nsJson);
}
[Fact]
public void DefaultJsonOptions()
{
var testObject = new SimpleTestObject
{
Id = 0,
Name = "Test",
};
CompareJson(testObject, JsonHelpers.Default, new Newtonsoft.Json.JsonSerializerSettings());
}
[Fact]
public void IndentedJsonOptions()
{
var testObject = new SimpleTestObject
{
Id = 10,
Name = "Test Name"
};
CompareJson(testObject, JsonHelpers.Indented, new Newtonsoft.Json.JsonSerializerSettings
{
Formatting = Newtonsoft.Json.Formatting.Indented,
});
}
[Fact]
public void NullValueHandlingJsonOptions()
{
var testObject = new SimpleTestObject
{
Id = 14,
Name = null,
};
CompareJson(testObject, JsonHelpers.IgnoreWritingNull, new Newtonsoft.Json.JsonSerializerSettings
{
NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore,
});
}
}
public class SimpleTestObject
{
public int Id { get; set; }
public string Name { get; set; }
}
}