nhyatt/bitwarden
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2025-06-30 23:52:50 -05:00
Code Activity

Support large organization sync (#1311)

Browse Source 
* Increase organization max seat size from 30k to 2b (#1274)

* Increase organization max seat size from 30k to 2b

* PR review. Do not modify unless state matches expected

* Organization sync simultaneous event reporting (#1275)

* Split up azure messages according to max size

* Allow simultaneous login of organization user events

* Early resolve small event lists

* Clarify logic

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>

* Improve readability

This comes at the cost of multiple serializations, but the
 improvement in wire-time should more than make up for this
 on message where serialization time matters

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>

* Queue emails (#1286)

* Extract common Azure queue methods

* Do not use internal entity framework namespace

* Prefer IEnumerable to IList unless needed

All of these implementations were just using `Count == 1`,
which is easily replicated. This will be used when abstracting Azure queues

* Add model for azure queue message

* Abstract Azure queue for reuse

* Creat service to enqueue mail messages for later processing

Azure queue mail service uses Azure queues.
Blocking just blocks until all the work is done -- This is
how emailing works today

* Provide mail queue service to DI

* Queue organization invite emails for later processing

All emails can later be added to this queue

* Create Admin hosted service to process enqueued mail messages

* Prefer constructors to static generators

* Mass delete organization users (#1287)

* Add delete many to Organization Users

* Correct formatting

* Remove erroneous migration

* Clarify parameter name

* Formatting fixes

* Simplify bump account revision sproc

* Formatting fixes

* Match file names to objects

* Indicate if large import is expected

* Early pull all existing users we were planning on inviting (#1290)

* Early pull all existing users we were planning on inviting

* Improve sproc name

* Batch upsert org users (#1289)

* Add UpsertMany sprocs to OrganizationUser

* Add method to create TVPs from any object.

Uses DbOrder attribute to generate.
Sproc will fail unless TVP column order matches that of the db type

* Combine migrations

* Correct formatting

* Include sql objects in sql project

* Keep consisten parameter names

* Batch deletes for performance

* Correct formatting

* consolidate migrations

* Use batch methods in OrganizationImport

* Declare @BatchSize

* Transaction names limited to 32 chars

Drop sproc before creating it if it exists

* Update import tests

* Allow for more users in org upgrades

* Fix formatting

* Improve class hierarchy structure

* Use name tuple types

* Fix formatting

* Front load all reflection

* Format constructor

* Simplify ToTvp as class-specific extension

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>
This commit is contained in:
Matt Gibson
2021-05-17 09:43:02 -05:00
committed by GitHub
parent 738a4c2bac
commit 785e788cb6
64 changed files with 1704 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
test/Core.Test/Services/HandlebarsMailServiceTests.cs
View File

@ -12,15 +12,18 @@ namespace Bit.Core.Test.Services
private readonly GlobalSettings _globalSettings;
private readonly IMailDeliveryService _mailDeliveryService;
private readonly IMailEnqueuingService _mailEnqueuingService;
public HandlebarsMailServiceTests()
{
_globalSettings = new GlobalSettings();
_mailDeliveryService = Substitute.For<IMailDeliveryService>();
_mailEnqueuingService = Substitute.For<IMailEnqueuingService>();
_sut = new HandlebarsMailService(
_globalSettings,
_mailDeliveryService
_mailDeliveryService,
_mailEnqueuingService
);
}

198
test/Core.Test/Services/OrganizationServiceTests.cs
View File

@ -7,7 +7,6 @@ using Bit.Core.Models.Table;
using Bit.Core.Models.Business;
using Bit.Core.Repositories;
using Bit.Core.Services;
using Microsoft.AspNetCore.DataProtection;
using NSubstitute;
using Xunit;
using Bit.Core.Test.AutoFixture;
@ -17,135 +16,108 @@ using Bit.Core.Test.AutoFixture.Attributes;
using Bit.Core.Test.AutoFixture.OrganizationFixtures;
using System.Text.Json;
using Organization = Bit.Core.Models.Table.Organization;
using System.Linq;
namespace Bit.Core.Test.Services
{
public class OrganizationServiceTests
{
[Fact]
public async Task OrgImportCreateNewUsers()
// [Fact]
[Theory, PaidOrganizationAutoData]
public async Task OrgImportCreateNewUsers(SutProvider<OrganizationService> sutProvider, Guid userId,
Organization org, List<OrganizationUserUserDetails> existingUsers, List<ImportedOrganizationUser> newUsers)
{
var orgRepo = Substitute.For<IOrganizationRepository>();
var orgUserRepo = Substitute.For<IOrganizationUserRepository>();
var collectionRepo = Substitute.For<ICollectionRepository>();
var userRepo = Substitute.For<IUserRepository>();
var groupRepo = Substitute.For<IGroupRepository>();
var dataProtector = Substitute.For<IDataProtector>();
var mailService = Substitute.For<IMailService>();
var pushNotService = Substitute.For<IPushNotificationService>();
var pushRegService = Substitute.For<IPushRegistrationService>();
var deviceRepo = Substitute.For<IDeviceRepository>();
var licenseService = Substitute.For<ILicensingService>();
var eventService = Substitute.For<IEventService>();
var installationRepo = Substitute.For<IInstallationRepository>();
var appCacheService = Substitute.For<IApplicationCacheService>();
var paymentService = Substitute.For<IPaymentService>();
var policyRepo = Substitute.For<IPolicyRepository>();
var ssoConfigRepo = Substitute.For<ISsoConfigRepository>();
var ssoUserRepo = Substitute.For<ISsoUserRepository>();
var referenceEventService = Substitute.For<IReferenceEventService>();
var globalSettings = Substitute.For<Settings.GlobalSettings>();
var taxRateRepository = Substitute.For<ITaxRateRepository>();
var orgService = new OrganizationService(orgRepo, orgUserRepo, collectionRepo, userRepo,
groupRepo, dataProtector, mailService, pushNotService, pushRegService, deviceRepo,
licenseService, eventService, installationRepo, appCacheService, paymentService, policyRepo,
ssoConfigRepo, ssoUserRepo, referenceEventService, globalSettings, taxRateRepository);
var id = Guid.NewGuid();
var userId = Guid.NewGuid();
var org = new Organization
org.UseDirectory = true;
newUsers.Add(new ImportedOrganizationUser
{
Id = id,
Name = "Test Org",
UseDirectory = true,
UseGroups = true,
Seats = 3
};
orgRepo.GetByIdAsync(id).Returns(org);
var existingUsers = new List<OrganizationUserUserDetails>();
existingUsers.Add(new OrganizationUserUserDetails
{
Id = Guid.NewGuid(),
ExternalId = "a",
Email = "a@test.com"
Email = existingUsers.First().Email,
ExternalId = existingUsers.First().ExternalId
});
orgUserRepo.GetManyDetailsByOrganizationAsync(id).Returns(existingUsers);
orgUserRepo.GetCountByOrganizationIdAsync(id).Returns(1);
var expectedNewUsersCount = newUsers.Count - 1;
var newUsers = new List<ImportedOrganizationUser>();
newUsers.Add(new ImportedOrganizationUser { Email = "a@test.com", ExternalId = "a" });
newUsers.Add(new ImportedOrganizationUser { Email = "b@test.com", ExternalId = "b" });
newUsers.Add(new ImportedOrganizationUser { Email = "c@test.com", ExternalId = "c" });
await orgService.ImportAsync(id, userId, null, newUsers, null, false);
sutProvider.GetDependency<IOrganizationRepository>().GetByIdAsync(org.Id).Returns(org);
sutProvider.GetDependency<IOrganizationUserRepository>().GetManyDetailsByOrganizationAsync(org.Id)
.Returns(existingUsers);
sutProvider.GetDependency<IOrganizationUserRepository>().GetCountByOrganizationIdAsync(org.Id)
.Returns(existingUsers.Count);
await orgUserRepo.DidNotReceive().UpsertAsync(Arg.Any<OrganizationUser>());
await orgUserRepo.Received(2).CreateAsync(Arg.Any<OrganizationUser>());
await sutProvider.Sut.ImportAsync(org.Id, userId, null, newUsers, null, false);
await sutProvider.GetDependency<IOrganizationUserRepository>().DidNotReceiveWithAnyArgs()
.UpsertAsync(default);
await sutProvider.GetDependency<IOrganizationUserRepository>().Received(1)
.UpsertManyAsync(Arg.Is<IEnumerable<OrganizationUser>>(users => users.Count() == 0));
await sutProvider.GetDependency<IOrganizationUserRepository>().DidNotReceiveWithAnyArgs()
.CreateAsync(default);
// Create new users
await sutProvider.GetDependency<IOrganizationUserRepository>().Received(1)
.CreateManyAsync(Arg.Is<IEnumerable<OrganizationUser>>(users => users.Count() == expectedNewUsersCount));
await sutProvider.GetDependency<IMailService>().Received(1)
.BulkSendOrganizationInviteEmailAsync(org.Name,
Arg.Is<IEnumerable<(OrganizationUser, string)>>(messages => messages.Count() == expectedNewUsersCount));
// Send events
await sutProvider.GetDependency<IEventService>().Received(1)
.LogOrganizationUserEventsAsync(Arg.Is<IEnumerable<(OrganizationUser, EventType, DateTime?)>>(events =>
events.Count() == expectedNewUsersCount));
await sutProvider.GetDependency<IReferenceEventService>().Received(1)
.RaiseEventAsync(Arg.Is<ReferenceEvent>(referenceEvent =>
referenceEvent.Type == ReferenceEventType.InvitedUsers && referenceEvent.Id == org.Id &&
referenceEvent.Users == expectedNewUsersCount));
}
[Fact]
public async Task OrgImportCreateNewUsersAndMarryExistingUser()
[Theory, PaidOrganizationAutoData]
public async Task OrgImportCreateNewUsersAndMarryExistingUser(SutProvider<OrganizationService> sutProvider,
Guid userId, Organization org, List<OrganizationUserUserDetails> existingUsers,
List<ImportedOrganizationUser> newUsers)
{
var orgRepo = Substitute.For<IOrganizationRepository>();
var orgUserRepo = Substitute.For<IOrganizationUserRepository>();
var collectionRepo = Substitute.For<ICollectionRepository>();
var userRepo = Substitute.For<IUserRepository>();
var groupRepo = Substitute.For<IGroupRepository>();
var dataProtector = Substitute.For<IDataProtector>();
var mailService = Substitute.For<IMailService>();
var pushNotService = Substitute.For<IPushNotificationService>();
var pushRegService = Substitute.For<IPushRegistrationService>();
var deviceRepo = Substitute.For<IDeviceRepository>();
var licenseService = Substitute.For<ILicensingService>();
var eventService = Substitute.For<IEventService>();
var installationRepo = Substitute.For<IInstallationRepository>();
var appCacheService = Substitute.For<IApplicationCacheService>();
var paymentService = Substitute.For<IPaymentService>();
var policyRepo = Substitute.For<IPolicyRepository>();
var ssoConfigRepo = Substitute.For<ISsoConfigRepository>();
var ssoUserRepo = Substitute.For<ISsoUserRepository>();
var referenceEventService = Substitute.For<IReferenceEventService>();
var globalSettings = Substitute.For<Settings.GlobalSettings>();
var taxRateRepo = Substitute.For<ITaxRateRepository>();
var orgService = new OrganizationService(orgRepo, orgUserRepo, collectionRepo, userRepo,
groupRepo, dataProtector, mailService, pushNotService, pushRegService, deviceRepo,
licenseService, eventService, installationRepo, appCacheService, paymentService, policyRepo,
ssoConfigRepo, ssoUserRepo, referenceEventService, globalSettings, taxRateRepo);
var id = Guid.NewGuid();
var userId = Guid.NewGuid();
var org = new Organization
org.UseDirectory = true;
var reInvitedUser = existingUsers.First();
reInvitedUser.ExternalId = null;
newUsers.Add(new ImportedOrganizationUser
{
Id = id,
Name = "Test Org",
UseDirectory = true,
UseGroups = true,
Seats = 3
};
orgRepo.GetByIdAsync(id).Returns(org);
var existingUserAId = Guid.NewGuid();
var existingUsers = new List<OrganizationUserUserDetails>();
existingUsers.Add(new OrganizationUserUserDetails
{
Id = existingUserAId,
// No external id here
Email = "a@test.com"
Email = reInvitedUser.Email,
ExternalId = reInvitedUser.Email,
});
orgUserRepo.GetManyDetailsByOrganizationAsync(id).Returns(existingUsers);
orgUserRepo.GetCountByOrganizationIdAsync(id).Returns(1);
orgUserRepo.GetByIdAsync(existingUserAId).Returns(new OrganizationUser { Id = existingUserAId });
var expectedNewUsersCount = newUsers.Count - 1;
var newUsers = new List<ImportedOrganizationUser>();
newUsers.Add(new ImportedOrganizationUser { Email = "a@test.com", ExternalId = "a" });
newUsers.Add(new ImportedOrganizationUser { Email = "b@test.com", ExternalId = "b" });
newUsers.Add(new ImportedOrganizationUser { Email = "c@test.com", ExternalId = "c" });
await orgService.ImportAsync(id, userId, null, newUsers, null, false);
sutProvider.GetDependency<IOrganizationRepository>().GetByIdAsync(org.Id).Returns(org);
sutProvider.GetDependency<IOrganizationUserRepository>().GetManyDetailsByOrganizationAsync(org.Id)
.Returns(existingUsers);
sutProvider.GetDependency<IOrganizationUserRepository>().GetCountByOrganizationIdAsync(org.Id)
.Returns(existingUsers.Count);
sutProvider.GetDependency<IOrganizationUserRepository>().GetByIdAsync(reInvitedUser.Id)
.Returns(new OrganizationUser { Id = reInvitedUser.Id });
await orgUserRepo.Received(1).UpsertAsync(Arg.Any<OrganizationUser>());
await orgUserRepo.Received(2).CreateAsync(Arg.Any<OrganizationUser>());
await sutProvider.Sut.ImportAsync(org.Id, userId, null, newUsers, null, false);
await sutProvider.GetDependency<IOrganizationUserRepository>().DidNotReceiveWithAnyArgs()
.UpsertAsync(default);
await sutProvider.GetDependency<IOrganizationUserRepository>().DidNotReceiveWithAnyArgs()
.CreateAsync(default);
await sutProvider.GetDependency<IOrganizationUserRepository>().DidNotReceiveWithAnyArgs()
.CreateAsync(default, default);
// Upserted existing user
await sutProvider.GetDependency<IOrganizationUserRepository>().Received(1)
.UpsertManyAsync(Arg.Is<IEnumerable<OrganizationUser>>(users => users.Count() == 1));
// Created and invited new users
await sutProvider.GetDependency<IOrganizationUserRepository>().Received(1)
.CreateManyAsync(Arg.Is<IEnumerable<OrganizationUser>>(users => users.Count() == expectedNewUsersCount));
await sutProvider.GetDependency<IMailService>().Received(1)
.BulkSendOrganizationInviteEmailAsync(org.Name,
Arg.Is<IEnumerable<(OrganizationUser, string)>>(messages => messages.Count() == expectedNewUsersCount));
// Sent events
await sutProvider.GetDependency<IEventService>().Received(1)
.LogOrganizationUserEventsAsync(Arg.Is<IEnumerable<(OrganizationUser, EventType, DateTime?)>>(events =>
events.Where(e => e.Item2 == EventType.OrganizationUser_Invited).Count() == expectedNewUsersCount));
await sutProvider.GetDependency<IReferenceEventService>().Received(1)
.RaiseEventAsync(Arg.Is<ReferenceEvent>(referenceEvent =>
referenceEvent.Type == ReferenceEventType.InvitedUsers && referenceEvent.Id == org.Id &&
referenceEvent.Users == expectedNewUsersCount));
}
[Theory, CustomAutoData(typeof(SutProviderCustomization))]