Support large organization sync (#1311)

* Increase organization max seat size from 30k to 2b (#1274)

* Increase organization max seat size from 30k to 2b

* PR review. Do not modify unless state matches expected

* Organization sync simultaneous event reporting (#1275)

* Split up azure messages according to max size

* Allow simultaneous login of organization user events

* Early resolve small event lists

* Clarify logic

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>

* Improve readability

This comes at the cost of multiple serializations, but the
 improvement in wire-time should more than make up for this
 on message where serialization time matters

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>

* Queue emails (#1286)

* Extract common Azure queue methods

* Do not use internal entity framework namespace

* Prefer IEnumerable to IList unless needed

All of these implementations were just using `Count == 1`,
which is easily replicated. This will be used when abstracting Azure queues

* Add model for azure queue message

* Abstract Azure queue for reuse

* Creat service to enqueue mail messages for later processing

Azure queue mail service uses Azure queues.
Blocking just blocks until all the work is done -- This is
how emailing works today

* Provide mail queue service to DI

* Queue organization invite emails for later processing

All emails can later be added to this queue

* Create Admin hosted service to process enqueued mail messages

* Prefer constructors to static generators

* Mass delete organization users (#1287)

* Add delete many to Organization Users

* Correct formatting

* Remove erroneous migration

* Clarify parameter name

* Formatting fixes

* Simplify bump account revision sproc

* Formatting fixes

* Match file names to objects

* Indicate if large import is expected

* Early pull all existing users we were planning on inviting (#1290)

* Early pull all existing users we were planning on inviting

* Improve sproc name

* Batch upsert org users (#1289)

* Add UpsertMany sprocs to OrganizationUser

* Add method to create TVPs from any object.

Uses DbOrder attribute to generate.
Sproc will fail unless TVP column order matches that of the db type

* Combine migrations

* Correct formatting

* Include sql objects in sql project

* Keep consisten parameter names

* Batch deletes for performance

* Correct formatting

* consolidate migrations

* Use batch methods in OrganizationImport

* Declare @BatchSize

* Transaction names limited to 32 chars

Drop sproc before creating it if it exists

* Update import tests

* Allow for more users in org upgrades

* Fix formatting

* Improve class hierarchy structure

* Use name tuple types

* Fix formatting

* Front load all reflection

* Format constructor

* Simplify ToTvp as class-specific extension

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>

src/Core/Repositories/IEventRepository.cs

@@ -17,6 +17,6 @@ namespace Bit.Core.Repositories
         Task<PagedResult<IEvent>> GetManyByCipherAsync(Cipher cipher, DateTime startDate, DateTime endDate,
             PageOptions pageOptions);
         Task CreateAsync(IEvent e);
-        Task CreateManyAsync(IList<IEvent> e);
+        Task CreateManyAsync(IEnumerable<IEvent> e);
     }
 }

src/Core/Repositories/IOrganizationUserRepository.cs

@@ -15,6 +15,7 @@ namespace Bit.Core.Repositories
         Task<ICollection<OrganizationUser>> GetManyByUserAsync(Guid userId);
         Task<ICollection<OrganizationUser>> GetManyByOrganizationAsync(Guid organizationId, OrganizationUserType? type);
         Task<int> GetCountByOrganizationAsync(Guid organizationId, string email, bool onlyRegisteredUsers);
+        Task<IEnumerable<string>> SelectKnownEmailsAsync(Guid organizationId, IEnumerable<string> emails, bool onlyRegisteredUsers);
         Task<OrganizationUser> GetByOrganizationAsync(Guid organizationId, Guid userId);
         Task<Tuple<OrganizationUser, ICollection<SelectionReadOnly>>> GetByIdWithCollectionsAsync(Guid id);
         Task<OrganizationUserUserDetails> GetDetailsByIdAsync(Guid id);
@@ -26,10 +27,14 @@ namespace Bit.Core.Repositories
         Task<OrganizationUserOrganizationDetails> GetDetailsByUserAsync(Guid userId, Guid organizationId,
             OrganizationUserStatusType? status = null);
         Task UpdateGroupsAsync(Guid orgUserId, IEnumerable<Guid> groupIds);
+        Task UpsertManyAsync(IEnumerable<OrganizationUser> organizationUsers);
         Task CreateAsync(OrganizationUser obj, IEnumerable<SelectionReadOnly> collections);
+        Task CreateManyAsync(IEnumerable<OrganizationUser> organizationIdUsers);
         Task ReplaceAsync(OrganizationUser obj, IEnumerable<SelectionReadOnly> collections);
+        Task ReplaceManyAsync(IEnumerable<OrganizationUser> organizationUsers);
         Task<ICollection<OrganizationUser>> GetManyByManyUsersAsync(IEnumerable<Guid> userIds);
         Task<ICollection<OrganizationUser>> GetManyAsync(IEnumerable<Guid> Ids);
+        Task DeleteManyAsync(IEnumerable<Guid> userIds);
         Task<OrganizationUser> GetByOrganizationEmailAsync(Guid organizationId, string email);
     }
 }

src/Core/Repositories/SqlServer/EventRepository.cs

@@ -74,14 +74,14 @@ namespace Bit.Core.Repositories.SqlServer
             await base.CreateAsync(ev);
         }
 
-        public async Task CreateManyAsync(IList<IEvent> entities)
+        public async Task CreateManyAsync(IEnumerable<IEvent> entities)
         {
             if (!entities?.Any() ?? true)
             {
                 return;
             }
 
-            if (entities.Count == 1)
+            if (!entities.Skip(1).Any())
             {
                 await CreateAsync(entities.First());
                 return;

src/Core/Repositories/SqlServer/OrganizationUserRepository.cs

@@ -76,6 +76,20 @@ namespace Bit.Core.Repositories.SqlServer
             }
         }
 
+        public async Task<IEnumerable<string>> SelectKnownEmailsAsync(Guid organizationId, IEnumerable<string> emails,
+            bool onlyRegisteredUsers)
+        {
+            using (var connection = new SqlConnection(ConnectionString))
+            {
+                var result = await connection.QueryAsync<string>(
+                    "[dbo].[OrganizationUser_SelectKnownEmails]",
+                    new { OrganizationId = organizationId, Emails = emails.ToArrayTVP("Email"), OnlyUsers = onlyRegisteredUsers },
+                    commandType: CommandType.StoredProcedure);
+
+                return result;
+            }
+        }
+
         public async Task<OrganizationUser> GetByOrganizationAsync(Guid organizationId, Guid userId)
         {
             using (var connection = new SqlConnection(ConnectionString))
@@ -285,5 +299,71 @@ namespace Bit.Core.Repositories.SqlServer
                 return results.SingleOrDefault();
             }
         }
+
+        public async Task DeleteManyAsync(IEnumerable<Guid> organizationUserIds)
+        {
+            using (var connection = new SqlConnection(ConnectionString))
+            {
+                await connection.ExecuteAsync("[dbo].[OrganizationUser_DeleteByIds]",
+                    new { Ids = organizationUserIds.ToGuidIdArrayTVP() }, commandType: CommandType.StoredProcedure);
+            }
+        }
+
+        public async Task UpsertManyAsync(IEnumerable<OrganizationUser> organizationUsers)
+        {
+            var createUsers = new List<OrganizationUser>();
+            var replaceUsers = new List<OrganizationUser>();
+            foreach (var organizationUser in organizationUsers)
+            {
+                if (organizationUser.Id.Equals(default))
+                {
+                    createUsers.Add(organizationUser);
+                }
+                else
+                {
+                    replaceUsers.Add(organizationUser);
+                }
+            }
+
+            await CreateManyAsync(createUsers);
+            await ReplaceManyAsync(replaceUsers);
+        }
+
+        public async Task CreateManyAsync(IEnumerable<OrganizationUser> organizationUsers)
+        {
+            if (!organizationUsers.Any())
+            {
+                return;
+            }
+
+            foreach(var organizationUser in organizationUsers)
+            {
+                organizationUser.SetNewId();
+            }
+
+            using (var connection = new SqlConnection(ConnectionString))
+            {
+                var results = await connection.ExecuteAsync(
+                    $"[{Schema}].[{Table}_CreateMany]",
+                    new { OrganizationUsersInput = organizationUsers.ToTvp() },
+                    commandType: CommandType.StoredProcedure);
+            }
+        }
+
+        public async Task ReplaceManyAsync(IEnumerable<OrganizationUser> organizationUsers)
+        {
+            if (!organizationUsers.Any())
+            {
+                return;
+            }
+
+            using (var connection = new SqlConnection(ConnectionString))
+            {
+                var results = await connection.ExecuteAsync(
+                    $"[{Schema}].[{Table}_UpdateMany]",
+                    new { OrganizationUsersInput = organizationUsers.ToTvp() },
+                    commandType: CommandType.StoredProcedure);
+            }
+        }
     }
 }

src/Core/Repositories/TableStorage/EventRepository.cs

@@ -62,14 +62,14 @@ namespace Bit.Core.Repositories.TableStorage
             await CreateEntityAsync(entity);
         }
 
-        public async Task CreateManyAsync(IList<IEvent> e)
+        public async Task CreateManyAsync(IEnumerable<IEvent> e)
         {
             if (!e?.Any() ?? true)
             {
                 return;
             }
 
-            if (e.Count == 1)
+            if (!e.Skip(1).Any())
             {
                 await CreateAsync(e.First());
                 return;