nhyatt/bitwarden
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2025-07-18 08:00:59 -05:00
Code Activity

Support large organization sync (#1311)

Browse Source 
* Increase organization max seat size from 30k to 2b (#1274)

* Increase organization max seat size from 30k to 2b

* PR review. Do not modify unless state matches expected

* Organization sync simultaneous event reporting (#1275)

* Split up azure messages according to max size

* Allow simultaneous login of organization user events

* Early resolve small event lists

* Clarify logic

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>

* Improve readability

This comes at the cost of multiple serializations, but the
 improvement in wire-time should more than make up for this
 on message where serialization time matters

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>

* Queue emails (#1286)

* Extract common Azure queue methods

* Do not use internal entity framework namespace

* Prefer IEnumerable to IList unless needed

All of these implementations were just using `Count == 1`,
which is easily replicated. This will be used when abstracting Azure queues

* Add model for azure queue message

* Abstract Azure queue for reuse

* Creat service to enqueue mail messages for later processing

Azure queue mail service uses Azure queues.
Blocking just blocks until all the work is done -- This is
how emailing works today

* Provide mail queue service to DI

* Queue organization invite emails for later processing

All emails can later be added to this queue

* Create Admin hosted service to process enqueued mail messages

* Prefer constructors to static generators

* Mass delete organization users (#1287)

* Add delete many to Organization Users

* Correct formatting

* Remove erroneous migration

* Clarify parameter name

* Formatting fixes

* Simplify bump account revision sproc

* Formatting fixes

* Match file names to objects

* Indicate if large import is expected

* Early pull all existing users we were planning on inviting (#1290)

* Early pull all existing users we were planning on inviting

* Improve sproc name

* Batch upsert org users (#1289)

* Add UpsertMany sprocs to OrganizationUser

* Add method to create TVPs from any object.

Uses DbOrder attribute to generate.
Sproc will fail unless TVP column order matches that of the db type

* Combine migrations

* Correct formatting

* Include sql objects in sql project

* Keep consisten parameter names

* Batch deletes for performance

* Correct formatting

* consolidate migrations

* Use batch methods in OrganizationImport

* Declare @BatchSize

* Transaction names limited to 32 chars

Drop sproc before creating it if it exists

* Update import tests

* Allow for more users in org upgrades

* Fix formatting

* Improve class hierarchy structure

* Use name tuple types

* Fix formatting

* Front load all reflection

* Format constructor

* Simplify ToTvp as class-specific extension

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>
This commit is contained in:
Matt Gibson
2021-05-17 09:43:02 -05:00
committed by GitHub
parent 738a4c2bac
commit 785e788cb6
64 changed files with 1704 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/Sql/Sql.sqlproj
View File

@ -108,8 +108,10 @@
<Build Include="dbo\Stored Procedures\Organization_ReadByUserId.sql" />
<Build Include="dbo\Stored Procedures\Organization_Update.sql" />
<Build Include="dbo\Stored Procedures\OrganizationUser_Create.sql" />
<Build Include="dbo\Stored Procedures\OrganizationUser_CreateMany.sql" />
<Build Include="dbo\Stored Procedures\Grant_DeleteByKey.sql" />
<Build Include="dbo\Stored Procedures\OrganizationUser_DeleteById.sql" />
<Build Include="dbo\Stored Procedures\OrganizationUser_DeleteByIds.sql" />
<Build Include="dbo\Stored Procedures\Grant_Delete.sql" />
<Build Include="dbo\Stored Procedures\OrganizationUser_ReadById.sql" />
<Build Include="dbo\Stored Procedures\OrganizationUser_ReadByIds.sql" />
@ -123,12 +125,16 @@
<Build Include="dbo\Stored Procedures\User_ReadAccountRevisionDateById.sql" />
<Build Include="dbo\Stored Procedures\OrganizationUser_ReadCountByOrganizationId.sql" />
<Build Include="dbo\Stored Procedures\OrganizationUser_ReadCountByOnlyOwner.sql" />
<Build Include="dbo\Stored Procedures\OrganizationUser_SelectKnownEmails.sql" />
<Build Include="dbo\Stored Procedures\OrganizationUser_Update.sql" />
<Build Include="dbo\Stored Procedures\OrganizationUser_UpdateMany.sql" />
<Build Include="dbo\Stored Procedures\OrganizationUserOrganizationDetails_ReadByUserIdStatus.sql" />
<Build Include="dbo\Stored Procedures\OrganizationUserUserDetails_ReadByOrganizationId.sql" />
<Build Include="dbo\Stored Procedures\User_BumpAccountRevisionDate.sql" />
<Build Include="dbo\Stored Procedures\User_BumpManyAccountRevisionDates.sql" />
<Build Include="dbo\Stored Procedures\User_BumpAccountRevisionDateByOrganizationId.sql" />
<Build Include="dbo\Stored Procedures\User_BumpAccountRevisionDateByOrganizationUserId.sql" />
<Build Include="dbo\Stored Procedures\User_BumpAccountRevisionDateByOrganizationUserIds.sql" />
<Build Include="dbo\Stored Procedures\Cipher_Delete.sql" />
<Build Include="dbo\Stored Procedures\User_ReadPublicKeyById.sql" />
<Build Include="dbo\Stored Procedures\Cipher_Move.sql" />
@ -202,7 +208,10 @@
<Build Include="dbo\Stored Procedures\Folder_ReadById.sql" />
<Build Include="dbo\Stored Procedures\Folder_ReadByUserId.sql" />
<Build Include="dbo\Stored Procedures\Organization_Create.sql" />
<Build Include="dbo\User Defined Types\EmailArray.sql" />
<Build Include="dbo\User Defined Types\GuidIdArray.sql" />
<Build Include="dbo\User Defined Types\OrganizationUserType.sql" />
<Build Include="dbo\User Defined Types\TwoGuidIdArray.sql" />
<Build Include="dbo\User Defined Types\SelectionReadOnlyArray.sql" />
<Build Include="dbo\Stored Procedures\Cipher_UpdateAttachment.sql" />
<Build Include="dbo\Stored Procedures\Organization_UpdateStorage.sql" />
@ -272,6 +281,7 @@
<Build Include="dbo\Stored Procedures\Cipher_SoftDelete.sql" />
<Build Include="dbo\Tables\SsoUser.sql" />
<Build Include="dbo\Stored Procedures\SsoUser_Delete.sql" />
<Build Include="dbo\Stored Procedures\SsoUser_DeleteMany.sql" />
<Build Include="dbo\Stored Procedures\SsoUser_Create.sql" />
<Build Include="dbo\Stored Procedures\User_ReadBySsoUserOrganizationIdExternalId.sql" />
<Build Include="dbo\Stored Procedures\SsoUser_Update.sql" />

40
src/Sql/dbo/Stored Procedures/OrganizationUser_CreateMany.sql Normal file
View File

@ -0,0 +1,40 @@
CREATE PROCEDURE [dbo].[OrganizationUser_CreateMany]
@OrganizationUsersInput [dbo].[OrganizationUserType] READONLY
AS
BEGIN
SET NOCOUNT ON
INSERT INTO [dbo].[OrganizationUser]
(
[Id],
[OrganizationId],
[UserId],
[Email],
[Key],
[Status],
[Type],
[AccessAll],
[ExternalId],
[CreationDate],
[RevisionDate],
[Permissions],
[ResetPasswordKey]
)
SELECT
OU.[Id],
OU.[OrganizationId],
OU.[UserId],
OU.[Email],
OU.[Key],
OU.[Status],
OU.[Type],
OU.[AccessAll],
OU.[ExternalId],
OU.[CreationDate],
OU.[RevisionDate],
OU.[Permissions],
OU.[ResetPasswordKey]
FROM
@OrganizationUsersInput OU
END
GO

83
src/Sql/dbo/Stored Procedures/OrganizationUser_DeleteByIds.sql Normal file
View File

@ -0,0 +1,83 @@
CREATE PROCEDURE [dbo].[OrganizationUser_DeleteByIds]
@Ids [dbo].[GuidIdArray] READONLY
AS
BEGIN
SET NOCOUNT ON
EXEC [dbo].[User_BumpAccountRevisionDateByOrganizationUserIds] @Ids
DECLARE @UserAndOrganizationIds [dbo].[TwoGuidIdArray]
INSERT INTO @UserAndOrganizationIds
(Id1, Id2)
SELECT
UserId,
OrganizationId
FROM
[dbo].[OrganizationUser] OU
INNER JOIN
@Ids OUIds ON OUIds.Id = OU.Id
WHERE
UserId IS NOT NULL AND
OrganizationId IS NOT NULL
BEGIN
EXEC [dbo].[SsoUser_DeleteMany] @UserAndOrganizationIds
END
DECLARE @BatchSize INT = 100
-- Delete CollectionUsers
WHILE @BatchSize > 0
BEGIN
BEGIN TRANSACTION CollectionUser_DeleteMany_CollectionUsers
DELETE TOP(@BatchSize) CU
FROM
[dbo].[CollectionUser] CU
INNER JOIN
@Ids I ON I.Id = CU.OrganizationUserId
SET @BatchSize = @@ROWCOUNT
COMMIT TRANSACTION CollectionUser_DeleteMany_CollectionUsers
END
SET @BatchSize = 100;
-- Delete GroupUsers
WHILE @BatchSize > 0
BEGIN
BEGIN TRANSACTION GroupUser_DeleteMany_GroupUsers
DELETE TOP(@BatchSize) GU
FROM
[dbo].[GroupUser] GU
INNER JOIN
@Ids I ON I.Id = GU.OrganizationUserId
SET @BatchSize = @@ROWCOUNT
COMMIT TRANSACTION GoupUser_DeleteMany_GroupUsers
END
SET @BatchSize = 100;
-- Delete OrganizationUsers
WHILE @BatchSize > 0
BEGIN
BEGIN TRANSACTION OrganizationUser_DeleteMany_OrganizationUsers
DELETE TOP(@BatchSize) OU
FROM
[dbo].[OrganizationUser] OU
INNER JOIN
@Ids I ON I.Id = OU.Id
SET @BatchSize = @@ROWCOUNT
COMMIT TRANSACTION OrganizationUser_DeleteMany_OrganizationUsers
END
END
GO

30
src/Sql/dbo/Stored Procedures/OrganizationUser_SelectKnownEmails.sql Normal file
View File

@ -0,0 +1,30 @@
CREATE PROCEDURE [dbo].[OrganizationUser_SelectKnownEmails]
@OrganizationId UNIQUEIDENTIFIER,
@Emails [dbo].[EmailArray] READONLY,
@OnlyUsers BIT
AS
BEGIN
SET NOCOUNT ON
SELECT
E.Email
FROM
@Emails E
INNER JOIN
(
SELECT
U.[Email] as 'UEmail',
OU.[Email] as 'OUEmail',
OU.OrganizationId
FROM
[dbo].[User] U
RIGHT JOIN
[dbo].[OrganizationUser] OU ON OU.[UserId] = U.[Id]
WHERE
OU.OrganizationId = @OrganizationId
) OUU ON OUU.[UEmail] = E.[Email] OR OUU.[OUEmail] = E.[Email]
WHERE
(@OnlyUsers = 0 AND (OUU.UEmail IS NOT NULL OR OUU.OUEmail IS NOT NULL)) OR
(@OnlyUsers = 1 AND (OUU.UEmail IS NOT NULL))
END

33
src/Sql/dbo/Stored Procedures/OrganizationUser_UpdateMany.sql Normal file
View File

@ -0,0 +1,33 @@
CREATE PROCEDURE [dbo].[OrganizationUser_UpdateMany]
@OrganizationUsersInput [dbo].[OrganizationUserType] READONLY
AS
BEGIN
SET NOCOUNT ON
UPDATE
OU
SET
[OrganizationId] = OUI.[OrganizationId],
[UserId] = OUI.[UserId],
[Email] = OUI.[Email],
[Key] = OUI.[Key],
[Status] = OUI.[Status],
[Type] = OUI.[Type],
[AccessAll] = OUI.[AccessAll],
[ExternalId] = OUI.[ExternalId],
[CreationDate] = OUI.[CreationDate],
[RevisionDate] = OUI.[RevisionDate],
[Permissions] = OUI.[Permissions],
[ResetPasswordKey] = OUI.[ResetPasswordKey]
FROM
[dbo].[OrganizationUser] OU
INNER JOIN
@OrganizationUsersInput OUI ON OU.Id = OUI.Id
EXEC [dbo].[User_BumpManyAccountRevisionDates]
(
SELECT UserId
FROM @OrganizationUsersInput
)
END
GO

2
src/Sql/dbo/Stored Procedures/Organization_Create.sql
View File

@ -11,7 +11,7 @@
@BillingEmail NVARCHAR(256),
@Plan NVARCHAR(50),
@PlanType TINYINT,
@Seats SMALLINT,
@Seats INT,
@MaxCollections SMALLINT,
@UsePolicies BIT,
@UseSso BIT,

2
src/Sql/dbo/Stored Procedures/Organization_Update.sql
View File

@ -11,7 +11,7 @@
@BillingEmail NVARCHAR(256),
@Plan NVARCHAR(50),
@PlanType TINYINT,
@Seats SMALLINT,
@Seats INT,
@MaxCollections SMALLINT,
@UsePolicies BIT,
@UseSso BIT,

34
src/Sql/dbo/Stored Procedures/SsoUser_DeleteMany.sql Normal file
View File

@ -0,0 +1,34 @@
CREATE PROCEDURE [dbo].[SsoUser_DeleteMany]
@UserAndOrganizationIds [dbo].[TwoGuidIdArray] READONLY
AS
BEGIN
SET NOCOUNT ON
SELECT
Id
INTO
#SSOIds
FROM
[dbo].[SsoUser] SU
INNER JOIN
@UserAndOrganizationIds UOI ON UOI.Id1 = SU.UserId AND UOI.Id2 = SU.OrganizationId
DECLARE @BatchSize INT = 100
-- Delete SSO Users
WHILE @BatchSize > 0
BEGIN
BEGIN TRANSACTION SsoUser_DeleteMany_SsoUsers
DELETE TOP(@BatchSize) SU
FROM
[dbo].[SsoUser] SU
INNER JOIN
#SSOIds ON #SSOIds.Id = SU.Id
SET @BatchSize = @@ROWCOUNT
COMMIT TRANSACTION SsoUser_DeleteMany_SsoUsers
END
END
GO

18
src/Sql/dbo/Stored Procedures/User_BumpAccountRevisionDateByOrganizationUserIds.sql Normal file
View File

@ -0,0 +1,18 @@
CREATE PROCEDURE [dbo].[User_BumpAccountRevisionDateByOrganizationUserIds]
@OrganizationUserIds [dbo].[GuidIdArray] READONLY
AS
BEGIN
SET NOCOUNT ON
UPDATE
U
SET
U.[AccountRevisionDate] = GETUTCDATE()
FROM
@OrganizationUserIds OUIDs
INNER JOIN
[dbo].[OrganizationUser] OU ON OUIDs.Id = OU.Id AND OU.[Status] = 2 -- Confirmed
INNER JOIN
[dbo].[User] U ON OU.UserId = U.Id
END
GO

16
src/Sql/dbo/Stored Procedures/User_BumpManyAccountRevisionDates.sql Normal file
View File

@ -0,0 +1,16 @@
CREATE PROCEDURE [dbo].[User_BumpManyAccountRevisionDates]
@Ids [dbo].[GuidIdArray] READONLY
AS
BEGIN
SET NOCOUNT ON
UPDATE
U
SET
[AccountRevisionDate] = GETUTCDATE()
FROM
[dbo].[User] U
INNER JOIN
@Ids IDs ON IDs.Id = U.Id
END
GO

2
src/Sql/dbo/Tables/Organization.sql
View File

@ -11,7 +11,7 @@
[BillingEmail] NVARCHAR (256) NOT NULL,
[Plan] NVARCHAR (50) NOT NULL,
[PlanType] TINYINT NOT NULL,
[Seats] SMALLINT NULL,
[Seats] INT NULL,
[MaxCollections] SMALLINT NULL,
[UsePolicies] BIT NOT NULL,
[UseSso] BIT NOT NULL,

3
src/Sql/dbo/User Defined Types/EmailArray.sql Normal file
View File

@ -0,0 +1,3 @@
CREATE TYPE [dbo].[EmailArray] AS TABLE (
[Email] NVARCHAR(256) NOT NULL);
GO

15
src/Sql/dbo/User Defined Types/OrganizationUserType.sql Normal file
View File

@ -0,0 +1,15 @@
CREATE TYPE [dbo].[OrganizationUserType] AS TABLE(
[Id] UNIQUEIDENTIFIER,
[OrganizationId] UNIQUEIDENTIFIER,
[UserId] UNIQUEIDENTIFIER,
[Email] NVARCHAR(256),
[Key] VARCHAR(MAX),
[Status] TINYINT,
[Type] TINYINT,
[AccessAll] BIT,
[ExternalId] NVARCHAR(300),
[CreationDate] DATETIME2(7),
[RevisionDate] DATETIME2(7),
[Permissions] NVARCHAR(MAX),
[ResetPasswordKey] VARCHAR(MAX)
)

4
src/Sql/dbo/User Defined Types/TwoGuidIdArray.sql Normal file
View File

@ -0,0 +1,4 @@
CREATE TYPE [dbo].[TwoGuidIdArray] AS TABLE (
[Id1] UNIQUEIDENTIFIER NOT NULL,
[Id2] UNIQUEIDENTIFIER NOT NULL);
GO