1
0
mirror of https://github.com/bitwarden/server.git synced 2025-06-30 07:36:14 -05:00

Support large organization sync (#1311)

* Increase organization max seat size from 30k to 2b (#1274)

* Increase organization max seat size from 30k to 2b

* PR review. Do not modify unless state matches expected

* Organization sync simultaneous event reporting (#1275)

* Split up azure messages according to max size

* Allow simultaneous login of organization user events

* Early resolve small event lists

* Clarify logic

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>

* Improve readability

This comes at the cost of multiple serializations, but the
 improvement in wire-time should more than make up for this
 on message where serialization time matters

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>

* Queue emails (#1286)

* Extract common Azure queue methods

* Do not use internal entity framework namespace

* Prefer IEnumerable to IList unless needed

All of these implementations were just using `Count == 1`,
which is easily replicated. This will be used when abstracting Azure queues

* Add model for azure queue message

* Abstract Azure queue for reuse

* Creat service to enqueue mail messages for later processing

Azure queue mail service uses Azure queues.
Blocking just blocks until all the work is done -- This is
how emailing works today

* Provide mail queue service to DI

* Queue organization invite emails for later processing

All emails can later be added to this queue

* Create Admin hosted service to process enqueued mail messages

* Prefer constructors to static generators

* Mass delete organization users (#1287)

* Add delete many to Organization Users

* Correct formatting

* Remove erroneous migration

* Clarify parameter name

* Formatting fixes

* Simplify bump account revision sproc

* Formatting fixes

* Match file names to objects

* Indicate if large import is expected

* Early pull all existing users we were planning on inviting (#1290)

* Early pull all existing users we were planning on inviting

* Improve sproc name

* Batch upsert org users (#1289)

* Add UpsertMany sprocs to OrganizationUser

* Add method to create TVPs from any object.

Uses DbOrder attribute to generate.
Sproc will fail unless TVP column order matches that of the db type

* Combine migrations

* Correct formatting

* Include sql objects in sql project

* Keep consisten parameter names

* Batch deletes for performance

* Correct formatting

* consolidate migrations

* Use batch methods in OrganizationImport

* Declare @BatchSize

* Transaction names limited to 32 chars

Drop sproc before creating it if it exists

* Update import tests

* Allow for more users in org upgrades

* Fix formatting

* Improve class hierarchy structure

* Use name tuple types

* Fix formatting

* Front load all reflection

* Format constructor

* Simplify ToTvp as class-specific extension

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>
This commit is contained in:
Matt Gibson
2021-05-17 09:43:02 -05:00
committed by GitHub
parent 738a4c2bac
commit 785e788cb6
64 changed files with 1704 additions and 234 deletions

View File

@ -0,0 +1,235 @@
IF EXISTS (
SELECT *
FROM INFORMATION_SCHEMA.COLUMNS
WHERE COLUMN_NAME = 'Seats' AND
DATA_TYPE = 'smallint' AND
TABLE_NAME = 'Organization')
BEGIN
ALTER TABLE [dbo].[Organization]
ALTER COLUMN [Seats] INT NULL
END
GO
IF OBJECT_ID('[dbo].[Organization_Create]') IS NOT NULL
BEGIN
DROP PROCEDURE [dbo].[Organization_Create]
END
GO
CREATE PROCEDURE [dbo].[Organization_Create]
@Id UNIQUEIDENTIFIER,
@Identifier NVARCHAR(50),
@Name NVARCHAR(50),
@BusinessName NVARCHAR(50),
@BusinessAddress1 NVARCHAR(50),
@BusinessAddress2 NVARCHAR(50),
@BusinessAddress3 NVARCHAR(50),
@BusinessCountry VARCHAR(2),
@BusinessTaxNumber NVARCHAR(30),
@BillingEmail NVARCHAR(256),
@Plan NVARCHAR(50),
@PlanType TINYINT,
@Seats INT,
@MaxCollections SMALLINT,
@UsePolicies BIT,
@UseSso BIT,
@UseGroups BIT,
@UseDirectory BIT,
@UseEvents BIT,
@UseTotp BIT,
@Use2fa BIT,
@UseApi BIT,
@SelfHost BIT,
@UsersGetPremium BIT,
@Storage BIGINT,
@MaxStorageGb SMALLINT,
@Gateway TINYINT,
@GatewayCustomerId VARCHAR(50),
@GatewaySubscriptionId VARCHAR(50),
@ReferenceData VARCHAR(MAX),
@Enabled BIT,
@LicenseKey VARCHAR(100),
@ApiKey VARCHAR(30),
@TwoFactorProviders NVARCHAR(MAX),
@ExpirationDate DATETIME2(7),
@CreationDate DATETIME2(7),
@RevisionDate DATETIME2(7)
AS
BEGIN
SET NOCOUNT ON
INSERT INTO [dbo].[Organization]
(
[Id],
[Identifier],
[Name],
[BusinessName],
[BusinessAddress1],
[BusinessAddress2],
[BusinessAddress3],
[BusinessCountry],
[BusinessTaxNumber],
[BillingEmail],
[Plan],
[PlanType],
[Seats],
[MaxCollections],
[UsePolicies],
[UseSso],
[UseGroups],
[UseDirectory],
[UseEvents],
[UseTotp],
[Use2fa],
[UseApi],
[SelfHost],
[UsersGetPremium],
[Storage],
[MaxStorageGb],
[Gateway],
[GatewayCustomerId],
[GatewaySubscriptionId],
[ReferenceData],
[Enabled],
[LicenseKey],
[ApiKey],
[TwoFactorProviders],
[ExpirationDate],
[CreationDate],
[RevisionDate]
)
VALUES
(
@Id,
@Identifier,
@Name,
@BusinessName,
@BusinessAddress1,
@BusinessAddress2,
@BusinessAddress3,
@BusinessCountry,
@BusinessTaxNumber,
@BillingEmail,
@Plan,
@PlanType,
@Seats,
@MaxCollections,
@UsePolicies,
@UseSso,
@UseGroups,
@UseDirectory,
@UseEvents,
@UseTotp,
@Use2fa,
@UseApi,
@SelfHost,
@UsersGetPremium,
@Storage,
@MaxStorageGb,
@Gateway,
@GatewayCustomerId,
@GatewaySubscriptionId,
@ReferenceData,
@Enabled,
@LicenseKey,
@ApiKey,
@TwoFactorProviders,
@ExpirationDate,
@CreationDate,
@RevisionDate
)
END
GO
-- Recreate procedure Organization_Update
IF OBJECT_ID('[dbo].[Organization_Update]') IS NOT NULL
BEGIN
DROP PROCEDURE [dbo].[Organization_Update]
END
GO
CREATE PROCEDURE [dbo].[Organization_Update]
@Id UNIQUEIDENTIFIER,
@Identifier NVARCHAR(50),
@Name NVARCHAR(50),
@BusinessName NVARCHAR(50),
@BusinessAddress1 NVARCHAR(50),
@BusinessAddress2 NVARCHAR(50),
@BusinessAddress3 NVARCHAR(50),
@BusinessCountry VARCHAR(2),
@BusinessTaxNumber NVARCHAR(30),
@BillingEmail NVARCHAR(256),
@Plan NVARCHAR(50),
@PlanType TINYINT,
@Seats INT,
@MaxCollections SMALLINT,
@UsePolicies BIT,
@UseSso BIT,
@UseGroups BIT,
@UseDirectory BIT,
@UseEvents BIT,
@UseTotp BIT,
@Use2fa BIT,
@UseApi BIT,
@SelfHost BIT,
@UsersGetPremium BIT,
@Storage BIGINT,
@MaxStorageGb SMALLINT,
@Gateway TINYINT,
@GatewayCustomerId VARCHAR(50),
@GatewaySubscriptionId VARCHAR(50),
@ReferenceData VARCHAR(MAX),
@Enabled BIT,
@LicenseKey VARCHAR(100),
@ApiKey VARCHAR(30),
@TwoFactorProviders NVARCHAR(MAX),
@ExpirationDate DATETIME2(7),
@CreationDate DATETIME2(7),
@RevisionDate DATETIME2(7)
AS
BEGIN
SET NOCOUNT ON
UPDATE
[dbo].[Organization]
SET
[Identifier] = @Identifier,
[Name] = @Name,
[BusinessName] = @BusinessName,
[BusinessAddress1] = @BusinessAddress1,
[BusinessAddress2] = @BusinessAddress2,
[BusinessAddress3] = @BusinessAddress3,
[BusinessCountry] = @BusinessCountry,
[BusinessTaxNumber] = @BusinessTaxNumber,
[BillingEmail] = @BillingEmail,
[Plan] = @Plan,
[PlanType] = @PlanType,
[Seats] = @Seats,
[MaxCollections] = @MaxCollections,
[UsePolicies] = @UsePolicies,
[UseSso] = @UseSso,
[UseGroups] = @UseGroups,
[UseDirectory] = @UseDirectory,
[UseEvents] = @UseEvents,
[UseTotp] = @UseTotp,
[Use2fa] = @Use2fa,
[UseApi] = @UseApi,
[SelfHost] = @SelfHost,
[UsersGetPremium] = @UsersGetPremium,
[Storage] = @Storage,
[MaxStorageGb] = @MaxStorageGb,
[Gateway] = @Gateway,
[GatewayCustomerId] = @GatewayCustomerId,
[GatewaySubscriptionId] = @GatewaySubscriptionId,
[ReferenceData] = @ReferenceData,
[Enabled] = @Enabled,
[LicenseKey] = @LicenseKey,
[ApiKey] = @ApiKey,
[TwoFactorProviders] = @TwoFactorProviders,
[ExpirationDate] = @ExpirationDate,
[CreationDate] = @CreationDate,
[RevisionDate] = @RevisionDate
WHERE
[Id] = @Id
END
GO

View File

@ -0,0 +1,183 @@
-- Create sproc to bump the revision date of a batch of users
IF OBJECT_ID('[dbo].[User_BumpAccountRevisionDateByOrganizationUserIds]') IS NOT NULL
BEGIN
DROP PROCEDURE [dbo].[User_BumpAccountRevisionDateByOrganizationUserIds]
END
GO
CREATE PROCEDURE [dbo].[User_BumpAccountRevisionDateByOrganizationUserIds]
@OrganizationUserIds [dbo].[GuidIdArray] READONLY
AS
BEGIN
SET NOCOUNT ON
SELECT
OU.UserId
INTO
#UserIds
FROM
[dbo].[OrganizationUser] OU
INNER JOIN
@OrganizationUserIds OUIds ON OUIds.Id = OU.Id
WHERE
OU.[Status] = 2 -- Confirmed
UPDATE
U
SET
U.[AccountRevisionDate] = GETUTCDATE()
FROM
[dbo].[User] U
INNER JOIN
#UserIds ON U.[Id] = #UserIds.[UserId]
END
GO
-- Create TwoGuidIdArray Type
IF NOT EXISTS (
SELECT
*
FROM
sys.types
WHERE
[Name] = 'TwoGuidIdArray' AND
is_user_defined = 1
)
CREATE TYPE [dbo].[TwoGuidIdArray] AS TABLE (
[Id1] UNIQUEIDENTIFIER NOT NULL,
[Id2] UNIQUEIDENTIFIER NOT NULL);
GO
-- Create sproc to delete batch of users
-- Parameter Ids are UserId, OrganizationId
IF OBJECT_ID('[dbo].[SsoUser_DeleteMany]') IS NOT NULL
BEGIN
DROP PROCEDURE [dbo].[SsoUser_DeleteMany]
END
GO
CREATE PROCEDURE [dbo].[SsoUser_DeleteMany]
@UserAndOrganizationIds [dbo].[TwoGuidIdArray] READONLY
AS
BEGIN
SET NOCOUNT ON
SELECT
Id
INTO
#SSOIds
FROM
[dbo].[SsoUser] SU
INNER JOIN
@UserAndOrganizationIds UOI ON UOI.Id1 = SU.UserId AND UOI.Id2 = SU.OrganizationId
DECLARE @BatchSize INT = 100
-- Delete SSO Users
WHILE @BatchSize > 0
BEGIN
BEGIN TRANSACTION SsoUser_DeleteMany_SsoUsers
DELETE TOP(@BatchSize) SU
FROM
[dbo].[SsoUser] SU
INNER JOIN
#SSOIDs ON #SSOIds.Id = SU.Id
SET @BatchSize = @@ROWCOUNT
COMMIT TRANSACTION SsoUser_DeleteMany_SsoUsers
END
END
GO
-- Create OrganizationUser Delete many by Id procedure
IF OBJECT_ID('[dbo].[OrganizationUser_DeleteByIds]') IS NOT NULL
BEGIN
DROP PROCEDURE [dbo].[OrganizationUser_DeleteByIds]
END
GO
CREATE PROCEDURE [dbo].[OrganizationUser_DeleteByIds]
@Ids [dbo].[GuidIdArray] READONLY
AS
BEGIN
SET NOCOUNT ON
EXEC [dbo].[User_BumpAccountRevisionDateByOrganizationUserIds] @Ids
DECLARE @UserAndOrganizationIds [dbo].[TwoGuidIdArray]
INSERT INTO @UserAndOrganizationIds
(Id1, Id2)
SELECT
UserId,
OrganizationId
FROM
[dbo].[OrganizationUser] OU
INNER JOIN
@Ids OUIds ON OUIds.Id = OU.Id
WHERE
UserId IS NOT NULL AND
OrganizationId IS NOT NULL
BEGIN
EXEC [dbo].[SsoUser_DeleteMany] @UserAndOrganizationIds
END
DECLARE @BatchSize INT = 100
-- Delete CollectionUsers
WHILE @BatchSize > 0
BEGIN
BEGIN TRANSACTION CollectionUser_DeleteMany_CUs
DELETE TOP(@BatchSize) CU
FROM
[dbo].[CollectionUser] CU
INNER JOIN
@Ids I ON I.Id = CU.OrganizationUserId
SET @BatchSize = @@ROWCOUNT
COMMIT TRANSACTION CollectionUser_DeleteMany_CUs
END
SET @BatchSize = 100;
-- Delete GroupUsers
WHILE @BatchSize > 0
BEGIN
BEGIN TRANSACTION GroupUser_DeleteMany_GroupUsers
DELETE TOP(@BatchSize) GU
FROM
[dbo].[GroupUser] GU
INNER JOIN
@Ids I ON I.Id = GU.OrganizationUserId
SET @BatchSize = @@ROWCOUNT
COMMIT TRANSACTION GoupUser_DeleteMany_GroupUsers
END
SET @BatchSize = 100;
-- Delete OrganizationUsers
WHILE @BatchSize > 0
BEGIN
BEGIN TRANSACTION OrganizationUser_DeleteMany_OUs
DELETE TOP(@BatchSize) OU
FROM
[dbo].[OrganizationUser] OU
INNER JOIN
@Ids I ON I.Id = OU.Id
SET @BatchSize = @@ROWCOUNT
COMMIT TRANSACTION OrganizationUser_DeleteMany_OUs
END
END
GO

View File

@ -0,0 +1,142 @@
-- Create OrganizationUser Type
IF NOT EXISTS (
SELECT
*
FROM
sys.types
WHERE
[Name] = 'OrganizationUserType' AND
is_user_defined = 1
)
BEGIN
CREATE TYPE [dbo].[OrganizationUserType] AS TABLE(
[Id] UNIQUEIDENTIFIER,
[OrganizationId] UNIQUEIDENTIFIER,
[UserId] UNIQUEIDENTIFIER,
[Email] NVARCHAR(256),
[Key] VARCHAR(MAX),
[Status] TINYINT,
[Type] TINYINT,
[AccessAll] BIT,
[ExternalId] NVARCHAR(300),
[CreationDate] DATETIME2(7),
[RevisionDate] DATETIME2(7),
[Permissions] NVARCHAR(MAX),
[ResetPasswordKey] VARCHAR(MAX)
)
END
GO
-- Create many sproc
IF OBJECT_ID('[dbo].[OrganizationUser_CreateMany]') IS NOT NULL
BEGIN
DROP PROCEDURE [dbo].[OrganizationUser_CreateMany]
END
GO
CREATE PROCEDURE [dbo].[OrganizationUser_CreateMany]
@OrganizationUsersInput [dbo].[OrganizationUserType] READONLY
AS
BEGIN
SET NOCOUNT ON
INSERT INTO [dbo].[OrganizationUser]
(
[Id],
[OrganizationId],
[UserId],
[Email],
[Key],
[Status],
[Type],
[AccessAll],
[ExternalId],
[CreationDate],
[RevisionDate],
[Permissions],
[ResetPasswordKey]
)
SELECT
OU.[Id],
OU.[OrganizationId],
OU.[UserId],
OU.[Email],
OU.[Key],
OU.[Status],
OU.[Type],
OU.[AccessAll],
OU.[ExternalId],
OU.[CreationDate],
OU.[RevisionDate],
OU.[Permissions],
OU.[ResetPasswordKey]
FROM
@OrganizationUsersInput OU
END
GO
-- Bump many user account revision dates
IF OBJECT_ID('[dbo].[User_BumpManyAccountRevisionDates]') IS NOT NULL
BEGIN
DROP PROCEDURE [dbo].[User_BumpManyAccountRevisionDates]
END
GO
CREATE PROCEDURE [dbo].[User_BumpManyAccountRevisionDates]
@Ids [dbo].[GuidIdArray] READONLY
AS
BEGIN
SET NOCOUNT ON
UPDATE
U
SET
[AccountRevisionDate] = GETUTCDATE()
FROM
[dbo].[User] U
INNER JOIN
@Ids IDs ON IDs.Id = U.Id
END
GO
-- Update many OrganizationUsers
IF OBJECT_ID('[dbo].[OrganizationUser_UpdateMany]') IS NOT NULL
BEGIN
DROP PROCEDURE [dbo].[OrganizationUser_UpdateMany]
END
GO
CREATE PROCEDURE [dbo].[OrganizationUser_UpdateMany]
@OrganizationUsersInput [dbo].[OrganizationUserType] READONLY
AS
BEGIN
SET NOCOUNT ON
UPDATE
OU
SET
[OrganizationId] = OUI.[OrganizationId],
[UserId] = OUI.[UserId],
[Email] = OUI.[Email],
[Key] = OUI.[Key],
[Status] = OUI.[Status],
[Type] = OUI.[Type],
[AccessAll] = OUI.[AccessAll],
[ExternalId] = OUI.[ExternalId],
[CreationDate] = OUI.[CreationDate],
[RevisionDate] = OUI.[RevisionDate],
[Permissions] = OUI.[Permissions],
[ResetPasswordKey] = OUI.[ResetPasswordKey]
FROM
[dbo].[OrganizationUser] OU
INNER JOIN
@OrganizationUsersInput OUI ON OU.Id = OUI.Id
EXEC [dbo].[User_BumpManyAccountRevisionDates]
(
SELECT UserId
FROM @OrganizationUsersInput
)
END
GO

View File

@ -0,0 +1,49 @@
-- Create EmailArray type
IF NOT EXISTS (
SELECT *
FROM sys.types
WHERE [Name] = 'EmailArray'
AND is_user_defined = 1
)
CREATE TYPE [dbo].[EmailArray] AS TABLE (
[Email] NVARCHAR(256) NOT NULL);
GO
IF OBJECT_ID('[dbo].[OrganizationUser_SelectKnownEmails]') IS NOT NULL
BEGIN
DROP PROCEDURE [dbo].[OrganizationUser_SelectKnownEmails]
END
GO
-- Create sproc to return existing users
CREATE PROCEDURE [dbo].[OrganizationUser_SelectKnownEmails]
@OrganizationId UNIQUEIDENTIFIER,
@Emails [dbo].[EmailArray] READONLY,
@OnlyUsers BIT
AS
BEGIN
SET NOCOUNT ON
SELECT
E.Email
FROM
@Emails E
INNER JOIN
(
SELECT
U.[Email] as 'UEmail',
OU.[Email] as 'OUEmail',
OU.OrganizationId
FROM
[dbo].[User] U
RIGHT JOIN
[dbo].[OrganizationUser] OU ON OU.[UserId] = U.[Id]
WHERE
OU.OrganizationId = @OrganizationId
) OUU ON OUU.[UEmail] = E.[Email] OR OUU.[OUEmail] = E.[Email]
WHERE
(@OnlyUsers = 0 AND (OUU.UEmail IS NOT NULL OR OUU.OUEmail IS NOT NULL)) OR
(@OnlyUsers = 1 AND (OUU.UEmail IS NOT NULL))
END
GO