Postgres & MySql Support For Self-Hosted Installations (#1386)

* EF Database Support Init (#1221)

* scaffolding for ef support

* deleted old postgres repos

* added tables to oncreate

* updated all the things to .NET 5

* Addition to #1221: Migrated DockerFiles from dotnet/3.1 to  5.0 (#1223)

* Migrated DockerFiles from dotnet/3.1 to  5.0

* Migrated SSO/Dockerfile from dotnet 3.1 to 5.0

Co-authored-by: Daniel James Smith <djsmith85@users.noreply.github.com>

* EFDatabaseSupport: Updated links and description in README.md and SETUP.md (#1232)

* Updated requirements in README.md

* Updated link to documentation of app-secrets

* upgraded dotnet version to 5.0

* Ef database support implementation examples (#1265)

* mostly finished testing the user repo

* finished testing user repo

* finished org, user, ssoconfig, and ssouser ef implementations

* removed unused prop

* fixed a sql file

* fixed a spacing issue

* fixed a spacing issue

* removed extra database creation

* refactoring

* MsSql => SqlServer

* refactoring

* code review fixes

* build fix

* code review

* continued attempts to fix the the build

* skipped another test

* finished all create test

* initial pass at several repos

* continued building out repos

* initial pass at several repos

* initial pass at device repo

* initial pass at collection repo

* initial run of all Entity Framework implementations

* signup, signin, create/edit ciphers works

* sync working

* all web vault pages seem to load with 100% 200s

* bulkcopy, folders, and favorites

* group and collection management

* sso, groups, emergency access, send

* get basic creates matching on all repos

* got everything building again post merge

* removed some IDE config files

* cleanup

* no more notimplemented methods in the cipher repo

* no more not implementeds everywhere

* cleaned up schema/navigation properties and fixed tests

* removed a sql comment that was written in c# style

* fixed build issues from merge

* removed unsupported db providers

* formatting

* code review refactors

* naming cleanup for queries

* added provider methods

* cipher repo cleanup

* implemented several missing procedures from the EF implementation surround account revision dates, keys, and storage

* fixed the build

* added a null check

* consolidated some cipher repo methods

* formatting fix

* cleaned up indentation of queries

* removed .idea file

* generated postgres migrations

* added mysql migrations

* formatting

* Bug Fixes & Formatting

* Formatting

* fixed a bug with bulk import when using MySql

* code review fixes

* fixed the build

* implemented new methods

* formatting

* fixed the build

* cleaned up select statements in ef queries

* formatting

* formatting

* formatting

Co-authored-by: Daniel James Smith <djsmith85@users.noreply.github.com>

src/Core/Models/EntityFramework/Cipher.cs

@@ -1,57 +1,14 @@
-using System.Text.Json;
+using System.Collections.Generic;
+using System.Text.Json;
 using AutoMapper;
 
 namespace Bit.Core.Models.EntityFramework
 {
     public class Cipher : Table.Cipher
     {
-        private JsonDocument _dataJson;
-        private JsonDocument _attachmentsJson;
-        private JsonDocument _favoritesJson;
-        private JsonDocument _foldersJson;
-
-        public User User { get; set; }
-        public Organization Organization { get; set; }
-        [IgnoreMap]
-        public JsonDocument DataJson
-        {
-            get => _dataJson;
-            set
-            {
-                Data = value?.ToString();
-                _dataJson = value;
-            }
-        }
-        [IgnoreMap]
-        public JsonDocument AttachmentsJson
-        {
-            get => _attachmentsJson;
-            set
-            {
-                Attachments = value?.ToString();
-                _attachmentsJson = value;
-            }
-        }
-        [IgnoreMap]
-        public JsonDocument FavoritesJson
-        {
-            get => _favoritesJson;
-            set
-            {
-                Favorites = value?.ToString();
-                _favoritesJson = value;
-            }
-        }
-        [IgnoreMap]
-        public JsonDocument FoldersJson
-        {
-            get => _foldersJson;
-            set
-            {
-                Folders = value?.ToString();
-                _foldersJson = value;
-            }
-        }
+        public virtual User User { get; set; }
+        public virtual Organization Organization { get; set; }
+        public virtual ICollection<CollectionCipher> CollectionCiphers { get; set; }
     }
 
     public class CipherMapperProfile : Profile

src/Core/Models/EntityFramework/Collection.cs

@@ -0,0 +1,22 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class Collection : Table.Collection
+    {
+        public virtual Organization Organization { get; set; }
+        public virtual ICollection<CollectionUser> CollectionUsers { get; set; }
+        public virtual ICollection<CollectionCipher> CollectionCiphers { get; set; }
+        public virtual ICollection<CollectionGroup> CollectionGroups { get; set; }
+    }
+
+    public class CollectionMapperProfile : Profile
+    {
+        public CollectionMapperProfile()
+        {
+            CreateMap<Table.Collection, Collection>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/CollectionCipher.cs

@@ -0,0 +1,20 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class CollectionCipher : Table.CollectionCipher
+    {
+        public virtual Cipher Cipher { get; set; }
+        public virtual Collection Collection { get; set; }
+    }
+
+    public class CollectionCipherMapperProfile : Profile
+    {
+        public CollectionCipherMapperProfile()
+        {
+            CreateMap<Table.CollectionCipher, CollectionCipher>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/CollectionGroup.cs

@@ -0,0 +1,18 @@
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class CollectionGroup : Table.CollectionGroup
+    {
+        public virtual Collection Collection { get; set; }
+        public virtual Group Group { get; set; }
+    }
+
+    public class CollectionGroupMapperProfile : Profile
+    {
+        public CollectionGroupMapperProfile()
+        {
+            CreateMap<Table.CollectionGroup, CollectionGroup>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/CollectionUser.cs

@@ -0,0 +1,18 @@
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class CollectionUser : Table.CollectionUser
+    {
+        public virtual Collection Collection { get; set; }
+        public virtual OrganizationUser OrganizationUser { get; set; }
+    }
+
+    public class CollectionUserMapperProfile : Profile
+    {
+        public CollectionUserMapperProfile()
+        {
+            CreateMap<Table.CollectionUser, CollectionUser>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Device.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class Device : Table.Device
+    {
+        public virtual User User { get; set; }
+    }
+
+    public class DeviceMapperProfile : Profile
+    {
+        public DeviceMapperProfile()
+        {
+            CreateMap<Table.Device, Device>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/EmergencyAccess.cs

@@ -0,0 +1,20 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class EmergencyAccess : Table.EmergencyAccess
+    {
+        public virtual User Grantee { get; set; }
+        public virtual User Grantor { get; set; }
+    }
+
+    public class EmergencyAccessMapperProfile : Profile
+    {
+        public EmergencyAccessMapperProfile()
+        {
+            CreateMap<Table.EmergencyAccess, EmergencyAccess>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Event.cs

@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class Event : Table.Event
+    {
+    }
+
+    public class EventMapperProfile : Profile
+    {
+        public EventMapperProfile()
+        {
+            CreateMap<Table.Event, Event>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Folder.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class Folder : Table.Folder
+    {
+        public virtual User User { get; set; }
+    }
+
+    public class FolderMapperProfile : Profile
+    {
+        public FolderMapperProfile()
+        {
+            CreateMap<Table.Folder, Folder>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Grant.cs

@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class Grant : Table.Grant
+    {
+    }
+
+    public class GrantMapperProfile : Profile
+    {
+        public GrantMapperProfile()
+        {
+            CreateMap<Table.Grant, Grant>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Group.cs

@@ -0,0 +1,20 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class Group : Table.Group
+    {
+        public virtual Organization Organization { get; set; }
+        public virtual ICollection<GroupUser> GroupUsers { get; set; }
+    }
+
+    public class GroupMapperProfile : Profile
+    {
+        public GroupMapperProfile()
+        {
+            CreateMap<Table.Group, Group>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/GroupUser.cs

@@ -0,0 +1,21 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class GroupUser : Table.GroupUser
+    {
+        public virtual Group Group { get; set; }
+        public virtual OrganizationUser OrganizationUser { get; set; }
+    }
+
+    public class GroupUserMapperProfile : Profile
+    {
+        public GroupUserMapperProfile()
+        {
+            CreateMap<Table.GroupUser, GroupUser>().ReverseMap();
+        }
+    }
+}
+

src/Core/Models/EntityFramework/Installation.cs

@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class Installation : Table.Installation
+    {
+    }
+
+    public class InstallationMapperProfile : Profile
+    {
+        public InstallationMapperProfile()
+        {
+            CreateMap<Table.Installation, Installation>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Organization.cs

@@ -1,25 +1,17 @@
 using System.Collections.Generic;
-using System.Text.Json;
 using AutoMapper;
 
 namespace Bit.Core.Models.EntityFramework
 {
     public class Organization : Table.Organization
     {
-        private JsonDocument _twoFactorProvidersJson;
-
-        public ICollection<Cipher> Ciphers { get; set; }
-        
-        [IgnoreMap]
-        public JsonDocument TwoFactorProvidersJson
-        {
-            get => _twoFactorProvidersJson;
-            set
-            {
-                TwoFactorProviders = value?.ToString();
-                _twoFactorProvidersJson = value;
-            }
-        }
+        public virtual ICollection<Cipher> Ciphers { get; set; }
+        public virtual ICollection<OrganizationUser> OrganizationUsers { get; set; }
+        public virtual ICollection<Group> Groups { get; set; }
+        public virtual ICollection<Policy> Policies { get; set; }
+        public virtual ICollection<SsoConfig> SsoConfigs { get; set; }
+        public virtual ICollection<SsoUser> SsoUsers { get; set; }
+        public virtual ICollection<Transaction> Transactions { get; set; }
     }
 
     public class OrganizationMapperProfile : Profile

src/Core/Models/EntityFramework/OrganizationUser.cs

@@ -0,0 +1,21 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class OrganizationUser : Table.OrganizationUser
+    {
+        public virtual Organization Organization { get; set; }
+        public virtual User User { get; set; }
+        public virtual ICollection<CollectionUser> CollectionUsers { get; set; }
+    }
+
+    public class OrganizationUserMapperProfile : Profile
+    {
+        public OrganizationUserMapperProfile()
+        {
+            CreateMap<Table.OrganizationUser, OrganizationUser>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Policy.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class Policy : Table.Policy
+    {
+        public virtual Organization Organization { get; set; }
+    }
+
+    public class PolicyMapperProfile : Profile
+    {
+        public PolicyMapperProfile()
+        {
+            CreateMap<Table.Policy, Policy>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Provider/Provider.cs

@@ -0,0 +1,16 @@
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework.Provider
+{
+    public class Provider : Table.Provider.Provider
+    {
+    }
+
+    public class ProviderMapperProfile : Profile
+    {
+        public ProviderMapperProfile()
+        {
+            CreateMap<Table.Provider.Provider, Provider>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Provider/ProviderOrganization.cs

@@ -0,0 +1,18 @@
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework.Provider
+{
+    public class ProviderOrganization : Table.Provider.ProviderOrganization
+    {
+        public virtual Provider Provider { get; set; }
+        public virtual Organization Organization { get; set; }
+    }
+
+    public class ProviderOrganizationMapperProfile : Profile
+    {
+        public ProviderOrganizationMapperProfile()
+        {
+            CreateMap<Table.Provider.ProviderOrganization, ProviderOrganization>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Provider/ProviderOrganizationProviderUser.cs

@@ -0,0 +1,18 @@
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework.Provider
+{
+    public class ProviderOrganizationProviderUser : Table.Provider.ProviderOrganizationProviderUser
+    {
+        public virtual ProviderOrganization ProviderOrganization { get; set; }
+        public virtual ProviderUser ProviderUser { get; set; }
+    }
+
+    public class ProviderOrganizationProviderUserMapperProfile : Profile
+    {
+        public ProviderOrganizationProviderUserMapperProfile()
+        {
+            CreateMap<Table.Provider.ProviderOrganizationProviderUser, ProviderOrganizationProviderUser>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Provider/ProviderUser.cs

@@ -0,0 +1,18 @@
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework.Provider
+{
+    public class ProviderUser : Table.Provider.ProviderUser
+    {
+        public virtual User User { get; set; }
+        public virtual Provider Provider { get; set; }
+    }
+
+    public class ProviderUserMapperProfile : Profile
+    {
+        public ProviderUserMapperProfile()
+        {
+            CreateMap<Table.Provider.ProviderUser, ProviderUser>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Role.cs

@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class Role : Table.Role
+    {
+    }
+
+    public class RoleMapperProfile : Profile
+    {
+        public RoleMapperProfile()
+        {
+            CreateMap<Table.Role, Role>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Send.cs

@@ -0,0 +1,20 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class Send : Table.Send
+    {
+        public virtual Organization Organization { get; set; }
+        public virtual User User { get; set; }
+    }
+
+    public class SendMapperProfile : Profile
+    {
+        public SendMapperProfile()
+        {
+            CreateMap<Table.Send, Send>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/SsoConfig.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class SsoConfig : Table.SsoConfig
+    {
+        public virtual Organization Organization { get; set; }
+    }
+
+    public class SsoConfigMapperProfile : Profile
+    {
+        public SsoConfigMapperProfile()
+        {
+            CreateMap<Table.SsoConfig, SsoConfig>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/SsoUser.cs

@@ -0,0 +1,20 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class SsoUser : Table.SsoUser
+    {
+        public virtual Organization Organization { get; set; }
+        public virtual User User { get; set; }
+    }
+
+    public class SsoUserMapperProfile : Profile
+    {
+        public SsoUserMapperProfile()
+        {
+            CreateMap<Table.SsoUser, SsoUser>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/TaxRate.cs

@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class TaxRate : Table.TaxRate
+    {
+    }
+
+    public class TaxRateMapperProfile : Profile
+    {
+        public TaxRateMapperProfile()
+        {
+            CreateMap<Table.TaxRate, TaxRate>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/Transaction.cs

@@ -0,0 +1,20 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class Transaction : Table.Transaction
+    {
+        public virtual Organization Organization { get; set; }
+        public virtual User User { get; set; }
+    }
+
+    public class TransactionMapperProfile : Profile
+    {
+        public TransactionMapperProfile()
+        {
+            CreateMap<Table.Transaction, Transaction>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/U2f.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+using System.Text.Json;
+using AutoMapper;
+
+namespace Bit.Core.Models.EntityFramework
+{
+    public class U2f : Table.U2f
+    {
+        public virtual User User { get; set; }
+    }
+
+    public class U2fMapperProfile : Profile
+    {
+        public U2fMapperProfile()
+        {
+            CreateMap<Table.U2f, U2f>().ReverseMap();
+        }
+    }
+}

src/Core/Models/EntityFramework/User.cs

@@ -6,20 +6,14 @@ namespace Bit.Core.Models.EntityFramework
 {
     public class User : Table.User
     {
-        private JsonDocument _twoFactorProvidersJson;
-
-        public ICollection<Cipher> Ciphers { get; set; }
-
-        [IgnoreMap]
-        public JsonDocument TwoFactorProvidersJson
-        {
-            get => _twoFactorProvidersJson;
-            set
-            {
-                TwoFactorProviders = value?.ToString();
-                _twoFactorProvidersJson = value;
-            }
-        }
+        public virtual ICollection<Cipher> Ciphers { get; set; }
+        public virtual ICollection<Folder> Folders { get; set; }
+        public virtual ICollection<CollectionUser> CollectionUsers { get; set; }
+        public virtual ICollection<GroupUser> GroupUsers { get; set; }
+        public virtual ICollection<OrganizationUser> OrganizationUsers { get; set; }
+        public virtual ICollection<SsoUser> SsoUsers { get; set; }
+        public virtual ICollection<Transaction> Transactions { get; set; }
+        public virtual ICollection<U2f> U2fs { get; set; }
     }
 
     public class UserMapperProfile : Profile