[BEEEP] Integration tests (#1945)

* Add api integration tests

* Add some stuff

* Make program mockable

* Work on IntegrationTests for Identity

* Formatting

* Update packages.lock.json

* Update more packages.lock.json

* Update all packages.lock.json

* Fix InMemory configuration

* Actually fix test configuration

* Fix tests for CI

* Fix event service

* Force EF EventRepository

* Add client_credentials test

* Remove Api.IntegrationTest

* Remove Api Program changes

* Cleanup

* Add more Auth-Email tests

* Run formatting

* Address some PR feedback

* Move integration stuff to it's own common project

* Ran linter

* Add shared project to test solution

* Remove sln changes

* Clean usings

* Add more coverage

* Address PR feedback

test/IntegrationTestCommon/Factories/IdentityApplicationFactory.cs

@@ -0,0 +1,49 @@
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Net.Http.Json;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Bit.Core.Enums;
+using Bit.Core.Models.Api.Request.Accounts;
+using Bit.Core.Utilities;
+using Bit.Identity;
+using Bit.Test.Common.Helpers;
+using Microsoft.AspNetCore.Http;
+
+namespace Bit.IntegrationTestCommon.Factories
+{
+    public class IdentityApplicationFactory : WebApplicationFactoryBase<Startup>
+    {
+        public const string DefaultDeviceIdentifier = "92b9d953-b9b6-4eaf-9d3e-11d57144dfeb";
+
+        public async Task<HttpContext> RegisterAsync(RegisterRequestModel model)
+        {
+            return await Server.PostAsync("/accounts/register", JsonContent.Create(model));
+        }
+
+        public async Task<(string Token, string RefreshToken)> TokenFromPasswordAsync(string username,
+            string password,
+            string deviceIdentifier = DefaultDeviceIdentifier,
+            string clientId = "web",
+            DeviceType deviceType = DeviceType.FirefoxBrowser,
+            string deviceName = "firefox")
+        {
+            var context = await Server.PostAsync("/connect/token", new FormUrlEncodedContent(new Dictionary<string, string>
+            {
+                { "scope", "api offline_access" },
+                { "client_id", clientId },
+                { "deviceType", ((int)deviceType).ToString() },
+                { "deviceIdentifier", deviceIdentifier },
+                { "deviceName", deviceName },
+                { "grant_type", "password" },
+                { "username", username },
+                { "password", password },
+            }), context => context.Request.Headers.Add("Auth-Email", CoreHelpers.Base64UrlEncodeString(username)));
+
+            using var body = await AssertHelper.AssertResponseTypeIs<JsonDocument>(context);
+            var root = body.RootElement;
+
+            return (root.GetProperty("access_token").GetString(), root.GetProperty("refresh_token").GetString());
+        }
+    }
+}

test/IntegrationTestCommon/Factories/WebApplicationFactoryBase.cs

@@ -0,0 +1,102 @@
+using System.Collections.Generic;
+using System.Linq;
+using AspNetCoreRateLimit;
+using Bit.Core.Repositories;
+using Bit.Core.Services;
+using Bit.Infrastructure.EntityFramework.Repositories;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.AspNetCore.TestHost;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Bit.IntegrationTestCommon.Factories
+{
+    public static class FactoryConstants
+    {
+        public const string DefaultDatabaseName = "test_database";
+        public const string WhitelistedIp = "1.1.1.1";
+    }
+
+    public abstract class WebApplicationFactoryBase<T> : WebApplicationFactory<T>
+        where T : class
+    {
+        /// <summary>
+        /// The database name to use for this instance of the factory. By default it will use a shared database name so all instances will connect to the same database during it's lifetime.
+        /// </summary>
+        /// <remarks>
+        /// This will need to be set BEFORE using the <c>Server</c> property
+        /// </remarks>
+        public string DatabaseName { get; set; } = FactoryConstants.DefaultDatabaseName;
+
+        /// <summary>
+        /// Configure the web host to use an EF in memory database
+        /// </summary>
+        protected override void ConfigureWebHost(IWebHostBuilder builder)
+        {
+            builder.ConfigureAppConfiguration(c =>
+            {
+                c.AddInMemoryCollection(new Dictionary<string, string>
+                {
+                    // Manually insert a EF provider so that ConfigureServices will add EF repositories but we will override
+                    // DbContextOptions to use an in memory database
+                    { "globalSettings:databaseProvider", "postgres" },
+                    { "globalSettings:postgreSql:connectionString", "Host=localhost;Username=test;Password=test;Database=test" },
+                });
+            });
+
+            builder.ConfigureTestServices(services =>
+            {
+                var dbContextOptions = services.First(sd => sd.ServiceType == typeof(DbContextOptions<DatabaseContext>));
+                services.Remove(dbContextOptions);
+                services.AddScoped(_ =>
+                {
+                    return new DbContextOptionsBuilder<DatabaseContext>()
+                        .UseInMemoryDatabase(DatabaseName)
+                        .Options;
+                });
+
+                // QUESTION: The normal licensing service should run fine on developer machines but not in CI
+                // should we have a fork here to leave the normal service for developers?
+                // TODO: Eventually add the license file to CI
+                var licensingService = services.First(sd => sd.ServiceType == typeof(ILicensingService));
+                services.Remove(licensingService);
+                services.AddSingleton<ILicensingService, NoopLicensingService>();
+
+                // FUTURE CONSIDERATION: Add way to run this self hosted/cloud, for now it is cloud only
+                var pushRegistrationService = services.First(sd => sd.ServiceType == typeof(IPushRegistrationService));
+                services.Remove(pushRegistrationService);
+                services.AddSingleton<IPushRegistrationService, NoopPushRegistrationService>();
+
+                // Even though we are cloud we currently set this up as cloud, we can use the EF/selfhosted service
+                // instead of using Noop for this service
+                // TODO: Install and use azurite in CI pipeline
+                var eventWriteService = services.First(sd => sd.ServiceType == typeof(IEventWriteService));
+                services.Remove(eventWriteService);
+                services.AddSingleton<IEventWriteService, RepositoryEventWriteService>();
+
+                var eventRepositoryService = services.First(sd => sd.ServiceType == typeof(IEventRepository));
+                services.Remove(eventRepositoryService);
+                services.AddSingleton<IEventRepository, EventRepository>();
+
+                // Our Rate limiter works so well that it begins to fail tests unless we carve out
+                // one whitelisted ip. We should still test the rate limiter though and they should change the Ip 
+                // to something that is NOT whitelisted
+                services.Configure<IpRateLimitOptions>(options =>
+                {
+                    options.IpWhitelist = new List<string>
+                    {
+                        FactoryConstants.WhitelistedIp,
+                    };
+                });
+            });
+        }
+
+        public DatabaseContext GetDatabaseContext()
+        {
+            var scope = Services.CreateScope();
+            return scope.ServiceProvider.GetRequiredService<DatabaseContext>();
+        }
+    }
+}

test/IntegrationTestCommon/Factories/WebApplicationFactoryExtensions.cs

@@ -0,0 +1,59 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.TestHost;
+using Microsoft.Extensions.Primitives;
+
+namespace Bit.IntegrationTestCommon.Factories
+{
+    public static class WebApplicationFactoryExtensions
+    {
+        private static async Task<HttpContext> SendAsync(this TestServer server,
+            HttpMethod method,
+            string requestUri,
+            HttpContent content = null,
+            Action<HttpContext> extraConfiguration = null)
+        {
+            return await server.SendAsync(httpContext =>
+            {
+                // Automatically set the whitelisted IP so normal tests do not run into rate limit issues
+                // to test rate limiter, use the extraConfiguration parameter to set Connection.RemoteIpAddress
+                // it runs after this so it will take precedence.
+                httpContext.Connection.RemoteIpAddress = IPAddress.Parse(FactoryConstants.WhitelistedIp);
+
+                httpContext.Request.Path = new PathString(requestUri);
+                httpContext.Request.Method = method.Method;
+
+                if (content != null)
+                {
+                    foreach (var header in content.Headers)
+                    {
+                        httpContext.Request.Headers.Add(header.Key, new StringValues(header.Value.ToArray()));
+                    }
+
+                    httpContext.Request.Body = content.ReadAsStream();
+                }
+
+                extraConfiguration?.Invoke(httpContext);
+            });
+        }
+        public static Task<HttpContext> PostAsync(this TestServer server,
+            string requestUri,
+            HttpContent content,
+            Action<HttpContext> extraConfiguration = null)
+            => SendAsync(server, HttpMethod.Post, requestUri, content, extraConfiguration);
+        public static Task<HttpContext> GetAsync(this TestServer server,
+            string requestUri,
+            Action<HttpContext> extraConfiguration = null)
+            => SendAsync(server, HttpMethod.Get, requestUri, content: null, extraConfiguration);
+        public static async Task<string> ReadBodyAsStringAsync(this HttpContext context)
+        {
+            using var sr = new StreamReader(context.Response.Body);
+            return await sr.ReadToEndAsync();
+        }
+    }
+}

test/IntegrationTestCommon/IntegrationTestCommon.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+  
+  <ItemGroup>
+    <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="5.0.15" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="5.0.15" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
+  </ItemGroup>
+    
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Identity\Identity.csproj" />
+    <ProjectReference Include="..\Common\Common.csproj" />
+  </ItemGroup>
+</Project>