[PS-93] Distributed Ip rate limiting (#2060)

* Upgrade AspNetCoreRateLimiter and enable redis distributed cache for rate limiting. - Upgrades AspNetCoreRateLimiter to 4.0.2, which required updating NewtonSoft.Json to 13.0.1. - Replaces Microsoft.Extensions.Caching.Redis with Microsoft.Extensions.Caching.StackExchangeRedis as the original was deprecated and conflicted with the latest AspNetCoreRateLimiter - Adds startup task to Program.cs for Api/Identity projects to support AspNetCoreRateLimiters breaking changes for seeding its stores. - Adds a Redis connection string option to GlobalSettings Signed-off-by: Shane Melton <smelton@bitwarden.com> * Cleanup Redis distributed cache registration - Add new AddDistributedCache service collection extension to add either a Memory or Redis distributed cache. - Remove distributed cache registration from Identity service collection extension. - Add IpRateLimitSeedStartupService.cs to run at application startup to seed the Ip rate limiting policies. Signed-off-by: Shane Melton <smelton@bitwarden.com> * Add caching configuration to SSO Startup.cs Signed-off-by: Shane Melton <smelton@bitwarden.com> * Add ProjectName as an instance name for Redis options Signed-off-by: Shane Melton <smelton@bitwarden.com> * Use distributed cache in CustomIpRateLimitMiddleware.cs Signed-off-by: Shane Melton <smelton@bitwarden.com> * Undo changes to Program.cs and launchSettings.json * Move new service collection extensions to SharedWeb * Upgrade Caching.StackExchangeRedis package to v6 * Cleanup and fix leftover merge conflicts * Remove use of Newtonsoft.Json in distributed cache extensions * Cleanup more formatting * Fix formatting * Fix startup issue caused by merge and fix integration test Signed-off-by: Shane Melton <smelton@bitwarden.com> * Linting fix Signed-off-by: Shane Melton <smelton@bitwarden.com>
2025-06-30 15:42:48 -05:00 · 2022-07-19 11:58:32 -07:00
parent 1764d2446e
commit 7d40b38352
39 changed files with 2331 additions and 1910 deletions
--- a/src/Core/Core.csproj
+++ b/src/Core/Core.csproj
@ -20,6 +20,7 @@
  </ItemGroup>

  <ItemGroup>
+    <PackageReference Include="AspNetCoreRateLimit.Redis" Version="1.0.1" />
    <PackageReference Include="AWSSDK.SimpleEmail" Version="3.7.0.150" />
    <PackageReference Include="AWSSDK.SQS" Version="3.7.2.47" />
    <PackageReference Include="Azure.Extensions.AspNetCore.DataProtection.Blobs" Version="1.2.1" />
@ -52,7 +53,7 @@
    <PackageReference Include="Stripe.net" Version="39.107.0" />
    <PackageReference Include="Otp.NET" Version="1.2.2" />
    <PackageReference Include="YubicoDotNetClient" Version="1.2.0" />
-    <PackageReference Include="Microsoft.Extensions.Caching.Redis" Version="2.2.0" />
+    <PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="6.0.6" />
  </ItemGroup>

  <ItemGroup>
--- a/src/Core/HostedServices/IpRateLimitSeedStartupService.cs
+++ b/src/Core/HostedServices/IpRateLimitSeedStartupService.cs
@ -0,0 +1,41 @@
+using AspNetCoreRateLimit;
+using Microsoft.Extensions.Hosting;
+
+namespace Bit.Core.HostedServices
+{
+    /// <summary>
+    /// A startup service that will seed the IP rate limiting stores with any values in the
+    /// GlobalSettings configuration.
+    /// </summary>
+    /// <remarks>
+    /// <para>Using an <see cref="IHostedService"/> here because it runs before the request processing pipeline
+    /// is configured, so that any rate limiting configuration is seeded/applied before any requests come in.
+    /// </para>
+    /// <para>
+    /// This is a cleaner alternative to modifying Program.cs in every project that requires rate limiting as
+    /// described/suggested here:
+    /// https://github.com/stefanprodan/AspNetCoreRateLimit/wiki/Version-3.0.0-Breaking-Changes
+    /// </para>
+    /// </remarks>
+    public class IpRateLimitSeedStartupService : IHostedService
+    {
+        private readonly IIpPolicyStore _ipPolicyStore;
+        private readonly IClientPolicyStore _clientPolicyStore;
+
+        public IpRateLimitSeedStartupService(IIpPolicyStore ipPolicyStore, IClientPolicyStore clientPolicyStore)
+        {
+            _ipPolicyStore = ipPolicyStore;
+            _clientPolicyStore = clientPolicyStore;
+        }
+
+        public async Task StartAsync(CancellationToken cancellationToken)
+        {
+            // Seed the policies from GlobalSettings
+            await _ipPolicyStore.SeedAsync();
+            await _clientPolicyStore.SeedAsync();
+        }
+
+        // noop
+        public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+    }
+}
--- a/src/Core/IdentityServer/ConfigureOpenIdConnectDistributedOptions.cs
+++ b/src/Core/IdentityServer/ConfigureOpenIdConnectDistributedOptions.cs
@ -2,7 +2,7 @@
 using IdentityServer4.Configuration;
 using Microsoft.AspNetCore.Authentication.Cookies;
 using Microsoft.AspNetCore.Http;
-using Microsoft.Extensions.Caching.Redis;
+using Microsoft.Extensions.Caching.StackExchangeRedis;
 using Microsoft.Extensions.Options;

 namespace Bit.Core.IdentityServer
--- a/src/Core/IdentityServer/RedisCacheTicketStore.cs
+++ b/src/Core/IdentityServer/RedisCacheTicketStore.cs
@ -1,8 +1,7 @@
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Authentication.Cookies;
 using Microsoft.Extensions.Caching.Distributed;
-using Microsoft.Extensions.Caching.Memory;
-using Microsoft.Extensions.Caching.Redis;
+using Microsoft.Extensions.Caching.StackExchangeRedis;

 namespace Bit.Core.IdentityServer
 {
--- a/src/Core/Settings/GlobalSettings.cs
+++ b/src/Core/Settings/GlobalSettings.cs
@ -49,6 +49,7 @@
        public virtual MailSettings Mail { get; set; } = new MailSettings();
        public virtual IConnectionStringSettings Storage { get; set; } = new ConnectionStringSettings();
        public virtual ConnectionStringSettings Events { get; set; } = new ConnectionStringSettings();
+        public virtual IConnectionStringSettings Redis { get; set; } = new ConnectionStringSettings();
        public virtual NotificationsSettings Notifications { get; set; } = new NotificationsSettings();
        public virtual IFileStorageSettings Attachment { get; set; }
        public virtual FileStorageSettings Send { get; set; }
--- a/src/Core/Utilities/CustomIpRateLimitMiddleware.cs
+++ b/src/Core/Utilities/CustomIpRateLimitMiddleware.cs
@ -2,7 +2,7 @@
 using Bit.Core.Models.Api;
 using Bit.Core.Services;
 using Microsoft.AspNetCore.Http;
-using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Caching.Distributed;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;

@ -10,13 +10,13 @@ namespace Bit.Core.Utilities
 {
    public class CustomIpRateLimitMiddleware : IpRateLimitMiddleware
    {
-        private readonly IpRateLimitOptions _options;
-        private readonly IMemoryCache _memoryCache;
        private readonly IBlockIpService _blockIpService;
        private readonly ILogger<CustomIpRateLimitMiddleware> _logger;
+        private readonly IDistributedCache _distributedCache;
+        private readonly IpRateLimitOptions _options;

        public CustomIpRateLimitMiddleware(
-            IMemoryCache memoryCache,
+            IDistributedCache distributedCache,
            IBlockIpService blockIpService,
            RequestDelegate next,
            IProcessingStrategy processingStrategy,
@ -26,7 +26,7 @@ namespace Bit.Core.Utilities
            ILogger<CustomIpRateLimitMiddleware> logger)
            : base(next, processingStrategy, options, policyStore, rateLimitConfiguration, logger)
        {
-            _memoryCache = memoryCache;
+            _distributedCache = distributedCache;
            _blockIpService = blockIpService;
            _options = options.Value;
            _logger = logger;
@ -34,12 +34,13 @@ namespace Bit.Core.Utilities

        public override Task ReturnQuotaExceededResponse(HttpContext httpContext, RateLimitRule rule, string retryAfter)
        {
-            var message = string.IsNullOrWhiteSpace(_options.QuotaExceededMessage) ?
-                $"Slow down! Too many requests. Try again in {rule.Period}." : _options.QuotaExceededMessage;
+            var message = string.IsNullOrWhiteSpace(_options.QuotaExceededMessage)
+                ? $"Slow down! Too many requests. Try again in {rule.Period}."
+                : _options.QuotaExceededMessage;
            httpContext.Response.Headers["Retry-After"] = retryAfter;
            httpContext.Response.StatusCode = _options.HttpStatusCode;
            var errorModel = new ErrorResponseModel { Message = message };
-            return httpContext.Response.WriteAsJsonAsync(errorModel, cancellationToken: httpContext.RequestAborted);
+            return httpContext.Response.WriteAsJsonAsync(errorModel, httpContext.RequestAborted);
        }

        protected override void LogBlockedRequest(HttpContext httpContext, ClientRequestIdentity identity,
@ -48,7 +49,7 @@ namespace Bit.Core.Utilities
            base.LogBlockedRequest(httpContext, identity, counter, rule);
            var key = $"blockedIp_{identity.ClientIp}";

-            _memoryCache.TryGetValue(key, out int blockedCount);
+            _distributedCache.TryGetValue(key, out int blockedCount);

            blockedCount++;
            if (blockedCount > 10)
@ -61,8 +62,8 @@ namespace Bit.Core.Utilities
            {
                _logger.LogInformation(Constants.BypassFiltersEventId, null,
                    "Request blocked {0}. \nInfo: \n{1}", identity.ClientIp, GetRequestInfo(httpContext));
-                _memoryCache.Set(key, blockedCount,
-                    new MemoryCacheEntryOptions().SetSlidingExpiration(new TimeSpan(0, 5, 0)));
+                _distributedCache.Set(key, blockedCount,
+                    new DistributedCacheEntryOptions().SetSlidingExpiration(new TimeSpan(0, 5, 0)));
            }
        }

--- a/src/Core/Utilities/DistributedCacheExtensions.cs
+++ b/src/Core/Utilities/DistributedCacheExtensions.cs
@ -0,0 +1,48 @@
+using System.Text.Json;
+using Microsoft.Extensions.Caching.Distributed;
+
+namespace Bit.Core.Utilities
+{
+    public static class DistributedCacheExtensions
+    {
+        public static void Set<T>(this IDistributedCache cache, string key, T value)
+        {
+            Set(cache, key, value, new DistributedCacheEntryOptions());
+        }
+
+        public static void Set<T>(this IDistributedCache cache, string key, T value,
+            DistributedCacheEntryOptions options)
+        {
+            var bytes = JsonSerializer.SerializeToUtf8Bytes(value);
+            cache.Set(key, bytes, options);
+        }
+
+        public static Task SetAsync<T>(this IDistributedCache cache, string key, T value)
+        {
+            return SetAsync(cache, key, value, new DistributedCacheEntryOptions());
+        }
+
+        public static Task SetAsync<T>(this IDistributedCache cache, string key, T value,
+            DistributedCacheEntryOptions options)
+        {
+            var bytes = JsonSerializer.SerializeToUtf8Bytes(value);
+            return cache.SetAsync(key, bytes, options);
+        }
+
+        public static bool TryGetValue<T>(this IDistributedCache cache, string key, out T value)
+        {
+            var val = cache.Get(key);
+            value = default;
+            if (val == null) return false;
+            try
+            {
+                value = JsonSerializer.Deserialize<T>(val);
+            }
+            catch
+            {
+                return false;
+            }
+            return true;
+        }
+    }
+}
--- a/src/Core/packages.lock.json
+++ b/src/Core/packages.lock.json
@ -14,6 +14,16 @@
          "Newtonsoft.Json": "13.0.1"
        }
      },
+      "AspNetCoreRateLimit.Redis": {
+        "type": "Direct",
+        "requested": "[1.0.1, )",
+        "resolved": "1.0.1",
+        "contentHash": "CsSGy/7SXt6iBOKg0xCvsRjb/ZHshbtr2Of1MHc912L2sLnZqadUrTboyXZC+ZlgEBeJ14GyjPTu8ZyfEhGUnw==",
+        "dependencies": {
+          "AspNetCoreRateLimit": "4.0.2",
+          "StackExchange.Redis": "2.5.43"
+        }
+      },
      "AWSSDK.SimpleEmail": {
        "type": "Direct",
        "requested": "[3.7.0.150, )",
@ -177,15 +187,15 @@
          "System.IdentityModel.Tokens.Jwt": "5.4.0"
        }
      },
-      "Microsoft.Extensions.Caching.Redis": {
+      "Microsoft.Extensions.Caching.StackExchangeRedis": {
        "type": "Direct",
-        "requested": "[2.2.0, )",
-        "resolved": "2.2.0",
-        "contentHash": "cb21miiGDVjlNl8TRBKIi7OEFdlKuV8d4ZoYqFOhKdZhzo7Sv+b8Puy3NLW3y/g+UDclt7FTh+Za7ykurtaVMQ==",
+        "requested": "[6.0.6, )",
+        "resolved": "6.0.6",
+        "contentHash": "bdVQpYm1hcHf0pyAypMjtDw3HjWQJ89UzloyyF1OBs56QlgA1naM498tP2Vjlho5vVRALMGPYzdRKCen8koubw==",
        "dependencies": {
-          "Microsoft.Extensions.Caching.Abstractions": "2.2.0",
-          "Microsoft.Extensions.Options": "2.2.0",
-          "StackExchange.Redis.StrongName": "1.2.6"
+          "Microsoft.Extensions.Caching.Abstractions": "6.0.0",
+          "Microsoft.Extensions.Options": "6.0.0",
+          "StackExchange.Redis": "2.2.4"
        }
      },
      "Microsoft.Extensions.Configuration.EnvironmentVariables": {
@ -811,8 +821,8 @@
      },
      "Microsoft.NETCore.Platforms": {
        "type": "Transitive",
-        "resolved": "2.0.0",
-        "contentHash": "VdLJOCXhZaEMY7Hm2GKiULmn7IEPFE4XC5LPSfBVCUIA8YLZVh846gtfBJalsPQF2PlzdD7ecX7DZEulJ402ZQ=="
+        "resolved": "5.0.0",
+        "contentHash": "VyPlqzH2wavqquTcYpkIIAQ6WdenuKoFN0BdYBbCWsclXacSOHNQn66Gt4z5NBqEYW0FAPm5rlvki9ZiCij5xQ=="
      },
      "Microsoft.NETCore.Targets": {
        "type": "Transitive",
@ -850,11 +860,11 @@
      },
      "Microsoft.Win32.Registry": {
        "type": "Transitive",
-        "resolved": "4.5.0",
-        "contentHash": "+FWlwd//+Tt56316p00hVePBCouXyEzT86Jb3+AuRotTND0IYn0OO3obs1gnQEs/txEnt+rF2JBGLItTG+Be6A==",
+        "resolved": "5.0.0",
+        "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==",
        "dependencies": {
-          "System.Security.AccessControl": "4.5.0",
-          "System.Security.Principal.Windows": "4.5.0"
+          "System.Security.AccessControl": "5.0.0",
+          "System.Security.Principal.Windows": "5.0.0"
        }
      },
      "Microsoft.Win32.SystemEvents": {
@ -931,6 +941,14 @@
          "libsodium": "[1.0.18, 1.0.19)"
        }
      },
+      "Pipelines.Sockets.Unofficial": {
+        "type": "Transitive",
+        "resolved": "2.2.2",
+        "contentHash": "Bhk0FWxH1paI+18zr1g5cTL+ebeuDcBCR+rRFO+fKEhretgjs7MF2Mc1P64FGLecWp4zKCUOPzngBNrqVyY7Zg==",
+        "dependencies": {
+          "System.IO.Pipelines": "5.0.1"
+        }
+      },
      "Portable.BouncyCastle": {
        "type": "Transitive",
        "resolved": "1.9.0",
@ -1145,34 +1163,13 @@
          "System.Text.Encoding.Extensions": "4.0.11"
        }
      },
-      "StackExchange.Redis.StrongName": {
+      "StackExchange.Redis": {
        "type": "Transitive",
-        "resolved": "1.2.6",
-        "contentHash": "UFmT1/JYu1PLiRwkyvEPVHk/tVTJa8Ka2rb9yzidzDoQARvhBVRpaWUeaP81373v54jupDBvAoGHGl0EY/HphQ==",
+        "resolved": "2.5.43",
+        "contentHash": "YQ38jVbX1b5mBi6lizESou+NpV6QZpeo6ofRR6qeuqJ8ePOmhcwhje3nDTNIGEkfPSK0sLuF6pR5rtFyq2F46g==",
        "dependencies": {
-          "NETStandard.Library": "1.6.1",
-          "System.Collections": "4.3.0",
-          "System.Collections.Concurrent": "4.3.0",
-          "System.Collections.NonGeneric": "4.3.0",
-          "System.Diagnostics.Tools": "4.3.0",
-          "System.IO.Compression": "4.3.0",
-          "System.IO.FileSystem": "4.3.0",
-          "System.Linq": "4.3.0",
-          "System.Net.NameResolution": "4.3.0",
-          "System.Net.Security": "4.3.0",
-          "System.Net.Sockets": "4.3.0",
-          "System.Reflection.Emit": "4.3.0",
-          "System.Reflection.Emit.Lightweight": "4.3.0",
-          "System.Reflection.TypeExtensions": "4.3.0",
-          "System.Runtime.Extensions": "4.3.0",
-          "System.Runtime.InteropServices.RuntimeInformation": "4.3.0",
-          "System.Security.Cryptography.Algorithms": "4.3.0",
-          "System.Security.Cryptography.X509Certificates": "4.3.0",
-          "System.Text.RegularExpressions": "4.3.0",
-          "System.Threading": "4.3.0",
-          "System.Threading.Thread": "4.3.0",
-          "System.Threading.ThreadPool": "4.3.0",
-          "System.Threading.Timer": "4.3.0"
+          "Pipelines.Sockets.Unofficial": "2.2.2",
+          "System.Diagnostics.PerformanceCounter": "5.0.0"
        }
      },
      "starkbank-ecdsa": {
@ -1227,15 +1224,15 @@
      },
      "System.Collections.NonGeneric": {
        "type": "Transitive",
-        "resolved": "4.3.0",
-        "contentHash": "prtjIEMhGUnQq6RnPEYLpFt8AtLbp9yq2zxOSrY7KJJZrw25Fi97IzBqY7iqssbM61Ek5b8f3MG/sG1N2sN5KA==",
+        "resolved": "4.0.1",
+        "contentHash": "hMxFT2RhhlffyCdKLDXjx8WEC5JfCvNozAZxCablAuFRH74SCV4AgzE8yJCh/73bFnEoZgJ9MJmkjQ0dJmnKqA==",
        "dependencies": {
-          "System.Diagnostics.Debug": "4.3.0",
-          "System.Globalization": "4.3.0",
-          "System.Resources.ResourceManager": "4.3.0",
-          "System.Runtime": "4.3.0",
-          "System.Runtime.Extensions": "4.3.0",
-          "System.Threading": "4.3.0"
+          "System.Diagnostics.Debug": "4.0.11",
+          "System.Globalization": "4.0.11",
+          "System.Resources.ResourceManager": "4.0.1",
+          "System.Runtime": "4.1.0",
+          "System.Runtime.Extensions": "4.1.0",
+          "System.Threading": "4.0.11"
        }
      },
      "System.Collections.Specialized": {
@ -1291,6 +1288,17 @@
          "System.Runtime.CompilerServices.Unsafe": "6.0.0"
        }
      },
+      "System.Diagnostics.PerformanceCounter": {
+        "type": "Transitive",
+        "resolved": "5.0.0",
+        "contentHash": "kcQWWtGVC3MWMNXdMDWfrmIlFZZ2OdoeT6pSNVRtk9+Sa7jwdPiMlNwb0ZQcS7NRlT92pCfmjRtkSWUW3RAKwg==",
+        "dependencies": {
+          "Microsoft.NETCore.Platforms": "5.0.0",
+          "Microsoft.Win32.Registry": "5.0.0",
+          "System.Configuration.ConfigurationManager": "5.0.0",
+          "System.Security.Principal.Windows": "5.0.0"
+        }
+      },
      "System.Diagnostics.Process": {
        "type": "Transitive",
        "resolved": "4.3.0",
@ -1516,6 +1524,11 @@
        "resolved": "6.0.0",
        "contentHash": "Rfm2jYCaUeGysFEZjDe7j1R4x6Z6BzumS/vUT5a1AA/AWJuGX71PoGB0RmpyX3VmrGqVnAwtfMn39OHR8Y/5+g=="
      },
+      "System.IO.Pipelines": {
+        "type": "Transitive",
+        "resolved": "5.0.1",
+        "contentHash": "qEePWsaq9LoEEIqhbGe6D5J8c9IqQOUuTzzV6wn1POlfdLkJliZY3OlB0j0f17uMWlqZYjH7txj+2YbyrIA8Yg=="
+      },
      "System.Linq": {
        "type": "Transitive",
        "resolved": "4.3.0",
@ -1616,23 +1629,23 @@
      },
      "System.Net.NameResolution": {
        "type": "Transitive",
-        "resolved": "4.3.0",
-        "contentHash": "AFYl08R7MrsrEjqpQWTZWBadqXyTzNDaWpMqyxhb0d6sGhV6xMDKueuBXlLL30gz+DIRY6MpdgnHWlCh5wmq9w==",
+        "resolved": "4.0.0",
+        "contentHash": "JdqRdM1Qym3YehqdKIi5LHrpypP4JMfxKQSNCJ2z4WawkG0il+N3XfNeJOxll2XrTnG7WgYYPoeiu/KOwg0DQw==",
        "dependencies": {
-          "Microsoft.NETCore.Platforms": "1.1.0",
-          "System.Collections": "4.3.0",
-          "System.Diagnostics.Tracing": "4.3.0",
-          "System.Globalization": "4.3.0",
-          "System.Net.Primitives": "4.3.0",
-          "System.Resources.ResourceManager": "4.3.0",
-          "System.Runtime": "4.3.0",
-          "System.Runtime.Extensions": "4.3.0",
-          "System.Runtime.Handles": "4.3.0",
-          "System.Runtime.InteropServices": "4.3.0",
-          "System.Security.Principal.Windows": "4.3.0",
-          "System.Threading": "4.3.0",
-          "System.Threading.Tasks": "4.3.0",
-          "runtime.native.System": "4.3.0"
+          "Microsoft.NETCore.Platforms": "1.0.1",
+          "System.Collections": "4.0.11",
+          "System.Diagnostics.Tracing": "4.1.0",
+          "System.Globalization": "4.0.11",
+          "System.Net.Primitives": "4.0.11",
+          "System.Resources.ResourceManager": "4.0.1",
+          "System.Runtime": "4.1.0",
+          "System.Runtime.Extensions": "4.1.0",
+          "System.Runtime.Handles": "4.0.1",
+          "System.Runtime.InteropServices": "4.1.0",
+          "System.Security.Principal.Windows": "4.0.0",
+          "System.Threading": "4.0.11",
+          "System.Threading.Tasks": "4.0.11",
+          "runtime.native.System": "4.0.0"
        }
      },
      "System.Net.NetworkInformation": {
@ -2208,11 +2221,8 @@
      },
      "System.Security.Principal.Windows": {
        "type": "Transitive",
-        "resolved": "4.5.0",
-        "contentHash": "U77HfRXlZlOeIXd//Yoj6Jnk8AXlbeisf1oq1os+hxOGVnuG+lGSfGqTwTZBoORFF6j/0q7HXIl8cqwQ9aUGqQ==",
-        "dependencies": {
-          "Microsoft.NETCore.Platforms": "2.0.0"
-        }
+        "resolved": "5.0.0",
+        "contentHash": "t0MGLukB5WAVU9bO3MGzvlGnyJPgUlcwerXn1kzBRjwLKixT96XV0Uza41W49gVd8zEMFu9vQEFlv0IOrytICA=="
      },
      "System.Security.SecureString": {
        "type": "Transitive",