using AspNetCoreRateLimit;
using AspNetCoreRateLimit.Redis;
using Bit.Core.Settings;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using StackExchange.Redis;
namespace Bit.Core.Utilities;
///
/// A modified version of that gracefully
/// handles a disrupted Redis connection. If the connection is down or the number of failed requests within
/// a given time period exceed the configured threshold, then rate limiting is temporarily disabled.
///
///
/// This is necessary to ensure the service does not become unresponsive due to Redis being out of service. As
/// the default implementation would throw an exception and exit the request pipeline for all requests.
///
public class CustomRedisProcessingStrategy : RedisProcessingStrategy
{
private readonly IConnectionMultiplexer _connectionMultiplexer;
private readonly ILogger _logger;
private readonly IMemoryCache _memoryCache;
private readonly GlobalSettings.DistributedIpRateLimitingSettings _distributedSettings;
private const string _redisTimeoutCacheKey = "IpRateLimitRedisTimeout";
public CustomRedisProcessingStrategy(
IConnectionMultiplexer connectionMultiplexer,
IRateLimitConfiguration config,
ILogger logger,
IMemoryCache memoryCache,
GlobalSettings globalSettings)
: base(connectionMultiplexer, config, logger)
{
_connectionMultiplexer = connectionMultiplexer;
_logger = logger;
_memoryCache = memoryCache;
_distributedSettings = globalSettings.DistributedIpRateLimiting;
}
public override async Task ProcessRequestAsync(ClientRequestIdentity requestIdentity,
RateLimitRule rule, ICounterKeyBuilder counterKeyBuilder, RateLimitOptions rateLimitOptions,
CancellationToken cancellationToken = default)
{
// If Redis is down entirely, skip rate limiting
if (!_connectionMultiplexer.IsConnected)
{
_logger.LogDebug("Redis connection is down, skipping IP rate limiting");
return SkipRateLimitResult();
}
// Check if any Redis timeouts have occurred recently
if (_memoryCache.TryGetValue(_redisTimeoutCacheKey, out var timeoutCounter))
{
// We've exceeded threshold, backoff Redis and skip rate limiting for now
if (timeoutCounter.Count >= _distributedSettings.MaxRedisTimeoutsThreshold)
{
_logger.LogDebug(
"Redis timeout threshold has been exceeded, backing off and skipping IP rate limiting");
return SkipRateLimitResult();
}
}
try
{
return await base.ProcessRequestAsync(requestIdentity, rule, counterKeyBuilder, rateLimitOptions, cancellationToken);
}
catch (RedisTimeoutException)
{
// If this is the first timeout we've had, start a new counter and sliding window
timeoutCounter ??= new TimeoutCounter()
{
Count = 0,
ExpiresAt = DateTime.UtcNow.AddSeconds(_distributedSettings.SlidingWindowSeconds)
};
timeoutCounter.Count++;
_memoryCache.Set(_redisTimeoutCacheKey, timeoutCounter,
new MemoryCacheEntryOptions { AbsoluteExpiration = timeoutCounter.ExpiresAt });
// Just because Redis timed out does not mean we should kill the request
return SkipRateLimitResult();
}
}
///
/// A RateLimitCounter result used when the rate limiting middleware should
/// fail open and allow the request to proceed without checking request limits.
///
private static RateLimitCounter SkipRateLimitResult()
{
return new RateLimitCounter { Count = 0, Timestamp = DateTime.UtcNow };
}
internal class TimeoutCounter
{
public DateTime ExpiresAt { get; init; }
public int Count { get; set; }
}
}