mirror of
https://github.com/bitwarden/server.git
synced 2025-07-19 08:30:59 -05:00
Run formatting (#2230)
This commit is contained in:
Justin Baur
@ -3,106 +3,105 @@ using Bit.Icons.Services;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
|
||||
namespace Bit.Icons.Controllers
|
||||
namespace Bit.Icons.Controllers;
|
||||
|
||||
[Route("")]
|
||||
public class IconsController : Controller
|
||||
{
|
||||
[Route("")]
|
||||
public class IconsController : Controller
|
||||
// Basic bwi-globe icon
|
||||
private static readonly byte[] _notFoundImage = Convert.FromBase64String("iVBORw0KGgoAAAANSUhEUg" +
|
||||
"AAABMAAAATCAQAAADYWf5HAAABu0lEQVR42nXSvWuTURTH8R+t0heI9Y04aJycdBLNJNrBFBU7OFgUER3q21I0bXK+JwZ" +
|
||||
"pXISm/QdcRB3EgqBBsNihsUbbgODQQSKCuKSDOApJuuhj8tCYQj/jvYfD795z1MZ+nBKrNKhSwrMxbZTrtRnqlEjZkB/x" +
|
||||
"C/xmhZrlc71qS0Up8yVzTCGucFNKD1JhORVd70SZNU4okNx5d4+U2UXRIpJFWLClsR79YzN88wQvLWNzzPKEeS/wkQGpW" +
|
||||
"VhhqhW8TtDJD3Mm1x/23zLSrZCdpBY8BueTNjHSbc+8wC9HlHgU5Aj5AW5zPdcVdpq0UcknWBSr/pjixO4gfp899Kd23p" +
|
||||
"M2qQCH7LkCnqAqGh73OK/8NPOcaibr90LrW/yWAnaUhqjaOSl9nFR2r5rsqo22ypn1B5IN8VOUMHVgOnNQIX+d62plcz6" +
|
||||
"rg1/jskK8CMb4we4pG6OWHtR/LBJkC2E4a7ZPkuX5ntumAOM2xxveclEhLvGH6XCmLPs735Eetrw63NnOgr9P9q1viC3x" +
|
||||
"lRUGOjImqFDuOBvrYYoaZU9z1uPpYae5NfdvbNVG2ZjDIlXq/oMi46lo++4vjjPBl2Dlg00AAAAASUVORK5CYII=");
|
||||
|
||||
private readonly IMemoryCache _memoryCache;
|
||||
private readonly IDomainMappingService _domainMappingService;
|
||||
private readonly IIconFetchingService _iconFetchingService;
|
||||
private readonly ILogger<IconsController> _logger;
|
||||
private readonly IconsSettings _iconsSettings;
|
||||
|
||||
public IconsController(
|
||||
IMemoryCache memoryCache,
|
||||
IDomainMappingService domainMappingService,
|
||||
IIconFetchingService iconFetchingService,
|
||||
ILogger<IconsController> logger,
|
||||
IconsSettings iconsSettings)
|
||||
{
|
||||
// Basic bwi-globe icon
|
||||
private static readonly byte[] _notFoundImage = Convert.FromBase64String("iVBORw0KGgoAAAANSUhEUg" +
|
||||
"AAABMAAAATCAQAAADYWf5HAAABu0lEQVR42nXSvWuTURTH8R+t0heI9Y04aJycdBLNJNrBFBU7OFgUER3q21I0bXK+JwZ" +
|
||||
"pXISm/QdcRB3EgqBBsNihsUbbgODQQSKCuKSDOApJuuhj8tCYQj/jvYfD795z1MZ+nBKrNKhSwrMxbZTrtRnqlEjZkB/x" +
|
||||
"C/xmhZrlc71qS0Up8yVzTCGucFNKD1JhORVd70SZNU4okNx5d4+U2UXRIpJFWLClsR79YzN88wQvLWNzzPKEeS/wkQGpW" +
|
||||
"VhhqhW8TtDJD3Mm1x/23zLSrZCdpBY8BueTNjHSbc+8wC9HlHgU5Aj5AW5zPdcVdpq0UcknWBSr/pjixO4gfp899Kd23p" +
|
||||
"M2qQCH7LkCnqAqGh73OK/8NPOcaibr90LrW/yWAnaUhqjaOSl9nFR2r5rsqo22ypn1B5IN8VOUMHVgOnNQIX+d62plcz6" +
|
||||
"rg1/jskK8CMb4we4pG6OWHtR/LBJkC2E4a7ZPkuX5ntumAOM2xxveclEhLvGH6XCmLPs735Eetrw63NnOgr9P9q1viC3x" +
|
||||
"lRUGOjImqFDuOBvrYYoaZU9z1uPpYae5NfdvbNVG2ZjDIlXq/oMi46lo++4vjjPBl2Dlg00AAAAASUVORK5CYII=");
|
||||
_memoryCache = memoryCache;
|
||||
_domainMappingService = domainMappingService;
|
||||
_iconFetchingService = iconFetchingService;
|
||||
_logger = logger;
|
||||
_iconsSettings = iconsSettings;
|
||||
}
|
||||
|
||||
private readonly IMemoryCache _memoryCache;
|
||||
private readonly IDomainMappingService _domainMappingService;
|
||||
private readonly IIconFetchingService _iconFetchingService;
|
||||
private readonly ILogger<IconsController> _logger;
|
||||
private readonly IconsSettings _iconsSettings;
|
||||
|
||||
public IconsController(
|
||||
IMemoryCache memoryCache,
|
||||
IDomainMappingService domainMappingService,
|
||||
IIconFetchingService iconFetchingService,
|
||||
ILogger<IconsController> logger,
|
||||
IconsSettings iconsSettings)
|
||||
[HttpGet("~/config")]
|
||||
public IActionResult GetConfig()
|
||||
{
|
||||
return new JsonResult(new
|
||||
{
|
||||
_memoryCache = memoryCache;
|
||||
_domainMappingService = domainMappingService;
|
||||
_iconFetchingService = iconFetchingService;
|
||||
_logger = logger;
|
||||
_iconsSettings = iconsSettings;
|
||||
CacheEnabled = _iconsSettings.CacheEnabled,
|
||||
CacheHours = _iconsSettings.CacheHours,
|
||||
CacheSizeLimit = _iconsSettings.CacheSizeLimit
|
||||
});
|
||||
}
|
||||
|
||||
[HttpGet("{hostname}/icon.png")]
|
||||
public async Task<IActionResult> Get(string hostname)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(hostname) || !hostname.Contains("."))
|
||||
{
|
||||
return new BadRequestResult();
|
||||
}
|
||||
|
||||
[HttpGet("~/config")]
|
||||
public IActionResult GetConfig()
|
||||
var url = $"http://{hostname}";
|
||||
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
|
||||
{
|
||||
return new JsonResult(new
|
||||
{
|
||||
CacheEnabled = _iconsSettings.CacheEnabled,
|
||||
CacheHours = _iconsSettings.CacheHours,
|
||||
CacheSizeLimit = _iconsSettings.CacheSizeLimit
|
||||
});
|
||||
return new BadRequestResult();
|
||||
}
|
||||
|
||||
[HttpGet("{hostname}/icon.png")]
|
||||
public async Task<IActionResult> Get(string hostname)
|
||||
var domain = uri.Host;
|
||||
// Convert sub.domain.com => domain.com
|
||||
//if(DomainName.TryParseBaseDomain(domain, out var baseDomain))
|
||||
//{
|
||||
// domain = baseDomain;
|
||||
//}
|
||||
|
||||
var mappedDomain = _domainMappingService.MapDomain(domain);
|
||||
if (!_iconsSettings.CacheEnabled || !_memoryCache.TryGetValue(mappedDomain, out Icon icon))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(hostname) || !hostname.Contains("."))
|
||||
var result = await _iconFetchingService.GetIconAsync(domain);
|
||||
if (result == null)
|
||||
{
|
||||
return new BadRequestResult();
|
||||
_logger.LogWarning("Null result returned for {0}.", domain);
|
||||
icon = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
icon = result.Icon;
|
||||
}
|
||||
|
||||
var url = $"http://{hostname}";
|
||||
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
|
||||
// Only cache not found and smaller images (<= 50kb)
|
||||
if (_iconsSettings.CacheEnabled && (icon == null || icon.Image.Length <= 50012))
|
||||
{
|
||||
return new BadRequestResult();
|
||||
}
|
||||
|
||||
var domain = uri.Host;
|
||||
// Convert sub.domain.com => domain.com
|
||||
//if(DomainName.TryParseBaseDomain(domain, out var baseDomain))
|
||||
//{
|
||||
// domain = baseDomain;
|
||||
//}
|
||||
|
||||
var mappedDomain = _domainMappingService.MapDomain(domain);
|
||||
if (!_iconsSettings.CacheEnabled || !_memoryCache.TryGetValue(mappedDomain, out Icon icon))
|
||||
{
|
||||
var result = await _iconFetchingService.GetIconAsync(domain);
|
||||
if (result == null)
|
||||
_logger.LogInformation("Cache icon for {0}.", domain);
|
||||
_memoryCache.Set(mappedDomain, icon, new MemoryCacheEntryOptions
|
||||
{
|
||||
_logger.LogWarning("Null result returned for {0}.", domain);
|
||||
icon = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
icon = result.Icon;
|
||||
}
|
||||
|
||||
// Only cache not found and smaller images (<= 50kb)
|
||||
if (_iconsSettings.CacheEnabled && (icon == null || icon.Image.Length <= 50012))
|
||||
{
|
||||
_logger.LogInformation("Cache icon for {0}.", domain);
|
||||
_memoryCache.Set(mappedDomain, icon, new MemoryCacheEntryOptions
|
||||
{
|
||||
AbsoluteExpirationRelativeToNow = new TimeSpan(_iconsSettings.CacheHours, 0, 0),
|
||||
Size = icon?.Image.Length ?? 0,
|
||||
Priority = icon == null ? CacheItemPriority.High : CacheItemPriority.Normal
|
||||
});
|
||||
}
|
||||
AbsoluteExpirationRelativeToNow = new TimeSpan(_iconsSettings.CacheHours, 0, 0),
|
||||
Size = icon?.Image.Length ?? 0,
|
||||
Priority = icon == null ? CacheItemPriority.High : CacheItemPriority.Normal
|
||||
});
|
||||
}
|
||||
|
||||
if (icon == null)
|
||||
{
|
||||
return new FileContentResult(_notFoundImage, "image/png");
|
||||
}
|
||||
|
||||
return new FileContentResult(icon.Image, icon.Format);
|
||||
}
|
||||
|
||||
if (icon == null)
|
||||
{
|
||||
return new FileContentResult(_notFoundImage, "image/png");
|
||||
}
|
||||
|
||||
return new FileContentResult(icon.Image, icon.Format);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,21 +1,20 @@
|
||||
using Bit.Core.Utilities;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace Bit.Icons.Controllers
|
||||
{
|
||||
public class InfoController : Controller
|
||||
{
|
||||
[HttpGet("~/alive")]
|
||||
[HttpGet("~/now")]
|
||||
public DateTime GetAlive()
|
||||
{
|
||||
return DateTime.UtcNow;
|
||||
}
|
||||
namespace Bit.Icons.Controllers;
|
||||
|
||||
[HttpGet("~/version")]
|
||||
public JsonResult GetVersion()
|
||||
{
|
||||
return Json(CoreHelpers.GetVersion());
|
||||
}
|
||||
public class InfoController : Controller
|
||||
{
|
||||
[HttpGet("~/alive")]
|
||||
[HttpGet("~/now")]
|
||||
public DateTime GetAlive()
|
||||
{
|
||||
return DateTime.UtcNow;
|
||||
}
|
||||
|
||||
[HttpGet("~/version")]
|
||||
public JsonResult GetVersion()
|
||||
{
|
||||
return Json(CoreHelpers.GetVersion());
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,9 +1,8 @@
|
||||
namespace Bit.Icons
|
||||
namespace Bit.Icons;
|
||||
|
||||
public class IconsSettings
|
||||
{
|
||||
public class IconsSettings
|
||||
{
|
||||
public virtual bool CacheEnabled { get; set; }
|
||||
public virtual int CacheHours { get; set; }
|
||||
public virtual long? CacheSizeLimit { get; set; }
|
||||
}
|
||||
public virtual bool CacheEnabled { get; set; }
|
||||
public virtual int CacheHours { get; set; }
|
||||
public virtual long? CacheSizeLimit { get; set; }
|
||||
}
|
||||
|
||||
@ -2,324 +2,323 @@
|
||||
using System.Reflection;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace Bit.Icons.Models
|
||||
namespace Bit.Icons.Models;
|
||||
|
||||
// ref: https://github.com/danesparza/domainname-parser
|
||||
public class DomainName
|
||||
{
|
||||
// ref: https://github.com/danesparza/domainname-parser
|
||||
public class DomainName
|
||||
private const string IpRegex = "^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\." +
|
||||
"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\." +
|
||||
"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\." +
|
||||
"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$";
|
||||
|
||||
private string _subDomain = string.Empty;
|
||||
private string _domain = string.Empty;
|
||||
private string _tld = string.Empty;
|
||||
private TLDRule _tldRule = null;
|
||||
|
||||
public string SubDomain => _subDomain;
|
||||
public string Domain => _domain;
|
||||
public string SLD => _domain;
|
||||
public string TLD => _tld;
|
||||
public TLDRule Rule => _tldRule;
|
||||
public string BaseDomain => $"{_domain}.{_tld}";
|
||||
|
||||
public DomainName(string TLD, string SLD, string SubDomain, TLDRule TLDRule)
|
||||
{
|
||||
private const string IpRegex = "^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\." +
|
||||
"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\." +
|
||||
"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\." +
|
||||
"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$";
|
||||
_tld = TLD;
|
||||
_domain = SLD;
|
||||
_subDomain = SubDomain;
|
||||
_tldRule = TLDRule;
|
||||
}
|
||||
|
||||
private string _subDomain = string.Empty;
|
||||
private string _domain = string.Empty;
|
||||
private string _tld = string.Empty;
|
||||
private TLDRule _tldRule = null;
|
||||
public static bool TryParse(string domainString, out DomainName result)
|
||||
{
|
||||
var retval = false;
|
||||
|
||||
public string SubDomain => _subDomain;
|
||||
public string Domain => _domain;
|
||||
public string SLD => _domain;
|
||||
public string TLD => _tld;
|
||||
public TLDRule Rule => _tldRule;
|
||||
public string BaseDomain => $"{_domain}.{_tld}";
|
||||
// Our temporary domain parts:
|
||||
var tld = string.Empty;
|
||||
var sld = string.Empty;
|
||||
var subdomain = string.Empty;
|
||||
TLDRule _tldrule = null;
|
||||
result = null;
|
||||
|
||||
public DomainName(string TLD, string SLD, string SubDomain, TLDRule TLDRule)
|
||||
try
|
||||
{
|
||||
_tld = TLD;
|
||||
_domain = SLD;
|
||||
_subDomain = SubDomain;
|
||||
_tldRule = TLDRule;
|
||||
// Try parsing the domain name ... this might throw formatting exceptions
|
||||
ParseDomainName(domainString, out tld, out sld, out subdomain, out _tldrule);
|
||||
// Construct a new DomainName object and return it
|
||||
result = new DomainName(tld, sld, subdomain, _tldrule);
|
||||
// Return 'true'
|
||||
retval = true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Looks like something bad happened -- return 'false'
|
||||
retval = false;
|
||||
}
|
||||
|
||||
public static bool TryParse(string domainString, out DomainName result)
|
||||
return retval;
|
||||
}
|
||||
|
||||
public static bool TryParseBaseDomain(string domainString, out string result)
|
||||
{
|
||||
if (Regex.IsMatch(domainString, IpRegex))
|
||||
{
|
||||
var retval = false;
|
||||
|
||||
// Our temporary domain parts:
|
||||
var tld = string.Empty;
|
||||
var sld = string.Empty;
|
||||
var subdomain = string.Empty;
|
||||
TLDRule _tldrule = null;
|
||||
result = null;
|
||||
|
||||
try
|
||||
{
|
||||
// Try parsing the domain name ... this might throw formatting exceptions
|
||||
ParseDomainName(domainString, out tld, out sld, out subdomain, out _tldrule);
|
||||
// Construct a new DomainName object and return it
|
||||
result = new DomainName(tld, sld, subdomain, _tldrule);
|
||||
// Return 'true'
|
||||
retval = true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Looks like something bad happened -- return 'false'
|
||||
retval = false;
|
||||
}
|
||||
|
||||
return retval;
|
||||
result = domainString;
|
||||
return true;
|
||||
}
|
||||
|
||||
public static bool TryParseBaseDomain(string domainString, out string result)
|
||||
{
|
||||
if (Regex.IsMatch(domainString, IpRegex))
|
||||
{
|
||||
result = domainString;
|
||||
return true;
|
||||
}
|
||||
DomainName domain;
|
||||
var retval = TryParse(domainString, out domain);
|
||||
result = domain?.BaseDomain;
|
||||
return retval;
|
||||
}
|
||||
|
||||
DomainName domain;
|
||||
var retval = TryParse(domainString, out domain);
|
||||
result = domain?.BaseDomain;
|
||||
return retval;
|
||||
private static void ParseDomainName(string domainString, out string TLD, out string SLD,
|
||||
out string SubDomain, out TLDRule MatchingRule)
|
||||
{
|
||||
// Make sure domain is all lowercase
|
||||
domainString = domainString.ToLower();
|
||||
|
||||
TLD = string.Empty;
|
||||
SLD = string.Empty;
|
||||
SubDomain = string.Empty;
|
||||
MatchingRule = null;
|
||||
|
||||
// If the fqdn is empty, we have a problem already
|
||||
if (domainString.Trim() == string.Empty)
|
||||
{
|
||||
throw new ArgumentException("The domain cannot be blank");
|
||||
}
|
||||
|
||||
private static void ParseDomainName(string domainString, out string TLD, out string SLD,
|
||||
out string SubDomain, out TLDRule MatchingRule)
|
||||
// Next, find the matching rule:
|
||||
MatchingRule = FindMatchingTLDRule(domainString);
|
||||
|
||||
// At this point, no rules match, we have a problem
|
||||
if (MatchingRule == null)
|
||||
{
|
||||
// Make sure domain is all lowercase
|
||||
domainString = domainString.ToLower();
|
||||
throw new FormatException("The domain does not have a recognized TLD");
|
||||
}
|
||||
|
||||
TLD = string.Empty;
|
||||
SLD = string.Empty;
|
||||
SubDomain = string.Empty;
|
||||
MatchingRule = null;
|
||||
// Based on the tld rule found, get the domain (and possibly the subdomain)
|
||||
var tempSudomainAndDomain = string.Empty;
|
||||
var tldIndex = 0;
|
||||
|
||||
// If the fqdn is empty, we have a problem already
|
||||
if (domainString.Trim() == string.Empty)
|
||||
// First, determine what type of rule we have, and set the TLD accordingly
|
||||
switch (MatchingRule.Type)
|
||||
{
|
||||
case TLDRule.RuleType.Normal:
|
||||
tldIndex = domainString.LastIndexOf("." + MatchingRule.Name);
|
||||
tempSudomainAndDomain = domainString.Substring(0, tldIndex);
|
||||
TLD = domainString.Substring(tldIndex + 1);
|
||||
break;
|
||||
case TLDRule.RuleType.Wildcard:
|
||||
// This finds the last portion of the TLD...
|
||||
tldIndex = domainString.LastIndexOf("." + MatchingRule.Name);
|
||||
tempSudomainAndDomain = domainString.Substring(0, tldIndex);
|
||||
|
||||
// But we need to find the wildcard portion of it:
|
||||
tldIndex = tempSudomainAndDomain.LastIndexOf(".");
|
||||
tempSudomainAndDomain = domainString.Substring(0, tldIndex);
|
||||
TLD = domainString.Substring(tldIndex + 1);
|
||||
break;
|
||||
case TLDRule.RuleType.Exception:
|
||||
tldIndex = domainString.LastIndexOf(".");
|
||||
tempSudomainAndDomain = domainString.Substring(0, tldIndex);
|
||||
TLD = domainString.Substring(tldIndex + 1);
|
||||
break;
|
||||
}
|
||||
|
||||
// See if we have a subdomain:
|
||||
List<string> lstRemainingParts = new List<string>(tempSudomainAndDomain.Split('.'));
|
||||
|
||||
// If we have 0 parts left, there is just a tld and no domain or subdomain
|
||||
// If we have 1 part, it's the domain, and there is no subdomain
|
||||
// If we have 2+ parts, the last part is the domain, the other parts (combined) are the subdomain
|
||||
if (lstRemainingParts.Count > 0)
|
||||
{
|
||||
// Set the domain:
|
||||
SLD = lstRemainingParts[lstRemainingParts.Count - 1];
|
||||
|
||||
// Set the subdomain, if there is one to set:
|
||||
if (lstRemainingParts.Count > 1)
|
||||
{
|
||||
throw new ArgumentException("The domain cannot be blank");
|
||||
// We strip off the trailing period, too
|
||||
SubDomain = tempSudomainAndDomain.Substring(0, tempSudomainAndDomain.Length - SLD.Length - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static TLDRule FindMatchingTLDRule(string domainString)
|
||||
{
|
||||
// Split our domain into parts (based on the '.')
|
||||
// ...Put these parts in a list
|
||||
// ...Make sure these parts are in reverse order
|
||||
// (we'll be checking rules from the right-most pat of the domain)
|
||||
var lstDomainParts = domainString.Split('.').ToList();
|
||||
lstDomainParts.Reverse();
|
||||
|
||||
// Begin building our partial domain to check rules with:
|
||||
var checkAgainst = string.Empty;
|
||||
|
||||
// Our 'matches' collection:
|
||||
var ruleMatches = new List<TLDRule>();
|
||||
|
||||
foreach (string domainPart in lstDomainParts)
|
||||
{
|
||||
// Add on our next domain part:
|
||||
checkAgainst = string.Format("{0}.{1}", domainPart, checkAgainst);
|
||||
|
||||
// If we end in a period, strip it off:
|
||||
if (checkAgainst.EndsWith("."))
|
||||
{
|
||||
checkAgainst = checkAgainst.Substring(0, checkAgainst.Length - 1);
|
||||
}
|
||||
|
||||
// Next, find the matching rule:
|
||||
MatchingRule = FindMatchingTLDRule(domainString);
|
||||
|
||||
// At this point, no rules match, we have a problem
|
||||
if (MatchingRule == null)
|
||||
var rules = Enum.GetValues(typeof(TLDRule.RuleType)).Cast<TLDRule.RuleType>();
|
||||
foreach (var rule in rules)
|
||||
{
|
||||
throw new FormatException("The domain does not have a recognized TLD");
|
||||
}
|
||||
|
||||
// Based on the tld rule found, get the domain (and possibly the subdomain)
|
||||
var tempSudomainAndDomain = string.Empty;
|
||||
var tldIndex = 0;
|
||||
|
||||
// First, determine what type of rule we have, and set the TLD accordingly
|
||||
switch (MatchingRule.Type)
|
||||
{
|
||||
case TLDRule.RuleType.Normal:
|
||||
tldIndex = domainString.LastIndexOf("." + MatchingRule.Name);
|
||||
tempSudomainAndDomain = domainString.Substring(0, tldIndex);
|
||||
TLD = domainString.Substring(tldIndex + 1);
|
||||
break;
|
||||
case TLDRule.RuleType.Wildcard:
|
||||
// This finds the last portion of the TLD...
|
||||
tldIndex = domainString.LastIndexOf("." + MatchingRule.Name);
|
||||
tempSudomainAndDomain = domainString.Substring(0, tldIndex);
|
||||
|
||||
// But we need to find the wildcard portion of it:
|
||||
tldIndex = tempSudomainAndDomain.LastIndexOf(".");
|
||||
tempSudomainAndDomain = domainString.Substring(0, tldIndex);
|
||||
TLD = domainString.Substring(tldIndex + 1);
|
||||
break;
|
||||
case TLDRule.RuleType.Exception:
|
||||
tldIndex = domainString.LastIndexOf(".");
|
||||
tempSudomainAndDomain = domainString.Substring(0, tldIndex);
|
||||
TLD = domainString.Substring(tldIndex + 1);
|
||||
break;
|
||||
}
|
||||
|
||||
// See if we have a subdomain:
|
||||
List<string> lstRemainingParts = new List<string>(tempSudomainAndDomain.Split('.'));
|
||||
|
||||
// If we have 0 parts left, there is just a tld and no domain or subdomain
|
||||
// If we have 1 part, it's the domain, and there is no subdomain
|
||||
// If we have 2+ parts, the last part is the domain, the other parts (combined) are the subdomain
|
||||
if (lstRemainingParts.Count > 0)
|
||||
{
|
||||
// Set the domain:
|
||||
SLD = lstRemainingParts[lstRemainingParts.Count - 1];
|
||||
|
||||
// Set the subdomain, if there is one to set:
|
||||
if (lstRemainingParts.Count > 1)
|
||||
// Try to match rule:
|
||||
TLDRule result;
|
||||
if (TLDRulesCache.Instance.TLDRuleLists[rule].TryGetValue(checkAgainst, out result))
|
||||
{
|
||||
// We strip off the trailing period, too
|
||||
SubDomain = tempSudomainAndDomain.Substring(0, tempSudomainAndDomain.Length - SLD.Length - 1);
|
||||
ruleMatches.Add(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static TLDRule FindMatchingTLDRule(string domainString)
|
||||
// Sort our matches list (longest rule wins, according to :
|
||||
var results = from match in ruleMatches
|
||||
orderby match.Name.Length descending
|
||||
select match;
|
||||
|
||||
// Take the top result (our primary match):
|
||||
var primaryMatch = results.Take(1).SingleOrDefault();
|
||||
return primaryMatch;
|
||||
}
|
||||
|
||||
public class TLDRule : IComparable<TLDRule>
|
||||
{
|
||||
public string Name { get; private set; }
|
||||
public RuleType Type { get; private set; }
|
||||
|
||||
public TLDRule(string RuleInfo)
|
||||
{
|
||||
// Split our domain into parts (based on the '.')
|
||||
// ...Put these parts in a list
|
||||
// ...Make sure these parts are in reverse order
|
||||
// (we'll be checking rules from the right-most pat of the domain)
|
||||
var lstDomainParts = domainString.Split('.').ToList();
|
||||
lstDomainParts.Reverse();
|
||||
|
||||
// Begin building our partial domain to check rules with:
|
||||
var checkAgainst = string.Empty;
|
||||
|
||||
// Our 'matches' collection:
|
||||
var ruleMatches = new List<TLDRule>();
|
||||
|
||||
foreach (string domainPart in lstDomainParts)
|
||||
// Parse the rule and set properties accordingly:
|
||||
if (RuleInfo.StartsWith("*"))
|
||||
{
|
||||
// Add on our next domain part:
|
||||
checkAgainst = string.Format("{0}.{1}", domainPart, checkAgainst);
|
||||
Type = RuleType.Wildcard;
|
||||
Name = RuleInfo.Substring(2);
|
||||
}
|
||||
else if (RuleInfo.StartsWith("!"))
|
||||
{
|
||||
Type = RuleType.Exception;
|
||||
Name = RuleInfo.Substring(1);
|
||||
}
|
||||
else
|
||||
{
|
||||
Type = RuleType.Normal;
|
||||
Name = RuleInfo;
|
||||
}
|
||||
}
|
||||
|
||||
// If we end in a period, strip it off:
|
||||
if (checkAgainst.EndsWith("."))
|
||||
{
|
||||
checkAgainst = checkAgainst.Substring(0, checkAgainst.Length - 1);
|
||||
}
|
||||
public int CompareTo(TLDRule other)
|
||||
{
|
||||
if (other == null)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
var rules = Enum.GetValues(typeof(TLDRule.RuleType)).Cast<TLDRule.RuleType>();
|
||||
foreach (var rule in rules)
|
||||
return Name.CompareTo(other.Name);
|
||||
}
|
||||
|
||||
public enum RuleType
|
||||
{
|
||||
Normal,
|
||||
Wildcard,
|
||||
Exception
|
||||
}
|
||||
}
|
||||
|
||||
public class TLDRulesCache
|
||||
{
|
||||
private static volatile TLDRulesCache _uniqueInstance;
|
||||
private static object _syncObj = new object();
|
||||
private static object _syncList = new object();
|
||||
|
||||
private TLDRulesCache()
|
||||
{
|
||||
// Initialize our internal list:
|
||||
TLDRuleLists = GetTLDRules();
|
||||
}
|
||||
|
||||
public static TLDRulesCache Instance
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_uniqueInstance == null)
|
||||
{
|
||||
// Try to match rule:
|
||||
TLDRule result;
|
||||
if (TLDRulesCache.Instance.TLDRuleLists[rule].TryGetValue(checkAgainst, out result))
|
||||
lock (_syncObj)
|
||||
{
|
||||
ruleMatches.Add(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort our matches list (longest rule wins, according to :
|
||||
var results = from match in ruleMatches
|
||||
orderby match.Name.Length descending
|
||||
select match;
|
||||
|
||||
// Take the top result (our primary match):
|
||||
var primaryMatch = results.Take(1).SingleOrDefault();
|
||||
return primaryMatch;
|
||||
}
|
||||
|
||||
public class TLDRule : IComparable<TLDRule>
|
||||
{
|
||||
public string Name { get; private set; }
|
||||
public RuleType Type { get; private set; }
|
||||
|
||||
public TLDRule(string RuleInfo)
|
||||
{
|
||||
// Parse the rule and set properties accordingly:
|
||||
if (RuleInfo.StartsWith("*"))
|
||||
{
|
||||
Type = RuleType.Wildcard;
|
||||
Name = RuleInfo.Substring(2);
|
||||
}
|
||||
else if (RuleInfo.StartsWith("!"))
|
||||
{
|
||||
Type = RuleType.Exception;
|
||||
Name = RuleInfo.Substring(1);
|
||||
}
|
||||
else
|
||||
{
|
||||
Type = RuleType.Normal;
|
||||
Name = RuleInfo;
|
||||
}
|
||||
}
|
||||
|
||||
public int CompareTo(TLDRule other)
|
||||
{
|
||||
if (other == null)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
return Name.CompareTo(other.Name);
|
||||
}
|
||||
|
||||
public enum RuleType
|
||||
{
|
||||
Normal,
|
||||
Wildcard,
|
||||
Exception
|
||||
}
|
||||
}
|
||||
|
||||
public class TLDRulesCache
|
||||
{
|
||||
private static volatile TLDRulesCache _uniqueInstance;
|
||||
private static object _syncObj = new object();
|
||||
private static object _syncList = new object();
|
||||
|
||||
private TLDRulesCache()
|
||||
{
|
||||
// Initialize our internal list:
|
||||
TLDRuleLists = GetTLDRules();
|
||||
}
|
||||
|
||||
public static TLDRulesCache Instance
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_uniqueInstance == null)
|
||||
{
|
||||
lock (_syncObj)
|
||||
if (_uniqueInstance == null)
|
||||
{
|
||||
if (_uniqueInstance == null)
|
||||
{
|
||||
_uniqueInstance = new TLDRulesCache();
|
||||
}
|
||||
_uniqueInstance = new TLDRulesCache();
|
||||
}
|
||||
}
|
||||
return (_uniqueInstance);
|
||||
}
|
||||
return (_uniqueInstance);
|
||||
}
|
||||
}
|
||||
|
||||
public IDictionary<TLDRule.RuleType, IDictionary<string, TLDRule>> TLDRuleLists { get; set; }
|
||||
|
||||
public static void Reset()
|
||||
{
|
||||
lock (_syncObj)
|
||||
{
|
||||
_uniqueInstance = null;
|
||||
}
|
||||
}
|
||||
|
||||
private IDictionary<TLDRule.RuleType, IDictionary<string, TLDRule>> GetTLDRules()
|
||||
{
|
||||
var results = new Dictionary<TLDRule.RuleType, IDictionary<string, TLDRule>>();
|
||||
var rules = Enum.GetValues(typeof(TLDRule.RuleType)).Cast<TLDRule.RuleType>();
|
||||
foreach (var rule in rules)
|
||||
{
|
||||
results[rule] = new Dictionary<string, TLDRule>(StringComparer.CurrentCultureIgnoreCase);
|
||||
}
|
||||
|
||||
public IDictionary<TLDRule.RuleType, IDictionary<string, TLDRule>> TLDRuleLists { get; set; }
|
||||
var ruleStrings = ReadRulesData();
|
||||
|
||||
public static void Reset()
|
||||
// Strip out any lines that are:
|
||||
// a.) A comment
|
||||
// b.) Blank
|
||||
var rulesStrings = ruleStrings
|
||||
.Where(ruleString => !ruleString.StartsWith("//") && ruleString.Trim().Length != 0);
|
||||
foreach (var ruleString in rulesStrings)
|
||||
{
|
||||
lock (_syncObj)
|
||||
{
|
||||
_uniqueInstance = null;
|
||||
}
|
||||
var result = new TLDRule(ruleString);
|
||||
results[result.Type][result.Name] = result;
|
||||
}
|
||||
|
||||
private IDictionary<TLDRule.RuleType, IDictionary<string, TLDRule>> GetTLDRules()
|
||||
// Return our results:
|
||||
Debug.WriteLine(string.Format("Loaded {0} rules into cache.",
|
||||
results.Values.Sum(r => r.Values.Count)));
|
||||
return results;
|
||||
}
|
||||
|
||||
private IEnumerable<string> ReadRulesData()
|
||||
{
|
||||
var assembly = typeof(TLDRulesCache).GetTypeInfo().Assembly;
|
||||
var stream = assembly.GetManifestResourceStream("Bit.Icons.Resources.public_suffix_list.dat");
|
||||
string line;
|
||||
using (var reader = new StreamReader(stream))
|
||||
{
|
||||
var results = new Dictionary<TLDRule.RuleType, IDictionary<string, TLDRule>>();
|
||||
var rules = Enum.GetValues(typeof(TLDRule.RuleType)).Cast<TLDRule.RuleType>();
|
||||
foreach (var rule in rules)
|
||||
while ((line = reader.ReadLine()) != null)
|
||||
{
|
||||
results[rule] = new Dictionary<string, TLDRule>(StringComparer.CurrentCultureIgnoreCase);
|
||||
}
|
||||
|
||||
var ruleStrings = ReadRulesData();
|
||||
|
||||
// Strip out any lines that are:
|
||||
// a.) A comment
|
||||
// b.) Blank
|
||||
var rulesStrings = ruleStrings
|
||||
.Where(ruleString => !ruleString.StartsWith("//") && ruleString.Trim().Length != 0);
|
||||
foreach (var ruleString in rulesStrings)
|
||||
{
|
||||
var result = new TLDRule(ruleString);
|
||||
results[result.Type][result.Name] = result;
|
||||
}
|
||||
|
||||
// Return our results:
|
||||
Debug.WriteLine(string.Format("Loaded {0} rules into cache.",
|
||||
results.Values.Sum(r => r.Values.Count)));
|
||||
return results;
|
||||
}
|
||||
|
||||
private IEnumerable<string> ReadRulesData()
|
||||
{
|
||||
var assembly = typeof(TLDRulesCache).GetTypeInfo().Assembly;
|
||||
var stream = assembly.GetManifestResourceStream("Bit.Icons.Resources.public_suffix_list.dat");
|
||||
string line;
|
||||
using (var reader = new StreamReader(stream))
|
||||
{
|
||||
while ((line = reader.ReadLine()) != null)
|
||||
{
|
||||
yield return line;
|
||||
}
|
||||
yield return line;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,8 +1,7 @@
|
||||
namespace Bit.Icons.Models
|
||||
namespace Bit.Icons.Models;
|
||||
|
||||
public class Icon
|
||||
{
|
||||
public class Icon
|
||||
{
|
||||
public byte[] Image { get; set; }
|
||||
public string Format { get; set; }
|
||||
}
|
||||
public byte[] Image { get; set; }
|
||||
public string Format { get; set; }
|
||||
}
|
||||
|
||||
@ -1,66 +1,65 @@
|
||||
namespace Bit.Icons.Models
|
||||
{
|
||||
public class IconResult
|
||||
{
|
||||
public IconResult(string href, string sizes)
|
||||
{
|
||||
Path = href;
|
||||
if (!string.IsNullOrWhiteSpace(sizes))
|
||||
{
|
||||
var sizeParts = sizes.Split('x');
|
||||
if (sizeParts.Length == 2 && int.TryParse(sizeParts[0].Trim(), out var width) &&
|
||||
int.TryParse(sizeParts[1].Trim(), out var height))
|
||||
{
|
||||
DefinedWidth = width;
|
||||
DefinedHeight = height;
|
||||
namespace Bit.Icons.Models;
|
||||
|
||||
if (width == height)
|
||||
public class IconResult
|
||||
{
|
||||
public IconResult(string href, string sizes)
|
||||
{
|
||||
Path = href;
|
||||
if (!string.IsNullOrWhiteSpace(sizes))
|
||||
{
|
||||
var sizeParts = sizes.Split('x');
|
||||
if (sizeParts.Length == 2 && int.TryParse(sizeParts[0].Trim(), out var width) &&
|
||||
int.TryParse(sizeParts[1].Trim(), out var height))
|
||||
{
|
||||
DefinedWidth = width;
|
||||
DefinedHeight = height;
|
||||
|
||||
if (width == height)
|
||||
{
|
||||
if (width == 32)
|
||||
{
|
||||
if (width == 32)
|
||||
{
|
||||
Priority = 1;
|
||||
}
|
||||
else if (width == 64)
|
||||
{
|
||||
Priority = 2;
|
||||
}
|
||||
else if (width >= 24 && width <= 128)
|
||||
{
|
||||
Priority = 3;
|
||||
}
|
||||
else if (width == 16)
|
||||
{
|
||||
Priority = 4;
|
||||
}
|
||||
else
|
||||
{
|
||||
Priority = 100;
|
||||
}
|
||||
Priority = 1;
|
||||
}
|
||||
else if (width == 64)
|
||||
{
|
||||
Priority = 2;
|
||||
}
|
||||
else if (width >= 24 && width <= 128)
|
||||
{
|
||||
Priority = 3;
|
||||
}
|
||||
else if (width == 16)
|
||||
{
|
||||
Priority = 4;
|
||||
}
|
||||
else
|
||||
{
|
||||
Priority = 100;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Priority == 0)
|
||||
{
|
||||
Priority = 200;
|
||||
}
|
||||
}
|
||||
|
||||
public IconResult(Uri uri, byte[] bytes, string format)
|
||||
if (Priority == 0)
|
||||
{
|
||||
Path = uri.ToString();
|
||||
Icon = new Icon
|
||||
{
|
||||
Image = bytes,
|
||||
Format = format
|
||||
};
|
||||
Priority = 10;
|
||||
Priority = 200;
|
||||
}
|
||||
|
||||
public string Path { get; set; }
|
||||
public int? DefinedWidth { get; set; }
|
||||
public int? DefinedHeight { get; set; }
|
||||
public Icon Icon { get; set; }
|
||||
public int Priority { get; set; }
|
||||
}
|
||||
|
||||
public IconResult(Uri uri, byte[] bytes, string format)
|
||||
{
|
||||
Path = uri.ToString();
|
||||
Icon = new Icon
|
||||
{
|
||||
Image = bytes,
|
||||
Format = format
|
||||
};
|
||||
Priority = 10;
|
||||
}
|
||||
|
||||
public string Path { get; set; }
|
||||
public int? DefinedWidth { get; set; }
|
||||
public int? DefinedHeight { get; set; }
|
||||
public Icon Icon { get; set; }
|
||||
public int Priority { get; set; }
|
||||
}
|
||||
|
||||
@ -1,22 +1,21 @@
|
||||
using Bit.Core.Utilities;
|
||||
using Serilog.Events;
|
||||
|
||||
namespace Bit.Icons
|
||||
namespace Bit.Icons;
|
||||
|
||||
public class Program
|
||||
{
|
||||
public class Program
|
||||
public static void Main(string[] args)
|
||||
{
|
||||
public static void Main(string[] args)
|
||||
{
|
||||
Host
|
||||
.CreateDefaultBuilder(args)
|
||||
.ConfigureWebHostDefaults(webBuilder =>
|
||||
{
|
||||
webBuilder.UseStartup<Startup>();
|
||||
webBuilder.ConfigureLogging((hostingContext, logging) =>
|
||||
logging.AddSerilog(hostingContext, e => e.Level >= LogEventLevel.Error));
|
||||
})
|
||||
.Build()
|
||||
.Run();
|
||||
}
|
||||
Host
|
||||
.CreateDefaultBuilder(args)
|
||||
.ConfigureWebHostDefaults(webBuilder =>
|
||||
{
|
||||
webBuilder.UseStartup<Startup>();
|
||||
webBuilder.ConfigureLogging((hostingContext, logging) =>
|
||||
logging.AddSerilog(hostingContext, e => e.Level >= LogEventLevel.Error));
|
||||
})
|
||||
.Build()
|
||||
.Run();
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,24 +1,23 @@
|
||||
namespace Bit.Icons.Services
|
||||
namespace Bit.Icons.Services;
|
||||
|
||||
public class DomainMappingService : IDomainMappingService
|
||||
{
|
||||
public class DomainMappingService : IDomainMappingService
|
||||
private readonly Dictionary<string, string> _map = new Dictionary<string, string>
|
||||
{
|
||||
private readonly Dictionary<string, string> _map = new Dictionary<string, string>
|
||||
{
|
||||
["login.yahoo.com"] = "yahoo.com",
|
||||
["accounts.google.com"] = "google.com",
|
||||
["photo.walgreens.com"] = "walgreens.com",
|
||||
["passport.yandex.com"] = "yandex.com",
|
||||
// TODO: Add others here
|
||||
};
|
||||
["login.yahoo.com"] = "yahoo.com",
|
||||
["accounts.google.com"] = "google.com",
|
||||
["photo.walgreens.com"] = "walgreens.com",
|
||||
["passport.yandex.com"] = "yandex.com",
|
||||
// TODO: Add others here
|
||||
};
|
||||
|
||||
public string MapDomain(string hostname)
|
||||
public string MapDomain(string hostname)
|
||||
{
|
||||
if (_map.ContainsKey(hostname))
|
||||
{
|
||||
if (_map.ContainsKey(hostname))
|
||||
{
|
||||
return _map[hostname];
|
||||
}
|
||||
|
||||
return hostname;
|
||||
return _map[hostname];
|
||||
}
|
||||
|
||||
return hostname;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
namespace Bit.Icons.Services
|
||||
namespace Bit.Icons.Services;
|
||||
|
||||
public interface IDomainMappingService
|
||||
{
|
||||
public interface IDomainMappingService
|
||||
{
|
||||
string MapDomain(string hostname);
|
||||
}
|
||||
string MapDomain(string hostname);
|
||||
}
|
||||
|
||||
@ -1,9 +1,8 @@
|
||||
using Bit.Icons.Models;
|
||||
|
||||
namespace Bit.Icons.Services
|
||||
namespace Bit.Icons.Services;
|
||||
|
||||
public interface IIconFetchingService
|
||||
{
|
||||
public interface IIconFetchingService
|
||||
{
|
||||
Task<IconResult> GetIconAsync(string domain);
|
||||
}
|
||||
Task<IconResult> GetIconAsync(string domain);
|
||||
}
|
||||
|
||||
@ -3,448 +3,447 @@ using System.Text;
|
||||
using AngleSharp.Html.Parser;
|
||||
using Bit.Icons.Models;
|
||||
|
||||
namespace Bit.Icons.Services
|
||||
namespace Bit.Icons.Services;
|
||||
|
||||
public class IconFetchingService : IIconFetchingService
|
||||
{
|
||||
public class IconFetchingService : IIconFetchingService
|
||||
private readonly HashSet<string> _iconRels =
|
||||
new HashSet<string> { "icon", "apple-touch-icon", "shortcut icon" };
|
||||
private readonly HashSet<string> _blacklistedRels =
|
||||
new HashSet<string> { "preload", "image_src", "preconnect", "canonical", "alternate", "stylesheet" };
|
||||
private readonly HashSet<string> _iconExtensions =
|
||||
new HashSet<string> { ".ico", ".png", ".jpg", ".jpeg" };
|
||||
|
||||
private readonly string _pngMediaType = "image/png";
|
||||
private readonly byte[] _pngHeader = new byte[] { 137, 80, 78, 71 };
|
||||
private readonly byte[] _webpHeader = Encoding.UTF8.GetBytes("RIFF");
|
||||
|
||||
private readonly string _icoMediaType = "image/x-icon";
|
||||
private readonly string _icoAltMediaType = "image/vnd.microsoft.icon";
|
||||
private readonly byte[] _icoHeader = new byte[] { 00, 00, 01, 00 };
|
||||
|
||||
private readonly string _jpegMediaType = "image/jpeg";
|
||||
private readonly byte[] _jpegHeader = new byte[] { 255, 216, 255 };
|
||||
|
||||
private readonly HashSet<string> _allowedMediaTypes;
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly ILogger<IIconFetchingService> _logger;
|
||||
|
||||
public IconFetchingService(ILogger<IIconFetchingService> logger)
|
||||
{
|
||||
private readonly HashSet<string> _iconRels =
|
||||
new HashSet<string> { "icon", "apple-touch-icon", "shortcut icon" };
|
||||
private readonly HashSet<string> _blacklistedRels =
|
||||
new HashSet<string> { "preload", "image_src", "preconnect", "canonical", "alternate", "stylesheet" };
|
||||
private readonly HashSet<string> _iconExtensions =
|
||||
new HashSet<string> { ".ico", ".png", ".jpg", ".jpeg" };
|
||||
|
||||
private readonly string _pngMediaType = "image/png";
|
||||
private readonly byte[] _pngHeader = new byte[] { 137, 80, 78, 71 };
|
||||
private readonly byte[] _webpHeader = Encoding.UTF8.GetBytes("RIFF");
|
||||
|
||||
private readonly string _icoMediaType = "image/x-icon";
|
||||
private readonly string _icoAltMediaType = "image/vnd.microsoft.icon";
|
||||
private readonly byte[] _icoHeader = new byte[] { 00, 00, 01, 00 };
|
||||
|
||||
private readonly string _jpegMediaType = "image/jpeg";
|
||||
private readonly byte[] _jpegHeader = new byte[] { 255, 216, 255 };
|
||||
|
||||
private readonly HashSet<string> _allowedMediaTypes;
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly ILogger<IIconFetchingService> _logger;
|
||||
|
||||
public IconFetchingService(ILogger<IIconFetchingService> logger)
|
||||
_logger = logger;
|
||||
_allowedMediaTypes = new HashSet<string>
|
||||
{
|
||||
_logger = logger;
|
||||
_allowedMediaTypes = new HashSet<string>
|
||||
{
|
||||
_pngMediaType,
|
||||
_icoMediaType,
|
||||
_icoAltMediaType,
|
||||
_jpegMediaType
|
||||
};
|
||||
_pngMediaType,
|
||||
_icoMediaType,
|
||||
_icoAltMediaType,
|
||||
_jpegMediaType
|
||||
};
|
||||
|
||||
_httpClient = new HttpClient(new HttpClientHandler
|
||||
{
|
||||
AllowAutoRedirect = false,
|
||||
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
|
||||
});
|
||||
_httpClient.Timeout = TimeSpan.FromSeconds(20);
|
||||
_httpClient.MaxResponseContentBufferSize = 5000000; // 5 MB
|
||||
_httpClient = new HttpClient(new HttpClientHandler
|
||||
{
|
||||
AllowAutoRedirect = false,
|
||||
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
|
||||
});
|
||||
_httpClient.Timeout = TimeSpan.FromSeconds(20);
|
||||
_httpClient.MaxResponseContentBufferSize = 5000000; // 5 MB
|
||||
}
|
||||
|
||||
public async Task<IconResult> GetIconAsync(string domain)
|
||||
{
|
||||
if (IPAddress.TryParse(domain, out _))
|
||||
{
|
||||
_logger.LogWarning("IP address: {0}.", domain);
|
||||
return null;
|
||||
}
|
||||
|
||||
public async Task<IconResult> GetIconAsync(string domain)
|
||||
if (!Uri.TryCreate($"https://{domain}", UriKind.Absolute, out var parsedHttpsUri))
|
||||
{
|
||||
if (IPAddress.TryParse(domain, out _))
|
||||
{
|
||||
_logger.LogWarning("IP address: {0}.", domain);
|
||||
return null;
|
||||
}
|
||||
_logger.LogWarning("Bad domain: {0}.", domain);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate($"https://{domain}", UriKind.Absolute, out var parsedHttpsUri))
|
||||
{
|
||||
_logger.LogWarning("Bad domain: {0}.", domain);
|
||||
return null;
|
||||
}
|
||||
var uri = parsedHttpsUri;
|
||||
var response = await GetAndFollowAsync(uri, 2);
|
||||
if ((response == null || !response.IsSuccessStatusCode) &&
|
||||
Uri.TryCreate($"http://{parsedHttpsUri.Host}", UriKind.Absolute, out var parsedHttpUri))
|
||||
{
|
||||
Cleanup(response);
|
||||
uri = parsedHttpUri;
|
||||
response = await GetAndFollowAsync(uri, 2);
|
||||
|
||||
var uri = parsedHttpsUri;
|
||||
var response = await GetAndFollowAsync(uri, 2);
|
||||
if ((response == null || !response.IsSuccessStatusCode) &&
|
||||
Uri.TryCreate($"http://{parsedHttpsUri.Host}", UriKind.Absolute, out var parsedHttpUri))
|
||||
if (response == null || !response.IsSuccessStatusCode)
|
||||
{
|
||||
Cleanup(response);
|
||||
uri = parsedHttpUri;
|
||||
response = await GetAndFollowAsync(uri, 2);
|
||||
|
||||
if (response == null || !response.IsSuccessStatusCode)
|
||||
var dotCount = domain.Count(c => c == '.');
|
||||
if (dotCount > 1 && DomainName.TryParseBaseDomain(domain, out var baseDomain) &&
|
||||
Uri.TryCreate($"https://{baseDomain}", UriKind.Absolute, out var parsedBaseUri))
|
||||
{
|
||||
var dotCount = domain.Count(c => c == '.');
|
||||
if (dotCount > 1 && DomainName.TryParseBaseDomain(domain, out var baseDomain) &&
|
||||
Uri.TryCreate($"https://{baseDomain}", UriKind.Absolute, out var parsedBaseUri))
|
||||
{
|
||||
Cleanup(response);
|
||||
uri = parsedBaseUri;
|
||||
response = await GetAndFollowAsync(uri, 2);
|
||||
}
|
||||
else if (dotCount < 2 &&
|
||||
Uri.TryCreate($"https://www.{parsedHttpsUri.Host}", UriKind.Absolute, out var parsedWwwUri))
|
||||
{
|
||||
Cleanup(response);
|
||||
uri = parsedWwwUri;
|
||||
response = await GetAndFollowAsync(uri, 2);
|
||||
}
|
||||
Cleanup(response);
|
||||
uri = parsedBaseUri;
|
||||
response = await GetAndFollowAsync(uri, 2);
|
||||
}
|
||||
else if (dotCount < 2 &&
|
||||
Uri.TryCreate($"https://www.{parsedHttpsUri.Host}", UriKind.Absolute, out var parsedWwwUri))
|
||||
{
|
||||
Cleanup(response);
|
||||
uri = parsedWwwUri;
|
||||
response = await GetAndFollowAsync(uri, 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (response?.Content == null || !response.IsSuccessStatusCode)
|
||||
if (response?.Content == null || !response.IsSuccessStatusCode)
|
||||
{
|
||||
_logger.LogWarning("Couldn't load a website for {0}: {1}.", domain,
|
||||
response?.StatusCode.ToString() ?? "null");
|
||||
Cleanup(response);
|
||||
return null;
|
||||
}
|
||||
|
||||
var parser = new HtmlParser();
|
||||
using (response)
|
||||
using (var htmlStream = await response.Content.ReadAsStreamAsync())
|
||||
using (var document = await parser.ParseDocumentAsync(htmlStream))
|
||||
{
|
||||
uri = response.RequestMessage.RequestUri;
|
||||
if (document.DocumentElement == null)
|
||||
{
|
||||
_logger.LogWarning("Couldn't load a website for {0}: {1}.", domain,
|
||||
response?.StatusCode.ToString() ?? "null");
|
||||
Cleanup(response);
|
||||
_logger.LogWarning("No DocumentElement for {0}.", domain);
|
||||
return null;
|
||||
}
|
||||
|
||||
var parser = new HtmlParser();
|
||||
using (response)
|
||||
using (var htmlStream = await response.Content.ReadAsStreamAsync())
|
||||
using (var document = await parser.ParseDocumentAsync(htmlStream))
|
||||
var baseUrl = "/";
|
||||
var baseUrlNode = document.QuerySelector("head base[href]");
|
||||
if (baseUrlNode != null)
|
||||
{
|
||||
uri = response.RequestMessage.RequestUri;
|
||||
if (document.DocumentElement == null)
|
||||
var hrefAttr = baseUrlNode.Attributes["href"];
|
||||
if (!string.IsNullOrWhiteSpace(hrefAttr?.Value))
|
||||
{
|
||||
_logger.LogWarning("No DocumentElement for {0}.", domain);
|
||||
return null;
|
||||
baseUrl = hrefAttr.Value;
|
||||
}
|
||||
|
||||
var baseUrl = "/";
|
||||
var baseUrlNode = document.QuerySelector("head base[href]");
|
||||
if (baseUrlNode != null)
|
||||
baseUrlNode = null;
|
||||
hrefAttr = null;
|
||||
}
|
||||
|
||||
var icons = new List<IconResult>();
|
||||
var links = document.QuerySelectorAll("head link[href]");
|
||||
if (links != null)
|
||||
{
|
||||
foreach (var link in links.Take(200))
|
||||
{
|
||||
var hrefAttr = baseUrlNode.Attributes["href"];
|
||||
if (!string.IsNullOrWhiteSpace(hrefAttr?.Value))
|
||||
var hrefAttr = link.Attributes["href"];
|
||||
if (string.IsNullOrWhiteSpace(hrefAttr?.Value))
|
||||
{
|
||||
baseUrl = hrefAttr.Value;
|
||||
continue;
|
||||
}
|
||||
|
||||
baseUrlNode = null;
|
||||
var relAttr = link.Attributes["rel"];
|
||||
var sizesAttr = link.Attributes["sizes"];
|
||||
if (relAttr != null && _iconRels.Contains(relAttr.Value.ToLower()))
|
||||
{
|
||||
icons.Add(new IconResult(hrefAttr.Value, sizesAttr?.Value));
|
||||
}
|
||||
else if (relAttr == null || !_blacklistedRels.Contains(relAttr.Value.ToLower()))
|
||||
{
|
||||
try
|
||||
{
|
||||
var extension = Path.GetExtension(hrefAttr.Value);
|
||||
if (_iconExtensions.Contains(extension.ToLower()))
|
||||
{
|
||||
icons.Add(new IconResult(hrefAttr.Value, sizesAttr?.Value));
|
||||
}
|
||||
}
|
||||
catch (ArgumentException) { }
|
||||
}
|
||||
|
||||
sizesAttr = null;
|
||||
relAttr = null;
|
||||
hrefAttr = null;
|
||||
}
|
||||
|
||||
var icons = new List<IconResult>();
|
||||
var links = document.QuerySelectorAll("head link[href]");
|
||||
if (links != null)
|
||||
{
|
||||
foreach (var link in links.Take(200))
|
||||
{
|
||||
var hrefAttr = link.Attributes["href"];
|
||||
if (string.IsNullOrWhiteSpace(hrefAttr?.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var relAttr = link.Attributes["rel"];
|
||||
var sizesAttr = link.Attributes["sizes"];
|
||||
if (relAttr != null && _iconRels.Contains(relAttr.Value.ToLower()))
|
||||
{
|
||||
icons.Add(new IconResult(hrefAttr.Value, sizesAttr?.Value));
|
||||
}
|
||||
else if (relAttr == null || !_blacklistedRels.Contains(relAttr.Value.ToLower()))
|
||||
{
|
||||
try
|
||||
{
|
||||
var extension = Path.GetExtension(hrefAttr.Value);
|
||||
if (_iconExtensions.Contains(extension.ToLower()))
|
||||
{
|
||||
icons.Add(new IconResult(hrefAttr.Value, sizesAttr?.Value));
|
||||
}
|
||||
}
|
||||
catch (ArgumentException) { }
|
||||
}
|
||||
|
||||
sizesAttr = null;
|
||||
relAttr = null;
|
||||
hrefAttr = null;
|
||||
}
|
||||
|
||||
links = null;
|
||||
}
|
||||
|
||||
var iconResultTasks = new List<Task>();
|
||||
foreach (var icon in icons.OrderBy(i => i.Priority).Take(10))
|
||||
{
|
||||
Uri iconUri = null;
|
||||
if (icon.Path.StartsWith("//") && Uri.TryCreate($"{GetScheme(uri)}://{icon.Path.Substring(2)}",
|
||||
UriKind.Absolute, out var slashUri))
|
||||
{
|
||||
iconUri = slashUri;
|
||||
}
|
||||
else if (Uri.TryCreate(icon.Path, UriKind.Relative, out var relUri))
|
||||
{
|
||||
iconUri = ResolveUri($"{GetScheme(uri)}://{uri.Host}", baseUrl, relUri.OriginalString);
|
||||
}
|
||||
else if (Uri.TryCreate(icon.Path, UriKind.Absolute, out var absUri))
|
||||
{
|
||||
iconUri = absUri;
|
||||
}
|
||||
|
||||
if (iconUri != null)
|
||||
{
|
||||
var task = GetIconAsync(iconUri).ContinueWith(async (r) =>
|
||||
{
|
||||
var result = await r;
|
||||
if (result != null)
|
||||
{
|
||||
icon.Path = iconUri.ToString();
|
||||
icon.Icon = result.Icon;
|
||||
}
|
||||
});
|
||||
iconResultTasks.Add(task);
|
||||
}
|
||||
}
|
||||
|
||||
await Task.WhenAll(iconResultTasks);
|
||||
if (!icons.Any(i => i.Icon != null))
|
||||
{
|
||||
var faviconUri = ResolveUri($"{GetScheme(uri)}://{uri.Host}", "favicon.ico");
|
||||
var result = await GetIconAsync(faviconUri);
|
||||
if (result != null)
|
||||
{
|
||||
icons.Add(result);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning("No favicon.ico found for {0}.", uri.Host);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return icons.Where(i => i.Icon != null).OrderBy(i => i.Priority).First();
|
||||
links = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IconResult> GetIconAsync(Uri uri)
|
||||
{
|
||||
using (var response = await GetAndFollowAsync(uri, 2))
|
||||
var iconResultTasks = new List<Task>();
|
||||
foreach (var icon in icons.OrderBy(i => i.Priority).Take(10))
|
||||
{
|
||||
if (response?.Content?.Headers == null || !response.IsSuccessStatusCode)
|
||||
Uri iconUri = null;
|
||||
if (icon.Path.StartsWith("//") && Uri.TryCreate($"{GetScheme(uri)}://{icon.Path.Substring(2)}",
|
||||
UriKind.Absolute, out var slashUri))
|
||||
{
|
||||
response?.Content?.Dispose();
|
||||
iconUri = slashUri;
|
||||
}
|
||||
else if (Uri.TryCreate(icon.Path, UriKind.Relative, out var relUri))
|
||||
{
|
||||
iconUri = ResolveUri($"{GetScheme(uri)}://{uri.Host}", baseUrl, relUri.OriginalString);
|
||||
}
|
||||
else if (Uri.TryCreate(icon.Path, UriKind.Absolute, out var absUri))
|
||||
{
|
||||
iconUri = absUri;
|
||||
}
|
||||
|
||||
if (iconUri != null)
|
||||
{
|
||||
var task = GetIconAsync(iconUri).ContinueWith(async (r) =>
|
||||
{
|
||||
var result = await r;
|
||||
if (result != null)
|
||||
{
|
||||
icon.Path = iconUri.ToString();
|
||||
icon.Icon = result.Icon;
|
||||
}
|
||||
});
|
||||
iconResultTasks.Add(task);
|
||||
}
|
||||
}
|
||||
|
||||
await Task.WhenAll(iconResultTasks);
|
||||
if (!icons.Any(i => i.Icon != null))
|
||||
{
|
||||
var faviconUri = ResolveUri($"{GetScheme(uri)}://{uri.Host}", "favicon.ico");
|
||||
var result = await GetIconAsync(faviconUri);
|
||||
if (result != null)
|
||||
{
|
||||
icons.Add(result);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning("No favicon.ico found for {0}.", uri.Host);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
var format = response.Content.Headers?.ContentType?.MediaType;
|
||||
var bytes = await response.Content.ReadAsByteArrayAsync();
|
||||
response.Content.Dispose();
|
||||
if (format == null || !_allowedMediaTypes.Contains(format))
|
||||
return icons.Where(i => i.Icon != null).OrderBy(i => i.Priority).First();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IconResult> GetIconAsync(Uri uri)
|
||||
{
|
||||
using (var response = await GetAndFollowAsync(uri, 2))
|
||||
{
|
||||
if (response?.Content?.Headers == null || !response.IsSuccessStatusCode)
|
||||
{
|
||||
response?.Content?.Dispose();
|
||||
return null;
|
||||
}
|
||||
|
||||
var format = response.Content.Headers?.ContentType?.MediaType;
|
||||
var bytes = await response.Content.ReadAsByteArrayAsync();
|
||||
response.Content.Dispose();
|
||||
if (format == null || !_allowedMediaTypes.Contains(format))
|
||||
{
|
||||
if (HeaderMatch(bytes, _icoHeader))
|
||||
{
|
||||
if (HeaderMatch(bytes, _icoHeader))
|
||||
{
|
||||
format = _icoMediaType;
|
||||
}
|
||||
else if (HeaderMatch(bytes, _pngHeader) || HeaderMatch(bytes, _webpHeader))
|
||||
{
|
||||
format = _pngMediaType;
|
||||
}
|
||||
else if (HeaderMatch(bytes, _jpegHeader))
|
||||
{
|
||||
format = _jpegMediaType;
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
format = _icoMediaType;
|
||||
}
|
||||
else if (HeaderMatch(bytes, _pngHeader) || HeaderMatch(bytes, _webpHeader))
|
||||
{
|
||||
format = _pngMediaType;
|
||||
}
|
||||
else if (HeaderMatch(bytes, _jpegHeader))
|
||||
{
|
||||
format = _jpegMediaType;
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new IconResult(uri, bytes, format);
|
||||
}
|
||||
|
||||
return new IconResult(uri, bytes, format);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<HttpResponseMessage> GetAndFollowAsync(Uri uri, int maxRedirectCount)
|
||||
{
|
||||
var response = await GetAsync(uri);
|
||||
if (response == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
return await FollowRedirectsAsync(response, maxRedirectCount);
|
||||
}
|
||||
|
||||
private async Task<HttpResponseMessage> GetAsync(Uri uri)
|
||||
{
|
||||
if (uri == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
private async Task<HttpResponseMessage> GetAndFollowAsync(Uri uri, int maxRedirectCount)
|
||||
// Prevent non-http(s) and non-default ports
|
||||
if ((uri.Scheme != "http" && uri.Scheme != "https") || !uri.IsDefaultPort)
|
||||
{
|
||||
var response = await GetAsync(uri);
|
||||
if (response == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
return await FollowRedirectsAsync(response, maxRedirectCount);
|
||||
return null;
|
||||
}
|
||||
|
||||
private async Task<HttpResponseMessage> GetAsync(Uri uri)
|
||||
// Prevent local hosts (localhost, bobs-pc, etc) and IP addresses
|
||||
if (!uri.Host.Contains(".") || IPAddress.TryParse(uri.Host, out _))
|
||||
{
|
||||
if (uri == null)
|
||||
return null;
|
||||
}
|
||||
|
||||
// Resolve host to make sure it is not an internal/private IP address
|
||||
try
|
||||
{
|
||||
var hostEntry = Dns.GetHostEntry(uri.Host);
|
||||
if (hostEntry?.AddressList.Any(ip => IsInternal(ip)) ?? true)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
// Prevent non-http(s) and non-default ports
|
||||
if ((uri.Scheme != "http" && uri.Scheme != "https") || !uri.IsDefaultPort)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
using (var message = new HttpRequestMessage())
|
||||
{
|
||||
message.RequestUri = uri;
|
||||
message.Method = HttpMethod.Get;
|
||||
|
||||
// Prevent local hosts (localhost, bobs-pc, etc) and IP addresses
|
||||
if (!uri.Host.Contains(".") || IPAddress.TryParse(uri.Host, out _))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
// Let's add some headers to look like we're coming from a web browser request. Some websites
|
||||
// will block our request without these.
|
||||
message.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
||||
"(KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299");
|
||||
message.Headers.Add("Accept-Language", "en-US,en;q=0.8");
|
||||
message.Headers.Add("Cache-Control", "no-cache");
|
||||
message.Headers.Add("Pragma", "no-cache");
|
||||
message.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;" +
|
||||
"q=0.9,image/webp,image/apng,*/*;q=0.8");
|
||||
|
||||
// Resolve host to make sure it is not an internal/private IP address
|
||||
try
|
||||
{
|
||||
var hostEntry = Dns.GetHostEntry(uri.Host);
|
||||
if (hostEntry?.AddressList.Any(ip => IsInternal(ip)) ?? true)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
return await _httpClient.SendAsync(message);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
using (var message = new HttpRequestMessage())
|
||||
{
|
||||
message.RequestUri = uri;
|
||||
message.Method = HttpMethod.Get;
|
||||
|
||||
// Let's add some headers to look like we're coming from a web browser request. Some websites
|
||||
// will block our request without these.
|
||||
message.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
||||
"(KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299");
|
||||
message.Headers.Add("Accept-Language", "en-US,en;q=0.8");
|
||||
message.Headers.Add("Cache-Control", "no-cache");
|
||||
message.Headers.Add("Pragma", "no-cache");
|
||||
message.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;" +
|
||||
"q=0.9,image/webp,image/apng,*/*;q=0.8");
|
||||
|
||||
try
|
||||
{
|
||||
return await _httpClient.SendAsync(message);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
private async Task<HttpResponseMessage> FollowRedirectsAsync(HttpResponseMessage response,
|
||||
int maxFollowCount, int followCount = 0)
|
||||
{
|
||||
if (response == null || response.IsSuccessStatusCode || followCount > maxFollowCount)
|
||||
{
|
||||
return response;
|
||||
}
|
||||
|
||||
private async Task<HttpResponseMessage> FollowRedirectsAsync(HttpResponseMessage response,
|
||||
int maxFollowCount, int followCount = 0)
|
||||
if (!(response.StatusCode == HttpStatusCode.Redirect ||
|
||||
response.StatusCode == HttpStatusCode.MovedPermanently ||
|
||||
response.StatusCode == HttpStatusCode.RedirectKeepVerb ||
|
||||
response.StatusCode == HttpStatusCode.SeeOther) ||
|
||||
response.Headers.Location == null)
|
||||
{
|
||||
if (response == null || response.IsSuccessStatusCode || followCount > maxFollowCount)
|
||||
{
|
||||
return response;
|
||||
}
|
||||
Cleanup(response);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!(response.StatusCode == HttpStatusCode.Redirect ||
|
||||
response.StatusCode == HttpStatusCode.MovedPermanently ||
|
||||
response.StatusCode == HttpStatusCode.RedirectKeepVerb ||
|
||||
response.StatusCode == HttpStatusCode.SeeOther) ||
|
||||
response.Headers.Location == null)
|
||||
Uri location = null;
|
||||
if (response.Headers.Location.IsAbsoluteUri)
|
||||
{
|
||||
if (response.Headers.Location.Scheme != "http" && response.Headers.Location.Scheme != "https")
|
||||
{
|
||||
Cleanup(response);
|
||||
return null;
|
||||
}
|
||||
|
||||
Uri location = null;
|
||||
if (response.Headers.Location.IsAbsoluteUri)
|
||||
{
|
||||
if (response.Headers.Location.Scheme != "http" && response.Headers.Location.Scheme != "https")
|
||||
if (Uri.TryCreate($"https://{response.Headers.Location.OriginalString}",
|
||||
UriKind.Absolute, out var newUri))
|
||||
{
|
||||
if (Uri.TryCreate($"https://{response.Headers.Location.OriginalString}",
|
||||
UriKind.Absolute, out var newUri))
|
||||
{
|
||||
location = newUri;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
location = response.Headers.Location;
|
||||
location = newUri;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
var requestUri = response.RequestMessage.RequestUri;
|
||||
location = ResolveUri($"{GetScheme(requestUri)}://{requestUri.Host}",
|
||||
response.Headers.Location.OriginalString);
|
||||
location = response.Headers.Location;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
var requestUri = response.RequestMessage.RequestUri;
|
||||
location = ResolveUri($"{GetScheme(requestUri)}://{requestUri.Host}",
|
||||
response.Headers.Location.OriginalString);
|
||||
}
|
||||
|
||||
Cleanup(response);
|
||||
var newResponse = await GetAsync(location);
|
||||
if (newResponse != null)
|
||||
Cleanup(response);
|
||||
var newResponse = await GetAsync(location);
|
||||
if (newResponse != null)
|
||||
{
|
||||
followCount++;
|
||||
var redirectedResponse = await FollowRedirectsAsync(newResponse, maxFollowCount, followCount);
|
||||
if (redirectedResponse != null)
|
||||
{
|
||||
followCount++;
|
||||
var redirectedResponse = await FollowRedirectsAsync(newResponse, maxFollowCount, followCount);
|
||||
if (redirectedResponse != null)
|
||||
if (redirectedResponse != newResponse)
|
||||
{
|
||||
if (redirectedResponse != newResponse)
|
||||
{
|
||||
Cleanup(newResponse);
|
||||
}
|
||||
return redirectedResponse;
|
||||
Cleanup(newResponse);
|
||||
}
|
||||
return redirectedResponse;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private bool HeaderMatch(byte[] imageBytes, byte[] header)
|
||||
{
|
||||
return imageBytes.Length >= header.Length && header.SequenceEqual(imageBytes.Take(header.Length));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private Uri ResolveUri(string baseUrl, params string[] paths)
|
||||
private bool HeaderMatch(byte[] imageBytes, byte[] header)
|
||||
{
|
||||
return imageBytes.Length >= header.Length && header.SequenceEqual(imageBytes.Take(header.Length));
|
||||
}
|
||||
|
||||
private Uri ResolveUri(string baseUrl, params string[] paths)
|
||||
{
|
||||
var url = baseUrl;
|
||||
foreach (var path in paths)
|
||||
{
|
||||
var url = baseUrl;
|
||||
foreach (var path in paths)
|
||||
if (Uri.TryCreate(new Uri(url), path, out var r))
|
||||
{
|
||||
if (Uri.TryCreate(new Uri(url), path, out var r))
|
||||
{
|
||||
url = r.ToString();
|
||||
}
|
||||
url = r.ToString();
|
||||
}
|
||||
return new Uri(url);
|
||||
}
|
||||
return new Uri(url);
|
||||
}
|
||||
|
||||
private void Cleanup(IDisposable obj)
|
||||
private void Cleanup(IDisposable obj)
|
||||
{
|
||||
obj?.Dispose();
|
||||
obj = null;
|
||||
}
|
||||
|
||||
private string GetScheme(Uri uri)
|
||||
{
|
||||
return uri != null && uri.Scheme == "http" ? "http" : "https";
|
||||
}
|
||||
|
||||
public static bool IsInternal(IPAddress ip)
|
||||
{
|
||||
if (IPAddress.IsLoopback(ip))
|
||||
{
|
||||
obj?.Dispose();
|
||||
obj = null;
|
||||
return true;
|
||||
}
|
||||
|
||||
private string GetScheme(Uri uri)
|
||||
var ipString = ip.ToString();
|
||||
if (ipString == "::1" || ipString == "::" || ipString.StartsWith("::ffff:"))
|
||||
{
|
||||
return uri != null && uri.Scheme == "http" ? "http" : "https";
|
||||
return true;
|
||||
}
|
||||
|
||||
public static bool IsInternal(IPAddress ip)
|
||||
// IPv6
|
||||
if (ip.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6)
|
||||
{
|
||||
if (IPAddress.IsLoopback(ip))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var ipString = ip.ToString();
|
||||
if (ipString == "::1" || ipString == "::" || ipString.StartsWith("::ffff:"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// IPv6
|
||||
if (ip.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6)
|
||||
{
|
||||
return ipString.StartsWith("fc") || ipString.StartsWith("fd") ||
|
||||
ipString.StartsWith("fe") || ipString.StartsWith("ff");
|
||||
}
|
||||
|
||||
// IPv4
|
||||
var bytes = ip.GetAddressBytes();
|
||||
return (bytes[0]) switch
|
||||
{
|
||||
0 => true,
|
||||
10 => true,
|
||||
127 => true,
|
||||
169 => bytes[1] == 254, // Cloud environments, such as AWS
|
||||
172 => bytes[1] < 32 && bytes[1] >= 16,
|
||||
192 => bytes[1] == 168,
|
||||
_ => false,
|
||||
};
|
||||
return ipString.StartsWith("fc") || ipString.StartsWith("fd") ||
|
||||
ipString.StartsWith("fe") || ipString.StartsWith("ff");
|
||||
}
|
||||
|
||||
// IPv4
|
||||
var bytes = ip.GetAddressBytes();
|
||||
return (bytes[0]) switch
|
||||
{
|
||||
0 => true,
|
||||
10 => true,
|
||||
127 => true,
|
||||
169 => bytes[1] == 254, // Cloud environments, such as AWS
|
||||
172 => bytes[1] < 32 && bytes[1] >= 16,
|
||||
192 => bytes[1] == 168,
|
||||
_ => false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@ -5,73 +5,72 @@ using Bit.Icons.Services;
|
||||
using Bit.SharedWeb.Utilities;
|
||||
using Microsoft.Net.Http.Headers;
|
||||
|
||||
namespace Bit.Icons
|
||||
namespace Bit.Icons;
|
||||
|
||||
public class Startup
|
||||
{
|
||||
public class Startup
|
||||
public Startup(IWebHostEnvironment env, IConfiguration configuration)
|
||||
{
|
||||
public Startup(IWebHostEnvironment env, IConfiguration configuration)
|
||||
CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("en-US");
|
||||
Configuration = configuration;
|
||||
Environment = env;
|
||||
}
|
||||
|
||||
public IConfiguration Configuration { get; }
|
||||
public IWebHostEnvironment Environment { get; }
|
||||
|
||||
public void ConfigureServices(IServiceCollection services)
|
||||
{
|
||||
// Options
|
||||
services.AddOptions();
|
||||
|
||||
// Settings
|
||||
var globalSettings = services.AddGlobalSettingsServices(Configuration, Environment);
|
||||
var iconsSettings = new IconsSettings();
|
||||
ConfigurationBinder.Bind(Configuration.GetSection("IconsSettings"), iconsSettings);
|
||||
services.AddSingleton(s => iconsSettings);
|
||||
|
||||
// Cache
|
||||
services.AddMemoryCache(options =>
|
||||
{
|
||||
CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("en-US");
|
||||
Configuration = configuration;
|
||||
Environment = env;
|
||||
options.SizeLimit = iconsSettings.CacheSizeLimit;
|
||||
});
|
||||
|
||||
// Services
|
||||
services.AddSingleton<IDomainMappingService, DomainMappingService>();
|
||||
services.AddSingleton<IIconFetchingService, IconFetchingService>();
|
||||
|
||||
// Mvc
|
||||
services.AddMvc();
|
||||
}
|
||||
|
||||
public void Configure(
|
||||
IApplicationBuilder app,
|
||||
IWebHostEnvironment env,
|
||||
IHostApplicationLifetime appLifetime,
|
||||
GlobalSettings globalSettings)
|
||||
{
|
||||
app.UseSerilog(env, appLifetime, globalSettings);
|
||||
|
||||
// Add general security headers
|
||||
app.UseMiddleware<SecurityHeadersMiddleware>();
|
||||
|
||||
if (env.IsDevelopment())
|
||||
{
|
||||
app.UseDeveloperExceptionPage();
|
||||
}
|
||||
|
||||
public IConfiguration Configuration { get; }
|
||||
public IWebHostEnvironment Environment { get; }
|
||||
|
||||
public void ConfigureServices(IServiceCollection services)
|
||||
app.Use(async (context, next) =>
|
||||
{
|
||||
// Options
|
||||
services.AddOptions();
|
||||
|
||||
// Settings
|
||||
var globalSettings = services.AddGlobalSettingsServices(Configuration, Environment);
|
||||
var iconsSettings = new IconsSettings();
|
||||
ConfigurationBinder.Bind(Configuration.GetSection("IconsSettings"), iconsSettings);
|
||||
services.AddSingleton(s => iconsSettings);
|
||||
|
||||
// Cache
|
||||
services.AddMemoryCache(options =>
|
||||
context.Response.GetTypedHeaders().CacheControl = new CacheControlHeaderValue
|
||||
{
|
||||
options.SizeLimit = iconsSettings.CacheSizeLimit;
|
||||
});
|
||||
Public = true,
|
||||
MaxAge = TimeSpan.FromDays(7)
|
||||
};
|
||||
await next();
|
||||
});
|
||||
|
||||
// Services
|
||||
services.AddSingleton<IDomainMappingService, DomainMappingService>();
|
||||
services.AddSingleton<IIconFetchingService, IconFetchingService>();
|
||||
|
||||
// Mvc
|
||||
services.AddMvc();
|
||||
}
|
||||
|
||||
public void Configure(
|
||||
IApplicationBuilder app,
|
||||
IWebHostEnvironment env,
|
||||
IHostApplicationLifetime appLifetime,
|
||||
GlobalSettings globalSettings)
|
||||
{
|
||||
app.UseSerilog(env, appLifetime, globalSettings);
|
||||
|
||||
// Add general security headers
|
||||
app.UseMiddleware<SecurityHeadersMiddleware>();
|
||||
|
||||
if (env.IsDevelopment())
|
||||
{
|
||||
app.UseDeveloperExceptionPage();
|
||||
}
|
||||
|
||||
app.Use(async (context, next) =>
|
||||
{
|
||||
context.Response.GetTypedHeaders().CacheControl = new CacheControlHeaderValue
|
||||
{
|
||||
Public = true,
|
||||
MaxAge = TimeSpan.FromDays(7)
|
||||
};
|
||||
await next();
|
||||
});
|
||||
|
||||
app.UseRouting();
|
||||
app.UseEndpoints(endpoints => endpoints.MapDefaultControllerRoute());
|
||||
}
|
||||
app.UseRouting();
|
||||
app.UseEndpoints(endpoints => endpoints.MapDefaultControllerRoute());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user