diff --git a/src/Icons/Controllers/IconsController.cs b/src/Icons/Controllers/IconsController.cs index a0ea8815f6..350af4111b 100644 --- a/src/Icons/Controllers/IconsController.cs +++ b/src/Icons/Controllers/IconsController.cs @@ -1,8 +1,4 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; using System.Threading.Tasks; using Bit.Icons.Models; using Bit.Icons.Services; @@ -14,37 +10,20 @@ namespace Bit.Icons.Controllers [Route("")] public class IconsController : Controller { - private static readonly HttpClient _httpClient = new HttpClient(new HttpClientHandler - { - AllowAutoRedirect = false, - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate - }); - private static string _pngMediaType = "image/png"; - private static byte[] _pngHeader = new byte[] { 137, 80, 78, 71 }; - private static string _icoMediaType = "image/x-icon"; - private static string _icoAltMediaType = "image/vnd.microsoft.icon"; - private static byte[] _icoHeader = new byte[] { 00, 00, 01, 00 }; - private static string _jpegMediaType = "image/jpeg"; - private static byte[] _jpegHeader = new byte[] { 255, 216, 255 }; - private static string _octetMediaType = "application/octet-stream"; - private static readonly HashSet _allowedMediaTypes = new HashSet{ - _pngMediaType, - _icoMediaType, - _icoAltMediaType, - _jpegMediaType, - _octetMediaType - }; private readonly IMemoryCache _memoryCache; private readonly IDomainMappingService _domainMappingService; + private readonly IIconFetchingService _iconFetchingService; private readonly IconsSettings _iconsSettings; public IconsController( IMemoryCache memoryCache, IDomainMappingService domainMappingService, + IIconFetchingService iconFetchingService, IconsSettings iconsSettings) { _memoryCache = memoryCache; _domainMappingService = domainMappingService; + _iconFetchingService = iconFetchingService; _iconsSettings = iconsSettings; } @@ -66,46 +45,16 @@ namespace Bit.Icons.Controllers var mappedDomain = _domainMappingService.MapDomain(uri.Host); if(!_memoryCache.TryGetValue(mappedDomain, out Icon icon)) { - var iconUrl = new Uri($"{_iconsSettings.BestIconBaseUrl}/icon" + - $"?url={mappedDomain}&size=16..32..256&fallback_icon_url=" + - $"https://raw.githubusercontent.com/bitwarden/web/master/src/images/fa-globe.png"); - var response = await _httpClient.GetAsync(iconUrl); - response = await FollowRedirectsAsync(response, 1); - if(!response.IsSuccessStatusCode || - !_allowedMediaTypes.Contains(response.Content.Headers.ContentType.MediaType)) + var result = await _iconFetchingService.GetIconAsync(mappedDomain); + if(result == null) { return new NotFoundResult(); } - var image = await response.Content.ReadAsByteArrayAsync(); - icon = new Icon - { - Image = image, - Format = response.Content.Headers.ContentType.MediaType - }; - - if(icon.Format == _octetMediaType) - { - if(HeaderMatch(icon, _icoHeader)) - { - icon.Format = _icoMediaType; - } - else if(HeaderMatch(icon, _pngHeader)) - { - icon.Format = _pngMediaType; - } - else if(HeaderMatch(icon, _jpegHeader)) - { - icon.Format = _jpegMediaType; - } - else - { - return new NotFoundResult(); - } - } + icon = result.Icon; // Only cache smaller images (<= 50kb) - if(image.Length <= 50012) + if(icon.Image.Length <= 50012) { _memoryCache.Set(mappedDomain, icon, new MemoryCacheEntryOptions { @@ -116,47 +65,5 @@ namespace Bit.Icons.Controllers return new FileContentResult(icon.Image, icon.Format); } - - private async Task FollowRedirectsAsync(HttpResponseMessage response, int followCount) - { - if(response.IsSuccessStatusCode || followCount > 2) - { - return response; - } - - if((response.StatusCode == HttpStatusCode.Redirect || response.StatusCode == HttpStatusCode.MovedPermanently) && - response.Headers.Contains("Location")) - { - var locationHeader = response.Headers.GetValues("Location").FirstOrDefault(); - if(!string.IsNullOrWhiteSpace(locationHeader) && - Uri.TryCreate(locationHeader, UriKind.Absolute, out Uri location)) - { - var message = new HttpRequestMessage - { - RequestUri = location, - Method = HttpMethod.Get - }; - - // Let's add some headers to look like we're coming from a web browser request. Some websites - // will block our request without these. - message.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); - message.Headers.Add("Accept-Language", "en-US,en;q=0.8"); - message.Headers.Add("Cache-Control", "no-cache"); - message.Headers.Add("Pragma", "no-cache"); - message.Headers.Add("Accept", "image/webp,image/apng,image/*,*/*;q=0.8"); - - response = await _httpClient.SendAsync(message); - response = await FollowRedirectsAsync(response, followCount++); - } - } - - return response; - } - - private bool HeaderMatch(Icon icon, byte[] header) - { - return icon.Image.Length >= header.Length && header.SequenceEqual(icon.Image.Take(header.Length)); - } } } diff --git a/src/Icons/Dockerfile b/src/Icons/Dockerfile index 2bc5c6c2e1..ddb8bde229 100644 --- a/src/Icons/Dockerfile +++ b/src/Icons/Dockerfile @@ -2,17 +2,9 @@ FROM microsoft/aspnetcore:2.0.6 RUN apt-get update \ && apt-get install -y --no-install-recommends \ - unzip \ gosu \ && rm -rf /var/lib/apt/lists/* -WORKDIR /tmp -COPY iconserver.sha256 . -RUN curl -L -o iconserver.zip https://github.com/mat/besticon/releases/download/v3.6.0/iconserver_linux_amd64.zip \ - && sha256sum -c iconserver.sha256 \ - && unzip iconserver.zip -d /etc/iconserver \ - && rm iconserver.* - ENV ASPNETCORE_URLS http://+:5000 WORKDIR /app EXPOSE 5000 diff --git a/src/Icons/Icons.csproj b/src/Icons/Icons.csproj index 4dc118d6ef..2509fb2bdf 100644 --- a/src/Icons/Icons.csproj +++ b/src/Icons/Icons.csproj @@ -9,6 +9,7 @@ + diff --git a/src/Icons/IconsSettings.cs b/src/Icons/IconsSettings.cs index 878f3f3170..809df7d09f 100644 --- a/src/Icons/IconsSettings.cs +++ b/src/Icons/IconsSettings.cs @@ -2,7 +2,6 @@ { public class IconsSettings { - public virtual string BestIconBaseUrl { get; set; } public virtual int CacheHours { get; set; } public virtual long? CacheSizeLimit { get; set; } } diff --git a/src/Icons/Models/IconResult.cs b/src/Icons/Models/IconResult.cs new file mode 100644 index 0000000000..d8a2c723e4 --- /dev/null +++ b/src/Icons/Models/IconResult.cs @@ -0,0 +1,70 @@ +using System; +using HtmlAgilityPack; + +namespace Bit.Icons.Models +{ + public class IconResult + { + public IconResult(string href, HtmlNode node) + { + Path = href; + var sizesAttr = node.Attributes["sizes"]; + if(!string.IsNullOrWhiteSpace(sizesAttr?.Value)) + { + var sizeParts = sizesAttr.Value.Split('x'); + if(sizeParts.Length == 2 && int.TryParse(sizeParts[0].Trim(), out var width) && + int.TryParse(sizeParts[1].Trim(), out var height)) + { + DefinedWidth = width; + DefinedHeight = height; + + if(width == height) + { + if(width == 32) + { + Priority = 1; + } + else if(width == 64) + { + Priority = 2; + } + else if(width >= 24 && width <= 128) + { + Priority = 3; + } + else if(width == 16) + { + Priority = 4; + } + else + { + Priority = 100; + } + } + } + } + + if(Priority == 0) + { + Priority = 200; + } + } + + public IconResult(Uri uri, byte[] bytes, string format) + { + Path = uri.ToString(); + Icon = new Icon + { + Image = bytes, + Format = format + }; + Priority = 10; + } + + public string Path { get; set; } + public int? DefinedWidth { get; set; } + public int? DefinedHeight { get; set; } + public Icon Icon { get; set; } + public int Priority { get; set; } + } +} diff --git a/src/Icons/Services/IIconFetchingService.cs b/src/Icons/Services/IIconFetchingService.cs new file mode 100644 index 0000000000..433917e5dd --- /dev/null +++ b/src/Icons/Services/IIconFetchingService.cs @@ -0,0 +1,10 @@ +using System.Threading.Tasks; +using Bit.Icons.Models; + +namespace Bit.Icons.Services +{ + public interface IIconFetchingService + { + Task GetIconAsync(string domain); + } +} diff --git a/src/Icons/Services/IconFetchingService.cs b/src/Icons/Services/IconFetchingService.cs new file mode 100644 index 0000000000..bdbae3c7e7 --- /dev/null +++ b/src/Icons/Services/IconFetchingService.cs @@ -0,0 +1,267 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Bit.Icons.Models; +using HtmlAgilityPack; + +namespace Bit.Icons.Services +{ + public class IconFetchingService : IIconFetchingService + { + private static HashSet _iconRels = new HashSet { "icon", "apple-touch-icon", "shortcut icon" }; + private static HashSet _iconExtensions = new HashSet { ".ico", ".png", ".jpg", ".jpeg" }; + private static readonly HttpClient _httpClient = new HttpClient(new HttpClientHandler + { + AllowAutoRedirect = false, + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }); + private static string _pngMediaType = "image/png"; + private static byte[] _pngHeader = new byte[] { 137, 80, 78, 71 }; + private static string _icoMediaType = "image/x-icon"; + private static string _icoAltMediaType = "image/vnd.microsoft.icon"; + private static byte[] _icoHeader = new byte[] { 00, 00, 01, 00 }; + private static string _jpegMediaType = "image/jpeg"; + private static byte[] _jpegHeader = new byte[] { 255, 216, 255 }; + private static string _octetMediaType = "application/octet-stream"; + private static readonly HashSet _allowedMediaTypes = new HashSet{ + _pngMediaType, + _icoMediaType, + _icoAltMediaType, + _jpegMediaType, + _octetMediaType + }; + + public async Task GetIconAsync(string domain) + { + var uri = new Uri($"http://{domain}"); + var response = await GetAndFollowAsync(uri, 2); + if(response == null || !response.IsSuccessStatusCode) + { + uri = new Uri($"https://{domain}"); + response = await GetAndFollowAsync(uri, 2); + } + + if(response?.Content == null || !response.IsSuccessStatusCode) + { + return null; + } + + if(response.Content.Headers?.ContentType?.MediaType != "text/html") + { + return null; + } + + uri = response.RequestMessage.RequestUri; + var html = await response.Content.ReadAsStringAsync(); + var doc = new HtmlDocument(); + doc.LoadHtml(html); + if(doc.DocumentNode == null) + { + return null; + } + + var icons = new List(); + var links = doc.DocumentNode.SelectNodes(@"//link[@href]"); + if(links != null) + { + foreach(var link in links) + { + var hrefAttr = link.Attributes["href"]; + if(string.IsNullOrWhiteSpace(hrefAttr?.Value)) + { + continue; + } + + var relAttr = link.Attributes["rel"]; + if(relAttr != null && _iconRels.Contains(relAttr.Value)) + { + icons.Add(new IconResult(hrefAttr.Value, link)); + } + else + { + var extension = Path.GetExtension(hrefAttr.Value); + if(_iconExtensions.Contains(extension)) + { + icons.Add(new IconResult(hrefAttr.Value, link)); + } + } + } + } + + var iconResultTasks = new List(); + foreach(var icon in icons) + { + Uri iconUri = null; + if(Uri.TryCreate(icon.Path, UriKind.Relative, out Uri relUri)) + { + iconUri = new Uri($"{uri.Scheme}://{uri.Host}/{relUri.OriginalString}"); + } + else if(Uri.TryCreate(icon.Path, UriKind.Absolute, out Uri absUri)) + { + iconUri = absUri; + } + + if(iconUri != null) + { + var task = GetIconAsync(iconUri).ContinueWith(async (r) => + { + var result = await r; + if(result != null) + { + icon.Path = iconUri.ToString(); + icon.Icon = result.Icon; + } + }); + iconResultTasks.Add(task); + } + } + + await Task.WhenAll(iconResultTasks); + if(!icons.Any(i => i.Icon != null)) + { + var faviconUri = new Uri($"{uri.Scheme}://{uri.Host}/favicon.ico"); + var result = await GetIconAsync(faviconUri); + if(result != null) + { + icons.Add(result); + } + else + { + return null; + } + } + + return icons.Where(i => i.Icon != null).OrderBy(i => i.Priority).First(); + } + + private async Task GetIconAsync(Uri uri) + { + var response = await GetAndFollowAsync(uri, 2); + if(response?.Content?.Headers == null || !response.IsSuccessStatusCode) + { + return null; + } + + var format = response.Content.Headers?.ContentType?.MediaType; + if(format == null || !_allowedMediaTypes.Contains(format)) + { + return null; + } + + var bytes = await response.Content.ReadAsByteArrayAsync(); + if(format == _octetMediaType) + { + if(HeaderMatch(bytes, _icoHeader)) + { + format = _icoMediaType; + } + else if(HeaderMatch(bytes, _pngHeader)) + { + format = _pngMediaType; + } + else if(HeaderMatch(bytes, _jpegHeader)) + { + format = _jpegMediaType; + } + else + { + return null; + } + } + + return new IconResult(uri, bytes, format); + } + + private async Task GetAndFollowAsync(Uri uri, int maxRedirectCount) + { + var response = await GetAsync(uri); + if(response == null) + { + return null; + } + return await FollowRedirectsAsync(response, maxRedirectCount); + } + + private async Task GetAsync(Uri uri) + { + var message = new HttpRequestMessage + { + RequestUri = uri, + Method = HttpMethod.Get + }; + + // Let's add some headers to look like we're coming from a web browser request. Some websites + // will block our request without these. + message.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); + message.Headers.Add("Accept-Language", "en-US,en;q=0.8"); + message.Headers.Add("Cache-Control", "no-cache"); + message.Headers.Add("Pragma", "no-cache"); + message.Headers.Add("Accept", "image/webp,image/apng,image/*,*/*;q=0.8"); + + try + { + return await _httpClient.SendAsync(message); + } + catch + { + return null; + } + } + + private async Task FollowRedirectsAsync(HttpResponseMessage response, + int maxFollowCount, int followCount = 0) + { + if(response.IsSuccessStatusCode || followCount > maxFollowCount) + { + return response; + } + + if(!(response.StatusCode == HttpStatusCode.Redirect || + response.StatusCode == HttpStatusCode.MovedPermanently || + response.StatusCode == HttpStatusCode.RedirectKeepVerb) || + !response.Headers.Contains("Location")) + { + return null; + } + + var locationHeader = response.Headers.GetValues("Location").FirstOrDefault(); + if(!string.IsNullOrWhiteSpace(locationHeader)) + { + if(!Uri.TryCreate(locationHeader, UriKind.Absolute, out Uri location)) + { + if(Uri.TryCreate(locationHeader, UriKind.Relative, out Uri relLocation)) + { + var requestUri = response.RequestMessage.RequestUri; + location = new Uri($"{requestUri.Scheme}://{requestUri.Host}/{relLocation.OriginalString}"); + } + else + { + return null; + } + } + + var newResponse = await GetAsync(location); + if(newResponse != null) + { + var redirectedResponse = await FollowRedirectsAsync(newResponse, maxFollowCount, followCount++); + if(redirectedResponse != null) + { + return redirectedResponse; + } + } + } + + return null; + } + + private bool HeaderMatch(byte[] imageBytes, byte[] header) + { + return imageBytes.Length >= header.Length && header.SequenceEqual(imageBytes.Take(header.Length)); + } + } +} diff --git a/src/Icons/Startup.cs b/src/Icons/Startup.cs index 2ae874036b..0573a44f23 100644 --- a/src/Icons/Startup.cs +++ b/src/Icons/Startup.cs @@ -35,6 +35,7 @@ namespace Bit.Icons // Services services.AddSingleton(); + services.AddSingleton(); // Mvc services.AddMvc(); diff --git a/src/Icons/appsettings.json b/src/Icons/appsettings.json index 7c2dc44628..63e7df9f9f 100644 --- a/src/Icons/appsettings.json +++ b/src/Icons/appsettings.json @@ -13,7 +13,6 @@ } }, "iconsSettings": { - "bestIconBaseUrl": "https://besticon-demo.herokuapp.com", "cacheHours": 24, "cacheSizeLimit": null } diff --git a/src/Icons/entrypoint.sh b/src/Icons/entrypoint.sh index 4ad8541614..f1a4fa0ba0 100644 --- a/src/Icons/entrypoint.sh +++ b/src/Icons/entrypoint.sh @@ -55,7 +55,4 @@ fi # The rest... chown -R $USERNAME:$GROUPNAME /app -chown -R $USERNAME:$GROUPNAME /etc/iconserver - -gosu $USERNAME:$GROUPNAME /etc/iconserver/iconserver & -gosu $USERNAME:$GROUPNAME dotnet /app/Icons.dll iconsSettings:bestIconBaseUrl=http://localhost:8080 +gosu $USERNAME:$GROUPNAME dotnet /app/Icons.dll diff --git a/src/Icons/iconserver.sha256 b/src/Icons/iconserver.sha256 deleted file mode 100644 index ddafb71e9d..0000000000 --- a/src/Icons/iconserver.sha256 +++ /dev/null @@ -1 +0,0 @@ -2fc9b9b34d4c4cba0cd90fe4d7ecf93e493c17ef240c1f4b757c9a293461f877 *iconserver.zip