new icon fetching service. remove besticon dep.

src/Icons/Controllers/IconsController.cs

@@ -1,8 +1,4 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net;
-using System.Net.Http;
 using System.Threading.Tasks;
 using Bit.Icons.Models;
 using Bit.Icons.Services;
@@ -14,37 +10,20 @@ namespace Bit.Icons.Controllers
     [Route("")]
     public class IconsController : Controller
     {
-        private static readonly HttpClient _httpClient = new HttpClient(new HttpClientHandler
-        {
-            AllowAutoRedirect = false,
-            AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
-        });
-        private static string _pngMediaType = "image/png";
-        private static byte[] _pngHeader = new byte[] { 137, 80, 78, 71 };
-        private static string _icoMediaType = "image/x-icon";
-        private static string _icoAltMediaType = "image/vnd.microsoft.icon";
-        private static byte[] _icoHeader = new byte[] { 00, 00, 01, 00 };
-        private static string _jpegMediaType = "image/jpeg";
-        private static byte[] _jpegHeader = new byte[] { 255, 216, 255 };
-        private static string _octetMediaType = "application/octet-stream";
-        private static readonly HashSet<string> _allowedMediaTypes = new HashSet<string>{
-            _pngMediaType,
-            _icoMediaType,
-            _icoAltMediaType,
-            _jpegMediaType,
-            _octetMediaType
-        };
         private readonly IMemoryCache _memoryCache;
         private readonly IDomainMappingService _domainMappingService;
+        private readonly IIconFetchingService _iconFetchingService;
         private readonly IconsSettings _iconsSettings;
 
         public IconsController(
             IMemoryCache memoryCache,
             IDomainMappingService domainMappingService,
+            IIconFetchingService iconFetchingService,
             IconsSettings iconsSettings)
         {
             _memoryCache = memoryCache;
             _domainMappingService = domainMappingService;
+            _iconFetchingService = iconFetchingService;
             _iconsSettings = iconsSettings;
         }
 
@@ -66,46 +45,16 @@ namespace Bit.Icons.Controllers
             var mappedDomain = _domainMappingService.MapDomain(uri.Host);
             if(!_memoryCache.TryGetValue(mappedDomain, out Icon icon))
             {
-                var iconUrl = new Uri($"{_iconsSettings.BestIconBaseUrl}/icon" +
+                var result = await _iconFetchingService.GetIconAsync(mappedDomain);
-                    $"?url={mappedDomain}&size=16..32..256&fallback_icon_url=" +
+                if(result == null)
-                    $"https://raw.githubusercontent.com/bitwarden/web/master/src/images/fa-globe.png");
-                var response = await _httpClient.GetAsync(iconUrl);
-                response = await FollowRedirectsAsync(response, 1);
-                if(!response.IsSuccessStatusCode ||
-                    !_allowedMediaTypes.Contains(response.Content.Headers.ContentType.MediaType))
                 {
                     return new NotFoundResult();
                 }
 
-                var image = await response.Content.ReadAsByteArrayAsync();
+                icon = result.Icon;
-                icon = new Icon
-                {
-                    Image = image,
-                    Format = response.Content.Headers.ContentType.MediaType
-                };
-
-                if(icon.Format == _octetMediaType)
-                {
-                    if(HeaderMatch(icon, _icoHeader))
-                    {
-                        icon.Format = _icoMediaType;
-                    }
-                    else if(HeaderMatch(icon, _pngHeader))
-                    {
-                        icon.Format = _pngMediaType;
-                    }
-                    else if(HeaderMatch(icon, _jpegHeader))
-                    {
-                        icon.Format = _jpegMediaType;
-                    }
-                    else
-                    {
-                        return new NotFoundResult();
-                    }
-                }
 
                 // Only cache smaller images (<= 50kb)
-                if(image.Length <= 50012)
+                if(icon.Image.Length <= 50012)
                 {
                     _memoryCache.Set(mappedDomain, icon, new MemoryCacheEntryOptions
                     {
@@ -116,47 +65,5 @@ namespace Bit.Icons.Controllers
 
             return new FileContentResult(icon.Image, icon.Format);
         }
-
-        private async Task<HttpResponseMessage> FollowRedirectsAsync(HttpResponseMessage response, int followCount)
-        {
-            if(response.IsSuccessStatusCode || followCount > 2)
-            {
-                return response;
-            }
-
-            if((response.StatusCode == HttpStatusCode.Redirect || response.StatusCode == HttpStatusCode.MovedPermanently) &&
-                response.Headers.Contains("Location"))
-            {
-                var locationHeader = response.Headers.GetValues("Location").FirstOrDefault();
-                if(!string.IsNullOrWhiteSpace(locationHeader) &&
-                    Uri.TryCreate(locationHeader, UriKind.Absolute, out Uri location))
-                {
-                    var message = new HttpRequestMessage
-                    {
-                        RequestUri = location,
-                        Method = HttpMethod.Get
-                    };
-
-                    // Let's add some headers to look like we're coming from a web browser request. Some websites
-                    // will block our request without these.
-                    message.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
-                        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36");
-                    message.Headers.Add("Accept-Language", "en-US,en;q=0.8");
-                    message.Headers.Add("Cache-Control", "no-cache");
-                    message.Headers.Add("Pragma", "no-cache");
-                    message.Headers.Add("Accept", "image/webp,image/apng,image/*,*/*;q=0.8");
-
-                    response = await _httpClient.SendAsync(message);
-                    response = await FollowRedirectsAsync(response, followCount++);
-                }
-            }
-
-            return response;
-        }
-
-        private bool HeaderMatch(Icon icon, byte[] header)
-        {
-            return icon.Image.Length >= header.Length && header.SequenceEqual(icon.Image.Take(header.Length));
-        }
     }
 }

src/Icons/Dockerfile

@@ -2,17 +2,9 @@ FROM microsoft/aspnetcore:2.0.6
 
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
-        unzip \
         gosu \
     && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /tmp
-COPY iconserver.sha256 .
-RUN curl -L -o iconserver.zip https://github.com/mat/besticon/releases/download/v3.6.0/iconserver_linux_amd64.zip \
-    && sha256sum -c iconserver.sha256 \
-    && unzip iconserver.zip -d /etc/iconserver \
-    && rm iconserver.*
-
 ENV ASPNETCORE_URLS http://+:5000
 WORKDIR /app
 EXPOSE 5000

src/Icons/Icons.csproj

@@ -9,6 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="HtmlAgilityPack" Version="1.8.2" />
     <PackageReference Include="Microsoft.AspNetCore.All" Version="2.0.6" />
   </ItemGroup>
 

src/Icons/IconsSettings.cs

@@ -2,7 +2,6 @@
 {
     public class IconsSettings
     {
-        public virtual string BestIconBaseUrl { get; set; }
         public virtual int CacheHours { get; set; }
         public virtual long? CacheSizeLimit { get; set; }
     }

src/Icons/Models/IconResult.cs

@@ -0,0 +1,70 @@
+using System;
+using HtmlAgilityPack;
+
+namespace Bit.Icons.Models
+{
+    public class IconResult
+    {
+        public IconResult(string href, HtmlNode node)
+        {
+            Path = href;
+            var sizesAttr = node.Attributes["sizes"];
+            if(!string.IsNullOrWhiteSpace(sizesAttr?.Value))
+            {
+                var sizeParts = sizesAttr.Value.Split('x');
+                if(sizeParts.Length == 2 && int.TryParse(sizeParts[0].Trim(), out var width) &&
+                    int.TryParse(sizeParts[1].Trim(), out var height))
+                {
+                    DefinedWidth = width;
+                    DefinedHeight = height;
+
+                    if(width == height)
+                    {
+                        if(width == 32)
+                        {
+                            Priority = 1;
+                        }
+                        else if(width == 64)
+                        {
+                            Priority = 2;
+                        }
+                        else if(width >= 24 && width <= 128)
+                        {
+                            Priority = 3;
+                        }
+                        else if(width == 16)
+                        {
+                            Priority = 4;
+                        }
+                        else
+                        {
+                            Priority = 100;
+                        }
+                    }
+                }
+            }
+
+            if(Priority == 0)
+            {
+                Priority = 200;
+            }
+        }
+
+        public IconResult(Uri uri, byte[] bytes, string format)
+        {
+            Path = uri.ToString();
+            Icon = new Icon
+            {
+                Image = bytes,
+                Format = format
+            };
+            Priority = 10;
+        }
+
+        public string Path { get; set; }
+        public int? DefinedWidth { get; set; }
+        public int? DefinedHeight { get; set; }
+        public Icon Icon { get; set; }
+        public int Priority { get; set; }
+    }
+}

src/Icons/Services/IIconFetchingService.cs

@@ -0,0 +1,10 @@
+using System.Threading.Tasks;
+using Bit.Icons.Models;
+
+namespace Bit.Icons.Services
+{
+    public interface IIconFetchingService
+    {
+        Task<IconResult> GetIconAsync(string domain);
+    }
+}

src/Icons/Services/IconFetchingService.cs

@@ -0,0 +1,267 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Bit.Icons.Models;
+using HtmlAgilityPack;
+
+namespace Bit.Icons.Services
+{
+    public class IconFetchingService : IIconFetchingService
+    {
+        private static HashSet<string> _iconRels = new HashSet<string> { "icon", "apple-touch-icon", "shortcut icon" };
+        private static HashSet<string> _iconExtensions = new HashSet<string> { ".ico", ".png", ".jpg", ".jpeg" };
+        private static readonly HttpClient _httpClient = new HttpClient(new HttpClientHandler
+        {
+            AllowAutoRedirect = false,
+            AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
+        });
+        private static string _pngMediaType = "image/png";
+        private static byte[] _pngHeader = new byte[] { 137, 80, 78, 71 };
+        private static string _icoMediaType = "image/x-icon";
+        private static string _icoAltMediaType = "image/vnd.microsoft.icon";
+        private static byte[] _icoHeader = new byte[] { 00, 00, 01, 00 };
+        private static string _jpegMediaType = "image/jpeg";
+        private static byte[] _jpegHeader = new byte[] { 255, 216, 255 };
+        private static string _octetMediaType = "application/octet-stream";
+        private static readonly HashSet<string> _allowedMediaTypes = new HashSet<string>{
+            _pngMediaType,
+            _icoMediaType,
+            _icoAltMediaType,
+            _jpegMediaType,
+            _octetMediaType
+        };
+
+        public async Task<IconResult> GetIconAsync(string domain)
+        {
+            var uri = new Uri($"http://{domain}");
+            var response = await GetAndFollowAsync(uri, 2);
+            if(response == null || !response.IsSuccessStatusCode)
+            {
+                uri = new Uri($"https://{domain}");
+                response = await GetAndFollowAsync(uri, 2);
+            }
+
+            if(response?.Content == null || !response.IsSuccessStatusCode)
+            {
+                return null;
+            }
+
+            if(response.Content.Headers?.ContentType?.MediaType != "text/html")
+            {
+                return null;
+            }
+
+            uri = response.RequestMessage.RequestUri;
+            var html = await response.Content.ReadAsStringAsync();
+            var doc = new HtmlDocument();
+            doc.LoadHtml(html);
+            if(doc.DocumentNode == null)
+            {
+                return null;
+            }
+
+            var icons = new List<IconResult>();
+            var links = doc.DocumentNode.SelectNodes(@"//link[@href]");
+            if(links != null)
+            {
+                foreach(var link in links)
+                {
+                    var hrefAttr = link.Attributes["href"];
+                    if(string.IsNullOrWhiteSpace(hrefAttr?.Value))
+                    {
+                        continue;
+                    }
+
+                    var relAttr = link.Attributes["rel"];
+                    if(relAttr != null && _iconRels.Contains(relAttr.Value))
+                    {
+                        icons.Add(new IconResult(hrefAttr.Value, link));
+                    }
+                    else
+                    {
+                        var extension = Path.GetExtension(hrefAttr.Value);
+                        if(_iconExtensions.Contains(extension))
+                        {
+                            icons.Add(new IconResult(hrefAttr.Value, link));
+                        }
+                    }
+                }
+            }
+
+            var iconResultTasks = new List<Task>();
+            foreach(var icon in icons)
+            {
+                Uri iconUri = null;
+                if(Uri.TryCreate(icon.Path, UriKind.Relative, out Uri relUri))
+                {
+                    iconUri = new Uri($"{uri.Scheme}://{uri.Host}/{relUri.OriginalString}");
+                }
+                else if(Uri.TryCreate(icon.Path, UriKind.Absolute, out Uri absUri))
+                {
+                    iconUri = absUri;
+                }
+
+                if(iconUri != null)
+                {
+                    var task = GetIconAsync(iconUri).ContinueWith(async (r) =>
+                    {
+                        var result = await r;
+                        if(result != null)
+                        {
+                            icon.Path = iconUri.ToString();
+                            icon.Icon = result.Icon;
+                        }
+                    });
+                    iconResultTasks.Add(task);
+                }
+            }
+
+            await Task.WhenAll(iconResultTasks);
+            if(!icons.Any(i => i.Icon != null))
+            {
+                var faviconUri = new Uri($"{uri.Scheme}://{uri.Host}/favicon.ico");
+                var result = await GetIconAsync(faviconUri);
+                if(result != null)
+                {
+                    icons.Add(result);
+                }
+                else
+                {
+                    return null;
+                }
+            }
+
+            return icons.Where(i => i.Icon != null).OrderBy(i => i.Priority).First();
+        }
+
+        private async Task<IconResult> GetIconAsync(Uri uri)
+        {
+            var response = await GetAndFollowAsync(uri, 2);
+            if(response?.Content?.Headers == null || !response.IsSuccessStatusCode)
+            {
+                return null;
+            }
+
+            var format = response.Content.Headers?.ContentType?.MediaType;
+            if(format == null || !_allowedMediaTypes.Contains(format))
+            {
+                return null;
+            }
+
+            var bytes = await response.Content.ReadAsByteArrayAsync();
+            if(format == _octetMediaType)
+            {
+                if(HeaderMatch(bytes, _icoHeader))
+                {
+                    format = _icoMediaType;
+                }
+                else if(HeaderMatch(bytes, _pngHeader))
+                {
+                    format = _pngMediaType;
+                }
+                else if(HeaderMatch(bytes, _jpegHeader))
+                {
+                    format = _jpegMediaType;
+                }
+                else
+                {
+                    return null;
+                }
+            }
+
+            return new IconResult(uri, bytes, format);
+        }
+
+        private async Task<HttpResponseMessage> GetAndFollowAsync(Uri uri, int maxRedirectCount)
+        {
+            var response = await GetAsync(uri);
+            if(response == null)
+            {
+                return null;
+            }
+            return await FollowRedirectsAsync(response, maxRedirectCount);
+        }
+
+        private async Task<HttpResponseMessage> GetAsync(Uri uri)
+        {
+            var message = new HttpRequestMessage
+            {
+                RequestUri = uri,
+                Method = HttpMethod.Get
+            };
+
+            // Let's add some headers to look like we're coming from a web browser request. Some websites
+            // will block our request without these.
+            message.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36");
+            message.Headers.Add("Accept-Language", "en-US,en;q=0.8");
+            message.Headers.Add("Cache-Control", "no-cache");
+            message.Headers.Add("Pragma", "no-cache");
+            message.Headers.Add("Accept", "image/webp,image/apng,image/*,*/*;q=0.8");
+
+            try
+            {
+                return await _httpClient.SendAsync(message);
+            }
+            catch
+            {
+                return null;
+            }
+        }
+
+        private async Task<HttpResponseMessage> FollowRedirectsAsync(HttpResponseMessage response,
+            int maxFollowCount, int followCount = 0)
+        {
+            if(response.IsSuccessStatusCode || followCount > maxFollowCount)
+            {
+                return response;
+            }
+
+            if(!(response.StatusCode == HttpStatusCode.Redirect ||
+                response.StatusCode == HttpStatusCode.MovedPermanently ||
+                response.StatusCode == HttpStatusCode.RedirectKeepVerb) ||
+                !response.Headers.Contains("Location"))
+            {
+                return null;
+            }
+
+            var locationHeader = response.Headers.GetValues("Location").FirstOrDefault();
+            if(!string.IsNullOrWhiteSpace(locationHeader))
+            {
+                if(!Uri.TryCreate(locationHeader, UriKind.Absolute, out Uri location))
+                {
+                    if(Uri.TryCreate(locationHeader, UriKind.Relative, out Uri relLocation))
+                    {
+                        var requestUri = response.RequestMessage.RequestUri;
+                        location = new Uri($"{requestUri.Scheme}://{requestUri.Host}/{relLocation.OriginalString}");
+                    }
+                    else
+                    {
+                        return null;
+                    }
+                }
+
+                var newResponse = await GetAsync(location);
+                if(newResponse != null)
+                {
+                    var redirectedResponse = await FollowRedirectsAsync(newResponse, maxFollowCount, followCount++);
+                    if(redirectedResponse != null)
+                    {
+                        return redirectedResponse;
+                    }
+                }
+            }
+
+            return null;
+        }
+
+        private bool HeaderMatch(byte[] imageBytes, byte[] header)
+        {
+            return imageBytes.Length >= header.Length && header.SequenceEqual(imageBytes.Take(header.Length));
+        }
+    }
+}

src/Icons/Startup.cs

@@ -35,6 +35,7 @@ namespace Bit.Icons
 
             // Services
             services.AddSingleton<IDomainMappingService, DomainMappingService>();
+            services.AddSingleton<IIconFetchingService, IconFetchingService>();
 
             // Mvc
             services.AddMvc();

src/Icons/appsettings.json

@@ -13,7 +13,6 @@
     }
   },
   "iconsSettings": {
-    "bestIconBaseUrl": "https://besticon-demo.herokuapp.com",
     "cacheHours": 24,
     "cacheSizeLimit": null
   }

src/Icons/entrypoint.sh

@@ -55,7 +55,4 @@ fi
 # The rest...
 
 chown -R $USERNAME:$GROUPNAME /app
-chown -R $USERNAME:$GROUPNAME /etc/iconserver
+gosu $USERNAME:$GROUPNAME dotnet /app/Icons.dll
-
-gosu $USERNAME:$GROUPNAME /etc/iconserver/iconserver &
-gosu $USERNAME:$GROUPNAME dotnet /app/Icons.dll iconsSettings:bestIconBaseUrl=http://localhost:8080

src/Icons/iconserver.sha256

@@ -1 +0,0 @@
-2fc9b9b34d4c4cba0cd90fe4d7ecf93e493c17ef240c1f4b757c9a293461f877 *iconserver.zip