mirror of
https://github.com/bitwarden/server.git
synced 2025-06-30 15:42:48 -05:00
Rewrite Icon fetching (#3023)
* Rewrite Icon fetching * Move validation to IconUri, Uri, or UriBuilder * `dotnet format` 🤖 * PR suggestions * Add not null compiler hint * Add twitter to test case * Move Uri manipulation to UriService * Implement MockedHttpClient Presents better, fluent handling of message matching and response building. * Add redirect handling tests * Add testing to models * More aggressively dispose content in icon link * Format 🤖 * Update icon lockfile * Convert to cloned stream for HttpResponseBuilder Content was being disposed when HttResponseMessage was being disposed. This avoids losing our reference to our content and allows multiple usages of the same `MockedHttpMessageResponse` * Move services to extension Extension is shared by testing and allows access to services from our service tests * Remove unused `using` * Prefer awaiting asyncs for better exception handling * `dotnet format` 🤖 * Await async * Update tests to use test TLD and ip ranges * Remove unused interfaces * Make assignments static when possible * Prefer invariant comparer to downcasing * Prefer injecting interface services to implementations * Prefer comparer set in HashSet initialization * Allow SVG icons * Filter out icons with unknown formats * Seek to beginning of MemoryStream after writing it * More appropriate to not return icon if it's invalid * Add svg icon test
This commit is contained in:
Matt Gibson
@ -20,6 +20,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\Icons\Icons.csproj" />
|
||||
<ProjectReference Include="..\Common\Common.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
38
test/Icons.Test/Models/IconHttpRequestTests.cs
Normal file
38
test/Icons.Test/Models/IconHttpRequestTests.cs
Normal file
@ -0,0 +1,38 @@
|
||||
using System.Net;
|
||||
using Bit.Icons.Models;
|
||||
using Bit.Icons.Services;
|
||||
using Bit.Test.Common.MockedHttpClient;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Net.Http.Headers;
|
||||
using NSubstitute;
|
||||
using Xunit;
|
||||
|
||||
namespace Bit.Icons.Test.Models;
|
||||
|
||||
public class IconHttpRequestTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task FetchAsync_FollowsTwoRedirectsAsync()
|
||||
{
|
||||
var handler = new MockedHttpMessageHandler();
|
||||
|
||||
var request = handler
|
||||
.Fallback
|
||||
.WithStatusCode(HttpStatusCode.Redirect)
|
||||
.WithContent("text/html", "<html><head><title>Redirect 2</title></head><body><a href=\"https://icon.test\">Redirect 3</a></body></html>")
|
||||
.WithHeader(HeaderNames.Location, "https://icon.test");
|
||||
|
||||
var clientFactory = Substitute.For<IHttpClientFactory>();
|
||||
clientFactory.CreateClient("Icons").Returns(handler.ToHttpClient());
|
||||
|
||||
var uriService = Substitute.For<IUriService>();
|
||||
uriService.TryGetUri(Arg.Any<Uri>(), out Arg.Any<IconUri>()).Returns(x =>
|
||||
{
|
||||
x[1] = new IconUri(new Uri("https://icon.test"), IPAddress.Parse("192.0.2.1"));
|
||||
return true;
|
||||
});
|
||||
var result = await IconHttpRequest.FetchAsync(new Uri("https://icon.test"), NullLogger<IIconFetchingService>.Instance, clientFactory, uriService);
|
||||
|
||||
Assert.Equal(3, request.NumberOfResponses); // Initial + 2 redirects
|
||||
}
|
||||
}
|
||||
101
test/Icons.Test/Models/IconHttpResponseTests.cs
Normal file
101
test/Icons.Test/Models/IconHttpResponseTests.cs
Normal file
@ -0,0 +1,101 @@
|
||||
using System.Net;
|
||||
using AngleSharp.Html.Parser;
|
||||
using Bit.Icons.Models;
|
||||
using Bit.Icons.Services;
|
||||
using Bit.Test.Common.Helpers;
|
||||
using Bit.Test.Common.MockedHttpClient;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using Xunit;
|
||||
|
||||
namespace Bit.Icons.Test.Models;
|
||||
|
||||
public class IconHttpResponseTests
|
||||
{
|
||||
private readonly IUriService _mockedUriService;
|
||||
private static readonly IHtmlParser _parser = new HtmlParser();
|
||||
|
||||
public IconHttpResponseTests()
|
||||
{
|
||||
_mockedUriService = Substitute.For<IUriService>();
|
||||
_mockedUriService.TryGetUri(Arg.Any<Uri>(), out Arg.Any<IconUri>()).Returns(x =>
|
||||
{
|
||||
x[1] = new IconUri(new Uri("https://icon.test"), IPAddress.Parse("192.0.2.1"));
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetrieveIconsAsync_Processes200LinksAsync()
|
||||
{
|
||||
var htmlBuilder = new HtmlBuilder();
|
||||
var headBuilder = new HtmlBuilder("head");
|
||||
for (var i = 0; i < 200; i++)
|
||||
{
|
||||
headBuilder.Append(UnusableLinkNode());
|
||||
}
|
||||
headBuilder.Append(UsableLinkNode());
|
||||
htmlBuilder.Append(headBuilder);
|
||||
var response = GetHttpResponseMessage(htmlBuilder.ToString());
|
||||
var sut = CurriedIconHttpResponse()(response);
|
||||
|
||||
var result = await sut.RetrieveIconsAsync(new Uri("https://icon.test"), "icon.test", _parser);
|
||||
|
||||
Assert.Empty(result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetrieveIconsAsync_Processes10IconsAsync()
|
||||
{
|
||||
var htmlBuilder = new HtmlBuilder();
|
||||
var headBuilder = new HtmlBuilder("head");
|
||||
for (var i = 0; i < 11; i++)
|
||||
{
|
||||
headBuilder.Append(UsableLinkNode());
|
||||
}
|
||||
htmlBuilder.Append(headBuilder);
|
||||
var response = GetHttpResponseMessage(htmlBuilder.ToString());
|
||||
var sut = CurriedIconHttpResponse()(response);
|
||||
|
||||
var result = await sut.RetrieveIconsAsync(new Uri("https://icon.test"), "icon.test", _parser);
|
||||
|
||||
Assert.Equal(10, result.Count());
|
||||
}
|
||||
|
||||
private static string UsableLinkNode()
|
||||
{
|
||||
return "<link rel=\"icon\" href=\"https://icon.test/favicon.ico\" />";
|
||||
}
|
||||
|
||||
private static string UnusableLinkNode()
|
||||
{
|
||||
// Empty href links are not usable
|
||||
return "<link rel=\"icon\" href=\"\" />";
|
||||
}
|
||||
|
||||
private static HttpResponseMessage GetHttpResponseMessage(string content)
|
||||
{
|
||||
return new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
RequestMessage = new HttpRequestMessage(HttpMethod.Get, "https://icon.test"),
|
||||
Content = new StringContent(content)
|
||||
};
|
||||
}
|
||||
|
||||
private Func<HttpResponseMessage, IconHttpResponse> CurriedIconHttpResponse()
|
||||
{
|
||||
return (HttpResponseMessage response) => new IconHttpResponse(response, NullLogger<IIconFetchingService>.Instance, UsableIconHttpClientFactory(), _mockedUriService);
|
||||
}
|
||||
|
||||
private static IHttpClientFactory UsableIconHttpClientFactory()
|
||||
{
|
||||
var substitute = Substitute.For<IHttpClientFactory>();
|
||||
var handler = new MockedHttpMessageHandler();
|
||||
handler.Fallback
|
||||
.WithStatusCode(HttpStatusCode.OK)
|
||||
.WithContent("image/png", new byte[] { 137, 80, 78, 71 });
|
||||
|
||||
substitute.CreateClient("Icons").Returns(handler.ToHttpClient());
|
||||
return substitute;
|
||||
}
|
||||
}
|
||||
85
test/Icons.Test/Models/IconLinkTests.cs
Normal file
85
test/Icons.Test/Models/IconLinkTests.cs
Normal file
@ -0,0 +1,85 @@
|
||||
using System.Net;
|
||||
using AngleSharp.Dom;
|
||||
using Bit.Icons.Models;
|
||||
using Bit.Icons.Services;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using NSubstitute;
|
||||
using Xunit;
|
||||
|
||||
namespace Bit.Icons.Test.Models;
|
||||
|
||||
public class IconLinkTests
|
||||
{
|
||||
private readonly IElement _element;
|
||||
private readonly Uri _uri = new("https://icon.test");
|
||||
private readonly ILogger<IIconFetchingService> _logger = Substitute.For<ILogger<IIconFetchingService>>();
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly IUriService _uriService;
|
||||
private readonly string _baseUrlPath = "/";
|
||||
|
||||
public IconLinkTests()
|
||||
{
|
||||
_element = Substitute.For<IElement>();
|
||||
_httpClientFactory = Substitute.For<IHttpClientFactory>();
|
||||
_uriService = Substitute.For<IUriService>();
|
||||
_uriService.TryGetUri(Arg.Any<Uri>(), out Arg.Any<IconUri>()).Returns(x =>
|
||||
{
|
||||
x[1] = new IconUri(new Uri("https://icon.test"), IPAddress.Parse("192.0.2.1"));
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WithNoHref_IsNotUsable()
|
||||
{
|
||||
_element.GetAttribute("href").Returns(string.Empty);
|
||||
|
||||
var result = new IconLink(_element, _uri, _baseUrlPath).IsUsable();
|
||||
|
||||
Assert.False(result);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null, false)]
|
||||
[InlineData("", false)]
|
||||
[InlineData(" ", false)]
|
||||
[InlineData("unusable", false)]
|
||||
[InlineData("ico", true)]
|
||||
public void WithNoRel_IsUsable(string extension, bool expectedResult)
|
||||
{
|
||||
SetAttributeValue("href", $"/favicon.{extension}");
|
||||
|
||||
var result = new IconLink(_element, _uri, _baseUrlPath).IsUsable();
|
||||
|
||||
Assert.Equal(expectedResult, result);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("icon", true)]
|
||||
[InlineData("stylesheet", false)]
|
||||
public void WithRel_IsUsable(string rel, bool expectedResult)
|
||||
{
|
||||
SetAttributeValue("href", "/favicon.ico");
|
||||
SetAttributeValue("rel", rel);
|
||||
|
||||
var result = new IconLink(_element, _uri, _baseUrlPath).IsUsable();
|
||||
|
||||
Assert.Equal(expectedResult, result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FetchAsync_Unvalidated_ReturnsNull()
|
||||
{
|
||||
var result = new IconLink(_element, _uri, _baseUrlPath).FetchAsync(_logger, _httpClientFactory, _uriService);
|
||||
|
||||
Assert.Null(result.Result);
|
||||
}
|
||||
|
||||
private void SetAttributeValue(string attribute, string value)
|
||||
{
|
||||
var attr = Substitute.For<IAttr>();
|
||||
attr.Value.Returns(value);
|
||||
|
||||
_element.Attributes[attribute].Returns(attr);
|
||||
}
|
||||
}
|
||||
22
test/Icons.Test/Models/IconUriTests.cs
Normal file
22
test/Icons.Test/Models/IconUriTests.cs
Normal file
@ -0,0 +1,22 @@
|
||||
using System.Net;
|
||||
using Bit.Icons.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace Bit.Icons.Test.Models;
|
||||
|
||||
public class IconUriTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("https://icon.test", "1.1.1.1", true)]
|
||||
[InlineData("https://icon.test:4443", "1.1.1.1", false)] // Non standard port
|
||||
[InlineData("http://test", "1.1.1.1", false)] // top level domain
|
||||
[InlineData("https://icon.test", "127.0.0.1", false)] // IP is internal
|
||||
[InlineData("https://icon.test", "::1", false)] // IP is internal
|
||||
[InlineData("https://1.1.1.1", "::1", false)] // host is IP
|
||||
public void IsValid(string uri, string ip, bool expectedResult)
|
||||
{
|
||||
var result = new IconUri(new Uri(uri), IPAddress.Parse(ip)).IsValid;
|
||||
|
||||
Assert.Equal(expectedResult, result);
|
||||
}
|
||||
}
|
||||
@ -1,25 +1,25 @@
|
||||
using Bit.Icons.Services;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Xunit;
|
||||
|
||||
namespace Bit.Icons.Test.Services;
|
||||
|
||||
public class IconFetchingServiceTests
|
||||
public class IconFetchingServiceTests : ServiceTestBase<IconFetchingService>
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("www.twitter.com")] // https site
|
||||
[InlineData("www.google.com")] // https site
|
||||
[InlineData("neverssl.com")] // http site
|
||||
[InlineData("ameritrade.com")]
|
||||
[InlineData("neopets.com")] // uses favicon.ico
|
||||
[InlineData("hopin.com")] // uses svg+xml format
|
||||
[InlineData("ameritrade.com")] // redirects to tdameritrace.com
|
||||
[InlineData("icloud.com")]
|
||||
[InlineData("bofa.com", Skip = "Broken in pipeline for .NET 6. Tracking link: https://bitwarden.atlassian.net/browse/PS-982")]
|
||||
public async Task GetIconAsync_Success(string domain)
|
||||
{
|
||||
var sut = new IconFetchingService(GetLogger());
|
||||
var sut = BuildSut();
|
||||
var result = await sut.GetIconAsync(domain);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.NotNull(result.Icon);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
@ -28,23 +28,12 @@ public class IconFetchingServiceTests
|
||||
[InlineData("localhost")]
|
||||
public async Task GetIconAsync_ReturnsNull(string domain)
|
||||
{
|
||||
var sut = new IconFetchingService(GetLogger());
|
||||
var sut = BuildSut();
|
||||
var result = await sut.GetIconAsync(domain);
|
||||
|
||||
Assert.Null(result);
|
||||
}
|
||||
|
||||
private static ILogger<IconFetchingService> GetLogger()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(b =>
|
||||
{
|
||||
b.ClearProviders();
|
||||
b.AddDebug();
|
||||
});
|
||||
|
||||
var provider = services.BuildServiceProvider();
|
||||
|
||||
return provider.GetRequiredService<ILogger<IconFetchingService>>();
|
||||
}
|
||||
private IconFetchingService BuildSut() =>
|
||||
GetService<IconFetchingService>();
|
||||
}
|
||||
|
||||
41
test/Icons.Test/Services/ServiceTestBase.cs
Normal file
41
test/Icons.Test/Services/ServiceTestBase.cs
Normal file
@ -0,0 +1,41 @@
|
||||
using Bit.Icons.Extensions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Bit.Icons.Test.Services;
|
||||
|
||||
public class ServiceTestBase
|
||||
{
|
||||
internal ServiceCollection _services = new();
|
||||
internal ServiceProvider _provider;
|
||||
|
||||
public ServiceTestBase()
|
||||
{
|
||||
_services = new ServiceCollection();
|
||||
_services.AddLogging(b =>
|
||||
{
|
||||
b.ClearProviders();
|
||||
b.AddDebug();
|
||||
});
|
||||
|
||||
_services.ConfigureHttpClients();
|
||||
_services.AddHtmlParsing();
|
||||
_services.AddServices();
|
||||
|
||||
_provider = _services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
public T GetService<T>() =>
|
||||
_provider.GetRequiredService<T>();
|
||||
}
|
||||
|
||||
public class ServiceTestBase<TSut> : ServiceTestBase where TSut : class
|
||||
{
|
||||
public ServiceTestBase() : base()
|
||||
{
|
||||
_services.AddTransient<TSut>();
|
||||
_provider = _services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
public TSut Sut => GetService<TSut>();
|
||||
}
|
||||
@ -73,6 +73,33 @@
|
||||
"StackExchange.Redis": "2.5.43"
|
||||
}
|
||||
},
|
||||
"AutoFixture": {
|
||||
"type": "Transitive",
|
||||
"resolved": "4.17.0",
|
||||
"contentHash": "efMRCG3Epc4QDELwdmQGf6/caQUleRXPRCnLAq5gLMpTuOTcOQWV12vEJ8qo678Rj97/TjjxHYu/34rGkXdVAA==",
|
||||
"dependencies": {
|
||||
"Fare": "[2.1.1, 3.0.0)",
|
||||
"System.ComponentModel.Annotations": "4.3.0"
|
||||
}
|
||||
},
|
||||
"AutoFixture.AutoNSubstitute": {
|
||||
"type": "Transitive",
|
||||
"resolved": "4.17.0",
|
||||
"contentHash": "iWsRiDQ7T8s6F4mvYbSvPTq0GDtxJD6D+E1Fu9gVbHUvJiNikC1yIDNTH+3tQF7RK864HH/3R8ETj9m2X8UXvg==",
|
||||
"dependencies": {
|
||||
"AutoFixture": "4.17.0",
|
||||
"NSubstitute": "[2.0.3, 5.0.0)"
|
||||
}
|
||||
},
|
||||
"AutoFixture.Xunit2": {
|
||||
"type": "Transitive",
|
||||
"resolved": "4.17.0",
|
||||
"contentHash": "lrURL/LhJLPkn2tSPUEW8Wscr5LoV2Mr8A+ikn5gwkofex3o7qWUsBswlLw+KCA7EOpeqwZOldp3k91zDF+48Q==",
|
||||
"dependencies": {
|
||||
"AutoFixture": "4.17.0",
|
||||
"xunit.extensibility.core": "[2.2.0, 3.0.0)"
|
||||
}
|
||||
},
|
||||
"AutoMapper": {
|
||||
"type": "Transitive",
|
||||
"resolved": "12.0.1",
|
||||
@ -246,6 +273,14 @@
|
||||
"Microsoft.Win32.Registry": "5.0.0"
|
||||
}
|
||||
},
|
||||
"Fare": {
|
||||
"type": "Transitive",
|
||||
"resolved": "2.1.1",
|
||||
"contentHash": "HaI8puqA66YU7/9cK4Sgbs1taUTP1Ssa4QT2PIzqJ7GvAbN1QgkjbRsjH+FSbMh1MJdvS0CIwQNLtFT+KF6KpA==",
|
||||
"dependencies": {
|
||||
"NETStandard.Library": "1.6.1"
|
||||
}
|
||||
},
|
||||
"Fido2": {
|
||||
"type": "Transitive",
|
||||
"resolved": "3.0.1",
|
||||
@ -326,6 +361,15 @@
|
||||
"IdentityModel": "4.4.0"
|
||||
}
|
||||
},
|
||||
"Kralizek.AutoFixture.Extensions.MockHttp": {
|
||||
"type": "Transitive",
|
||||
"resolved": "1.2.0",
|
||||
"contentHash": "6zmks7/5mVczazv910N7V2EdiU6B+rY61lwdgVO0o2iZuTI6KI3T+Hgkrjv0eGOKYucq2OMC+gnAc5Ej2ajoTQ==",
|
||||
"dependencies": {
|
||||
"AutoFixture": "4.11.0",
|
||||
"RichardSzalay.MockHttp": "6.0.0"
|
||||
}
|
||||
},
|
||||
"LaunchDarkly.Cache": {
|
||||
"type": "Transitive",
|
||||
"resolved": "1.0.2",
|
||||
@ -1148,6 +1192,11 @@
|
||||
"System.Diagnostics.DiagnosticSource": "4.7.1"
|
||||
}
|
||||
},
|
||||
"RichardSzalay.MockHttp": {
|
||||
"type": "Transitive",
|
||||
"resolved": "6.0.0",
|
||||
"contentHash": "bStGNqIX/MGYtML7K3EzdsE/k5HGVAcg7XgN23TQXGXqxNC9fvYFR94fA0sGM5hAT36R+BBGet6ZDQxXL/IPxg=="
|
||||
},
|
||||
"runtime.debian.8-x64.runtime.native.System.Security.Cryptography.OpenSsl": {
|
||||
"type": "Transitive",
|
||||
"resolved": "4.3.2",
|
||||
@ -1568,6 +1617,24 @@
|
||||
"System.Runtime": "4.3.0"
|
||||
}
|
||||
},
|
||||
"System.ComponentModel.Annotations": {
|
||||
"type": "Transitive",
|
||||
"resolved": "4.3.0",
|
||||
"contentHash": "SY2RLItHt43rd8J9D8M8e8NM4m+9WLN2uUd9G0n1I4hj/7w+v3pzK6ZBjexlG1/2xvLKQsqir3UGVSyBTXMLWA==",
|
||||
"dependencies": {
|
||||
"System.Collections": "4.3.0",
|
||||
"System.ComponentModel": "4.3.0",
|
||||
"System.Globalization": "4.3.0",
|
||||
"System.Linq": "4.3.0",
|
||||
"System.Reflection": "4.3.0",
|
||||
"System.Reflection.Extensions": "4.3.0",
|
||||
"System.Resources.ResourceManager": "4.3.0",
|
||||
"System.Runtime": "4.3.0",
|
||||
"System.Runtime.Extensions": "4.3.0",
|
||||
"System.Text.RegularExpressions": "4.3.0",
|
||||
"System.Threading": "4.3.0"
|
||||
}
|
||||
},
|
||||
"System.ComponentModel.Primitives": {
|
||||
"type": "Transitive",
|
||||
"resolved": "4.3.0",
|
||||
@ -2770,6 +2837,18 @@
|
||||
"NETStandard.Library": "1.6.1"
|
||||
}
|
||||
},
|
||||
"common": {
|
||||
"type": "Project",
|
||||
"dependencies": {
|
||||
"AutoFixture.AutoNSubstitute": "[4.17.0, )",
|
||||
"AutoFixture.Xunit2": "[4.17.0, )",
|
||||
"Core": "[2023.5.1, )",
|
||||
"Kralizek.AutoFixture.Extensions.MockHttp": "[1.2.0, )",
|
||||
"Microsoft.NET.Test.Sdk": "[17.1.0, )",
|
||||
"NSubstitute": "[4.3.0, )",
|
||||
"xunit": "[2.4.1, )"
|
||||
}
|
||||
},
|
||||
"core": {
|
||||
"type": "Project",
|
||||
"dependencies": {
|
||||
@ -2850,4 +2929,4 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user