Check the status code, honor the encoding and reuse the HttpClient when downloading

Core.Test/NetRelated/DefaultDownloaderTest.cs

@@ -0,0 +1,128 @@
+using System;
+using System.Collections.Generic;
+using System.Net;
+using System.Net.Http;
+using System.Text;
+using System.Threading.Tasks;
+using Core.Extensions.NetRelated;
+using Core.Net.Impl;
+using Xunit;
+
+namespace Core.Test.NetRelated;
+
+public class DefaultDownloaderTest
+{
+    private static HttpResponseMessage Text(string text, HttpStatusCode status = HttpStatusCode.OK)
+    {
+        return new HttpResponseMessage(status) { Content = new StringContent(text, Encoding.UTF8, "text/plain") };
+    }
+
+    [Fact]
+    public async Task UsesTheGivenClientAndDoesNotDisposeIt()
+    {
+        var handler = new StubHandler(_ => Text("body"));
+        using var client = new HttpClient(handler);
+        var downloader = new DefaultDownloader(client);
+
+        Assert.Equal("body", downloader.DownloadToString("https://example.com/a"));
+        Assert.Equal("body", downloader.DownloadToString("https://example.com/b"));
+
+        Assert.Equal(2, handler.RequestCount);
+        // a disposed client throws an ObjectDisposedException
+        Assert.Equal("body", await client.GetStringAsync("https://example.com/c"));
+    }
+
+    [Fact]
+    public void FailsForAnErrorStatus()
+    {
+        var handler = new StubHandler(_ => Text("error page", HttpStatusCode.NotFound));
+        using var client = new HttpClient(handler);
+        var downloader = new DefaultDownloader(client);
+
+        Assert.False(downloader.TryDownloadToString("https://example.com/missing", out var result, "fallback"));
+        Assert.Equal("fallback", result);
+    }
+
+    [Fact]
+    public void TheClientIsRequired()
+    {
+        Assert.Throws<ArgumentNullException>(() => new DefaultDownloader(null!));
+    }
+
+    [Fact]
+    public void DownloaderWithCredentialsSendsTheCredentialsAndCanBeUsedRepeatedly()
+    {
+        var seen = new List<string>();
+
+        // the listener answers a request without credentials with a 401 itself
+        using var server = new LocalHttpServer(context =>
+        {
+            var identity = (HttpListenerBasicIdentity)context.User!.Identity!;
+            lock (seen)
+                seen.Add($"{identity.Name}:{identity.Password}");
+            LocalHttpServer.WriteText(context, "secret");
+        }, AuthenticationSchemes.Basic);
+
+        var downloader = new DownloaderWithCredentials(new NetworkCredential("user", "pass"));
+
+        Assert.Equal("secret", downloader.DownloadToString(server.Url("/secret")));
+        Assert.Equal("secret", downloader.DownloadToString(server.Url("/secret")));
+
+        lock (seen)
+            Assert.Equal(new[] { "user:pass", "user:pass" }, seen);
+    }
+
+    [Fact]
+    public void ThrowsTheHttpRequestExceptionItselfForAnErrorStatus()
+    {
+        using var client = new HttpClient(new StubHandler(_ => Text("error page", HttpStatusCode.NotFound)));
+
+        // not wrapped in an AggregateException
+        Assert.Throws<HttpRequestException>(() => new DefaultDownloader(client).DownloadToString("https://example.com/missing"));
+    }
+
+    [Fact]
+    public void ThrowsTheHttpRequestExceptionItselfIfTheServerCannotBeReached()
+    {
+        using var client = new HttpClient(new StubHandler(_ => throw new HttpRequestException("no route to host")));
+
+        var exception = Assert.Throws<HttpRequestException>(() => new DefaultDownloader(client).DownloadToString("https://example.com/"));
+        Assert.Equal("no route to host", exception.Message);
+    }
+
+    [Fact]
+    public void TheCredentialsAreRequired()
+    {
+        Assert.Throws<ArgumentNullException>(() => new DownloaderWithCredentials(null!));
+    }
+
+    [Fact]
+    public void HonorsTheTimeoutOfTheGivenClient()
+    {
+        using var client = new HttpClient(new HangingHandler()) { Timeout = TimeSpan.FromMilliseconds(100) };
+
+        Assert.ThrowsAny<OperationCanceledException>(() => new DefaultDownloader(client).DownloadToString("https://example.com/slow"));
+    }
+
+    [Fact]
+    public void AnAlreadyDisposedClientFailsWhenItIsUsed()
+    {
+        var client = new HttpClient(new StubHandler(_ => Text("body")));
+        client.Dispose();
+
+        // the downloader does not own the client, so it does not check it: using it is the caller's job
+        Assert.Throws<ObjectDisposedException>(() => new DefaultDownloader(client).DownloadToString("https://example.com/"));
+    }
+
+    /// <summary>
+    /// Never answers, so only the timeout of the client ends the request.
+    /// </summary>
+    private sealed class HangingHandler : HttpMessageHandler
+    {
+        protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, System.Threading.CancellationToken cancellationToken)
+        {
+            await Task.Delay(System.Threading.Timeout.InfiniteTimeSpan, cancellationToken);
+            return new HttpResponseMessage();
+        }
+    }
+}

Core.Test/NetRelated/DownloadHeaderTest.cs

@@ -42,44 +42,12 @@ private static HttpResponseMessage CreateHeadResponse()
         return response;
     }
 
-    private sealed class StubHandler : HttpMessageHandler
-    {
-        private readonly Func<HttpRequestMessage, HttpResponseMessage> _respond;
-
-        public StubHandler(Func<HttpRequestMessage, HttpResponseMessage> respond)
-        {
-            _respond = respond;
-        }
-
-        public HttpRequestMessage? LastRequest { get; private set; }
-
-        protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
-        {
-            LastRequest = request;
-            return Task.FromResult(_respond(request));
-        }
-    }
-
-    private static void WithFakeServer(StubHandler handler, Action action)
-    {
-        var original = HttpChannelExt.SharedHttpClient;
-        HttpChannelExt.SharedHttpClient = new Lazy<HttpClient>(() => new HttpClient(handler));
-        try
-        {
-            action();
-        }
-        finally
-        {
-            HttpChannelExt.SharedHttpClient = original;
-        }
-    }
-
     [Fact]
     public void ReadsTheValuesOfTheResponseHeaders()
     {
         var handler = new StubHandler(_ => CreateHeadResponse());
 
-        WithFakeServer(handler, () =>
+        SharedHttpClientSwap.Use(handler, () =>
         {
             var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file");
 
@@ -96,7 +64,7 @@ public void ReadsTheContentHeaders()
     {
         var handler = new StubHandler(_ => CreateHeadResponse());
 
-        WithFakeServer(handler, () =>
+        SharedHttpClientSwap.Use(handler, () =>
         {
             var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file");
 
@@ -111,7 +79,7 @@ public void JoinsMultipleValuesOfOneHeader()
     {
         var handler = new StubHandler(_ => CreateHeadResponse());
 
-        WithFakeServer(handler, () =>
+        SharedHttpClientSwap.Use(handler, () =>
         {
             var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file");
 
@@ -124,7 +92,7 @@ public void NoValueIsTheNameOfAType()
     {
         var handler = new StubHandler(_ => CreateHeadResponse());
 
-        WithFakeServer(handler, () =>
+        SharedHttpClientSwap.Use(handler, () =>
         {
             var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file");
 
@@ -141,7 +109,7 @@ public void SendsAHeadRequestWithTheAuthorization()
         var handler = new StubHandler(_ => CreateHeadResponse());
         var authorization = new AuthenticationHeaderValue("Bearer", "token");
 
-        WithFakeServer(handler, () =>
+        SharedHttpClientSwap.Use(handler, () =>
         {
             new DefaultHttpChannel().DownloadHeader("https://example.com/file", authorization);
 
@@ -163,7 +131,7 @@ public void KeepsSeveralSetCookieHeadersApart()
             return response;
         });
 
-        WithFakeServer(handler, () =>
+        SharedHttpClientSwap.Use(handler, () =>
         {
             var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file");
 

Core.Test/NetRelated/DownloadToStringTest.cs

@@ -0,0 +1,141 @@
+using System.Net;
+using System.Net.Http;
+using System.Text;
+using Core.Extensions.NetRelated;
+using Core.Net.Impl;
+using Xunit;
+
+namespace Core.Test.NetRelated;
+
+/// <summary>
+/// Tests <see cref="HttpChannelExt.DownloadToString"/> and the downloader on top of it against a fake server.
+/// </summary>
+[Collection(SharedHttpClientCollection.Name)]
+public class DownloadToStringTest
+{
+    private const string Url = "https://example.com/text";
+    private const string Text = "Grüße aus Köln";
+    private static readonly Encoding Latin1 = Encoding.GetEncoding("ISO-8859-1");
+
+    private static HttpResponseMessage Response(byte[] body, string? contentType, HttpStatusCode status = HttpStatusCode.OK)
+    {
+        var response = new HttpResponseMessage(status) { Content = new ByteArrayContent(body) };
+        if (contentType != null)
+            response.Content.Headers.TryAddWithoutValidation("Content-Type", contentType);
+        return response;
+    }
+
+    private static StubHandler Serve(byte[] body, string? contentType, HttpStatusCode status = HttpStatusCode.OK)
+    {
+        return new StubHandler(_ => Response(body, contentType, status));
+    }
+
+    [Theory]
+    [InlineData(HttpStatusCode.NotFound)]
+    [InlineData(HttpStatusCode.InternalServerError)]
+    public void ThrowsForAnErrorStatusInsteadOfReturningTheErrorPage(HttpStatusCode status)
+    {
+        var handler = Serve(Encoding.UTF8.GetBytes("<html>error page</html>"), "text/html", status);
+
+        SharedHttpClientSwap.Use(handler, () =>
+        {
+            var exception = Assert.Throws<HttpRequestException>(() => new DefaultHttpChannel().DownloadToString(Url));
+            Assert.Contains(((int)status).ToString(), exception.Message);
+        });
+    }
+
+    [Fact]
+    public void TryDownloadToStringFailsForAnErrorStatus()
+    {
+        var handler = Serve(Encoding.UTF8.GetBytes("<html>error page</html>"), "text/html", HttpStatusCode.NotFound);
+
+        SharedHttpClientSwap.Use(handler, () =>
+        {
+            var downloader = new HttpChannelDownloader();
+
+            Assert.False(downloader.TryDownloadToString(Url, out var result, "fallback"));
+            Assert.Equal("fallback", result);
+        });
+    }
+
+    [Fact]
+    public void UsesTheCharsetOfTheServerWithoutAnExplicitEncoding()
+    {
+        var handler = Serve(Latin1.GetBytes(Text), "text/plain; charset=iso-8859-1");
+
+        SharedHttpClientSwap.Use(handler, () =>
+            Assert.Equal(Text, new DefaultHttpChannel().DownloadToString(Url)));
+    }
+
+    [Fact]
+    public void UsesTheExplicitEncoding()
+    {
+        // the server does not say which charset it uses, so the default (UTF-8) would garble the text
+        var handler = Serve(Latin1.GetBytes(Text), "text/plain");
+
+        SharedHttpClientSwap.Use(handler, () =>
+            Assert.Equal(Text, new DefaultHttpChannel().DownloadToString(Url, Latin1)));
+    }
+
+    [Fact]
+    public void TheExplicitEncodingWinsOverTheCharsetOfTheServer()
+    {
+        var handler = Serve(Encoding.UTF8.GetBytes(Text), "text/plain; charset=iso-8859-1");
+
+        SharedHttpClientSwap.Use(handler, () =>
+            Assert.Equal(Text, new DefaultHttpChannel().DownloadToString(Url, Encoding.UTF8)));
+    }
+
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public void TheByteOrderMarkIsNotPartOfTheText(bool explicitEncoding)
+    {
+        var body = new byte[] { 0xEF, 0xBB, 0xBF };
+        body = Concat(body, Encoding.UTF8.GetBytes(Text));
+        var handler = Serve(body, "text/plain; charset=utf-8");
+
+        SharedHttpClientSwap.Use(handler, () =>
+        {
+            var text = new DefaultHttpChannel().DownloadToString(Url, explicitEncoding ? Encoding.UTF8 : null);
+            Assert.Equal(Text, text);
+        });
+    }
+
+    [Fact]
+    public void AByteOrderMarkWinsOverTheExplicitEncoding()
+    {
+        // the caller says Latin-1, but the content is UTF-8 with a byte order mark: the mark tells the truth
+        var body = Concat(new byte[] { 0xEF, 0xBB, 0xBF }, Encoding.UTF8.GetBytes(Text));
+        var handler = Serve(body, "text/plain");
+
+        SharedHttpClientSwap.Use(handler, () =>
+            Assert.Equal(Text, new DefaultHttpChannel().DownloadToString(Url, Latin1)));
+    }
+
+    [Fact]
+    public void HttpChannelDownloaderUsesTheCharsetOfTheServerByDefault()
+    {
+        var handler = Serve(Latin1.GetBytes(Text), "text/plain; charset=iso-8859-1");
+
+        SharedHttpClientSwap.Use(handler, () =>
+            Assert.Equal(Text, new HttpChannelDownloader().DownloadToString(Url)));
+    }
+
+    [Fact]
+    public void HttpChannelDownloaderUsesTheGivenEncoding()
+    {
+        var handler = Serve(Latin1.GetBytes(Text), "text/plain");
+
+        SharedHttpClientSwap.Use(handler, () =>
+            Assert.Equal(Text, new HttpChannelDownloader(encoding: Latin1).DownloadToString(Url)));
+    }
+
+    private static byte[] Concat(byte[] first, byte[] second)
+    {
+        var result = new byte[first.Length + second.Length];
+        first.CopyTo(result, 0);
+        second.CopyTo(result, first.Length);
+        return result;
+    }
+}