Fix DownloadHeader for real servers

Core.Test/NetRelated/DownloadHeaderTest.cs

@@ -0,0 +1,180 @@
+using System;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Threading;
+using System.Threading.Tasks;
+using Core.Extensions.NetRelated;
+using Core.Net.Impl;
+using Xunit;
+
+namespace Core.Test.NetRelated;
+
+/// <summary>
+/// Tests <see cref="HttpChannelExt.DownloadHeader"/> against a fake server, so no internet access is needed.
+/// </summary>
+[Collection(SharedHttpClientCollection.Name)]
+public class DownloadHeaderTest
+{
+    private static readonly DateTimeOffset LastModified = new(2015, 10, 21, 7, 28, 0, TimeSpan.Zero);
+    private static readonly DateTimeOffset Date = new(1994, 11, 15, 8, 12, 31, TimeSpan.Zero);
+
+    /// <summary>
+    /// A response like a real server answers a HEAD request: Content-* headers belong to the content headers,
+    /// all others to the response headers.
+    /// </summary>
+    private static HttpResponseMessage CreateHeadResponse()
+    {
+        var response = new HttpResponseMessage(HttpStatusCode.OK)
+        {
+            Content = new ByteArrayContent(Array.Empty<byte>())
+        };
+        response.Content.Headers.ContentLength = 1234;
+        response.Content.Headers.ContentType = new MediaTypeHeaderValue("text/plain") { CharSet = "utf-8" };
+        response.Content.Headers.LastModified = LastModified;
+
+        response.Headers.Server.ParseAdd("TestServer/1.0");
+        response.Headers.ETag = new EntityTagHeaderValue("\"abc\"");
+        response.Headers.AcceptRanges.Add("bytes");
+        response.Headers.Date = Date;
+        response.Headers.Location = new Uri("https://example.com/moved");
+        response.Headers.TryAddWithoutValidation("X-Multi", new[] { "first", "second" });
+        return response;
+    }
+
+    private sealed class StubHandler : HttpMessageHandler
+    {
+        private readonly Func<HttpRequestMessage, HttpResponseMessage> _respond;
+
+        public StubHandler(Func<HttpRequestMessage, HttpResponseMessage> respond)
+        {
+            _respond = respond;
+        }
+
+        public HttpRequestMessage? LastRequest { get; private set; }
+
+        protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+        {
+            LastRequest = request;
+            return Task.FromResult(_respond(request));
+        }
+    }
+
+    private static void WithFakeServer(StubHandler handler, Action action)
+    {
+        var original = HttpChannelExt.SharedHttpClient;
+        HttpChannelExt.SharedHttpClient = new Lazy<HttpClient>(() => new HttpClient(handler));
+        try
+        {
+            action();
+        }
+        finally
+        {
+            HttpChannelExt.SharedHttpClient = original;
+        }
+    }
+
+    [Fact]
+    public void ReadsTheValuesOfTheResponseHeaders()
+    {
+        var handler = new StubHandler(_ => CreateHeadResponse());
+
+        WithFakeServer(handler, () =>
+        {
+            var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file");
+
+            Assert.Equal("TestServer/1.0", header.Server);
+            Assert.Equal("\"abc\"", header.EntityTag);
+            Assert.True(header.AcceptRanges);
+            Assert.Equal("https://example.com/moved", header.Location);
+            Assert.Equal(Date.UtcDateTime, header.CreatedAtUtc);
+        });
+    }
+
+    [Fact]
+    public void ReadsTheContentHeaders()
+    {
+        var handler = new StubHandler(_ => CreateHeadResponse());
+
+        WithFakeServer(handler, () =>
+        {
+            var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file");
+
+            Assert.Equal(1234, header.ContentLength);
+            Assert.Equal("text/plain; charset=utf-8", header.ContentType);
+            Assert.Equal(LastModified.UtcDateTime, header.LastModifiedUtc);
+        });
+    }
+
+    [Fact]
+    public void JoinsMultipleValuesOfOneHeader()
+    {
+        var handler = new StubHandler(_ => CreateHeadResponse());
+
+        WithFakeServer(handler, () =>
+        {
+            var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file");
+
+            Assert.Equal("first, second", Assert.IsType<HttpHeader>(header).RawDictionary["X-Multi"]);
+        });
+    }
+
+    [Fact]
+    public void NoValueIsTheNameOfAType()
+    {
+        var handler = new StubHandler(_ => CreateHeadResponse());
+
+        WithFakeServer(handler, () =>
+        {
+            var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file");
+
+            var raw = Assert.IsType<HttpHeader>(header).RawDictionary;
+            Assert.NotEmpty(raw);
+            foreach (var pair in raw)
+                Assert.DoesNotContain("System.", pair.Value);
+        });
+    }
+
+    [Fact]
+    public void SendsAHeadRequestWithTheAuthorization()
+    {
+        var handler = new StubHandler(_ => CreateHeadResponse());
+        var authorization = new AuthenticationHeaderValue("Bearer", "token");
+
+        WithFakeServer(handler, () =>
+        {
+            new DefaultHttpChannel().DownloadHeader("https://example.com/file", authorization);
+
+            Assert.Equal(HttpMethod.Head, handler.LastRequest!.Method);
+            Assert.Equal(authorization, handler.LastRequest.Headers.Authorization);
+        });
+    }
+
+    [Fact]
+    public void KeepsSeveralSetCookieHeadersApart()
+    {
+        // a comma is part of a valid cookie (Expires), so the cookies must not be joined with a comma
+        const string first = "a=1; Path=/; Expires=Wed, 21 Oct 2015 07:28:00 GMT";
+        const string second = "b=2; Path=/";
+        var handler = new StubHandler(_ =>
+        {
+            var response = CreateHeadResponse();
+            response.Headers.TryAddWithoutValidation("Set-Cookie", new[] { first, second });
+            return response;
+        });
+
+        WithFakeServer(handler, () =>
+        {
+            var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file");
+
+            var httpHeader = Assert.IsType<HttpHeader>(header);
+            var expected = first + HttpHeader.SetCookieSeparator + second;
+            Assert.Equal(expected, header.SetCookie);
+            Assert.Equal(expected, httpHeader.RawDictionary["Set-Cookie"]);
+            Assert.Equal(new[] { first, second }, httpHeader.SetCookies);
+
+            // every other header with several values is still joined with a comma
+            Assert.Equal("first, second", httpHeader.RawDictionary["X-Multi"]);
+        });
+    }
+}

Core.Test/NetRelated/HttpChannelTest.cs

@@ -6,6 +6,7 @@
 namespace Core.Test.NetRelated;
 
 [Trait("Category", "Network")]
+[Collection(SharedHttpClientCollection.Name)]
 public class HttpChannelTest
 {
     [Fact]
@@ -15,4 +16,20 @@ public void BasicDownload()
         var site = client.DownloadToString("https://ipinfo.io/ip").Trim('\n', ' ', '\r');
         Assert.Matches(RegExLib.IpV4Address, site);
     }
+
+    [Fact]
+    public void DownloadHeaderFromARealServer()
+    {
+        var header = new DefaultHttpChannel().DownloadHeader("https://www.example.com/");
+
+        Assert.StartsWith("text/html", header.ContentType);
+        Assert.NotNull(header.CreatedAtUtc);
+    }
+
+    [Fact]
+    public void TryDownloadHeaderFromARealServer()
+    {
+        Assert.True(new DefaultHttpChannel().TryDownloadHeader("https://www.example.com/", out var header));
+        Assert.StartsWith("text/html", header.ContentType);
+    }
 }

Core.Test/NetRelated/HttpHeaderTest.cs

@@ -0,0 +1,82 @@
+using System;
+using System.Collections.Generic;
+using Core.Net.Impl;
+using Xunit;
+
+namespace Core.Test.NetRelated;
+
+public class HttpHeaderTest
+{
+    [Theory]
+    [InlineData("Content-Length", "Content-Type")]
+    [InlineData("content-length", "content-type")] // HTTP/2 sends all header names in lower case
+    [InlineData("CONTENT-LENGTH", "CONTENT-TYPE")]
+    public void HeaderNamesAreCaseInsensitive(string lengthKey, string typeKey)
+    {
+        var raw = new Dictionary<string, string>
+        {
+            { lengthKey, "42" },
+            { typeKey, "text/html" },
+        };
+
+        var header = new HttpHeader(raw);
+
+        Assert.Equal(42, header.ContentLength);
+        Assert.Equal("text/html", header.ContentType);
+    }
+
+    [Fact]
+    public void RawDictionaryIsTheGivenDictionary()
+    {
+        var raw = new Dictionary<string, string> { { "content-length", "42" } };
+
+        var header = new HttpHeader(raw);
+
+        Assert.Same(raw, header.RawDictionary);
+    }
+
+    [Fact]
+    public void ParsesDatesIndependentOfTheCurrentCulture()
+    {
+        var raw = new Dictionary<string, string>
+        {
+            { "date", "Tue, 15 Nov 1994 08:12:31 GMT" },
+            { "last-modified", "Wed, 21 Oct 2015 07:28:00 GMT" },
+        };
+
+        var header = new HttpHeader(raw);
+
+        Assert.Equal(new DateTime(1994, 11, 15, 8, 12, 31, DateTimeKind.Utc), header.CreatedAtUtc);
+        Assert.Equal(new DateTime(2015, 10, 21, 7, 28, 0, DateTimeKind.Utc), header.LastModifiedUtc);
+    }
+
+    [Fact]
+    public void SetCookiesIsEmptyWithoutTheHeader()
+    {
+        var header = new HttpHeader(new Dictionary<string, string>());
+
+        Assert.Null(header.SetCookie);
+        Assert.Empty(header.SetCookies);
+    }
+
+    [Fact]
+    public void SetCookiesWithOneCookie()
+    {
+        var header = new HttpHeader(new Dictionary<string, string> { { "set-cookie", "a=1; Path=/" } });
+
+        Assert.Equal("a=1; Path=/", header.SetCookie);
+        Assert.Equal(new[] { "a=1; Path=/" }, header.SetCookies);
+    }
+
+    [Fact]
+    public void SetCookiesSplitsAtTheSeparatorAndNotAtCommas()
+    {
+        const string first = "a=1; Expires=Wed, 21 Oct 2015 07:28:00 GMT";
+        const string second = "b=2";
+        var raw = new Dictionary<string, string> { { HttpHeader.SetCookieKey, first + HttpHeader.SetCookieSeparator + second } };
+
+        var header = new HttpHeader(raw);
+
+        Assert.Equal(new[] { first, second }, header.SetCookies);
+    }
+}

Core.Test/NetRelated/SharedHttpClientCollection.cs

@@ -0,0 +1,10 @@
+namespace Core.Test.NetRelated;
+
+/// <summary>
+/// Tests that replace or use the process-wide <c>HttpChannelExt.SharedHttpClient</c> must not run in parallel.
+/// Put them into this collection: <c>[Collection(SharedHttpClientCollection.Name)]</c>.
+/// </summary>
+public static class SharedHttpClientCollection
+{
+    public const string Name = "SharedHttpClient";
+}

Core/Extensions/NetRelated/HttpChannelExt.cs

@@ -38,12 +38,29 @@ public static IHttpHeader DownloadHeader(this IHttpChannel channel, string url,
 
         request.Method = HttpMethod.Head;
 
-        var result = SharedHttpClient.Value.SendAsync(request).Result;
+        using var result = SharedHttpClient.Value.SendAsync(request).Result;
 
-        var dict = result.Headers.ToDictionary(key => key.Key, v => v.Value.ToString() ?? "");
+        // Header names are case-insensitive. Content-Length, Content-Type, Last-Modified etc. are content headers,
+        // all others response headers, so both collections are needed. A header can have several values.
+        var dict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        AddHeaders(dict, result.Headers);
+        if (result.Content != null)
+            AddHeaders(dict, result.Content.Headers);
         return new HttpHeader(dict);
     }
 
+    private static void AddHeaders(Dictionary<string, string> target, IEnumerable<KeyValuePair<string, IEnumerable<string>>> headers)
+    {
+        foreach (var header in headers)
+        {
+            // several values are joined with a comma; Set-Cookie is the exception, see HttpHeader.SetCookieSeparator
+            var separator = string.Equals(header.Key, HttpHeader.SetCookieKey, StringComparison.OrdinalIgnoreCase)
+                ? HttpHeader.SetCookieSeparator
+                : ", ";
+            target[header.Key] = string.Join(separator, header.Value);
+        }
+    }
+
     public static bool TryDownloadHeader(this IHttpChannel channel, string url, out IHttpHeader header)
     {
         header = new HttpHeader("plain", 0);