diff --git a/Core.Test/NetRelated/DownloadHeaderTest.cs b/Core.Test/NetRelated/DownloadHeaderTest.cs new file mode 100644 index 0000000..a801f1d --- /dev/null +++ b/Core.Test/NetRelated/DownloadHeaderTest.cs @@ -0,0 +1,180 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using Core.Extensions.NetRelated; +using Core.Net.Impl; +using Xunit; + +namespace Core.Test.NetRelated; + +/// +/// Tests against a fake server, so no internet access is needed. +/// +[Collection(SharedHttpClientCollection.Name)] +public class DownloadHeaderTest +{ + private static readonly DateTimeOffset LastModified = new(2015, 10, 21, 7, 28, 0, TimeSpan.Zero); + private static readonly DateTimeOffset Date = new(1994, 11, 15, 8, 12, 31, TimeSpan.Zero); + + /// + /// A response like a real server answers a HEAD request: Content-* headers belong to the content headers, + /// all others to the response headers. + /// + private static HttpResponseMessage CreateHeadResponse() + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(Array.Empty()) + }; + response.Content.Headers.ContentLength = 1234; + response.Content.Headers.ContentType = new MediaTypeHeaderValue("text/plain") { CharSet = "utf-8" }; + response.Content.Headers.LastModified = LastModified; + + response.Headers.Server.ParseAdd("TestServer/1.0"); + response.Headers.ETag = new EntityTagHeaderValue("\"abc\""); + response.Headers.AcceptRanges.Add("bytes"); + response.Headers.Date = Date; + response.Headers.Location = new Uri("https://example.com/moved"); + response.Headers.TryAddWithoutValidation("X-Multi", new[] { "first", "second" }); + return response; + } + + private sealed class StubHandler : HttpMessageHandler + { + private readonly Func _respond; + + public StubHandler(Func respond) + { + _respond = respond; + } + + public HttpRequestMessage? LastRequest { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + LastRequest = request; + return Task.FromResult(_respond(request)); + } + } + + private static void WithFakeServer(StubHandler handler, Action action) + { + var original = HttpChannelExt.SharedHttpClient; + HttpChannelExt.SharedHttpClient = new Lazy(() => new HttpClient(handler)); + try + { + action(); + } + finally + { + HttpChannelExt.SharedHttpClient = original; + } + } + + [Fact] + public void ReadsTheValuesOfTheResponseHeaders() + { + var handler = new StubHandler(_ => CreateHeadResponse()); + + WithFakeServer(handler, () => + { + var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file"); + + Assert.Equal("TestServer/1.0", header.Server); + Assert.Equal("\"abc\"", header.EntityTag); + Assert.True(header.AcceptRanges); + Assert.Equal("https://example.com/moved", header.Location); + Assert.Equal(Date.UtcDateTime, header.CreatedAtUtc); + }); + } + + [Fact] + public void ReadsTheContentHeaders() + { + var handler = new StubHandler(_ => CreateHeadResponse()); + + WithFakeServer(handler, () => + { + var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file"); + + Assert.Equal(1234, header.ContentLength); + Assert.Equal("text/plain; charset=utf-8", header.ContentType); + Assert.Equal(LastModified.UtcDateTime, header.LastModifiedUtc); + }); + } + + [Fact] + public void JoinsMultipleValuesOfOneHeader() + { + var handler = new StubHandler(_ => CreateHeadResponse()); + + WithFakeServer(handler, () => + { + var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file"); + + Assert.Equal("first, second", Assert.IsType(header).RawDictionary["X-Multi"]); + }); + } + + [Fact] + public void NoValueIsTheNameOfAType() + { + var handler = new StubHandler(_ => CreateHeadResponse()); + + WithFakeServer(handler, () => + { + var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file"); + + var raw = Assert.IsType(header).RawDictionary; + Assert.NotEmpty(raw); + foreach (var pair in raw) + Assert.DoesNotContain("System.", pair.Value); + }); + } + + [Fact] + public void SendsAHeadRequestWithTheAuthorization() + { + var handler = new StubHandler(_ => CreateHeadResponse()); + var authorization = new AuthenticationHeaderValue("Bearer", "token"); + + WithFakeServer(handler, () => + { + new DefaultHttpChannel().DownloadHeader("https://example.com/file", authorization); + + Assert.Equal(HttpMethod.Head, handler.LastRequest!.Method); + Assert.Equal(authorization, handler.LastRequest.Headers.Authorization); + }); + } + + [Fact] + public void KeepsSeveralSetCookieHeadersApart() + { + // a comma is part of a valid cookie (Expires), so the cookies must not be joined with a comma + const string first = "a=1; Path=/; Expires=Wed, 21 Oct 2015 07:28:00 GMT"; + const string second = "b=2; Path=/"; + var handler = new StubHandler(_ => + { + var response = CreateHeadResponse(); + response.Headers.TryAddWithoutValidation("Set-Cookie", new[] { first, second }); + return response; + }); + + WithFakeServer(handler, () => + { + var header = new DefaultHttpChannel().DownloadHeader("https://example.com/file"); + + var httpHeader = Assert.IsType(header); + var expected = first + HttpHeader.SetCookieSeparator + second; + Assert.Equal(expected, header.SetCookie); + Assert.Equal(expected, httpHeader.RawDictionary["Set-Cookie"]); + Assert.Equal(new[] { first, second }, httpHeader.SetCookies); + + // every other header with several values is still joined with a comma + Assert.Equal("first, second", httpHeader.RawDictionary["X-Multi"]); + }); + } +} diff --git a/Core.Test/NetRelated/HttpChannelTest.cs b/Core.Test/NetRelated/HttpChannelTest.cs index 6c313cd..d2a69a1 100644 --- a/Core.Test/NetRelated/HttpChannelTest.cs +++ b/Core.Test/NetRelated/HttpChannelTest.cs @@ -6,6 +6,7 @@ namespace Core.Test.NetRelated; [Trait("Category", "Network")] +[Collection(SharedHttpClientCollection.Name)] public class HttpChannelTest { [Fact] @@ -15,4 +16,20 @@ public void BasicDownload() var site = client.DownloadToString("https://ipinfo.io/ip").Trim('\n', ' ', '\r'); Assert.Matches(RegExLib.IpV4Address, site); } + + [Fact] + public void DownloadHeaderFromARealServer() + { + var header = new DefaultHttpChannel().DownloadHeader("https://www.example.com/"); + + Assert.StartsWith("text/html", header.ContentType); + Assert.NotNull(header.CreatedAtUtc); + } + + [Fact] + public void TryDownloadHeaderFromARealServer() + { + Assert.True(new DefaultHttpChannel().TryDownloadHeader("https://www.example.com/", out var header)); + Assert.StartsWith("text/html", header.ContentType); + } } \ No newline at end of file diff --git a/Core.Test/NetRelated/HttpHeaderTest.cs b/Core.Test/NetRelated/HttpHeaderTest.cs new file mode 100644 index 0000000..d960958 --- /dev/null +++ b/Core.Test/NetRelated/HttpHeaderTest.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using Core.Net.Impl; +using Xunit; + +namespace Core.Test.NetRelated; + +public class HttpHeaderTest +{ + [Theory] + [InlineData("Content-Length", "Content-Type")] + [InlineData("content-length", "content-type")] // HTTP/2 sends all header names in lower case + [InlineData("CONTENT-LENGTH", "CONTENT-TYPE")] + public void HeaderNamesAreCaseInsensitive(string lengthKey, string typeKey) + { + var raw = new Dictionary + { + { lengthKey, "42" }, + { typeKey, "text/html" }, + }; + + var header = new HttpHeader(raw); + + Assert.Equal(42, header.ContentLength); + Assert.Equal("text/html", header.ContentType); + } + + [Fact] + public void RawDictionaryIsTheGivenDictionary() + { + var raw = new Dictionary { { "content-length", "42" } }; + + var header = new HttpHeader(raw); + + Assert.Same(raw, header.RawDictionary); + } + + [Fact] + public void ParsesDatesIndependentOfTheCurrentCulture() + { + var raw = new Dictionary + { + { "date", "Tue, 15 Nov 1994 08:12:31 GMT" }, + { "last-modified", "Wed, 21 Oct 2015 07:28:00 GMT" }, + }; + + var header = new HttpHeader(raw); + + Assert.Equal(new DateTime(1994, 11, 15, 8, 12, 31, DateTimeKind.Utc), header.CreatedAtUtc); + Assert.Equal(new DateTime(2015, 10, 21, 7, 28, 0, DateTimeKind.Utc), header.LastModifiedUtc); + } + + [Fact] + public void SetCookiesIsEmptyWithoutTheHeader() + { + var header = new HttpHeader(new Dictionary()); + + Assert.Null(header.SetCookie); + Assert.Empty(header.SetCookies); + } + + [Fact] + public void SetCookiesWithOneCookie() + { + var header = new HttpHeader(new Dictionary { { "set-cookie", "a=1; Path=/" } }); + + Assert.Equal("a=1; Path=/", header.SetCookie); + Assert.Equal(new[] { "a=1; Path=/" }, header.SetCookies); + } + + [Fact] + public void SetCookiesSplitsAtTheSeparatorAndNotAtCommas() + { + const string first = "a=1; Expires=Wed, 21 Oct 2015 07:28:00 GMT"; + const string second = "b=2"; + var raw = new Dictionary { { HttpHeader.SetCookieKey, first + HttpHeader.SetCookieSeparator + second } }; + + var header = new HttpHeader(raw); + + Assert.Equal(new[] { first, second }, header.SetCookies); + } +} diff --git a/Core.Test/NetRelated/SharedHttpClientCollection.cs b/Core.Test/NetRelated/SharedHttpClientCollection.cs new file mode 100644 index 0000000..c1f0dca --- /dev/null +++ b/Core.Test/NetRelated/SharedHttpClientCollection.cs @@ -0,0 +1,10 @@ +namespace Core.Test.NetRelated; + +/// +/// Tests that replace or use the process-wide HttpChannelExt.SharedHttpClient must not run in parallel. +/// Put them into this collection: [Collection(SharedHttpClientCollection.Name)]. +/// +public static class SharedHttpClientCollection +{ + public const string Name = "SharedHttpClient"; +} diff --git a/Core/Extensions/NetRelated/HttpChannelExt.cs b/Core/Extensions/NetRelated/HttpChannelExt.cs index b1525a8..1fbabf6 100644 --- a/Core/Extensions/NetRelated/HttpChannelExt.cs +++ b/Core/Extensions/NetRelated/HttpChannelExt.cs @@ -38,12 +38,29 @@ public static IHttpHeader DownloadHeader(this IHttpChannel channel, string url, request.Method = HttpMethod.Head; - var result = SharedHttpClient.Value.SendAsync(request).Result; + using var result = SharedHttpClient.Value.SendAsync(request).Result; - var dict = result.Headers.ToDictionary(key => key.Key, v => v.Value.ToString() ?? ""); + // Header names are case-insensitive. Content-Length, Content-Type, Last-Modified etc. are content headers, + // all others response headers, so both collections are needed. A header can have several values. + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + AddHeaders(dict, result.Headers); + if (result.Content != null) + AddHeaders(dict, result.Content.Headers); return new HttpHeader(dict); } + private static void AddHeaders(Dictionary target, IEnumerable>> headers) + { + foreach (var header in headers) + { + // several values are joined with a comma; Set-Cookie is the exception, see HttpHeader.SetCookieSeparator + var separator = string.Equals(header.Key, HttpHeader.SetCookieKey, StringComparison.OrdinalIgnoreCase) + ? HttpHeader.SetCookieSeparator + : ", "; + target[header.Key] = string.Join(separator, header.Value); + } + } + public static bool TryDownloadHeader(this IHttpChannel channel, string url, out IHttpHeader header) { header = new HttpHeader("plain", 0); diff --git a/Core/Net/Impl/HttpHeader.cs b/Core/Net/Impl/HttpHeader.cs index 9097ccd..8829b2b 100644 --- a/Core/Net/Impl/HttpHeader.cs +++ b/Core/Net/Impl/HttpHeader.cs @@ -20,6 +20,14 @@ public class HttpHeader : IHttpHeader // non standard but read somewhere public const string StatusKey = "Status"; + /// + /// Separates several Set-Cookie values in and in . + /// Other headers with several values are joined with a comma, but Set-Cookie must not be (RFC 6265, section 3; + /// RFC 9110, section 5.3): a comma can be part of a cookie, e.g. in Expires=Wed, 21 Oct 2015 07:28:00 GMT. + /// A line feed can never be part of a header value, so it keeps the cookies apart without loss. + /// + public const string SetCookieSeparator = "\n"; + /// /// Initializes the minimal information. Used for http responses from servers that doesn't support http headers /// @@ -36,37 +44,45 @@ public HttpHeader(IReadOnlyDictionary headerDict) { RawDictionary = headerDict; - if (headerDict.TryGetValue(AcceptRangesKey, out var acceptRangesValue)) + // header names are case-insensitive, e.g. HTTP/2 sends them in lower case + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var pair in headerDict) + headers[pair.Key] = pair.Value; + + if (headers.TryGetValue(AcceptRangesKey, out var acceptRangesValue)) AcceptRanges = acceptRangesValue == "bytes"; - if (headerDict.TryGetValue(ConnectionKey, out var connectionValue)) + if (headers.TryGetValue(ConnectionKey, out var connectionValue)) Connection = connectionValue; - if (headerDict.TryGetValue(ContentLengthKey, out var lengthValue)) + if (headers.TryGetValue(ContentLengthKey, out var lengthValue)) ContentLength = long.Parse(lengthValue, NumberStyles.Integer, CultureInfo.InvariantCulture); - if (headerDict.TryGetValue(ContentTypeKey, out var contentTypeValue)) + if (headers.TryGetValue(ContentTypeKey, out var contentTypeValue)) ContentType = contentTypeValue; - if (headerDict.TryGetValue(DateKey, out var dateValue)) + if (headers.TryGetValue(DateKey, out var dateValue)) CreatedAtUtc = DateTime.Parse(dateValue, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal); - if (headerDict.TryGetValue(ETagKey, out var eTagValue)) + if (headers.TryGetValue(ETagKey, out var eTagValue)) EntityTag = eTagValue; - if (headerDict.TryGetValue(LastModifiedKey, out var lastModifiedValue)) + if (headers.TryGetValue(LastModifiedKey, out var lastModifiedValue)) LastModifiedUtc = DateTime.Parse(lastModifiedValue, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal); - if (headerDict.TryGetValue(LocationKey, out var locationValue)) + if (headers.TryGetValue(LocationKey, out var locationValue)) Location = locationValue; - if (headerDict.TryGetValue(ServerKey, out var serverValue)) + if (headers.TryGetValue(ServerKey, out var serverValue)) Server = serverValue; - if (headerDict.TryGetValue(SetCookieKey, out var setCookieValue)) + if (headers.TryGetValue(SetCookieKey, out var setCookieValue)) + { SetCookie = setCookieValue; + SetCookies = setCookieValue.Split(SetCookieSeparatorChars, StringSplitOptions.RemoveEmptyEntries); + } - if (headerDict.TryGetValue(StatusKey, out var statusValue)) + if (headers.TryGetValue(StatusKey, out var statusValue)) Status = statusValue; } @@ -84,8 +100,18 @@ public HttpHeader(IReadOnlyDictionary headerDict) public string? Server { get; } + /// + /// All Set-Cookie values, joined with . + /// public string? SetCookie { get; } + /// + /// The Set-Cookie values, one entry per cookie. Empty if the header is not present. + /// + public IReadOnlyList SetCookies { get; } = Array.Empty(); + + private static readonly char[] SetCookieSeparatorChars = SetCookieSeparator.ToCharArray(); + public string? EntityTag { get; } public DateTime? LastModifiedUtc { get; }