Skip to content

Commit eb765b7

Browse files
buyaa-nstephentoub
buyaa-n
and
stephentoub
authored
Base 64 decoder, reject input when unused bits are not 0 (#105262)
* Base 64 decoder, reject input when unused bits are not 0 * Update invalid json test value, fix issue in Base64Url validation * Apply feedbacks * Update comments * Apply suggestions from code review Co-authored-by: Stephen Toub <stoub@microsoft.com> --------- Co-authored-by: Stephen Toub <stoub@microsoft.com>
1 parent 99c9f5b commit eb765b7

8 files changed

Lines changed: 139 additions & 72 deletions

File tree

‎src/libraries/System.Memory/tests/Base64/Base64DecoderUnitTests.cs‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,7 @@ public void DecodingOutputTooSmall()
179179
{
180180
Span<byte> source = new byte[12];
181181
Base64TestHelper.InitializeDecodableBytes(source);
182+
source[9] = 65; // make sure unused bits set to 0
182183
source[10] = Base64TestHelper.EncodingPad;
183184
source[11] = Base64TestHelper.EncodingPad;
184185

@@ -193,6 +194,7 @@ public void DecodingOutputTooSmall()
193194
{
194195
Span<byte> source = new byte[12];
195196
Base64TestHelper.InitializeDecodableBytes(source);
197+
source[10] = 77; // make sure unused bits set to 0
196198
source[11] = Base64TestHelper.EncodingPad;
197199

198200
Span<byte> decodedBytes = new byte[7];
@@ -270,6 +272,23 @@ public void BasicDecodingWithFinalBlockTrueKnownInputDone(string inputString, in
270272
Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, expectedWritten, source, decodedBytes));
271273
}
272274

275+
[Theory]
276+
[InlineData("AR==")]
277+
[InlineData("AQJ=")]
278+
[InlineData("AQIDBB==")]
279+
[InlineData("AQIDBAV=")]
280+
[InlineData("AQIDBAUHCAkKCwwNDz==")]
281+
[InlineData("AQIDBAUHCAkKCwwNDxD=")]
282+
public void BasicDecodingWithNonZeroUnusedBits(string inputString)
283+
{
284+
Span<byte> source = Encoding.ASCII.GetBytes(inputString);
285+
Span<byte> decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)];
286+
287+
Assert.False(Base64.IsValid(inputString));
288+
Assert.Equal(OperationStatus.InvalidData, Base64.DecodeFromUtf8(source, decodedBytes, out int _, out int _));
289+
Assert.Equal(OperationStatus.InvalidData, Base64.DecodeFromUtf8InPlace(source, out int _));
290+
}
291+
273292
[Theory]
274293
[InlineData("A", 0, 0)]
275294
[InlineData("A===", 0, 0)]
@@ -468,10 +487,9 @@ public void DecodingInvalidBytesPadding(bool isFinalBlock)
468487

469488
// The last byte or the last 2 bytes being the padding character is valid, if isFinalBlock = true
470489
{
471-
Span<byte> source = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 };
490+
Span<byte> source = new byte[] { 50, 50, 50, 50, 80, 65,
491+
Base64TestHelper.EncodingPad, Base64TestHelper.EncodingPad }; // valid input - "2222PA=="
472492
Span<byte> decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)];
473-
source[6] = Base64TestHelper.EncodingPad;
474-
source[7] = Base64TestHelper.EncodingPad; // valid input - "2222PP=="
475493

476494
OperationStatus expectedStatus = isFinalBlock ? OperationStatus.Done : OperationStatus.InvalidData;
477495
int expectedConsumed = isFinalBlock ? source.Length : 4;
@@ -482,9 +500,9 @@ public void DecodingInvalidBytesPadding(bool isFinalBlock)
482500
Assert.Equal(expectedWritten, decodedByteCount);
483501
Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, expectedWritten, source, decodedBytes));
484502

485-
source = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 };
503+
source = new byte[] { 50, 50, 50, 50, 80, 80, 77, 80 };
486504
decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)];
487-
source[7] = Base64TestHelper.EncodingPad; // valid input - "2222PPP="
505+
source[7] = Base64TestHelper.EncodingPad; // valid input - "2222PPM="
488506

489507
expectedConsumed = isFinalBlock ? source.Length : 4;
490508
expectedWritten = isFinalBlock ? 5 : 3;
@@ -661,9 +679,8 @@ public void DecodeInPlaceInvalidBytesPadding()
661679

662680
// The last byte or the last 2 bytes being the padding character is valid
663681
{
664-
Span<byte> buffer = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 };
665-
buffer[6] = Base64TestHelper.EncodingPad;
666-
buffer[7] = Base64TestHelper.EncodingPad; // valid input - "2222PP=="
682+
Span<byte> buffer = new byte[] { 50, 50, 50, 50, 80, 65,
683+
Base64TestHelper.EncodingPad, Base64TestHelper.EncodingPad }; // valid input - "2222PA=="
667684
string sourceString = Encoding.ASCII.GetString(buffer.ToArray());
668685
Assert.Equal(OperationStatus.Done, Base64.DecodeFromUtf8InPlace(buffer, out int bytesWritten));
669686
Assert.Equal(4, bytesWritten);
@@ -672,8 +689,8 @@ public void DecodeInPlaceInvalidBytesPadding()
672689
}
673690

674691
{
675-
Span<byte> buffer = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 };
676-
buffer[7] = Base64TestHelper.EncodingPad; // valid input - "2222PPP="
692+
Span<byte> buffer = new byte[] { 50, 50, 50, 50, 80, 80, 77, 80 };
693+
buffer[7] = Base64TestHelper.EncodingPad; // valid input - "2222PPM="
677694
string sourceString = Encoding.ASCII.GetString(buffer.ToArray());
678695
Assert.Equal(OperationStatus.Done, Base64.DecodeFromUtf8InPlace(buffer, out int bytesWritten));
679696
Assert.Equal(5, bytesWritten);

‎src/libraries/System.Memory/tests/Base64Url/Base64UrlDecoderUnitTests.cs‎

Lines changed: 46 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ public void BasicDecoding()
2323

2424
Span<byte> source = new byte[numBytes];
2525
Base64TestHelper.InitializeUrlDecodableBytes(source, numBytes);
26+
source[numBytes - 1] = 65; // make sure unused bits set 0
2627

2728
Span<byte> decodedBytes = new byte[Base64Url.GetMaxDecodedLength(source.Length)];
2829
Assert.Equal(OperationStatus.Done, Base64Url.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount));
@@ -46,6 +47,7 @@ public void BasicDecodingByteArrayReturnOverload()
4647

4748
Span<byte> source = new byte[numBytes];
4849
Base64TestHelper.InitializeUrlDecodableBytes(source, numBytes);
50+
source[numBytes - 1] = 65; // make sure unused bits set 0
4951

5052
Span<byte> decodedBytes = Base64Url.DecodeFromUtf8(source);
5153
Assert.Equal(decodedBytes.Length, Base64Url.GetMaxDecodedLength(source.Length));
@@ -197,6 +199,7 @@ public void DecodingOutputTooSmall()
197199
{
198200
Span<byte> source = new byte[12];
199201
Base64TestHelper.InitializeUrlDecodableBytes(source);
202+
source[9] = 65; // make sure unused bits set 0
200203
source[10] = Base64TestHelper.EncodingPad;
201204
source[11] = Base64TestHelper.EncodingPad;
202205

@@ -211,6 +214,7 @@ public void DecodingOutputTooSmall()
211214
{
212215
Span<byte> source = new byte[12];
213216
Base64TestHelper.InitializeUrlDecodableBytes(source);
217+
source[10] = 77; // make sure unused bits set 0
214218
source[11] = Base64TestHelper.EncodingPad;
215219

216220
Span<byte> decodedBytes = new byte[7];
@@ -287,6 +291,23 @@ public void BasicDecodingWithFinalBlockTrueKnownInputDone(string inputString, in
287291
Assert.True(Base64TestHelper.VerifyUrlDecodingCorrectness(inputString.Length, expectedWritten, source, decodedBytes));
288292
}
289293

294+
[Theory]
295+
[InlineData("AR")]
296+
[InlineData("AQJ")]
297+
[InlineData("AQIDBB%%")]
298+
[InlineData("AQIDBAV%")]
299+
[InlineData("AQIDBAUHCAkKCwwNDz")]
300+
[InlineData("AQIDBAUHCAkKCwwNDxD")]
301+
public void BasicDecodingWithNonZeroUnusedBits(string inputString)
302+
{
303+
byte[] source = Encoding.ASCII.GetBytes(inputString);
304+
Span<byte> decodedBytes = new byte[Base64Url.GetMaxDecodedLength(source.Length)];
305+
306+
Assert.False(Base64Url.IsValid(inputString.AsSpan()));
307+
Assert.Equal(OperationStatus.InvalidData, Base64Url.DecodeFromUtf8(source, decodedBytes, out int _, out int _));
308+
Assert.Throws<FormatException>(() => Base64Url.DecodeFromUtf8InPlace(source));
309+
}
310+
290311
[Theory]
291312
[InlineData("A", 0, 0, OperationStatus.InvalidData)]
292313
[InlineData("A===", 0, 0, OperationStatus.InvalidData)]
@@ -460,7 +481,7 @@ public void DecodingInvalidBytes(bool isFinalBlock)
460481
// When isFinalBlock = true input that is not a multiple of 4 is invalid for Base64, but valid for Base64Url
461482
if (isFinalBlock)
462483
{
463-
Span<byte> source = "2222PPP"u8.ToArray(); // incomplete input
484+
Span<byte> source = "2222PPM"u8.ToArray(); // incomplete input
464485
Span<byte> decodedBytes = new byte[Base64Url.GetMaxDecodedLength(source.Length)];
465486
Assert.Equal(5, Base64Url.DecodeFromUtf8(source, decodedBytes));
466487
Assert.True(Base64TestHelper.VerifyUrlDecodingCorrectness(7, 5, source, decodedBytes));
@@ -517,10 +538,9 @@ public void DecodingInvalidBytesPadding(bool isFinalBlock)
517538

518539
// The last byte or the last 2 bytes being the padding character is valid, if isFinalBlock = true
519540
{
520-
Span<byte> source = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 };
541+
Span<byte> source = new byte[] { 50, 50, 50, 50, 80, 65,
542+
Base64TestHelper.EncodingPad, Base64TestHelper.EncodingPad }; // valid input - "2222PA=="
521543
Span<byte> decodedBytes = new byte[Base64Url.GetMaxDecodedLength(source.Length)];
522-
source[6] = Base64TestHelper.EncodingPad;
523-
source[7] = Base64TestHelper.EncodingPad; // valid input - "2222PP=="
524544

525545
OperationStatus expectedStatus = isFinalBlock ? OperationStatus.Done : OperationStatus.InvalidData;
526546
int expectedConsumed = isFinalBlock ? source.Length : 4;
@@ -531,9 +551,9 @@ public void DecodingInvalidBytesPadding(bool isFinalBlock)
531551
Assert.Equal(expectedWritten, decodedByteCount);
532552
Assert.True(Base64TestHelper.VerifyUrlDecodingCorrectness(expectedConsumed, expectedWritten, source, decodedBytes));
533553

534-
source = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 };
554+
source = new byte[] { 50, 50, 50, 50, 80, 80, 77, 80 };
535555
decodedBytes = new byte[Base64Url.GetMaxDecodedLength(source.Length)];
536-
source[7] = Base64TestHelper.UrlEncodingPad; // valid input - "2222PPP="
556+
source[7] = Base64TestHelper.UrlEncodingPad; // valid input - "2222PPM="
537557

538558
expectedConsumed = isFinalBlock ? source.Length : 4;
539559
expectedWritten = isFinalBlock ? 5 : 3;
@@ -685,9 +705,8 @@ public void DecodeInPlaceInvalidBytesPaddingThrowsFormatException()
685705

686706
// The last byte or the last 2 bytes being the padding character is valid
687707
{
688-
Span<byte> buffer = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 };
689-
buffer[6] = Base64TestHelper.UrlEncodingPad;
690-
buffer[7] = Base64TestHelper.EncodingPad; // valid input - "2222PP=="
708+
Span<byte> buffer = new byte[] { 50, 50, 50, 50, 80, 65,
709+
Base64TestHelper.UrlEncodingPad, Base64TestHelper.EncodingPad }; // valid input - "2222PA=="
691710
string sourceString = Encoding.ASCII.GetString(buffer.ToArray());
692711
int bytesWritten = Base64Url.DecodeFromUtf8InPlace(buffer);
693712

@@ -696,8 +715,7 @@ public void DecodeInPlaceInvalidBytesPaddingThrowsFormatException()
696715
}
697716

698717
{
699-
Span<byte> buffer = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 };
700-
buffer[7] = Base64TestHelper.EncodingPad; // valid input - "2222PPP="
718+
Span<byte> buffer = new byte[] { 50, 50, 50, 50, 80, 80, 77, Base64TestHelper.EncodingPad }; // valid input - "2222PPM="
701719
string sourceString = Encoding.ASCII.GetString(buffer.ToArray());
702720
int bytesWritten = Base64Url.DecodeFromUtf8InPlace(buffer);
703721

@@ -707,7 +725,7 @@ public void DecodeInPlaceInvalidBytesPaddingThrowsFormatException()
707725

708726
// The last byte or the last 2 bytes being the padding character is valid
709727
{
710-
Span<byte> buffer = new byte[] { 50, 50, 50, 50, 80, 80 }; // valid input without padding "2222PP"
728+
Span<byte> buffer = new byte[] { 50, 50, 50, 50, 80, 65 }; // valid input without padding "2222PA"
711729

712730
string sourceString = Encoding.ASCII.GetString(buffer.ToArray());
713731
int bytesWritten = Base64Url.DecodeFromUtf8InPlace(buffer);
@@ -775,12 +793,12 @@ public void DecodingInPlaceWithOnlyCharsToBeIgnored(string utf8WithCharsToBeIgno
775793
}
776794

777795
[Theory]
778-
[InlineData(new byte[] { 0xa, 0xa, 0x2d, 0x2d }, 251)]
779-
[InlineData(new byte[] { 0xa, 0x5f, 0xa, 0x2d }, 255)]
780-
[InlineData(new byte[] { 0x5f, 0x5f, 0xa, 0xa }, 255)]
781-
[InlineData(new byte[] { 0x70, 0xa, 0x61, 0xa }, 165)]
782-
[InlineData(new byte[] { 0xa, 0x70, 0xa, 0x61, 0xa }, 165)]
783-
[InlineData(new byte[] { 0x70, 0xa, 0x61, 0xa, 0x3d, 0x3d }, 165)]
796+
[InlineData(new byte[] { 0xa, 0xa, 0x2d, 0x77 }, 251)]
797+
[InlineData(new byte[] { 0xa, 0x5f, 0xa, 0x77 }, 255)]
798+
[InlineData(new byte[] { 0x5f, 0x77, 0xa, 0xa }, 255)]
799+
[InlineData(new byte[] { 0x70, 0xa, 0x51, 0xa }, 165)]
800+
[InlineData(new byte[] { 0xa, 0x70, 0xa, 0x51, 0xa }, 165)]
801+
[InlineData(new byte[] { 0x70, 0xa, 0x51, 0xa, 0x3d, 0x3d }, 165)]
784802
public void DecodingLessThan4BytesWithWhiteSpaces(byte[] utf8Bytes, byte decoded)
785803
{
786804
Assert.True(Base64Url.IsValid(utf8Bytes, out int decodedLength));
@@ -802,12 +820,12 @@ public void DecodingLessThan4BytesWithWhiteSpaces(byte[] utf8Bytes, byte decoded
802820
}
803821

804822
[Theory]
805-
[InlineData(new char[] { '\r', '\r', '-', '-' }, 251)]
806-
[InlineData(new char[] { '\r', '_', '\r', '-' }, 255)]
807-
[InlineData(new char[] { '_', '_', '\r', '\r' }, 255)]
808-
[InlineData(new char[] { 'p', '\r', 'a', '\r' }, 165)]
809-
[InlineData(new char[] { '\r', 'p', '\r', 'a', '\r' }, 165)]
810-
[InlineData(new char[] { 'p', '\r', 'a', '\r', '=', '=' }, 165)]
823+
[InlineData(new char[] { '\r', '\r', '-', 'w' }, 251)]
824+
[InlineData(new char[] { '\r', '_', '\r', 'w' }, 255)]
825+
[InlineData(new char[] { '_', 'w', '\r', '\r' }, 255)]
826+
[InlineData(new char[] { 'p', '\r', 'Q', '\r' }, 165)]
827+
[InlineData(new char[] { '\r', 'p', '\r', 'Q', '\r' }, 165)]
828+
[InlineData(new char[] { 'p', '\r', 'Q', '\r', '=', '=' }, 165)]
811829
public void DecodingLessThan4CharsWithWhiteSpaces(char[] utf8Bytes, byte decoded)
812830
{
813831
Assert.True(Base64Url.IsValid(utf8Bytes, out int decodedLength));
@@ -825,8 +843,8 @@ public void DecodingLessThan4CharsWithWhiteSpaces(char[] utf8Bytes, byte decoded
825843
}
826844

827845
[Theory]
828-
[InlineData(new byte[] { 0x4a, 0x74, 0xa, 0x4a, 0x4a, 0x74, 0xa, 0x4a }, new byte[] { 38, 210, 73, 180 })]
829-
[InlineData(new byte[] { 0xa, 0x2d, 0x56, 0xa, 0xa, 0xa, 0x2d, 0x4a, 0x4a, 0x4a, }, new byte[] { 249, 95, 137, 36 })]
846+
[InlineData(new byte[] { 0x4a, 0x74, 0xa, 0x4a, 0x4a, 0x74, 0xa, 0x41 }, new byte[] { 38, 210, 73, 180 })]
847+
[InlineData(new byte[] { 0xa, 0x2d, 0x56, 0xa, 0xa, 0xa, 0x2d, 0x4a, 0x4a, 0x41, }, new byte[] { 249, 95, 137, 36 })]
830848
public void DecodingNotMultipleOf4WithWhiteSpace(byte[] utf8Bytes, byte[] decoded)
831849
{
832850
Assert.True(Base64Url.IsValid(utf8Bytes, out int decodedLength));
@@ -847,8 +865,8 @@ public void DecodingNotMultipleOf4WithWhiteSpace(byte[] utf8Bytes, byte[] decode
847865
}
848866

849867
[Theory]
850-
[InlineData(new char[] { 'J', 't', '\r', 'J', 'J', 't', '\r', 'J' }, new byte[] { 38, 210, 73, 180 })]
851-
[InlineData(new char[] { '\r', '-', 'V', '\r', '\r', '\r', '-', 'J', 'J', 'J', }, new byte[] { 249, 95, 137, 36 })]
868+
[InlineData(new char[] { 'J', 't', '\r', 'J', 'J', 't', '\r', 'A' }, new byte[] { 38, 210, 73, 180 })]
869+
[InlineData(new char[] { '\r', '-', 'V', '\r', '\r', '\r', '-', 'J', 'J', 'A', }, new byte[] { 249, 95, 137, 36 })]
852870
public void DecodingNotMultipleOf4CharsWithWhiteSpace(char[] utf8Bytes, byte[] decoded)
853871
{
854872
Assert.True(Base64Url.IsValid(utf8Bytes, out int decodedLength));

‎src/libraries/System.Memory/tests/Base64Url/Base64UrlUnicodeAPIsUnitTests.cs‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ public void DecodeWithLargeSpan()
8080

8181
Span<char> source = new char[numBytes];
8282
Base64TestHelper.InitializeUrlDecodableChars(source, numBytes);
83+
source[numBytes - 1] = 'A'; // make sure unused bits set 0
8384

8485
Span<byte> decodedBytes = new byte[Base64Url.GetMaxDecodedLength(source.Length)];
8586
Assert.Equal(OperationStatus.Done, Base64Url.DecodeFromChars(source, decodedBytes, out int consumed, out int decodedByteCount));
@@ -262,45 +263,46 @@ public static void Roundtrip()
262263
}
263264

264265
[Fact]
265-
public static void PartialRoundtripWithoutPadding()
266+
public static void RoundtripWithoutPadding()
266267
{
267-
string input = "ab";
268+
string input = "ag";
268269
Verify(input, result =>
269270
{
270271
Assert.Equal(1, result.Length);
271272

272273
string roundtrippedString = Base64Url.EncodeToString(result);
273-
Assert.NotEqual(input, roundtrippedString);
274-
Assert.Equal(input[0], roundtrippedString[0]);
274+
Assert.Equal(input, roundtrippedString);
275275
});
276276
}
277277

278278
[Fact]
279-
public static void PartialRoundtripWithPadding2()
279+
public static void RoundtripWithPadding2()
280280
{
281-
string input = "ab==";
281+
string input = "ag==";
282282
Verify(input, result =>
283283
{
284284
Assert.Equal(1, result.Length);
285285

286286
string roundtrippedString = Base64Url.EncodeToString(result);
287-
Assert.NotEqual(input, roundtrippedString);
287+
Assert.NotEqual(input, roundtrippedString); // Padding character omitted
288288
Assert.Equal(input[0], roundtrippedString[0]);
289+
Assert.Equal(input[1], roundtrippedString[1]);
289290
});
290291
}
291292

292293
[Fact]
293-
public static void PartialRoundtripWithPadding1()
294+
public static void RoundtripWithPadding1()
294295
{
295-
string input = "789=";
296+
string input = "788=";
296297
Verify(input, result =>
297298
{
298299
Assert.Equal(2, result.Length);
299300

300301
string roundtrippedString = Base64Url.EncodeToString(result);
301-
Assert.NotEqual(input, roundtrippedString);
302+
Assert.NotEqual(input, roundtrippedString); // Padding character omitted
302303
Assert.Equal(input[0], roundtrippedString[0]);
303304
Assert.Equal(input[1], roundtrippedString[1]);
305+
Assert.Equal(input[2], roundtrippedString[2]);
304306
});
305307
}
306308

0 commit comments

Comments
 (0)