From f54edf35705b6c26bcc8e1a7ef003bc9cb8df80d Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Thu, 4 Jun 2026 11:50:33 -0700 Subject: [PATCH 1/3] Disable Server GC for 32-bit crossgen2/ilc hosts crossgen2 and ilc enable Server GC unconditionally. On a 32-bit host process the ~2 GB user-mode address space is largely consumed up front by Server GC's per-heap segment reservations (observed ~1.5 GB / 75% reserved across 4 heaps with only ~296 MB committed). Compiling a very large method then fails to allocate the contiguous output-image buffer (MemoryStream.ToArray in EmitChecksums, ~10 MB for HugeArray1) and the process fail-fasts with OutOfMemory, even though real memory use is low. Use Workstation GC when the compiler process architecture is 32-bit (x86/arm/armel). The process architecture is CrossHostArch when cross-building, otherwise TargetArchitecture, mirroring the existing TargetArchitectureForSharedLibraries logic. Cross-targeted product builds run crossgen2 as a 64-bit host and keep Server GC, so only the genuinely 32-bit tool process is affected. Fixes https://github.com/dotnet/runtime/issues/128531 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/coreclr/tools/aot/AotCompilerCommon.props | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/coreclr/tools/aot/AotCompilerCommon.props b/src/coreclr/tools/aot/AotCompilerCommon.props index 800a9cff0cc9f6..62318cf70566c3 100644 --- a/src/coreclr/tools/aot/AotCompilerCommon.props +++ b/src/coreclr/tools/aot/AotCompilerCommon.props @@ -1,6 +1,16 @@ - true + + <_AotCompilerHostArch Condition="'$(CrossHostArch)' != ''">$(CrossHostArch) + <_AotCompilerHostArch Condition="'$(CrossHostArch)' == ''">$(TargetArchitecture) + + + true + false false true Speed From 334e4c079fb46137124efba198c524ea4c57bf9c Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Fri, 5 Jun 2026 09:45:50 -0700 Subject: [PATCH 2/3] Avoid full-image copy when computing R2R/AOT checksums EmitChecksums copied the entire output image into a MemoryStream and then into a contiguous byte[] (MemoryStream.ToArray) so that each checksum and the deterministic PE timestamp could be computed over the original, unmodified bytes. That copy is the size of the output image - 10s to 100s of MB for large R2R images, and 1+ GB for native AOT with debug info. On a 32-bit compiler host the ~2 GB user-mode address space is easily too fragmented to satisfy that single contiguous allocation, so compiling a very large method fail-fasts with OutOfMemory even though real memory use is low. Compute each checksum directly from the seekable output stream instead of from a full in-memory copy, and defer writing the checksum/timestamp values until all of them have been computed. Deferring the writes preserves the previous behavior of hashing the image before any checksum is written, so the output is byte-for-byte identical (verified by crossgen'ing a framework assembly before and after the change). IChecksumNode.EmitChecksum now takes the output Stream rather than a ReadOnlySpan over the whole image. Fixes https://github.com/dotnet/runtime/issues/128531 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../DependencyAnalysis/ISymbolNode.cs | 10 +++++++- .../Compiler/ObjectWriter/ObjectWriter.cs | 25 ++++++++++++------- .../Compiler/ObjectWriter/PEObjectWriter.cs | 19 +++++++++----- .../ReadyToRun/DebugDirectoryEntryNode.cs | 8 +++--- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/src/coreclr/tools/Common/Compiler/DependencyAnalysis/ISymbolNode.cs b/src/coreclr/tools/Common/Compiler/DependencyAnalysis/ISymbolNode.cs index 742e0bd9505f6b..c44efbb394bd8f 100644 --- a/src/coreclr/tools/Common/Compiler/DependencyAnalysis/ISymbolNode.cs +++ b/src/coreclr/tools/Common/Compiler/DependencyAnalysis/ISymbolNode.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.IO; using ILCompiler.DependencyAnalysisFramework; using Internal.Text; @@ -97,7 +98,14 @@ public interface IChecksumNode : ISymbolNode { int ChecksumSize { get; } - void EmitChecksum(ReadOnlySpan outputBlob, Span checksumLocation); + /// + /// Computes the checksum over the complete output image. + /// is a seekable stream positioned at the start of the image and contains the image as it + /// is before any checksums are written, so the result must only depend on those original + /// bytes. The method may read the stream and leave its position changed, but must not + /// modify the stream contents; the caller writes the result. + /// + void EmitChecksum(Stream outputStream, Span checksumLocation); } /// diff --git a/src/coreclr/tools/Common/Compiler/ObjectWriter/ObjectWriter.cs b/src/coreclr/tools/Common/Compiler/ObjectWriter/ObjectWriter.cs index 6229e6d43aaf45..c2cf6b7fdf4505 100644 --- a/src/coreclr/tools/Common/Compiler/ObjectWriter/ObjectWriter.cs +++ b/src/coreclr/tools/Common/Compiler/ObjectWriter/ObjectWriter.cs @@ -668,15 +668,20 @@ private static string GetNodeTypeName(Type nodeType) private void EmitChecksums(Stream outputFileStream, List checksumRelocations) { - MemoryStream originalOutputStream = new(); - outputFileStream.Seek(0, SeekOrigin.Begin); - outputFileStream.CopyTo(originalOutputStream); - byte[] originalOutput = originalOutputStream.ToArray(); - EmitChecksumsForObject(outputFileStream, checksumRelocations, originalOutput); + // Defer writing the computed values until all checksums are computed so each one is + // calculated over the original image and not a value written by an earlier checksum. + List<(long Offset, byte[] Value)> pendingWrites = ComputeChecksums(outputFileStream, checksumRelocations); + + foreach ((long offset, byte[] value) in pendingWrites) + { + outputFileStream.Seek(offset, SeekOrigin.Begin); + outputFileStream.Write(value); + } } - private protected virtual void EmitChecksumsForObject(Stream outputFileStream, List checksumRelocations, ReadOnlySpan originalOutput) + private protected virtual List<(long Offset, byte[] Value)> ComputeChecksums(Stream outputFileStream, List checksumRelocations) { + List<(long Offset, byte[] Value)> pendingWrites = []; foreach (var block in checksumRelocations) { foreach (var reloc in block.ChecksumRelocations) @@ -684,13 +689,15 @@ private protected virtual void EmitChecksumsForObject(Stream outputFileStream, L IChecksumNode checksum = (IChecksumNode)reloc.Target; byte[] checksumValue = new byte[checksum.ChecksumSize]; - checksum.EmitChecksum(originalOutput, checksumValue); + outputFileStream.Seek(0, SeekOrigin.Begin); + checksum.EmitChecksum(outputFileStream, checksumValue); var checksumOffset = (long)_outputSectionLayout[block.SectionIndex].FilePosition + block.Offset + reloc.Offset; - outputFileStream.Seek(checksumOffset, SeekOrigin.Begin); - outputFileStream.Write(checksumValue); + pendingWrites.Add((checksumOffset, checksumValue)); } } + + return pendingWrites; } partial void HandleControlFlowForRelocation(ISymbolNode relocTarget, Utf8String relocSymbolName); diff --git a/src/coreclr/tools/Common/Compiler/ObjectWriter/PEObjectWriter.cs b/src/coreclr/tools/Common/Compiler/ObjectWriter/PEObjectWriter.cs index ad67397800d1ea..ebbb1a33ebc115 100644 --- a/src/coreclr/tools/Common/Compiler/ObjectWriter/PEObjectWriter.cs +++ b/src/coreclr/tools/Common/Compiler/ObjectWriter/PEObjectWriter.cs @@ -11,7 +11,6 @@ using System.Reflection.Metadata; using System.Reflection.PortableExecutable; using System.Security.Cryptography; -using System.Text; using ILCompiler.DependencyAnalysis; @@ -868,17 +867,25 @@ private void PopulateDataDirectoryForWellKnownSymbolIfPresent(OptionalHeaderData } } - private protected override void EmitChecksumsForObject(Stream outputFileStream, List checksumRelocations, ReadOnlySpan originalOutput) + private protected override List<(long Offset, byte[] Value)> ComputeChecksums(Stream outputFileStream, List checksumRelocations) { - base.EmitChecksumsForObject(outputFileStream, checksumRelocations, originalOutput); + List<(long Offset, byte[] Value)> pendingWrites = base.ComputeChecksums(outputFileStream, checksumRelocations); if (_coffTimestamp is null) { // If we were not provided a deterministic timestamp, generate one from a hash of the content. - outputFileStream.Seek(_coffHeaderOffset + CoffHeader.TimeDateStampOffset(bigObj: false), SeekOrigin.Begin); - using BinaryWriter writer = new(outputFileStream, Encoding.UTF8, leaveOpen: true); - writer.Write(BlobContentId.FromHash(SHA256.HashData(originalOutput)).Stamp); + outputFileStream.Seek(0, SeekOrigin.Begin); + Span hash = stackalloc byte[SHA256.HashSizeInBytes]; + SHA256.HashData(outputFileStream, hash); + + byte[] timestamp = new byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(timestamp, BlobContentId.FromHash(hash.ToArray()).Stamp); + + long timestampOffset = _coffHeaderOffset + CoffHeader.TimeDateStampOffset(bigObj: false); + pendingWrites.Add((timestampOffset, timestamp)); } + + return pendingWrites; } private unsafe void ResolveRelocations(int sectionIndex, List symbolicRelocations, long imageBase, MemoryStream stream) diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryEntryNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryEntryNode.cs index fcac97dee0ccef..bc31ff907f5fea 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryEntryNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryEntryNode.cs @@ -244,11 +244,13 @@ private class RSDSChecksumNode : DependencyNodeCore, IChecksumNode { public int ChecksumSize => 16; - public void EmitChecksum(ReadOnlySpan outputBlob, Span checksumLocation) + public void EmitChecksum(Stream outputStream, Span checksumLocation) { Debug.Assert(checksumLocation.Length == ChecksumSize); - // Take the first 16 bytes of the SHA256 hash as the RSDS checksum. - SHA256.HashData(outputBlob)[0..ChecksumSize].CopyTo(checksumLocation); + // Take the first 16 bytes of the SHA256 hash of the image as the RSDS checksum. + Span hash = stackalloc byte[SHA256.HashSizeInBytes]; + SHA256.HashData(outputStream, hash); + hash[0..ChecksumSize].CopyTo(checksumLocation); } public override bool InterestingForDynamicDependencyAnalysis => false; From 7ef512dfd5fef99cfc90a686fed76ee48f382158 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Fri, 5 Jun 2026 09:51:20 -0700 Subject: [PATCH 3/3] Revert "Disable Server GC for 32-bit crossgen2/ilc hosts" This reverts commit f54edf35705b6c26bcc8e1a7ef003bc9cb8df80d. --- src/coreclr/tools/aot/AotCompilerCommon.props | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/coreclr/tools/aot/AotCompilerCommon.props b/src/coreclr/tools/aot/AotCompilerCommon.props index 62318cf70566c3..800a9cff0cc9f6 100644 --- a/src/coreclr/tools/aot/AotCompilerCommon.props +++ b/src/coreclr/tools/aot/AotCompilerCommon.props @@ -1,16 +1,6 @@ - - <_AotCompilerHostArch Condition="'$(CrossHostArch)' != ''">$(CrossHostArch) - <_AotCompilerHostArch Condition="'$(CrossHostArch)' == ''">$(TargetArchitecture) - - - true - false + true false true Speed