NativeAOT stack overflow with recursive GVM delegate on struct interface call

[hez2010]

Found in #128702. See details in https://dev.azure.com/dnceng-public/public/_build/results?buildId=1480580&view=ms.vss-test-web.build-test-results-tab&runId=40860540&resultId=116527&paneView=debug

Repro:

GenericVirtualMethodTests.RuntimeLookupDelegate();

class Assert
{
    public static void Equal<T>(T x, T y)
    {
        if (!EqualityComparer<T>.Default.Equals(x, y)) throw new Exception($"Expected: {x}, Actual: {y}");
    }
}

public class GenericVirtualMethodTests
{
    public static void RuntimeLookupDelegate()
    {
        RuntimeLookupDelegateGenericVirtual.TestGenericMethodOnNonGenericType<int>();
    }
}

internal class RuntimeLookupDelegateGenericVirtual
{
    internal static void TestGenericMethodOnNonGenericType<T>()
    {
        IBase test5 = new DerivedStruct();
        Delegate m3 = test5.Foo<List<T>>();
        Delegate m4 = test5.Foo<List<List<T>>>;
        Assert.Equal(m3.Method, m4.Method);
    }
}

internal interface IBase
{
    public virtual Delegate Foo<U>()
    {
        return Foo<U>;
    }
}

internal struct DerivedStruct : IBase
{
    public Delegate Foo<U>()
    {
        return Foo<List<U>>;
    }
}

Publish and run with NativeAOT:

Process is terminating due to StackOverflowException.

Stacktrace under Checked configuration:

System.OutOfMemoryException: Insufficient memory to continue the execution of the program.
   at System.Text.StringBuilder.ToString()
   at Internal.TypeSystem.NoMetadata.NoMetadataType.ToString()
   at Internal.TypeSystem.TypeSystemContext.ResolveGenericInstantiation(DefType, Instantiation)
   at Internal.Runtime.TypeLoader.NativeLayoutInfoLoadContext.GetInstantiationType(NativeParser&, UInt32)
   at Internal.Runtime.TypeLoader.NativeLayoutInfoLoadContext.GetType(NativeParser&)
   at Internal.Runtime.TypeLoader.NativeLayoutInfoLoadContext.GetTypeSequence(NativeParser&)
   at Internal.Runtime.TypeLoader.NativeLayoutInfoLoadContext.GetMethod(NativeParser&)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.ParseAndCreateCell(NativeLayoutInfoLoadContext, NativeParser&)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.BuildDictionary(TypeBuilder, NativeLayoutInfoLoadContext, NativeParser)
   at Internal.Runtime.TypeLoader.TypeBuilder.ParseNativeLayoutInfo(InstantiatedMethod)
   at Internal.Runtime.TypeLoader.TypeBuilder.PrepareMethod(MethodDesc)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.MethodCell.Prepare(TypeBuilder)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.BuildDictionary(TypeBuilder, NativeLayoutInfoLoadContext, NativeParser)
   ...
   at Internal.Runtime.TypeLoader.TypeBuilder.ParseNativeLayoutInfo(InstantiatedMethod)
   at Internal.Runtime.TypeLoader.TypeBuilder.PrepareMethod(MethodDesc)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.MethodCell.Prepare(TypeBuilder)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.BuildDictionary(TypeBuilder, NativeLayoutInfoLoadContext, NativeParser)
   at Internal.Runtime.TypeLoader.TypeBuilder.ParseNativeLayoutInfo(InstantiatedMethod)
   at Internal.Runtime.TypeLoader.TypeBuilder.PrepareMethod(MethodDesc)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.MethodCell.Prepare(TypeBuilder)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.BuildDictionary(TypeBuilder, NativeLayoutInfoLoadContext, NativeParser)
   at Internal.Runtime.TypeLoader.TypeBuilder.ParseNativeLayoutInfo(InstantiatedMethod)
   at Internal.Runtime.TypeLoader.TypeBuilder.PrepareMethod(MethodDesc)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.MethodCell.Prepare(TypeBuilder)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.BuildDictionary(TypeBuilder, NativeLayoutInfoLoadContext, NativeParser)
   at Internal.Runtime.TypeLoader.TypeBuilder.ParseNativeLayoutInfo(InstantiatedMethod)
   at Internal.Runtime.TypeLoader.TypeBuilder.PrepareMethod(MethodDesc)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.MethodCell.Prepare(TypeBuilder)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.BuildDictionary(TypeBuilder, NativeLayoutInfoLoadContext, NativeParser)
   at Internal.Runtime.TypeLoader.TypeBuilder.ParseNativeLayoutInfo(InstantiatedMethod)
   at Internal.Runtime.TypeLoader.TypeBuilder.PrepareMethod(MethodDesc)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.MethodCell.Prepare(TypeBuilder)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.BuildDictionary(TypeBuilder, NativeLayoutInfoLoadContext, NativeParser)
   at Internal.Runtime.TypeLoader.TypeBuilder.ParseNativeLayoutInfo(InstantiatedMethod)
   at Internal.Runtime.TypeLoader.TypeBuilder.PrepareMethod(MethodDesc)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.MethodCell.Prepare(TypeBuilder)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.BuildDictionary(TypeBuilder, NativeLayoutInfoLoadContext, NativeParser)
   at Internal.Runtime.TypeLoader.TypeBuilder.ParseNativeLayoutInfo(InstantiatedMethod)
   at Internal.Runtime.TypeLoader.TypeBuilder.PrepareMethod(MethodDesc)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.MethodCell.Prepare(TypeBuilder)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.BuildDictionary(TypeBuilder, NativeLayoutInfoLoadContext, NativeParser)
   at Internal.Runtime.TypeLoader.TypeBuilder.ParseNativeLayoutInfo(InstantiatedMethod)
   at Internal.Runtime.TypeLoader.TypeBuilder.PrepareMethod(MethodDesc)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.MethodCell.Prepare(TypeBuilder)
   at Internal.Runtime.TypeLoader.GenericDictionaryCell.BuildDictionary(TypeBuilder, NativeLayoutInfoLoadContext, NativeParser)
   at Internal.Runtime.TypeLoader.TypeBuilder.ParseNativeLayoutInfo(InstantiatedMethod)
   at Internal.Runtime.TypeLoader.TypeBuilder.PrepareMethod(MethodDesc)
   at Internal.Runtime.TypeLoader.TypeBuilder.BuildMethod(InstantiatedMethod)
   at Internal.Runtime.TypeLoader.TypeBuilder.TryBuildGenericMethod(InstantiatedMethod, IntPtr&)
   at Internal.Runtime.TypeLoader.TypeLoaderEnvironment.TryGetGenericVirtualMethodPointer(InstantiatedMethod, IntPtr&, IntPtr&)
   at Internal.Runtime.TypeLoader.TypeLoaderEnvironment.ResolveGenericVirtualMethodTarget(RuntimeTypeHandle, RuntimeMethodHandle)
   at System.Runtime.TypeLoaderExports.<>c.<GVMLookupForSlotSlow>b__8_0(IntPtr context, IntPtr signature, Object contextObject, IntPtr& auxResult)
   at System.Runtime.TypeLoaderExports.CacheMiss(IntPtr, IntPtr, RuntimeObjectFactory, Object)
   at System.Runtime.TypeLoaderExports.GVMLookupForSlotSlow(Object, RuntimeMethodHandle)
   at System.Runtime.TypeLoaderExports.GVMLookupForSlot(Object, RuntimeMethodHandle)
   at RuntimeLookupDelegateGenericVirtual.TestGenericMethodOnNonGenericType[T]()
   at GenericVirtualMethodTests.RuntimeLookupDelegate()
   at Program.<<Main>$>g__TestExecutor117|0_118(StreamWriter, StreamWriter, Program.<>c__DisplayClass0_0&)

cc: @MichalStrehovsky

[dotnet-policy-service]

Tagging subscribers to this area: @agocke, @dotnet/ilc-contrib
See info in area-owners.md if you want to be subscribed.