fix shutdown tests

src/libraries/Microsoft.Extensions.Hosting/tests/FunctionalTests/IntegrationTesting/src/Deployers/ApplicationDeployer.cs

@@ -102,8 +102,8 @@ protected void CleanPublishedOutput()
 
         protected string GetDotNetExeForArchitecture()
         {
-            var executableName = DotnetCommandName;
-            // We expect x64 dotnet.exe to be on the path but we have to go searching for the x86 version.
+            var executableName = GetHostDotNetExecutable();
+            // We expect the x64 dotnet host to be resolvable, but we have to go searching for the x86 version.
             if (DotNetCommands.IsRunningX86OnX64(DeploymentParameters.RuntimeArchitecture))
             {
                 executableName = DotNetCommands.GetDotNetExecutable(DeploymentParameters.RuntimeArchitecture);
@@ -116,15 +116,36 @@ protected string GetDotNetExeForArchitecture()
             return executableName;
         }
 
+        // The runtime libraries Helix harness runs tests against the testhost via $RUNTIME_PATH/dotnet by
+        // absolute path and doesn't add dotnet to PATH (that's only done for workload tests), so the bare
+        // command name can fail to launch on machines without a global dotnet. Resolve the muxer next to the
+        // running shared framework instead, falling back to PATH for local runs.
+        private static string GetHostDotNetExecutable()
+        {
+            var runtimeDirectory = Path.GetDirectoryName(typeof(object).Assembly.Location);
+            if (!string.IsNullOrEmpty(runtimeDirectory))
+            {
+                // runtimeDirectory is <testhost>/shared/Microsoft.NETCore.App/<version>; the muxer lives at <testhost>/dotnet.
+                string dotnetRoot = Path.GetFullPath(Path.Combine(runtimeDirectory, "..", "..", ".."));
+                string muxer = Path.Combine(dotnetRoot, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "dotnet.exe" : "dotnet");
+                if (File.Exists(muxer))
+                {
+                    return muxer;
+                }
+            }
+
+            return DotnetCommandName;
+        }
+
         protected void ShutDownIfAnyHostProcess(Process hostProcess)
         {
-            if (hostProcess != null && !hostProcess.HasExited)
+            if (hostProcess is not null && IsRunning(hostProcess))
             {
                 Logger.LogInformation("Attempting to cancel process {0}", hostProcess.Id);
 
                 // Shutdown the host process.
                 hostProcess.KillTree();
-                if (!hostProcess.HasExited)
+                if (IsRunning(hostProcess))
                 {
                     Logger.LogWarning("Unable to terminate the host process with process Id '{processId}", hostProcess.Id);
                 }
@@ -139,6 +160,21 @@ protected void ShutDownIfAnyHostProcess(Process hostProcess)
             }
         }
 
+        // Process.HasExited throws InvalidOperationException ("No process is associated with this object")
+        // when the process was never started. Treat that as "not running" so disposal after a failed
+        // deployment doesn't mask the original start failure with a misleading exception.
+        private static bool IsRunning(Process hostProcess)
+        {
+            try
+            {
+                return !hostProcess.HasExited;
+            }
+            catch (InvalidOperationException)
+            {
+                return false;
+            }
+        }
+
         protected void AddEnvironmentVariablesToProcess(ProcessStartInfo startInfo, IDictionary<string, string> environmentVariables)
         {
             var environment = startInfo.Environment;

src/libraries/Microsoft.Extensions.Hosting/tests/FunctionalTests/IntegrationTesting/src/Deployers/SelfHostDeployer.cs

@@ -122,10 +122,44 @@ protected async Task<CancellationToken> StartSelfHostAsync()
                 AddEnvironmentVariablesToProcess(startInfo, DeploymentParameters.EnvironmentVariables);
 
                 var started = new TaskCompletionSource<object>();
+                var hostExitTokenSource = new CancellationTokenSource();
+
+                await StartHostWithRetryAsync(startInfo, executableName, started, hostExitTokenSource);
+
+                if (HostProcess.HasExited)
+                {
+                    Logger.LogError("Host process {processName} {pid} exited with code {exitCode} or failed to start.", startInfo.FileName, HostProcess.Id, HostProcess.ExitCode);
+                    throw new Exception("Failed to start host");
+                }
+
+                Logger.LogInformation("Started {fileName}. Process Id : {processId}", startInfo.FileName, HostProcess.Id);
 
-                HostProcess = new Process() { StartInfo = startInfo };
-                HostProcess.EnableRaisingEvents = true;
-                HostProcess.OutputDataReceived += (sender, dataArgs) =>
+                // Host may not write startup messages, in which case assume it started
+                if (DeploymentParameters.StatusMessagesEnabled)
+                {
+                    // The timeout here is large, because we don't know how long the test could need
+                    // We cover a lot of error cases above, but I want to make sure we eventually give up and don't hang the build
+                    // just in case we missed one -anurse
+                    await started.Task.WaitAsync(TimeSpan.FromMinutes(10));
+                }
+
+                return hostExitTokenSource.Token;
+            }
+        }
+
+        // Launching the host process can fail transiently on constrained CI/Helix machines (for example a
+        // failed fork or a momentarily unavailable executable). Retry a few times before giving up so a
+        // one-off launch failure doesn't fail the test; the final failure is rethrown with its real cause.
+        private async Task StartHostWithRetryAsync(ProcessStartInfo startInfo, string executableName, TaskCompletionSource<object> started, CancellationTokenSource hostExitTokenSource)
+        {
+            const int MaxAttempts = 3;
+            TimeSpan retryDelay = TimeSpan.FromSeconds(2);
+
+            for (int attempt = 1; ; attempt++)
+            {
+                var process = new Process() { StartInfo = startInfo };
+                process.EnableRaisingEvents = true;
+                process.OutputDataReceived += (sender, dataArgs) =>
                 {
                     if (string.Equals(dataArgs.Data, ApplicationStartedMessage))
                     {
@@ -134,44 +168,35 @@ protected async Task<CancellationToken> StartSelfHostAsync()
 
                     OutputReceived?.Invoke(sender, dataArgs);
                 };
-                var hostExitTokenSource = new CancellationTokenSource();
-                HostProcess.Exited += (sender, e) =>
+                process.Exited += (sender, e) =>
                 {
-                    Logger.LogInformation("host process ID {pid} shut down", HostProcess.Id);
+                    Logger.LogInformation("host process ID {pid} shut down", process.Id);
 
                     // If TrySetResult was called above, this will just silently fail to set the new state, which is what we want
-                    started.TrySetException(new Exception($"Command exited unexpectedly with exit code: {HostProcess.ExitCode}"));
+                    started.TrySetException(new Exception($"Command exited unexpectedly with exit code: {process.ExitCode}"));
 
                     TriggerHostShutdown(hostExitTokenSource);
                 };
 
+                HostProcess = process;
+
                 try
                 {
-                    HostProcess.StartAndCaptureOutAndErrToLogger(executableName, Logger);
-                }
-                catch (Exception ex)
-                {
-                    Logger.LogError("Error occurred while starting the process. Exception: {exception}", ex.ToString());
+                    process.StartAndCaptureOutAndErrToLogger(executableName, Logger);
+                    return;
                 }
-
-                if (HostProcess.HasExited)
+                catch (Exception ex) when (attempt < MaxAttempts)
                 {
-                    Logger.LogError("Host process {processName} {pid} exited with code {exitCode} or failed to start.", startInfo.FileName, HostProcess.Id, HostProcess.ExitCode);
-                    throw new Exception("Failed to start host");
+                    Logger.LogWarning("Attempt {attempt} of {maxAttempts} to start the host process failed; retrying in {delaySeconds}s. Exception: {exception}",
+                        attempt, MaxAttempts, retryDelay.TotalSeconds, ex.ToString());
+                    process.Dispose();
+                    await Task.Delay(retryDelay);
                 }
-
-                Logger.LogInformation("Started {fileName}. Process Id : {processId}", startInfo.FileName, HostProcess.Id);
-
-                // Host may not write startup messages, in which case assume it started
-                if (DeploymentParameters.StatusMessagesEnabled)
+                catch (Exception ex)
                 {
-                    // The timeout here is large, because we don't know how long the test could need
-                    // We cover a lot of error cases above, but I want to make sure we eventually give up and don't hang the build
-                    // just in case we missed one -anurse
-                    await started.Task.WaitAsync(TimeSpan.FromMinutes(10));
+                    Logger.LogError("Failed to start the host process after {maxAttempts} attempts. Exception: {exception}", MaxAttempts, ex.ToString());
+                    throw;
                 }
-
-                return hostExitTokenSource.Token;
             }
         }