diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index c8a46ad2..8a5c2865 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -100,7 +100,7 @@ exampleosdiskname examplevmname exthandlers fde -FFF +fff FFFF FFFFFFFF fffi @@ -211,6 +211,7 @@ pgpkey pgrep pidof pkgversion +pls portaddr portpair postinst @@ -319,6 +320,7 @@ Unregistering unregisters unspec uzers +VCpus vcruntime vendored vflji diff --git a/e2etest/GuestProxyAgentTest/Extensions/ModelExtensions.cs b/e2etest/GuestProxyAgentTest/Extensions/ModelExtensions.cs index 6b73ef2a..3e7208ea 100644 --- a/e2etest/GuestProxyAgentTest/Extensions/ModelExtensions.cs +++ b/e2etest/GuestProxyAgentTest/Extensions/ModelExtensions.cs @@ -12,7 +12,7 @@ namespace GuestProxyAgentTest.Extensions /// public static class ModelExtensions { - public static TestCaseResultDetails ToTestResultDetails(this RunCommandOutputDetails runCommandOutputDetails, Action logger = null!, bool downloadContentFromBlob = true) + public static TestCaseResultDetails ToTestResultDetails(this RunCommandOutputDetails runCommandOutputDetails, TestLogger logger = null!, bool downloadContentFromBlob = true) { return new TestCaseResultDetails { @@ -24,7 +24,7 @@ public static TestCaseResultDetails ToTestResultDetails(this RunCommandOutputDet }.DownloadContentIfFromBlob(logger); } - public static TestCaseResultDetails DownloadContentIfFromBlob(this TestCaseResultDetails testCaseResultDetails, Action logger = null!) + public static TestCaseResultDetails DownloadContentIfFromBlob(this TestCaseResultDetails testCaseResultDetails, TestLogger logger = null!) { if(!testCaseResultDetails.FromBlob) { @@ -96,7 +96,7 @@ public static void WriteJUnitTestResult(this TestCaseResultDetails testCaseResul /// /// /// - public static T SafeDeserializedCustomOutAs(this TestCaseResultDetails testCaseResultDetails) where T: class + public static T SafeDeserializedCustomOutAs(this TestCaseResultDetails testCaseResultDetails, TestLogger logger) where T: class { try { @@ -104,7 +104,7 @@ public static T SafeDeserializedCustomOutAs(this TestCaseResultDetails testCa } catch (Exception ex) { - Console.WriteLine("Deserialized custom out json string failed with exception: " + ex.ToString()); + logger.Log("Deserialized custom out json string failed with exception: " + ex.ToString()); } return null; } diff --git a/e2etest/GuestProxyAgentTest/GuestProxyAgentScenarioTests.cs b/e2etest/GuestProxyAgentTest/GuestProxyAgentScenarioTests.cs index 90df2803..42357211 100644 --- a/e2etest/GuestProxyAgentTest/GuestProxyAgentScenarioTests.cs +++ b/e2etest/GuestProxyAgentTest/GuestProxyAgentScenarioTests.cs @@ -13,6 +13,12 @@ namespace GuestProxyAgentTest /// public class GuestProxyAgentScenarioTests { + private readonly TestLogger logger; + public GuestProxyAgentScenarioTests(TestLogger logger) + { + this.logger = logger; + } + /// /// Main function to start each scenario test /// @@ -74,7 +80,7 @@ public async Task StartAsync(List testScenarioList) var message = string.Format("Test Group: {0}, Test Scenario: {1}: {2} ({3})" , testScenario.testGroupName, testScenario.testScenarioName , testScenarioStatusDetails.Result, testScenarioStatusDetails.ErrorMessage); - Console.WriteLine(message); + logger.Log(message); } } var stopMonitor = new ManualResetEvent(false); @@ -92,14 +98,14 @@ public async Task StartAsync(List testScenarioList) } catch (Exception ex) { - Console.WriteLine($"Test execution exception: {ex.Message}"); + logger.Log($"Test execution exception: {ex.Message}"); } stopMonitor.Set(); foreach (var groupName in groupTestResultBuilderMap.Keys) { - Console.WriteLine("building test result report for test group: " + groupName); + logger.Log("building test result report for test group: " + groupName); groupTestResultBuilderMap[groupName].Build(); } @@ -114,7 +120,7 @@ private void ConsolePrintTestScenariosStatusSummary(IEnumerable x.Status == ScenarioTestStatus.Running).Count()}" + $", failed {testScenarioStatusDetailsList.Where(x => x.Status == ScenarioTestStatus.Completed && x.Result == ScenarioTestResult.Failed).Count()}" + $", success {testScenarioStatusDetailsList.Where(x => x.Status == ScenarioTestStatus.Completed && x.Result == ScenarioTestResult.Succeed).Count()}. "; - Console.WriteLine(message); + logger.Log(message); } private void ConsolePrintTestScenariosDetailsSummary(IEnumerable testScenariosStatusDetailsList) @@ -130,7 +136,7 @@ private void ConsolePrintTestScenariosDetailsSummary(IEnumerable static async Task Main(string[] args) { + TestLogger logger = new TestLogger("GuestProxyAgentTests"); var testConfigFilePath = args[0]; var testResultFolder = args[1]; var guestProxyAgentZipFilePath = args[2]; @@ -29,14 +30,14 @@ static async Task Main(string[] args) { test_arm64 = true; } - + TestCommonUtilities.TestSetup(guestProxyAgentZipFilePath, testConfigFilePath, testResultFolder, proxyAgentVersion); VMHelper.Instance.CleanupOldTestResourcesAndForget(); - await new GuestProxyAgentScenarioTests().StartAsync(TestMapReader.ReadFlattenTestScenarioSettingFromTestMap(test_arm64)); - Console.WriteLine("E2E Test run completed."); + await new GuestProxyAgentScenarioTests(logger).StartAsync(TestMapReader.ReadFlattenTestScenarioSettingFromTestMap(test_arm64)); + logger.Log("E2E Test run completed."); } } } \ No newline at end of file diff --git a/e2etest/GuestProxyAgentTest/Scripts/GuestProxyAgentExtensionValidation.ps1 b/e2etest/GuestProxyAgentTest/Scripts/GuestProxyAgentExtensionValidation.ps1 index e2362587..c8150268 100644 --- a/e2etest/GuestProxyAgentTest/Scripts/GuestProxyAgentExtensionValidation.ps1 +++ b/e2etest/GuestProxyAgentTest/Scripts/GuestProxyAgentExtensionValidation.ps1 @@ -117,13 +117,16 @@ if ($guestProxyAgentExtensionVersion) { $proxyAgentStatus = $json.status.substatus[1].formattedMessage.message $jsonObject = $proxyAgentStatus | ConvertFrom-json $extractedVersion = $jsonObject.version + # Compare the extracted version with the version obtained from the executable if ($extractedVersion -ne $proxyAgentVersion) { - Write-Output "$((Get-Date).ToUniversalTime()) - Error, the proxy agent version [ $extractedVersions ] does not match the version [ $proxyAgentVersion ]" + Write-Output "$((Get-Date).ToUniversalTime()) - Error, the proxy agent version [ $extractedVersion ] does not match the version [ $proxyAgentVersion ]" $guestProxyAgentExtensionVersion = $false } if ($expectedProxyAgentVersion -ne "0") { $cleanExpectedProxyAgentVersion = $expectedProxyAgentVersion.Trim() - if ($extractedVersion -eq $cleanExpectedProxyAgentVersion){ + # Compare only the major, minor, and patch versions, ignoring any additional labels + # as the inputted expectedProxyAgentVersion only contains 3 parts, but the extractedVersion, it is file version, starts to have 4 parts in windows + if (([System.Version]$extractedVersion).ToString(3) -eq ([System.Version]$cleanExpectedProxyAgentVersion).ToString(3)){ Write-Output "$((Get-Date).ToUniversalTime()) - After Update Version check: The proxy agent version matches the expected and extracted version" } else { Write-Output "$((Get-Date).ToUniversalTime()) - After Update Version check: Error, the proxy agent version [ $extractedVersion ] does not match expected version [ $cleanExpectedProxyAgentVersion ]" diff --git a/e2etest/GuestProxyAgentTest/TestCases/GuestProxyAgentExtensionValidationCase.cs b/e2etest/GuestProxyAgentTest/TestCases/GuestProxyAgentExtensionValidationCase.cs index 93231129..fe77ea17 100644 --- a/e2etest/GuestProxyAgentTest/TestCases/GuestProxyAgentExtensionValidationCase.cs +++ b/e2etest/GuestProxyAgentTest/TestCases/GuestProxyAgentExtensionValidationCase.cs @@ -21,10 +21,10 @@ public override async Task StartAsync(TestCaseExecutionContext context) { List<(string, string)> parameterList = new List<(string, string)>(); parameterList.Add(("expectedProxyAgentVersion", expectedProxyAgentVersion)); - context.TestResultDetails = (await RunScriptViaRunCommandV2Async(context, Constants.GUEST_PROXY_AGENT_EXTENSION_VALIDATION_SCRIPT_NAME, parameterList)).ToTestResultDetails(ConsoleLog); + context.TestResultDetails = (await RunScriptViaRunCommandV2Async(context, Constants.GUEST_PROXY_AGENT_EXTENSION_VALIDATION_SCRIPT_NAME, parameterList)).ToTestResultDetails(context.Logger); if (context.TestResultDetails.Succeed && context.TestResultDetails.CustomOut != null) { - var validationDetails = context.TestResultDetails.SafeDeserializedCustomOutAs(); + var validationDetails = context.TestResultDetails.SafeDeserializedCustomOutAs(context.Logger); if (validationDetails != null && validationDetails.guestProxyAgentExtensionServiceExist && validationDetails.guestProxyAgentExtensionProcessExist diff --git a/e2etest/GuestProxyAgentTest/TestCases/GuestProxyAgentLoadedModulesValidationCase.cs b/e2etest/GuestProxyAgentTest/TestCases/GuestProxyAgentLoadedModulesValidationCase.cs index ca59f7a1..69754dad 100644 --- a/e2etest/GuestProxyAgentTest/TestCases/GuestProxyAgentLoadedModulesValidationCase.cs +++ b/e2etest/GuestProxyAgentTest/TestCases/GuestProxyAgentLoadedModulesValidationCase.cs @@ -22,11 +22,11 @@ public override async Task StartAsync(TestCaseExecutionContext context) context.TestResultDetails = (await RunScriptViaRunCommandV2Async(context, "GuestProxyAgentLoadedModulesValidation.ps1", new List<(string, string)> { ("loadedModulesBaseLineSAS", System.Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(baseLineModulesSas))) - })).ToTestResultDetails(ConsoleLog); + })).ToTestResultDetails(context.Logger); if (context.TestResultDetails.Succeed && context.TestResultDetails.CustomOut != null) { - var validationDetails = context.TestResultDetails.SafeDeserializedCustomOutAs(); + var validationDetails = context.TestResultDetails.SafeDeserializedCustomOutAs(context.Logger); // if the validation result is match or no new added modules, then consider the case as succeed. if (validationDetails != null && (validationDetails.IsMatch || validationDetails.NewAddedModules == null || validationDetails.NewAddedModules.Count == 0)) diff --git a/e2etest/GuestProxyAgentTest/TestCases/GuestProxyAgentValidationCase.cs b/e2etest/GuestProxyAgentTest/TestCases/GuestProxyAgentValidationCase.cs index acd55770..074a0bc5 100644 --- a/e2etest/GuestProxyAgentTest/TestCases/GuestProxyAgentValidationCase.cs +++ b/e2etest/GuestProxyAgentTest/TestCases/GuestProxyAgentValidationCase.cs @@ -38,10 +38,10 @@ public override async Task StartAsync(TestCaseExecutionContext context) { List<(string, string)> parameterList = new List<(string, string)>(); parameterList.Add(("expectedSecureChannelState", expectedSecureChannelState)); - context.TestResultDetails = (await RunScriptViaRunCommandV2Async(context, Constants.GUEST_PROXY_AGENT_VALIDATION_SCRIPT_NAME, parameterList)).ToTestResultDetails(ConsoleLog); + context.TestResultDetails = (await RunScriptViaRunCommandV2Async(context, Constants.GUEST_PROXY_AGENT_VALIDATION_SCRIPT_NAME, parameterList)).ToTestResultDetails(context.Logger); if (context.TestResultDetails.Succeed && context.TestResultDetails.CustomOut != null) { - var validationDetails = context.TestResultDetails.SafeDeserializedCustomOutAs(); + var validationDetails = context.TestResultDetails.SafeDeserializedCustomOutAs(context.Logger); // check the validation json output, if the guest proxy agent service was installed and runing and guest proxy agent process exists and log was generate, // then consider it as succeed, otherwise fail the case. if (validationDetails != null diff --git a/e2etest/GuestProxyAgentTest/TestCases/IMDSPingTestCase.cs b/e2etest/GuestProxyAgentTest/TestCases/IMDSPingTestCase.cs index d0d092e3..8e18ec87 100644 --- a/e2etest/GuestProxyAgentTest/TestCases/IMDSPingTestCase.cs +++ b/e2etest/GuestProxyAgentTest/TestCases/IMDSPingTestCase.cs @@ -19,7 +19,7 @@ public override async Task StartAsync(TestCaseExecutionContext context) { List<(string, string)> parameterList = new List<(string, string)>(); parameterList.Add(("imdsSecureChannelEnabled", ImdsSecureChannelEnabled.ToString())); - context.TestResultDetails = (await RunScriptViaRunCommandV2Async(context, Constants.IMDS_PING_TEST_SCRIPT_NAME, parameterList, false)).ToTestResultDetails(ConsoleLog); + context.TestResultDetails = (await RunScriptViaRunCommandV2Async(context, Constants.IMDS_PING_TEST_SCRIPT_NAME, parameterList, false)).ToTestResultDetails(context.Logger); } } } diff --git a/e2etest/GuestProxyAgentTest/TestCases/InstallOrUpdateGuestProxyAgentCase.cs b/e2etest/GuestProxyAgentTest/TestCases/InstallOrUpdateGuestProxyAgentCase.cs index 661751a8..f328998f 100644 --- a/e2etest/GuestProxyAgentTest/TestCases/InstallOrUpdateGuestProxyAgentCase.cs +++ b/e2etest/GuestProxyAgentTest/TestCases/InstallOrUpdateGuestProxyAgentCase.cs @@ -18,7 +18,7 @@ public InstallOrUpdateGuestProxyAgentCase() : base("InstallOrUpdateGuestProxyAge public override async Task StartAsync(TestCaseExecutionContext context) { - var runCommandRes = await RunCommandRunner.ExecuteRunCommandOnVM(context.VirtualMachineResource, new RunCommandSettingBuilder() + var runCommandRes = await RunCommandRunner.ExecuteRunCommandOnVM(context.Logger, context.VirtualMachineResource, new RunCommandSettingBuilder() .TestScenarioSetting(context.ScenarioSetting) .RunCommandName("InstallOrUpdateProxyAgentMsi") .ScriptFullPath(Path.Combine(TestSetting.Instance.scriptsFolder, Constants.INSTALL_GUEST_PROXY_AGENT_SCRIPT_NAME)) diff --git a/e2etest/GuestProxyAgentTest/TestCases/InstallOrUpdateGuestProxyAgentExtensionCase.cs b/e2etest/GuestProxyAgentTest/TestCases/InstallOrUpdateGuestProxyAgentExtensionCase.cs index df9b2aa7..61860671 100644 --- a/e2etest/GuestProxyAgentTest/TestCases/InstallOrUpdateGuestProxyAgentExtensionCase.cs +++ b/e2etest/GuestProxyAgentTest/TestCases/InstallOrUpdateGuestProxyAgentExtensionCase.cs @@ -18,7 +18,7 @@ public InstallOrUpdateGuestProxyAgentExtensionCase() : base("InstallOrUpdateGues public override async Task StartAsync(TestCaseExecutionContext context) { - var runCommandRes = await RunCommandRunner.ExecuteRunCommandOnVM(context.VirtualMachineResource, new RunCommandSettingBuilder() + var runCommandRes = await RunCommandRunner.ExecuteRunCommandOnVM(context.Logger, context.VirtualMachineResource, new RunCommandSettingBuilder() .TestScenarioSetting(context.ScenarioSetting) .RunCommandName("InstallGuestProxyAgentExtension") .ScriptFullPath(Path.Combine(TestSetting.Instance.scriptsFolder, Constants.INSTALL_GUEST_PROXY_AGENT_EXTENSION_SCRIPT_NAME)) diff --git a/e2etest/GuestProxyAgentTest/TestCases/InstallOrUpdateGuestProxyAgentPackageCase.cs b/e2etest/GuestProxyAgentTest/TestCases/InstallOrUpdateGuestProxyAgentPackageCase.cs index 0df5bfc6..adb7da0a 100644 --- a/e2etest/GuestProxyAgentTest/TestCases/InstallOrUpdateGuestProxyAgentPackageCase.cs +++ b/e2etest/GuestProxyAgentTest/TestCases/InstallOrUpdateGuestProxyAgentPackageCase.cs @@ -17,7 +17,7 @@ public InstallOrUpdateGuestProxyAgentPackageCase() : base("InstallOrUpdateGuestP public override async Task StartAsync(TestCaseExecutionContext context) { - var runCommandRes = await RunCommandRunner.ExecuteRunCommandOnVM(context.VirtualMachineResource, new RunCommandSettingBuilder() + var runCommandRes = await RunCommandRunner.ExecuteRunCommandOnVM(context.Logger, context.VirtualMachineResource, new RunCommandSettingBuilder() .TestScenarioSetting(context.ScenarioSetting) .RunCommandName("InstallOrUpdateProxyAgentPackage") .ScriptFullPath(Path.Combine(TestSetting.Instance.scriptsFolder, Constants.INSTALL_LINUX_GUEST_PROXY_AGENT_PACKAGE_SCRIPT_NAME)) diff --git a/e2etest/GuestProxyAgentTest/TestCases/LocalIPBindingCase.cs b/e2etest/GuestProxyAgentTest/TestCases/LocalIPBindingCase.cs index 1f876b7f..4504587b 100644 --- a/e2etest/GuestProxyAgentTest/TestCases/LocalIPBindingCase.cs +++ b/e2etest/GuestProxyAgentTest/TestCases/LocalIPBindingCase.cs @@ -18,7 +18,7 @@ public override async Task StartAsync(TestCaseExecutionContext context) { List<(string, string)> parameterList = new List<(string, string)>(); parameterList.Add(("imdsSecureChannelEnabled", ImdsSecureChannelEnabled.ToString())); - context.TestResultDetails = (await RunScriptViaRunCommandV2Async(context, "PingTestOnBindingLocalIP.ps1", parameterList, false)).ToTestResultDetails(ConsoleLog); + context.TestResultDetails = (await RunScriptViaRunCommandV2Async(context, "PingTestOnBindingLocalIP.ps1", parameterList, false)).ToTestResultDetails(context.Logger); } } } diff --git a/e2etest/GuestProxyAgentTest/TestCases/TCPPortScalabilityCase.cs b/e2etest/GuestProxyAgentTest/TestCases/TCPPortScalabilityCase.cs index 288a22a4..604e4a34 100644 --- a/e2etest/GuestProxyAgentTest/TestCases/TCPPortScalabilityCase.cs +++ b/e2etest/GuestProxyAgentTest/TestCases/TCPPortScalabilityCase.cs @@ -16,7 +16,7 @@ public TCPPortScalabilityCase(bool imdsSecureChannelEnabled) : base("TCPPortScal public override async Task StartAsync(TestCaseExecutionContext context) { - context.TestResultDetails = (await RunScriptViaRunCommandV2Async(context, "ConfigTCPPortScalability.ps1", null!, false)).ToTestResultDetails(ConsoleLog); + context.TestResultDetails = (await RunScriptViaRunCommandV2Async(context, "ConfigTCPPortScalability.ps1", null!, false)).ToTestResultDetails(context.Logger); if(!context.TestResultDetails.Succeed) { return; @@ -26,7 +26,7 @@ public override async Task StartAsync(TestCaseExecutionContext context) await vmr.RestartAsync(Azure.WaitUntil.Completed); List<(string, string)> parameterList = new List<(string, string)>(); parameterList.Add(("imdsSecureChannelEnabled", ImdsSecureChannelEnabled.ToString())); - context.TestResultDetails = (await RunScriptViaRunCommandV2Async(context, "IMDSPingTest.ps1", parameterList, false)).ToTestResultDetails(ConsoleLog); + context.TestResultDetails = (await RunScriptViaRunCommandV2Async(context, "IMDSPingTest.ps1", parameterList, false)).ToTestResultDetails(context.Logger); } } } diff --git a/e2etest/GuestProxyAgentTest/TestCases/TestCaseBase.cs b/e2etest/GuestProxyAgentTest/TestCases/TestCaseBase.cs index 84134de8..d2f4a4f5 100644 --- a/e2etest/GuestProxyAgentTest/TestCases/TestCaseBase.cs +++ b/e2etest/GuestProxyAgentTest/TestCases/TestCaseBase.cs @@ -65,10 +65,10 @@ protected async Task RunScriptViaRunCommandV2Async(Test if (includeCustomJsonOutputSasParam) { var custJsonPath = Path.Combine(Path.GetTempPath(), $"{testScenarioSetting.testGroupName}_{testScenarioSetting.testScenarioName}_{TestCaseName}.json"); - using (File.CreateText(custJsonPath)) ConsoleLog("Created empty test file for customized json output file."); + using (File.CreateText(custJsonPath)) context.Logger.Log("Created empty test file for customized json output file."); custJsonSas = StorageHelper.Instance.Upload2SharedBlob(Constants.SHARED_E2E_TEST_OUTPUT_CONTAINER_NAME, custJsonPath, "customOutputJson.json", testScenarioSetting.TestScenarioStorageFolderPrefix); } - return await RunCommandRunner.ExecuteRunCommandOnVM(context.VirtualMachineResource, new RunCommandSettingBuilder() + return await RunCommandRunner.ExecuteRunCommandOnVM(context.Logger, context.VirtualMachineResource, new RunCommandSettingBuilder() .TestScenarioSetting(testScenarioSetting) .RunCommandName(TestCaseName) .ScriptFullPath(Path.Combine(TestSetting.Instance.scriptsFolder, scriptFileName)) @@ -78,8 +78,6 @@ protected async Task RunScriptViaRunCommandV2Async(Test .AddParameters(parameterList)); } - protected void ConsoleLog(string message) { Console.WriteLine($"[{TestCaseName}]: " + message); } - protected string FormatVMExtensionData(VirtualMachineExtensionData data) { if (data == null) diff --git a/e2etest/GuestProxyAgentTest/TestScenarios/TestScenarioBase.cs b/e2etest/GuestProxyAgentTest/TestScenarios/TestScenarioBase.cs index 4381952c..979ab88f 100644 --- a/e2etest/GuestProxyAgentTest/TestScenarios/TestScenarioBase.cs +++ b/e2etest/GuestProxyAgentTest/TestScenarios/TestScenarioBase.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft Corporation // SPDX-License-Identifier: MIT +using Azure; using Azure.ResourceManager.Compute; using GuestProxyAgentTest.Extensions; using GuestProxyAgentTest.Models; @@ -7,6 +8,7 @@ using GuestProxyAgentTest.TestCases; using GuestProxyAgentTest.Utilities; using System.Diagnostics; +using System.Text.RegularExpressions; namespace GuestProxyAgentTest.TestScenarios { @@ -19,17 +21,25 @@ public abstract class TestScenarioBase private VMBuilder _vmBuilder = null!; private JunitTestResultBuilder _junitTestResultBuilder = null!; private List _testCases = new List(); + protected TestLogger Logger + { + get; private set; + } + protected bool EnableProxyAgentForNewVM { get; set; } public TestScenarioBase() { + Logger = new TestLogger(this.LogPrefix); TestScenarioSetup(); } public TestScenarioBase TestScenarioSetting(TestScenarioSetting testScenarioSetting) { this._testScenarioSetting = testScenarioSetting; + // refresh the Logger with new LogPrefix which is based on the test scenario setting + Logger = new TestLogger(this.LogPrefix); this._vmBuilder = new VMBuilder().LoadTestCaseSetting(testScenarioSetting); return this; } @@ -59,13 +69,20 @@ private string LogPrefix get { // _testScenarioSetting may still null in constructor functions - return "Test Group: " + _testScenarioSetting?.testGroupName + ", Test Scenario: " + _testScenarioSetting?.testScenarioName + ": "; + if (_testScenarioSetting == null) + { + return "Test Scenario: "+this.GetType().Name; + } + else + { + return "Test Group: " + _testScenarioSetting?.testGroupName + ", Test Scenario: " + _testScenarioSetting?.testScenarioName; + } } } protected void ConsoleLog(string msg) { - Console.WriteLine(LogPrefix + msg); + Logger.Log(msg); } protected void PreCheck() @@ -127,7 +144,7 @@ public async Task StartAsync(TestScenarioStatusDetails testScenarioStatusDetails } catch (Exception ex) { - Console.WriteLine("Collect GA Logs error: " + ex.Message); + ConsoleLog("Collect GA Logs error: " + ex.Message); } try @@ -137,11 +154,27 @@ public async Task StartAsync(TestScenarioStatusDetails testScenarioStatusDetails } catch (Exception ex) { - Console.WriteLine("Cleanup azure resources exception: " + ex.Message); + ConsoleLog("Cleanup azure resources exception: " + ex.Message); } } } + /// + /// Try to parse alternative VM sizes from the AllocationFailed error message. + /// Expected pattern: "Alternative VM sizes for the same region: Standard_D2as_v5, Standard_D4as_v5." + /// + private static List ParseAlternativeVmSizes(string errorMessage) + { + var alternativeSizes = new List(); + var match = Regex.Match(errorMessage, @"Alternative VM sizes for the same region:\s*(.+?)\.?\s*$", RegexOptions.Multiline); + if (match.Success) + { + var sizes = match.Groups[1].Value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + alternativeSizes.AddRange(sizes.Where(s => !string.IsNullOrWhiteSpace(s))); + } + return alternativeSizes; + } + private async Task DoStartAsync(TestScenarioStatusDetails testScenarioStatusDetails, CancellationToken cancellationToken) { try @@ -155,13 +188,67 @@ private async Task DoStartAsync(TestScenarioStatusDetails testScenarioStatusDeta try { ConsoleLog(string.Format("Creating {0} VM...", _testScenarioSetting.VMImageDetails.IsArm64 ? "ARM64" : "AMD64")); - vmr = await _vmBuilder.Build(this.EnableProxyAgentForNewVM, cancellationToken); + vmr = await _vmBuilder.Build(this.Logger, this.EnableProxyAgentForNewVM, cancellationToken); ConsoleLog("VM Create succeed"); sw.Stop(); _junitTestResultBuilder.AddSuccessTestResult(_testScenarioSetting.testScenarioName, vmCreateTestName, "VM Create succeed", "", sw.ElapsedMilliseconds); } catch (Exception ex) { + ConsoleLog($"VM first create failed with exception: {ex.GetType().Name} - {ex.Message}"); + + // catch ErrorCode: AllocationFailed and retry with different VMSize if possible, + // as sometimes the allocation failure is caused by the specific VM size is not available in the region, + // but other VM sizes are still available. + if (ex is RequestFailedException rfEx && rfEx.ErrorCode == "AllocationFailed") + { + var alternativeSizes = ParseAlternativeVmSizes(rfEx.Message); + bool retrySucceeded = false; + if (alternativeSizes.Count > 0) + { + ConsoleLog($"AllocationFailed for VM size '{TestSetting.Instance.vmSize}'. Retrying with alternative VM sizes: {string.Join(", ", alternativeSizes)}"); + foreach (var altSize in alternativeSizes) + { + try + { + ConsoleLog($"Retrying VM creation with VM size: {altSize}"); + vmr = await _vmBuilder.Build(this.Logger, this.EnableProxyAgentForNewVM, altSize, false, cancellationToken); + ConsoleLog($"VM Create succeed with alternative VM size: {altSize}"); + retrySucceeded = true; + break; + } + catch (RequestFailedException retryRfEx) when (retryRfEx.ErrorCode == "AllocationFailed") + { + ConsoleLog($"AllocationFailed for alternative VM size '{altSize}', trying next alternative if available."); + continue; + } + catch (Exception retryEx) + { + // NOT AllocationFailed exception, assume retry succeeded but with other exception, break the loop to avoid retrying other sizes + retrySucceeded = true; + ConsoleLog($"VM creation failed with alternative VM size '{altSize}' with exception: {retryEx.Message}. Not retrying other sizes."); + break; + } + } + } + else + { + var availableVMSize = await _vmBuilder.GetAvailableVmSizeAsync(this.Logger); + ConsoleLog($"AllocationFailed but no alternative VM sizes found in the error message, last retry with available VM size `{availableVMSize}`."); + vmr = await _vmBuilder.Build(this.Logger, this.EnableProxyAgentForNewVM, availableVMSize, false, cancellationToken); + ConsoleLog($"VM Create succeed with available VM size {availableVMSize}."); + retrySucceeded = true; + } + + if (!retrySucceeded) + { + // All alternative sizes also failed, rethrow the original exception + sw.Stop(); + _junitTestResultBuilder.AddFailureTestResult(testScenarioStatusDetails.ScenarioName, vmCreateTestName, "", ex.Message + ex.StackTrace ?? "", "", sw.ElapsedMilliseconds); + throw; + } + } + // if the VM Creation operation failed, try check the VM instance view for 5 minutes var startTime = DateTime.UtcNow; while (true) @@ -195,6 +282,7 @@ private async Task DoStartAsync(TestScenarioStatusDetails testScenarioStatusDeta } catch (Exception ex) { + ConsoleLog("ExceptionType: " + ex.GetType().FullName); testScenarioStatusDetails.ErrorMessage = ex.Message; testScenarioStatusDetails.Result = ScenarioTestResult.Failed; ConsoleLog("Exception occurs: " + ex.Message); @@ -217,11 +305,12 @@ private async Task ScenarioTestAsync(VirtualMachineResource vmr, TestScenarioSta break; } - TestCaseExecutionContext context = new TestCaseExecutionContext(vmr, _testScenarioSetting, cancellationToken); + TestCaseExecutionContext context = new TestCaseExecutionContext(this.Logger, vmr, _testScenarioSetting, cancellationToken); Stopwatch sw = Stopwatch.StartNew(); try { + ConsoleLog($"Starting test case: {testCase.TestCaseName}"); testCase.Result = TestCaseResult.Running; await testCase.StartAsync(context); sw.Stop(); @@ -250,7 +339,7 @@ private async Task ScenarioTestAsync(VirtualMachineResource vmr, TestScenarioSta finally { testCase.Result = context.TestResultDetails.Succeed ? TestCaseResult.Succeed : TestCaseResult.Failed; - ConsoleLog($"Scenario case {testCase.TestCaseName} finished with result: {(context.TestResultDetails.Succeed ? "Succeed" : "Failed")} and duration: " + sw.ElapsedMilliseconds + "ms"); + ConsoleLog($"Test case {testCase.TestCaseName} finished with result: {(context.TestResultDetails.Succeed ? "Succeed" : "Failed")} and duration: " + sw.ElapsedMilliseconds + "ms"); SaveResultFile(context.TestResultDetails.CustomOut, $"TestCases/{testCase.TestCaseName}", "customOut.txt", context.TestResultDetails.FromBlob); SaveResultFile(context.TestResultDetails.StdErr, $"TestCases/{testCase.TestCaseName}", "stdErr.txt", context.TestResultDetails.FromBlob); SaveResultFile(context.TestResultDetails.StdOut, $"TestCases/{testCase.TestCaseName}", "stdOut.txt", context.TestResultDetails.FromBlob); @@ -269,7 +358,7 @@ private async Task CollectGALogsOnVMAsync() } var logZipSas = StorageHelper.Instance.Upload2SharedBlob(Constants.SHARED_E2E_TEST_OUTPUT_CONTAINER_NAME, logZipPath, _testScenarioSetting.TestScenarioStorageFolderPrefix); - var collectGALogOutput = await RunCommandRunner.ExecuteRunCommandOnVM(vmr, new RunCommandSettingBuilder() + var collectGALogOutput = await RunCommandRunner.ExecuteRunCommandOnVM(this.Logger, vmr, new RunCommandSettingBuilder() .TestScenarioSetting(_testScenarioSetting) .RunCommandName("CollectInVMGALog") .ScriptFullPath(Path.Combine(TestSetting.Instance.scriptsFolder, Constants.COLLECT_INVM_GA_LOG_SCRIPT_NAME)) @@ -296,7 +385,7 @@ private void SaveResultFile(string fileContentOrSas, string parentFolderName, st if (isFromSas) { - TestCommonUtilities.DownloadFile(fileContentOrSas, filePath, ConsoleLog); + TestCommonUtilities.DownloadFile(fileContentOrSas, filePath, this.Logger); } else { @@ -315,12 +404,21 @@ public class TestCaseExecutionContext private VirtualMachineResource _vmr = null!; private TestScenarioSetting _testScenarioSetting = null!; private CancellationToken _cancellationToken; + private TestLogger _logger = null!; /// /// TestResultDetails for a particular test case /// public TestCaseResultDetails TestResultDetails { get; set; } = new TestCaseResultDetails(); + public TestLogger Logger + { + get + { + return _logger; + } + } + public TestScenarioSetting ScenarioSetting { get @@ -348,8 +446,9 @@ public CancellationToken CancellationToken } } - public TestCaseExecutionContext(VirtualMachineResource vmr, TestScenarioSetting testScenarioSetting, CancellationToken cancellationToken) + public TestCaseExecutionContext(TestLogger logger, VirtualMachineResource vmr, TestScenarioSetting testScenarioSetting, CancellationToken cancellationToken) { + _logger = logger; _vmr = vmr; _testScenarioSetting = testScenarioSetting; _cancellationToken = cancellationToken; diff --git a/e2etest/GuestProxyAgentTest/Utilities/RunCommandRunner.cs b/e2etest/GuestProxyAgentTest/Utilities/RunCommandRunner.cs index 7e44f9fe..8d9ac89b 100644 --- a/e2etest/GuestProxyAgentTest/Utilities/RunCommandRunner.cs +++ b/e2etest/GuestProxyAgentTest/Utilities/RunCommandRunner.cs @@ -21,13 +21,14 @@ public class RunCommandRunner /// cancellation token /// parameter setter for the run command script /// - public static async Task ExecuteRunCommandOnVM(VirtualMachineResource vmr + public static async Task ExecuteRunCommandOnVM(TestLogger logger, + VirtualMachineResource vmr , RunCommandSettingBuilder runCommandSettingBuilder , CancellationToken cancellationToken , Func runCommandParameterSetter = null!) { var vmrcs = vmr.GetVirtualMachineRunCommands(); - Console.WriteLine("Creating runcommand on vm."); + logger.Log("Creating runcommand on vm."); if (null != runCommandParameterSetter) { diff --git a/e2etest/GuestProxyAgentTest/Utilities/TestCommonUtilities.cs b/e2etest/GuestProxyAgentTest/Utilities/TestCommonUtilities.cs index a9a0a8a4..e01c6df9 100644 --- a/e2etest/GuestProxyAgentTest/Utilities/TestCommonUtilities.cs +++ b/e2etest/GuestProxyAgentTest/Utilities/TestCommonUtilities.cs @@ -32,7 +32,7 @@ public static void TestSetup(string guestProxyAgentZipFilePath, string testConfi /// download url /// retry count, default value is 5 /// - public static (bool, string) DownloadContentAsString(string url, Action logger = null!, int retryCnt = 5) + public static (bool, string) DownloadContentAsString(string url, TestLogger logger = null!, int retryCnt = 5) { if (url == null || url.Length == 0) { @@ -58,14 +58,14 @@ public static (bool, string) DownloadContentAsString(string url, Action catch (Exception ex) { errMessage = string.Format("Download content failed, attempted: {0} times, exception: {1}", cnt, ex.ToString()); - logger?.Invoke(errMessage); + logger?.Log(errMessage); } Thread.Sleep(1000); } return (false, errMessage); } - public static bool DownloadFile(string url, string filePath, Action logger = null!, int retryCnt = 5) + public static bool DownloadFile(string url, string filePath, TestLogger logger = null!, int retryCnt = 5) { if (null == url || url.Length == 0) { @@ -93,7 +93,7 @@ public static bool DownloadFile(string url, string filePath, Action logg catch (Exception ex) { var errMessage = string.Format("Download file failed, attempted: {0} times, exception: {1}", cnt, ex.ToString()); - logger?.Invoke(errMessage); + logger.Log(errMessage); } } return false; diff --git a/e2etest/GuestProxyAgentTest/Utilities/TestLogger.cs b/e2etest/GuestProxyAgentTest/Utilities/TestLogger.cs new file mode 100644 index 00000000..23e712bb --- /dev/null +++ b/e2etest/GuestProxyAgentTest/Utilities/TestLogger.cs @@ -0,0 +1,17 @@ +namespace GuestProxyAgentTest.Utilities +{ + public class TestLogger + { + public TestLogger(string prefix) + { + this.Prefix = prefix; + } + + public string Prefix { get; } + + public void Log(string message) + { + Console.WriteLine($"[{this.Prefix}] - {DateTime.Now:yyyy-MM-ddTHH:mm:ss.fff} - {message}"); + } + } +} diff --git a/e2etest/GuestProxyAgentTest/Utilities/VMBuilder.cs b/e2etest/GuestProxyAgentTest/Utilities/VMBuilder.cs index f96d2518..eace53cd 100644 --- a/e2etest/GuestProxyAgentTest/Utilities/VMBuilder.cs +++ b/e2etest/GuestProxyAgentTest/Utilities/VMBuilder.cs @@ -8,6 +8,7 @@ using Azure.ResourceManager.Network; using Azure.ResourceManager.Resources; using GuestProxyAgentTest.Settings; +using System.Linq; namespace GuestProxyAgentTest.Utilities { @@ -55,30 +56,132 @@ public VMBuilder LoadTestCaseSetting(TestScenarioSetting testScenarioSetting) /// Build Build and return the VirtualMachine based on the setting /// /// - public async Task Build(bool enableProxyAgent, CancellationToken cancellationToken) + // Preferred fallback VM sizes in order of preference, grouped by architecture. + private static readonly string[] FALLBACK_VM_SIZES_X64 = new string[] + { + "Standard_D2as_v5" + }; + + private static readonly string[] FALLBACK_VM_SIZES_ARM64 = new string[] + { + "Standard_B2pls_v5", + }; + + public async Task Build(TestLogger logger, bool enableProxyAgent, CancellationToken cancellationToken) + { + return await Build(logger, enableProxyAgent, null, true, cancellationToken); + } + + /// + /// Build and return the VirtualMachine based on the setting, with an optional VM size override + /// + /// + /// If not null, overrides the default VM size from TestSetting + /// true to delete RG if already exists + /// + /// + public async Task Build(TestLogger logger, bool enableProxyAgent, string vmSizeOverride, bool deleteExistingResourceGroup, CancellationToken cancellationToken) { PreCheck(); ArmClient client = new(new GuestProxyAgentE2ETokenCredential(), defaultSubscriptionId: TestSetting.Instance.subscriptionId); var sub = await client.GetDefaultSubscriptionAsync(); + + string vmSizeToUse = vmSizeOverride ?? TestSetting.Instance.vmSize; + if (vmSizeOverride == null) + { + // Resolve an available VM size before creating resources + var resolvedVmSize = await GetAvailableVmSizeAsync(logger); + logger.Log($"Resolved VM size: {resolvedVmSize}"); + vmSizeToUse = resolvedVmSize; + } + var rgs = sub.GetResourceGroups(); - if (await rgs.ExistsAsync(rgName)) + if (deleteExistingResourceGroup && await rgs.ExistsAsync(rgName)) { - Console.WriteLine($"Resource group: {rgName} already exists, cleaning it up."); + logger.Log($"Resource group: {rgName} already exists, cleaning it up."); await (await rgs.GetAsync(rgName)).Value.DeleteAsync(WaitUntil.Completed); } - Console.WriteLine("Creating resource group: " + rgName); + logger.Log("Creating resource group: " + rgName); var rgData = new ResourceGroupData(TestSetting.Instance.location); rgData.Tags.Add(Constants.COULD_CLEANUP_TAG_NAME, "true"); var rgr = rgs.CreateOrUpdate(WaitUntil.Completed, rgName, rgData).Value; VirtualMachineCollection vmCollection = rgr.GetVirtualMachines(); - Console.WriteLine("Creating virtual machine..."); - var vmr = (await vmCollection.CreateOrUpdateAsync(WaitUntil.Completed, this.vmName, await DoCreateVMData(rgr, enableProxyAgent), cancellationToken: cancellationToken)).Value; - Console.WriteLine("Virtual machine created, with id: " + vmr.Id); + logger.Log("Creating virtual machine..."); + var vmr = (await vmCollection.CreateOrUpdateAsync(WaitUntil.Completed, this.vmName, + await DoCreateVMData(logger, rgr, enableProxyAgent, vmSizeToUse), cancellationToken: cancellationToken)).Value; + logger.Log("Virtual machine created, with id: " + vmr.Id); return vmr; } + /// + /// Check if the configured VM size is available in the target location. + /// If not, try fallback sizes. Returns the first available VM size. + /// + internal async Task GetAvailableVmSizeAsync(TestLogger logger) + { + ArmClient client = new(new GuestProxyAgentE2ETokenCredential(), defaultSubscriptionId: TestSetting.Instance.subscriptionId); + var sub = await client.GetDefaultSubscriptionAsync(); + + // Collect available VM SKUs with their vCPU count and architecture + var availableSkus = new List<(string Name, int VCpus, string Architecture)>(); + await foreach (var sku in sub.GetComputeResourceSkusAsync(filter: $"location eq '{TestSetting.Instance.location}'")) + { + if (sku.ResourceType != null + && string.Equals(sku.ResourceType, "virtualMachines", StringComparison.OrdinalIgnoreCase) + && !sku.Restrictions.Any(r => r.ReasonCode == ComputeResourceSkuRestrictionsReasonCode.NotAvailableForSubscription)) + { + var vCpuCap = sku.Capabilities.FirstOrDefault(c => string.Equals(c.Name, "vCPUs", StringComparison.OrdinalIgnoreCase)); + int vCpus = vCpuCap != null && int.TryParse(vCpuCap.Value, out var v) ? v : 0; + + var archCap = sku.Capabilities.FirstOrDefault(c => string.Equals(c.Name, "CpuArchitectureType", StringComparison.OrdinalIgnoreCase)); + string arch = archCap?.Value ?? "x64"; + + availableSkus.Add((sku.Name, vCpus, arch)); + } + } + + var availableNames = new HashSet(availableSkus.Select(s => s.Name), StringComparer.OrdinalIgnoreCase); + bool isArm64 = this.testScenarioSetting.VMImageDetails.IsArm64; + string requiredArch = isArm64 ? "Arm64" : "x64"; + + var configuredSize = TestSetting.Instance.vmSize; + if (availableNames.Contains(configuredSize)) + { + logger.Log($"Configured VM size '{configuredSize}' is available."); + return configuredSize; + } + + logger.Log($"WARNING: Configured VM size '{configuredSize}' is not available in '{TestSetting.Instance.location}'. Searching for a fallback..."); + + // First try the explicit fallback list + var fallbacks = isArm64 ? FALLBACK_VM_SIZES_ARM64 : FALLBACK_VM_SIZES_X64; + foreach (var fallback in fallbacks) + { + if (availableNames.Contains(fallback)) + { + logger.Log($"Using fallback VM size: '{fallback}'"); + return fallback; + } + } + + // If no explicit fallback is available, pick the first available 2 vCPU size matching the required architecture + var autoSelected = availableSkus + .Where(s => s.VCpus == 2 && string.Equals(s.Architecture, requiredArch, StringComparison.OrdinalIgnoreCase)) + .Select(s => s.Name) + .FirstOrDefault(); + if (autoSelected != null) + { + logger.Log($"Using auto-selected 2 vCPU {requiredArch} VM size: '{autoSelected}'"); + return autoSelected; + } + + // If none of the preferred fallbacks are available, return the configured size and let Azure report the error. + logger.Log($"WARNING: No fallback VM size is available either. Proceeding with configured size '{configuredSize}'."); + return configuredSize; + } + public async Task GetVirtualMachineResource() { PreCheck(); @@ -87,13 +190,13 @@ public async Task GetVirtualMachineResource() return sub.GetResourceGroups().Get(this.rgName).Value.GetVirtualMachine(this.vmName); } - private async Task DoCreateVMData(ResourceGroupResource rgr, bool enableProxyAgent) + private async Task DoCreateVMData(TestLogger logger, ResourceGroupResource rgr, bool enableProxyAgent, string vmSize) { var vmData = new VirtualMachineData(TestSetting.Instance.location) { HardwareProfile = new VirtualMachineHardwareProfile() { - VmSize = new VirtualMachineSizeType(TestSetting.Instance.vmSize), + VmSize = new VirtualMachineSizeType(vmSize), }, StorageProfile = new VirtualMachineStorageProfile() { @@ -120,7 +223,7 @@ private async Task DoCreateVMData(ResourceGroupResource rgr, AdminUsername = this.adminUsername, AdminPassword = this.adminPassword, }, - NetworkProfile = await DoCreateVMNetWorkProfile(rgr), + NetworkProfile = await DoCreateVMNetWorkProfile(logger, rgr), }; if (enableProxyAgent) @@ -189,9 +292,9 @@ private async Task DoCreateVMData(ResourceGroupResource rgr, return vmData; } - private async Task DoCreateVMNetWorkProfile(ResourceGroupResource rgr) + private async Task DoCreateVMNetWorkProfile(TestLogger logger, ResourceGroupResource rgr) { - Console.WriteLine("Creating network profile"); + logger.Log("Creating network profile"); var vns = rgr.GetVirtualNetworks(); await vns.CreateOrUpdateAsync(WaitUntil.Completed, this.vNetName, new VirtualNetworkData { @@ -214,7 +317,7 @@ private async Task DoCreateVMNetWorkProfile(Resour var pips = rgr.GetPublicIPAddresses(); - Console.WriteLine("Creating public ip address."); + logger.Log("Creating public ip address."); await pips.CreateOrUpdateAsync(WaitUntil.Completed, this.pubIpName, new PublicIPAddressData { Location = TestSetting.Instance.location @@ -222,7 +325,7 @@ private async Task DoCreateVMNetWorkProfile(Resour var nifs = rgr.GetNetworkInterfaces(); - Console.WriteLine("Creating network interface."); + logger.Log("Creating network interface."); await nifs.CreateOrUpdateAsync(WaitUntil.Completed, this.netInfName, new NetworkInterfaceData() { IPConfigurations =