From e747d58008359009491846899a0fd29d1a94877e Mon Sep 17 00:00:00 2001 From: Aayush Maini Date: Wed, 21 May 2025 09:49:05 -0700 Subject: [PATCH 1/5] Use Go CLI scan instead of go.sum if available --- .../go/Go117ComponentDetector.cs | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs index cc7da78f9..8ed6a7992 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs @@ -97,7 +97,10 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID return; } - var record = new GoGraphTelemetryRecord(); + using var record = new GoGraphTelemetryRecord(); + record.WasGoCliDisabled = false; + record.WasGoFallbackStrategyUsed = false; + var fileExtension = Path.GetExtension(file.Location).ToUpperInvariant(); switch (fileExtension) { @@ -123,7 +126,24 @@ await GoDependencyGraphUtility.GenerateAndPopulateDependencyGraphAsync( case ".SUM": { this.Logger.LogDebug("Found Go.sum: {Location}", file.Location); - await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); + + // check if we can use Go CLI instead + var wasGoCliScanSuccessful = false; + if (!this.IsGoCliManuallyDisabled()) + { + wasGoCliScanSuccessful = await this.goParserFactory.CreateParser(GoParserType.GoCLI, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); + } + + this.Logger.LogDebug("Status of Go CLI scan when considering {GoSumLocation}: {Status}", file.Location, wasGoCliScanSuccessful); + + // If Go CLI scan was not successful/disabled, scan go.sum because this go.sum was recorded due to go.mod + // containing go < 1.17. So go.mod is incomplete. We need to parse go.sum to make list of dependencies complete + if (!wasGoCliScanSuccessful) + { + this.Logger.LogDebug("Go CLI scan when considering {GoSumLocation} was not successful. Falling back to scanning go.sum", file.Location); + await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); + } + break; } From 4f6ad193d9b2e64b651fae5fb76cbef0a188a971 Mon Sep 17 00:00:00 2001 From: Aayush Maini Date: Wed, 21 May 2025 11:24:06 -0700 Subject: [PATCH 2/5] Track project roots with successful CLI scan --- .../go/Go117ComponentDetector.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs index 8ed6a7992..5dd60f5aa 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs @@ -143,6 +143,10 @@ await GoDependencyGraphUtility.GenerateAndPopulateDependencyGraphAsync( this.Logger.LogDebug("Go CLI scan when considering {GoSumLocation} was not successful. Falling back to scanning go.sum", file.Location); await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); } + else + { + this.projectRoots.Add(projectRootDirectory.FullName); + } break; } From 33fcab7765e6db874e33e2a292f409454b116348 Mon Sep 17 00:00:00 2001 From: Aayush Maini Date: Wed, 21 May 2025 12:43:54 -0700 Subject: [PATCH 3/5] Add unit test for new CLI logic --- .../go/Go117ComponentDetector.cs | 6 ++- .../Go117ComponentDetectorTests.cs | 50 +++++++++++++++---- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs index 5dd60f5aa..6b065aa60 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs @@ -98,7 +98,8 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } using var record = new GoGraphTelemetryRecord(); - record.WasGoCliDisabled = false; + var wasGoCliDisabled = this.IsGoCliManuallyDisabled(); + record.WasGoCliDisabled = wasGoCliDisabled; record.WasGoFallbackStrategyUsed = false; var fileExtension = Path.GetExtension(file.Location).ToUpperInvariant(); @@ -129,7 +130,7 @@ await GoDependencyGraphUtility.GenerateAndPopulateDependencyGraphAsync( // check if we can use Go CLI instead var wasGoCliScanSuccessful = false; - if (!this.IsGoCliManuallyDisabled()) + if (!wasGoCliDisabled) { wasGoCliScanSuccessful = await this.goParserFactory.CreateParser(GoParserType.GoCLI, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); } @@ -140,6 +141,7 @@ await GoDependencyGraphUtility.GenerateAndPopulateDependencyGraphAsync( // containing go < 1.17. So go.mod is incomplete. We need to parse go.sum to make list of dependencies complete if (!wasGoCliScanSuccessful) { + record.WasGoFallbackStrategyUsed = true; this.Logger.LogDebug("Go CLI scan when considering {GoSumLocation} was not successful. Falling back to scanning go.sum", file.Location); await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs index be8ab4739..a9b901538 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs @@ -139,29 +139,57 @@ public async Task Go117ModDetector_GoModFileFound_GoModParserIsExecuted() goModParserMock.Verify(parser => parser.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); } - [TestMethod] - public async Task Go117ModDetector_GoSumFileFound_GoSumParserIsExecuted() + /// + /// Verifies that if Go CLI is enabled/available and succeeds, go.sum file is not parsed and vice-versa. + /// + /// Task. + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task Go117Detector_GoSum_GoSumParserExecuted(bool goCliSucceeds) { + var nInvocationsOfSumParser = goCliSucceeds ? 0 : 1; var goSumParserMock = new Mock(); + var goCliParserMock = new Mock(); this.mockParserFactory.Setup(x => x.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(goSumParserMock.Object); + this.mockParserFactory.Setup(x => x.CreateParser(GoParserType.GoCLI, It.IsAny())).Returns(goCliParserMock.Object); - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, null, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) - .ReturnsAsync(true); + // Setup go cli parser to succeed/fail + goCliParserMock.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(goCliSucceeds); - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, default, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = "go version go1.10.6 windows/amd64", - }); + // Setup go sum parser to succeed + goSumParserMock.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.sum", string.Empty) .ExecuteDetectorAsync(); scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + this.mockParserFactory.Verify(clm => clm.CreateParser(GoParserType.GoSum, It.IsAny()), nInvocationsOfSumParser == 0 ? Times.Never : Times.Once); + } + + /// + /// Verifies that if Go CLI is disabled, go.sum is parsed. + /// + /// Task. + [TestMethod] + public async Task Go117Detector_GoSum_GoSumParserExecutedIfCliDisabled() + { + var goSumParserMock = new Mock(); + this.mockParserFactory.Setup(x => x.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(goSumParserMock.Object); + + // Setup environment variable to disable CLI scan + this.envVarService.Setup(s => s.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(true); + + // Setup go sum parser to succed + goSumParserMock.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); - goSumParserMock.Verify(parser => parser.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("go.sum", string.Empty) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + this.mockParserFactory.Verify(clm => clm.CreateParser(GoParserType.GoSum, It.IsAny()), Times.Once); } [TestMethod] From 4458e9a6083ef1c405acaf2a53b6a23e470ae55e Mon Sep 17 00:00:00 2001 From: Aayush Maini Date: Wed, 21 May 2025 15:08:27 -0700 Subject: [PATCH 4/5] CR: Fix method name that filters go.sum --- .../go/Go117ComponentDetector.cs | 2 +- .../go/GoComponentDetector.cs | 2 +- .../go/Utils/GoDetectorUtils.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs index 6b065aa60..c358fabe9 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs @@ -75,7 +75,7 @@ protected override Task> OnPrepareDetectionAsync( return true; } - return GoDetectorUtils.ShouldRemoveGoSumFromDetection(goSumFilePath: processRequest.ComponentStream.Location, goModFile, this.Logger); + return GoDetectorUtils.ShouldIncludeGoSumFromDetection(goSumFilePath: processRequest.ComponentStream.Location, goModFile, this.Logger); } finally { diff --git a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs index bf924ae03..f2b0f361a 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs @@ -94,7 +94,7 @@ protected override Task> OnPrepareDetectionAsync( return true; } - return GoDetectorUtils.ShouldRemoveGoSumFromDetection(goSumFilePath: processRequest.ComponentStream.Location, goModFile, this.Logger); + return GoDetectorUtils.ShouldIncludeGoSumFromDetection(goSumFilePath: processRequest.ComponentStream.Location, goModFile, this.Logger); } finally { diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Utils/GoDetectorUtils.cs b/src/Microsoft.ComponentDetection.Detectors/go/Utils/GoDetectorUtils.cs index d27931a56..ba90234ca 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Utils/GoDetectorUtils.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Utils/GoDetectorUtils.cs @@ -15,7 +15,7 @@ public static class GoDetectorUtils /// Component stream representing the adjacent go.mod file. /// The logger to use for logging messages. /// True if the adjacent go.mod file is present and has a go version >= 1.17. - public static bool ShouldRemoveGoSumFromDetection(string goSumFilePath, ComponentStream adjacentGoModFile, ILogger logger) + public static bool ShouldIncludeGoSumFromDetection(string goSumFilePath, ComponentStream adjacentGoModFile, ILogger logger) { using var reader = new StreamReader(adjacentGoModFile.Stream); var goModFileContents = reader.ReadToEnd(); From 4690c4f7b73537aecd6f64d4e14f6cee63c1a495 Mon Sep 17 00:00:00 2001 From: Aayush Maini Date: Wed, 21 May 2025 15:20:57 -0700 Subject: [PATCH 5/5] CR: Bump Go117 version --- .../go/Go117ComponentDetector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs index c358fabe9..c7e0e7430 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs @@ -48,7 +48,7 @@ public Go117ComponentDetector( public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Go]; - public override int Version => 1; + public override int Version => 2; protected override Task> OnPrepareDetectionAsync( IObservable processRequests,