From 81e64469bd5ea22e73b7ac9b20bc6ba62d933bca Mon Sep 17 00:00:00 2001 From: Aayush Maini Date: Wed, 4 Jun 2025 23:48:19 -0700 Subject: [PATCH 1/5] Ignore local go package references --- .../go/Parsers/GoModParser.cs | 99 ++++++++++++++++++- 1 file changed, 96 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs index 582a83a14..499acec51 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs @@ -1,5 +1,7 @@ namespace Microsoft.ComponentDetection.Detectors.Go; +using System; +using System.Collections.Generic; using System.IO; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -15,11 +17,50 @@ public class GoModParser : IGoParser public GoModParser(ILogger logger) => this.logger = logger; + /// + /// Checks whether the input path is a potential local file system path + /// 1. '.' checks whether the path is relative to current directory. + /// 2. '..' checks whether the path is relative to some ancestor directory. + /// 3. IsRootedPath checks whether it is an absolute path. + /// + /// Candidate path. + /// true if potential local file system path. + private static bool IsLocalPath(string path) + { + return path.StartsWith('.') || path.StartsWith("..") || Path.IsPathRooted(path); + } + + /// + /// Tries to extract source token from replace directive. + /// + /// String containing a directive after replace token. + /// HashSet where the token is placed if replace directive substitutes a local path. + private static void TryExtractReplaceDirective(string directiveLine, HashSet replaceDirectives) + { + var parts = directiveLine.Split("=>", StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 2) + { + var source = parts[0].Trim().Split(' ')[0]; + var target = parts[1].Trim(); + + if (IsLocalPath(target)) + { + replaceDirectives.Add(source); + } + } + } + public async Task ParseAsync( ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file, GoGraphTelemetryRecord record) { + // Collect replace directives that point to a local path + var replaceDirectives = await this.GetAllReplacePathDirectivesAsync(file); + + // Rewind stream after reading replace directives + file.Stream.Seek(0, SeekOrigin.Begin); + using var reader = new StreamReader(file.Stream); // There can be multiple require( ) sections in go 1.17+. loop over all of them. @@ -38,7 +79,7 @@ public async Task ParseAsync( // are listed in the require () section if (line.StartsWith(StartString)) { - this.TryRegisterDependencyFromModLine(line[StartString.Length..], singleFileComponentRecorder); + this.TryRegisterDependencyFromModLine(file, line[StartString.Length..], singleFileComponentRecorder, replaceDirectives); } line = await reader.ReadLineAsync(); @@ -47,14 +88,14 @@ public async Task ParseAsync( // Stopping at the first ) restrict the detection to only the require section. while ((line = await reader.ReadLineAsync()) != null && !line.EndsWith(')')) { - this.TryRegisterDependencyFromModLine(line, singleFileComponentRecorder); + this.TryRegisterDependencyFromModLine(file, line, singleFileComponentRecorder, replaceDirectives); } } return true; } - private void TryRegisterDependencyFromModLine(string line, ISingleFileComponentRecorder singleFileComponentRecorder) + private void TryRegisterDependencyFromModLine(IComponentStream file, string line, ISingleFileComponentRecorder singleFileComponentRecorder, HashSet replaceDirectives) { if (line.Trim().StartsWith("//")) { @@ -64,6 +105,15 @@ private void TryRegisterDependencyFromModLine(string line, ISingleFileComponentR if (this.TryToCreateGoComponentFromModLine(line, out var goComponent)) { + if (replaceDirectives.Contains(goComponent.Name)) + { + // Skip registering this dependency since it's replaced by a local path + // we will be reading this dependency somewhere else + this.logger.LogInformation("Skipping {GoComponentId} from {Location} because it's a local reference.", goComponent.Id, file.Location); + return; + } + + this.logger.LogError("Registering {GoComponent} from {Location}", goComponent.Name, file.Location); singleFileComponentRecorder.RegisterUsage(new DetectedComponent(goComponent)); } else @@ -90,4 +140,47 @@ private bool TryToCreateGoComponentFromModLine(string line, out GoComponent goCo return true; } + + private async Task> GetAllReplacePathDirectivesAsync(IComponentStream file) + { + var replacedDirectives = new HashSet(StringComparer.OrdinalIgnoreCase); + const string singleReplaceDirectiveBegin = "replace "; + const string multiReplaceDirectiveBegin = "replace ("; + using (var reader = new StreamReader(file.Stream, leaveOpen: true)) + { + while (!reader.EndOfStream) + { + var line = await reader.ReadLineAsync(); + if (line == null) + { + continue; + } + + line = line.Trim(); + + // Multiline block: replace ( + if (line.StartsWith(multiReplaceDirectiveBegin)) + { + while ((line = await reader.ReadLineAsync()) != null) + { + line = line.Trim(); + if (line == ")") + { + break; + } + + TryExtractReplaceDirective(line, replacedDirectives); + } + } + else if (line.StartsWith(singleReplaceDirectiveBegin)) + { + // single line block: replace + var directiveContent = line[singleReplaceDirectiveBegin.Length..].Trim(); + TryExtractReplaceDirective(directiveContent, replacedDirectives); + } + } + } + + return replacedDirectives; + } } From d5f4915f886e34e8624346086c2a6f018716f544 Mon Sep 17 00:00:00 2001 From: Aayush Maini Date: Thu, 5 Jun 2025 00:16:10 -0700 Subject: [PATCH 2/5] Add UT to verify that local references are ignored --- .../Go117ComponentDetectorTests.cs | 42 +++++++++++++++++++ ....ComponentDetection.Detectors.Tests.csproj | 7 ++++ .../TestFiles/go_WithLocalReferences.mod | 18 ++++++++ 3 files changed, 67 insertions(+) create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/TestFiles/go_WithLocalReferences.mod diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs index a9b901538..df36b08cb 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs @@ -8,6 +8,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using FluentAssertions; using Microsoft.ComponentDetection.Common.Telemetry.Records; using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Detectors.Go; using Microsoft.ComponentDetection.TestsUtilities; using Microsoft.Extensions.Logging; @@ -212,4 +213,45 @@ public async Task Go117ModDetector_ExecutingGoVersionFails_DetectorDoesNotFail() goModParserMock.Verify(parser => parser.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); } + + [TestMethod] + public async Task Go117ModDetector_VerifyLocalReferencesIgnored() + { + var goModFilePath = ".\\TestFiles\\go_WithLocalReferences.mod"; // Replace with your actual file path + var fileStream = new FileStream(goModFilePath, FileMode.Open, FileAccess.Read); + + var goModParser = new GoModParser(this.mockLogger.Object); + var mockSingleFileComponentRecorder = new Mock(); + + var capturedComponents = new List(); + var expectedComponentIds = new List() + { + "github.com/grafana/grafana-app-sdk v0.23.1 - Go", + "k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f - Go", + }; + + mockSingleFileComponentRecorder + .Setup(m => m.RegisterUsage( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback((comp, _, _, _, _, _) => + { + capturedComponents.Add(comp); + }); + + var mockComponentStream = new Mock(); + mockComponentStream.Setup(mcs => mcs.Stream).Returns(fileStream); + mockComponentStream.Setup(mcs => mcs.Location).Returns("Location"); + + var result = await goModParser.ParseAsync(mockSingleFileComponentRecorder.Object, mockComponentStream.Object, new GoGraphTelemetryRecord()); + result.Should().BeTrue(); + capturedComponents + .Select(c => c.Component.Id) + .Should() + .BeEquivalentTo(expectedComponentIds); + } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj b/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj index b219414bc..2b9b4a22a 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj @@ -56,6 +56,13 @@ Always + + Always + + + + + diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/TestFiles/go_WithLocalReferences.mod b/test/Microsoft.ComponentDetection.Detectors.Tests/TestFiles/go_WithLocalReferences.mod new file mode 100644 index 000000000..1c52e09d2 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/TestFiles/go_WithLocalReferences.mod @@ -0,0 +1,18 @@ +module github.com/Go117Tests + +go 1.23.5 + +replace github.com/grafana/grafana => ../../.. + +require ( + github.com/grafana/grafana v0.0.0-00010101000000-000000000000 + github.com/grafana/grafana-app-sdk v0.23.1 + k8s.io/apimachinery v0.32.0 + k8s.io/apiserver v0.32.0 + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f +) + +replace ( + k8s.io/apimachinery => ../ + k8s.io/apiserver => ./a/b/c +) From 306d41866a266ccf8e1f9d91eca3e16d7c08199f Mon Sep 17 00:00:00 2001 From: Aayush Maini Date: Thu, 5 Jun 2025 00:21:28 -0700 Subject: [PATCH 3/5] Bump Go117 detector version --- .../go/Go117ComponentDetector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs index c7e0e7430..957b9748a 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs @@ -48,7 +48,7 @@ public Go117ComponentDetector( public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Go]; - public override int Version => 2; + public override int Version => 3; protected override Task> OnPrepareDetectionAsync( IObservable processRequests, From a8be033772634df4b2d1cac5f1684364531ea27b Mon Sep 17 00:00:00 2001 From: Aayush Maini Date: Thu, 5 Jun 2025 00:45:36 -0700 Subject: [PATCH 4/5] Add other replace directives in UT go.mod file --- .../TestFiles/go_WithLocalReferences.mod | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/TestFiles/go_WithLocalReferences.mod b/test/Microsoft.ComponentDetection.Detectors.Tests/TestFiles/go_WithLocalReferences.mod index 1c52e09d2..3b2af55dc 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/TestFiles/go_WithLocalReferences.mod +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/TestFiles/go_WithLocalReferences.mod @@ -3,6 +3,8 @@ module github.com/Go117Tests go 1.23.5 replace github.com/grafana/grafana => ../../.. +replace k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f => k8s.io/kube-openapi v1.1.1 + require ( github.com/grafana/grafana v0.0.0-00010101000000-000000000000 @@ -16,3 +18,5 @@ replace ( k8s.io/apimachinery => ../ k8s.io/apiserver => ./a/b/c ) + +replace github.com/grafana/grafana-app-sdk => github.com/grafana/grafana-app-sdk v0.22.1 From af53a5320211aba432e3c252aa6a593482567988 Mon Sep 17 00:00:00 2001 From: Aayush Maini Date: Thu, 5 Jun 2025 00:51:45 -0700 Subject: [PATCH 5/5] CR: Use unix path for test file --- .../Go117ComponentDetectorTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs index df36b08cb..57323b780 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs @@ -217,7 +217,7 @@ public async Task Go117ModDetector_ExecutingGoVersionFails_DetectorDoesNotFail() [TestMethod] public async Task Go117ModDetector_VerifyLocalReferencesIgnored() { - var goModFilePath = ".\\TestFiles\\go_WithLocalReferences.mod"; // Replace with your actual file path + var goModFilePath = "./TestFiles/go_WithLocalReferences.mod"; // Replace with your actual file path var fileStream = new FileStream(goModFilePath, FileMode.Open, FileAccess.Read); var goModParser = new GoModParser(this.mockLogger.Object);