Skip to content

User/aamaini/go sort requests #1432

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AMaini503 merged 6 commits into from
Jun 11, 2025
Merged

User/aamaini/go sort requests #1432

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1a6afdb
Process go process requests in order of depth
Jun 11, 2025
bf6b17b
Add UTs to verify traversal order in go detector
Jun 11, 2025
9dcd9e8
Fix paths for cross-plat UTs
Jun 11, 2025
1a827d2
Fix another path in UT
Jun 11, 2025
f6104bd
CR: Change level of a noisy trace
Jun 11, 2025
fc46c8f
Bump detector version
Jun 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 25 additions & 6 deletions src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,14 @@ public GoComponentDetector(

public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = [ComponentType.Go];

public override int Version => 9;
public override int Version => 10;

protected override Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(
protected async override Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(
IObservable<ProcessRequest> processRequests,
IDictionary<string, string> detectorArgs,
CancellationToken cancellationToken = default)
{
var goModProcessRequests = processRequests.Where(processRequest =>
var filteredGoProcessRequests = await processRequests.Where(processRequest =>
{
if (Path.GetFileName(processRequest.ComponentStream.Location) != "go.sum")
{
Expand All @@ -81,9 +81,15 @@ protected override Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(
{
goModFile?.Stream.Dispose();
}
});
}).ToList(); // Materialize the filtered items for sorting

return Task.FromResult(goModProcessRequests);
// Sort by depth: shallow files (fewer directory segments) come first
var sortedGoProcessRequests = filteredGoProcessRequests
.OrderBy(pr => pr.ComponentStream.Location.Count(c => c == Path.DirectorySeparatorChar))
.ThenBy(pr => pr.ComponentStream.Location)
.ToList();

return sortedGoProcessRequests.ToObservable();
}

protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default)
Expand All @@ -108,7 +114,20 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
case ".MOD":
{
this.Logger.LogDebug("Found Go.mod: {Location}", file.Location);
await this.goParserFactory.CreateParser(GoParserType.GoMod, this.Logger).ParseAsync(singleFileComponentRecorder, file, record);
var wasModParsedSuccessfully = await this.goParserFactory.CreateParser(GoParserType.GoMod, this.Logger).ParseAsync(singleFileComponentRecorder, file, record);

// Check if go.mod was parsed successfully and Go version is >= 1.17 in go.mod
if (wasModParsedSuccessfully &&
!string.IsNullOrEmpty(record.GoModVersion) &&
System.Version.TryParse(record.GoModVersion, out var goVersion) &&
goVersion >= new Version(1, 17))
{
this.projectRoots.Add(projectRootDirectory.FullName);
}
else
{
this.Logger.LogDebug("Not adding {Root} to processed roots: {ParseSuccess} {GoModVersion}", projectRootDirectory.FullName, wasModParsedSuccessfully, record.GoModVersion);
}

if (await this.ShouldRunGoGraphAsync())
{
Expand Down
224 changes: 224 additions & 0 deletions test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -983,4 +983,228 @@ public async Task GoModDetector_VerifyLocalReferencesIgnored()
.Should()
.BeEquivalentTo(expectedComponentIds);
}

/// <summary>
/// Verify that nested directories are skipped once root is processed.
/// Assume root GoModVersion is >= 1.17.
/// </summary>
/// <returns>Task.</returns>
[TestMethod]
public async Task GoDetector_GoMod_VerifyNestedRootsUnderGTE117_AreSkipped()
{
var processedFiles = new List<string>();
this.SetupMockGoModParser();
this.mockGoModParser
.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()))
.ReturnsAsync(true)
.Callback<ISingleFileComponentRecorder, IComponentStream, GoGraphTelemetryRecord>((_, file, record) =>
{
processedFiles.Add(file.Location);
record.GoModVersion = "1.18";
});

var root = Path.Combine("C:", "root");
var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "a", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "b", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "d", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "a", "go.mod"))
.ExecuteDetectorAsync();

scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
processedFiles.Should().ContainSingle();
processedFiles.Should().OnlyContain(p => p == Path.Combine(root, "go.mod"));
}

/// <summary>
/// Verify that nested roots under go mod less than 1.17 are not skipped.
/// </summary>
/// <returns>Task.</returns>
[TestMethod]
public async Task GoDetector_GoMod_VerifyNestedRootsUnderLT117AreNotSkipped()
{
var root = Path.Combine("C:", "root");
var processedFiles = new List<string>();
this.SetupMockGoModParser();
this.mockGoModParser
.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()))
.ReturnsAsync(true)
.Callback<ISingleFileComponentRecorder, IComponentStream, GoGraphTelemetryRecord>((_, file, record) =>
{
processedFiles.Add(file.Location);
var rootMod = Path.Combine(root, "go.mod");
var aMod = Path.Combine(root, "a", "go.mod");
var bMod = Path.Combine(root, "b", "go.mod");
record.GoModVersion = file.Location switch
{
var loc when loc == rootMod => "1.16",
var loc when loc == aMod => "1.16",
var loc when loc == bMod => "1.17",
_ => null,
};
});

var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "a", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "b", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "d", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "a", "go.mod"))
.ExecuteDetectorAsync();

scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
processedFiles.Should().HaveCount(5);
processedFiles.Should().ContainInOrder(
Path.Combine(root, "go.mod"),
Path.Combine(root, "a", "go.mod"),
Path.Combine(root, "b", "go.mod"),
Path.Combine(root, "a", "a", "go.mod"),
Path.Combine(root, "a", "b", "go.mod"));
}

/// <summary>
/// Verify that nested roots are not skipped if parent go.mod parsing fails.
/// </summary>
/// <returns>Task.</returns>
[TestMethod]
public async Task GoDetector_GoMod_VerifyNestedRootsAreNotSkippedIfParentParseFails()
{
var processedFiles = new List<string>();
var root = Path.Combine("C:", "root");
this.SetupMockGoModParser();

this.mockGoModParser
.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()))
.ReturnsAsync((ISingleFileComponentRecorder recorder, IComponentStream file, GoGraphTelemetryRecord record) =>
{
processedFiles.Add(file.Location);
var aMod = Path.Combine(root, "a", "go.mod");
var bMod = Path.Combine(root, "b", "go.mod");
record.GoModVersion = file.Location switch
{
var loc when loc == bMod => "1.18",
_ => "1.16",
};

// Simulate parse failure only for C:\root\a\go.mod
if (file.Location == aMod)
{
return false;
}

return true;
});

var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "a", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "b", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "d", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "a", "go.mod"))
.ExecuteDetectorAsync();

scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
processedFiles.Should().HaveCount(5);
processedFiles.Should().ContainInOrder(
Path.Combine(root, "go.mod"),
Path.Combine(root, "a", "go.mod"),
Path.Combine(root, "b", "go.mod"),
Path.Combine(root, "a", "a", "go.mod"),
Path.Combine(root, "a", "b", "go.mod"));
}

/// <summary>
/// Verify that nested directories are skipped once root is processed.
/// Assume root GoModVersion is >= 1.17.
/// </summary>
/// <returns>Task.</returns>
[TestMethod]
public async Task GoDetector_GoSum_VerifyNestedRootsUnderGoSum_AreSkipped()
{
var processedFiles = new List<string>();
var root = Path.Combine("C:", "root");
this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false);
this.SetupMockGoCLIParser();
this.mockGoCliParser
.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()))
.ReturnsAsync(true)
.Callback<ISingleFileComponentRecorder, IComponentStream, GoGraphTelemetryRecord>((_, file, record) =>
{
processedFiles.Add(file.Location);
});

var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "a", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "b", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "go.mod"))
.WithFile("go.sum", string.Empty, fileLocation: Path.Combine(root, "go.sum"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "d", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "a", "go.mod"))
.ExecuteDetectorAsync();

scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
processedFiles.Should().ContainSingle();
processedFiles.Should().OnlyContain(p => p == Path.Combine(root, "go.sum"));
}

[TestMethod]
public async Task GoDetector_GoSum_VerifyNestedRootsAreNotSkippedIfParentParseFails()
{
var processedFiles = new List<string>();
var root = Path.Combine("C:", "root");
this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false);
this.SetupMockGoModParser();
this.SetupMockGoCLIParser();
this.SetupMockGoSumParser();

this.mockGoModParser
.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()))
.ReturnsAsync((ISingleFileComponentRecorder recorder, IComponentStream file, GoGraphTelemetryRecord record) =>
{
processedFiles.Add(file.Location);
var bMod = Path.Combine(root, "b", "go.mod");
record.GoModVersion = file.Location switch
{
var loc when loc == bMod => "1.18",
_ => "1.16",
};

return true;
});

this.mockGoCliParser
.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()))
.ReturnsAsync((ISingleFileComponentRecorder recorder, IComponentStream file, GoGraphTelemetryRecord record) =>
{
processedFiles.Add(file.Location);
return file.Location != Path.Combine(root, "a", "go.sum");
});

var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile("go.sum", string.Empty, fileLocation: Path.Combine(root, "a", "go.sum"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "a", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "a", "b", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "d", "go.mod"))
.WithFile("go.mod", string.Empty, fileLocation: Path.Combine(root, "b", "a", "go.mod"))
.ExecuteDetectorAsync();

scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
processedFiles.Should().HaveCount(5);
processedFiles.Should().ContainInOrder(
Path.Combine(root, "go.mod"),
Path.Combine(root, "a", "go.sum"),
Path.Combine(root, "b", "go.mod"),
Path.Combine(root, "a", "a", "go.mod"),
Path.Combine(root, "a", "b", "go.mod"));
}
}
Loading