Go CLI detector enhancement (go list -m all) (#105)

grvillic · web-flow · commit 6e1d270d35a8 · 2022-04-21T16:23:13.000-07:00
* Go CLI enhancement, include only modules in build list
diff --git a/docs/detectors/go.md b/docs/detectors/go.md
@@ -22,10 +22,11 @@ Improved go detection depends on the following to successfully run:
 
 - Go v1.11+.
 
-Go detection is performed by parsing output from executing `go mod graph`.
 Full dependency graph generation is supported if Go v1.11+ is present on the build agent.
 If no Go v1.11+ is present, fallback detection strategy is performed.
 
+Go detection is performed by parsing output from executing [go list -m -json all](1). To generate the graph, the command [go mod graph](2) is executed, this only adds edges between the components that were already registered by `go list`.
+
 As we validate this opt-in behavior, we will eventually graduate it to the default detection strategy.
 
 ## Known limitations
@@ -35,11 +36,14 @@ Dev dependency tagging is not supported.
 Go detection will fallback if no Go v1.11+ is present.
 
 Due to the nature of `go.sum` containing references for all dependencies, including historical, no-longer-needed dependencies; the fallback strategy can result in over detection.
-Executing `go mod tidy` before detection via fallback is encouraged.
+Executing [go mod tidy](https://go.dev/ref/mod#go-mod-tidy) before detection via fallback is encouraged.
+
+Some legacy dependencies may report stale transitive dependencies in their manifests, in this case you can remove them safely from your binaries by using [exclude directive](https://go.dev/doc/modules/gomod-ref#exclude).
 
 ## Environment Variables
 
-If the environment variable `EnableGoCliScan` is set, to any value, the Go detector uses [`go mod graph`][1] to discover Go dependencies.
+If the environment variable `EnableGoCliScan` is set, to any value, the Go detector uses [`go list -m -json all`][1] to discover Go dependencies.
 If the environment variable is not present, we fall back to parsing `go.mod` and `go.sum` ourselves.
 
-[1]: https://go.dev/ref/mod#go-mod-graph
+[1]: https://go.dev/ref/mod#go-list-m
+[2]: https://go.dev/ref/mod#go-mod-graph
diff --git a/docs/feature-overview.md b/docs/feature-overview.md
@@ -6,7 +6,7 @@
 | Conda (Python) (Beta) | <ul><li>environment.yml</li><li>environment.yaml</li></ul> | <ul><li>Conda v4.10.2+</li></ul> | ❌ | ❌ |
 | Linux (Debian, Alpine, Rhel, Centos, Fedora, Ubuntu)| <ul><li>(via [syft](https://github.com/anchore/syft))</li></ul> | - | - | - | - |
 | Gradle | <ul><li>*.lockfile</li></ul> | <ul><li>Gradle 7 or prior using [Single File lock](https://docs.gradle.org/6.8.1/userguide/dependency_locking.html#single_lock_file_per_project)</li></ul> | ❌ | ❌ |
-| Go | <ul><li>*go mod graph*</li></ul>Fallback</br><ul><li>go.mod</li><li>go.sum</li></ul> | <ul><li>Go 1.11+ (will fallback if not present)</li></ul> | ❌ | ✔ (root idenditication only for fallback) |
+| Go | <ul><li>*go list -m -json all*</li><li>*go mod graph* (edge information only)</li></ul>Fallback</br><ul><li>go.mod</li><li>go.sum</li></ul> | <ul><li>Go 1.11+ (will fallback if not present)</li></ul> | ❌ | ✔ (root idenditication only for fallback) |
 | Maven | <ul><li>pom.xml</li><li>*mvn dependency:tree -f {pom.xml}*</li></ul> | <ul><li>Maven</li><li>Maven Dependency Plugin (auto-installed with Maven)</li></ul> | ✔ (test dependency scope) | ✔ |
 | NPM | <ul><li>package.json</li><li>package-lock.json</li><li>npm-shrinkwrap.json</li><li>lerna.json</li></ul> | - | ✔ (dev-dependencies in package.json, dev flag in package-lock.json) | ✔ |
 | Yarn (v1, v2) | <ul><li>package.json</li><li>yarn.lock</li></ul> | - | ✔ (dev-dependencies in package.json) | ✔ |
diff --git a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs
@@ -9,6 +9,7 @@
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.Internal;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
+using Newtonsoft.Json;
 
 namespace Microsoft.ComponentDetection.Detectors.Go
 {
@@ -33,7 +34,7 @@ public class GoComponentDetector : FileComponentDetector
 
         public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = new[] { ComponentType.Go };
 
-        public override int Version => 4;
+        public override int Version => 5;
 
         private HashSet<string> projectRoots = new HashSet<string>();
 
@@ -104,22 +105,31 @@ private async Task<bool> UseGoCliToScan(string location, ISingleFileComponentRec
             var projectRootDirectory = Directory.GetParent(location);
             record.ProjectRoot = projectRootDirectory.FullName;
 
-            var isGoAvailable = await CommandLineInvocationService.CanCommandBeLocated("go", null, workingDirectory: projectRootDirectory, new List<string> { "version" }.ToArray());
+            var isGoAvailable = await CommandLineInvocationService.CanCommandBeLocated("go", null, workingDirectory: projectRootDirectory, new[] { "version" });
             record.IsGoAvailable = isGoAvailable;
 
             if (!isGoAvailable)
             {
                 return false;
             }
 
+            var goDependenciesProcess = await CommandLineInvocationService.ExecuteCommand("go", null, workingDirectory: projectRootDirectory, new[] { "list", "-m", "-json", "all" });
+            if (goDependenciesProcess.ExitCode != 0)
+            {
+                Logger.LogError($"Go CLI could not get dependency build list at location: {location}. Fallback go.sum/go.mod parsing will be used.");
+                return false;
+            }
+
+            RecordBuildDependencies(goDependenciesProcess.StdOut, singleFileComponentRecorder);
+
             var generateGraphProcess = await CommandLineInvocationService.ExecuteCommand("go", null, workingDirectory: projectRootDirectory, new List<string> { "mod", "graph" }.ToArray());
             if (generateGraphProcess.ExitCode == 0)
             {
                 PopulateDependencyGraph(generateGraphProcess.StdOut, singleFileComponentRecorder);
                 record.WasGraphSuccessful = true;
             }
 
-            return record.WasGraphSuccessful;
+            return true;
         }
 
         private void ParseGoModFile(
@@ -200,7 +210,10 @@ private bool TryToCreateGoComponentFromSumLine(string line, out GoComponent goCo
             return false;
         }
 
-        private void PopulateDependencyGraph(string goGraphOutput, ISingleFileComponentRecorder singleFileComponentRecorder)
+        /// <summary>
+        /// This command only adds edges between parent and child components, it does not add nor remove any entries from the existing build list.
+        /// </summary>
+        private void PopulateDependencyGraph(string goGraphOutput, ISingleFileComponentRecorder componentRecorder)
         {
             // Yes, go always returns \n even on Windows
             var graphRelationships = goGraphOutput.Split('\n');
@@ -210,35 +223,29 @@ private void PopulateDependencyGraph(string goGraphOutput, ISingleFileComponentR
                 var components = relationship.Split(' ');
                 if (components.Length != 2)
                 {
-                    Logger.LogWarning("Unexpected output from go mod graph:");
+                    Logger.LogWarning("Unexpected relationship output from go mod graph:");
                     Logger.LogWarning(relationship);
                     continue;
                 }
 
                 GoComponent parentComponent;
                 GoComponent childComponent;
 
-                var parentPart = components[0];
-                var childPart = components[1];
-
-                var isParentParsed = TryCreateGoComponentFromRelationshipPart(parentPart, out parentComponent);
-                var isChildParsed = TryCreateGoComponentFromRelationshipPart(childPart, out childComponent);
+                var isParentParsed = TryCreateGoComponentFromRelationshipPart(components[0], out parentComponent);
+                var isChildParsed = TryCreateGoComponentFromRelationshipPart(components[1], out childComponent);
 
-                // If the parent component doesn't have a version, it means it's one of the 'main' modules
-                // The imports of the main modules are explicitly referenced
-                if (!isParentParsed && isChildParsed)
+                if (!isParentParsed)
                 {
-                    singleFileComponentRecorder.RegisterUsage(new DetectedComponent(childComponent), isExplicitReferencedDependency: true);
+                    // These are explicit dependencies, we already have those recorded
+                    continue;
                 }
-                else if (isParentParsed && isChildParsed)
+
+                if (isChildParsed)
                 {
-                    // Go can have a cyclic dependency between modules, which could cause child components to be listed first than parents. Reproducible with Go 1.16
-                    if (singleFileComponentRecorder.GetComponent(parentComponent.Id) == null)
+                    if (IsModuleInBuildList(componentRecorder, parentComponent) && IsModuleInBuildList(componentRecorder, childComponent))
                     {
-                        singleFileComponentRecorder.RegisterUsage(new DetectedComponent(parentComponent));
+                        componentRecorder.RegisterUsage(new DetectedComponent(childComponent), parentComponentId: parentComponent.Id);
                     }
-
-                    singleFileComponentRecorder.RegisterUsage(new DetectedComponent(childComponent), parentComponentId: parentComponent.Id);
                 }
                 else
                 {
@@ -247,6 +254,46 @@ private void PopulateDependencyGraph(string goGraphOutput, ISingleFileComponentR
             }
         }
 
+        private bool IsModuleInBuildList(ISingleFileComponentRecorder singleFileComponentRecorder, GoComponent component)
+        {
+            return singleFileComponentRecorder.GetComponent(component.Id) != null;
+        }
+
+        private void RecordBuildDependencies(string goListOutput, ISingleFileComponentRecorder singleFileComponentRecorder)
+        {
+            var goBuildModules = new List<GoBuildModule>();
+            var reader = new JsonTextReader(new StringReader(goListOutput));
+            reader.SupportMultipleContent = true;
+
+            while (reader.Read())
+            {
+                var serializer = new JsonSerializer();
+                var buildModule = serializer.Deserialize<GoBuildModule>(reader);
+
+                goBuildModules.Add(buildModule);
+            }
+
+            foreach (var dependency in goBuildModules)
+            {
+                if (dependency.Main)
+                {
+                    // main is the entry point module (superfluous as we already have the file location)
+                    continue;
+                }
+
+                var goComponent = new GoComponent(dependency.Path, dependency.Version);
+
+                if (dependency.Indirect)
+                {
+                    singleFileComponentRecorder.RegisterUsage(new DetectedComponent(goComponent));
+                }
+                else
+                {
+                    singleFileComponentRecorder.RegisterUsage(new DetectedComponent(goComponent), isExplicitReferencedDependency: true);
+                }
+            }
+        }
+
         private bool TryCreateGoComponentFromRelationshipPart(string relationship, out GoComponent goComponent)
         {
             var componentParts = relationship.Split('@');
@@ -264,5 +311,16 @@ private bool IsGoCliManuallyEnabled()
         {
             return EnvVarService.DoesEnvironmentVariableExist("EnableGoCliScan");
         }
+
+        private class GoBuildModule
+        {
+            public string Path { get; set; }
+
+            public bool Main { get; set; }
+
+            public string Version { get; set; }
+
+            public bool Indirect { get; set; }
+        }
     }
 }
diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs
@@ -311,11 +311,47 @@ public async Task TestGoDetector_GoGraphCommandThrows()
         [TestMethod]
         public async Task TestGoDetector_GoGraphHappyPath()
         {
-            var goGraph = "example.com/mainModule some-package@v1.2.3\nsome-package@v1.2.3 other@v1.0.0\nsome-package@v1.2.3 test@v2.0.0\ntest@v2.0.0 a@v1.5.0";
+            var buildDependencies = @"{
+    ""Path"": ""some-package"",
+    ""Version"": ""v1.2.3"",
+    ""Time"": ""2021-12-06T23:04:27Z"",
+    ""Indirect"": true,
+    ""GoMod"": ""C:\\test\\go.mod"",
+    ""GoVersion"": ""1.11""
+}" + "\n" + @"{
+    ""Path"": ""test"",
+    ""Version"": ""v2.0.0"",
+    ""Time"": ""2021-12-06T23:04:27Z"",
+    ""Indirect"": true,
+    ""GoMod"": ""C:\\test\\go.mod"",
+    ""GoVersion"": ""1.11""
+}" + "\n" + @"{
+    ""Path"": ""other"",
+    ""Version"": ""v1.2.0"",
+    ""Time"": ""2021-12-06T23:04:27Z"",
+    ""Indirect"": true,
+    ""GoMod"": ""C:\\test\\go.mod"",
+    ""GoVersion"": ""1.11""
+}" + "\n" + @"{
+    ""Path"": ""a"",
+    ""Version"": ""v1.5.0"",
+    ""Time"": ""2020-05-19T17:02:07Z"",
+    ""Indirect"": true,
+    ""GoMod"": ""C:\\test\\go.mod"",
+    ""GoVersion"": ""1.11""
+}";
+            var goGraph = "example.com/mainModule some-package@v1.2.3\nsome-package@v1.2.3 other@v1.0.0\nsome-package@v1.2.3 other@v1.2.0\ntest@v2.0.0 a@v1.5.0";
 
             commandLineMock.Setup(x => x.CanCommandBeLocated("go", null, It.IsAny<DirectoryInfo>(), It.IsAny<string[]>()))
                 .ReturnsAsync(true);
 
+            commandLineMock.Setup(x => x.ExecuteCommand("go", null, It.IsAny<DirectoryInfo>(), new[] { "list", "-m", "-json", "all" }))
+                .ReturnsAsync(new CommandLineExecutionResult
+                {
+                    ExitCode = 0,
+                    StdOut = buildDependencies,
+                });
+
             commandLineMock.Setup(x => x.ExecuteCommand("go", null, It.IsAny<DirectoryInfo>(), new[] { "mod", "graph" }))
                 .ReturnsAsync(new CommandLineExecutionResult
                 {
@@ -333,17 +369,51 @@ public async Task TestGoDetector_GoGraphHappyPath()
 
             var detectedComponents = componentRecorder.GetDetectedComponents();
             Assert.AreEqual(4, detectedComponents.Count());
+            detectedComponents.Where(component => component.Component.Id == "other v1.0.0 - Go").Should().HaveCount(0);
+            detectedComponents.Where(component => component.Component.Id == "other v1.2.0 - Go").Should().HaveCount(1);
+            detectedComponents.Where(component => component.Component.Id == "some-package v1.2.3 - Go").Should().HaveCount(1);
+            detectedComponents.Where(component => component.Component.Id == "test v2.0.0 - Go").Should().HaveCount(1);
+            detectedComponents.Where(component => component.Component.Id == "a v1.5.0 - Go").Should().HaveCount(1);
         }
 
         [TestMethod]
         public async Task TestGoDetector_GoGraphCyclicDependencies()
         {
+            var buildDependencies = @"{
+    ""Path"": ""github.com/prometheus/common"",
+    ""Version"": ""v0.32.1"",
+    ""Time"": ""2021-12-06T23:04:27Z"",
+    ""Indirect"": true,
+    ""GoMod"": ""C:\\test\\go.mod"",
+    ""GoVersion"": ""1.11""
+}" + "\n" + @"{
+    ""Path"": ""github.com/prometheus/client_golang"",
+    ""Version"": ""v1.11.0"",
+    ""Time"": ""2021-12-06T23:04:27Z"",
+    ""Indirect"": true,
+    ""GoMod"": ""C:\\test\\go.mod"",
+    ""GoVersion"": ""1.11""
+}" + "\n" + @"{
+    ""Path"": ""github.com/prometheus/client_golang"",
+    ""Version"": ""v1.12.1"",
+    ""Time"": ""2021-12-06T23:04:27Z"",
+    ""Indirect"": true,
+    ""GoMod"": ""C:\\test\\go.mod"",
+    ""GoVersion"": ""1.11""
+}";
             var goGraph = @"
 github.com/prometheus/common@v0.32.1 github.com/prometheus/client_golang@v1.11.0
 github.com/prometheus/client_golang@v1.12.1 github.com/prometheus/common@v0.32.1";
             commandLineMock.Setup(x => x.CanCommandBeLocated("go", null, It.IsAny<DirectoryInfo>(), It.IsAny<string[]>()))
                 .ReturnsAsync(true);
 
+            commandLineMock.Setup(x => x.ExecuteCommand("go", null, It.IsAny<DirectoryInfo>(), new[] { "list", "-m", "-json", "all" }))
+                .ReturnsAsync(new CommandLineExecutionResult
+                {
+                    ExitCode = 0,
+                    StdOut = buildDependencies,
+                });
+
             commandLineMock.Setup(x => x.ExecuteCommand("go", null, It.IsAny<DirectoryInfo>(), new[] { "mod", "graph" }))
                 .ReturnsAsync(new CommandLineExecutionResult
                 {
