XRAY-138689 - Add Poetry support for jf ca (jfrog#768)

Author: Phavya-jfrog

artifactory_test.go

@@ -225,6 +225,13 @@ func clearOrRedirectLocalCacheIfNeeded(t *testing.T, projectType project.Project
 			envVarCallbackFunc()
 			createTempDirCallback()
 		}
+	case project.Poetry:
+		poetryTempCachePath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
+		envVarCallbackFunc := clientTests.SetEnvWithCallbackAndAssert(t, "POETRY_CACHE_DIR", poetryTempCachePath)
+		callbackFunc = func() {
+			envVarCallbackFunc()
+			createTempDirCallback()
+		}
 	}
 	return
 }

cli/docs/flags.go

@@ -317,7 +317,7 @@ var flagsMap = map[string]components.Flag{
 	WorkingDirs:         components.NewStringFlag(WorkingDirs, "A comma-separated(,) list of relative working directories, to determine the audit targets locations. If flag isn't provided, a recursive scan is triggered from the root directory of the project."),
 	OutputDir:           components.NewStringFlag(OutputDir, "Target directory to save partial results to.", components.SetHiddenStrFlag()),
 	UploadRepoPath:      components.NewStringFlag(UploadRepoPath, "Artifactory repository name or path to upload the cyclonedx file to. If no name or path are provided, a local generic repository will be created which will automatically be indexed by Xray.", components.WithStrDefaultValue("import-cdx-scan-results")),
-	SkipAutoInstall:     components.NewBoolFlag(SkipAutoInstall, "Set to true to skip auto-install of dependencies in un-built modules. Currently supported only for some package managers.", components.SetHiddenBoolFlag()),
+	SkipAutoInstall:     components.NewBoolFlag(SkipAutoInstall, "Set to true to skip auto-install of dependencies in un-built modules. Currently supported for Yarn, NPM, Pip, and Poetry.", components.SetHiddenBoolFlag()),
 	AllowPartialResults: components.NewBoolFlag(AllowPartialResults, "Set to true to allow partial results and continuance of the scan in case of certain errors.", components.SetHiddenBoolFlag()),
 	ExclusionsAudit: components.NewStringFlag(
 		Exclusions,

commands/curation/curationaudit.go

@@ -121,6 +121,9 @@ var supportedTech = map[techutils.Technology]func(ca *CurationAuditCommand) (boo
 		return ca.checkSupportByVersionOrEnv(techutils.Gem, MinArtiGradleGemSupport)
 	},
 	techutils.Docker: func(ca *CurationAuditCommand) (bool, error) { return true, nil },
+	techutils.Poetry: func(ca *CurationAuditCommand) (bool, error) {
+		return ca.checkSupportByVersionOrEnv(techutils.Poetry, MinArtiPassThroughSupport)
+	},
 }
 
 func (ca *CurationAuditCommand) checkSupportByVersionOrEnv(tech techutils.Technology, minArtiVersion string) (bool, error) {
@@ -1603,7 +1606,7 @@ func getUrlNameAndVersionByTech(tech techutils.Technology, node *xrayUtils.Graph
 		return getGradleNameScopeAndVersion(node.Id, artiUrl, repo, node)
 	case techutils.Gem:
 		return getGemNameScopeAndVersion(node.Id, artiUrl, repo)
-	case techutils.Pip:
+	case techutils.Pip, techutils.Poetry:
 		downloadUrls, name, version = getPythonNameVersion(node.Id, downloadUrlsMap)
 		return
 	case techutils.Go:
@@ -1643,7 +1646,7 @@ func getPythonNameVersion(id string, downloadUrlsMap map[string]string) (downloa
 	if dl, ok := downloadUrlsMap[normalizedId]; ok {
 		downloadUrls = []string{dl}
 	} else {
-		log.Warn(fmt.Sprintf("couldn't find download url for node id %s in report.json", id))
+		log.Warn(fmt.Sprintf("Couldn't find download URL for node ID %s", id))
 	}
 	return
 }

commands/curation/curationaudit_test.go

@@ -1838,6 +1838,141 @@ func TestFetchNodesStatusConcurrentMapWrite(t *testing.T) {
 	})
 	assert.Equal(t, numNodes, count, "expected all %d packages to be recorded as blocked", numNodes)
 }
+// =============================================================================
+// Tests for Poetry support added to curationaudit.go.
+// Covers the new dispatcher case (Pip, Poetry -> getPythonNameVersion) and the
+// supportedTech registration.
+// =============================================================================
+
+func Test_getPythonNameVersion(t *testing.T) {
+	const exampleUrl = "https://test.jfrog.io/artifactory/api/pypi/pypi-remote/packages/aa/bb/flask-2.0.0-py3-none-any.whl"
+
+	tests := []struct {
+		name             string
+		id               string
+		downloadUrlsMap  map[string]string
+		wantDownloadUrls []string
+		wantName         string
+		wantVersion      string
+	}{
+		{
+			name:             "pip id with matching download url",
+			id:               "pypi://flask:2.0.0",
+			downloadUrlsMap:  map[string]string{"pypi://flask:2.0.0": exampleUrl},
+			wantDownloadUrls: []string{exampleUrl},
+			wantName:         "flask",
+			wantVersion:      "2.0.0",
+		},
+		{
+			name:             "poetry id with matching download url (same pypi:// prefix)",
+			id:               "pypi://click:8.0.1",
+			downloadUrlsMap:  map[string]string{"pypi://click:8.0.1": exampleUrl},
+			wantDownloadUrls: []string{exampleUrl},
+			wantName:         "click",
+			wantVersion:      "8.0.1",
+		},
+		{
+			name:             "id present in map but no entry returns name+version only",
+			id:               "pypi://requests:2.31.0",
+			downloadUrlsMap:  map[string]string{"pypi://other:1.0.0": exampleUrl},
+			wantDownloadUrls: nil,
+			wantName:         "requests",
+			wantVersion:      "2.31.0",
+		},
+		{
+			name:             "nil downloadUrlsMap returns name+version only",
+			id:               "pypi://requests:2.31.0",
+			downloadUrlsMap:  nil,
+			wantDownloadUrls: nil,
+			wantName:         "requests",
+			wantVersion:      "2.31.0",
+		},
+		{
+			name:             "malformed id (no version separator) returns empty",
+			id:               "pypi://malformed",
+			downloadUrlsMap:  nil,
+			wantDownloadUrls: nil,
+			wantName:         "",
+			wantVersion:      "",
+		},
+		{
+			name:             "hyphenated name resolved via normalization fallback",
+			id:               "pypi://Flask-Babel:1.0",
+			downloadUrlsMap:  map[string]string{"pypi://flask_babel:1.0": exampleUrl},
+			wantDownloadUrls: []string{exampleUrl},
+			wantName:         "Flask-Babel",
+			wantVersion:      "1.0",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotDownloadUrls, gotName, gotVersion := getPythonNameVersion(tt.id, tt.downloadUrlsMap)
+			assert.Equal(t, tt.wantDownloadUrls, gotDownloadUrls, "downloadUrls mismatch")
+			assert.Equal(t, tt.wantName, gotName, "name mismatch")
+			assert.Equal(t, tt.wantVersion, gotVersion, "version mismatch")
+		})
+	}
+}
+
+// TestGetBlockedPackageDetails_403UnparsableBodyReturnsBlocked verifies that
+// getBlockedPackageDetails returns a blocked PackageStatus (no error) when a 403
+// response body cannot be resolved to a known curation block reason:
+// (1) the body is not valid JSON (e.g. an HTML error page), or
+// (2) the body is valid JSON but the Errors array is empty.
+// In both cases the 403 itself is treated as authoritative — the package is
+// recorded as blocked with an unknown policy rather than being dropped silently.
+func TestGetBlockedPackageDetails_403UnparsableBodyReturnsBlocked(t *testing.T) {
+	tests := []struct {
+		name     string
+		respBody string
+	}{
+		{
+			name:     "non-JSON body (HTML error page)",
+			respBody: "<html><body><h1>403 Forbidden</h1></body></html>",
+		},
+		{
+			name:     "JSON body with empty errors list",
+			respBody: `{"errors":[]}`,
+		},
+	}
+
+	const (
+		pkgName    = "telnyx"
+		pkgVersion = "4.87.1"
+	)
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			serverMock, _, rtManager := coreCommonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusForbidden)
+				_, _ = w.Write([]byte(tt.respBody))
+			})
+			defer serverMock.Close()
+
+			rtAuth := rtManager.GetConfig().GetServiceDetails()
+			httpClientDetails := rtAuth.CreateHttpClientDetails()
+			analyzer := treeAnalyzer{
+				rtManager:            rtManager,
+				rtAuth:               rtAuth,
+				httpClientDetails:    httpClientDetails,
+				extractPoliciesRegex: regexp.MustCompile(extractPoliciesRegexTemplate),
+				url:                  rtAuth.GetUrl(),
+				repo:                 "pypi-remote",
+				tech:                 techutils.Poetry,
+			}
+			packageUrl := fmt.Sprintf("%sapi/pypi/pypi-remote/packages/%s-%s.tar.gz", rtAuth.GetUrl(), pkgName, pkgVersion)
+
+			got, err := analyzer.getBlockedPackageDetails(packageUrl, pkgName, pkgVersion)
+
+			require.NoError(t, err, "unparsable 403 body should not surface as an error")
+			require.NotNil(t, got, "a blocked PackageStatus must be returned when the 403 block reason is unknown")
+			assert.Equal(t, blocked, got.Action)
+			assert.Equal(t, BlockingReasonUnknown, got.BlockingReason)
+			assert.Equal(t, pkgName, got.PackageName)
+			assert.Equal(t, pkgVersion, got.PackageVersion)
+		})
+	}
+}
 
 // TestFetchCvsBlockedStatusTransitive verifies the CVS fallback for a transitive range blocker:
 // the range is resolved to the newest satisfying version and the policy is recovered from the 403 probe.

curation_test.go

@@ -134,8 +134,71 @@ func TestDockerCurationAudit(t *testing.T) {
 	assert.Equal(t, "Image is not Docker Hub official", results[0].Policy[0].Condition)
 }
 
-func curationServer(t *testing.T, expectedRequest map[string]bool, requestToFail map[string]bool) (*httptest.Server, *config.ServerDetails) {
+func TestPoetryCurationAudit(t *testing.T) {
+	integration.InitCurationTest(t)
+	const repo = "pypi-curation"
+	tempDirPath, cleanUp := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "package-managers", "python", "poetry", "poetry-curation-project"))
+	defer cleanUp()
+
+	blockedURL := "/api/pypi/" + repo + "/packages/aa/urllib3-1.26.20-py2.py3-none-any.whl"
+	expectedRequest := map[string]bool{blockedURL: false}
+	requestToFail := map[string]bool{blockedURL: false}
+	serverMock, config := curationServer(t, expectedRequest, requestToFail, map[string]string{
+		"urllib3": `<a href="../../packages/aa/urllib3-1.26.20-py2.py3-none-any.whl">urllib3-1.26.20-py2.py3-none-any.whl</a>`,
+	})
+	defer serverMock.Close()
+
+	cleanUpHome := integration.UseTestHomeWithDefaultXrayConfig(t)
+	defer cleanUpHome()
+
+	config.User = "admin"
+	config.Password = "password"
+	config.ServerId = "test"
+	configCmd := commonCommands.NewConfigCommand(commonCommands.AddOrEdit, config.ServerId).SetDetails(config).SetUseBasicAuthOnly(true).SetInteractive(false)
+	assert.NoError(t, configCmd.Run())
+
+	localXrayCli := securityTests.PlatformCli.WithoutCredentials()
+	workingDirsFlag := fmt.Sprintf("--working-dirs=%s", tempDirPath)
+	output := localXrayCli.RunCliCmdWithOutput(t, "curation-audit", "--format="+string(format.Json), workingDirsFlag)
+
+	expectedResp := getPoetryCurationExpectedResponse(config, repo)
+	var got []curation.PackageStatus
+	bracketIndex := strings.Index(output, "[")
+	require.Less(t, 0, bracketIndex, "Unexpected Curation output with missing '['")
+	err := json.Unmarshal([]byte(output[bracketIndex:]), &got)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedResp, got)
+	for k, v := range expectedRequest {
+		assert.Truef(t, v, "didn't receive expected HEAD request for package url %s", k)
+	}
+}
+
+func getPoetryCurationExpectedResponse(config *config.ServerDetails, repo string) []curation.PackageStatus {
+	return []curation.PackageStatus{
+		{
+			Action:            "blocked",
+			PackageName:       "urllib3",
+			PackageVersion:    "1.26.20",
+			BlockedPackageUrl: config.ArtifactoryUrl + "api/pypi/" + repo + "/packages/aa/urllib3-1.26.20-py2.py3-none-any.whl",
+			BlockingReason:    curation.BlockingReasonPolicy,
+			ParentName:        "urllib3",
+			ParentVersion:     "1.26.20",
+			DepRelation:       "direct",
+			PkgType:           "poetry",
+			Policy: []curation.Policy{
+				{Policy: "pol1", Condition: "cond1", Explanation: "explanation", Recommendation: "recommendation"},
+				{Policy: "pol2", Condition: "cond2", Explanation: "explanation2", Recommendation: "recommendation2"},
+			},
+		},
+	}
+}
+
+func curationServer(t *testing.T, expectedRequest map[string]bool, requestToFail map[string]bool, simpleIndex ...map[string]string) (*httptest.Server, *config.ServerDetails) {
 	mapLockReadWrite := sync.Mutex{}
+	var index map[string]string
+	if len(simpleIndex) > 0 {
+		index = simpleIndex[0]
+	}
 	serverMock, config, _ := commonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) {
 		if r.Method == http.MethodHead {
 			mapLockReadWrite.Lock()
@@ -146,15 +209,26 @@ func curationServer(t *testing.T, expectedRequest map[string]bool, requestToFail
 			if _, exist := requestToFail[r.RequestURI]; exist {
 				w.WriteHeader(http.StatusForbidden)
 			}
+			return
 		}
 		if r.Method == http.MethodGet {
 			if r.RequestURI == "/api/system/version" {
-				_, err := w.Write([]byte(`{"version": "7.0.0"}`))
+				_, err := w.Write([]byte(`{"version": "7.82.0"}`))
 				require.NoError(t, err)
-				w.WriteHeader(http.StatusOK)
 				return
 			}
-
+			if r.RequestURI == "/api/v1/system/version" {
+				_, err := w.Write([]byte(`{"xray_version": "3.92.0"}`))
+				require.NoError(t, err)
+				return
+			}
+			for name, href := range index {
+				if strings.HasSuffix(r.URL.Path, "/simple/"+name+"/") {
+					_, err := w.Write([]byte("<html><body>" + href + "</body></html>"))
+					require.NoError(t, err)
+					return
+				}
+			}
 			if _, exist := requestToFail[r.RequestURI]; exist {
 				w.WriteHeader(http.StatusForbidden)
 				_, err := w.Write([]byte("{\n    \"errors\": [\n        {\n            \"status\": 403,\n            " +

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/magiconair/properties v1.8.10
 	github.com/owenrumney/go-sarif/v3 v3.2.3
 	github.com/package-url/packageurl-go v0.1.3
+	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
 	github.com/urfave/cli v1.22.17
 	github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74
@@ -122,7 +123,6 @@ require (
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
-	github.com/spf13/viper v1.21.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/ulikunitz/xz v0.5.15 // indirect
 	github.com/vbatts/tar-split v0.12.2 // indirect

sca/bom/buildinfo/technologies/common_test.go

@@ -142,6 +142,7 @@ func TestSuspectCurationBlockedError(t *testing.T) {
 	mvnOutput2 := "status code: 500, reason phrase: Server Error (500)"
 	pipOutput := "because of HTTP error 403 Client Error: Forbidden for url"
 	goOutput := "Failed running Go command: 403 Forbidden"
+	poetryOutput := "because of HTTP error 403 Client Error: Forbidden for url"
 
 	tests := []struct {
 		name          string
@@ -190,6 +191,19 @@ func TestSuspectCurationBlockedError(t *testing.T) {
 			output:        goOutput,
 			expect:        fmt.Sprintf(CurationErrorMsgToUserTemplate, techutils.Go),
 		},
+		{
+			name:          "poetry 403 error (pass-through disabled)",
+			isCurationCmd: true,
+			tech:          techutils.Poetry,
+			output:        poetryOutput,
+			expect:        fmt.Sprintf(CurationErrorMsgToUserTemplate, techutils.Poetry),
+		},
+		{
+			name:          "poetry not pass through error",
+			isCurationCmd: true,
+			tech:          techutils.Poetry,
+			output:        "http error 401",
+		},
 		{
 			name:          "not a supported tech",
 			isCurationCmd: true,