From 4da13afeb09f47fed656afc0bc3526d96a5309c0 Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Sun, 22 Jan 2023 16:15:40 +0100 Subject: [PATCH 1/5] DPKG support: added parser and tests --- pkg/lockfile/dpkg-status.go | 105 ++++++++++++++++ pkg/lockfile/dpkg-status_test.go | 128 ++++++++++++++++++++ pkg/lockfile/fixtures/dpkg/empty_status | 0 pkg/lockfile/fixtures/dpkg/malformed_status | 78 ++++++++++++ pkg/lockfile/fixtures/dpkg/multiple_status | 74 +++++++++++ pkg/lockfile/fixtures/dpkg/not_status | 26 ++++ pkg/lockfile/fixtures/dpkg/shuffled_status | 22 ++++ pkg/lockfile/fixtures/dpkg/single_status | 33 +++++ 8 files changed, 466 insertions(+) create mode 100644 pkg/lockfile/dpkg-status.go create mode 100644 pkg/lockfile/dpkg-status_test.go create mode 100644 pkg/lockfile/fixtures/dpkg/empty_status create mode 100644 pkg/lockfile/fixtures/dpkg/malformed_status create mode 100644 pkg/lockfile/fixtures/dpkg/multiple_status create mode 100644 pkg/lockfile/fixtures/dpkg/not_status create mode 100644 pkg/lockfile/fixtures/dpkg/shuffled_status create mode 100644 pkg/lockfile/fixtures/dpkg/single_status diff --git a/pkg/lockfile/dpkg-status.go b/pkg/lockfile/dpkg-status.go new file mode 100644 index 00000000000..b863878c459 --- /dev/null +++ b/pkg/lockfile/dpkg-status.go @@ -0,0 +1,105 @@ +package lockfile + +import ( + "bufio" + "fmt" + "os" + "strings" +) + +const DebianEcosystem Ecosystem = "Debian" + +func groupDpkgPackageLines(scanner *bufio.Scanner) [][]string { + var groups [][]string + var group []string + + for scanner.Scan() { + line := scanner.Text() + + if line == "" { + if len(group) > 0 { + groups = append(groups, group) + } + group = make([]string, 0) + + continue + } + group = append(group, line) + } + + if len(group) > 0 { + groups = append(groups, group) + } + + return groups +} + +func parseDpkgPackageGroup(group []string, pathToLockfile string) PackageDetails { + var pkg = PackageDetails{ + Ecosystem: DebianEcosystem, + CompareAs: DebianEcosystem, + } + + // TODO File SPECS: + for _, line := range group { + switch { + case strings.HasPrefix(line, "Package:"): + pkg.Name = strings.TrimPrefix(line, "Package:") + pkg.Name = strings.TrimSpace(pkg.Name) + case strings.HasPrefix(line, "Version:"): + pkg.Version = strings.TrimPrefix(line, "Version:") + pkg.Version = strings.TrimSpace(pkg.Version) + } + } + + if pkg.Version == "" { + pkgPrintName := pkg.Name + if pkgPrintName == "" { + pkgPrintName = "" + } + + _, _ = fmt.Fprintf( + os.Stderr, + "warning: malformed DPKG status file. Found no version number in record. Package %s. File: %s\n", + pkgPrintName, + pathToLockfile, + ) + } + + return pkg +} + +func ParseDpkgStatus(pathToLockfile string) ([]PackageDetails, error) { + file, err := os.Open(pathToLockfile) + if err != nil { + return []PackageDetails{}, fmt.Errorf("could not open %s: %w", pathToLockfile, err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + packageGroups := groupDpkgPackageLines(scanner) + + packages := make([]PackageDetails, 0, len(packageGroups)) + + for _, group := range packageGroups { + pkg := parseDpkgPackageGroup(group, pathToLockfile) + + if pkg.Name == "" { + _, _ = fmt.Fprintf( + os.Stderr, + "warning: malformed DPKG status file. Found no package name in record. File: %s\n", + pathToLockfile, + ) + + continue + } + + packages = append(packages, pkg) + } + + if err := scanner.Err(); err != nil { + return packages, fmt.Errorf("error while scanning %s: %w", pathToLockfile, err) + } + + return packages, nil +} diff --git a/pkg/lockfile/dpkg-status_test.go b/pkg/lockfile/dpkg-status_test.go new file mode 100644 index 00000000000..f36f9a131ee --- /dev/null +++ b/pkg/lockfile/dpkg-status_test.go @@ -0,0 +1,128 @@ +package lockfile_test + +import ( + "testing" + + "github.com/google/osv-scanner/pkg/lockfile" +) + +func TestDpkgStatus_FileDoesNotExist(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/does-not-exist") + + expectErrContaining(t, err, "could not open") + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestDpkgStatus_Empty(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/empty_status") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestDpkgStatus_NotAStatus(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/not_status") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestDpkgStatus_Malformed(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/malformed_status") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "bash", + Version: "", + Ecosystem: lockfile.DebianEcosystem, + CompareAs: lockfile.DebianEcosystem, + }, + }) +} + +func TestDpkgStatus_Single(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/single_status") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "sudo", + Version: "1.8.27-1+deb10u1", + Ecosystem: lockfile.DebianEcosystem, + CompareAs: lockfile.DebianEcosystem, + }, + }) +} + +func TestDpkgStatus_Shuffled(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/shuffled_status") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "libc6", + Version: "2.31-13+deb11u5", + Ecosystem: lockfile.DebianEcosystem, + CompareAs: lockfile.DebianEcosystem, + }, + }) +} + +func TestDpkgStatus_Multiple(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/multiple_status") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "bash", + Version: "5.1-2+deb11u1", + Ecosystem: lockfile.DebianEcosystem, + CompareAs: lockfile.DebianEcosystem, + }, + { + Name: "bsdutils", + Version: "1:2.36.1-8+deb11u1", + Ecosystem: lockfile.DebianEcosystem, + CompareAs: lockfile.DebianEcosystem, + }, + { + Name: "libc6", + Version: "2.31-13+deb11u5", + Ecosystem: lockfile.DebianEcosystem, + CompareAs: lockfile.DebianEcosystem, + }, + }) +} diff --git a/pkg/lockfile/fixtures/dpkg/empty_status b/pkg/lockfile/fixtures/dpkg/empty_status new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pkg/lockfile/fixtures/dpkg/malformed_status b/pkg/lockfile/fixtures/dpkg/malformed_status new file mode 100644 index 00000000000..ca342b79847 --- /dev/null +++ b/pkg/lockfile/fixtures/dpkg/malformed_status @@ -0,0 +1,78 @@ +Malformed DPKG + +no version: + +Package: bash +Essential: yes +Status: install ok installed +Priority: required +Section: shells +Installed-Size: 6469 +Maintainer: redacted +Architecture: amd64 +Multi-Arch: foreign +Replaces: bash-completion (<< 20060301-0), bash-doc (<= 2.05-1) +Depends: base-files (>= 2.1.12), debianutils (>= 2.15) +Pre-Depends: libc6 (>= 2.25), libtinfo6 (>= 6) +Recommends: bash-completion (>= 20060301-0) +Suggests: bash-doc +Conflicts: bash-completion (<< 20060301-0) +Conffiles: + /etc/bash.bashrc 89269e1298235f1b12b4c16e4065ad0d + /etc/skel/.bash_logout 22bfb8c1dd94b5f3813a2b25da67463f + /etc/skel/.bashrc ee35a240758f374832e809ae0ea4883a + /etc/skel/.profile f4e81ade7d6f9fb342541152d08e7a97 +Description: GNU Bourne Again SHell + Bash is an sh-compatible command language interpreter that executes + commands read from the standard input or from a file. Bash also + incorporates useful features from the Korn and C shells (ksh and csh). + . + Bash is ultimately intended to be a conformant implementation of the + IEEE POSIX Shell and Tools specification (IEEE Working Group 1003.2). + . + The Programmable Completion Code, by Ian Macdonald, is now found in + the bash-completion package. +Homepage: http://tiswww.case.edu/php/chet/bash/bashtop.html + + + +No package: + + + +Essential: yes +Status: install ok installed +Priority: required +Section: utils +Installed-Size: 394 +Maintainer: redacted +Architecture: amd64 +Multi-Arch: foreign +Source: util-linux (2.36.1-8+deb11u1) +Version: 1:2.36.1-8+deb11u1 +Pre-Depends: libc6 (>= 2.17), libsystemd0 +Recommends: bsdextrautils +Description: basic utilities from 4.4BSD-Lite + This package contains the bare minimum of BSD utilities needed for a Debian + system: logger, renice, script, scriptlive, scriptreplay and wall. The + remaining standard BSD utilities are provided by bsdextrautils. +Homepage: http://www.kernel.org/pub/linux/utils/util-linux/ + + +Nothing... + + +Status: install ok installed +Priority: optional +Section: libs +Installed-Size: 12837 +Maintainer: redacted +Architecture: amd64 +Multi-Arch: same +Source: glibc +Replaces: libc6-amd64 +Description: GNU C Library: Shared libraries + Contains the standard libraries that are used by nearly all programs on + the system. This package includes shared versions of the standard C library + and the standard math library, as well as many others. +Homepage: https://www.gnu.org/software/libc/libc.html diff --git a/pkg/lockfile/fixtures/dpkg/multiple_status b/pkg/lockfile/fixtures/dpkg/multiple_status new file mode 100644 index 00000000000..4c30f8e6982 --- /dev/null +++ b/pkg/lockfile/fixtures/dpkg/multiple_status @@ -0,0 +1,74 @@ +Package: bash +Essential: yes +Status: install ok installed +Priority: required +Section: shells +Installed-Size: 6469 +Maintainer: redacted +Architecture: amd64 +Multi-Arch: foreign +Version: 5.1-2+deb11u1 +Replaces: bash-completion (<< 20060301-0), bash-doc (<= 2.05-1) +Depends: base-files (>= 2.1.12), debianutils (>= 2.15) +Pre-Depends: libc6 (>= 2.25), libtinfo6 (>= 6) +Recommends: bash-completion (>= 20060301-0) +Suggests: bash-doc +Conflicts: bash-completion (<< 20060301-0) +Conffiles: + /etc/bash.bashrc 89269e1298235f1b12b4c16e4065ad0d + /etc/skel/.bash_logout 22bfb8c1dd94b5f3813a2b25da67463f + /etc/skel/.bashrc ee35a240758f374832e809ae0ea4883a + /etc/skel/.profile f4e81ade7d6f9fb342541152d08e7a97 +Description: GNU Bourne Again SHell + Bash is an sh-compatible command language interpreter that executes + commands read from the standard input or from a file. Bash also + incorporates useful features from the Korn and C shells (ksh and csh). + . + Bash is ultimately intended to be a conformant implementation of the + IEEE POSIX Shell and Tools specification (IEEE Working Group 1003.2). + . + The Programmable Completion Code, by Ian Macdonald, is now found in + the bash-completion package. +Homepage: http://tiswww.case.edu/php/chet/bash/bashtop.html + +Package: bsdutils +Essential: yes +Status: install ok installed +Priority: required +Section: utils +Installed-Size: 394 +Maintainer: redacted +Architecture: amd64 +Multi-Arch: foreign +Source: util-linux (2.36.1-8+deb11u1) +Version: 1:2.36.1-8+deb11u1 +Pre-Depends: libc6 (>= 2.17), libsystemd0 +Recommends: bsdextrautils +Description: basic utilities from 4.4BSD-Lite + This package contains the bare minimum of BSD utilities needed for a Debian + system: logger, renice, script, scriptlive, scriptreplay and wall. The + remaining standard BSD utilities are provided by bsdextrautils. +Homepage: http://www.kernel.org/pub/linux/utils/util-linux/ + +Package: libc6 +Status: install ok installed +Priority: optional +Section: libs +Installed-Size: 12837 +Maintainer: redacted +Architecture: amd64 +Multi-Arch: same +Source: glibc +Version: 2.31-13+deb11u5 +Replaces: libc6-amd64 +Depends: libgcc-s1, libcrypt1 +Recommends: libidn2-0 (>= 2.0.5~), libnss-nis, libnss-nisplus +Suggests: glibc-doc, debconf | debconf-2.0, libc-l10n, locales +Breaks: busybox (<< 1.30.1-6), hurd (<< 1:0.9.git20170910-1), ioquake3 (<< 1.36+u20200211.f2c61c1~dfsg-2~), iraf-fitsutil (<< 2018.07.06-4), libgegl-0.4-0 (<< 0.4.18), libtirpc1 (<< 0.2.3), locales (<< 2.31), locales-all (<< 2.31), macs (<< 2.2.7.1-3~), nocache (<< 1.1-1~), nscd (<< 2.31), openarena (<< 0.8.8+dfsg-4~), openssh-server (<< 1:8.1p1-5), r-cran-later (<< 0.7.5+dfsg-2), wcc (<< 0.0.2+dfsg-3) +Conffiles: + /etc/ld.so.conf.d/x86_64-linux-gnu.conf d4e7a7b88a71b5ffd9e2644e71a0cfab +Description: GNU C Library: Shared libraries + Contains the standard libraries that are used by nearly all programs on + the system. This package includes shared versions of the standard C library + and the standard math library, as well as many others. +Homepage: https://www.gnu.org/software/libc/libc.html \ No newline at end of file diff --git a/pkg/lockfile/fixtures/dpkg/not_status b/pkg/lockfile/fixtures/dpkg/not_status new file mode 100644 index 00000000000..a2455426cb2 --- /dev/null +++ b/pkg/lockfile/fixtures/dpkg/not_status @@ -0,0 +1,26 @@ +Not a dpkg status file! + + +astroid==2.5.1 + # via pylint +beautifulsoup4==4.9.3 + # via metadata-parser +boto3==1.17.19 + # via -r requirements.in +botocore==1.20.19 + # via + # boto3 + # s3transfer +certifi==2020.12.5 + # via requests +chardet==4.0.0 + # via requests +circus==0.17.1 + # via -r requirements.in +click==7.1.2 + # via pip-tools +django-debug-toolbar==3.2.1 + # via -r requirements.in +django-filter==2.4.0 + # via -r requirements.in +django-nose==1.4.7 \ No newline at end of file diff --git a/pkg/lockfile/fixtures/dpkg/shuffled_status b/pkg/lockfile/fixtures/dpkg/shuffled_status new file mode 100644 index 00000000000..d875ac5f46a --- /dev/null +++ b/pkg/lockfile/fixtures/dpkg/shuffled_status @@ -0,0 +1,22 @@ +Breaks: busybox (<< 1.30.1-6), hurd (<< 1:0.9.git20170910-1), ioquake3 (<< 1.36+u20200211.f2c61c1~dfsg-2~), iraf-fitsutil (<< 2018.07.06-4), libgegl-0.4-0 (<< 0.4.18), libtirpc1 (<< 0.2.3), locales (<< 2.31), locales-all (<< 2.31), macs (<< 2.2.7.1-3~), nocache (<< 1.1-1~), nscd (<< 2.31), openarena (<< 0.8.8+dfsg-4~), openssh-server (<< 1:8.1p1-5), r-cran-later (<< 0.7.5+dfsg-2), wcc (<< 0.0.2+dfsg-3) +Status: install ok installed +Architecture: amd64 +Suggests: glibc-doc, debconf | debconf-2.0, libc-l10n, locales + Contains the standard libraries that are used by nearly all programs on +Depends: libgcc-s1, libcrypt1 +Version: 2.31-13+deb11u5 +Description: GNU C Library: Shared libraries +Replaces: libc6-amd64 +Installed-Size: 12837 +Section: libs +Multi-Arch: same +Maintainer: redacted +Homepage: https://www.gnu.org/software/libc/libc.html + /etc/ld.so.conf.d/x86_64-linux-gnu.conf d4e7a7b88a71b5ffd9e2644e71a0cfab +Conffiles: +Source: glibc + and the standard math library, as well as many others. +Recommends: libidn2-0 (>= 2.0.5~), libnss-nis, libnss-nisplus + the system. This package includes shared versions of the standard C library +Priority: optional +Package: libc6 \ No newline at end of file diff --git a/pkg/lockfile/fixtures/dpkg/single_status b/pkg/lockfile/fixtures/dpkg/single_status new file mode 100644 index 00000000000..12d1d03d896 --- /dev/null +++ b/pkg/lockfile/fixtures/dpkg/single_status @@ -0,0 +1,33 @@ +Package: sudo +Status: install ok installed +Priority: important +Section: admin +Installed-Size: 4337 +Maintainer: redacted +Architecture: amd64 +Version: 1.8.27-1+deb10u1 +Replaces: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~) +Provides: apt-transport-https (= 2.2.4) +Depends: adduser, gpgv | gpgv2 | gpgv1, libapt-pkg6.0 (>= 2.2.4), debian-archive-keyring, libc6 (>= 2.15), libgcc-s1 (>= 3.0), libgnutls30 (>= 3.7.0), libseccomp2 (>= 2.4.2), libstdc++6 (>= 9), libsystemd0 +Recommends: ca-certificates +Suggests: apt-doc, aptitude | synaptic | wajig, dpkg-dev (>= 1.17.2), gnupg | gnupg2 | gnupg1, powermgmt-base +Breaks: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~), aptitude (<< 0.8.10) +Conffiles: + /etc/apt/apt.conf.d/01autoremove ab6540f7278a05a4b7f9e58afcaa5f46 + /etc/cron.daily/apt-compat 49e9b2cfa17849700d4db735d04244f3 + /etc/kernel/postinst.d/apt-auto-removal 6486b24d4c496e7d6a443178869a019b + /etc/logrotate.d/apt 179f2ed4f85cbaca12fa3d69c2a4a1c3 +Description: commandline package manager + This package provides commandline tools for searching and + managing as well as querying information about packages + as a low-level access to all features of the libapt-pkg library. + . + These include: + * apt-get for retrieval of packages and information about them + from authenticated sources and for installation, upgrade and + removal of packages together with their dependencies + * apt-cache for querying available information about installed + as well as installable packages + * apt-cdrom to use removable media as a source for packages + * apt-config as an interface to the configuration settings + * apt-key as an interface to manage authentication keys \ No newline at end of file From f880d8a2c51bc104b2fb7861669c8c456bc935a3 Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Fri, 27 Jan 2023 09:11:47 +0100 Subject: [PATCH 2/5] DPKG: corrected linter error (goconst) --- pkg/lockfile/dpkg-status.go | 2 +- pkg/lockfile/parse.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/lockfile/dpkg-status.go b/pkg/lockfile/dpkg-status.go index b863878c459..0937ad53753 100644 --- a/pkg/lockfile/dpkg-status.go +++ b/pkg/lockfile/dpkg-status.go @@ -55,7 +55,7 @@ func parseDpkgPackageGroup(group []string, pathToLockfile string) PackageDetails if pkg.Version == "" { pkgPrintName := pkg.Name if pkgPrintName == "" { - pkgPrintName = "" + pkgPrintName = unknownPkgName } _, _ = fmt.Fprintf( diff --git a/pkg/lockfile/parse.go b/pkg/lockfile/parse.go index 8ca8d83105a..dcd6c8a32ec 100644 --- a/pkg/lockfile/parse.go +++ b/pkg/lockfile/parse.go @@ -8,6 +8,10 @@ import ( "strings" ) +// To avoid linter error (goconst) on some parsers. +// String used when pkgName is not present +const unknownPkgName = "" + func FindParser(pathToLockfile string, parseAs string) (PackageDetailsParser, string) { if parseAs == "" { parseAs = filepath.Base(pathToLockfile) From 0fe45247a3540b25826530abc00962c2f0add403 Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Fri, 3 Feb 2023 09:07:44 +0100 Subject: [PATCH 3/5] chore(dpkg-parser): Update comment on pkgName not determined while scanning Co-authored-by: Gareth Jones --- pkg/lockfile/parse.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/lockfile/parse.go b/pkg/lockfile/parse.go index dcd6c8a32ec..06cd480f5ed 100644 --- a/pkg/lockfile/parse.go +++ b/pkg/lockfile/parse.go @@ -8,8 +8,8 @@ import ( "strings" ) -// To avoid linter error (goconst) on some parsers. -// String used when pkgName is not present +// Represents when a package name could not be determined while parsing. +// Currently, parsers are expected to omit such packages from their results. const unknownPkgName = "" func FindParser(pathToLockfile string, parseAs string) (PackageDetailsParser, string) { From e4dd1c0dd9d86ba46c93a084b5445699d1f00f24 Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Sun, 12 Feb 2023 17:04:42 +0100 Subject: [PATCH 4/5] fix(dpkg-parser): Now Source field is used when present as required by osv.dev --- pkg/lockfile/dpkg-status.go | 70 +++++++++++++++++-- pkg/lockfile/dpkg-status_test.go | 33 +++++++-- pkg/lockfile/fixtures/dpkg/malformed_status | 26 ++++++- pkg/lockfile/fixtures/dpkg/multiple_status | 25 +++++++ .../fixtures/dpkg/source_ver_override_status | 20 ++++++ pkg/lockfile/parse.go | 1 + 6 files changed, 164 insertions(+), 11 deletions(-) create mode 100644 pkg/lockfile/fixtures/dpkg/source_ver_override_status diff --git a/pkg/lockfile/dpkg-status.go b/pkg/lockfile/dpkg-status.go index 0937ad53753..7fcaf597fbe 100644 --- a/pkg/lockfile/dpkg-status.go +++ b/pkg/lockfile/dpkg-status.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "os" + "regexp" "strings" ) @@ -34,21 +35,72 @@ func groupDpkgPackageLines(scanner *bufio.Scanner) [][]string { return groups } +// Return name and version if "Source" field contains them +func parseSourceField(source string) (string, string) { + // Pattern: name (version) + re := regexp.MustCompile(`^(.*)\((.*)\)`) + matches := re.FindStringSubmatch(source) + if len(matches) == 3 { + return strings.TrimSpace(matches[1]), strings.TrimSpace(matches[2]) + } + // If it not matches the pattern "name (version)", it is only "name" + return strings.TrimSpace(source), "" +} + func parseDpkgPackageGroup(group []string, pathToLockfile string) PackageDetails { var pkg = PackageDetails{ Ecosystem: DebianEcosystem, CompareAs: DebianEcosystem, } - // TODO File SPECS: + sourcePresent := false + sourceHasVersion := false for _, line := range group { switch { - case strings.HasPrefix(line, "Package:"): - pkg.Name = strings.TrimPrefix(line, "Package:") - pkg.Name = strings.TrimSpace(pkg.Name) + // Status field SPECS: http://www.fifi.org/doc/libapt-pkg-doc/dpkg-tech.html/ch1.html#s1.2 + case strings.HasPrefix(line, "Status:"): + status := strings.TrimPrefix(line, "Status:") + tokens := strings.Fields(status) + // Staus field is malformed. Expected: "Status: Want Flag Status" + if len(tokens) != 3 { + _, _ = fmt.Fprintf( + os.Stderr, + "warning: malformed DPKG status file. Found no valid \"Source\" field. File: %s\n", + pathToLockfile, + ) + + return PackageDetails{} + } + // Status field has correct number of fields but package is not installed or has only config files left + // various other field values indicate partial install/uninstall (e.g. failure of some pre/post install scripts) + // since it's not clear if failure has left package active on system, cautiously add it to queries to osv.dev + if tokens[2] == "not-installed" || tokens[2] == "config-files" { + return PackageDetails{} + } + + case strings.HasPrefix(line, "Source:"): + sourcePresent = true + source := strings.TrimPrefix(line, "Source:") + name, version := parseSourceField(source) + pkg.Name = name // can be "" + if version != "" { + sourceHasVersion = true + pkg.Version = version + } + + // If Source fieald has no version, use Version field case strings.HasPrefix(line, "Version:"): - pkg.Version = strings.TrimPrefix(line, "Version:") - pkg.Version = strings.TrimSpace(pkg.Version) + if !sourceHasVersion { + pkg.Version = strings.TrimPrefix(line, "Version:") + pkg.Version = strings.TrimSpace(pkg.Version) + } + + // Some packages have no Source field (e.g. sudo) so we use Package value + case strings.HasPrefix(line, "Package:"): + if !sourcePresent { + pkg.Name = strings.TrimPrefix(line, "Package:") + pkg.Name = strings.TrimSpace(pkg.Name) + } } } @@ -84,6 +136,12 @@ func ParseDpkgStatus(pathToLockfile string) ([]PackageDetails, error) { for _, group := range packageGroups { pkg := parseDpkgPackageGroup(group, pathToLockfile) + // PackageDetails does not contain any field that represent a "not installed" state + // To manage this state and avoid false positives, empty struct means "not installed" so skip it + if (PackageDetails{}) == pkg { + continue + } + if pkg.Name == "" { _, _ = fmt.Fprintf( os.Stderr, diff --git a/pkg/lockfile/dpkg-status_test.go b/pkg/lockfile/dpkg-status_test.go index f36f9a131ee..2cbf73c68cb 100644 --- a/pkg/lockfile/dpkg-status_test.go +++ b/pkg/lockfile/dpkg-status_test.go @@ -55,6 +55,12 @@ func TestDpkgStatus_Malformed(t *testing.T) { Ecosystem: lockfile.DebianEcosystem, CompareAs: lockfile.DebianEcosystem, }, + { + Name: "util-linux", + Version: "2.36.1-8+deb11u1", + Ecosystem: lockfile.DebianEcosystem, + CompareAs: lockfile.DebianEcosystem, + }, }) } @@ -88,7 +94,7 @@ func TestDpkgStatus_Shuffled(t *testing.T) { expectPackages(t, packages, []lockfile.PackageDetails{ { - Name: "libc6", + Name: "glibc", Version: "2.31-13+deb11u5", Ecosystem: lockfile.DebianEcosystem, CompareAs: lockfile.DebianEcosystem, @@ -113,16 +119,35 @@ func TestDpkgStatus_Multiple(t *testing.T) { CompareAs: lockfile.DebianEcosystem, }, { - Name: "bsdutils", - Version: "1:2.36.1-8+deb11u1", + Name: "util-linux", + Version: "2.36.1-8+deb11u1", Ecosystem: lockfile.DebianEcosystem, CompareAs: lockfile.DebianEcosystem, }, { - Name: "libc6", + Name: "glibc", Version: "2.31-13+deb11u5", Ecosystem: lockfile.DebianEcosystem, CompareAs: lockfile.DebianEcosystem, }, }) } + +func TestDpkgStatus_Source_Ver_Override(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseDpkgStatus("fixtures/dpkg/source_ver_override_status") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "lvm2", + Version: "2.02.176-4.1ubuntu3", + Ecosystem: lockfile.DebianEcosystem, + CompareAs: lockfile.DebianEcosystem, + }, + }) +} diff --git a/pkg/lockfile/fixtures/dpkg/malformed_status b/pkg/lockfile/fixtures/dpkg/malformed_status index ca342b79847..cf4b500b3da 100644 --- a/pkg/lockfile/fixtures/dpkg/malformed_status +++ b/pkg/lockfile/fixtures/dpkg/malformed_status @@ -58,6 +58,31 @@ Description: basic utilities from 4.4BSD-Lite remaining standard BSD utilities are provided by bsdextrautils. Homepage: http://www.kernel.org/pub/linux/utils/util-linux/ +#Bad status + +Package: libc6 +Status: bad-status +Priority: optional +Section: libs +Installed-Size: 12837 +Maintainer: redacted +Architecture: amd64 +Multi-Arch: same +Source: glibc +Version: 2.31-13+deb11u5 +Replaces: libc6-amd64 +Depends: libgcc-s1, libcrypt1 +Recommends: libidn2-0 (>= 2.0.5~), libnss-nis, libnss-nisplus +Suggests: glibc-doc, debconf | debconf-2.0, libc-l10n, locales +Breaks: busybox (<< 1.30.1-6), hurd (<< 1:0.9.git20170910-1), ioquake3 (<< 1.36+u20200211.f2c61c1~dfsg-2~), iraf-fitsutil (<< 2018.07.06-4), libgegl-0.4-0 (<< 0.4.18), libtirpc1 (<< 0.2.3), locales (<< 2.31), locales-all (<< 2.31), macs (<< 2.2.7.1-3~), nocache (<< 1.1-1~), nscd (<< 2.31), openarena (<< 0.8.8+dfsg-4~), openssh-server (<< 1:8.1p1-5), r-cran-later (<< 0.7.5+dfsg-2), wcc (<< 0.0.2+dfsg-3) +Conffiles: + /etc/ld.so.conf.d/x86_64-linux-gnu.conf d4e7a7b88a71b5ffd9e2644e71a0cfab +Description: GNU C Library: Shared libraries + Contains the standard libraries that are used by nearly all programs on + the system. This package includes shared versions of the standard C library + and the standard math library, as well as many others. +Homepage: https://www.gnu.org/software/libc/libc.html + Nothing... @@ -69,7 +94,6 @@ Installed-Size: 12837 Maintainer: redacted Architecture: amd64 Multi-Arch: same -Source: glibc Replaces: libc6-amd64 Description: GNU C Library: Shared libraries Contains the standard libraries that are used by nearly all programs on diff --git a/pkg/lockfile/fixtures/dpkg/multiple_status b/pkg/lockfile/fixtures/dpkg/multiple_status index 4c30f8e6982..59d87423645 100644 --- a/pkg/lockfile/fixtures/dpkg/multiple_status +++ b/pkg/lockfile/fixtures/dpkg/multiple_status @@ -65,6 +65,31 @@ Depends: libgcc-s1, libcrypt1 Recommends: libidn2-0 (>= 2.0.5~), libnss-nis, libnss-nisplus Suggests: glibc-doc, debconf | debconf-2.0, libc-l10n, locales Breaks: busybox (<< 1.30.1-6), hurd (<< 1:0.9.git20170910-1), ioquake3 (<< 1.36+u20200211.f2c61c1~dfsg-2~), iraf-fitsutil (<< 2018.07.06-4), libgegl-0.4-0 (<< 0.4.18), libtirpc1 (<< 0.2.3), locales (<< 2.31), locales-all (<< 2.31), macs (<< 2.2.7.1-3~), nocache (<< 1.1-1~), nscd (<< 2.31), openarena (<< 0.8.8+dfsg-4~), openssh-server (<< 1:8.1p1-5), r-cran-later (<< 0.7.5+dfsg-2), wcc (<< 0.0.2+dfsg-3) +Conffiles: + /etc/ld.so.conf.d/x86_64-linux-gnu.conf d4e7a7b88a71b5ffd9e2644e71a0cfab +Description: GNU C Library: Shared libraries + Contains the standard libraries that are used by nearly all programs on + the system. This package includes shared versions of the standard C library + and the standard math library, as well as many others. +Homepage: https://www.gnu.org/software/libc/libc.html + +#Status not installed + +Package: libc6 +Status: install ok not-installed +Priority: optional +Section: libs +Installed-Size: 12837 +Maintainer: redacted +Architecture: amd64 +Multi-Arch: same +Source: glibc +Version: 2.31-13+deb11u5 +Replaces: libc6-amd64 +Depends: libgcc-s1, libcrypt1 +Recommends: libidn2-0 (>= 2.0.5~), libnss-nis, libnss-nisplus +Suggests: glibc-doc, debconf | debconf-2.0, libc-l10n, locales +Breaks: busybox (<< 1.30.1-6), hurd (<< 1:0.9.git20170910-1), ioquake3 (<< 1.36+u20200211.f2c61c1~dfsg-2~), iraf-fitsutil (<< 2018.07.06-4), libgegl-0.4-0 (<< 0.4.18), libtirpc1 (<< 0.2.3), locales (<< 2.31), locales-all (<< 2.31), macs (<< 2.2.7.1-3~), nocache (<< 1.1-1~), nscd (<< 2.31), openarena (<< 0.8.8+dfsg-4~), openssh-server (<< 1:8.1p1-5), r-cran-later (<< 0.7.5+dfsg-2), wcc (<< 0.0.2+dfsg-3) Conffiles: /etc/ld.so.conf.d/x86_64-linux-gnu.conf d4e7a7b88a71b5ffd9e2644e71a0cfab Description: GNU C Library: Shared libraries diff --git a/pkg/lockfile/fixtures/dpkg/source_ver_override_status b/pkg/lockfile/fixtures/dpkg/source_ver_override_status new file mode 100644 index 00000000000..feb9fc1255d --- /dev/null +++ b/pkg/lockfile/fixtures/dpkg/source_ver_override_status @@ -0,0 +1,20 @@ +Depends: libc6 (>= 2.14), libdevmapper-event1.02.1 (>= 2:1.02.110), libdevmapper1.02.1 (>= 2:1.02.141), liblvm2cmd2.02 (>= 2.02.176) +Architecture: amd64 + volume management, while keeping knowledge of the underlying device layout +Section: admin +Version: 2:1.02.145-4.1ubuntu3 +Original-Maintainer: redacted + in user-space. This makes it useful for not only LVM, but software raid, + . + The Linux Kernel Device Mapper is the LVM (Linux Logical Volume Management) +Source: lvm2 (2.02.176-4.1ubuntu3) +Description: Linux Kernel Device Mapper event daemon +Maintainer: redacted +Status: install ok installed +Homepage: http://sources.redhat.com/lvm2/ +Package: dmeventd +Installed-Size: 188 + This package contains a daemon to monitor events of devmapper devices. +Priority: optional + and other drivers that create "virtual" block devices. + Team's implementation of a minimalistic kernel-space driver that handles \ No newline at end of file diff --git a/pkg/lockfile/parse.go b/pkg/lockfile/parse.go index 06cd480f5ed..76c60d0656a 100644 --- a/pkg/lockfile/parse.go +++ b/pkg/lockfile/parse.go @@ -10,6 +10,7 @@ import ( // Represents when a package name could not be determined while parsing. // Currently, parsers are expected to omit such packages from their results. +// Using a const is required to avoid linter error (goconst) due to multiple usage in parsers. const unknownPkgName = "" func FindParser(pathToLockfile string, parseAs string) (PackageDetailsParser, string) { From ffd952e473672ef661858917f0d7b35b1198ea2e Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Sat, 18 Feb 2023 10:09:30 +0100 Subject: [PATCH 5/5] chore(dpkg-parser): code optimization --- pkg/lockfile/dpkg-status.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/lockfile/dpkg-status.go b/pkg/lockfile/dpkg-status.go index 7fcaf597fbe..4fec19328ee 100644 --- a/pkg/lockfile/dpkg-status.go +++ b/pkg/lockfile/dpkg-status.go @@ -17,15 +17,14 @@ func groupDpkgPackageLines(scanner *bufio.Scanner) [][]string { for scanner.Scan() { line := scanner.Text() - if line == "" { - if len(group) > 0 { - groups = append(groups, group) - } - group = make([]string, 0) - + if line != "" { + group = append(group, line) continue } - group = append(group, line) + if len(group) > 0 { + groups = append(groups, group) + } + group = make([]string, 0) } if len(group) > 0 { @@ -88,7 +87,7 @@ func parseDpkgPackageGroup(group []string, pathToLockfile string) PackageDetails pkg.Version = version } - // If Source fieald has no version, use Version field + // If Source field has no version, use Version field case strings.HasPrefix(line, "Version:"): if !sourceHasVersion { pkg.Version = strings.TrimPrefix(line, "Version:")