refactor(vulnfeeds): Uncouple Debian conversion from combine-to-osv converter

vulnfeeds/cmd/combine-to-osv/README.md

@@ -9,7 +9,6 @@ Combine [`PackageInfo`](https://github.com/google/osv.dev/blob/2c22e9534a521c6c6
 To address the generation of CVE records from multiple disparate sources (all requiring a common record prefix):
 
 * Alpine, by [this code](../alpine)
-* Debian, by [this code](../debian)
 * the NVD, by [this code](../nvd-cve-osv)
 
 ## How

vulnfeeds/cmd/combine-to-osv/main.go

@@ -24,8 +24,6 @@ const (
 
 	alpineEcosystem          = "Alpine"
 	alpineSecurityTrackerURL = "https://security.alpinelinux.org/vuln"
-	debianEcosystem          = "Debian"
-	debianSecurityTrackerURL = "https://security-tracker.debian.org/tracker"
 )
 
 var Logger utility.LoggerWrapper
@@ -170,14 +168,10 @@ func combineIntoOSV(loadedCves map[cves.CVEID]cves.Vulnerability, allParts map[c
 			}
 		}
 
-		addedDebianURL := false
 		addedAlpineURL := false
 		for _, pkgInfo := range allParts[cveID] {
 			convertedCve.AddPkgInfo(pkgInfo)
-			if strings.HasPrefix(pkgInfo.Ecosystem, debianEcosystem) && !addedDebianURL {
-				addReference(string(cveID), debianEcosystem, convertedCve)
-				addedDebianURL = true
-			} else if strings.HasPrefix(pkgInfo.Ecosystem, alpineEcosystem) && !addedAlpineURL {
+			if strings.HasPrefix(pkgInfo.Ecosystem, alpineEcosystem) && !addedAlpineURL {
 				addReference(string(cveID), alpineEcosystem, convertedCve)
 				addedAlpineURL = true
 			}
@@ -249,11 +243,8 @@ func loadAllCVEs(cvePath string) map[cves.CVEID]cves.Vulnerability {
 // addReference adds the related security tracker URL to a given vulnerability's references
 func addReference(cveID string, ecosystem string, convertedCve *vulns.Vulnerability) {
 	securityReference := osvschema.Reference{Type: osvschema.ReferenceAdvisory}
-	switch ecosystem {
-	case alpineEcosystem:
+	if ecosystem == alpineEcosystem {
 		securityReference.URL, _ = url.JoinPath(alpineSecurityTrackerURL, cveID)
-	case debianEcosystem:
-		securityReference.URL, _ = url.JoinPath(debianSecurityTrackerURL, cveID)
 	}
 
 	if securityReference.URL == "" {

vulnfeeds/cmd/combine-to-osv/main_test.go

@@ -104,13 +104,14 @@ func TestCombineIntoOSV(t *testing.T) {
 		if len(combinedOSV[cve].Affected) != len(allParts[cve]) {
 			t.Errorf("Affected lengths for %s do not match", cve)
 		}
+
 		found := false
 		switch cve {
 		case "CVE-2018-1000500":
 			for _, reference := range combinedOSV[cve].References {
 				if reference.Type == "ADVISORY" &&
 					reference.URL == "https://security-tracker.debian.org/tracker/CVE-2018-1000500" {
-					found = true
+					t.Errorf("Found unexpected Debian advisory URL for %s", cve)
 				}
 			}
 		case "CVE-2022-33745":
@@ -128,12 +129,11 @@ func TestCombineIntoOSV(t *testing.T) {
 				}
 			}
 		}
-		if !found {
+		if !found && cve != "CVE-2018-1000500" {
 			t.Errorf("%s doesn't have all expected references", cve)
 		}
 	}
 }
-
 func TestGetModifiedTime(t *testing.T) {
 	_, err := getModifiedTime("../../test_data/parts/debian/CVE-2016-1585.debian.json")
 	if err != nil {

vulnfeeds/cmd/debian/debian_security_tracker.go

@@ -7,11 +7,11 @@ type Release struct {
 	Urgency      string            `json:"urgency"`
 }
 
-type CVE struct {
+type DebianCVE struct {
 	Description string `json:"description"`
 	DebianBug   int
 	Scope       string             `json:"scope"`
 	Releases    map[string]Release `json:"releases"`
 }
 
-type DebianSecurityTrackerData map[string]map[string]CVE
+type DebianSecurityTrackerData map[string]map[string]DebianCVE

vulnfeeds/cmd/debian/main.go

@@ -4,21 +4,24 @@ package main
 import (
 	"encoding/csv"
 	"encoding/json"
+	"flag"
 	"fmt"
 	"net/http"
 	"os"
 	"path"
 	"sort"
 	"strconv"
+	"time"
 
 	"github.com/google/osv/vulnfeeds/faulttolerant"
 	"github.com/google/osv/vulnfeeds/models"
 	"github.com/google/osv/vulnfeeds/utility"
 	"github.com/google/osv/vulnfeeds/vulns"
+	"github.com/ossf/osv-schema/bindings/go/osvschema"
 )
 
 const (
-	debianOutputPathDefault  = "parts/debian"
+	debianOutputPathDefault  = "debian_osv"
 	debianDistroInfoURL      = "https://debian.pages.debian.net/distro-info-data/debian.csv"
 	debianSecurityTrackerURL = "https://security-tracker.debian.org/tracker/data/json"
 )
@@ -30,7 +33,10 @@ func main() {
 	Logger, logCleanup = utility.CreateLoggerWrapper("debian-osv")
 	defer logCleanup()
 
-	err := os.MkdirAll(debianOutputPathDefault, 0755)
+	debianOutputPath := flag.String("output_path", debianOutputPathDefault, "Path to output OSV files.")
+	flag.Parse()
+
+	err := os.MkdirAll(*debianOutputPath, 0755)
 	if err != nil {
 		Logger.Fatalf("Can't create output path: %s", err)
 	}
@@ -45,14 +51,96 @@ func main() {
 		Logger.Fatalf("Failed to get Debian distro info data: %s", err)
 	}
 
-	cvePkgInfos := generateDebianSecurityTrackerOSV(debianData, debianReleaseMap)
-	if err = writeToOutput(cvePkgInfos); err != nil {
+	osvCves := generateOSVFromDebianTracker(debianData, debianReleaseMap)
+
+	if err = writeToOutput(osvCves, *debianOutputPath); err != nil {
 		Logger.Fatalf("Failed to write OSV output file: %s", err)
 	}
 
 	Logger.Infof("Debian CVE conversion succeeded.")
 }
 
+// generateOSVFromDebianTracker converts Debian Security Tracker entries to OSV format.
+func generateOSVFromDebianTracker(debianData DebianSecurityTrackerData, debianReleaseMap map[string]string) map[string]*vulns.Vulnerability {
+	Logger.Infof("Converting Debian Security Tracker data to OSV.")
+	osvCves := make(map[string]*vulns.Vulnerability)
+
+	// Sorts packages to ensure results remain consistent between runs.
+	pkgNames := make([]string, 0, len(debianData))
+	for name := range debianData {
+		pkgNames = append(pkgNames, name)
+	}
+	sort.Strings(pkgNames)
+
+	// Sorts releases to ensure pkgInfos remain consistent between runs.
+	releaseNames := make([]string, 0, len(debianReleaseMap))
+	for k := range debianReleaseMap {
+		releaseNames = append(releaseNames, k)
+	}
+
+	sort.Slice(releaseNames, func(i, j int) bool {
+		vi, _ := strconv.ParseFloat(debianReleaseMap[releaseNames[i]], 64)
+		vj, _ := strconv.ParseFloat(debianReleaseMap[releaseNames[j]], 64)
+
+		return vi < vj
+	})
+
+	for _, pkgName := range pkgNames {
+		pkg := debianData[pkgName]
+		for cveID, cveData := range pkg {
+			v, ok := osvCves[cveID]
+			if !ok {
+				v = &vulns.Vulnerability{
+					Vulnerability: osvschema.Vulnerability{
+						ID:       "DEBIAN-" + cveID,
+						Upstream: []string{cveID},
+						Modified: time.Now().UTC(),
+						Details:  cveData.Description,
+						References: []osvschema.Reference{
+							{
+								Type: "ADVISORY",
+								URL:  "https://security-tracker.debian.org/tracker/" + cveID,
+							},
+						},
+					},
+				}
+				osvCves[cveID] = v
+			}
+
+			for _, releaseName := range releaseNames {
+				// For reference on urgency levels, see: https://security-team.debian.org/security_tracker.html#severity-levels
+				release, ok := cveData.Releases[releaseName]
+				if !ok {
+					continue
+				}
+				debianVersion, ok := debianReleaseMap[releaseName]
+				if !ok {
+					continue
+				}
+
+				if release.Status == "resolved" && release.FixedVersion == "0" { // not affected
+					continue
+				}
+
+				pkgInfo := vulns.PackageInfo{
+					PkgName:   pkgName,
+					Ecosystem: "Debian:" + debianVersion,
+					EcosystemSpecific: map[string]any{
+						"urgency": release.Urgency,
+					},
+				}
+
+				if release.Status == "resolved" {
+					pkgInfo.VersionInfo.AffectedVersions = append(pkgInfo.VersionInfo.AffectedVersions, models.AffectedVersion{Fixed: release.FixedVersion})
+				}
+				v.AddPkgInfo(pkgInfo)
+			}
+		}
+	}
+
+	return osvCves
+}
+
 // getDebianReleaseMap gets the Debian version number, excluding testing and experimental versions.
 func getDebianReleaseMap() (map[string]string, error) {
 	releaseMap := make(map[string]string)
@@ -100,98 +188,24 @@ func getDebianReleaseMap() (map[string]string, error) {
 	return releaseMap, err
 }
 
-// updateOSVPkgInfos adds new release entries to osvPkgInfos.
-func updateOSVPkgInfos(pkgName string, cveID string, releases map[string]Release, osvPkgInfos map[string][]vulns.PackageInfo, debianReleaseMap map[string]string, releaseNames []string) {
-	//nolint:prealloc
-	var pkgInfos []vulns.PackageInfo
-	if value, ok := osvPkgInfos[cveID]; ok {
-		pkgInfos = value
-	}
-
-	for _, releaseName := range releaseNames {
-		// For reference on urgency levels, see: https://security-team.debian.org/security_tracker.html#severity-levels
-		release, ok := releases[releaseName]
-		if !ok {
-			continue
-		}
-		debianVersion, ok := debianReleaseMap[releaseName]
-		if !ok {
-			continue
-		}
-
-		pkgInfo := vulns.PackageInfo{
-			PkgName:   pkgName,
-			Ecosystem: "Debian:" + debianVersion,
-		}
-		pkgInfo.EcosystemSpecific = make(map[string]any)
-
-		pkgInfo.VersionInfo = models.VersionInfo{
-			AffectedVersions: []models.AffectedVersion{{Introduced: "0"}},
-		}
-		if release.Status == "resolved" {
-			if release.FixedVersion == "0" { // not affected
-				continue
-			}
-			pkgInfo.VersionInfo.AffectedVersions = append(pkgInfo.VersionInfo.AffectedVersions, models.AffectedVersion{Fixed: release.FixedVersion})
-		}
-		pkgInfo.EcosystemSpecific["urgency"] = release.Urgency
-		pkgInfos = append(pkgInfos, pkgInfo)
-	}
-	if pkgInfos != nil {
-		osvPkgInfos[cveID] = pkgInfos
-	}
-}
-
-// generateDebianSecurityTrackerOSV converts Debian Security Tracker entries to OSV PackageInfo format.
-func generateDebianSecurityTrackerOSV(debianData DebianSecurityTrackerData, debianReleaseMap map[string]string) map[string][]vulns.PackageInfo {
-	Logger.Infof("Converting Debian Security Tracker data to OSV package infos.")
-	osvPkgInfos := make(map[string][]vulns.PackageInfo)
-
-	// Sorts packages to ensure results remain consistent between runs.
-	pkgNames := make([]string, 0, len(debianData))
-	for name := range debianData {
-		pkgNames = append(pkgNames, name)
-	}
-	sort.Strings(pkgNames)
-
-	// Sorts releases to ensure pkgInfos remain consistent between runs.
-	releaseNames := make([]string, 0, len(debianReleaseMap))
-	for k := range debianReleaseMap {
-		releaseNames = append(releaseNames, k)
-	}
-
-	sort.Slice(releaseNames, func(i, j int) bool {
-		vi, _ := strconv.ParseFloat(debianReleaseMap[releaseNames[i]], 64)
-		vj, _ := strconv.ParseFloat(debianReleaseMap[releaseNames[j]], 64)
-
-		return vi < vj
-	})
-
-	for _, pkgName := range pkgNames {
-		pkg := debianData[pkgName]
-		for cveID, cve := range pkg {
-			updateOSVPkgInfos(pkgName, cveID, cve.Releases, osvPkgInfos, debianReleaseMap, releaseNames)
-		}
-	}
-
-	return osvPkgInfos
-}
-
-func writeToOutput(cvePkgInfos map[string][]vulns.PackageInfo) error {
-	Logger.Infof("Writing package infos to the output.")
-	for cveID := range cvePkgInfos {
-		pkgInfos := cvePkgInfos[cveID]
-		file, err := os.OpenFile(path.Join(debianOutputPathDefault, cveID+".debian.json"), os.O_CREATE|os.O_RDWR, 0644)
+func writeToOutput(osvCves map[string]*vulns.Vulnerability, debianOutputPath string) error {
+	Logger.Infof("Writing OSV files to the output.")
+	for cveID, osv := range osvCves {
+		file, err := os.OpenFile(path.Join(debianOutputPath, cveID+".json"), os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
 		if err != nil {
 			return err
 		}
+
 		encoder := json.NewEncoder(file)
 		encoder.SetIndent("", "  ")
-		err = encoder.Encode(&pkgInfos)
+		err = encoder.Encode(osv)
+		closeErr := file.Close()
 		if err != nil {
 			return err
 		}
-		_ = file.Close()
+		if closeErr != nil {
+			return closeErr
+		}
 	}
 
 	return nil