refactor(vulnfeeds): Uncouple Debian conversion from combine-to-osv converter (#3894)

Starting to deal with #2465 

Closes #3899

---------

Co-authored-by: Rex P <[email protected]>

Author: jess-lowe

deployment/clouddeploy/gke-workers/environments/oss-vdb-test/debian-cve-convert.yaml

@@ -13,4 +13,4 @@ spec:
             - name: GOOGLE_CLOUD_PROJECT
               value: oss-vdb-test
             - name: OUTPUT_GCS_BUCKET
-              value: osv-test-cve-osv-conversion
+              value: osv-test-debian-osv

deployment/clouddeploy/gke-workers/environments/oss-vdb/debian-cve-convert.yaml

@@ -13,4 +13,4 @@ spec:
             - name: GOOGLE_CLOUD_PROJECT
               value: oss-vdb
             - name: OUTPUT_GCS_BUCKET
-              value: cve-osv-conversion
+              value: debian-osv

vulnfeeds/cmd/combine-to-osv/README.md

@@ -9,7 +9,6 @@ Combine [`PackageInfo`](https://github.com/google/osv.dev/blob/2c22e9534a521c6c6
 To address the generation of CVE records from multiple disparate sources (all requiring a common record prefix):
 
 * Alpine, by [this code](../alpine)
-* Debian, by [this code](../debian)
 * the NVD, by [this code](../nvd-cve-osv)
 
 ## How

vulnfeeds/cmd/combine-to-osv/main.go

@@ -25,8 +25,6 @@ const (
 
 	alpineEcosystem          = "Alpine"
 	alpineSecurityTrackerURL = "https://security.alpinelinux.org/vuln"
-	debianEcosystem          = "Debian"
-	debianSecurityTrackerURL = "https://security-tracker.debian.org/tracker"
 )
 
 func main() {
@@ -47,7 +45,7 @@ func main() {
 		logger.Fatal("Can't create output path", slog.Any("err", err))
 	}
 
-	allCves := loadAllCVEs(*cvePath)
+	allCves := vulns.LoadAllCVEs(*cvePath)
 	allParts, cveModifiedMap := loadParts(*partsInputPath)
 	combinedData := combineIntoOSV(allCves, allParts, *cveListPath, cveModifiedMap)
 	writeOSVFile(combinedData, *osvOutputPath)
@@ -166,14 +164,10 @@ func combineIntoOSV(loadedCves map[cves.CVEID]cves.Vulnerability, allParts map[c
 			}
 		}
 
-		addedDebianURL := false
 		addedAlpineURL := false
 		for _, pkgInfo := range allParts[cveID] {
 			convertedCve.AddPkgInfo(pkgInfo)
-			if strings.HasPrefix(pkgInfo.Ecosystem, debianEcosystem) && !addedDebianURL {
-				addReference(string(cveID), debianEcosystem, convertedCve)
-				addedDebianURL = true
-			} else if strings.HasPrefix(pkgInfo.Ecosystem, alpineEcosystem) && !addedAlpineURL {
+			if strings.HasPrefix(pkgInfo.Ecosystem, alpineEcosystem) && !addedAlpineURL {
 				addReference(string(cveID), alpineEcosystem, convertedCve)
 				addedAlpineURL = true
 			}
@@ -209,47 +203,11 @@ func writeOSVFile(osvData map[cves.CVEID]*vulns.Vulnerability, osvOutputPath str
 	logger.Info("Successfully written OSV files", slog.Int("count", len(osvData)))
 }
 
-// loadAllCVEs loads the downloaded CVE's from the NVD database into memory.
-func loadAllCVEs(cvePath string) map[cves.CVEID]cves.Vulnerability {
-	dir, err := os.ReadDir(cvePath)
-	if err != nil {
-		logger.Fatal("Failed to read dir", slog.String("path", cvePath), slog.Any("err", err))
-	}
-
-	result := make(map[cves.CVEID]cves.Vulnerability)
-
-	for _, entry := range dir {
-		if !strings.HasSuffix(entry.Name(), ".json") {
-			continue
-		}
-		file, err := os.Open(path.Join(cvePath, entry.Name()))
-		if err != nil {
-			logger.Fatal("Failed to open CVE JSON", slog.String("path", path.Join(cvePath, entry.Name())), slog.Any("err", err))
-		}
-		var nvdcve cves.CVEAPIJSON20Schema
-		err = json.NewDecoder(file).Decode(&nvdcve)
-		if err != nil {
-			logger.Fatal("Failed to decode JSON", slog.String("file", file.Name()), slog.Any("err", err))
-		}
-
-		for _, item := range nvdcve.Vulnerabilities {
-			result[item.CVE.ID] = item
-		}
-		logger.Info("Loaded CVE "+entry.Name(), slog.String("cve", entry.Name()))
-		file.Close()
-	}
-
-	return result
-}
-
 // addReference adds the related security tracker URL to a given vulnerability's references
 func addReference(cveID string, ecosystem string, convertedCve *vulns.Vulnerability) {
 	securityReference := osvschema.Reference{Type: osvschema.ReferenceAdvisory}
-	switch ecosystem {
-	case alpineEcosystem:
+	if ecosystem == alpineEcosystem {
 		securityReference.URL, _ = url.JoinPath(alpineSecurityTrackerURL, cveID)
-	case debianEcosystem:
-		securityReference.URL, _ = url.JoinPath(debianSecurityTrackerURL, cveID)
 	}
 
 	if securityReference.URL == "" {

vulnfeeds/cmd/combine-to-osv/main_test.go

@@ -104,13 +104,14 @@ func TestCombineIntoOSV(t *testing.T) {
 		if len(combinedOSV[cve].Affected) != len(allParts[cve]) {
 			t.Errorf("Affected lengths for %s do not match", cve)
 		}
+
 		found := false
 		switch cve {
 		case "CVE-2018-1000500":
 			for _, reference := range combinedOSV[cve].References {
 				if reference.Type == "ADVISORY" &&
 					reference.URL == "https://security-tracker.debian.org/tracker/CVE-2018-1000500" {
-					found = true
+					t.Errorf("Found unexpected Debian advisory URL for %s", cve)
 				}
 			}
 		case "CVE-2022-33745":
@@ -128,12 +129,11 @@ func TestCombineIntoOSV(t *testing.T) {
 				}
 			}
 		}
-		if !found {
+		if !found && cve != "CVE-2018-1000500" {
 			t.Errorf("%s doesn't have all expected references", cve)
 		}
 	}
 }
-
 func TestGetModifiedTime(t *testing.T) {
 	_, err := getModifiedTime("../../test_data/parts/debian/CVE-2016-1585.debian.json")
 	if err != nil {

vulnfeeds/cmd/debian/debian_security_tracker.go

@@ -7,11 +7,11 @@ type Release struct {
 	Urgency      string            `json:"urgency"`
 }
 
-type CVE struct {
+type DebianCVE struct {
 	Description string `json:"description"`
 	DebianBug   int
 	Scope       string             `json:"scope"`
 	Releases    map[string]Release `json:"releases"`
 }
 
-type DebianSecurityTrackerData map[string]map[string]CVE
+type DebianSecurityTrackerData map[string]map[string]DebianCVE