fix(debian): make fault-tolerant HTTP downloads (#2674)

I noticed a recent spate of 503s in the error logging that may have
benefited from this

Author: andrewpollock

.gitignore

@@ -1,3 +1,5 @@
+go.work
+go.work.sum
 *.pyc
 __pycache__/
 .idea/

vulnfeeds/cmd/debian/main.go

@@ -11,6 +11,7 @@ import (
 	"strconv"
 
 	"github.com/google/osv/vulnfeeds/cves"
+	"github.com/google/osv/vulnfeeds/faulttolerant"
 	"github.com/google/osv/vulnfeeds/utility"
 	"github.com/google/osv/vulnfeeds/vulns"
 )
@@ -54,7 +55,7 @@ func main() {
 // getDebianReleaseMap gets the Debian version number, excluding testing and experimental versions.
 func getDebianReleaseMap() (map[string]string, error) {
 	releaseMap := make(map[string]string)
-	res, err := http.Get(debianDistroInfoURL)
+	res, err := faulttolerant.Get(debianDistroInfoURL)
 	if err != nil {
 		return releaseMap, err
 	}
@@ -194,7 +195,7 @@ func writeToOutput(cvePkgInfos map[string][]vulns.PackageInfo) error {
 
 // downloadDebianSecurityTracker download Debian json file
 func downloadDebianSecurityTracker() (DebianSecurityTrackerData, error) {
-	res, err := http.Get(debianSecurityTrackerURL)
+	res, err := faulttolerant.Get(debianSecurityTrackerURL)
 	if err != nil {
 		return nil, err
 	}

vulnfeeds/faulttolerant/http.go

@@ -0,0 +1,69 @@
+package faulttolerant
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/sethvargo/go-retry"
+)
+
+// Make a HTTP GET request for url and retry 3 times, with an exponential backoff.
+func Get(url string) (resp *http.Response, err error) {
+	backoff := retry.NewExponential(1 * time.Second)
+	if err := retry.Do(context.Background(), retry.WithMaxRetries(3, backoff), func(ctx context.Context) error {
+		req, err := http.NewRequest("GET", url, nil)
+		if err != nil {
+			return err
+		}
+
+		r, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return err
+		}
+
+		switch r.StatusCode / 100 {
+		case 4:
+			return fmt.Errorf("bad response: %v", r.StatusCode)
+		case 5:
+			return retry.RetryableError(fmt.Errorf("bad response: %v", r.StatusCode))
+		default:
+			resp = r
+			return nil
+		}
+	}); err != nil {
+		return nil, fmt.Errorf("fail: %q: %v", url, err)
+	}
+	return resp, err
+}
+
+// Make a HTTP HEAD request for url and retry 3 times, with an exponential backoff.
+func Head(url string) (resp *http.Response, err error) {
+	backoff := retry.NewExponential(1 * time.Second)
+	if err := retry.Do(context.Background(), retry.WithMaxRetries(3, backoff), func(ctx context.Context) error {
+		req, err := http.NewRequest("HEAD", url, nil)
+		if err != nil {
+			return err
+		}
+
+		r, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return err
+		}
+		defer r.Body.Close()
+
+		switch r.StatusCode / 100 {
+		case 4:
+			return fmt.Errorf("bad response: %v", r.StatusCode)
+		case 5:
+			return retry.RetryableError(fmt.Errorf("bad response: %v", r.StatusCode))
+		default:
+			resp = r
+			return nil
+		}
+	}); err != nil {
+		return nil, fmt.Errorf("fail: %q: %v", url, err)
+	}
+	return resp, err
+}