fix: Update exporter requested resources & scheduling (#4202)

Doubling `num_workers` to 400 allows the exporter to run in ~10 minutes.
We've wanted the exporter to run every 15 minutes since forever, so do
that now that it can.

Also, fixed vulns using camelCase in all.zip and ecosystems.txt not
containing a newline at end of file

Author: michaelkedar

deployment/clouddeploy/gke-workers/base/exporter.yaml

@@ -5,7 +5,7 @@ metadata:
   labels:
     cronLastSuccessfulTimeMins: "180"
 spec:
-  schedule: "*/30 * * * *"
+  schedule: "*/15 * * * *"
   concurrencyPolicy: Forbid
   # If the previous job overruns by more than a minute,
   # Don't try to immediately queue up the next job
@@ -26,9 +26,9 @@ spec:
             imagePullPolicy: Always
             resources:
               requests:
-                cpu: "12"
-                memory: "48Gi"
+                cpu: "10"
+                memory: "40Gi"
               limits:
-                cpu: "24"
-                memory: "128Gi"
+                cpu: "20"
+                memory: "64Gi"
           restartPolicy: Never

deployment/clouddeploy/gke-workers/environments/oss-vdb-test/exporter.yaml

@@ -15,6 +15,7 @@ spec:
             args:
               # TODO(michaelkedar): single source of truth w/ terraform config
               - "--uploadToGCS=true"
+              - "--num_workers=400"
               - "--bucket=osv-test-vulnerabilities"
               - "--osv_vulns_bucket=osv-test-vulnerabilities"
 

deployment/clouddeploy/gke-workers/environments/oss-vdb/exporter.yaml

@@ -14,6 +14,7 @@ spec:
               value: oss-vdb
             args:
               - "--uploadToGCS=true"
+              - "--num_workers=400"
               - "--bucket=osv-vulnerabilities"
               - "--osv_vulns_bucket=osv-vulnerabilities"
 

go/cmd/exporter/worker.go

@@ -169,7 +169,7 @@ WorkLoop:
 			if !ok {
 				break WorkLoop
 			}
-			b, err := protojson.MarshalOptions{}.Marshal(v.Vulnerability)
+			b, err := protoMarshaller.Marshal(v.Vulnerability)
 			if err != nil {
 				logger.Error("failed to marshal vulnerability to json", slog.String("id", v.GetId()), slog.Any("err", err))
 				continue
@@ -185,7 +185,7 @@ WorkLoop:
 	writeZIP(ctx, allZipFilename, allVulns, outCh)
 	ecos := slices.Collect(maps.Keys(ecosystems))
 	slices.Sort(ecos)
-	ecoString := strings.Join(ecos, "\n")
+	ecoString := strings.Join(ecos, "\n") + "\n"
 	write(ctx, ecosystemsFilename, []byte(ecoString), "text/plain", outCh)
 	logger.Info("all-ecosystem worker finished processing")
 }