feat(cve5): Add flag to reject on failure

vulnfeeds/cmd/cve-bulk-converter/main.go

@@ -23,6 +23,7 @@ var (
 	years          = flag.String("years", "2022,2023,2024,2025", "A comma-separated list of years to process.")
 	workers        = flag.Int("workers", 30, "The number of concurrent workers to use for processing CVEs.")
 	cnaAllowList   = flag.String("cnas-allowlist", "", "A comma-separated list of CNAs to process. If not provided, defaults to cna_allowlist.txt.")
+	rejectFailed   = flag.Bool("reject-failed", false, "If set, OSV records with a failed conversion outcome will not be generated.")
 )
 
 //go:embed cna_allowlist.txt
@@ -32,7 +33,7 @@ func main() {
 	flag.Parse()
 	logger.InitGlobalLogger()
 
-	logger.Info("Commencing Linux CVE to OSV conversion run")
+	logger.Info("Commencing CVE to OSV conversion run")
 	if err := os.MkdirAll(*localOutputDir, 0755); err != nil {
 		logger.Fatal("Failed to create local output directory", slog.Any("err", err))
 	}
@@ -54,7 +55,7 @@ func main() {
 	// Start the worker pool.
 	for range *workers {
 		wg.Add(1)
-		go worker(&wg, jobs, *localOutputDir, cnaList)
+		go worker(&wg, jobs, *localOutputDir, cnaList, *rejectFailed)
 	}
 
 	// Discover files and send them to the workers.
@@ -71,6 +72,7 @@ func main() {
 		logger.Info("Processing CVEs for year", slog.String("year", year))
 		err := filepath.Walk(yearDir, func(path string, info os.FileInfo, err error) error {
 			if err != nil {
+				logger.Info("Error walking directory for year", slog.String("year", year), slog.Any("err", err))
 				return err
 			}
 			if !info.IsDir() && strings.HasSuffix(info.Name(), ".json") {
@@ -90,7 +92,7 @@ func main() {
 }
 
 // worker is a function that processes CVE files from the jobs channel.
-func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string) {
+func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string, rejectFailed bool) {
 	defer wg.Done()
 	for path := range jobs {
 		data, err := os.ReadFile(path)
@@ -126,10 +128,17 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string
 		}
 
 		// Perform the conversion and export the results.
-		if err = cvelist2osv.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, sourceLink); err != nil {
+		metrics, err := cvelist2osv.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, sourceLink)
+		if err != nil {
 			logger.Warn("Failed to generate an OSV record", slog.String("cve", string(cveID)), slog.Any("err", err))
 		} else {
-			logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName))
+			if rejectFailed && metrics.Outcome != cvelist2osv.Successful {
+				logger.Info("Rejecting failed OSV record", slog.String("cve", string(cveID)), slog.String("outcome", metrics.Outcome.String()))
+				osvFile.Close()
+				os.Remove(osvFile.Name())
+			} else {
+				logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName), slog.String("outcome", metrics.Outcome.String()))
+			}
 		}
 
 		metricsFile.Close()

vulnfeeds/cmd/cve-single-converter/main.go

@@ -51,10 +51,10 @@ func main() {
 	}
 
 	// Perform the conversion and export the results.
-	if err = cvelist2osv.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, ""); err != nil {
+	if metrics, err := cvelist2osv.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, ""); err != nil {
 		logger.Warn("Failed to generate an OSV record", slog.String("cve", string(cveID)), slog.Any("err", err))
 	} else {
-		logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName))
+		logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName), slog.String("outcome", metrics.Outcome.String()))
 	}
 
 	metricsFile.Close()

vulnfeeds/cvelist2osv/common.go

@@ -53,6 +53,27 @@ const (
 	FixUnresolvable                            // Partial resolution of versions, resulting in a false positive.
 )
 
+func (c ConversionOutcome) String() string {
+	switch c {
+	case Successful:
+		return "Successful"
+	case Rejected:
+		return "Rejected"
+	case NoSoftware:
+		return "NoSoftware"
+	case NoRepos:
+		return "NoRepos"
+	case NoCommitRanges:
+		return "NoCommitRanges"
+	case NoRanges:
+		return "NoRanges"
+	case FixUnresolvable:
+		return "FixUnresolvable"
+	default:
+		return "Unknown"
+	}
+}
+
 // String returns the string representation of a VersionRangeType.
 func (vrt VersionRangeType) String() string {
 	switch vrt {

vulnfeeds/cvelist2osv/converter.go

@@ -229,7 +229,7 @@ func determineOutcome(metrics *ConversionMetrics) {
 
 // ConvertAndExportCVEToOSV is the main function for this file. It takes a CVE,
 // converts it into an OSV record, collects metrics, and writes both to disk.
-func ConvertAndExportCVEToOSV(cve cves.CVE5, vulnSink io.Writer, metricsSink io.Writer, sourceLink string) error {
+func ConvertAndExportCVEToOSV(cve cves.CVE5, vulnSink io.Writer, metricsSink io.Writer, sourceLink string) (*ConversionMetrics, error) {
 	cveID := cve.Metadata.CVEID
 	cnaAssigner := cve.Metadata.AssignerShortName
 	references := identifyPossibleURLs(cve)
@@ -261,21 +261,21 @@ func ConvertAndExportCVEToOSV(cve cves.CVE5, vulnSink io.Writer, metricsSink io.
 	err := v.ToJSON(vulnSink)
 	if err != nil {
 		logger.Info("Failed to write", slog.Any("err", err))
-		return err
+		return &metrics, err
 	}
 
 	marshalledMetrics, err := json.MarshalIndent(&metrics, "", "  ")
 	if err != nil {
 		logger.Info("Failed to marshal", slog.Any("err", err))
-		return err
+		return &metrics, err
 	}
 	_, err = metricsSink.Write(marshalledMetrics)
 	if err != nil {
 		logger.Info("Failed to write", slog.Any("err", err))
-		return err
+		return &metrics, err
 	}
 
-	return nil
+	return &metrics, nil
 }
 
 // identifyPossibleURLs extracts all URLs from a CVE object.

vulnfeeds/cvelist2osv/converter_test.go

@@ -583,7 +583,7 @@ func TestConvertAndExportCVEToOSV(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			vWriter := bytes.NewBuffer(nil)
 			mWriter := bytes.NewBuffer(nil)
-			err := ConvertAndExportCVEToOSV(tc.cve, vWriter, mWriter, "")
+			_, err := ConvertAndExportCVEToOSV(tc.cve, vWriter, mWriter, "")
 			if err != nil {
 				t.Errorf("Unexpected error from ConvertAndExportCVEToOSV: %v", err)
 			}