diff --git a/vulnfeeds/cmd/cve-bulk-converter/main.go b/vulnfeeds/cmd/cve-bulk-converter/main.go index 065d098c4b4..9f34f7c4cf7 100644 --- a/vulnfeeds/cmd/cve-bulk-converter/main.go +++ b/vulnfeeds/cmd/cve-bulk-converter/main.go @@ -23,6 +23,7 @@ var ( years = flag.String("years", "2022,2023,2024,2025", "A comma-separated list of years to process.") workers = flag.Int("workers", 30, "The number of concurrent workers to use for processing CVEs.") cnaAllowList = flag.String("cnas-allowlist", "", "A comma-separated list of CNAs to process. If not provided, defaults to cna_allowlist.txt.") + rejectFailed = flag.Bool("reject-failed", false, "If set, OSV records with a failed conversion outcome will not be generated.") ) //go:embed cna_allowlist.txt @@ -32,7 +33,7 @@ func main() { flag.Parse() logger.InitGlobalLogger() - logger.Info("Commencing Linux CVE to OSV conversion run") + logger.Info("Commencing CVE to OSV conversion run") if err := os.MkdirAll(*localOutputDir, 0755); err != nil { logger.Fatal("Failed to create local output directory", slog.Any("err", err)) } @@ -54,7 +55,7 @@ func main() { // Start the worker pool. for range *workers { wg.Add(1) - go worker(&wg, jobs, *localOutputDir, cnaList) + go worker(&wg, jobs, *localOutputDir, cnaList, *rejectFailed) } // Discover files and send them to the workers. @@ -71,6 +72,7 @@ func main() { logger.Info("Processing CVEs for year", slog.String("year", year)) err := filepath.Walk(yearDir, func(path string, info os.FileInfo, err error) error { if err != nil { + logger.Info("Error walking directory for year", slog.String("year", year), slog.Any("err", err)) return err } if !info.IsDir() && strings.HasSuffix(info.Name(), ".json") { @@ -90,7 +92,7 @@ func main() { } // worker is a function that processes CVE files from the jobs channel. -func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string) { +func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string, rejectFailed bool) { defer wg.Done() for path := range jobs { data, err := os.ReadFile(path) @@ -126,10 +128,17 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string } // Perform the conversion and export the results. - if err = cvelist2osv.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, sourceLink); err != nil { + metrics, err := cvelist2osv.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, sourceLink) + if err != nil { logger.Warn("Failed to generate an OSV record", slog.String("cve", string(cveID)), slog.Any("err", err)) } else { - logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName)) + if rejectFailed && metrics.Outcome != cvelist2osv.Successful { + logger.Info("Rejecting failed OSV record", slog.String("cve", string(cveID)), slog.String("outcome", metrics.Outcome.String())) + osvFile.Close() + os.Remove(osvFile.Name()) + } else { + logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName), slog.String("outcome", metrics.Outcome.String())) + } } metricsFile.Close() diff --git a/vulnfeeds/cmd/cve-single-converter/main.go b/vulnfeeds/cmd/cve-single-converter/main.go index 471e95fc161..a2af6573492 100644 --- a/vulnfeeds/cmd/cve-single-converter/main.go +++ b/vulnfeeds/cmd/cve-single-converter/main.go @@ -51,10 +51,10 @@ func main() { } // Perform the conversion and export the results. - if err = cvelist2osv.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, ""); err != nil { + if metrics, err := cvelist2osv.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, ""); err != nil { logger.Warn("Failed to generate an OSV record", slog.String("cve", string(cveID)), slog.Any("err", err)) } else { - logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName)) + logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName), slog.String("outcome", metrics.Outcome.String())) } metricsFile.Close() diff --git a/vulnfeeds/cvelist2osv/common.go b/vulnfeeds/cvelist2osv/common.go index da185c331c5..c853091e176 100644 --- a/vulnfeeds/cvelist2osv/common.go +++ b/vulnfeeds/cvelist2osv/common.go @@ -53,6 +53,27 @@ const ( FixUnresolvable // Partial resolution of versions, resulting in a false positive. ) +func (c ConversionOutcome) String() string { + switch c { + case Successful: + return "Successful" + case Rejected: + return "Rejected" + case NoSoftware: + return "NoSoftware" + case NoRepos: + return "NoRepos" + case NoCommitRanges: + return "NoCommitRanges" + case NoRanges: + return "NoRanges" + case FixUnresolvable: + return "FixUnresolvable" + default: + return "Unknown" + } +} + // String returns the string representation of a VersionRangeType. func (vrt VersionRangeType) String() string { switch vrt { diff --git a/vulnfeeds/cvelist2osv/converter.go b/vulnfeeds/cvelist2osv/converter.go index 3f643659d8c..c525e57a41e 100644 --- a/vulnfeeds/cvelist2osv/converter.go +++ b/vulnfeeds/cvelist2osv/converter.go @@ -229,7 +229,7 @@ func determineOutcome(metrics *ConversionMetrics) { // ConvertAndExportCVEToOSV is the main function for this file. It takes a CVE, // converts it into an OSV record, collects metrics, and writes both to disk. -func ConvertAndExportCVEToOSV(cve cves.CVE5, vulnSink io.Writer, metricsSink io.Writer, sourceLink string) error { +func ConvertAndExportCVEToOSV(cve cves.CVE5, vulnSink io.Writer, metricsSink io.Writer, sourceLink string) (*ConversionMetrics, error) { cveID := cve.Metadata.CVEID cnaAssigner := cve.Metadata.AssignerShortName references := identifyPossibleURLs(cve) @@ -261,21 +261,21 @@ func ConvertAndExportCVEToOSV(cve cves.CVE5, vulnSink io.Writer, metricsSink io. err := v.ToJSON(vulnSink) if err != nil { logger.Info("Failed to write", slog.Any("err", err)) - return err + return &metrics, err } marshalledMetrics, err := json.MarshalIndent(&metrics, "", " ") if err != nil { logger.Info("Failed to marshal", slog.Any("err", err)) - return err + return &metrics, err } _, err = metricsSink.Write(marshalledMetrics) if err != nil { logger.Info("Failed to write", slog.Any("err", err)) - return err + return &metrics, err } - return nil + return &metrics, nil } // identifyPossibleURLs extracts all URLs from a CVE object. diff --git a/vulnfeeds/cvelist2osv/converter_test.go b/vulnfeeds/cvelist2osv/converter_test.go index 7ce0523525b..5d73721851d 100644 --- a/vulnfeeds/cvelist2osv/converter_test.go +++ b/vulnfeeds/cvelist2osv/converter_test.go @@ -583,7 +583,7 @@ func TestConvertAndExportCVEToOSV(t *testing.T) { t.Run(tc.name, func(t *testing.T) { vWriter := bytes.NewBuffer(nil) mWriter := bytes.NewBuffer(nil) - err := ConvertAndExportCVEToOSV(tc.cve, vWriter, mWriter, "") + _, err := ConvertAndExportCVEToOSV(tc.cve, vWriter, mWriter, "") if err != nil { t.Errorf("Unexpected error from ConvertAndExportCVEToOSV: %v", err) }