Improve the process for importing affected and fixed commits

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importer.py

@@ -402,18 +402,25 @@ class AffectedPackageV2:
     """
     Relate a Package URL with a range of affected versions and fixed versions.
     The Package URL must *not* have a version.
-    AffectedPackage must contain either ``affected_version_range`` or ``fixed_version_range``.
+    AffectedPackage must contain either ``affected_version_range`` or ``fixed_version_range`` or ``affected_by_commits`` or ``fixed_by_commits``.
     """
 
     package: PackageURL
     affected_version_range: Optional[VersionRange] = None
     fixed_version_range: Optional[VersionRange] = None
+    fixed_by_commits: List[CodeCommitData] = dataclasses.field(default_factory=list)
+    affected_by_commits: List[CodeCommitData] = dataclasses.field(default_factory=list)
 
     def __post_init__(self):
         if self.package.version:
             raise ValueError(f"Affected Package URL {self.package!r} cannot have a version.")
 
-        if not (self.affected_version_range or self.fixed_version_range):
+        if not (
+            self.affected_version_range
+            or self.fixed_version_range
+            or self.affected_by_commits
+            or self.fixed_by_commits
+        ):
             raise ValueError(
                 f"Affected Package {self.package!r} should have either fixed version range or an "
                 "affected version range."
@@ -430,6 +437,8 @@ def _cmp_key(self):
             str(self.package),
             str(self.affected_version_range or ""),
             str(self.fixed_version_range or ""),
+            str(self.affected_by_commits or []),
+            str(self.fixed_by_commits or []),
         )
 
     def to_dict(self):
@@ -443,6 +452,12 @@ def to_dict(self):
             "package": purl_to_dict(self.package),
             "affected_version_range": affected_version_range,
             "fixed_version_range": fixed_version_range,
+            "affected_by_commits": [
+                affected_by_commit.to_dict() for affected_by_commit in self.affected_by_commits
+            ],
+            "fixed_by_commits": [
+                fixed_by_commit.to_dict() for fixed_by_commit in self.fixed_by_commits
+            ],
         }
 
     @classmethod
@@ -454,6 +469,8 @@ def from_dict(cls, affected_pkg: dict):
         fixed_version_range = None
         affected_range = affected_pkg["affected_version_range"]
         fixed_range = affected_pkg["fixed_version_range"]
+        affected_by_commits = affected_pkg.get("affected_by_commits") or []
+        fixed_by_commits = affected_pkg.get("fixed_by_commits") or []
 
         try:
             affected_version_range = VersionRange.from_string(affected_range)
@@ -475,6 +492,13 @@ def from_dict(cls, affected_pkg: dict):
             package=package,
             affected_version_range=affected_version_range,
             fixed_version_range=fixed_version_range,
+            affected_by_commits=[
+                CodeCommitData.from_dict(affected_by_commit)
+                for affected_by_commit in affected_by_commits
+            ],
+            fixed_by_commits=[
+                CodeCommitData.from_dict(fixed_by_commit) for fixed_by_commit in fixed_by_commits
+            ],
         )
 
 
@@ -502,8 +526,6 @@ class AdvisoryData:
     date_published: Optional[datetime.datetime] = None
     weaknesses: List[int] = dataclasses.field(default_factory=list)
     severities: List[VulnerabilitySeverity] = dataclasses.field(default_factory=list)
-    fixed_by_commits: List[CodeCommitData] = dataclasses.field(default_factory=list)
-    affected_by_commits: List[CodeCommitData] = dataclasses.field(default_factory=list)
     url: Optional[str] = None
     original_advisory_text: Optional[str] = None
 
@@ -536,12 +558,6 @@ def to_dict(self):
                 "severities": [sev.to_dict() for sev in self.severities],
                 "date_published": self.date_published.isoformat() if self.date_published else None,
                 "weaknesses": self.weaknesses,
-                "affected_by_commits": [
-                    affected_by_commit.to_dict() for affected_by_commit in self.affected_by_commits
-                ],
-                "fixed_by_commits": [
-                    fixed_by_commit.to_dict() for fixed_by_commit in self.fixed_by_commits
-                ],
                 "url": self.url if self.url else "",
             }
         return {
@@ -602,8 +618,6 @@ class AdvisoryDataV2:
     date_published: Optional[datetime.datetime] = None
     weaknesses: List[int] = dataclasses.field(default_factory=list)
     url: Optional[str] = None
-    fixed_by_commits: List[CodeCommitData] = dataclasses.field(default_factory=list)
-    affected_by_commits: List[CodeCommitData] = dataclasses.field(default_factory=list)
 
     def __post_init__(self):
         if self.date_published and not self.date_published.tzinfo:
@@ -627,12 +641,6 @@ def to_dict(self):
             "references": [ref.to_dict() for ref in self.references],
             "date_published": self.date_published.isoformat() if self.date_published else None,
             "weaknesses": self.weaknesses,
-            "affected_by_commits": [
-                affected_by_commit.to_dict() for affected_by_commit in self.affected_by_commits
-            ],
-            "fixed_by_commits": [
-                fixed_by_commit.to_dict() for fixed_by_commit in self.fixed_by_commits
-            ],
             "url": self.url if self.url else "",
         }
 
@@ -652,14 +660,6 @@ def from_dict(cls, advisory_data):
             if date_published
             else None,
             "weaknesses": advisory_data["weaknesses"],
-            "affected_by_commits": [
-                CodeCommitData.from_dict(affected_by_commit)
-                for affected_by_commit in advisory_data["affected_by_commits"]
-            ],
-            "fixed_by_commits": [
-                CodeCommitData.from_dict(fixed_by_commit)
-                for fixed_by_commit in advisory_data["fixed_by_commits"]
-            ],
             "url": advisory_data.get("url") or None,
         }
         return cls(**transformed)

vulnerabilities/importers/curl.py

@@ -97,7 +97,7 @@ def parse_advisory_data(raw_data) -> AdvisoryData:
         ...     ]
         ... }
         >>> parse_advisory_data(raw_data)
-        AdvisoryData(advisory_id='', aliases=['CVE-2024-2379'], summary='QUIC certificate check bypass with wolfSSL', affected_packages=[AffectedPackage(package=PackageURL(type='generic', namespace='curl.se', name='curl', version=None, qualifiers={}, subpath=None), affected_version_range=GenericVersionRange(constraints=(VersionConstraint(comparator='=', version=SemverVersion(string='8.6.0')),)), fixed_version=SemverVersion(string='8.7.0'))], references=[Reference(reference_id='', reference_type='', url='https://curl.se/docs/CVE-2024-2379.html', severities=[VulnerabilitySeverity(system=Cvssv3ScoringSystem(identifier='cvssv3.1', name='CVSSv3.1 Base Score', url='https://www.first.org/cvss/v3-1/', notes='CVSSv3.1 base score and vector'), value='Low', scoring_elements='', published_at=None, url=None)]), Reference(reference_id='', reference_type='', url='https://hackerone.com/reports/2410774', severities=[])], references_v2=[], date_published=datetime.datetime(2024, 3, 27, 8, 0, tzinfo=datetime.timezone.utc), weaknesses=[297], severities=[], fixed_by_commits=[], affected_by_commits=[], url='https://curl.se/docs/CVE-2024-2379.json', original_advisory_text=None)
+        AdvisoryData(advisory_id='', aliases=['CVE-2024-2379'], summary='QUIC certificate check bypass with wolfSSL', affected_packages=[AffectedPackage(package=PackageURL(type='generic', namespace='curl.se', name='curl', version=None, qualifiers={}, subpath=None), affected_version_range=GenericVersionRange(constraints=(VersionConstraint(comparator='=', version=SemverVersion(string='8.6.0')),)), fixed_version=SemverVersion(string='8.7.0'))], references=[Reference(reference_id='', reference_type='', url='https://curl.se/docs/CVE-2024-2379.html', severities=[VulnerabilitySeverity(system=Cvssv3ScoringSystem(identifier='cvssv3.1', name='CVSSv3.1 Base Score', url='https://www.first.org/cvss/v3-1/', notes='CVSSv3.1 base score and vector'), value='Low', scoring_elements='', published_at=None, url=None)]), Reference(reference_id='', reference_type='', url='https://hackerone.com/reports/2410774', severities=[])], references_v2=[], date_published=datetime.datetime(2024, 3, 27, 8, 0, tzinfo=datetime.timezone.utc), weaknesses=[297], severities=[], url='https://curl.se/docs/CVE-2024-2379.json', original_advisory_text=None)
     """
 
     affected = get_item(raw_data, "affected")[0] if len(get_item(raw_data, "affected")) > 0 else []

vulnerabilities/importers/osv.py

@@ -134,8 +134,6 @@ def parse_advisory_data_v2(
     references = get_references_v2(raw_data=raw_data)
 
     affected_packages = []
-    fixed_by_commits = []
-    affected_by_commits = []
     for affected_pkg in raw_data.get("affected") or []:
         purl = get_affected_purl(affected_pkg=affected_pkg, raw_id=advisory_id)
 
@@ -148,6 +146,8 @@ def parse_advisory_data_v2(
             )
 
         fixed_versions = []
+        fixed_by_commits = []
+        affected_by_commits = []
         fixed_version_range = None
         for fixed_range in affected_pkg.get("ranges") or []:
             fixed_version, (introduced_commits, fixed_commits) = get_fixed_versions_and_commits(
@@ -174,6 +174,8 @@ def parse_advisory_data_v2(
                     package=purl,
                     affected_version_range=affected_version_range,
                     fixed_version_range=fixed_version_range,
+                    affected_by_commits=affected_by_commits,
+                    fixed_by_commits=fixed_by_commits,
                 )
             )
 
@@ -193,8 +195,6 @@ def parse_advisory_data_v2(
         affected_packages=affected_packages,
         date_published=date_published,
         weaknesses=weaknesses,
-        fixed_by_commits=fixed_by_commits,
-        affected_by_commits=affected_by_commits,
         url=advisory_url,
         original_advisory_text=advisory_text or json.dumps(raw_data, indent=2, ensure_ascii=False),
     )

vulnerabilities/pipes/advisory.py

@@ -188,8 +188,6 @@ def insert_advisory_v2(
     severities = get_or_create_advisory_severities(severities=advisory.severities)
     weaknesses = get_or_create_advisory_weaknesses(weaknesses=advisory.weaknesses)
     content_id = compute_content_id(advisory_data=advisory)
-    affected_by_commits = get_or_create_advisory_code_commits(advisory.affected_by_commits)
-    fixed_by_commits = get_or_create_advisory_code_commits(advisory.fixed_by_commits)
 
     try:
         default_data = {
@@ -257,8 +255,12 @@ def insert_advisory_v2(
             impact.affecting_packages.add(*affected_packages_v2)
             impact.fixed_by_packages.add(*fixed_packages_v2)
 
-            impact.affecting_commits.add(*affected_by_commits)
-            impact.fixed_by_commits.add(*fixed_by_commits)
+            affected_commit_v2 = get_or_create_advisory_code_commits(
+                affected_pkg.affected_by_commits
+            )
+            fixed_commit_v2 = get_or_create_advisory_code_commits(affected_pkg.fixed_by_commits)
+            impact.affecting_commits.add(*affected_commit_v2)
+            impact.fixed_by_commits.add(*fixed_commit_v2)
 
     return advisory_obj
 

vulnerabilities/tests/pipelines/v2_importers/test_github_osv_importer_v2.py

@@ -84,7 +84,7 @@ def delete(self):
     assert advisory.original_advisory_text.strip().startswith("{")
     assert advisory.affected_packages
     assert advisory.affected_packages[0].package.type == "pypi"
-    assert advisory.affected_by_commits == [
+    assert advisory.affected_packages[0].affected_by_commits == [
         CodeCommitData(
             commit_hash="4b825dc642cb6eb9a060e54bf8d69288fbee4904",
             vcs_url="https://github.com/aboutcode-org/vulnerablecode",
@@ -101,7 +101,7 @@ def delete(self):
         ),
     ]
 
-    assert advisory.fixed_by_commits == [
+    assert advisory.affected_packages[0].fixed_by_commits == [
         CodeCommitData(
             commit_hash="10081dd502dcfc0953de333fe8afb399db5f2a88",
             vcs_url="https://github.com/aboutcode-org/vulnerablecode",

vulnerabilities/tests/pipes/test_advisory.py

@@ -46,19 +46,6 @@ def setUp(self):
                     affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"),
                 )
             ],
-            references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")],
-            affected_by_commits=[
-                CodeCommitData(
-                    commit_hash="9ff29db8ec3adefefce0d37c3c9b5b2c22e59fac",
-                    vcs_url="https://github.com/aboutcode-org/vulnerablecode",
-                )
-            ],
-            fixed_by_commits=[
-                CodeCommitData(
-                    commit_hash="9ff29db8ec3adefefce0d37c3c9b5b2c22e59fac",
-                    vcs_url="https://github.com/aboutcode-org/vulnerablecode",
-                )
-            ],
             date_published=timezone.now(),
             url="https://test.com",
         )

vulnerabilities/tests/pipes/test_vulnerablecode_importer_pipeline_v2.py

@@ -53,26 +53,38 @@ def dummy_advisory():
                 package=PackageURL.from_string("pkg:npm/foobar"),
                 affected_version_range=VersionRange.from_string("vers:npm/<=1.2.3"),
                 fixed_version_range=VersionRange.from_string("vers:npm/1.2.4"),
+                fixed_by_commits=[
+                    CodeCommitData(
+                        commit_hash="9ff29db8ec3adefefce0d37c3c9b5b2c22e59fac",
+                        vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+                    )
+                ],
+                affected_by_commits=[
+                    CodeCommitData(
+                        commit_hash="ab99939678dc36b3bee0f366493df1aeef521df4",
+                        vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+                    )
+                ],
             ),
             AffectedPackageV2(
                 package=PackageURL.from_string("pkg:npm/foobar"),
                 affected_version_range=VersionRange.from_string("vers:npm/<=3.2.3"),
                 fixed_version_range=VersionRange.from_string("vers:npm/3.2.4"),
+                fixed_by_commits=[
+                    CodeCommitData(
+                        commit_hash="9ff29db8ec3adefefce0d37c3c9b5b2c22e59fac",
+                        vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+                    )
+                ],
+                affected_by_commits=[
+                    CodeCommitData(
+                        commit_hash="ab99939678dc36b3bee0f366493df1aeef521df4",
+                        vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+                    )
+                ],
             ),
         ],
         advisory_id="ADV-123",
-        fixed_by_commits=[
-            CodeCommitData(
-                commit_hash="9ff29db8ec3adefefce0d37c3c9b5b2c22e59fac",
-                vcs_url="https://github.com/aboutcode-org/vulnerablecode",
-            )
-        ],
-        affected_by_commits=[
-            CodeCommitData(
-                commit_hash="ab99939678dc36b3bee0f366493df1aeef521df4",
-                vcs_url="https://github.com/aboutcode-org/vulnerablecode",
-            )
-        ],
         date_published=datetime.now() - timedelta(days=10),
         url="https://example.com/advisory/1",
     )

vulnerabilities/tests/test_data/archlinux/archlinux_advisoryv2-expected.json

@@ -16,7 +16,9 @@
           "subpath": ""
         },
         "affected_version_range": "vers:alpm/2.3.0-1",
-        "fixed_version_range": "vers:alpm/2.4.0-1"
+        "fixed_version_range": "vers:alpm/2.4.0-1",
+        "affected_by_commits": [],
+        "fixed_by_commits": []
       }
     ],
     "references_v2": [
@@ -29,8 +31,6 @@
     "severities": [],
     "date_published": null,
     "weaknesses": [],
-    "affected_by_commits": [],
-    "fixed_by_commits": [],
     "url": "https://security.archlinux.org/AVG-2781.json"
   },
   {
@@ -52,7 +52,9 @@
           "subpath": ""
         },
         "affected_version_range": "vers:alpm/2.36.3-1",
-        "fixed_version_range": "vers:alpm/2.36.4-1"
+        "fixed_version_range": "vers:alpm/2.36.4-1",
+        "affected_by_commits": [],
+        "fixed_by_commits": []
       }
     ],
     "references_v2": [
@@ -65,8 +67,6 @@
     "severities": [],
     "date_published": null,
     "weaknesses": [],
-    "affected_by_commits": [],
-    "fixed_by_commits": [],
     "url": "https://security.archlinux.org/AVG-2780.json"
   },
   {
@@ -87,7 +87,9 @@
           "subpath": ""
         },
         "affected_version_range": "vers:alpm/1.0.6-5",
-        "fixed_version_range": "vers:alpm/1.0.6-6"
+        "fixed_version_range": "vers:alpm/1.0.6-6",
+        "affected_by_commits": [],
+        "fixed_by_commits": []
       }
     ],
     "references_v2": [
@@ -105,8 +107,6 @@
     "severities": [],
     "date_published": null,
     "weaknesses": [],
-    "affected_by_commits": [],
-    "fixed_by_commits": [],
     "url": "https://security.archlinux.org/AVG-4.json"
   }
 ]