Merge branch 'main' into add-advisory-v2-api-endpoint-2224

Author: shivamshrma09

.github/workflows/pypi-release.yml

@@ -21,12 +21,12 @@ on:
 jobs:
   build-pypi-distribs:
     name: Build and publish library to PyPI
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.12
 
@@ -37,7 +37,7 @@ jobs:
         run: python -m build --sdist --wheel --outdir dist/
 
       - name: Upload built archives
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: pypi_archives
           path: dist/*
@@ -47,37 +47,39 @@ jobs:
     name: Create GH release
     needs:
       - build-pypi-distribs
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       - name: Download built archives
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           name: pypi_archives
           path: dist
 
       - name: Create GH release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
-          draft: true
+          draft: false
+          generate_release_notes: true
           files: dist/*
 
 
   create-pypi-release:
     name: Create PyPI release
     needs:
       - create-gh-release
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
+    environment: pypi-publish
+    permissions:
+      id-token: write
 
     steps:
       - name: Download built archives
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           name: pypi_archives
           path: dist
 
       - name: Publish to PyPI
-        if: startsWith(github.ref, 'refs/tags')
-        uses: pypa/gh-action-pypi-publish@master
-        with:
-          password: ${{ secrets.PYPI_API_TOKEN }}
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: pypa/gh-action-pypi-publish@release/v1

CHANGELOG.rst

@@ -13,10 +13,16 @@ Version v37.0.0
 - We have added new models AdvisoryV2, AdvisoryAlias, AdvisoryReference, AdvisorySeverity, AdvisoryWeakness, PackageV2 and CodeFixV2.
 - We are using ``avid`` as an internal advisory ID for uniquely identifying advisories.
 - We have a new route ``/v2`` which only support package search which has information on packages that are reported to be affected or fixing by advisories.
-- This version introduces ``/api/v2/advisories-packages`` which has information on packages that are reported to be affected or fixing by advisories.
+- This version introduces ``/api/v3/packages`` which has information on packages that are reported to be affected or fixing by advisories.
 - Pipeline Dashboard improvements #1920.
 - Throttle API requests based on user permissions #1909.
 - Add pipeline to compute Advisory ToDos #1764
+- Use related advisory severity to calculate exploitibility, weighted severity and risk scores
+- Migrate all importers to use the new advisory models. All new advisories have a unique AVID and all importers will use this AVID as the unique identifier for advisories instead of CVE ID or other identifiers used by the data sources #1881.
+- Handle advisories with same and related data https://github.com/aboutcode-org/vulnerablecode/issues/2099.
+- Add a pipeline for exporting VulnerableCode data to FederatedCode #2110.
+- Plan storing of exploits and EPSS based advisories #2069.
+
 
 Version v36.1.3
 ---------------------

PIPELINES-AVID.rst

@@ -0,0 +1,74 @@
+.. list-table:: Pipeline AVID Mapping
+   :header-rows: 1
+   :widths: 35 65
+
+   * - pipeline name
+     - AVID
+   * - alpine_linux_importer_v2
+     - {package_name}/{distroversion}/{version}/{vulnerability_id}
+   * - aosp_dataset_fix_commits
+     - CVE ID of the record
+   * - apache_httpd_importer_v2
+     - CVE ID of the record
+   * - apache_kafka_importer_v2
+     - CVE ID of the record
+   * - apache_tomcat_importer_v2
+     - {page_id}/{cve_id}
+   * - archlinux_importer_v2
+     - AVG ID of the record
+   * - curl_importer_v2
+     - CURL-CVE ID of the record
+   * - debian_importer_v2
+     - {package_name}/{debian_record_id}
+   * - elixir_security_importer_v2
+     - {package_name}/{file_id}
+   * - epss_importer_v2
+     - CVE ID of the record
+   * - fireeye_importer_v2
+     - {file_id}
+   * - gentoo_importer_v2
+     - GLSA ID of the record
+   * - github_osv_importer_v2
+     - ID of the OSV record
+   * - gitlab_importer_v2
+     - Identifier of the GitLab community advisory record
+   * - istio_importer_v2
+     - ISTIO-SECURITY-<ID>
+   * - mattermost_importer_v2
+     - MMSA-<ID>
+   * - mozilla_importer_v2
+     - MFSA-<ID>
+   * - nginx_importer_v2
+     - First alias of the record
+   * - nodejs_security_wg
+     - NPM-<ID>
+   * - nvd_importer_v2
+     - CVE ID of the record
+   * - openssl_importer_v2
+     - CVE ID of the record
+   * - oss_fuzz_importer_v2
+     - ID of the OSV record
+   * - postgresql_importer_v2
+     - CVE ID of the record
+   * - project-kb-msr-2019_v2
+     - Vulnerability ID of the record
+   * - project-kb-statements_v2
+     - Vulnerability ID of the record
+   * - pypa_importer_v2
+     - ID of the OSV record
+   * - pysec_importer_v2
+     - ID of the OSV record
+   * - redhat_importer_v2
+     - RHSA ID of the record
+   * - retiredotnet_importer_v2
+     - retiredotnet-{file_id}
+   * - ruby_importer_v2
+     - {file_id}
+   * - suse_importer_v2
+     - CVE ID of the record
+   * - ubuntu_osv_importer_v2
+     - ID of the OSV record
+   * - vulnrichment_importer_v2
+     - CVE ID of the record
+   * - xen_importer_v2
+     - XSA-<ID>

requirements.txt

@@ -40,7 +40,7 @@ docutils==0.17.1
 drf-spectacular==0.24.2
 drf-spectacular-sidecar==2022.10.1
 executing==0.8.3
-fetchcode==0.6.0
+fetchcode==0.8.2
 freezegun==1.2.1
 frozenlist==1.3.0
 gitdb==4.0.9

setup.cfg

@@ -91,7 +91,7 @@ install_requires =
     # networking
     GitPython>=3.1.17
     requests>=2.25.1
-    fetchcode>=0.6.0
+    fetchcode>=0.8.2
 
     #pipeline
     aboutcode.pipeline>=0.1.0

vulnerabilities/improvers/__init__.py

@@ -20,6 +20,7 @@
 from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline
 from vulnerabilities.pipelines import remove_duplicate_advisories
 from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees
+from vulnerabilities.pipelines.v2_improvers import compute_advisory_content_hash
 from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2
 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2
 from vulnerabilities.pipelines.v2_improvers import (
@@ -74,5 +75,6 @@
         compute_advisory_todo.ComputeToDo,
         collect_ssvc_trees.CollectSSVCPipeline,
         relate_severities.RelateSeveritiesPipeline,
+        compute_advisory_content_hash.ComputeAdvisoryContentHash,
     ]
 )

vulnerabilities/migrations/0116_advisoryv2_advisory_content_hash.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.2.11 on 2026-03-11 08:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0115_impactedpackageaffecting_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="advisoryv2",
+            name="advisory_content_hash",
+            field=models.CharField(
+                blank=True,
+                help_text="A unique hash computed from the content of the advisory used to identify advisories with the same content.",
+                max_length=64,
+                null=True,
+            ),
+        ),
+    ]

vulnerabilities/models.py

@@ -3010,6 +3010,13 @@ class AdvisoryV2(models.Model):
         help_text="Related advisories that are used to calculate the severity of this advisory.",
     )
 
+    advisory_content_hash = models.CharField(
+        max_length=64,
+        blank=True,
+        null=True,
+        help_text="A unique hash computed from the content of the advisory used to identify advisories with the same content.",
+    )
+
     @property
     def risk_score(self):
         """
@@ -3078,35 +3085,6 @@ def get_aliases(self):
         """
         return self.aliases.all()
 
-    def compute_advisory_content(self):
-        """
-        Compute a unique content hash for an advisory by normalizing its data and hashing it.
-
-        :param advisory: An Advisory object
-        :return: SHA-256 hash digest as content hash
-        """
-        normalized_data = {
-            "summary": normalize_text(self.summary),
-            "impacted_packages": sorted(
-                [impact.to_dict() for impact in self.impacted_packages.all()],
-                key=lambda x: json.dumps(x, sort_keys=True),
-            ),
-            "patches": sorted(
-                [patch.to_patch_data().to_dict() for patch in self.patches.all()],
-                key=lambda x: json.dumps(x, sort_keys=True),
-            ),
-            "severities": sorted(
-                [sev.to_vulnerability_severity_data().to_dict() for sev in self.severities.all()],
-                key=lambda x: (x.get("system"), x.get("value")),
-            ),
-            "weaknesses": normalize_list([weakness.cwe_id for weakness in self.weaknesses.all()]),
-        }
-
-        normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True)
-        content_hash = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest()
-
-        return content_hash
-
     alias = get_aliases
 
 

vulnerabilities/pipelines/enhance_with_exploitdb.py

@@ -78,19 +78,16 @@ def add_exploit(self):
 
 
 def add_vulnerability_exploit(row, logger):
-    vulnerabilities = set()
-
     aliases = row["codes"].split(";") if row["codes"] else []
 
     if not aliases:
         return 0
 
-    for raw_alias in aliases:
-        try:
-            if alias := Alias.objects.get(alias=raw_alias):
-                vulnerabilities.add(alias.vulnerability)
-        except Alias.DoesNotExist:
-            continue
+    vulnerabilities = (
+        Alias.objects.filter(alias__in=aliases, vulnerability__isnull=False)
+        .values_list("vulnerability_id", flat=True)
+        .distinct()
+    )
 
     if not vulnerabilities:
         logger(f"No vulnerability found for aliases {aliases}")
@@ -104,7 +101,7 @@ def add_vulnerability_exploit(row, logger):
         add_exploit_references(row["codes"], row["source_url"], row["file"], vulnerability, logger)
         try:
             Exploit.objects.update_or_create(
-                vulnerability=vulnerability,
+                vulnerability_id=vulnerability,
                 data_source="Exploit-DB",
                 defaults={
                     "date_added": date_added,
@@ -125,7 +122,7 @@ def add_vulnerability_exploit(row, logger):
     return 1
 
 
-def add_exploit_references(ref_id, direct_url, path, vul, logger):
+def add_exploit_references(ref_id, direct_url, path, vul_id, logger):
     url_map = {
         "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}",
         "direct_url": direct_url,
@@ -144,7 +141,7 @@ def add_exploit_references(ref_id, direct_url, path, vul, logger):
 
                 if created:
                     VulnerabilityRelatedReference.objects.get_or_create(
-                        vulnerability=vul,
+                        vulnerability_id=vul_id,
                         reference=ref,
                     )
 

vulnerabilities/pipelines/enhance_with_kev.py

@@ -71,26 +71,29 @@ def add_vulnerability_exploit(kev_vul, logger):
     if not cve_id:
         return 0
 
-    vulnerability = None
-    try:
-        if alias := Alias.objects.get(alias=cve_id):
-            vulnerability = alias.vulnerability
-    except Alias.DoesNotExist:
+    vulnerabilities = (
+        Alias.objects.filter(alias=cve_id, vulnerability__isnull=False)
+        .values_list("vulnerability", flat=True)
+        .distinct()
+    )
+
+    if not vulnerabilities:
         logger(f"No vulnerability found for aliases {cve_id}")
         return 0
 
-    Exploit.objects.update_or_create(
-        vulnerability=vulnerability,
-        data_source="KEV",
-        defaults={
-            "description": kev_vul["shortDescription"],
-            "date_added": kev_vul["dateAdded"],
-            "required_action": kev_vul["requiredAction"],
-            "due_date": kev_vul["dueDate"],
-            "notes": kev_vul["notes"],
-            "known_ransomware_campaign_use": True
-            if kev_vul["knownRansomwareCampaignUse"] == "Known"
-            else False,
-        },
-    )
+    for vulnerability in vulnerabilities:
+        Exploit.objects.update_or_create(
+            vulnerability_id=vulnerability,
+            data_source="KEV",
+            defaults={
+                "description": kev_vul["shortDescription"],
+                "date_added": kev_vul["dateAdded"],
+                "required_action": kev_vul["requiredAction"],
+                "due_date": kev_vul["dueDate"],
+                "notes": kev_vul["notes"],
+                "known_ransomware_campaign_use": True
+                if kev_vul["knownRansomwareCampaignUse"] == "Known"
+                else False,
+            },
+        )
     return 1