Merge pull request #2196 from aboutcode-org/review_v2_pipelines

Review all v2 pipelines

Author: TG1999

CHANGELOG.rst

@@ -13,10 +13,16 @@ Version v37.0.0
 - We have added new models AdvisoryV2, AdvisoryAlias, AdvisoryReference, AdvisorySeverity, AdvisoryWeakness, PackageV2 and CodeFixV2.
 - We are using ``avid`` as an internal advisory ID for uniquely identifying advisories.
 - We have a new route ``/v2`` which only support package search which has information on packages that are reported to be affected or fixing by advisories.
-- This version introduces ``/api/v2/advisories-packages`` which has information on packages that are reported to be affected or fixing by advisories.
+- This version introduces ``/api/v3/packages`` which has information on packages that are reported to be affected or fixing by advisories.
 - Pipeline Dashboard improvements #1920.
 - Throttle API requests based on user permissions #1909.
 - Add pipeline to compute Advisory ToDos #1764
+- Use related advisory severity to calculate exploitibility, weighted severity and risk scores
+- Migrate all importers to use the new advisory models. All new advisories have a unique AVID and all importers will use this AVID as the unique identifier for advisories instead of CVE ID or other identifiers used by the data sources #1881.
+- Handle advisories with same and related data https://github.com/aboutcode-org/vulnerablecode/issues/2099.
+- Add a pipeline for exporting VulnerableCode data to FederatedCode #2110.
+- Plan storing of exploits and EPSS based advisories #2069.
+
 
 Version v36.1.3
 ---------------------

PIPELINES-AVID.rst

@@ -0,0 +1,74 @@
+.. list-table:: Pipeline AVID Mapping
+   :header-rows: 1
+   :widths: 35 65
+
+   * - pipeline name
+     - AVID
+   * - alpine_linux_importer_v2
+     - {package_name}/{distroversion}/{version}/{vulnerability_id}
+   * - aosp_dataset_fix_commits
+     - CVE ID of the record
+   * - apache_httpd_importer_v2
+     - CVE ID of the record
+   * - apache_kafka_importer_v2
+     - CVE ID of the record
+   * - apache_tomcat_importer_v2
+     - {page_id}/{cve_id}
+   * - archlinux_importer_v2
+     - AVG ID of the record
+   * - curl_importer_v2
+     - CURL-CVE ID of the record
+   * - debian_importer_v2
+     - {package_name}/{debian_record_id}
+   * - elixir_security_importer_v2
+     - {package_name}/{file_id}
+   * - epss_importer_v2
+     - CVE ID of the record
+   * - fireeye_importer_v2
+     - {file_id}
+   * - gentoo_importer_v2
+     - GLSA ID of the record
+   * - github_osv_importer_v2
+     - ID of the OSV record
+   * - gitlab_importer_v2
+     - Identifier of the GitLab community advisory record
+   * - istio_importer_v2
+     - ISTIO-SECURITY-<ID>
+   * - mattermost_importer_v2
+     - MMSA-<ID>
+   * - mozilla_importer_v2
+     - MFSA-<ID>
+   * - nginx_importer_v2
+     - First alias of the record
+   * - nodejs_security_wg
+     - NPM-<ID>
+   * - nvd_importer_v2
+     - CVE ID of the record
+   * - openssl_importer_v2
+     - CVE ID of the record
+   * - oss_fuzz_importer_v2
+     - ID of the OSV record
+   * - postgresql_importer_v2
+     - CVE ID of the record
+   * - project-kb-msr-2019_v2
+     - Vulnerability ID of the record
+   * - project-kb-statements_v2
+     - Vulnerability ID of the record
+   * - pypa_importer_v2
+     - ID of the OSV record
+   * - pysec_importer_v2
+     - ID of the OSV record
+   * - redhat_importer_v2
+     - RHSA ID of the record
+   * - retiredotnet_importer_v2
+     - retiredotnet-{file_id}
+   * - ruby_importer_v2
+     - {file_id}
+   * - suse_importer_v2
+     - CVE ID of the record
+   * - ubuntu_osv_importer_v2
+     - ID of the OSV record
+   * - vulnrichment_importer_v2
+     - CVE ID of the record
+   * - xen_importer_v2
+     - XSA-<ID>

vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 import logging
 from pathlib import Path
 from typing import Any
@@ -244,4 +245,5 @@ def load_advisories(
                 references=references,
                 affected_packages=affected_packages,
                 url=url,
+                original_advisory_text=json.dumps(pkg_infos, indent=2, ensure_ascii=False),
             )

vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py

@@ -111,6 +111,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                         summary=advisory_list[0].summary,
                         affected_packages=affected_packages,
                         url=page_url,
+                        original_advisory_text=str(content),
                     )
 
             except Exception as e:

vulnerabilities/pipelines/v2_importers/archlinux_importer.py

@@ -14,10 +14,13 @@
 from packageurl import PackageURL
 from univers.version_range import ArchLinuxVersionRange
 
+from vulnerabilities import severity_systems
 from vulnerabilities.importer import AdvisoryDataV2
 from vulnerabilities.importer import AffectedPackageV2
 from vulnerabilities.importer import ReferenceV2
+from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.severity_systems import SCORING_SYSTEMS
 from vulnerabilities.utils import fetch_response
 
 
@@ -53,7 +56,9 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
     def parse_advisory(self, record) -> AdvisoryDataV2:
         affected_packages = []
         references = []
+        severities = []
         avg_name = record.get("name")
+        severity = record.get("severity")
         aliases = record.get("issues", [])
         aliases.extend(record.get("advisories", []))
         summary = record.get("type", "")
@@ -92,13 +97,26 @@ def parse_advisory(self, record) -> AdvisoryDataV2:
                 )
             )
 
+        if severity not in severity_systems.ARCHLINUX.choices:
+            self.log(f"Unknown severity {severity} for {avg_name}")
+            severity = None
+        if severity:
+            severities = [
+                VulnerabilitySeverity(
+                    system=severity_systems.ARCHLINUX,
+                    value=severity,
+                    url="https://security.archlinux.org/{avg_name}.json",
+                )
+            ]
+
         return AdvisoryDataV2(
             advisory_id=avg_name,
             aliases=aliases,
             summary=summary,
             references=references,
             affected_packages=affected_packages,
+            severities=severities,
             weaknesses=[],
             url=f"https://security.archlinux.org/{avg_name}.json",
-            original_advisory_text=json.dumps(record),
+            original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False),
         )

vulnerabilities/pipelines/v2_importers/debian_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 import re
 from typing import Any
 from typing import Iterable
@@ -171,6 +172,7 @@ def parse(self, pkg_name: str, records: Mapping[str, Any]) -> Iterable[AdvisoryD
                 references=references,
                 weaknesses=weaknesses,
                 url=f"https://security-tracker.debian.org/tracker/{record_identifier}",
+                original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False),
             )
 
 

vulnerabilities/pipelines/v2_importers/epss_importer_v2.py

@@ -83,4 +83,5 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                 severities=[severity],
                 references=[references],
                 url=self.advisory_url,
+                original_advisory_text=",".join(epss_row),
             )

vulnerabilities/pipelines/v2_importers/istio_importer.py

@@ -42,6 +42,7 @@ class IstioImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     spdx_license_expression = "Apache-2.0"
     license_url = "https://github.com/istio/istio.io/blob/master/LICENSE"
     repo_url = "git+https://github.com/istio/istio.io"
+    run_once = True
 
     precedence = 200
 

vulnerabilities/pipelines/v2_importers/mattermost_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 from typing import Iterable
 
 from packageurl import PackageURL
@@ -122,5 +123,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                 summary=details,
                 references=[reference],
                 affected_packages=affected_packages,
+                severities=severities,
                 url=self.url,
+                original_advisory_text=json.dumps(advisory, indent=2, ensure_ascii=False),
             )

vulnerabilities/pipelines/v2_importers/nginx_importer.py

@@ -62,7 +62,7 @@ def collect_advisories(self):
         vulnerability_list = soup.select("li p")
         for vulnerability_info in vulnerability_list:
             ngnix_advisory = parse_advisory_data_from_paragraph(vulnerability_info)
-            yield to_advisory_data(ngnix_advisory)
+            yield to_advisory_data(ngnix_advisory, vulnerability_info)
 
 
 class NginxAdvisory(NamedTuple):
@@ -79,7 +79,7 @@ def to_dict(self):
         return self._asdict()
 
 
-def to_advisory_data(nginx_adv: NginxAdvisory) -> AdvisoryDataV2:
+def to_advisory_data(nginx_adv: NginxAdvisory, vulnerability_info) -> AdvisoryDataV2:
     """
     Return AdvisoryDataV2 from an NginxAdvisory tuple.
     """
@@ -150,6 +150,7 @@ def to_advisory_data(nginx_adv: NginxAdvisory) -> AdvisoryDataV2:
         references=nginx_adv.references,
         patches=nginx_adv.patches,
         url="https://nginx.org/en/security_advisories.html",
+        original_advisory_text=str(vulnerability_info),
     )