Add github pipeline

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

vulnerabilities/importers/__init__.py

@@ -43,10 +43,12 @@
 from vulnerabilities.pipelines import nvd_importer
 from vulnerabilities.pipelines import pypa_importer
 from vulnerabilities.pipelines import pysec_importer
+from vulnerabilities.pipelines.v2_importers import github_importer as github_importer_v2
 from vulnerabilities.pipelines.v2_importers import nvd_importer as nvd_importer_v2
 
 IMPORTERS_REGISTRY = [
     nvd_importer_v2.NVDImporterPipeline,
+    github_importer_v2.GitHubAPIImporterPipeline,
     nvd_importer.NVDImporterPipeline,
     github_importer.GitHubAPIImporterPipeline,
     gitlab_importer.GitLabImporterPipeline,

vulnerabilities/models.py

@@ -164,6 +164,7 @@ def with_package_counts(self):
         )
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class VulnerabilitySeverity(models.Model):
     url = models.URLField(
         max_length=1024,
@@ -203,6 +204,7 @@ class Meta:
         ordering = ["url", "scoring_system", "value"]
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class VulnerabilityStatusType(models.IntegerChoices):
     """List of vulnerability statuses."""
 
@@ -211,6 +213,7 @@ class VulnerabilityStatusType(models.IntegerChoices):
     INVALID = 3, "Invalid"
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class Vulnerability(models.Model):
     """
     A software vulnerability with a unique identifier and alternate ``aliases``.
@@ -503,6 +506,7 @@ def get_cwes(self):
 Database.get_cwes = get_cwes
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class Weakness(models.Model):
     """
     A Common Weakness Enumeration model
@@ -549,6 +553,7 @@ def to_dict(self):
         return {"cwe_id": self.cwe_id, "name": self.name, "description": self.description}
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class VulnerabilityReferenceQuerySet(BaseQuerySet):
     def for_cpe(self):
         """
@@ -557,6 +562,7 @@ def for_cpe(self):
         return self.filter(reference_id__startswith="cpe")
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class VulnerabilityReference(models.Model):
     """
     A reference to a vulnerability such as a security advisory from a Linux distribution or language
@@ -614,6 +620,7 @@ def is_cpe(self):
         return self.reference_id.startswith("cpe")
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class VulnerabilityRelatedReference(models.Model):
     """
     A reference related to a vulnerability.
@@ -634,6 +641,7 @@ class Meta:
         ordering = ["vulnerability", "reference"]
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class PackageQuerySet(BaseQuerySet, PackageURLQuerySet):
     def get_fixed_by_package_versions(self, purl: PackageURL, fix=True):
         """
@@ -800,6 +808,7 @@ def get_purl_query_lookups(purl):
     return purl_to_dict(plain_purl, with_empty=False)
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class Package(PackageURLMixin):
     """
     A software package with related vulnerabilities.
@@ -1132,6 +1141,7 @@ def affecting_vulns(self):
         )
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class PackageRelatedVulnerabilityBase(models.Model):
     """
     Abstract base class for package-vulnerability relations.
@@ -1228,11 +1238,13 @@ def add_package_vulnerability_changelog(self, advisory):
         )
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class FixingPackageRelatedVulnerability(PackageRelatedVulnerabilityBase):
     class Meta(PackageRelatedVulnerabilityBase.Meta):
         verbose_name_plural = "Fixing Package Related Vulnerabilities"
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class AffectedByPackageRelatedVulnerability(PackageRelatedVulnerabilityBase):
 
     severities = models.ManyToManyField(
@@ -1254,6 +1266,7 @@ def for_cve(self):
         return self.filter(alias__startswith="CVE")
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class Alias(models.Model):
     """
     An alias is a unique vulnerability identifier in some database, such as
@@ -1311,6 +1324,7 @@ class AdvisoryQuerySet(BaseQuerySet):
     pass
 
 
+# FIXME: Remove when migration from Vulnerability to Advisory is completed
 class Advisory(models.Model):
     """
     An advisory represents data directly obtained from upstream transformed
@@ -2055,6 +2069,8 @@ class AdvisoryV2(models.Model):
         help_text="A list of packages that are reported by this advisory.",
     )
 
+    # TODO: Add Advisory Status
+
     objects = AdvisoryQuerySet.as_manager()
 
     class Meta:
@@ -2083,6 +2099,16 @@ def to_advisory_data(self) -> "AdvisoryDataV2":
         )
 
 
+class PackageQuerySetV2(BaseQuerySet, PackageURLQuerySet):
+    def get_or_create_from_purl(self, purl: Union[PackageURL, str]):
+        """
+        Return a new or existing Package given a ``purl`` PackageURL object or PURL string.
+        """
+        package, is_created = PackageV2.objects.get_or_create(**purl_to_dict(purl=purl))
+
+        return package, is_created
+
+
 class PackageV2(PackageURLMixin):
     """
     A software package with related vulnerabilities.
@@ -2155,6 +2181,8 @@ def save(self, *args, **kwargs):
         self.plain_package_url = str(plain_purl)
         super().save(*args, **kwargs)
 
+    objects = PackageQuerySetV2.as_manager()
+
     @property
     def calculate_version_rank(self):
         """
@@ -2177,11 +2205,3 @@ def calculate_version_rank(self):
                 package.version_rank = rank
             Package.objects.bulk_update(sorted_packages, fields=["version_rank"])
         return self.version_rank
-
-    def get_or_create_from_purl(self, purl: Union[PackageURL, str]):
-        """
-        Return a new or existing Package given a ``purl`` PackageURL object or PURL string.
-        """
-        package, is_created = Package.objects.get_or_create(**purl_to_dict(purl=purl))
-
-        return package, is_created

vulnerabilities/pipelines/__init__.py

@@ -13,19 +13,29 @@
 from timeit import default_timer as timer
 from traceback import format_exc as traceback_format_exc
 from typing import Iterable
+from typing import List
+from typing import Optional
 
 from aboutcode.pipeline import BasePipeline
 from aboutcode.pipeline import LoopProgress
 from aboutcode.pipeline import humanize_time
+from fetchcode import package_versions
+from packageurl import PackageURL
 
 from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import UnMergeablePackageError
 from vulnerabilities.improver import MAX_CONFIDENCE
 from vulnerabilities.models import Advisory
 from vulnerabilities.models import PackageV2
 from vulnerabilities.pipes.advisory import import_advisory
 from vulnerabilities.pipes.advisory import insert_advisory
 from vulnerabilities.pipes.advisory import insert_advisory_v2
+from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage
 from vulnerabilities.utils import classproperty
+from vulnerabilities.utils import get_affected_packages_by_patched_package
+from vulnerabilities.utils import nearest_patched_package
+from vulnerabilities.utils import resolve_version_range
 
 module_logger = logging.getLogger(__name__)
 
@@ -210,13 +220,12 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
     repo_url = None
     importer_name = None
     advisory_confidence = MAX_CONFIDENCE
+    ignorable_versions = []
+    unfurl_version_ranges = False
 
     @classmethod
     def steps(cls):
-        return (
-            cls.collect_and_store_advisories,
-            cls.import_new_advisories,
-        )
+        return (cls.collect_and_store_advisories,)
 
     def collect_advisories(self) -> Iterable[AdvisoryData]:
         """
@@ -270,6 +279,15 @@ def get_advisory_packages(self, advisory_data: AdvisoryData) -> list:
             affected_purls.extend(package_affected_purls)
             fixed_purls.extend(package_fixed_purls)
 
+
+        if self.unfurl_version_ranges:
+            vulnerable_pvs, fixed_pvs = self.get_impacted_packages(
+                affected_packages=advisory_data.affected_packages,
+                advisory_date_published=advisory_data.date_published,
+            )
+            affected_purls.extend(vulnerable_pvs)
+            fixed_purls.extend(fixed_pvs)
+
         vulnerable_packages = []
         fixed_packages = []
 
@@ -282,3 +300,143 @@ def get_advisory_packages(self, advisory_data: AdvisoryData) -> list:
             fixed_packages.append(fixed_package)
 
         return vulnerable_packages, fixed_packages
+
+    def get_published_package_versions(
+        self, package_url: PackageURL, until: Optional[datetime] = None
+    ) -> List[str]:
+        """
+        Return a list of versions published before `until` for the `package_url`
+        """
+        versions = package_versions.versions(str(package_url))
+        versions_before_until = []
+        for version in versions or []:
+            if until and version.release_date and version.release_date > until:
+                continue
+            versions_before_until.append(version.value)
+
+        return versions_before_until
+
+    def get_impacted_packages(self, affected_packages, advisory_date_published):
+        """
+        Return a tuple of lists of affected and fixed PackageURLs
+        """
+        if not affected_packages:
+            return [], []
+
+        mergable = True
+
+        # TODO: We should never had the exception in first place
+        try:
+            purl, affected_version_ranges, fixed_versions = AffectedPackage.merge(affected_packages)
+        except UnMergeablePackageError:
+            self.log(f"Cannot merge with different purls {affected_packages!r}", logging.ERROR)
+            mergable = False
+
+        if not mergable:
+            for affected_package in affected_packages:
+                purl = affected_package.package
+                affected_version_range = affected_package.affected_version_range
+                fixed_version = affected_package.fixed_version
+                pkg_type = purl.type
+                pkg_namespace = purl.namespace
+                pkg_name = purl.name
+                if not affected_version_range and fixed_version:
+                    # FIXME: Handle the receving end to address the concern of looping the data
+                    return [], [
+                        PackageURL(
+                            type=pkg_type,
+                            namespace=pkg_namespace,
+                            name=pkg_name,
+                            version=str(fixed_version),
+                        )
+                    ]
+                else:
+                    # FIXME: Handle the receving end to address the concern of looping the data
+                    valid_versions = self.get_published_package_versions(
+                        package_url=purl, until=advisory_date_published
+                    )
+                    return self.resolve_package_versions(
+                        affected_version_range=affected_version_range,
+                        pkg_type=pkg_type,
+                        pkg_namespace=pkg_namespace,
+                        pkg_name=pkg_name,
+                        valid_versions=valid_versions,
+                    )
+
+        else:
+            pkg_type = purl.type
+            pkg_namespace = purl.namespace
+            pkg_name = purl.name
+            pkg_qualifiers = purl.qualifiers
+            fixed_purls = [
+                PackageURL(
+                    type=pkg_type,
+                    namespace=pkg_namespace,
+                    name=pkg_name,
+                    version=str(version),
+                    qualifiers=pkg_qualifiers,
+                )
+                for version in fixed_versions
+            ]
+            if not affected_version_ranges:
+                return [], fixed_purls
+            else:
+                valid_versions = self.get_published_package_versions(
+                    package_url=purl, until=advisory_date_published
+                )
+                for affected_version_range in affected_version_ranges:
+                    return self.resolve_package_versions(
+                        affected_version_range=affected_version_range,
+                        pkg_type=pkg_type,
+                        pkg_namespace=pkg_namespace,
+                        pkg_name=pkg_name,
+                        valid_versions=valid_versions,
+                    )
+
+    def resolve_package_versions(
+        self,
+        affected_version_range,
+        pkg_type,
+        pkg_namespace,
+        pkg_name,
+        valid_versions,
+    ):
+        """
+        Return a tuple of lists of ``affected_packages`` and ``fixed_packages`` PackageURL for the given `affected_version_range` and `valid_versions`.
+
+        ``valid_versions`` are the valid version listed on the package registry for that package
+        
+        """
+        aff_vers, unaff_vers = resolve_version_range(
+            affected_version_range=affected_version_range,
+            ignorable_versions=self.ignorable_versions,
+            package_versions=valid_versions,
+        )
+
+        affected_purls = list(
+            self.expand_verion_range_to_purls(pkg_type, pkg_namespace, pkg_name, aff_vers)
+        )
+
+        unaffected_purls = list(
+            self.expand_verion_range_to_purls(pkg_type, pkg_namespace, pkg_name, unaff_vers)
+        )
+
+        fixed_packages = []
+        affected_packages = []
+
+        patched_packages = nearest_patched_package(
+                vulnerable_packages=affected_purls, resolved_packages=unaffected_purls
+            )
+
+        for (fixed_package, affected_purls,) in get_affected_packages_by_patched_package(
+            patched_packages
+        ).items():
+            if fixed_package:
+                fixed_packages.append(fixed_package)
+            affected_packages.extend(affected_purls)
+
+        return affected_packages, fixed_packages
+
+    def expand_verion_range_to_purls(self, pkg_type, pkg_namespace, pkg_name, versions):
+        for version in versions:
+            yield PackageURL(type=pkg_type, namespace=pkg_namespace, name=pkg_name, version=version)