Add model changes and support new advisory ingestion

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

vulnerabilities/importer.py

@@ -413,6 +413,7 @@ class AdvisoryDataV2:
     date_published must be aware datetime
     """
 
+    advisory_id: str = ""
     aliases: List[str] = dataclasses.field(default_factory=list)
     summary: Optional[str] = ""
     affected_packages: List[AffectedPackage] = dataclasses.field(default_factory=list)

vulnerabilities/pipelines/__init__.py

@@ -19,10 +19,12 @@
 from aboutcode.pipeline import humanize_time
 
 from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AdvisoryDataV2
 from vulnerabilities.improver import MAX_CONFIDENCE
 from vulnerabilities.models import Advisory
 from vulnerabilities.pipes.advisory import import_advisory
 from vulnerabilities.pipes.advisory import insert_advisory
+from vulnerabilities.pipes.advisory import insert_advisory_v2
 from vulnerabilities.utils import classproperty
 
 module_logger = logging.getLogger(__name__)
@@ -149,12 +151,20 @@ def collect_and_store_advisories(self):
 
         progress = LoopProgress(total_iterations=estimated_advisory_count, logger=self.log)
         for advisory in progress.iter(self.collect_advisories()):
-            if _obj := insert_advisory(
-                advisory=advisory,
-                pipeline_id=self.pipeline_id,
-                logger=self.log,
-            ):
-                collected_advisory_count += 1
+            if isinstance(advisory, AdvisoryData):
+                if _obj := insert_advisory(
+                    advisory=advisory,
+                    pipeline_id=self.pipeline_id,
+                    logger=self.log,
+                ):
+                    collected_advisory_count += 1
+            if isinstance(advisory, AdvisoryDataV2):
+                if _obj := insert_advisory_v2(
+                    advisory=advisory,
+                    pipeline_id=self.pipeline_id,
+                    logger=self.log,
+                ):
+                    collected_advisory_count += 1
 
         self.log(f"Successfully collected {collected_advisory_count:,d} advisories")
 

vulnerabilities/pipelines/alpine_linux_importer.py

@@ -19,6 +19,7 @@
 from univers.versions import AlpineLinuxVersion
 
 from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AdvisoryDataV2
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline
 from vulnerabilities.references import WireSharkReference
@@ -288,3 +289,26 @@ def load_advisories(
                 aliases=aliases,
                 url=url,
             )
+
+            if any(is_cve(alias) for alias in aliases):
+                advisory_id = next((alias for alias in aliases if is_cve(alias)), None)
+                aliases.remove(advisory_id)
+                yield AdvisoryDataV2(
+                    references=references,
+                    affected_packages=affected_packages,
+                    url=url,
+                    advisory_id=advisory_id,
+                    aliases=aliases,
+                )
+
+            else:
+                aliases.sort()
+                advisory_id = aliases[0]
+                aliases = aliases[1:]
+                yield AdvisoryDataV2(
+                    references=references,
+                    affected_packages=affected_packages,
+                    url=url,
+                    advisory_id=advisory_id,
+                    aliases=aliases,
+                )

vulnerabilities/pipes/advisory.py

@@ -18,8 +18,11 @@
 from django.db.models.query import QuerySet
 
 from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AdvisoryDataV2
 from vulnerabilities.improver import MAX_CONFIDENCE
 from vulnerabilities.models import Advisory
+from vulnerabilities.models import AdvisoryAlias
+from vulnerabilities.models import AdvisoryV2
 from vulnerabilities.models import AffectedByPackageRelatedVulnerability
 from vulnerabilities.models import Alias
 from vulnerabilities.models import FixingPackageRelatedVulnerability
@@ -36,6 +39,12 @@ def get_or_create_aliases(aliases: List) -> QuerySet:
     return Alias.objects.filter(alias__in=aliases)
 
 
+def get_or_create_aliases_v2(aliases: List) -> QuerySet:
+    for alias in aliases:
+        AdvisoryAlias.objects.get_or_create(alias=alias)
+    return AdvisoryAlias.objects.filter(alias__in=aliases)
+
+
 def insert_advisory(advisory: AdvisoryData, pipeline_id: str, logger: Callable = None):
     from vulnerabilities.utils import compute_content_id
 
@@ -74,6 +83,42 @@ def insert_advisory(advisory: AdvisoryData, pipeline_id: str, logger: Callable =
     return advisory_obj
 
 
+def insert_advisory_v2(advisory: AdvisoryDataV2, pipeline_id: str, logger: Callable = None):
+    from vulnerabilities.utils import compute_content_id
+
+    advisory_obj = None
+    aliases = get_or_create_aliases_v2(aliases=advisory.aliases)
+    content_id = compute_content_id(advisory_data=advisory)
+    try:
+        default_data = {
+            "summary": advisory.summary,
+            "date_published": advisory.date_published,
+            "created_by": pipeline_id,
+            "date_collected": datetime.now(timezone.utc),
+            "advisory_id": advisory.advisory_id,
+        }
+
+        advisory_obj, _ = AdvisoryV2.objects.get_or_create(
+            unique_content_id=content_id,
+            url=advisory.url,
+            defaults=default_data,
+        )
+        advisory_obj.aliases.add(*aliases)
+    except Advisory.MultipleObjectsReturned:
+        logger.error(
+            f"Multiple Advisories returned: unique_content_id: {content_id}, url: {advisory.url}, advisory: {advisory!r}"
+        )
+        raise
+    except Exception as e:
+        if logger:
+            logger(
+                f"Error while processing {advisory!r} with aliases {advisory.aliases!r}: {e!r} \n {traceback_format_exc()}",
+                level=logging.ERROR,
+            )
+
+    return advisory_obj
+
+
 @transaction.atomic
 def import_advisory(
     advisory: Advisory,

vulnerabilities/utils.py

@@ -39,7 +39,7 @@
 from univers.version_range import NginxVersionRange
 from univers.version_range import VersionRange
 
-from aboutcode.hashid import build_vcid  # NOQA
+from aboutcode.hashid import build_vcid
 
 logger = logging.getLogger(__name__)
 
@@ -595,6 +595,7 @@ def compute_content_id(advisory_data):
 
     # Normalize fields
     from vulnerabilities.importer import AdvisoryData
+    from vulnerabilities.importer import AdvisoryDataV2
     from vulnerabilities.models import Advisory
 
     if isinstance(advisory_data, Advisory):
@@ -609,7 +610,7 @@ def compute_content_id(advisory_data):
         }
         normalized_data["url"] = advisory_data.url
 
-    elif isinstance(advisory_data, AdvisoryData):
+    elif isinstance(advisory_data, AdvisoryData) or isinstance(advisory_data, AdvisoryDataV2):
         normalized_data = {
             "aliases": normalize_list(advisory_data.aliases),
             "summary": normalize_text(advisory_data.summary),