Change architecture to import and marking advisories as usable

CHANGELOG.rst

@@ -5,7 +5,12 @@ next release
 ---------------------
 
 - WARNING: Vulnerablecode V1 API and UI has stopped supporting Ubuntu OVAL advisories, please shift to V3 API for new Ubuntu advisories.
-- Add attribute ``pipeline_id`` to AdvisoryV2 to track the pipeline that created the advisory, also rename existing ``datasource_id`` and AVIDs. 
+- WARNING: We will deprecate improver pipelines for calculating package version rank and grouping advisories for packages.
+- Add attribute ``pipeline_id`` to AdvisoryV2 to track the pipeline that created the advisory, also rename existing ``datasource_id`` and AVIDs.
+- We will group advisories for packages and calculate package risk score and advisory risk score only when an advisory is completely unfurled,
+  this will improve consistency of the data. In future if we change our grouping approach we have to make a migration to mark all advisories 
+  as not unfurled, so they can be goruped again with the new algo.
+
 
 Version v38.6.0
 ---------------------

vulnerabilities/importers/__init__.py

@@ -120,39 +120,8 @@
         ubuntu_osv_importer_v2.UbuntuOSVImporterPipeline,
         alpine_linux_importer_v2.AlpineLinuxImporterPipeline,
         linux_kernel_importer_v2.LinuxKernelPipeline,
-        github_importer.GitHubAPIImporterPipeline,
-        gitlab_importer.GitLabImporterPipeline,
-        github_osv.GithubOSVImporter,
-        pypa_importer.PyPaImporterPipeline,
-        npm_importer.NpmImporterPipeline,
-        nginx_importer.NginxImporterPipeline,
-        pysec_importer.PyPIImporterPipeline,
-        fireeye_importer_v2.FireeyeImporterPipeline,
-        apache_tomcat.ApacheTomcatImporter,
-        postgresql.PostgreSQLImporter,
-        debian.DebianImporter,
-        curl.CurlImporter,
-        epss.EPSSImporter,
-        vulnrichment.VulnrichImporter,
-        alpine_linux_importer.AlpineLinuxImporterPipeline,
-        ruby.RubyImporter,
-        apache_kafka.ApacheKafkaImporter,
         openssl_importer_v2.OpenSSLImporterPipeline,
-        redhat.RedhatImporter,
-        archlinux.ArchlinuxImporter,
-        debian_oval.DebianOvalImporter,
-        retiredotnet.RetireDotnetImporter,
-        apache_httpd.ApacheHTTPDImporter,
-        mozilla.MozillaImporter,
-        gentoo.GentooImporter,
-        istio.IstioImporter,
-        project_kb_msr2019.ProjectKBMSRImporter,
-        suse_scores.SUSESeverityScoreImporter,
-        elixir_security.ElixirSecurityImporter,
-        xen.XenImporter,
-        ubuntu_usn.UbuntuUSNImporter,
-        fireeye.FireyeImporter,
-        oss_fuzz.OSSFuzzImporter,
+        fireeye_importer_v2.FireeyeImporterPipeline,
         collect_fix_commits_v2.CollectLinuxFixCommitsPipeline,
         collect_fix_commits_v2.CollectBusyBoxFixCommitsPipeline,
         collect_fix_commits_v2.CollectNginxFixCommitsPipeline,
@@ -190,6 +159,37 @@
         collect_fix_commits_v2.CollectGitFixCommitsPipeline,
         collect_fix_commits_v2.CollectJenkinsFixCommitsPipeline,
         collect_fix_commits_v2.CollectGitlabFixCommitsPipeline,
+        github_importer.GitHubAPIImporterPipeline,
+        gitlab_importer.GitLabImporterPipeline,
+        github_osv.GithubOSVImporter,
+        pypa_importer.PyPaImporterPipeline,
+        npm_importer.NpmImporterPipeline,
+        nginx_importer.NginxImporterPipeline,
+        pysec_importer.PyPIImporterPipeline,
+        apache_tomcat.ApacheTomcatImporter,
+        postgresql.PostgreSQLImporter,
+        debian.DebianImporter,
+        curl.CurlImporter,
+        epss.EPSSImporter,
+        vulnrichment.VulnrichImporter,
+        alpine_linux_importer.AlpineLinuxImporterPipeline,
+        apache_kafka.ApacheKafkaImporter,
+        ruby.RubyImporter,
+        redhat.RedhatImporter,
+        archlinux.ArchlinuxImporter,
+        debian_oval.DebianOvalImporter,
+        retiredotnet.RetireDotnetImporter,
+        apache_httpd.ApacheHTTPDImporter,
+        mozilla.MozillaImporter,
+        gentoo.GentooImporter,
+        istio.IstioImporter,
+        project_kb_msr2019.ProjectKBMSRImporter,
+        suse_scores.SUSESeverityScoreImporter,
+        elixir_security.ElixirSecurityImporter,
+        xen.XenImporter,
+        ubuntu_usn.UbuntuUSNImporter,
+        fireeye.FireyeImporter,
+        oss_fuzz.OSSFuzzImporter,
     ]
 )
 

vulnerabilities/improvers/__init__.py

@@ -9,37 +9,45 @@
 
 from vulnerabilities.improvers import valid_versions
 from vulnerabilities.improvers import vulnerability_status
-from vulnerabilities.pipelines import add_cvss31_to_CVEs
 from vulnerabilities.pipelines import compute_package_risk
 from vulnerabilities.pipelines import compute_package_version_rank
 from vulnerabilities.pipelines import enhance_with_exploitdb
 from vulnerabilities.pipelines import enhance_with_kev
 from vulnerabilities.pipelines import enhance_with_metasploit
 from vulnerabilities.pipelines import flag_ghost_packages
 from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline
-from vulnerabilities.pipelines import remove_duplicate_advisories
 from vulnerabilities.pipelines.v2_improvers import archive_urls
 from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees
 from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2
 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2
-from vulnerabilities.pipelines.v2_improvers import (
-    computer_package_version_rank as compute_version_rank_v2,
-)
 from vulnerabilities.pipelines.v2_improvers import enhance_with_exploitdb as exploitdb_v2
 from vulnerabilities.pipelines.v2_improvers import enhance_with_github_poc
 from vulnerabilities.pipelines.v2_improvers import enhance_with_kev as enhance_with_kev_v2
 from vulnerabilities.pipelines.v2_improvers import (
     enhance_with_metasploit as enhance_with_metasploit_v2,
 )
 from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2
-from vulnerabilities.pipelines.v2_improvers import group_advisories_for_packages
+from vulnerabilities.pipelines.v2_improvers import mark_unfurl_version_range
 from vulnerabilities.pipelines.v2_improvers import reference_collect_commits
 from vulnerabilities.pipelines.v2_improvers import relate_severities
 from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2
 from vulnerabilities.utils import create_registry
 
 IMPROVERS_REGISTRY = create_registry(
     [
+        exploitdb_v2.ExploitDBImproverPipeline,
+        enhance_with_kev_v2.VulnerabilityKevPipeline,
+        flag_ghost_packages_v2.FlagGhostPackagePipeline,
+        enhance_with_metasploit_v2.MetasploitImproverPipeline,
+        compute_package_risk_v2.ComputePackageRiskPipeline,
+        unfurl_version_range_v2.UnfurlVersionRangePipeline,
+        collect_ssvc_trees.CollectSSVCPipeline,
+        relate_severities.RelateSeveritiesPipeline,
+        archive_urls.ArchiveImproverPipeline,
+        compute_advisory_todo_v2.ComputeToDo,
+        reference_collect_commits.CollectReferencesFixCommitsPipeline,
+        enhance_with_github_poc.GithubPocsImproverPipeline,
+        mark_unfurl_version_range.MarkUnfurlVersionRangePipeline,
         valid_versions.GitHubBasicImprover,
         valid_versions.GitLabBasicImprover,
         valid_versions.NginxBasicImprover,
@@ -62,22 +70,6 @@
         enhance_with_exploitdb.ExploitDBImproverPipeline,
         compute_package_risk.ComputePackageRiskPipeline,
         compute_package_version_rank.ComputeVersionRankPipeline,
-        add_cvss31_to_CVEs.CVEAdvisoryMappingPipeline,
-        remove_duplicate_advisories.RemoveDuplicateAdvisoriesPipeline,
         populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline,
-        exploitdb_v2.ExploitDBImproverPipeline,
-        enhance_with_kev_v2.VulnerabilityKevPipeline,
-        flag_ghost_packages_v2.FlagGhostPackagePipeline,
-        enhance_with_metasploit_v2.MetasploitImproverPipeline,
-        compute_package_risk_v2.ComputePackageRiskPipeline,
-        compute_version_rank_v2.ComputeVersionRankPipeline,
-        unfurl_version_range_v2.UnfurlVersionRangePipeline,
-        collect_ssvc_trees.CollectSSVCPipeline,
-        relate_severities.RelateSeveritiesPipeline,
-        archive_urls.ArchiveImproverPipeline,
-        group_advisories_for_packages.GroupAdvisoriesForPackages,
-        compute_advisory_todo_v2.ComputeToDo,
-        reference_collect_commits.CollectReferencesFixCommitsPipeline,
-        enhance_with_github_poc.GithubPocsImproverPipeline,
     ]
 )

vulnerabilities/migrations/0134_advisoryv2__all_impacts_unfurled_and_more.py

@@ -0,0 +1,66 @@
+# Generated by Django 5.2.11 on 2026-06-01 19:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0133_alter_advisorytodov2_issue_detail"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="advisoryv2",
+            name="_all_impacts_unfurled",
+            field=models.BooleanField(
+                default=False,
+                help_text="Indicates whether all impacts for this advisory have been unfurled.",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="advisory_id",
+            field=models.CharField(
+                db_index=True,
+                help_text="An advisory is a unique vulnerability identifier in some database, such as PYSEC-2020-2233",
+                max_length=200,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="avid",
+            field=models.CharField(
+                help_text="Unique ID for the datasource used for this advisory .e.g.: pysec_importer_v2/PYSEC-2020-2233",
+                max_length=250,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="datasource_id",
+            field=models.CharField(
+                db_index=True,
+                help_text="Unique ID for the datasource used for this advisory .e.g.: nginx",
+                max_length=50,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="pipeline_id",
+            field=models.CharField(
+                db_index=True,
+                help_text="Unique ID for the pipeline used for this advisory .e.g.: nginx_importer_v2",
+                max_length=50,
+            ),
+        ),
+        migrations.AlterUniqueTogether(
+            name="advisoryset",
+            unique_together={("package", "relation_type", "primary_advisory")},
+        ),
+        migrations.AddIndex(
+            model_name="impactedpackage",
+            index=models.Index(
+                fields=["advisory", "last_range_unfurl_at"], name="vulnerabili_advisor_1e3414_idx"
+            ),
+        ),
+    ]

vulnerabilities/migrations/0135_advisoryv2__all_impacts_unfurled_successfully_and_more.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.2.11 on 2026-06-03 12:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0134_advisoryv2__all_impacts_unfurled_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="advisoryv2",
+            name="_all_impacts_unfurled_successfully",
+            field=models.BooleanField(
+                default=False,
+                help_text="Indicates whether all impacts for this advisory have been unfurled successfully.",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="advisoryv2",
+            index=models.Index(
+                fields=["_all_impacts_unfurled", "id"], name="advisory_unfurled_idx"
+            ),
+        ),
+    ]

vulnerabilities/migrations/0136_alter_advisoryv2__all_impacts_unfurled_and_more.py

@@ -0,0 +1,61 @@
+# Generated by Django 5.2.11 on 2026-06-05 05:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0135_advisoryv2__all_impacts_unfurled_successfully_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="_all_impacts_unfurled",
+            field=models.BooleanField(
+                db_index=True,
+                default=False,
+                help_text="Indicates whether all impacts for this advisory have been unfurled.",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="_all_impacts_unfurled_successfully",
+            field=models.BooleanField(
+                db_index=True,
+                default=False,
+                help_text="Indicates whether all impacts for this advisory have been unfurled successfully.",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="advisoryv2",
+            index=models.Index(
+                fields=["is_latest", "_all_impacts_unfurled"], name="vulnerabili_is_late_5c48b4_idx"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="impactedpackageaffecting",
+            index=models.Index(
+                fields=["package", "impacted_package"], name="vulnerabili_package_22fb39_idx"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="impactedpackageaffecting",
+            index=models.Index(
+                fields=["impacted_package", "package"], name="vulnerabili_impacte_1904b9_idx"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="impactedpackagefixedby",
+            index=models.Index(
+                fields=["package", "impacted_package"], name="vulnerabili_package_f6588f_idx"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="impactedpackagefixedby",
+            index=models.Index(
+                fields=["impacted_package", "package"], name="vulnerabili_impacte_390971_idx"
+            ),
+        ),
+    ]

vulnerabilities/migrations/0137_alter_advisoryset_relation_type_and_more.py

@@ -0,0 +1,28 @@
+# Generated by Django 5.2.11 on 2026-06-08 06:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0136_alter_advisoryv2__all_impacts_unfurled_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="advisoryset",
+            name="relation_type",
+            field=models.CharField(
+                choices=[("affecting", "Affecting"), ("fixing", "Fixing")],
+                db_index=True,
+                max_length=20,
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="advisoryset",
+            index=models.Index(
+                fields=["package", "relation_type"], name="vulnerabili_package_13a5a3_idx"
+            ),
+        ),
+    ]