diff --git a/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py b/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py index d82a7f3d6..c38b37bfa 100644 --- a/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py +++ b/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py @@ -45,7 +45,7 @@ def steps(cls): cls.fetch_federation_config, cls.clone_federation_repository, cls.load_checkpoint, - cls.publish_package_related_advisories, + # cls.publish_package_related_advisories, cls.publish_advisories, cls.save_checkpoint, cls.delete_working_dir, @@ -150,7 +150,7 @@ def publish_advisories(self): batch_size = 4000 chunk_size = 1000 files_to_commit = set() - advisory_qs = export.advisory_prefetched_qs(self.checkpoint) + advisory_qs = export.advisory_prefetched_qs() advisory_count = advisory_qs.count() self.log(f"Exporting {advisory_count} advisory.") diff --git a/vulnerabilities/pipes/export.py b/vulnerabilities/pipes/export.py index ccef8d082..4fd0f3b18 100644 --- a/vulnerabilities/pipes/export.py +++ b/vulnerabilities/pipes/export.py @@ -5,7 +5,6 @@ # See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # - import saneyaml from django.db.models import Prefetch @@ -13,6 +12,7 @@ from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import ImpactedPackageAffecting from vulnerabilities.models import ImpactedPackageFixedBy +from vulnerabilities.models import PackageCommitPatch from vulnerabilities.models import PackageV2 # Note! @@ -94,17 +94,49 @@ def get_package_related_advisory(packages): return package.package_url, package_vulnerabilities -def advisory_prefetched_qs(checkpoint): - qs = AdvisoryV2.objects.order_by("date_collected").prefetch_related( - "impacted_packages", - "aliases", - "references", - "severities", - "weaknesses", +def advisory_prefetched_qs(): + return ( + AdvisoryV2.objects.order_by("date_collected") + .filter(is_latest=True) + .exclude(datasource_id="epss") + .prefetch_related( + "aliases", + "references", + "severities", + "weaknesses", + "exploits", + Prefetch( + "impacted_packages", + queryset=ImpactedPackage.objects.prefetch_related( + Prefetch( + "affecting_packages", + queryset=PackageV2.objects.only("package_url", "is_ghost"), + ), + Prefetch( + "fixed_by_packages", + queryset=PackageV2.objects.only("package_url", "is_ghost"), + ), + Prefetch( + "introduced_by_package_commit_patches", + queryset=PackageCommitPatch.objects.only( + "commit_hash", + "vcs_url", + "patch_text", + ), + ), + Prefetch( + "fixed_by_package_commit_patches", + queryset=PackageCommitPatch.objects.only( + "commit_hash", + "vcs_url", + "patch_text", + ), + ), + ), + ), + ) ) - return qs.filter(date_collected__gte=checkpoint) if checkpoint else qs - def serialize_severity(sev): return { @@ -129,22 +161,44 @@ def serialize_commit_patches(patches): { "vcs_url": p.vcs_url, "commit": p.commit_hash, + "patch_text": p.patch_text, } for p in patches.all() ] +def serialize_advisory_exploit(exploit): + return { + "description": exploit.description, + "required_action": exploit.required_action, + "due_date": exploit.due_date, + "notes": exploit.notes, + "known_ransomware_campaign_use": exploit.known_ransomware_campaign_use, + "source_date_published": exploit.source_date_published, + "exploit_type": exploit.exploit_type, + "platform": exploit.platform, + "source_date_updated": exploit.source_date_updated, + "data_source": exploit.data_source, + "source_url": exploit.source_url, + "record_id": exploit.record_id, + } + + def serialize_advisory(advisory): """Return a plain data mapping serialized from advisory object.""" aliases = sorted([a.alias for a in advisory.aliases.all()]) severities = [serialize_severity(sev) for sev in advisory.severities.all()] weaknesses = [wkns.cwe for wkns in advisory.weaknesses.all()] references = [serialize_references(ref) for ref in advisory.references.all()] + exploits = [serialize_advisory_exploit(e) for e in advisory.exploits.all()] + impacts = [ { "purl": impact.base_purl, "affected_versions": impact.affecting_vers, "fixed_versions": impact.fixed_vers, + "affected_packages": [pkg.package_url for pkg in impact.affecting_packages.all()], + "fixed_packages": [pkg.package_url for pkg in impact.fixed_by_packages.all()], "fixed_in_commits": serialize_commit_patches( impact.fixed_by_package_commit_patches, ), @@ -165,6 +219,7 @@ def serialize_advisory(advisory): "severities": severities, "weaknesses": weaknesses, "references": references, + "exploits": exploits, } diff --git a/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py b/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py index d418a621f..b31fc4a30 100644 --- a/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py +++ b/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py @@ -149,15 +149,15 @@ def test_vulnerabilities_federation_v2(self, mock_check_fed, mock_commit, mock_c self.assertEqual(exit_code, 0) - result_advisories_yml = next(self.working_dir.rglob("1.2.4/advisories.yml")) + # result_advisories_yml = next(self.working_dir.rglob("1.2.4/advisories.yml")) result_advisory1_yml = next(self.working_dir.rglob("ADV-001.yml")) result_advisory2_yml = next(self.working_dir.rglob("ADV-002.yml")) - expected_advisories_yml = TEST_DATA / "1.2.4" / "advisories-expected.yml" + # expected_advisories_yml = TEST_DATA / "1.2.4" / "advisories-expected.yml" expected_advisory1_yml = TEST_DATA / "ADV-001-expected.yml" expected_advisory2_yml = TEST_DATA / "ADV-002-expected.yml" - util_tests.check_results_and_expected_files(result_advisories_yml, expected_advisories_yml) + # util_tests.check_results_and_expected_files(result_advisories_yml, expected_advisories_yml) util_tests.check_results_and_expected_files(result_advisory1_yml, expected_advisory1_yml) util_tests.check_results_and_expected_files(result_advisory2_yml, expected_advisory2_yml) diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-001-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-001-expected.yml index c50efefd7..5989c32c3 100644 --- a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-001-expected.yml +++ b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-001-expected.yml @@ -8,8 +8,13 @@ impacted_packages: - purl: pkg:npm/foobar affected_versions: vers:npm/<=1.2.3 fixed_versions: vers:npm/1.2.4 + affected_packages: + - pkg:npm/foobar@1.2.3 + fixed_packages: + - pkg:npm/foobar@1.2.4 fixed_in_commits: [] introduced_in_commits: [] severities: [] weaknesses: [] references: [] +exploits: [] diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-002-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-002-expected.yml index 14fa46564..1e2edd670 100644 --- a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-002-expected.yml +++ b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-002-expected.yml @@ -8,9 +8,14 @@ impacted_packages: - purl: pkg:npm/foobar affected_versions: vers:npm/>=1.2.4 fixed_versions: vers:npm/2.0.0 + affected_packages: + - pkg:npm/foobar@1.2.4 + fixed_packages: + - pkg:npm/foobar@2.0.0 fixed_in_commits: - vcs_url: https://foobar.vcs/ commit: 982f801f + patch_text: introduced_in_commits: [] severities: - score: '8.8' @@ -25,3 +30,4 @@ references: - url: https://example.com/vuln1 reference_type: reference_id: +exploits: []