diff --git a/vulnerabilities/api_v2.py b/vulnerabilities/api_v2.py index aa26e5f39..c45dbfebe 100644 --- a/vulnerabilities/api_v2.py +++ b/vulnerabilities/api_v2.py @@ -349,7 +349,7 @@ def get_fixing_vulnerabilities(self, obj): # Ghost package should not fix any vulnerability. if obj.is_ghost: return [] - return [adv.advisory_id for adv in obj.fixing_advisories.all()] + return [adv.avid for adv in obj.fixing_advisories.all()] class PackageurlListSerializer(serializers.Serializer): diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 9cef5e0fa..4e0f015da 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -160,7 +160,7 @@ def __post_init__(self): self.reference_id = str(self.reference_id) def __lt__(self, other): - if not isinstance(other, Reference): + if not isinstance(other, ReferenceV2): return NotImplemented return self._cmp_key() < other._cmp_key() diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 3dd914a92..a72ef981e 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -42,6 +42,9 @@ from vulnerabilities.pipelines import pypa_importer from vulnerabilities.pipelines import pysec_importer from vulnerabilities.pipelines.v2_importers import apache_httpd_importer as apache_httpd_v2 +from vulnerabilities.pipelines.v2_importers import ( + elixir_security_importer as elixir_security_importer_v2, +) from vulnerabilities.pipelines.v2_importers import github_importer as github_importer_v2 from vulnerabilities.pipelines.v2_importers import gitlab_importer as gitlab_importer_v2 from vulnerabilities.pipelines.v2_importers import npm_importer as npm_importer_v2 @@ -54,6 +57,7 @@ IMPORTERS_REGISTRY = create_registry( [ nvd_importer_v2.NVDImporterPipeline, + elixir_security_importer_v2.ElixirSecurityImporterPipeline, github_importer_v2.GitHubAPIImporterPipeline, npm_importer_v2.NpmImporterPipeline, vulnrichment_importer_v2.VulnrichImporterPipeline, diff --git a/vulnerabilities/importers/osv.py b/vulnerabilities/importers/osv.py index 01f2d8023..76be8ef0f 100644 --- a/vulnerabilities/importers/osv.py +++ b/vulnerabilities/importers/osv.py @@ -23,6 +23,7 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Reference +from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import build_description @@ -268,7 +269,7 @@ def get_references_v2(raw_data) -> List[Reference]: if not url: logger.error(f"Reference without URL : {ref!r} for OSV id: {raw_data['id']!r}") continue - references.append(Reference(url=ref["url"])) + references.append(ReferenceV2(url=ref["url"])) return references diff --git a/vulnerabilities/migrations/0097_alter_advisoryv2_advisory_id_and_more.py b/vulnerabilities/migrations/0097_alter_advisoryv2_advisory_id_and_more.py new file mode 100644 index 000000000..f52711a03 --- /dev/null +++ b/vulnerabilities/migrations/0097_alter_advisoryv2_advisory_id_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.22 on 2025-07-03 16:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0096_alter_pipelineschedule_run_interval"), + ] + + operations = [ + migrations.AlterField( + model_name="advisoryv2", + name="advisory_id", + field=models.CharField( + help_text="An advisory is a unique vulnerability identifier in some database, such as PYSEC-2020-2233", + max_length=500, + ), + ), + migrations.AlterField( + model_name="advisoryv2", + name="datasource_id", + field=models.CharField( + help_text="Unique ID for the datasource used for this advisory .e.g.: nginx_importer_v2", + max_length=200, + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index ba7b4215e..df3b72a94 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2657,7 +2657,7 @@ class AdvisoryV2(models.Model): # This is similar to a type or a namespace datasource_id = models.CharField( - max_length=100, + max_length=200, blank=False, null=False, help_text="Unique ID for the datasource used for this advisory ." "e.g.: nginx_importer_v2", @@ -2665,7 +2665,7 @@ class AdvisoryV2(models.Model): # This is similar to a name advisory_id = models.CharField( - max_length=50, + max_length=500, blank=False, null=False, unique=False, diff --git a/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py b/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py index 90ea32b75..77b55f7dc 100644 --- a/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py +++ b/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py @@ -21,7 +21,7 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference +from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 from vulnerabilities.severity_systems import APACHE_HTTPD @@ -260,7 +260,7 @@ def to_advisory(self, data): ) ) break - reference = Reference( + reference = ReferenceV2( reference_id=alias, url=urllib.parse.urljoin(self.base_url, f"{alias}.json"), ) diff --git a/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py b/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py index 902dd5248..d12f7d947 100644 --- a/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py +++ b/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py @@ -18,7 +18,7 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference +from vulnerabilities.importer import ReferenceV2 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 from vulnerabilities.utils import is_cve from vulnerabilities.utils import load_yaml @@ -83,7 +83,7 @@ def process_file(self, file, base_path) -> Iterable[AdvisoryData]: references = [] link = yaml_file.get("link") or "" if link: - references.append(Reference(url=link)) + references.append(ReferenceV2(url=link)) constraints = [] vrc = HexVersionRange.version_class diff --git a/vulnerabilities/pipelines/v2_importers/github_importer.py b/vulnerabilities/pipelines/v2_importers/github_importer.py index 46fb95ea3..55ac93716 100644 --- a/vulnerabilities/pipelines/v2_importers/github_importer.py +++ b/vulnerabilities/pipelines/v2_importers/github_importer.py @@ -24,7 +24,7 @@ from vulnerabilities import utils from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference +from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 from vulnerabilities.utils import dedupe @@ -271,7 +271,7 @@ def process_response( references = get_item(advisory, "references") or [] if references: urls = (ref["url"] for ref in references) - references = [Reference.from_url(u) for u in urls] + references = [ReferenceV2.from_url(u) for u in urls] date_published = get_item(advisory, "publishedAt") if date_published: diff --git a/vulnerabilities/pipelines/v2_importers/gitlab_importer.py b/vulnerabilities/pipelines/v2_importers/gitlab_importer.py index 1f175f07f..a51875ddf 100644 --- a/vulnerabilities/pipelines/v2_importers/gitlab_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gitlab_importer.py @@ -26,7 +26,7 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference +from vulnerabilities.importer import ReferenceV2 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 from vulnerabilities.utils import build_description from vulnerabilities.utils import get_advisory_url @@ -237,9 +237,7 @@ def parse_gitlab_advisory( aliases.remove(advisory_id) summary = build_description(gitlab_advisory.get("title"), gitlab_advisory.get("description")) urls = gitlab_advisory.get("urls") - references = [Reference.from_url(u) for u in urls] - - print(references) + references = [ReferenceV2.from_url(u) for u in urls] cwe_ids = gitlab_advisory.get("cwe_ids") or [] cwe_list = list(map(get_cwe_id, cwe_ids)) @@ -247,6 +245,7 @@ def parse_gitlab_advisory( date_published = dateparser.parse(gitlab_advisory.get("pubdate")) date_published = date_published.replace(tzinfo=pytz.UTC) package_slug = gitlab_advisory.get("package_slug") + advisory_id = f"{package_slug}/{advisory_id}" if package_slug else advisory_id advisory_url = get_advisory_url( file=file, base_path=base_path, @@ -264,7 +263,7 @@ def parse_gitlab_advisory( return AdvisoryData( aliases=aliases, summary=summary, - references=references, + references_v2=references, date_published=date_published, url=advisory_url, ) diff --git a/vulnerabilities/pipelines/v2_importers/npm_importer.py b/vulnerabilities/pipelines/v2_importers/npm_importer.py index 19d21c987..67e2a4355 100644 --- a/vulnerabilities/pipelines/v2_importers/npm_importer.py +++ b/vulnerabilities/pipelines/v2_importers/npm_importer.py @@ -20,7 +20,7 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference +from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 from vulnerabilities.severity_systems import CVSSV2 @@ -100,14 +100,14 @@ def to_advisory_data(self, file: Path) -> Iterable[AdvisoryData]: self.log(f"Advisory ID not found in {file}") return - advisory_reference = Reference( + advisory_reference = ReferenceV2( url=f"https://github.com/nodejs/security-wg/blob/main/vuln/npm/{id}.json", reference_id=id, ) for ref in data.get("references") or []: references.append( - Reference( + ReferenceV2( url=ref, ) ) diff --git a/vulnerabilities/pipelines/v2_importers/nvd_importer.py b/vulnerabilities/pipelines/v2_importers/nvd_importer.py index 1166ac8ef..a6654d6b1 100644 --- a/vulnerabilities/pipelines/v2_importers/nvd_importer.py +++ b/vulnerabilities/pipelines/v2_importers/nvd_importer.py @@ -20,7 +20,7 @@ from vulnerabilities import severity_systems from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Reference +from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 from vulnerabilities.utils import get_cwe_id @@ -267,11 +267,11 @@ def references(self): # we track each CPE as a reference for now for cpe in self.cpes: cpe_url = f"https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query={cpe}" - references.append(Reference(reference_id=cpe, url=cpe_url)) + references.append(ReferenceV2(reference_id=cpe, url=cpe_url)) # FIXME: we also add the CVE proper as a reference, but is this correct? references.append( - Reference( + ReferenceV2( url=f"https://nvd.nist.gov/vuln/detail/{self.cve_id}", reference_id=self.cve_id, ) @@ -283,7 +283,7 @@ def references(self): for ru in self.reference_urls if ru != f"https://nvd.nist.gov/vuln/detail/{self.cve_id}" ] - references.extend([Reference(url=url) for url in ref_urls]) + references.extend([ReferenceV2(url=url) for url in ref_urls]) return references diff --git a/vulnerabilities/pipelines/v2_importers/postgresql_importer.py b/vulnerabilities/pipelines/v2_importers/postgresql_importer.py index 2f5a49439..2f98a7ce4 100644 --- a/vulnerabilities/pipelines/v2_importers/postgresql_importer.py +++ b/vulnerabilities/pipelines/v2_importers/postgresql_importer.py @@ -19,7 +19,7 @@ from vulnerabilities import severity_systems from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference +from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 @@ -122,12 +122,12 @@ def to_advisories(self, data): pass references = [] + severities = [] vector_link_tag = severity_score_col.find("a") for a_tag in ref_col.select("a"): link = a_tag.attrs["href"] if link.startswith("/"): link = urlparse.urljoin("https://www.postgresql.org/", link) - severities = [] if "support/security/CVE" in link and vector_link_tag: parsed_link = urlparse.urlparse(vector_link_tag["href"]) cvss3_vector = urlparse.parse_qs(parsed_link.query).get("vector", [""])[0] @@ -139,7 +139,7 @@ def to_advisories(self, data): scoring_elements=cvss3_vector, ) ) - references.append(Reference(url=link, severities=severities)) + references.append(ReferenceV2(url=link)) if cve_id: advisories.append( @@ -148,6 +148,7 @@ def to_advisories(self, data): aliases=[], summary=summary, references_v2=references, + severities=severities, affected_packages=affected_packages, url=f"https://www.postgresql.org/support/security/{cve_id}", ) diff --git a/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py b/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py index b2ddfd3cd..85b51e4ec 100644 --- a/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py +++ b/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py @@ -8,7 +8,7 @@ from fetchcode.vcs import fetch_via_vcs from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Reference +from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.models import VulnerabilityReference from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 @@ -151,7 +151,7 @@ def parse_cve_advisory(self, raw_data, advisory_url): ref_type = vul_ref_types.get(tag_type) url = ref.get("url") - reference = Reference( + reference = ReferenceV2( reference_id=get_reference_id(url), url=url, reference_type=ref_type, @@ -160,7 +160,7 @@ def parse_cve_advisory(self, raw_data, advisory_url): references.append(reference) cpes_ref = [ - Reference( + ReferenceV2( reference_id=cpe, reference_type=VulnerabilityReference.OTHER, url=f"https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query={cpe}", diff --git a/vulnerabilities/tests/pipelines/test_gitlab_v2_importer.py b/vulnerabilities/tests/pipelines/test_gitlab_v2_importer.py index 6e5c8eb15..1c7e03d31 100644 --- a/vulnerabilities/tests/pipelines/test_gitlab_v2_importer.py +++ b/vulnerabilities/tests/pipelines/test_gitlab_v2_importer.py @@ -92,7 +92,7 @@ def test_collect_advisories(mock_gitlab_yaml, mock_vcs_response, mock_fetch_via_ advisory = advisories[0] assert isinstance(advisory, AdvisoryData) - assert advisory.advisory_id == "CVE-2022-0001" + assert advisory.advisory_id == "pypi/package_name/CVE-2022-0001" assert advisory.summary == "Example vulnerability\nExample description" assert advisory.references_v2[0].url == "https://example.com/advisory" assert advisory.affected_packages[0].package.name == "package-name" diff --git a/vulnerabilities/tests/pipelines/test_postgresql_v2_importer.py b/vulnerabilities/tests/pipelines/test_postgresql_v2_importer.py index da077f3ed..154ad1fc4 100644 --- a/vulnerabilities/tests/pipelines/test_postgresql_v2_importer.py +++ b/vulnerabilities/tests/pipelines/test_postgresql_v2_importer.py @@ -148,7 +148,7 @@ def test_cvss_parsing(mock_get, importer): assert len(advisories) == 1 reference = advisories[0].references_v2[0] - severity = reference.severities[0] + severity = advisories[0].severities[0] assert severity.system.identifier == "cvssv3" assert severity.value == "9.8" assert "AV:N/AC:L/PR:N/UI:N" in severity.scoring_elements