From 5b93bae86a8a5ecf7452d16c5b6765c59e563ff8 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Thu, 27 Aug 2026 08:35:55 -0400 Subject: [PATCH 1/2] Add vulnerabilities to the PyPI JSON API Serve Warehouse-shaped vulnerability data from stored OSV reports, and let remotes opt in to scan the new repository version after sync. Assisted By: Cursor Grok 4.6 Co-authored-by: Cursor --- CHANGES/1365.feature | 1 + docs/user/guides/sync.md | 10 ++ docs/user/guides/vulnerability_report.md | 10 ++ .../0025_pythonremote_vulnerabilities.py | 16 ++ pulp_python/app/models.py | 7 +- pulp_python/app/pypi/serializers.py | 3 + pulp_python/app/serializers.py | 6 + pulp_python/app/tasks/__init__.py | 2 +- pulp_python/app/tasks/sync.py | 5 +- pulp_python/app/tasks/vulnerability_report.py | 11 ++ pulp_python/app/utils.py | 59 ++++++- pulp_python/app/viewsets.py | 11 +- .../tests/functional/api/test_pypi_apis.py | 5 +- .../api/test_pypi_json_vulnerabilities.py | 155 ++++++++++++++++++ .../api/test_vulnerability_report.py | 17 +- .../tests/unit/test_vulnerabilities.py | 84 ++++++++++ 16 files changed, 377 insertions(+), 25 deletions(-) create mode 100644 CHANGES/1365.feature create mode 100644 pulp_python/app/migrations/0025_pythonremote_vulnerabilities.py create mode 100644 pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py create mode 100644 pulp_python/tests/unit/test_vulnerabilities.py diff --git a/CHANGES/1365.feature b/CHANGES/1365.feature new file mode 100644 index 000000000..60360f66d --- /dev/null +++ b/CHANGES/1365.feature @@ -0,0 +1 @@ +Added a `vulnerabilities` field to the PyPI JSON API, populated from OSV scan reports. Remotes can opt in to scan the new repository version after sync. diff --git a/docs/user/guides/sync.md b/docs/user/guides/sync.md index cb2dab88a..8b546fd12 100644 --- a/docs/user/guides/sync.md +++ b/docs/user/guides/sync.md @@ -154,6 +154,16 @@ pulp python remote create \ --keep-latest-packages 5 ``` +Set `vulnerabilities` on the remote to scan the new repository version after each successful sync. The scan runs as a follow-up task and does not fail the sync if OSV is unreachable. Results are stored as vulnerability reports and exposed on the JSON API. + +```bash +pulp python remote create \ + --name 'scanned-remote' \ + --url 'https://pypi.org/' \ + --includes '["django==5.2.1"]' \ + --vulnerabilities +``` + Reference: [Python Remote Usage](site:pulp_python/restapi/#tag/Remotes:-Python) ### Creating a remote to sync all of PyPI diff --git a/docs/user/guides/vulnerability_report.md b/docs/user/guides/vulnerability_report.md index 234b7f8ac..b206eb6c6 100644 --- a/docs/user/guides/vulnerability_report.md +++ b/docs/user/guides/vulnerability_report.md @@ -81,6 +81,16 @@ The report contains detailed information about each vulnerability, including: - **References**: Links to advisories and patches - **Repository and Content**: Pulp `RepositoryVersion` and `Content` impacted +## JSON API + +The PyPI JSON endpoints (`pypi//json` and `pypi///json`) include a +`vulnerabilities` array for the selected version. Pulp trims stored OSV reports to the Warehouse +shape (`id`, `source`, `link`, `aliases`, `details`, `summary`, `fixed_in`, `withdrawn`). The key is +always present; it is an empty list until a scan has run. + +Enable `vulnerabilities` on a remote to scan automatically after sync, or scan a repository version +manually as shown above. + ## Example Workflow Here's a complete example of scanning a repository for vulnerabilities: diff --git a/pulp_python/app/migrations/0025_pythonremote_vulnerabilities.py b/pulp_python/app/migrations/0025_pythonremote_vulnerabilities.py new file mode 100644 index 000000000..c6d4712c7 --- /dev/null +++ b/pulp_python/app/migrations/0025_pythonremote_vulnerabilities.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("python", "0024_pythonrepository_error_on_reject"), + ] + + operations = [ + migrations.AddField( + model_name="pythonremote", + name="vulnerabilities", + field=models.BooleanField(default=False), + ), + ] diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index e8e6c26dc..902bf5a82 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -130,7 +130,11 @@ def content_handler(self, path): if not settings.DOMAIN_ENABLED: domain = None json_body = python_content_to_json( - self.base_path, package_content, version=version, domain=domain + self.base_path, + package_content, + version=version, + domain=domain, + repository_version=self.publication.repository_version, ) if json_body: return json_response(json_body, headers=headers) @@ -350,6 +354,7 @@ class PythonRemote(Remote, AutoAddObjPermsMixin): models.CharField(max_length=10, blank=True), choices=PLATFORMS, default=list ) provenance = models.BooleanField(default=False) + vulnerabilities = models.BooleanField(default=False) def get_remote_artifact_url(self, relative_path=None, request=None): """Get url for remote_artifact""" diff --git a/pulp_python/app/pypi/serializers.py b/pulp_python/app/pypi/serializers.py index bfa1a0ae3..e75475ae2 100644 --- a/pulp_python/app/pypi/serializers.py +++ b/pulp_python/app/pypi/serializers.py @@ -35,6 +35,9 @@ class PackageMetadataSerializer(serializers.Serializer): info = serializers.JSONField(help_text=_("Core metadata of the package")) releases = serializers.JSONField(help_text=_("List of all the releases of the package")) urls = serializers.JSONField() + vulnerabilities = serializers.JSONField( + help_text=_("Known vulnerabilities for the selected package version."), + ) class PackageUploadSerializer(serializers.Serializer): diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 038bb3584..c59a53980 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -789,6 +789,11 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer): help_text=_("Whether to sync available provenances for Python packages."), default=False, ) + vulnerabilities = serializers.BooleanField( + required=False, + help_text=_("Whether to scan the new repository version for vulnerabilities after a sync."), + default=False, + ) def validate_includes(self, value): """Validates the includes""" @@ -821,6 +826,7 @@ class Meta: "keep_latest_packages", "exclude_platforms", "provenance", + "vulnerabilities", ) model = python_models.PythonRemote diff --git a/pulp_python/app/tasks/__init__.py b/pulp_python/app/tasks/__init__.py index 6c193dddb..f9407d6d1 100644 --- a/pulp_python/app/tasks/__init__.py +++ b/pulp_python/app/tasks/__init__.py @@ -6,5 +6,5 @@ from .repair import repair # noqa:F401 from .sync import sync # noqa:F401 from .upload import upload, upload_group # noqa:F401 -from .vulnerability_report import get_repo_version_content # noqa:F401 +from .vulnerability_report import dispatch_scan, get_repo_version_content # noqa:F401 from .yank import aunyank_package, ayank_package # noqa:F401 diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index dd5e38265..ab58ca94b 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -29,6 +29,7 @@ PythonRemote, ) from pulp_python.app.provenance import Provenance +from pulp_python.app.tasks.vulnerability_report import dispatch_scan from pulp_python.app.utils import PYPI_LAST_SERIAL, aget_remote_simple_page, parse_metadata logger = logging.getLogger(__name__) @@ -56,7 +57,9 @@ def sync(remote_pk, repository_pk, mirror): raise SyncError("A remote must have a url attribute to sync.") first_stage = PythonBanderStage(remote) - DeclarativeVersion(first_stage, repository, mirror).create() + new_version = DeclarativeVersion(first_stage, repository, mirror).create() + if new_version and remote.vulnerabilities: + dispatch_scan(repository, new_version) def create_bandersnatch_config(remote): diff --git a/pulp_python/app/tasks/vulnerability_report.py b/pulp_python/app/tasks/vulnerability_report.py index 8d5352ca1..a6c4e5b65 100644 --- a/pulp_python/app/tasks/vulnerability_report.py +++ b/pulp_python/app/tasks/vulnerability_report.py @@ -1,5 +1,6 @@ from pulpcore.plugin.models import RepositoryVersion from pulpcore.plugin.sync import sync_to_async_iterable +from pulpcore.plugin.tasking import check_content, dispatch from pulp_python.app.models import PythonPackageContent @@ -28,3 +29,13 @@ def _build_osv_data(name, ecosystem, version=None): if version: osv_data["version"] = version return osv_data + + +def dispatch_scan(repository, repository_version): + """Dispatch a vulnerability scan for a repository version.""" + func = f"{get_repo_version_content.__module__}.{get_repo_version_content.__name__}" + return dispatch( + check_content, + shared_resources=[repository], + args=[func, [str(repository_version.pk)]], + ) diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 9e08c77c6..f9d19c55b 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -17,11 +17,11 @@ from jinja2 import Template from packaging.requirements import Requirement from packaging.utils import canonicalize_name -from packaging.version import InvalidVersion, parse +from packaging.version import InvalidVersion, Version, parse from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage from pulpcore.plugin.exceptions import TimeoutException -from pulpcore.plugin.models import Artifact, Remote +from pulpcore.plugin.models import Artifact, Remote, VulnerabilityReport from pulpcore.plugin.util import get_domain log = logging.getLogger(__name__) @@ -377,6 +377,7 @@ def python_content_to_json( last_serial: int releases: Dict urls: Dict + vulnerabilities: List Returns None if version is specified but not found within content_query """ @@ -407,9 +408,63 @@ def python_content_to_json( full_metadata["info"] = python_content_to_info(latest_content[0]) full_metadata["releases"] = python_content_to_releases(all_content, base_path, domain) full_metadata["urls"] = python_content_to_urls(latest_content, base_path, domain) + full_metadata["vulnerabilities"] = _vulnerabilities_for_content(latest_content) return full_metadata +def _osv_fixed_in(vuln): + """Extract PEP 440 fixed versions from an OSV vulnerability record.""" + fixed = [] + seen = set() + for affected in vuln.get("affected") or []: + for range_ in affected.get("ranges") or []: + for event in range_.get("events") or []: + if "fixed" not in event: + continue + version = event["fixed"] + if version in seen: + continue + try: + Version(version) + except InvalidVersion: + continue + seen.add(version) + fixed.append(version) + return fixed + + +def osv_to_pypi_vulnerabilities(vulns): + """Trim OSV vulnerability records to the Warehouse JSON API shape.""" + seen = {} + for vuln in vulns or []: + vuln_id = vuln.get("id") + if not vuln_id or vuln_id in seen: + continue + seen[vuln_id] = { + "id": vuln_id, + "source": "osv", + "link": f"https://osv.dev/vulnerability/{vuln_id}", + "aliases": vuln.get("aliases") or [], + "details": vuln.get("details"), + "summary": vuln.get("summary"), + "fixed_in": _osv_fixed_in(vuln), + "withdrawn": vuln.get("withdrawn"), + } + return list(seen.values()) + + +def _vulnerabilities_for_content(contents): + """Load VulnerabilityReports for content units and trim them to Warehouse shape.""" + if not contents: + return [] + reports = VulnerabilityReport.objects.filter(content_id__in=[c.pk for c in contents]) + merged = [] + for vulns in reports.values_list("vulns", flat=True): + if vulns: + merged.extend(vulns) + return osv_to_pypi_vulnerabilities(merged) + + def latest_content_version(all_content, version): """ Walks through the content list and finds the instances that are the latest version. diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 6c73575f4..400c9398b 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -26,7 +26,7 @@ RepositoryAddRemoveContentSerializer, RepositorySyncURLSerializer, ) -from pulpcore.plugin.tasking import check_content, dispatch +from pulpcore.plugin.tasking import dispatch from pulpcore.plugin.util import extract_pk from pulp_python.app import models as python_models @@ -356,14 +356,7 @@ def scan(self, request, repository_pk, **kwargs): Scan a repository version for vulnerabilities. """ repository_version = self.get_object() - func = ( - f"{tasks.get_repo_version_content.__module__}.{tasks.get_repo_version_content.__name__}" - ) - task = dispatch( - check_content, - shared_resources=[repository_version.repository], - args=[func, [repository_version.pk]], - ) + task = tasks.dispatch_scan(repository_version.repository, repository_version) return core_viewsets.OperationPostponedResponse(task, request) diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index a742863fb..4cedbbd04 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -324,6 +324,7 @@ def assert_pypi_json(package): package["releases"][version], "Failed to match version", ) + assert package["vulnerabilities"] == [] def assert_download_info(expected, received, message="Failed to match"): @@ -377,6 +378,8 @@ def test_upload_time_reflects_repo_addition( # JSON API json_resp = requests.get(urljoin(distro.base_url, "pypi/twine/json")) assert json_resp.status_code == 200 - json_time = datetime.fromisoformat(json_resp.json()["urls"][0]["upload_time"]) + package = json_resp.json() + json_time = datetime.fromisoformat(package["urls"][0]["upload_time"]) assert json_time > content_created assert json_time == simple_time + assert package["vulnerabilities"] == [] diff --git a/pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py b/pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py new file mode 100644 index 000000000..3d5c3df54 --- /dev/null +++ b/pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py @@ -0,0 +1,155 @@ +from urllib.parse import urljoin, urlsplit + +import pytest +import requests + +from pulp_python.tests.functional.constants import ( + PYPI_URL, + VULNERABILITY_REPORT_TEST_PACKAGE_NAME, + VULNERABILITY_REPORT_TEST_PACKAGES, +) + +WAREHOUSE_VULN_KEYS = { + "id", + "source", + "link", + "aliases", + "details", + "summary", + "fixed_in", + "withdrawn", +} + + +def _index_url(distro, bindings_cfg): + """Build the index URL using the same origin the API client uses.""" + path = urlsplit(distro.base_url).path + if not path.endswith("/"): + path += "/" + return bindings_cfg.host.rstrip("/") + path + + +def _wait_for_child_tasks(pulpcore_bindings, monitor_task, parent_task): + parent = pulpcore_bindings.TasksApi.read(parent_task.pulp_href) + children = parent.child_tasks or [] + assert children, "expected a follow-up vulnerability scan task" + for child in children: + href = child if isinstance(child, str) else child.pulp_href + monitor_task(href) + + +def _assert_warehouse_vulnerabilities(package): + vulns = package["vulnerabilities"] + assert vulns + ids = [vuln["id"] for vuln in vulns] + assert len(ids) == len(set(ids)) + for vuln in vulns: + assert WAREHOUSE_VULN_KEYS <= vuln.keys() + assert vuln["source"] == "osv" + assert vuln["id"] + assert vuln["link"] == f"https://osv.dev/vulnerability/{vuln['id']}" + assert isinstance(vuln["aliases"], list) + assert isinstance(vuln["fixed_in"], list) + + +@pytest.mark.parallel +def test_pypi_json_vulnerabilities_from_manual_scan( + bindings_cfg, + pulpcore_bindings, + python_bindings, + python_remote_factory, + python_repo, + python_distribution_factory, + monitor_task, +): + """A remote without vulnerabilities does not scan; a later scan fills the JSON API.""" + remote = python_remote_factory(url=PYPI_URL, includes=VULNERABILITY_REPORT_TEST_PACKAGES) + sync_task = monitor_task( + python_bindings.RepositoriesPythonApi.sync( + python_repo.pulp_href, dict(remote=remote.pulp_href) + ).task + ) + assert not sync_task.child_tasks + + repo = python_bindings.RepositoriesPythonApi.read(python_repo.pulp_href) + distro = python_distribution_factory(repository=repo) + name = VULNERABILITY_REPORT_TEST_PACKAGE_NAME.lower() + index = _index_url(distro, bindings_cfg) + version_url = urljoin(index, f"pypi/{name}/5.2.1/json") + project_url = urljoin(index, f"pypi/{name}/json") + + scan_task = python_bindings.RepositoriesPythonVersionsApi.scan(repo.latest_version_href) + monitor_task(scan_task.task) + + project = requests.get(project_url).json() + version = requests.get(version_url).json() + _assert_warehouse_vulnerabilities(project) + _assert_warehouse_vulnerabilities(version) + assert [v["id"] for v in project["vulnerabilities"]] == [ + v["id"] for v in version["vulnerabilities"] + ] + + packages = python_bindings.ContentPackagesApi.list( + name=VULNERABILITY_REPORT_TEST_PACKAGE_NAME, + repository_version=repo.latest_version_href, + ) + assert packages.count >= 2 + for content in packages.results: + assert content.vuln_report is not None + report = pulpcore_bindings.VulnReportApi.read(content.vuln_report) + assert report.vulns + assert "affected" in report.vulns[0] + + +@pytest.mark.parallel +def test_sync_dispatches_vulnerability_scan( + bindings_cfg, + pulpcore_bindings, + python_bindings, + python_remote_factory, + python_repo, + python_distribution_factory, + monitor_task, +): + """A remote with vulnerabilities=True scans the new repository version after sync.""" + remote = python_remote_factory( + url=PYPI_URL, + includes=VULNERABILITY_REPORT_TEST_PACKAGES, + vulnerabilities=True, + ) + sync_task = monitor_task( + python_bindings.RepositoriesPythonApi.sync( + python_repo.pulp_href, dict(remote=remote.pulp_href) + ).task + ) + _wait_for_child_tasks(pulpcore_bindings, monitor_task, sync_task) + + repo = python_bindings.RepositoriesPythonApi.read(python_repo.pulp_href) + distro = python_distribution_factory(repository=repo) + name = VULNERABILITY_REPORT_TEST_PACKAGE_NAME.lower() + package = requests.get(urljoin(_index_url(distro, bindings_cfg), f"pypi/{name}/json")).json() + _assert_warehouse_vulnerabilities(package) + + +@pytest.mark.parallel +def test_uploaded_package_json_vulnerabilities_empty_until_scan( + bindings_cfg, + python_bindings, + python_content_factory, + python_repo_factory, + python_distribution_factory, + monitor_task, +): + """Uploaded packages have an empty vulnerabilities list until a scan runs.""" + repo = python_repo_factory() + python_content_factory(repository=repo) + distro = python_distribution_factory(repository=repo) + url = urljoin(_index_url(distro, bindings_cfg), "pypi/shelf-reader/json") + + package = requests.get(url).json() + assert package["vulnerabilities"] == [] + + repo = python_bindings.RepositoriesPythonApi.read(repo.pulp_href) + monitor_task(python_bindings.RepositoriesPythonVersionsApi.scan(repo.latest_version_href).task) + package = requests.get(url).json() + assert isinstance(package["vulnerabilities"], list) diff --git a/pulp_python/tests/functional/api/test_vulnerability_report.py b/pulp_python/tests/functional/api/test_vulnerability_report.py index 1630c147f..95d7326e2 100644 --- a/pulp_python/tests/functional/api/test_vulnerability_report.py +++ b/pulp_python/tests/functional/api/test_vulnerability_report.py @@ -28,21 +28,18 @@ def test_vulnerability_report( ) monitor_task(response.task) - # checks - vulns_list = pulpcore_bindings.VulnReportApi.list() - assert len(vulns_list.results) > 0 - for results in vulns_list.results: - assert len(results.vulns) > 0 - for vuln in results.vulns: - assert VULNERABILITY_REPORT_TEST_PACKAGE_NAME.lower() in ( - affected["package"]["name"] for affected in vuln["affected"] - ) - repo_version = python_bindings.RepositoriesPythonVersionsApi.read(latest_version_href) assert repo_version.vuln_report is not None python_packages = python_bindings.ContentPackagesApi.list( name=VULNERABILITY_REPORT_TEST_PACKAGE_NAME, repository_version=latest_version_href ) + assert python_packages.count > 0 for content in python_packages.results: assert content.vuln_report is not None + report = pulpcore_bindings.VulnReportApi.read(content.vuln_report) + assert len(report.vulns) > 0 + for vuln in report.vulns: + assert VULNERABILITY_REPORT_TEST_PACKAGE_NAME.lower() in ( + affected["package"]["name"] for affected in vuln["affected"] + ) diff --git a/pulp_python/tests/unit/test_vulnerabilities.py b/pulp_python/tests/unit/test_vulnerabilities.py new file mode 100644 index 000000000..ec491cceb --- /dev/null +++ b/pulp_python/tests/unit/test_vulnerabilities.py @@ -0,0 +1,84 @@ +from pulp_python.app.utils import osv_to_pypi_vulnerabilities + + +def test_osv_to_pypi_maps_fixed_versions_and_skips_git_shas(): + vulns = [ + { + "id": "PYSEC-2020-35", + "aliases": ["CVE-2020-7471"], + "details": "SQL injection", + "summary": None, + "withdrawn": None, + "affected": [ + { + "package": {"name": "django", "ecosystem": "PyPI"}, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + {"introduced": "3.0"}, + {"fixed": "3.0.3"}, + {"fixed": "eb31d845323618d688ad429479c6dda973056136"}, + ], + } + ], + } + ], + } + ] + + result = osv_to_pypi_vulnerabilities(vulns) + + assert result == [ + { + "id": "PYSEC-2020-35", + "source": "osv", + "link": "https://osv.dev/vulnerability/PYSEC-2020-35", + "aliases": ["CVE-2020-7471"], + "details": "SQL injection", + "summary": None, + "fixed_in": ["3.0.3"], + "withdrawn": None, + } + ] + + +def test_osv_to_pypi_deduplicates_by_id(): + vulns = [ + {"id": "GHSA-aaaa", "aliases": ["CVE-1"], "details": "first"}, + {"id": "GHSA-aaaa", "aliases": ["CVE-1"], "details": "duplicate"}, + {"id": "GHSA-bbbb", "aliases": [], "details": "other"}, + ] + + result = osv_to_pypi_vulnerabilities(vulns) + + assert [v["id"] for v in result] == ["GHSA-aaaa", "GHSA-bbbb"] + assert result[0]["details"] == "first" + + +def test_osv_to_pypi_preserves_withdrawn_and_missing_summary(): + vulns = [ + { + "id": "PYSEC-2022-XXX", + "aliases": ["CVE-2022-XXXXX"], + "details": "A long description.", + "withdrawn": "2022-06-28T16:39:06Z", + "affected": [ + { + "ranges": [{"events": [{"fixed": "1.2.3"}]}], + } + ], + } + ] + + result = osv_to_pypi_vulnerabilities(vulns) + + assert result[0]["summary"] is None + assert result[0]["withdrawn"] == "2022-06-28T16:39:06Z" + assert result[0]["fixed_in"] == ["1.2.3"] + + +def test_osv_to_pypi_empty_and_missing_ids(): + assert osv_to_pypi_vulnerabilities(None) == [] + assert osv_to_pypi_vulnerabilities([]) == [] + assert osv_to_pypi_vulnerabilities([{"details": "no id"}]) == [] From 7724bae22c68d012cc5fc6d63572ba64661b8922 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Thu, 27 Aug 2026 11:00:10 -0400 Subject: [PATCH 2/2] Scope JSON vulnerabilities to the scanned repository version Filter OSV reports by repository version so another index cannot show vulns until it is scanned. Move the Warehouse trim helper out of Django-backed utils so unit tests collect without loading apps. Assisted By: Cursor Grok 4.6 Co-authored-by: Cursor --- pulp_python/app/osv.py | 42 +++++++++++++ pulp_python/app/utils.py | 60 ++++--------------- .../api/test_pypi_json_vulnerabilities.py | 16 +++++ .../tests/unit/test_vulnerabilities.py | 2 +- 4 files changed, 72 insertions(+), 48 deletions(-) create mode 100644 pulp_python/app/osv.py diff --git a/pulp_python/app/osv.py b/pulp_python/app/osv.py new file mode 100644 index 000000000..154fbb17f --- /dev/null +++ b/pulp_python/app/osv.py @@ -0,0 +1,42 @@ +from packaging.version import InvalidVersion, Version + + +def _osv_fixed_in(vuln): + """Extract PEP 440 fixed versions from an OSV vulnerability record.""" + fixed = [] + seen = set() + for affected in vuln.get("affected") or []: + for range_ in affected.get("ranges") or []: + for event in range_.get("events") or []: + if "fixed" not in event: + continue + version = event["fixed"] + if version in seen: + continue + try: + Version(version) + except InvalidVersion: + continue + seen.add(version) + fixed.append(version) + return fixed + + +def osv_to_pypi_vulnerabilities(vulns): + """Trim OSV vulnerability records to the Warehouse JSON API shape.""" + seen = {} + for vuln in vulns or []: + vuln_id = vuln.get("id") + if not vuln_id or vuln_id in seen: + continue + seen[vuln_id] = { + "id": vuln_id, + "source": "osv", + "link": f"https://osv.dev/vulnerability/{vuln_id}", + "aliases": vuln.get("aliases") or [], + "details": vuln.get("details"), + "summary": vuln.get("summary"), + "fixed_in": _osv_fixed_in(vuln), + "withdrawn": vuln.get("withdrawn"), + } + return list(seen.values()) diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index f9d19c55b..7fba7c98e 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -17,13 +17,15 @@ from jinja2 import Template from packaging.requirements import Requirement from packaging.utils import canonicalize_name -from packaging.version import InvalidVersion, Version, parse +from packaging.version import InvalidVersion, parse from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage from pulpcore.plugin.exceptions import TimeoutException from pulpcore.plugin.models import Artifact, Remote, VulnerabilityReport from pulpcore.plugin.util import get_domain +from pulp_python.app.osv import osv_to_pypi_vulnerabilities + log = logging.getLogger(__name__) @@ -408,56 +410,20 @@ def python_content_to_json( full_metadata["info"] = python_content_to_info(latest_content[0]) full_metadata["releases"] = python_content_to_releases(all_content, base_path, domain) full_metadata["urls"] = python_content_to_urls(latest_content, base_path, domain) - full_metadata["vulnerabilities"] = _vulnerabilities_for_content(latest_content) + full_metadata["vulnerabilities"] = _vulnerabilities_for_content( + latest_content, repository_version + ) return full_metadata -def _osv_fixed_in(vuln): - """Extract PEP 440 fixed versions from an OSV vulnerability record.""" - fixed = [] - seen = set() - for affected in vuln.get("affected") or []: - for range_ in affected.get("ranges") or []: - for event in range_.get("events") or []: - if "fixed" not in event: - continue - version = event["fixed"] - if version in seen: - continue - try: - Version(version) - except InvalidVersion: - continue - seen.add(version) - fixed.append(version) - return fixed - - -def osv_to_pypi_vulnerabilities(vulns): - """Trim OSV vulnerability records to the Warehouse JSON API shape.""" - seen = {} - for vuln in vulns or []: - vuln_id = vuln.get("id") - if not vuln_id or vuln_id in seen: - continue - seen[vuln_id] = { - "id": vuln_id, - "source": "osv", - "link": f"https://osv.dev/vulnerability/{vuln_id}", - "aliases": vuln.get("aliases") or [], - "details": vuln.get("details"), - "summary": vuln.get("summary"), - "fixed_in": _osv_fixed_in(vuln), - "withdrawn": vuln.get("withdrawn"), - } - return list(seen.values()) - - -def _vulnerabilities_for_content(contents): - """Load VulnerabilityReports for content units and trim them to Warehouse shape.""" - if not contents: +def _vulnerabilities_for_content(contents, repository_version=None): + """Load VulnerabilityReports scanned for this repository version and trim to Warehouse shape.""" + if not contents or repository_version is None: return [] - reports = VulnerabilityReport.objects.filter(content_id__in=[c.pk for c in contents]) + reports = VulnerabilityReport.objects.filter( + content_id__in=[c.pk for c in contents], + repo_versions=repository_version, + ) merged = [] for vulns in reports.values_list("vulns", flat=True): if vulns: diff --git a/pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py b/pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py index 3d5c3df54..97bb33c79 100644 --- a/pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py +++ b/pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py @@ -59,6 +59,7 @@ def test_pypi_json_vulnerabilities_from_manual_scan( python_bindings, python_remote_factory, python_repo, + python_repo_factory, python_distribution_factory, monitor_task, ): @@ -94,11 +95,26 @@ def test_pypi_json_vulnerabilities_from_manual_scan( repository_version=repo.latest_version_href, ) assert packages.count >= 2 + hrefs = [] for content in packages.results: assert content.vuln_report is not None report = pulpcore_bindings.VulnReportApi.read(content.vuln_report) assert report.vulns assert "affected" in report.vulns[0] + hrefs.append(content.pulp_href) + + other = python_repo_factory() + monitor_task( + python_bindings.RepositoriesPythonApi.modify( + other.pulp_href, {"add_content_units": hrefs} + ).task + ) + other = python_bindings.RepositoriesPythonApi.read(other.pulp_href) + other_distro = python_distribution_factory(repository=other) + other_json = requests.get( + urljoin(_index_url(other_distro, bindings_cfg), f"pypi/{name}/json") + ).json() + assert other_json["vulnerabilities"] == [] @pytest.mark.parallel diff --git a/pulp_python/tests/unit/test_vulnerabilities.py b/pulp_python/tests/unit/test_vulnerabilities.py index ec491cceb..cd55a1a34 100644 --- a/pulp_python/tests/unit/test_vulnerabilities.py +++ b/pulp_python/tests/unit/test_vulnerabilities.py @@ -1,4 +1,4 @@ -from pulp_python.app.utils import osv_to_pypi_vulnerabilities +from pulp_python.app.osv import osv_to_pypi_vulnerabilities def test_osv_to_pypi_maps_fixed_versions_and_skips_git_shas():