diff --git a/CHANGES/1365.feature b/CHANGES/1365.feature new file mode 100644 index 000000000..60360f66d --- /dev/null +++ b/CHANGES/1365.feature @@ -0,0 +1 @@ +Added a `vulnerabilities` field to the PyPI JSON API, populated from OSV scan reports. Remotes can opt in to scan the new repository version after sync. diff --git a/docs/user/guides/sync.md b/docs/user/guides/sync.md index cb2dab88a..8b546fd12 100644 --- a/docs/user/guides/sync.md +++ b/docs/user/guides/sync.md @@ -154,6 +154,16 @@ pulp python remote create \ --keep-latest-packages 5 ``` +Set `vulnerabilities` on the remote to scan the new repository version after each successful sync. The scan runs as a follow-up task and does not fail the sync if OSV is unreachable. Results are stored as vulnerability reports and exposed on the JSON API. + +```bash +pulp python remote create \ + --name 'scanned-remote' \ + --url 'https://pypi.org/' \ + --includes '["django==5.2.1"]' \ + --vulnerabilities +``` + Reference: [Python Remote Usage](site:pulp_python/restapi/#tag/Remotes:-Python) ### Creating a remote to sync all of PyPI diff --git a/docs/user/guides/vulnerability_report.md b/docs/user/guides/vulnerability_report.md index 234b7f8ac..b206eb6c6 100644 --- a/docs/user/guides/vulnerability_report.md +++ b/docs/user/guides/vulnerability_report.md @@ -81,6 +81,16 @@ The report contains detailed information about each vulnerability, including: - **References**: Links to advisories and patches - **Repository and Content**: Pulp `RepositoryVersion` and `Content` impacted +## JSON API + +The PyPI JSON endpoints (`pypi//json` and `pypi///json`) include a +`vulnerabilities` array for the selected version. Pulp trims stored OSV reports to the Warehouse +shape (`id`, `source`, `link`, `aliases`, `details`, `summary`, `fixed_in`, `withdrawn`). The key is +always present; it is an empty list until a scan has run. + +Enable `vulnerabilities` on a remote to scan automatically after sync, or scan a repository version +manually as shown above. + ## Example Workflow Here's a complete example of scanning a repository for vulnerabilities: diff --git a/pulp_python/app/migrations/0025_pythonremote_vulnerabilities.py b/pulp_python/app/migrations/0025_pythonremote_vulnerabilities.py new file mode 100644 index 000000000..c6d4712c7 --- /dev/null +++ b/pulp_python/app/migrations/0025_pythonremote_vulnerabilities.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("python", "0024_pythonrepository_error_on_reject"), + ] + + operations = [ + migrations.AddField( + model_name="pythonremote", + name="vulnerabilities", + field=models.BooleanField(default=False), + ), + ] diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index e8e6c26dc..902bf5a82 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -130,7 +130,11 @@ def content_handler(self, path): if not settings.DOMAIN_ENABLED: domain = None json_body = python_content_to_json( - self.base_path, package_content, version=version, domain=domain + self.base_path, + package_content, + version=version, + domain=domain, + repository_version=self.publication.repository_version, ) if json_body: return json_response(json_body, headers=headers) @@ -350,6 +354,7 @@ class PythonRemote(Remote, AutoAddObjPermsMixin): models.CharField(max_length=10, blank=True), choices=PLATFORMS, default=list ) provenance = models.BooleanField(default=False) + vulnerabilities = models.BooleanField(default=False) def get_remote_artifact_url(self, relative_path=None, request=None): """Get url for remote_artifact""" diff --git a/pulp_python/app/osv.py b/pulp_python/app/osv.py new file mode 100644 index 000000000..154fbb17f --- /dev/null +++ b/pulp_python/app/osv.py @@ -0,0 +1,42 @@ +from packaging.version import InvalidVersion, Version + + +def _osv_fixed_in(vuln): + """Extract PEP 440 fixed versions from an OSV vulnerability record.""" + fixed = [] + seen = set() + for affected in vuln.get("affected") or []: + for range_ in affected.get("ranges") or []: + for event in range_.get("events") or []: + if "fixed" not in event: + continue + version = event["fixed"] + if version in seen: + continue + try: + Version(version) + except InvalidVersion: + continue + seen.add(version) + fixed.append(version) + return fixed + + +def osv_to_pypi_vulnerabilities(vulns): + """Trim OSV vulnerability records to the Warehouse JSON API shape.""" + seen = {} + for vuln in vulns or []: + vuln_id = vuln.get("id") + if not vuln_id or vuln_id in seen: + continue + seen[vuln_id] = { + "id": vuln_id, + "source": "osv", + "link": f"https://osv.dev/vulnerability/{vuln_id}", + "aliases": vuln.get("aliases") or [], + "details": vuln.get("details"), + "summary": vuln.get("summary"), + "fixed_in": _osv_fixed_in(vuln), + "withdrawn": vuln.get("withdrawn"), + } + return list(seen.values()) diff --git a/pulp_python/app/pypi/serializers.py b/pulp_python/app/pypi/serializers.py index bfa1a0ae3..e75475ae2 100644 --- a/pulp_python/app/pypi/serializers.py +++ b/pulp_python/app/pypi/serializers.py @@ -35,6 +35,9 @@ class PackageMetadataSerializer(serializers.Serializer): info = serializers.JSONField(help_text=_("Core metadata of the package")) releases = serializers.JSONField(help_text=_("List of all the releases of the package")) urls = serializers.JSONField() + vulnerabilities = serializers.JSONField( + help_text=_("Known vulnerabilities for the selected package version."), + ) class PackageUploadSerializer(serializers.Serializer): diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 038bb3584..c59a53980 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -789,6 +789,11 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer): help_text=_("Whether to sync available provenances for Python packages."), default=False, ) + vulnerabilities = serializers.BooleanField( + required=False, + help_text=_("Whether to scan the new repository version for vulnerabilities after a sync."), + default=False, + ) def validate_includes(self, value): """Validates the includes""" @@ -821,6 +826,7 @@ class Meta: "keep_latest_packages", "exclude_platforms", "provenance", + "vulnerabilities", ) model = python_models.PythonRemote diff --git a/pulp_python/app/tasks/__init__.py b/pulp_python/app/tasks/__init__.py index 6c193dddb..f9407d6d1 100644 --- a/pulp_python/app/tasks/__init__.py +++ b/pulp_python/app/tasks/__init__.py @@ -6,5 +6,5 @@ from .repair import repair # noqa:F401 from .sync import sync # noqa:F401 from .upload import upload, upload_group # noqa:F401 -from .vulnerability_report import get_repo_version_content # noqa:F401 +from .vulnerability_report import dispatch_scan, get_repo_version_content # noqa:F401 from .yank import aunyank_package, ayank_package # noqa:F401 diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index dd5e38265..ab58ca94b 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -29,6 +29,7 @@ PythonRemote, ) from pulp_python.app.provenance import Provenance +from pulp_python.app.tasks.vulnerability_report import dispatch_scan from pulp_python.app.utils import PYPI_LAST_SERIAL, aget_remote_simple_page, parse_metadata logger = logging.getLogger(__name__) @@ -56,7 +57,9 @@ def sync(remote_pk, repository_pk, mirror): raise SyncError("A remote must have a url attribute to sync.") first_stage = PythonBanderStage(remote) - DeclarativeVersion(first_stage, repository, mirror).create() + new_version = DeclarativeVersion(first_stage, repository, mirror).create() + if new_version and remote.vulnerabilities: + dispatch_scan(repository, new_version) def create_bandersnatch_config(remote): diff --git a/pulp_python/app/tasks/vulnerability_report.py b/pulp_python/app/tasks/vulnerability_report.py index 8d5352ca1..a6c4e5b65 100644 --- a/pulp_python/app/tasks/vulnerability_report.py +++ b/pulp_python/app/tasks/vulnerability_report.py @@ -1,5 +1,6 @@ from pulpcore.plugin.models import RepositoryVersion from pulpcore.plugin.sync import sync_to_async_iterable +from pulpcore.plugin.tasking import check_content, dispatch from pulp_python.app.models import PythonPackageContent @@ -28,3 +29,13 @@ def _build_osv_data(name, ecosystem, version=None): if version: osv_data["version"] = version return osv_data + + +def dispatch_scan(repository, repository_version): + """Dispatch a vulnerability scan for a repository version.""" + func = f"{get_repo_version_content.__module__}.{get_repo_version_content.__name__}" + return dispatch( + check_content, + shared_resources=[repository], + args=[func, [str(repository_version.pk)]], + ) diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 9e08c77c6..7fba7c98e 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -21,9 +21,11 @@ from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage from pulpcore.plugin.exceptions import TimeoutException -from pulpcore.plugin.models import Artifact, Remote +from pulpcore.plugin.models import Artifact, Remote, VulnerabilityReport from pulpcore.plugin.util import get_domain +from pulp_python.app.osv import osv_to_pypi_vulnerabilities + log = logging.getLogger(__name__) @@ -377,6 +379,7 @@ def python_content_to_json( last_serial: int releases: Dict urls: Dict + vulnerabilities: List Returns None if version is specified but not found within content_query """ @@ -407,9 +410,27 @@ def python_content_to_json( full_metadata["info"] = python_content_to_info(latest_content[0]) full_metadata["releases"] = python_content_to_releases(all_content, base_path, domain) full_metadata["urls"] = python_content_to_urls(latest_content, base_path, domain) + full_metadata["vulnerabilities"] = _vulnerabilities_for_content( + latest_content, repository_version + ) return full_metadata +def _vulnerabilities_for_content(contents, repository_version=None): + """Load VulnerabilityReports scanned for this repository version and trim to Warehouse shape.""" + if not contents or repository_version is None: + return [] + reports = VulnerabilityReport.objects.filter( + content_id__in=[c.pk for c in contents], + repo_versions=repository_version, + ) + merged = [] + for vulns in reports.values_list("vulns", flat=True): + if vulns: + merged.extend(vulns) + return osv_to_pypi_vulnerabilities(merged) + + def latest_content_version(all_content, version): """ Walks through the content list and finds the instances that are the latest version. diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 6c73575f4..400c9398b 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -26,7 +26,7 @@ RepositoryAddRemoveContentSerializer, RepositorySyncURLSerializer, ) -from pulpcore.plugin.tasking import check_content, dispatch +from pulpcore.plugin.tasking import dispatch from pulpcore.plugin.util import extract_pk from pulp_python.app import models as python_models @@ -356,14 +356,7 @@ def scan(self, request, repository_pk, **kwargs): Scan a repository version for vulnerabilities. """ repository_version = self.get_object() - func = ( - f"{tasks.get_repo_version_content.__module__}.{tasks.get_repo_version_content.__name__}" - ) - task = dispatch( - check_content, - shared_resources=[repository_version.repository], - args=[func, [repository_version.pk]], - ) + task = tasks.dispatch_scan(repository_version.repository, repository_version) return core_viewsets.OperationPostponedResponse(task, request) diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index a742863fb..4cedbbd04 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -324,6 +324,7 @@ def assert_pypi_json(package): package["releases"][version], "Failed to match version", ) + assert package["vulnerabilities"] == [] def assert_download_info(expected, received, message="Failed to match"): @@ -377,6 +378,8 @@ def test_upload_time_reflects_repo_addition( # JSON API json_resp = requests.get(urljoin(distro.base_url, "pypi/twine/json")) assert json_resp.status_code == 200 - json_time = datetime.fromisoformat(json_resp.json()["urls"][0]["upload_time"]) + package = json_resp.json() + json_time = datetime.fromisoformat(package["urls"][0]["upload_time"]) assert json_time > content_created assert json_time == simple_time + assert package["vulnerabilities"] == [] diff --git a/pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py b/pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py new file mode 100644 index 000000000..97bb33c79 --- /dev/null +++ b/pulp_python/tests/functional/api/test_pypi_json_vulnerabilities.py @@ -0,0 +1,171 @@ +from urllib.parse import urljoin, urlsplit + +import pytest +import requests + +from pulp_python.tests.functional.constants import ( + PYPI_URL, + VULNERABILITY_REPORT_TEST_PACKAGE_NAME, + VULNERABILITY_REPORT_TEST_PACKAGES, +) + +WAREHOUSE_VULN_KEYS = { + "id", + "source", + "link", + "aliases", + "details", + "summary", + "fixed_in", + "withdrawn", +} + + +def _index_url(distro, bindings_cfg): + """Build the index URL using the same origin the API client uses.""" + path = urlsplit(distro.base_url).path + if not path.endswith("/"): + path += "/" + return bindings_cfg.host.rstrip("/") + path + + +def _wait_for_child_tasks(pulpcore_bindings, monitor_task, parent_task): + parent = pulpcore_bindings.TasksApi.read(parent_task.pulp_href) + children = parent.child_tasks or [] + assert children, "expected a follow-up vulnerability scan task" + for child in children: + href = child if isinstance(child, str) else child.pulp_href + monitor_task(href) + + +def _assert_warehouse_vulnerabilities(package): + vulns = package["vulnerabilities"] + assert vulns + ids = [vuln["id"] for vuln in vulns] + assert len(ids) == len(set(ids)) + for vuln in vulns: + assert WAREHOUSE_VULN_KEYS <= vuln.keys() + assert vuln["source"] == "osv" + assert vuln["id"] + assert vuln["link"] == f"https://osv.dev/vulnerability/{vuln['id']}" + assert isinstance(vuln["aliases"], list) + assert isinstance(vuln["fixed_in"], list) + + +@pytest.mark.parallel +def test_pypi_json_vulnerabilities_from_manual_scan( + bindings_cfg, + pulpcore_bindings, + python_bindings, + python_remote_factory, + python_repo, + python_repo_factory, + python_distribution_factory, + monitor_task, +): + """A remote without vulnerabilities does not scan; a later scan fills the JSON API.""" + remote = python_remote_factory(url=PYPI_URL, includes=VULNERABILITY_REPORT_TEST_PACKAGES) + sync_task = monitor_task( + python_bindings.RepositoriesPythonApi.sync( + python_repo.pulp_href, dict(remote=remote.pulp_href) + ).task + ) + assert not sync_task.child_tasks + + repo = python_bindings.RepositoriesPythonApi.read(python_repo.pulp_href) + distro = python_distribution_factory(repository=repo) + name = VULNERABILITY_REPORT_TEST_PACKAGE_NAME.lower() + index = _index_url(distro, bindings_cfg) + version_url = urljoin(index, f"pypi/{name}/5.2.1/json") + project_url = urljoin(index, f"pypi/{name}/json") + + scan_task = python_bindings.RepositoriesPythonVersionsApi.scan(repo.latest_version_href) + monitor_task(scan_task.task) + + project = requests.get(project_url).json() + version = requests.get(version_url).json() + _assert_warehouse_vulnerabilities(project) + _assert_warehouse_vulnerabilities(version) + assert [v["id"] for v in project["vulnerabilities"]] == [ + v["id"] for v in version["vulnerabilities"] + ] + + packages = python_bindings.ContentPackagesApi.list( + name=VULNERABILITY_REPORT_TEST_PACKAGE_NAME, + repository_version=repo.latest_version_href, + ) + assert packages.count >= 2 + hrefs = [] + for content in packages.results: + assert content.vuln_report is not None + report = pulpcore_bindings.VulnReportApi.read(content.vuln_report) + assert report.vulns + assert "affected" in report.vulns[0] + hrefs.append(content.pulp_href) + + other = python_repo_factory() + monitor_task( + python_bindings.RepositoriesPythonApi.modify( + other.pulp_href, {"add_content_units": hrefs} + ).task + ) + other = python_bindings.RepositoriesPythonApi.read(other.pulp_href) + other_distro = python_distribution_factory(repository=other) + other_json = requests.get( + urljoin(_index_url(other_distro, bindings_cfg), f"pypi/{name}/json") + ).json() + assert other_json["vulnerabilities"] == [] + + +@pytest.mark.parallel +def test_sync_dispatches_vulnerability_scan( + bindings_cfg, + pulpcore_bindings, + python_bindings, + python_remote_factory, + python_repo, + python_distribution_factory, + monitor_task, +): + """A remote with vulnerabilities=True scans the new repository version after sync.""" + remote = python_remote_factory( + url=PYPI_URL, + includes=VULNERABILITY_REPORT_TEST_PACKAGES, + vulnerabilities=True, + ) + sync_task = monitor_task( + python_bindings.RepositoriesPythonApi.sync( + python_repo.pulp_href, dict(remote=remote.pulp_href) + ).task + ) + _wait_for_child_tasks(pulpcore_bindings, monitor_task, sync_task) + + repo = python_bindings.RepositoriesPythonApi.read(python_repo.pulp_href) + distro = python_distribution_factory(repository=repo) + name = VULNERABILITY_REPORT_TEST_PACKAGE_NAME.lower() + package = requests.get(urljoin(_index_url(distro, bindings_cfg), f"pypi/{name}/json")).json() + _assert_warehouse_vulnerabilities(package) + + +@pytest.mark.parallel +def test_uploaded_package_json_vulnerabilities_empty_until_scan( + bindings_cfg, + python_bindings, + python_content_factory, + python_repo_factory, + python_distribution_factory, + monitor_task, +): + """Uploaded packages have an empty vulnerabilities list until a scan runs.""" + repo = python_repo_factory() + python_content_factory(repository=repo) + distro = python_distribution_factory(repository=repo) + url = urljoin(_index_url(distro, bindings_cfg), "pypi/shelf-reader/json") + + package = requests.get(url).json() + assert package["vulnerabilities"] == [] + + repo = python_bindings.RepositoriesPythonApi.read(repo.pulp_href) + monitor_task(python_bindings.RepositoriesPythonVersionsApi.scan(repo.latest_version_href).task) + package = requests.get(url).json() + assert isinstance(package["vulnerabilities"], list) diff --git a/pulp_python/tests/functional/api/test_vulnerability_report.py b/pulp_python/tests/functional/api/test_vulnerability_report.py index 1630c147f..95d7326e2 100644 --- a/pulp_python/tests/functional/api/test_vulnerability_report.py +++ b/pulp_python/tests/functional/api/test_vulnerability_report.py @@ -28,21 +28,18 @@ def test_vulnerability_report( ) monitor_task(response.task) - # checks - vulns_list = pulpcore_bindings.VulnReportApi.list() - assert len(vulns_list.results) > 0 - for results in vulns_list.results: - assert len(results.vulns) > 0 - for vuln in results.vulns: - assert VULNERABILITY_REPORT_TEST_PACKAGE_NAME.lower() in ( - affected["package"]["name"] for affected in vuln["affected"] - ) - repo_version = python_bindings.RepositoriesPythonVersionsApi.read(latest_version_href) assert repo_version.vuln_report is not None python_packages = python_bindings.ContentPackagesApi.list( name=VULNERABILITY_REPORT_TEST_PACKAGE_NAME, repository_version=latest_version_href ) + assert python_packages.count > 0 for content in python_packages.results: assert content.vuln_report is not None + report = pulpcore_bindings.VulnReportApi.read(content.vuln_report) + assert len(report.vulns) > 0 + for vuln in report.vulns: + assert VULNERABILITY_REPORT_TEST_PACKAGE_NAME.lower() in ( + affected["package"]["name"] for affected in vuln["affected"] + ) diff --git a/pulp_python/tests/unit/test_vulnerabilities.py b/pulp_python/tests/unit/test_vulnerabilities.py new file mode 100644 index 000000000..cd55a1a34 --- /dev/null +++ b/pulp_python/tests/unit/test_vulnerabilities.py @@ -0,0 +1,84 @@ +from pulp_python.app.osv import osv_to_pypi_vulnerabilities + + +def test_osv_to_pypi_maps_fixed_versions_and_skips_git_shas(): + vulns = [ + { + "id": "PYSEC-2020-35", + "aliases": ["CVE-2020-7471"], + "details": "SQL injection", + "summary": None, + "withdrawn": None, + "affected": [ + { + "package": {"name": "django", "ecosystem": "PyPI"}, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + {"introduced": "3.0"}, + {"fixed": "3.0.3"}, + {"fixed": "eb31d845323618d688ad429479c6dda973056136"}, + ], + } + ], + } + ], + } + ] + + result = osv_to_pypi_vulnerabilities(vulns) + + assert result == [ + { + "id": "PYSEC-2020-35", + "source": "osv", + "link": "https://osv.dev/vulnerability/PYSEC-2020-35", + "aliases": ["CVE-2020-7471"], + "details": "SQL injection", + "summary": None, + "fixed_in": ["3.0.3"], + "withdrawn": None, + } + ] + + +def test_osv_to_pypi_deduplicates_by_id(): + vulns = [ + {"id": "GHSA-aaaa", "aliases": ["CVE-1"], "details": "first"}, + {"id": "GHSA-aaaa", "aliases": ["CVE-1"], "details": "duplicate"}, + {"id": "GHSA-bbbb", "aliases": [], "details": "other"}, + ] + + result = osv_to_pypi_vulnerabilities(vulns) + + assert [v["id"] for v in result] == ["GHSA-aaaa", "GHSA-bbbb"] + assert result[0]["details"] == "first" + + +def test_osv_to_pypi_preserves_withdrawn_and_missing_summary(): + vulns = [ + { + "id": "PYSEC-2022-XXX", + "aliases": ["CVE-2022-XXXXX"], + "details": "A long description.", + "withdrawn": "2022-06-28T16:39:06Z", + "affected": [ + { + "ranges": [{"events": [{"fixed": "1.2.3"}]}], + } + ], + } + ] + + result = osv_to_pypi_vulnerabilities(vulns) + + assert result[0]["summary"] is None + assert result[0]["withdrawn"] == "2022-06-28T16:39:06Z" + assert result[0]["fixed_in"] == ["1.2.3"] + + +def test_osv_to_pypi_empty_and_missing_ids(): + assert osv_to_pypi_vulnerabilities(None) == [] + assert osv_to_pypi_vulnerabilities([]) == [] + assert osv_to_pypi_vulnerabilities([{"details": "no id"}]) == []