diff --git a/.gitignore b/.gitignore index 29c94b654..444a89de7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ # Distribution / packaging .Python env/ +.venv/ build/ develop-eggs/ dist/ @@ -61,3 +62,6 @@ target/ # PyCharm .idea + +# VS Code / Cursor +.vscode/ diff --git a/CHANGES/1358.feature b/CHANGES/1358.feature new file mode 100644 index 000000000..a3a40f519 --- /dev/null +++ b/CHANGES/1358.feature @@ -0,0 +1 @@ +Added repository package catalog and metrics endpoints, plus ``collapse_builds`` and ``base_version`` on the Python package content API. Existing installs pick up access policy for the new actions on migrate unless the policy was customized. diff --git a/docs/index.md b/docs/index.md index 80c26e434..d5f244a4b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,6 +12,7 @@ The REST API documentation for `pulp_python` is available [here](site:pulp_pytho - [Create local mirrors of PyPI](site:pulp_python/docs/user/guides/sync/) that you have full control over - [Upload your own Python packages](site:pulp_python/docs/user/guides/upload/) +- [Browse the package catalog](site:pulp_python/docs/user/guides/catalog/) over the REST API - [Perform pip install](site:pulp_python/docs/user/guides/host/) from your Pulp Python repositories - Download packages on-demand to reduce disk usage - Every operation creates a restorable snapshot with Versioned Repositories diff --git a/docs/user/guides/_SUMMARY.md b/docs/user/guides/_SUMMARY.md index d9c01bdb2..36699ede5 100644 --- a/docs/user/guides/_SUMMARY.md +++ b/docs/user/guides/_SUMMARY.md @@ -1,6 +1,7 @@ * [Set up your own PyPI](pypi.md) * [Sync from Remote Repositories](sync.md) * [Upload and Manage Content](upload.md) +* [Browse the package catalog](catalog.md) * [Host Python Content](host.md) * [Vulnerability Report](vulnerability_report.md) * [Attestation Hosting](attestation.md) diff --git a/docs/user/guides/catalog.md b/docs/user/guides/catalog.md new file mode 100644 index 000000000..65a35ddc2 --- /dev/null +++ b/docs/user/guides/catalog.md @@ -0,0 +1,96 @@ +# Browse the package catalog + +Pulp CLI commands for these endpoints are generated from the OpenAPI spec in a separate package; until that is updated, use HTTP. + +The content list (`/pulp/api/v3/content/python/packages/`) returns **one row per distribution file** (wheel, sdist, …). For catalog UIs and automation that need **one row per package name**, plus repository metrics, use the repository package index. + +These endpoints default to the **latest complete repository version**. `{pulp_id}` is the repository UUID. Pass `repository_version` (HREF or PRN) to read a specific version of that repository. + +## List packages + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/?limit=10" +``` + +Pagination `count` is the number of **distinct packages** (`name_normalized`), not files. + +Each row includes both a simple version list and per-version metadata: + +```json +{ + "name": "shelf-reader", + "name_normalized": "shelf-reader", + "versions": ["0.1"], + "latest_releases": [ + { + "version": "0.1", + "release": "", + "created_at": "2026-08-10T10:45:08.099362Z" + } + ] +} +``` + +`set(versions)` is always the same as `set(latest_releases[].version)`. There is one `latest_releases` entry per **logical version** (after stripping a trailing rebuild suffix `\.[a-zA-Z]+-\d+$`), not per wheel or sdist. + +`created_at` is when that logical version entered the repository: the earliest `RepositoryContent.pulp_created` among its files, falling back to the content unit's `pulp_created`. `release` is empty until Python rebuilds are stored. + +### Prefix search + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/" \ + name_normalized__istartswith==shelf +``` + +`name_normalized__istartswith` and `name__istartswith` are case-insensitive (`ILIKE`). Prefix search belongs on this index, not on the flat content list. + +## Repository metrics + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/metrics/" +``` + +```json +{ + "package_count": 3, + "version_count": 9, + "build_count": 9 +} +``` + +Counts use Python package content units in that repository version (not filtered by `packagetype`): + +| Field | Identity | +|-------|----------| +| `package_count` | distinct `name_normalized` | +| `version_count` | distinct `(name_normalized, base_version)` after rebuild-suffix strip | +| `build_count` | distinct `(name_normalized, full version)` | + +Until rebuild suffixes exist, `version_count` equals `build_count`. + +## List versions of a package + +Use the existing content API. Pass `packagetype=sdist` for one representative file per PEP version (retry with `packagetype=bdist_wheel` if a release is wheel-only). + +`collapse_builds=true` keeps one unit per logical version (`name_normalized` + `base_version`), the one with the latest `pulp_created`. Do not nest rebuilds on this list. Clients can drain Pulp `next` if the page is full. + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/content/python/packages/" \ + name==shelf-reader \ + packagetype==sdist \ + collapse_builds==true \ + repository_version=="${LATEST_VERSION_HREF}" +``` + +Every content row includes `base_version` (stripped version; equal to `version` when there is no suffix). + +## Get one version + +Omit `collapse_builds`. Filter with `name`, `version`, and `packagetype=sdist`: + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/content/python/packages/" \ + name==shelf-reader \ + version==0.1 \ + packagetype==sdist +``` diff --git a/pulp_python/app/catalog.py b/pulp_python/app/catalog.py new file mode 100644 index 000000000..9c56ea525 --- /dev/null +++ b/pulp_python/app/catalog.py @@ -0,0 +1,166 @@ +"""Helpers for repository package catalog, metrics, and rebuild collapse.""" + +from collections import defaultdict + +from django.db.models import CharField, Func, Max, Min, Q, Value +from packaging.version import InvalidVersion, Version + +from pulp_python.app.models import PythonPackageContent + +# POSIX regex for REGEXP_REPLACE. PostgreSQL does not treat ``\d`` as digits. +BUILD_SUFFIX_PG_REGEX = r"\.[a-zA-Z]+-[0-9]+$" + + +def base_version_annotation(field_name="version"): + """SQL expression that strips a trailing rebuild suffix from ``version``. + + PostgreSQL POSIX regex does not treat ``\\d`` as digits, so the SQL pattern + uses ``[0-9]`` while the Python pattern in ``strip_build_suffix`` uses ``\\d``. + Implemented with ``REGEXP_REPLACE`` so it does not depend on Django's + ``RegexpReplace`` (not present in every Django 4.2/5.2 packaging Pulp uses). + """ + return Func( + field_name, + Value(BUILD_SUFFIX_PG_REGEX), + Value(""), + function="REGEXP_REPLACE", + output_field=CharField(), + ) + + +def collapse_python_builds(queryset): + """Keep one content unit per ``(name_normalized, base_version)``. + + ``base_version`` is ``version`` with a trailing rebuild suffix stripped. + The unit with the latest ``pulp_created`` is kept. Callers that want one + row per logical version (not per wheel/sdist) should also filter + ``packagetype``. + """ + return ( + queryset.prefetch_related(None) + .annotate(_collapse_base_version=base_version_annotation()) + .order_by("name_normalized", "_collapse_base_version", "-pulp_created") + .distinct("name_normalized", "_collapse_base_version") + ) + + +def python_packages_in_version(repository_version): + """Python package content contained in ``repository_version``.""" + if repository_version is None: + return PythonPackageContent.objects.none() + return PythonPackageContent.objects.filter(pk__in=repository_version.content) + + +def apply_package_prefix_filters(queryset, name_normalized_prefix=None, name_prefix=None): + """Apply case-insensitive prefix filters used by the package index.""" + if name_normalized_prefix: + queryset = queryset.filter(name_normalized__istartswith=name_normalized_prefix) + if name_prefix: + queryset = queryset.filter(name__istartswith=name_prefix) + return queryset + + +def distinct_package_names_qs(content_qs): + """One row per distinct ``name_normalized``, ordered for stable pagination.""" + return ( + content_qs.order_by() + .values("name_normalized") + .annotate(name=Max("name")) + .order_by("name_normalized") + ) + + +def _version_sort_key(version): + try: + return (0, Version(version)) + except InvalidVersion: + return (1, version) + + +def assemble_package_index(content_qs, name_rows, repository, repository_version): + """Build package-index dicts for ``name_rows``. + + ``created_at`` is the earliest repository-membership time + (``RepositoryContent.pulp_created``) of any file of that logical version + in ``repository_version``, falling back to the content unit's ``pulp_created``. + """ + if not name_rows or repository_version is None: + return [] + + names = [row["name_normalized"] for row in name_rows] + name_by_normalized = {row["name_normalized"]: row["name"] for row in name_rows} + + in_this_version = Q( + version_memberships__repository=repository, + version_memberships__version_added__number__lte=repository_version.number, + ) & ( + Q(version_memberships__version_removed__isnull=True) + | Q(version_memberships__version_removed__number__gt=repository_version.number) + ) + + release_rows = ( + content_qs.filter(name_normalized__in=names) + .annotate(_base_version=base_version_annotation()) + .values("name_normalized", "_base_version") + .annotate( + membership_created=Min( + "version_memberships__pulp_created", + filter=in_this_version, + ), + unit_created=Min("pulp_created"), + ) + ) + + releases_by_name = defaultdict(list) + for rel in release_rows: + releases_by_name[rel["name_normalized"]].append(rel) + + result = [] + for row in name_rows: + normalized = row["name_normalized"] + rels = sorted( + releases_by_name.get(normalized, []), + key=lambda item: _version_sort_key(item["_base_version"]), + ) + versions = [item["_base_version"] for item in rels] + latest_releases = [ + { + "version": item["_base_version"], + "release": "", + "created_at": item["membership_created"] or item["unit_created"], + } + for item in rels + ] + result.append( + { + "name": name_by_normalized[normalized], + "name_normalized": normalized, + "versions": versions, + "latest_releases": latest_releases, + } + ) + return result + + +def repository_metrics(content_qs): + """Distinct package / logical-version / build counts for package content. + + Identity is always ``PythonPackageContent`` (not filtered by packagetype): + + * ``package_count``: distinct ``name_normalized`` + * ``version_count``: distinct ``(name_normalized, base_version)`` + * ``build_count``: distinct ``(name_normalized, version)`` + + Until rebuild suffixes exist, ``version_count`` equals ``build_count``. + """ + content_qs = content_qs.order_by() + return { + "package_count": content_qs.values("name_normalized").distinct().count(), + "version_count": ( + content_qs.annotate(_base_version=base_version_annotation()) + .values("name_normalized", "_base_version") + .distinct() + .count() + ), + "build_count": content_qs.values("name_normalized", "version").distinct().count(), + } diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 038bb3584..c6601ddd5 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -33,6 +33,7 @@ canonicalize_name, get_project_metadata_from_file, parse_project_metadata, + strip_build_suffix, ) log = logging.getLogger(__name__) @@ -231,6 +232,16 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa help_text=_("The packages version number."), read_only=True, ) + base_version = serializers.SerializerMethodField( + help_text=_( + "The package version with a trailing rebuild suffix stripped " + r"(matching \.[a-zA-Z]+-\d+$). Equal to version when no suffix is present." + ), + ) + + def get_base_version(self, obj): + return strip_build_suffix(obj.version) + # Version 1.1 classifiers = serializers.JSONField( required=False, @@ -518,6 +529,7 @@ class Meta: "platform", "summary", "version", + "base_version", "classifiers", "download_url", "supported_platform", @@ -636,11 +648,70 @@ class Meta: "packagetype", "name", "version", + "base_version", "sha256", ) model = python_models.PythonPackageContent +class PythonPackageReleaseSerializer(serializers.Serializer): + """One logical version on the repository package index.""" + + version = serializers.CharField( + help_text=_("Logical version key (rebuild suffix stripped)."), + ) + release = serializers.CharField( + help_text=_( + "Rebuild/release qualifier within the version line. Empty until rebuilds exist." + ), + allow_blank=True, + ) + created_at = serializers.DateTimeField( + help_text=_( + "When this logical version entered the repository: the earliest " + "RepositoryContent.pulp_created among files of this version, falling back to " + "the content unit's pulp_created." + ), + ) + + +class PythonRepositoryPackageSerializer(serializers.Serializer): + """One distinct package in a repository version (not per wheel/sdist file).""" + + name = serializers.CharField(help_text=_("A representative project name for this package.")) + name_normalized = serializers.CharField( + help_text=_("PEP 503 normalized package name. Index rows are unique on this field."), + ) + versions = serializers.ListField( + child=serializers.CharField(), + help_text=_( + "Distinct logical version keys after rebuild-suffix strip. " + "The set of values matches latest_releases[].version." + ), + ) + latest_releases = PythonPackageReleaseSerializer( + many=True, + help_text=_( + "One object per logical version (not per distribution file). " + "set(versions) === set(latest_releases[].version)." + ), + ) + + +class PythonRepositoryMetricsSerializer(serializers.Serializer): + """Distinct package / version / build counts for a repository version.""" + + package_count = serializers.IntegerField( + help_text=_("Distinct name_normalized values among Python package content units."), + ) + version_count = serializers.IntegerField( + help_text=_("Distinct (name_normalized, base_version) pairs after rebuild-suffix strip."), + ) + build_count = serializers.IntegerField( + help_text=_("Distinct (name_normalized, full version) pairs among package content units."), + ) + + class PackageProvenanceSerializer(core_serializers.NoArtifactContentUploadSerializer): """ A Serializer for PackageProvenance. diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 9e08c77c6..aed0a050c 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -26,6 +26,16 @@ log = logging.getLogger(__name__) +# Rebuild suffix (Lightwell-style): 5.3.18.rhlw-00003 -> 5.3.18. Not hard-coded to "rhlw". +BUILD_SUFFIX_RE = re.compile(r"\.[a-zA-Z]+-\d+$") + + +def strip_build_suffix(version): + """Return ``version`` with a trailing rebuild suffix removed, else unchanged.""" + if not version: + return version + return BUILD_SUFFIX_RE.sub("", version) + PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL" """TODO This serial constant is temporary until Python repositories implements serials""" diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 6c73575f4..be21e40e0 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -4,7 +4,13 @@ from django.db import transaction from django_filters import CharFilter from django_filters.rest_framework import filters as drf_filters -from drf_spectacular.utils import extend_schema, extend_schema_view +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import ( + OpenApiParameter, + extend_schema, + extend_schema_view, + inline_serializer, +) from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.utils import canonicalize_name from rest_framework import status @@ -16,7 +22,7 @@ RetrieveModelMixin, ) from rest_framework.response import Response -from rest_framework.serializers import ValidationError +from rest_framework.serializers import IntegerField, URLField, ValidationError from pulpcore.plugin import viewsets as core_viewsets from pulpcore.plugin.actions import ModifyRepositoryActionMixin @@ -32,6 +38,14 @@ from pulp_python.app import models as python_models from pulp_python.app import serializers as python_serializers from pulp_python.app import tasks +from pulp_python.app.catalog import ( + apply_package_prefix_filters, + assemble_package_index, + collapse_python_builds, + distinct_package_names_qs, + python_packages_in_version, + repository_metrics, +) class PythonRepositoryViewSet( @@ -64,7 +78,7 @@ class PythonRepositoryViewSet( ], }, { - "action": ["retrieve"], + "action": ["retrieve", "packages", "metrics"], "principal": "authenticated", "effect": "allow", "condition": "has_model_or_domain_or_obj_perms:python.view_pythonrepository", @@ -138,6 +152,24 @@ class PythonRepositoryViewSet( "python.pythonrepository_viewer": ["python.view_pythonrepository"], } + def filter_queryset(self, queryset): + """Do not apply the repository FilterSet to package-index query params.""" + if getattr(self, "action", None) in ("packages", "metrics"): + return queryset + return super().filter_queryset(queryset) + + def _requested_repository_version(self, repository): + """Resolve optional ``repository_version`` href/PRN, else latest complete version.""" + href = self.request.query_params.get("repository_version") + if not href: + return repository.latest_version() + repo_version = self.get_resource(href, RepositoryVersion) + if repo_version.repository_id != repository.pk: + raise ValidationError( + {"repository_version": "Must be a version of this repository."} + ) + return repo_version + @extend_schema( description="Trigger an asynchronous task to create a new repository version.", summary="Modify Repository Content", @@ -247,6 +279,122 @@ def sync(self, request, pk, **kwargs): ) return core_viewsets.OperationPostponedResponse(result, request) + @extend_schema( + summary="List packages", + description=( + "Return one row per distinct package name in a repository version " + "(latest complete version if repository_version is omitted). " + "Pagination count is the number of distinct packages, not files. " + "Each row includes versions (logical version keys after rebuild-suffix strip) " + "and latest_releases (one object per logical version). " + "created_at is the earliest repository-membership time " + "(RepositoryContent.pulp_created) of any file of that logical version, " + "falling back to the content unit's pulp_created. " + "release is empty until Python rebuilds are stored. " + "set(versions) === set(latest_releases[].version)." + ), + parameters=[ + OpenApiParameter( + name="repository_version", + type=OpenApiTypes.URI, + location=OpenApiParameter.QUERY, + required=False, + description=( + "HREF or PRN of a version of this repository. " + "Defaults to the latest complete version." + ), + ), + OpenApiParameter( + name="name_normalized__istartswith", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Case-insensitive prefix on the PEP 503 normalized package name.", + ), + OpenApiParameter( + name="name__istartswith", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Case-insensitive prefix on the original package name.", + ), + ], + responses={ + 200: inline_serializer( + name="PaginatedPythonRepositoryPackageList", + fields={ + "count": IntegerField(), + "next": URLField(allow_null=True), + "previous": URLField(allow_null=True), + "results": python_serializers.PythonRepositoryPackageSerializer(many=True), + }, + ) + }, + ) + @action( + detail=True, + methods=["get"], + serializer_class=python_serializers.PythonRepositoryPackageSerializer, + ) + def packages(self, request, pk): + """List distinct packages in a repository version.""" + repository = self.get_object() + repo_version = self._requested_repository_version(repository) + content_qs = python_packages_in_version(repo_version) + content_qs = apply_package_prefix_filters( + content_qs, + name_normalized_prefix=request.query_params.get("name_normalized__istartswith"), + name_prefix=request.query_params.get("name__istartswith"), + ) + names_qs = distinct_package_names_qs(content_qs) + page = self.paginate_queryset(names_qs) + rows = assemble_package_index( + content_qs, + page if page is not None else list(names_qs), + repository, + repo_version, + ) + serializer = self.get_serializer(rows, many=True) + if page is not None: + return self.get_paginated_response(serializer.data) + return Response(serializer.data) + + @extend_schema( + summary="Repository metrics", + description=( + "Distinct counts for Python package content in a repository version " + "(latest complete version if repository_version is omitted). " + "package_count is distinct name_normalized. version_count is distinct " + "(name_normalized, base_version) after rebuild-suffix strip. build_count is " + "distinct (name_normalized, full version). Counts are not filtered by " + "packagetype. Until rebuild suffixes exist, version_count equals build_count." + ), + parameters=[ + OpenApiParameter( + name="repository_version", + type=OpenApiTypes.URI, + location=OpenApiParameter.QUERY, + required=False, + description=( + "HREF or PRN of a version of this repository. " + "Defaults to the latest complete version." + ), + ), + ], + responses={200: python_serializers.PythonRepositoryMetricsSerializer}, + ) + @action( + detail=True, + methods=["get"], + serializer_class=python_serializers.PythonRepositoryMetricsSerializer, + ) + def metrics(self, request, pk): + """Return package / version / build counts for a repository version.""" + repository = self.get_object() + repo_version = self._requested_repository_version(repository) + serializer = self.get_serializer( + repository_metrics(python_packages_in_version(repo_version)) + ) + return Response(serializer.data) + class PythonBlocklistEntryViewSet( core_viewsets.NamedModelViewSet, @@ -506,6 +654,25 @@ class PythonPackageContentFilter(core_viewsets.ContentFilter): field_name="version", help_text="Filter by PEP 440 version specifier (e.g., >=2.4,<3.0 or ~=1.26)", ) + collapse_builds = drf_filters.BooleanFilter( + method="filter_collapse_builds", + help_text=( + "When true, collapse rebuilds of the same logical version: strip a trailing " + r"suffix matching \.[a-zA-Z]+-\d+$ from version, then keep one content unit " + "per (name_normalized, base_version) with the latest pulp_created. " + "Pass packagetype=sdist so wheel and sdist files are not collapsed together. " + "Default false." + ), + ) + + def filter_collapse_builds(self, qs, name, value): + """Documented on the FilterSet; applied in the viewset after ordering. + + DISTINCT ON requires ORDER BY to start with the distinct columns. The + viewset applies collapse after other filter backends so that ordering + cannot break it. + """ + return qs class Meta: model = python_models.PythonPackageContent @@ -539,6 +706,18 @@ class PythonPackageSingleArtifactContentUploadViewSet( minimal_serializer_class = python_serializers.MinimalPythonPackageContentSerializer filterset_class = PythonPackageContentFilter + def filter_queryset(self, queryset): + """Apply ``collapse_builds`` after other backends so DISTINCT ON stays valid.""" + queryset = super().filter_queryset(queryset) + if getattr(self, "action", "") != "list": + return queryset + raw = self.request.query_params.get("collapse_builds") + if raw is None or raw == "": + return queryset + if str(raw).lower() in ("true", "t", "yes", "y", "1"): + return collapse_python_builds(queryset) + return queryset + DEFAULT_ACCESS_POLICY = { "statements": [ { diff --git a/pulp_python/tests/functional/api/test_catalog.py b/pulp_python/tests/functional/api/test_catalog.py new file mode 100644 index 000000000..62d1e2961 --- /dev/null +++ b/pulp_python/tests/functional/api/test_catalog.py @@ -0,0 +1,229 @@ +"""Catalog API tests. + +Generated client methods are unavailable until `oci-env generate-client` is rerun. +""" +from urllib.parse import urljoin + +import pytest +import requests + +from pulp_python.tests.functional.constants import PYTHON_SM_PROJECT_SPECIFIER + + +def _api_get(bindings_cfg, path, **params): + url = urljoin(bindings_cfg.host + "/", path.lstrip("/")) + response = requests.get(url, params=params, auth=(bindings_cfg.username, bindings_cfg.password)) + assert response.status_code == 200, response.text + return response.json() + + +def _content_packages_path(repo_href): + marker = "/api/v3/" + idx = repo_href.find(marker) + assert idx != -1, repo_href + return f"{repo_href[: idx + len(marker)]}content/python/packages/" + + +def _assert_package_row(pkg): + assert pkg["name"] + assert pkg["name_normalized"] + assert set(pkg["versions"]) == {rel["version"] for rel in pkg["latest_releases"]} + for rel in pkg["latest_releases"]: + assert rel["release"] == "" + assert rel["created_at"] + + +@pytest.fixture +def sm_repo(python_repo_with_sync, python_remote_factory): + remote = python_remote_factory(includes=PYTHON_SM_PROJECT_SPECIFIER) + return python_repo_with_sync(remote) + + +@pytest.mark.parallel +def test_package_list_grouping_and_pagination(bindings_cfg, sm_repo): + """Package index is one row per name, and count is distinct packages not files.""" + data = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=1) + assert data["count"] == 3 + assert len(data["results"]) == 1 + _assert_package_row(data["results"][0]) + + page2 = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=1, offset=1) + assert page2["count"] == 3 + assert page2["results"][0]["name_normalized"] != data["results"][0]["name_normalized"] + + all_rows = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=100)["results"] + assert {pkg["name_normalized"] for pkg in all_rows} == {"aiohttp", "celery", "django"} + django = next(pkg for pkg in all_rows if pkg["name_normalized"] == "django") + assert set(django["versions"]) == {"1.10.1", "1.10.2", "1.10.3", "1.10.4"} + # Dual-field contract: one latest_releases entry per logical version, not per wheel/sdist. + assert len(django["latest_releases"]) == 4 + + +@pytest.mark.parallel +def test_package_list_istartswith(bindings_cfg, sm_repo): + """Prefix search is case-insensitive on the package index.""" + data = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", name_normalized__istartswith="djan" + ) + assert data["count"] == 1 + assert data["results"][0]["name_normalized"] == "django" + + data = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", name_normalized__istartswith="DJAN" + ) + assert data["count"] == 1 + assert data["results"][0]["name_normalized"] == "django" + + data = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", name__istartswith="Cel") + assert data["count"] == 1 + assert data["results"][0]["name_normalized"] == "celery" + + data = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", name_normalized__istartswith="shelf" + ) + assert data["count"] == 0 + + +@pytest.mark.parallel +def test_package_list_empty_repository(bindings_cfg, python_repo_factory): + repo = python_repo_factory() + data = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert data["count"] == 0 + assert data["results"] == [] + + +@pytest.mark.parallel +def test_repository_metrics(bindings_cfg, sm_repo, python_repo_factory): + """Metrics count distinct packages / logical versions / builds, not files.""" + data = _api_get(bindings_cfg, f"{sm_repo.pulp_href}metrics/") + assert data["package_count"] == 3 + # aiohttp 3 + celery 2 + Django 4; no rebuild suffixes in fixtures. + assert data["version_count"] == 9 + assert data["build_count"] == 9 + assert data["version_count"] == data["build_count"] + + empty = _api_get(bindings_cfg, f"{python_repo_factory().pulp_href}metrics/") + assert empty == {"package_count": 0, "version_count": 0, "build_count": 0} + + +@pytest.mark.parallel +def test_packages_and_metrics_repository_version(bindings_cfg, sm_repo, python_repo_factory): + """repository_version selects a snapshot; omitted uses the latest complete version.""" + latest_href = sm_repo.latest_version_href + v0_href = f"{sm_repo.pulp_href}versions/0/" + + default_pkgs = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/") + explicit_pkgs = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", repository_version=latest_href + ) + assert default_pkgs["count"] == explicit_pkgs["count"] == 3 + + v0_pkgs = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", repository_version=v0_href) + assert v0_pkgs["count"] == 0 + assert v0_pkgs["results"] == [] + + default_metrics = _api_get(bindings_cfg, f"{sm_repo.pulp_href}metrics/") + explicit_metrics = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}metrics/", repository_version=latest_href + ) + assert default_metrics == explicit_metrics + v0_metrics = _api_get(bindings_cfg, f"{sm_repo.pulp_href}metrics/", repository_version=v0_href) + assert v0_metrics == {"package_count": 0, "version_count": 0, "build_count": 0} + + other = python_repo_factory() + url = urljoin(bindings_cfg.host + "/", f"{sm_repo.pulp_href}packages/".lstrip("/")) + response = requests.get( + url, + params={"repository_version": other.latest_version_href}, + auth=(bindings_cfg.username, bindings_cfg.password), + ) + assert response.status_code == 400, response.text + + +@pytest.mark.parallel +def test_collapse_builds_and_base_version(bindings_cfg, sm_repo): + """collapse_builds keeps one unit per logical version; base_version is always present.""" + path = _content_packages_path(sm_repo.pulp_href) + repo_version = sm_repo.latest_version_href + + expanded = _api_get( + bindings_cfg, + path, + name="Django", + repository_version=repo_version, + collapse_builds="false", + limit=100, + ) + collapsed = _api_get( + bindings_cfg, + path, + name="Django", + repository_version=repo_version, + collapse_builds="true", + limit=100, + ) + # Wheel + sdist per Django version collapse when packagetype is omitted. + assert expanded["count"] == 8 + assert collapsed["count"] == 4 + assert {item["base_version"] for item in collapsed["results"]} == { + "1.10.1", + "1.10.2", + "1.10.3", + "1.10.4", + } + for item in expanded["results"] + collapsed["results"]: + assert item["base_version"] == item["version"] + + sdist_false = _api_get( + bindings_cfg, + path, + name="Django", + packagetype="sdist", + repository_version=repo_version, + collapse_builds="false", + limit=100, + ) + sdist_true = _api_get( + bindings_cfg, + path, + name="Django", + packagetype="sdist", + repository_version=repo_version, + collapse_builds="true", + limit=100, + ) + assert sdist_false["count"] == 4 + assert sdist_true["count"] == 4 + assert {item["version"] for item in sdist_true["results"]} == { + "1.10.1", + "1.10.2", + "1.10.3", + "1.10.4", + } + + +@pytest.mark.parallel +def test_package_get_base_version_without_collapse(bindings_cfg, python_repo_with_sync): + """PackageGet uses the content list without collapse_builds; base_version is still present.""" + repo = python_repo_with_sync() + path = _content_packages_path(repo.pulp_href) + data = _api_get( + bindings_cfg, + path, + name="shelf-reader", + version="0.1", + packagetype="sdist", + ) + assert data["count"] == 1 + item = data["results"][0] + assert item["version"] == "0.1" + assert item["base_version"] == "0.1" + assert "collapse_builds" not in item + + pkgs = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert pkgs["count"] == 1 + row = pkgs["results"][0] + _assert_package_row(row) + assert row["name_normalized"] == "shelf-reader" + assert row["versions"] == ["0.1"] + assert len(row["latest_releases"]) == 1 diff --git a/pulp_python/tests/unit/test_catalog.py b/pulp_python/tests/unit/test_catalog.py new file mode 100644 index 000000000..bcc0bc002 --- /dev/null +++ b/pulp_python/tests/unit/test_catalog.py @@ -0,0 +1,21 @@ +import pytest + +from pulp_python.app.utils import strip_build_suffix + + +@pytest.mark.parametrize( + "version,expected", + [ + ("0.1", "0.1"), + ("5.3.18", "5.3.18"), + ("5.3.18.rhlw-00003", "5.3.18"), + ("1.0.0.abc-1", "1.0.0"), + ("1.0.0.ABC-99", "1.0.0"), + ("1.0.foo-bar", "1.0.foo-bar"), + ("1.0.rhlw-00003.extra", "1.0.rhlw-00003.extra"), + ("", ""), + (None, None), + ], +) +def test_strip_build_suffix(version, expected): + assert strip_build_suffix(version) == expected