diff --git a/CHANGES/1358.feature b/CHANGES/1358.feature new file mode 100644 index 00000000..2d8c205a --- /dev/null +++ b/CHANGES/1358.feature @@ -0,0 +1 @@ +Added repository and repository-version endpoints that list Python packages and version update times with limit/offset pagination. diff --git a/docs/index.md b/docs/index.md index 80c26e43..ff9bb16d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,6 +12,7 @@ The REST API documentation for `pulp_python` is available [here](site:pulp_pytho - [Create local mirrors of PyPI](site:pulp_python/docs/user/guides/sync/) that you have full control over - [Upload your own Python packages](site:pulp_python/docs/user/guides/upload/) +- [Browse the package catalog](site:pulp_python/docs/user/guides/catalog/) over the REST API - [Perform pip install](site:pulp_python/docs/user/guides/host/) from your Pulp Python repositories - Download packages on-demand to reduce disk usage - Every operation creates a restorable snapshot with Versioned Repositories @@ -34,5 +35,4 @@ Users may also find pulpcore’s conceptual docs useful. This documentation falls into two main categories: 1. `How-to Guides` shows the **major features** of the Python plugin, with links to reference docs. -2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each -minor feature, including all fields and options. +2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each minor feature, including all fields and options. diff --git a/docs/user/guides/_SUMMARY.md b/docs/user/guides/_SUMMARY.md index d9c01bdb..36699ede 100644 --- a/docs/user/guides/_SUMMARY.md +++ b/docs/user/guides/_SUMMARY.md @@ -1,6 +1,7 @@ * [Set up your own PyPI](pypi.md) * [Sync from Remote Repositories](sync.md) * [Upload and Manage Content](upload.md) +* [Browse the package catalog](catalog.md) * [Host Python Content](host.md) * [Vulnerability Report](vulnerability_report.md) * [Attestation Hosting](attestation.md) diff --git a/docs/user/guides/catalog.md b/docs/user/guides/catalog.md new file mode 100644 index 00000000..10025f25 --- /dev/null +++ b/docs/user/guides/catalog.md @@ -0,0 +1,60 @@ +# Browse the package catalog + +The package catalog returns one row per normalized package name. Use the +repository URL to read its latest complete repository version, or a repository +version URL to read a specific complete version. `${REPO_PK}` is the repository UUID. + +## List packages + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/?limit=10" +``` + +To read a specific repository version, set `${VERSION_NUMBER}` to its number: + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/versions/${VERSION_NUMBER}/packages/?limit=10" +``` + +`count` is the number of distinct packages, not files. `summary`, `description`, +`author`, `author_email`, and `license` come from the highest package version in +PEP 440 order. Metadata comes from an sdist of that version when available. +Otherwise, it comes from that version's newest file. + +`versions` has one entry per stored version string, in descending PEP 440 order. +A wheel and an sdist of the same version share one entry. Different version +strings, such as `1.0.0` and `1.0.0+test.1`, produce separate entries. + +Each version's `last_updated` is the latest `pulp_created` timestamp among its +files in the selected repository version. This records when Pulp created the +content, not the upstream release date. + +```json +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "name": "shelf-reader", + "name_normalized": "shelf-reader", + "summary": "A small example package", + "description": "", + "author": "", + "author_email": "", + "license": "", + "versions": [ + {"version": "0.1", "last_updated": "2026-11-11T12:00:00Z"} + ] + } + ] +} +``` + +Results are ordered by `name_normalized`. Use `limit` and `offset` to page +through the catalog. Use a repository version URL to keep every page on the +same repository version. Missing or incomplete repository versions return HTTP 404. + +Filter names with `name_normalized`, `name_normalized__icontains`, or +`name_normalized__istartswith`. Use `ordering=-name_normalized` to list names +in descending order. diff --git a/pulp_python/app/catalog.py b/pulp_python/app/catalog.py new file mode 100644 index 00000000..7f41d868 --- /dev/null +++ b/pulp_python/app/catalog.py @@ -0,0 +1,102 @@ +"""Helpers for the repository package catalog.""" + +from django import VERSION as DJANGO_VERSION +from django.contrib.postgres.aggregates import JSONBAgg +from django.db.models import Case, IntegerField, Value, When +from packaging.version import InvalidVersion, Version + +from pulp_python.app.models import PythonPackageContent + + +def _version_sort_key(version): + """Sort invalid version strings after valid PEP 440 versions.""" + try: + return (True, Version(version)) + except InvalidVersion: + return (False, version) + + +class PackageVersions(JSONBAgg): + """Aggregate values by version, keeping the last value for duplicate keys.""" + + function = "JSONB_OBJECT_AGG" + arity = 2 + + +def package_catalog_queryset(content, ordering=("name_normalized",)): + """Collect version timestamps and metadata IDs for each normalized package name. + + Wheels and sdists with the same version share an aggregate entry. + Each timestamp is the newest file's `pulp_created`. Use sdist metadata when available. + """ + # Aggregate ordering selects the newest timestamp and prefers sdist metadata. + ordering_argument = "order_by" if DJANGO_VERSION >= (5, 2) else "ordering" + file_ordering = { + ordering_argument: ( + "version", + Case( + When(packagetype="sdist", then=Value(1)), + default=Value(0), + output_field=IntegerField(), + ), + "pulp_created", + "-pk", + ) + } + return ( + content.order_by(*ordering) + .values("name_normalized") + .annotate( + versions=PackageVersions( + "version", + "pulp_created", + **{ordering_argument: ("pulp_created",)}, + ), + metadata_ids=PackageVersions( + "version", + "pk", + **file_ordering, + ), + ) + ) + + +def assemble_package_index(package_rows): + """Sort each package's versions and fetch metadata for its highest version in one query.""" + if not package_rows: + return [] + + for row in package_rows: + row["versions"] = sorted( + ( + {"version": version, "last_updated": updated} + for version, updated in row["versions"].items() + ), + key=lambda unit: (_version_sort_key(unit["version"]), unit["version"]), + reverse=True, + ) + + fields = ( + "name", + "name_normalized", + "summary", + "description", + "author", + "author_email", + "license", + ) + metadata = { + str(unit["pk"]): {field: unit[field] for field in fields} + for unit in PythonPackageContent.objects.filter( + pk__in=[row["metadata_ids"][row["versions"][0]["version"]] for row in package_rows] + ) + .order_by() + .values("pk", *fields) + } + return [ + { + **metadata[row["metadata_ids"][row["versions"][0]["version"]]], + "versions": row["versions"], + } + for row in package_rows + ] diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index 8d580a2f..ae974832 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -349,7 +349,7 @@ def list(self, request, path): if self.should_redirect(repo_version=repo_version): return redirect(urljoin(self.base_content_url, f"{path}/simple/")) names = ( - content.order_by("name_normalized") + content.order_by("name_normalized", "name") .values_list("name", flat=True) .distinct("name_normalized") .iterator() diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index c59a5398..e6031ed7 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -19,6 +19,7 @@ from pulpcore.plugin.util import get_current_authenticated_user, get_domain, get_prn, reverse from pulp_python.app import models as python_models +from pulp_python.app.catalog import assemble_package_index from pulp_python.app.provenance import ( AnyPublisher, Attestation, @@ -641,6 +642,66 @@ class Meta: model = python_models.PythonPackageContent +class PythonRepositoryPackageVersionSerializer(serializers.Serializer): + """One stored version of a package.""" + + version = serializers.CharField(help_text=_("The package's version number.")) + last_updated = serializers.DateTimeField( + help_text=_( + "The latest `pulp_created` timestamp for this version's files " + "in the selected repository version." + ), + ) + + +class PythonRepositoryPackageListSerializer(serializers.ListSerializer): + """Load package metadata for the current page.""" + + def to_representation(self, data): + rows = assemble_package_index(list(data)) + return super().to_representation(rows) + + +class PythonRepositoryPackageSerializer(serializers.Serializer): + """One distinct package name and its stored versions.""" + + name = serializers.CharField( + help_text=_("The package name from its metadata."), + ) + name_normalized = serializers.CharField( + help_text=_("The package name normalized according to PEP 503."), + ) + summary = serializers.CharField( + allow_blank=True, + help_text=_("A one-line summary of what the package does."), + ) + description = serializers.CharField( + allow_blank=True, + help_text=_("The package description."), + ) + author = serializers.CharField( + allow_blank=True, + help_text=_("Author information from the package metadata."), + ) + author_email = serializers.CharField( + allow_blank=True, + help_text=_("The author's e-mail address."), + ) + license = serializers.CharField( + allow_blank=True, + help_text=_("The license declared by the package."), + ) + versions = PythonRepositoryPackageVersionSerializer( + many=True, + help_text=_( + "Package versions in descending PEP 440 order, with an update time for each version." + ), + ) + + class Meta: + list_serializer_class = PythonRepositoryPackageListSerializer + + class PackageProvenanceSerializer(core_serializers.NoArtifactContentUploadSerializer): """ A Serializer for PackageProvenance. diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 400c9398..cce1192d 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -2,7 +2,9 @@ from bandersnatch.configuration import BandersnatchConfig from django.db import transaction -from django_filters import CharFilter +from django.http import Http404 +from django_filters import CharFilter, OrderingFilter +from django_filters.rest_framework import DjangoFilterBackend, FilterSet from django_filters.rest_framework import filters as drf_filters from drf_spectacular.utils import extend_schema, extend_schema_view from packaging.specifiers import InvalidSpecifier, SpecifierSet @@ -27,15 +29,87 @@ RepositorySyncURLSerializer, ) from pulpcore.plugin.tasking import dispatch -from pulpcore.plugin.util import extract_pk +from pulpcore.plugin.util import extract_pk, get_domain_pk from pulp_python.app import models as python_models from pulp_python.app import serializers as python_serializers from pulp_python.app import tasks +from pulp_python.app.catalog import package_catalog_queryset + + +class PythonRepositoryPackageFilter(FilterSet): + """Filter names and order packages in a repository version.""" + + ordering = OrderingFilter(fields=("name_normalized",)) + + class Meta: + model = python_models.PythonPackageContent + fields = {"name_normalized": core_viewsets.NAME_FILTER_OPTIONS} + + def filter_queryset(self, queryset): + return package_catalog_queryset( + super().filter_queryset(queryset), + ordering=self.form.cleaned_data["ordering"] or ("name_normalized",), + ) + + +class PythonRepositoryPackageFilterBackend(DjangoFilterBackend): + """Apply content filters without filtering the parent resource lookup.""" + + def get_filterset_class(self, view, queryset=None): + return view.filterset_class + + def filter_queryset(self, request, queryset, view): + # Package filters apply to content, not the parent resource lookup. + if queryset.model is not python_models.PythonPackageContent: + return queryset + return super().filter_queryset(request, queryset, view) + + +class PythonRepositoryPackagesMixin: + """Browse packages on a repository or a complete repository version.""" + + @extend_schema( + summary="List packages", + description=( + "Return one row per distinct package name in this repository version, " + "or the latest complete version when called on a repository. " + "Pagination counts package names, not distribution files." + ), + responses={200: python_serializers.PythonRepositoryPackageSerializer(many=True)}, + filters=True, + ) + @action( + detail=True, + methods=["get"], + serializer_class=python_serializers.PythonRepositoryPackageSerializer, + filterset_class=PythonRepositoryPackageFilter, + filter_backends=(PythonRepositoryPackageFilterBackend,), + ) + def packages(self, request, **kwargs): + """List packages from the repository version identified by the resource URL.""" + resource = self.get_object() + if isinstance(resource, RepositoryVersion): + if resource.repository.pulp_domain_id != get_domain_pk(): + raise Http404 + version = resource + else: + version = resource.latest_version() + content = python_models.PythonPackageContent.objects.all() + content = version.get_content(content) if version is not None else content.none() + queryset = self.filter_queryset(content) + page = self.paginate_queryset(queryset) + serializer = self.get_serializer(page if page is not None else queryset, many=True) + if page is not None: + return self.get_paginated_response(serializer.data) + return Response(serializer.data) class PythonRepositoryViewSet( - core_viewsets.RepositoryViewSet, ModifyRepositoryActionMixin, core_viewsets.RolesMixin + PythonRepositoryPackagesMixin, + core_viewsets.RepositoryViewSet, + ModifyRepositoryActionMixin, + core_viewsets.RolesMixin, ): """ PythonRepository represents a single Python repository, to which content can be @@ -64,7 +138,7 @@ class PythonRepositoryViewSet( ], }, { - "action": ["retrieve"], + "action": ["retrieve", "packages"], "principal": "authenticated", "effect": "allow", "condition": "has_model_or_domain_or_obj_perms:python.view_pythonrepository", @@ -303,7 +377,9 @@ def get_serializer_context(self): return context -class PythonRepositoryVersionViewSet(core_viewsets.RepositoryVersionViewSet): +class PythonRepositoryVersionViewSet( + PythonRepositoryPackagesMixin, core_viewsets.RepositoryVersionViewSet +): """ PythonRepositoryVersion represents a single Python repository version. """ @@ -313,7 +389,7 @@ class PythonRepositoryVersionViewSet(core_viewsets.RepositoryVersionViewSet): DEFAULT_ACCESS_POLICY = { "statements": [ { - "action": ["list", "retrieve"], + "action": ["list", "retrieve", "packages"], "principal": "authenticated", "effect": "allow", "condition": "has_repository_model_or_domain_or_obj_perms:python.view_pythonrepository", # noqa: E501 diff --git a/pulp_python/tests/functional/api/test_catalog.py b/pulp_python/tests/functional/api/test_catalog.py new file mode 100644 index 00000000..bed4168a --- /dev/null +++ b/pulp_python/tests/functional/api/test_catalog.py @@ -0,0 +1,159 @@ +"""Catalog API tests.""" + +import io +import tarfile + +import pytest + +from pulp_python.tests.functional.constants import PYTHON_SM_PROJECT_SPECIFIER + + +@pytest.fixture(params=["repository", "version"]) +def catalog(request, python_bindings): + """Call either catalog action through its generated client.""" + version = request.param == "version" + api = ( + python_bindings.RepositoriesPythonVersionsApi + if version + else python_bindings.RepositoriesPythonApi + ) + + def packages(repo, **params): + href = repo.latest_version_href if version else repo.pulp_href + return api.packages(href, **params) + + return packages + + +def test_package_list_schema(pulp_openapi_schema_pk_path_set): + """Both actions expose package filters without a repository-version filter.""" + operations = [ + methods["get"] + for path, methods in pulp_openapi_schema_pk_path_set["paths"].items() + if "/repositories/python/python/" in path and path.endswith("/packages/") + ] + assert len(operations) == 2 + for operation in operations: + params = {item["name"] for item in operation["parameters"] if item["in"] == "query"} + assert {"name_normalized", "ordering", "limit", "offset"} <= params + assert "repository_version" not in params + + +@pytest.mark.parallel +def test_package_list(python_repo_with_sync, python_remote_factory, catalog): + """Group and page package names, with name filtering and ordering.""" + repo = python_repo_with_sync(python_remote_factory(includes=PYTHON_SM_PROJECT_SPECIFIER)) + first = catalog(repo, limit=2) + second = catalog(repo, limit=2, offset=2) + assert first.count == second.count == 3 + assert [pkg.name_normalized for pkg in first.results + second.results] == [ + "aiohttp", + "celery", + "django", + ] + django = second.results[0] + assert [item.version for item in django.versions] == ["1.10.4", "1.10.3", "1.10.2", "1.10.1"] + assert set(django.versions[0].to_dict()) == {"version", "last_updated"} + for params in ({"name_normalized": "django"}, {"name_normalized__istartswith": "DJAN"}): + filtered = catalog(repo, **params) + assert filtered.count == 1 + assert filtered.results[0].name_normalized == "django" + descending = catalog(repo, ordering=["-name_normalized"], limit=2) + assert [pkg.name_normalized for pkg in descending.results] == ["django", "celery"] + + +@pytest.mark.parallel +def test_package_list_snapshots(python_bindings, python_repo_with_sync, monitor_task): + """Metadata and update times use only files in the requested repository version.""" + repo = python_repo_with_sync() + repositories = python_bindings.RepositoriesPythonApi + versions = python_bindings.RepositoriesPythonVersionsApi + empty = versions.packages(f"{repo.pulp_href}versions/0/") + assert empty.count == 0 + assert empty.results == [] + + files = python_bindings.ContentPackagesApi.list( + repository_version=repo.latest_version_href + ).results + sdist = next(item for item in files if item.packagetype == "sdist") + newest = max(files, key=lambda item: item.pulp_created) + remaining = next(item for item in files if item.pulp_href != newest.pulp_href) + original = repositories.packages(repo.pulp_href) + assert original.count == 1 + row = original.results[0] + assert row.description == sdist.description + assert [(item.version, item.last_updated) for item in row.versions] == [ + ("0.1", newest.pulp_created) + ] + + monitor_task( + repositories.modify(repo.pulp_href, {"remove_content_units": [newest.pulp_href]}).task + ) + current = repositories.packages(repo.pulp_href).results[0] + assert current.versions[0].last_updated == remaining.pulp_created + assert current.description == remaining.description + assert versions.packages(repo.latest_version_href).to_dict() == original.to_dict() + with pytest.raises(python_bindings.ApiException) as exc: + versions.packages(f"{repo.pulp_href}versions/999/") + assert exc.value.status == 404 + + +@pytest.mark.parallel +def test_package_list_versions(python_content_factory, python_repo, python_bindings, tmp_path): + """PEP 440 ordering selects metadata and preserves separate rebuild versions.""" + content = {} + for version in ("1.10", "1.9", "1.9+test.1"): + filename = f"catalog-test-{version}.tar.gz" + path = tmp_path / filename + metadata = ( + f"Metadata-Version: 2.4\nName: catalog-test\nVersion: {version}\n\nVersion {version}\n" + ).encode() + with tarfile.open(path, "w:gz") as archive: + info = tarfile.TarInfo(f"catalog-test-{version}/PKG-INFO") + info.size = len(metadata) + archive.addfile(info, io.BytesIO(metadata)) + content[version] = python_content_factory(filename, file=str(path), repository=python_repo) + + data = python_bindings.RepositoriesPythonApi.packages(python_repo.pulp_href) + assert data.count == 1 + row = data.results[0] + assert row.description.strip() == "Version 1.10" + assert [(item.version, item.last_updated) for item in row.versions] == [ + (version, content[version].pulp_created) for version in ("1.10", "1.9+test.1", "1.9") + ] + + +@pytest.mark.parallel +def test_package_list_permissions( + python_bindings, python_repo, python_repo_factory, gen_user, catalog +): + """Both actions require view permission on the parent repository.""" + other = python_repo_factory() + viewer = gen_user(object_roles=[("python.pythonrepository_viewer", python_repo.pulp_href)]) + with viewer: + assert catalog(python_repo).results == [] + with pytest.raises(python_bindings.ApiException) as exc: + catalog(other) + assert exc.value.status in (403, 404) + with gen_user(model_roles=["python.pythonrepository_viewer"]): + assert catalog(other).results == [] + + +@pytest.mark.parallel +def test_package_list_domains( + python_bindings, domain_factory, python_repo_factory, gen_user, pulp_domain_enabled, catalog +): + """Both actions enforce domain boundaries and accept domain viewer roles.""" + if not pulp_domain_enabled: + pytest.skip("Domain not enabled") + domain = domain_factory() + repo = python_repo_factory(pulp_domain=domain.name) + with gen_user(domain_roles=[("python.pythonrepository_viewer", domain.pulp_href)]): + assert catalog(repo).results == [] + repo.pulp_href = repo.pulp_href.replace(f"/{domain.name}/api/", "/default/api/") + repo.latest_version_href = repo.latest_version_href.replace( + f"/{domain.name}/api/", "/default/api/" + ) + with pytest.raises(python_bindings.ApiException) as exc: + catalog(repo) + assert exc.value.status == 404