feat: serve PyPI and Simple JSON from the content app

CHANGES/+content-handler-json.feature

@@ -0,0 +1 @@
+Added `PythonDistribution.content_handler_json` so the content app can serve PyPI JSON and PEP 691 Simple JSON when the client prefers `application/json`. Requires a pulpcore that includes `Distribution.content_handler_json` (issue 7887).

docs/user/guides/host.md

@@ -103,6 +103,22 @@ pip install --trusted-host localhost shelf-reader
 
 See the [pip docs](https://pip.pypa.io/en/stable/topics/configuration) for more details.
 
+## JSON from the content app
+
+When pulpcore's content app is asked for JSON (`Accept: application/json`), Python distributions
+serve the same PyPI JSON and PEP 691 Simple JSON already available from the `/pypi/` API. Clients
+do not need the `/json` URL suffix or the PyPI Simple media types:
+
+```bash
+http "${CONTENT_ORIGIN}${CONTENT_PATH_PREFIX}foo/simple/" Accept:application/json
+http "${CONTENT_ORIGIN}${CONTENT_PATH_PREFIX}foo/simple/shelf-reader/" Accept:application/json
+http "${CONTENT_ORIGIN}${CONTENT_PATH_PREFIX}foo/pypi/shelf-reader/" Accept:application/json
+```
+
+The distribution root still returns pulpcore's generic file listing. Package artifacts remain binary
+regardless of `Accept`. This requires a pulpcore that includes `Distribution.content_handler_json`
+(pulpcore issue 7887).
+
 
 ## Migrating off Publications
 

pulp_python/app/models.py

@@ -8,6 +8,7 @@
 from django.contrib.postgres.fields import ArrayField
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
+from django.db.models import F, FilteredRelation, Q
 from django_lifecycle import (
     BEFORE_SAVE,
     hook,
@@ -36,10 +37,14 @@
 from .utils import (
     PYPI_LAST_SERIAL,
     PYPI_SERIAL_CONSTANT,
+    PYPI_SIMPLE_V1_JSON,
     artifact_to_metadata_artifact,
     artifact_to_python_content_data,
+    build_content_url,
     canonicalize_name,
     python_content_to_json,
+    write_simple_detail_json,
+    write_simple_index_json,
 )
 
 log = getLogger(__name__)
@@ -126,7 +131,7 @@ def content_handler(self, path):
                 pk__in=self.publication.repository_version.content, name_normalized=normalized
             )
             # TODO Change this value to the Repo's serial value when implemented
-            headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)}
+            headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT), "Vary": "Accept"}
             if not settings.DOMAIN_ENABLED:
                 domain = None
             json_body = python_content_to_json(
@@ -137,6 +142,111 @@ def content_handler(self, path):
 
         return None
 
+    def content_handler_json(self, path):
+        """
+        Handler to serve a JSON representation of the content at ``path`` for this Distribution.
+
+        Called by pulpcore's content app when a client's ``Accept`` header prefers JSON,
+        instead of pulpcore's generic recursive file listing. This reuses the same
+        serialization already used by the ``pypi/*/json`` and ``pypi/simple`` PyPI APIs, so
+        requests made directly against the content app (e.g. by a UI or ``pip``-compatible
+        tool) get the same structured data without needing to know those separate URL
+        conventions.
+
+        Args:
+            path (str): The path being requested
+        Returns:
+            None if there is no JSON representation to serve at path. Otherwise a
+            JSON-serializable dict, or an aiohttp.web.Response for full header control.
+        """
+        path = PurePath(path)
+        parts = path.parts
+        if not parts:
+            # Distro root: let pulpcore's generic recursive listing handle it.
+            return None
+
+        _, repo_version, _ = self.get_repository_publication_and_version()
+        if repo_version is None:
+            return None
+        content = PythonPackageContent.objects.filter(pk__in=repo_version.content)
+        domain = get_domain() if settings.DOMAIN_ENABLED else None
+
+        if parts[0] == "pypi" and 2 <= len(parts) <= 3 and parts[-1] != "json":
+            # e.g. pypi/<name>/ or pypi/<name>/<version>/ (the *-suffixed /json paths are
+            # already handled by content_handler, unconditionally on Accept).
+            name = parts[1]
+            version = parts[2] if len(parts) == 3 else None
+            normalized = canonicalize_name(name)
+            package_content = content.filter(name_normalized=normalized)
+            headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT), "Vary": "Accept"}
+            yank_markers = dict(
+                PackageYank.objects.filter(
+                    pk__in=repo_version.content, name_normalized=normalized
+                ).values_list("version", "yanked_reason")
+            )
+            json_body = python_content_to_json(
+                self.base_path,
+                package_content,
+                version=version,
+                domain=domain,
+                repository_version=repo_version,
+                yank_markers=yank_markers,
+            )
+            if json_body:
+                return json_response(json_body, headers=headers)
+            return None
+
+        if parts[0] == "simple":
+            if len(parts) == 1:
+                names = (
+                    content.order_by("name_normalized")
+                    .values_list("name", flat=True)
+                    .distinct("name_normalized")
+                )
+                headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT), "Vary": "Accept"}
+                return json_response(
+                    write_simple_index_json(list(names)),
+                    headers=headers,
+                    content_type=PYPI_SIMPLE_V1_JSON,
+                )
+
+            if len(parts) != 2:
+                return None
+
+            normalized = canonicalize_name(parts[1])
+            packages = content.filter(name_normalized=normalized).annotate(
+                active_membership=FilteredRelation(
+                    "version_memberships",
+                    condition=Q(
+                        version_memberships__repository=repo_version.repository,
+                        version_memberships__version_removed=None,
+                    ),
+                ),
+                repo_added_time=F("active_membership__pulp_created"),
+            )
+            if not packages.exists():
+                return None
+            releases = [
+                {
+                    "filename": p.filename,
+                    "sha256": p.sha256,
+                    "metadata_sha256": p.metadata_sha256,
+                    "requires_python": p.requires_python,
+                    "size": p.size,
+                    "upload_time": p.repo_added_time or p.pulp_created,
+                    "version": p.version,
+                    "url": build_content_url(self.base_path, p.filename, domain=domain),
+                }
+                for p in packages
+            ]
+            return json_response(
+                write_simple_detail_json(normalized, releases),
+                headers={PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT), "Vary": "Accept"},
+                content_type=PYPI_SIMPLE_V1_JSON,
+            )
+
+        return None
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         permissions = [

pulp_python/app/utils.py

@@ -506,6 +506,20 @@ def python_content_to_urls(contents, base_path, domain=None):
     return [python_content_to_download_info(content, base_path, domain) for content in contents]
 
 
+def build_content_url(base_path, filename, domain=None):
+    """
+    Builds the absolute content-app URL for a filename served under a distribution's base_path.
+    """
+    origin = settings.CONTENT_ORIGIN or settings.PYPI_API_HOSTNAME or ""
+    origin = origin.strip("/")
+    prefix = settings.CONTENT_PATH_PREFIX.strip("/")
+    base_path = base_path.strip("/")
+    components = [origin, prefix, base_path, filename]
+    if domain:
+        components.insert(2, domain.name)
+    return "/".join(components)
+
+
 def python_content_to_download_info(content, base_path, domain=None):
     """
     Takes in a PythonPackageContent and base path of the distribution to create a dictionary of
@@ -524,14 +538,7 @@ def find_artifact():
         relative_path__endswith=".metadata"
     ).first()
     artifact = find_artifact()
-    origin = settings.CONTENT_ORIGIN or settings.PYPI_API_HOSTNAME or ""
-    origin = origin.strip("/")
-    prefix = settings.CONTENT_PATH_PREFIX.strip("/")
-    base_path = base_path.strip("/")
-    components = [origin, prefix, base_path, content.filename]
-    if domain:
-        components.insert(2, domain.name)
-    url = "/".join(components)
+    url = build_content_url(base_path, content.filename, domain=domain)
     md5 = artifact.md5 if artifact and artifact.md5 else ""
     return {
         "comment_text": "",

pulp_python/tests/functional/api/test_content_handler_json.py

@@ -0,0 +1,141 @@
+"""Content-app JSON via Accept: application/json (PythonDistribution.content_handler_json).
+
+These tests hit the content app, not the DRF ``/pypi/`` API. They require pulpcore
+Phase 1 (issue 7887): ``Distribution.content_handler_json`` plus Handler Accept
+negotiation. When CI pulpcore is unpatched, skip — unit tests still cover the
+handler return values.
+"""
+
+from importlib.util import find_spec
+from pathlib import Path
+from urllib.parse import urljoin
+
+import pytest
+import requests
+
+from pulp_python.tests.functional.constants import (
+    PYPI_SIMPLE_V1_JSON,
+    PYTHON_WHEEL_FILENAME,
+    PYTHON_XS_PROJECT_SPECIFIER,
+)
+
+JSON_ACCEPT = {"Accept": "application/json"}
+
+
+def _pulpcore_content_app_json_enabled():
+    """True when pulpcore will invoke Distribution.content_handler_json on JSON Accept.
+
+    Inspect source files instead of importing Django models, which are not ready
+    during pytest collection. Use find_spec so editable installs are resolved.
+    """
+    handler = find_spec("pulpcore.content.handler")
+    if not handler or not handler.origin:
+        return False
+    handler_path = Path(handler.origin)
+    publication_path = handler_path.resolve().parents[1] / "app" / "models" / "publication.py"
+    try:
+        handler_src = handler_path.read_text()
+        publication_src = publication_path.read_text()
+    except OSError:
+        return False
+    return "def negotiate_json" in handler_src and "def content_handler_json" in publication_src
+
+
+pytestmark = pytest.mark.skipif(
+    not _pulpcore_content_app_json_enabled(),
+    reason=(
+        "Requires pulpcore content-app Accept negotiation (Handler.negotiate_json and "
+        "Distribution.content_handler_json from pulpcore issue 7887). Unit tests cover "
+        "PythonDistribution.content_handler_json without that pulpcore change."
+    ),
+)
+
+
+def _content_url(pulp_content_url, distro, rel=""):
+    return urljoin(pulp_content_url, f"{distro.base_path}/{rel}")
+
+
+@pytest.mark.parallel
+def test_content_app_simple_index_and_detail_json(
+    python_remote_factory, python_repo_with_sync, python_distribution_factory, pulp_content_url
+):
+    """``.../simple/`` and ``.../simple/<name>/`` return PEP 691 Simple JSON."""
+    remote = python_remote_factory(includes=PYTHON_XS_PROJECT_SPECIFIER)
+    repo = python_repo_with_sync(remote)
+    distro = python_distribution_factory(repository=repo)
+
+    index = requests.get(_content_url(pulp_content_url, distro, "simple/"), headers=JSON_ACCEPT)
+    assert index.status_code == 200
+    assert PYPI_SIMPLE_V1_JSON in index.headers["Content-Type"]
+    index_body = index.json()
+    assert index_body["meta"]["api-version"] == "1.1"
+    names = [project["name"] for project in index_body["projects"]]
+    assert any("shelf" in name.lower() for name in names)
+
+    detail = requests.get(
+        _content_url(pulp_content_url, distro, "simple/shelf-reader/"), headers=JSON_ACCEPT
+    )
+    assert detail.status_code == 200
+    assert PYPI_SIMPLE_V1_JSON in detail.headers["Content-Type"]
+    detail_body = detail.json()
+    assert detail_body["name"] == "shelf-reader"
+    assert detail_body["files"]
+    assert detail_body["versions"]
+    filenames = [f["filename"] for f in detail_body["files"]]
+    assert PYTHON_WHEEL_FILENAME in filenames or any("shelf" in f for f in filenames)
+
+
+@pytest.mark.parallel
+def test_content_app_pypi_json_without_json_suffix(
+    python_remote_factory, python_repo_with_sync, python_distribution_factory, pulp_content_url
+):
+    """``.../pypi/<name>/`` returns PyPI JSON without the ``/json`` URL convention."""
+    remote = python_remote_factory(includes=PYTHON_XS_PROJECT_SPECIFIER)
+    repo = python_repo_with_sync(remote)
+    distro = python_distribution_factory(repository=repo)
+
+    response = requests.get(
+        _content_url(pulp_content_url, distro, "pypi/shelf-reader/"), headers=JSON_ACCEPT
+    )
+    assert response.status_code == 200
+    assert "application/json" in response.headers["Content-Type"]
+    body = response.json()
+    assert "info" in body
+    assert "releases" in body
+    assert "urls" in body
+    assert "shelf" in body["info"]["name"].lower()
+
+
+@pytest.mark.parallel
+def test_content_app_distro_root_is_generic_listing(
+    python_remote_factory, python_repo_with_sync, python_distribution_factory, pulp_content_url
+):
+    """Distribution root still uses pulpcore's generic JSON listing, not PyPI/Simple JSON."""
+    remote = python_remote_factory(includes=PYTHON_XS_PROJECT_SPECIFIER)
+    repo = python_repo_with_sync(remote)
+    distro = python_distribution_factory(repository=repo)
+
+    response = requests.get(_content_url(pulp_content_url, distro), headers=JSON_ACCEPT)
+    assert response.status_code == 200
+    assert "application/json" in response.headers["Content-Type"]
+    body = response.json()
+    assert "packages" in body
+    assert "projects" not in body
+    assert "info" not in body
+
+
+@pytest.mark.parallel
+def test_content_app_artifact_stays_binary(
+    python_remote_factory, python_repo_with_sync, python_distribution_factory, pulp_content_url
+):
+    """Concrete package artifacts stay binary even when Accept prefers JSON."""
+    remote = python_remote_factory(includes=PYTHON_XS_PROJECT_SPECIFIER, policy="immediate")
+    repo = python_repo_with_sync(remote)
+    distro = python_distribution_factory(repository=repo)
+
+    response = requests.get(
+        _content_url(pulp_content_url, distro, PYTHON_WHEEL_FILENAME), headers=JSON_ACCEPT
+    )
+    assert response.status_code == 200
+    assert "application/json" not in response.headers.get("Content-Type", "")
+    assert response.content[:2] == b"PK"  # zip/wheel local file header

pulp_python/tests/unit/conftest.py

@@ -0,0 +1,9 @@
+import os
+
+import django
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pulpcore.app.settings")
+
+
+def pytest_configure(config):
+    django.setup()