From bcc7bb88130992abada4b1e21c745f61c3e84892 Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Wed, 19 Aug 2026 08:41:29 -0400 Subject: [PATCH 1/2] ci_scripts: pypi_riscv64_check.py: add script Add a script for checking PyPI download metrics to identify which which of the top-50 binary wheels are lacking riscv64 builds. It also has logic for creating issues when new binary wheels are detected lacking riscv64 builds, which we aren't already tracking in the python-wheels issue list. A summary issue is generated with tables listing relevant wheel details and linking to any new per-package issues opened. AI-Generated: Uses Claude Sonnet 5 Signed-off-by: Trevor Gamblin --- ci_scripts/pypi_riscv64_check.py | 312 +++++++++++++++++++++++++++++++ 1 file changed, 312 insertions(+) create mode 100644 ci_scripts/pypi_riscv64_check.py diff --git a/ci_scripts/pypi_riscv64_check.py b/ci_scripts/pypi_riscv64_check.py new file mode 100644 index 00000000..bbb26a23 --- /dev/null +++ b/ci_scripts/pypi_riscv64_check.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +""" +pypi_riscv64_check.py + +Fetches the top N PyPI packages by download count, filters down to those +shipping platform-specific (binary) wheels, and flags which ones already +provide riscv64 wheels in their latest release on PyPI. + +Data source: hugovk/top-pypi-packages (updated monthly, via GitHub raw) +Wheel info: PyPI JSON API https://pypi.org/pypi/{package}/json + +Usage: + python pypi_riscv64_check.py # default: top 50 + python pypi_riscv64_check.py --top 30 # top 30 binary-wheel packages + python pypi_riscv64_check.py --exclude-riscv64 # hide packages that already have it + python pypi_riscv64_check.py --create-issues # file GitHub issues for gaps (needs GH_TOKEN) +""" + +import argparse +import json +import re +import subprocess +import sys +import time +import urllib.error +import urllib.request + +TOP_PACKAGES_URL = ( + "https://raw.githubusercontent.com/hugovk/top-pypi-packages/" + "main/top-pypi-packages.min.json" +) +PYPI_JSON_URL = "https://pypi.org/pypi/{package}/json" +REGISTRY_SIMPLE_URL = "https://pypi.riseproject.dev/simple/{package}/" +REQUEST_DELAY = 0.5 # seconds between PyPI API calls (be polite) +ISSUE_LABELS = ["riscv64-check", "wheel"] + + +def gh(*args: str) -> subprocess.CompletedProcess: + return subprocess.run(["gh", *args], capture_output=True, text=True, check=True) + + +def find_existing_issue(title: str) -> int | None: + """Return open issue number matching exact title with the riscv64-check label, else None.""" + try: + result = gh( + "issue", "list", + "--label", ISSUE_LABELS[0], + "--state", "open", + "--search", f'"{title}" in:title', + "--json", "number,title", + "--limit", "50", + ) + except subprocess.CalledProcessError as e: + print(f" [!] gh issue list failed: {e.stderr or e}", file=sys.stderr) + return None + + try: + issues = json.loads(result.stdout) + except json.JSONDecodeError: + return None + + for issue in issues: + if issue.get("title") == title: + return issue.get("number") + return None + + +def create_issue(title: str, body: str) -> int | None: + args = ["issue", "create", "--title", title, "--body", body] + for label in ISSUE_LABELS: + args += ["--label", label] + try: + result = gh(*args) + except subprocess.CalledProcessError as e: + print(f" [!] gh issue create failed: {e.stderr or e}", file=sys.stderr) + return None + + for line in result.stdout.splitlines(): + line = line.strip() + if line.startswith("http") and "/issues/" in line: + try: + return int(line.rsplit("/", 1)[-1]) + except ValueError: + pass + return None + + +def open_issues_for_missing(packages: list[str]) -> dict[str, int]: + """Open an issue for each package not already tracked by an open riscv64-check + issue. Returns {package: issue_number} for every package that ends up with + one open, whether just created or pre-existing.""" + issues = {} + for name in packages: + title = f"{name} riscv64 support" + existing = find_existing_issue(title) + if existing is not None: + issues[name] = existing + continue + body = ( + f"`{name}` ships binary wheels but no riscv64 wheels on PyPI, " + "and is not present in the RISE riscv64 registry.\n\n" + "Detected by the monthly `pypi-riscv64-check` workflow." + ) + number = create_issue(title, body) + if number is not None: + print(f" [+] Created issue #{number} for {name}") + issues[name] = number + else: + print(f" [!] Failed to create issue for {name}") + return issues + + +def open_summary_issue(report: str, issue_map: dict[str, int]) -> None: + """Open (or skip, if one already exists) a monthly summary issue linking + to each per-package issue and notifying the maintainers.""" + title = f"PyPI riscv64 check - {time.strftime('%Y-%m')}" + if find_existing_issue(title) is not None: + print(f" [=] Summary issue for {title!r} already open, skipping") + return + + lines = ["@threexc @justeph monthly PyPI riscv64 wheel report:", "", "```", report, "```"] + if issue_map: + lines += ["", "Per-package issues:"] + lines += [f"- #{number} — {name}" for name, number in issue_map.items()] + body = "\n".join(lines) + + number = create_issue(title, body) + if number is not None: + print(f" [+] Created summary issue #{number}") + else: + print(" [!] Failed to create summary issue") + + +def normalise(name: str) -> str: + """PEP 503 normalisation for simple-index package names.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Packages missing from the registry get 30x-redirected to pypi.org + instead of 404ing, so redirects must be treated as "not present" + rather than followed.""" + + def redirect_request(self, *args, **kwargs): + return None + + +_REGISTRY_OPENER = urllib.request.build_opener(_NoRedirect) + + +def in_rise_registry(name: str) -> bool: + """Whether the package has any wheels in the RISE riscv64 registry.""" + req = urllib.request.Request( + REGISTRY_SIMPLE_URL.format(package=normalise(name)), + headers={"User-Agent": "pypi-riscv64-checker/1.0"}, + ) + try: + _REGISTRY_OPENER.open(req, timeout=15) + return True + except urllib.error.HTTPError as exc: + if exc.code in (302, 303, 404): + return False + print(f" [WARN] Could not check registry for {name}: {exc}") + return False + except Exception as exc: + print(f" [WARN] Could not check registry for {name}: {exc}") + return False + + +def fetch_json(url: str) -> dict | list: + req = urllib.request.Request( + url, headers={"User-Agent": "pypi-riscv64-checker/1.0"} + ) + with urllib.request.urlopen(req, timeout=15) as r: + return json.loads(r.read()) + + +def analyse_package(name: str, download_count: int) -> dict | None: + """ + Returns a result dict if the package ships at least one binary wheel, + otherwise None (pure-Python or fetch error). + + A binary wheel is any .whl file whose filename does NOT end with + 'none-any.whl' (the platform-independent tag). + """ + try: + info = fetch_json(PYPI_JSON_URL.format(package=name)) + except Exception as exc: + print(f" [WARN] Could not fetch {name}: {exc}") + return None + + wheel_files = [ + u["filename"] + for u in info.get("urls", []) + if u["packagetype"] == "bdist_wheel" + ] + + binary_wheels = [f for f in wheel_files if not f.endswith("none-any.whl")] + if not binary_wheels: + return None # pure-Python or no wheels at all + + riscv64_wheels = [f for f in wheel_files if "riscv64" in f.lower()] + has_riscv64 = len(riscv64_wheels) > 0 + + return { + "project": name, + "download_count": download_count, + "has_riscv64": has_riscv64, + "in_rise_registry": None if has_riscv64 else in_rise_registry(name), + } + + +def fmt_count(n: int) -> str: + """Format large numbers with M/B suffixes.""" + if n >= 1_000_000_000: + return f"{n / 1_000_000_000:.2f}B" + if n >= 1_000_000: + return f"{n / 1_000_000:.1f}M" + return str(n) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--top", type=int, default=50, + help="Number of binary-wheel packages to collect (default: 50)" + ) + parser.add_argument( + "--exclude-riscv64", action="store_true", + help="Exclude packages that already have riscv64 wheels" + ) + parser.add_argument( + "--create-issues", action="store_true", + help="Open a GitHub issue for each package missing riscv64 wheels " + "upstream and in the RISE registry, unless one is already open" + ) + args = parser.parse_args() + + print("Fetching top-packages dataset …") + dataset = fetch_json(TOP_PACKAGES_URL) + all_packages = dataset["rows"] + last_update = dataset.get("last_update", "unknown") + print(f" Dataset last updated: {last_update}") + print(f" Scanning for top {args.top} binary-wheel packages …\n") + + results = [] + scanned = 0 + + for pkg in all_packages: + if len(results) >= args.top: + break + + name = pkg["project"] + dl = pkg["download_count"] + result = analyse_package(name, dl) + scanned += 1 + + if result is None: + continue + + if args.exclude_riscv64 and result["has_riscv64"]: + print(f" SKIP (riscv64 exists): {name}") + continue + + results.append(result) + time.sleep(REQUEST_DELAY) + + # --- Build table --- + report_lines = [ + f"{'Rank':<5} {'Package':<35} {'Downloads':>12} {'riscv64':>10} {'RISE registry':>14}", + "-" * 82, + ] + for i, r in enumerate(results, 1): + riscv_col = "✓" if r["has_riscv64"] else "✗" + if r["has_riscv64"]: + registry_col = "N/A" + else: + registry_col = "✓" if r["in_rise_registry"] else "✗" + report_lines.append( + f"{i:<5} {r['project']:<35} " + f"{fmt_count(r['download_count']):>12} " + f"{riscv_col:>10} " + f"{registry_col:>14}" + ) + + no_riscv = [r for r in results if not r["has_riscv64"]] + missing = [r["project"] for r in no_riscv if not r["in_rise_registry"]] + covered = [r["project"] for r in no_riscv if r["in_rise_registry"]] + + report_lines += [ + "", + f"Packages WITHOUT riscv64 wheels upstream and NOT in RISE registry ({len(missing)}/{len(results)}):", + *(f" - {name}" for name in missing), + "", + f"Packages WITHOUT riscv64 wheels upstream ALREADY in RISE registry ({len(covered)}/{len(results)}):", + *(f" - {name}" for name in covered), + "", + f"Scanned {scanned} packages total.", + ] + + report = "\n".join(report_lines) + print() + print(report) + + if args.create_issues: + print() + issue_map = open_issues_for_missing(missing) + open_summary_issue(report, issue_map) + + +if __name__ == "__main__": + main() From 08921d2719e1904b62444ed4d61c30d5f4630be8 Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Wed, 19 Aug 2026 08:46:01 -0400 Subject: [PATCH 2/2] workflows: pypi-riscv64-check: add Add a simple workflow which runs the pypi_riscv64_check.py script once per month. The hugovk/top-pypi-packages repository it uses as a reference updates on the first day of every month, so make the workflow run on the second day. AI-Generated: Uses Claude Sonnet 5 Signed-off-by: Trevor Gamblin --- .github/workflows/pypi-riscv64-check.yml | 32 ++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/pypi-riscv64-check.yml diff --git a/.github/workflows/pypi-riscv64-check.yml b/.github/workflows/pypi-riscv64-check.yml new file mode 100644 index 00000000..3ae44418 --- /dev/null +++ b/.github/workflows/pypi-riscv64-check.yml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT + +name: Check top PyPI packages for riscv64 wheels + +on: + schedule: + - cron: '0 2 2 * *' + workflow_dispatch: + push: + paths: + - 'ci_scripts/pypi_riscv64_check.py' + - '.github/workflows/pypi-riscv64-check.yml' + +permissions: + contents: read + issues: write + +jobs: + pypi_riscv64_check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3' + + - name: Run riscv64 wheel check and open issues for new gaps + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python3 ci_scripts/pypi_riscv64_check.py --create-issues