From 63a6c88ce5e03aecaf80115723384932a97923ec Mon Sep 17 00:00:00 2001 From: Julien Stephan Date: Mon, 26 Oct 2026 16:24:52 +0100 Subject: [PATCH] ci: mark published packages' issues "Available in RISE PyPI" A package's " riscv64 support" issue should move to "Available in RISE PyPI" in the Python Wheels project once its wheels are published. Doing it when update_doc.py opens the docs PR is too early: /simple/ is only regenerated by website.yml after that PR is merged. Add a sync-project-status job after website.yml's deploy that runs the new ci_scripts/sync_project_status.py. For every active package in docs/packages/*.yaml it takes the oldest open issue and, unless its status is already "Available in RISE PyPI" or "Available Upstream", adds it to the project and sets the status. The issue stays open; only a deprecation closes it. It compares state instead of diffing the push, so runs dropped by the concurrency queue or failed runs are caught up by the next deploy. Move the issue and project helpers into ci_scripts/github_issues.py, shared by sync_project_status.py, check_versions.py and update_doc.py (REPO, normalize_name). check_versions.py's find_upstream_issue() now fetches the issues once per run and picks the oldest open issue whose title matches exactly, instead of the newest fuzzy search hit. The project is matched by node ID, since its title is "Python Wheels - riseproject-dev/python-wheels". Document the job, and drop the closing-keyword advice from the porting steps: the issue must stay open. Signed-off-by: Julien Stephan --- .github/workflows/website.yml | 30 ++++ ci_scripts/check_versions.py | 41 +++-- ci_scripts/github_issues.py | 144 ++++++++++++++++++ ci_scripts/sync_project_status.py | 94 ++++++++++++ ci_scripts/update_doc.py | 9 +- docs/workflows.md | 10 +- .../references/pr-and-publishing.md | 18 ++- 7 files changed, 305 insertions(+), 41 deletions(-) create mode 100644 ci_scripts/github_issues.py create mode 100755 ci_scripts/sync_project_status.py diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index c6bef9d5cc74..8cf1b4f271df 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -60,3 +60,33 @@ jobs: with: path: requests-cache.sqlite key: ${{ steps.dashboard-requests-cache.outputs.cache-primary-key }} + + # A package is officially available once deploy has published /simple/, so + # we can now move the issue Status to "Available in RISE PyPI". + # Compares docs/packages with the project rather than diffing this push, + # so pushes the concurrency queue skipped are caught up. + sync-project-status: + needs: deploy + if: ${{ vars.RISEPROJECT_APP_CLIENT_ID != '' }} + runs-on: ubuntu-latest + permissions: + contents: read + steps: + # GITHUB_TOKEN cannot write organization projects; the App needs the + # organization "Projects: read and write" permission. + - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: app-token + with: + client-id: ${{ vars.RISEPROJECT_APP_CLIENT_ID }} + private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: | + ci_scripts + docs/packages + + - name: Mark published packages in the Python Wheels project + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: python3 ci_scripts/sync_project_status.py diff --git a/ci_scripts/check_versions.py b/ci_scripts/check_versions.py index d400c9ff2243..17a0e9e759f2 100755 --- a/ci_scripts/check_versions.py +++ b/ci_scripts/check_versions.py @@ -16,7 +16,6 @@ import json import re import shutil -import subprocess import sys import time from datetime import datetime, timedelta, timezone @@ -27,14 +26,14 @@ import requests import yaml +from github_issues import fetch_issues, normalize_name, oldest_open_issue from pending_versions import base_versions, RISCV64_PYTHONS -from update_doc import normalize_name, version_blocks +from update_doc import version_blocks ROOT_DIR = Path(__file__).resolve().parent.parent DOCS_DIR = ROOT_DIR / "docs" / "packages" PATCHES_DIR = ROOT_DIR / "patches" -REPO = "riseproject-dev/python-wheels" # Python versions riscv64 can build (cp314t -> "3.14t"); an upstream riscv64 # wheel for any of them deprecates our own build. "3.14t" is the free-threaded @@ -261,31 +260,25 @@ def target_files_lines(pythons: List[str]) -> List[str]: return [f" {line}" for line in dumped.splitlines()] -def find_upstream_issue(package: str) -> Optional[str]: - """ - Find issue for a package in the wheel_builder's Upstream milestone. +_issues: Optional[Dict] = None - All issues in the upstream milestone follow the same title format: - {package} riscv64 support +def find_upstream_issue(package: str) -> Optional[str]: """ - try: - issue_title = f"{package} riscv64 support" - result = subprocess.run([ - "gh", "issue", "list", - "--search", issue_title, - "--json", "number", - "--jq", ".[0].number", - "--repo", REPO, - ], capture_output=True, text=True, timeout=30) - - if result.returncode != 0: - return None + Number of the package's oldest open "{package} riscv64 support" issue. - return result.stdout.strip() or None - - except (subprocess.TimeoutExpired, subprocess.CalledProcessError, Exception): - return None + The issues are fetched once and reused for every package of the run. + """ + global _issues + if _issues is None: + try: + # Project fields are not needed here, and the nightly token cannot read them. + _issues = fetch_issues(with_project=False) + except RuntimeError as e: + print(f"warning: {e}", file=sys.stderr) + _issues = {} + issue = oldest_open_issue(_issues, package) + return str(issue["number"]) if issue else None def deprecate_package(package: str) -> bool: diff --git a/ci_scripts/github_issues.py b/ci_scripts/github_issues.py new file mode 100644 index 000000000000..aec4cc4b360e --- /dev/null +++ b/ci_scripts/github_issues.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +"""Shared helpers for the " riscv64 support" issues and their status in +the "Python Wheels" project. + +Uses the ``gh`` CLI. Reading project fields needs the ``read:project`` scope, +changing them the ``project`` scope (for the GitHub App: the organization +"Projects: read and write" permission). +""" + +import json +import re +import subprocess +import sys + +OWNER = "riseproject-dev" +REPO_NAME = "python-wheels" +REPO = f"{OWNER}/{REPO_NAME}" + +# Projects > "Python Wheels - riseproject-dev/python-wheels". Node IDs rather +# than names, so that the edit works with any gh version (item-edit only +# recently learned --field/--value) and a project rename changes nothing. +PROJECT_NUMBER = 4 +PROJECT_ID = "PVT_kwDOCRlTBM4BbcwJ" +STATUS_FIELD_ID = "PVTSSF_lADOCRlTBM4BbcwJzhWNMvs" +STATUS_TODO = "Todo" +STATUS_RISE = "Available in RISE PyPI" +STATUS_UPSTREAM = "Available Upstream" +STATUS_OPTION_IDS = { + STATUS_TODO: "f75ad846", + STATUS_RISE: "47fc9ee4", + STATUS_UPSTREAM: "98236657", +} + +TITLE_RE = re.compile(r"^\s*(.+?)\s+riscv64\s+support\s*$", re.IGNORECASE) + +QUERY = """ +query($owner: String!, $repo: String!, $cursor: String) { + repository(owner: $owner, name: $repo) { + issues(first: 100, after: $cursor, orderBy: {field: CREATED_AT, direction: ASC}) { + pageInfo { hasNextPage endCursor } + nodes { + number title url state + %s + } + } + } +} +""" + +PROJECT_FIELDS = """ + projectItems(first: 10) { + nodes { + project { id } + status: fieldValueByName(name: "Status") { + ... on ProjectV2ItemFieldSingleSelectValue { name } + } + } + }""" + + +def normalize_name(name): + """ + https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization + """ + return re.sub(r"[-_.]+", "-", name).lower() + + +def fetch_issues(owner=OWNER, repo=REPO_NAME, with_project=True, strict=False): + """ + Return {normalised package name: [issue, ...]} for " riscv64 support" + issues, open and closed, oldest first. + + Without the read:project scope the project fields are dropped with a + warning, unless `strict`, where that and any other GraphQL error raise + RuntimeError instead. + """ + query = QUERY % (PROJECT_FIELDS if with_project else "") + by_name = {} + cursor = None + while True: + cmd = ["gh", "api", "graphql", "-f", f"query={query}", + "-F", f"owner={owner}", "-F", f"repo={repo}"] + if cursor: + cmd += ["-F", f"cursor={cursor}"] + proc = subprocess.run(cmd, capture_output=True, text=True) + try: + data = json.loads(proc.stdout or "{}") + except json.JSONDecodeError: + data = {} + if with_project and not strict and "read:project" in proc.stderr: + print("warning: token lacks the read:project scope, project status unavailable" + " (gh auth refresh -h github.com -s read:project)", file=sys.stderr) + return fetch_issues(owner, repo, with_project=False) + if "data" not in data or data["data"] is None: + raise RuntimeError(f"gh api graphql failed: {proc.stderr.strip() or data}") + if data.get("errors"): + msgs = "; ".join(sorted({e.get("message", "") for e in data["errors"]})) + if strict: + raise RuntimeError(f"gh api graphql failed: {msgs}") + print(f"warning: {msgs}", file=sys.stderr) + issues = data["data"]["repository"]["issues"] + for issue in issues["nodes"]: + m = TITLE_RE.match(issue["title"]) + if m: + by_name.setdefault(normalize_name(m.group(1)), []).append(issue) + if not issues["pageInfo"]["hasNextPage"]: + return by_name + cursor = issues["pageInfo"]["endCursor"] + + +def oldest_open_issue(issues, package): + """The package's oldest open issue in fetch_issues() output, or None.""" + for issue in issues.get(normalize_name(package), []): + if issue["state"] == "OPEN": + return issue + return None + + +def project_status(issue): + if "projectItems" not in issue: + return "(unknown)" + for item in (issue.get("projectItems") or {}).get("nodes") or []: + if item and item["project"]["id"] == PROJECT_ID: + return (item.get("status") or {}).get("name") or "(no status)" + return "(not in project)" + + +def set_issue_status(issue_url, status): + """ + Add the issue to the project (a no-op when it is already there) and set its + Status. Raises subprocess.CalledProcessError when gh fails. + """ + item_id = subprocess.run( + ["gh", "project", "item-add", str(PROJECT_NUMBER), "--owner", OWNER, + "--url", issue_url, "--format", "json", "--jq", ".id"], + capture_output=True, text=True, check=True, + ).stdout.strip() + subprocess.run( + ["gh", "project", "item-edit", "--id", item_id, + "--project-id", PROJECT_ID, "--field-id", STATUS_FIELD_ID, + "--single-select-option-id", STATUS_OPTION_IDS[status]], + capture_output=True, text=True, check=True, + ) diff --git a/ci_scripts/sync_project_status.py b/ci_scripts/sync_project_status.py new file mode 100755 index 000000000000..0ddded6f8e91 --- /dev/null +++ b/ci_scripts/sync_project_status.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +"""Mark every package we publish as "Available in RISE PyPI" in the project. + +For each active package in docs/packages/*.yaml, take its oldest open +" riscv64 support" issue and, unless its Status already says it is +available (in RISE PyPI, or upstream after a deprecation), add it to the +"Python Wheels" project and set Status to "Available in RISE PyPI". + +The website workflow runs this after each deploy, so a package only flips once +its docs PR is merged and /simple/ lists it. It compares state rather than +reacting to one push, so a skipped or failed run is caught up by the next one. + +Needs a token with the ``project`` scope (the GitHub App: organization +"Projects: read and write"): + + gh auth refresh -h github.com -s project + ci_scripts/sync_project_status.py --dry-run +""" + +import argparse +import subprocess +import sys +import time +from pathlib import Path + +from github_issues import ( + STATUS_RISE, + STATUS_UPSTREAM, + fetch_issues, + oldest_open_issue, + project_status, + set_issue_status, +) + +DOCS_DIR = Path(__file__).resolve().parent.parent / "docs" / "packages" + +# Each update is two mutations. Usually only a package or two change per run, +# but a backlog would otherwise trip GitHub's secondary rate limit. +PAUSE_SECONDS = 1.5 + + +def read_packages(): + """Active package names: docs/packages/.yaml not marked `deprecated:`.""" + return [path.stem for path in sorted(DOCS_DIR.glob("*.yaml")) + if not path.read_text().startswith("deprecated:")] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--dry-run", action="store_true", + help="Print the issues that would change, change nothing") + args = parser.parse_args() + + try: + issues = fetch_issues(strict=True) + except RuntimeError as e: + sys.exit(str(e)) + + no_issue, failed, changed = [], [], 0 + for package in read_packages(): + issue = oldest_open_issue(issues, package) + if issue is None: + no_issue.append(package) + continue + status = project_status(issue) + if status in (STATUS_RISE, STATUS_UPSTREAM): + continue + prefix = "[dry-run] would set" if args.dry_run else "[+] set" + print(f"{prefix} {package} #{issue['number']}: {status} -> {STATUS_RISE}") + if args.dry_run: + changed += 1 + continue + try: + set_issue_status(issue["url"], STATUS_RISE) + changed += 1 + time.sleep(PAUSE_SECONDS) + except subprocess.CalledProcessError as e: + print(f"::warning::{package} #{issue['number']}: {e.stderr.strip()}") + failed.append(package) + + print(f"{changed} issue(s) {'to update' if args.dry_run else 'updated'}, " + f"{len(failed)} failed, {len(no_issue)} package(s) without an open issue") + if no_issue: + print("::group::Packages without an open ' riscv64 support' issue") + print("\n".join(no_issue)) + print("::endgroup::") + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/ci_scripts/update_doc.py b/ci_scripts/update_doc.py index e0d0271ba6ea..6738292ee136 100644 --- a/ci_scripts/update_doc.py +++ b/ci_scripts/update_doc.py @@ -32,7 +32,7 @@ import yaml -REPO = "riseproject-dev/python-wheels" +from github_issues import REPO, normalize_name CI_SCRIPTS_DIR = Path(os.path.dirname(os.path.abspath(__file__))) DOCS_DIR = (CI_SCRIPTS_DIR / ".." / "docs" / "packages").resolve() ARTIFACTS_PATH = os.environ.get("ARTIFACTS_PATH", "dist") @@ -47,13 +47,6 @@ def find_wheel_files(path): return sorted(Path(path).glob("*.whl")) -def normalize_name(name): - """ - https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization - """ - return re.sub(r"[-_.]+", "-", name).lower() - - # "License :: OSI Approved :: BSD License" -> "BSD" TROVE_LICENSE_PREFIX = "License :: " diff --git a/docs/workflows.md b/docs/workflows.md index 1b82fe36ae05..49c370eb8fc4 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -287,7 +287,12 @@ version is no longer pending. That push has two effects: - the package build workflow may be triggered by its YAML path, but `_setup.yml` returns `[]`, so all guarded build/publish jobs skip; - `website.yml` regenerates package pages and the static PEP 503/691 Simple API, - builds the Jekyll site, and deploys it to `pypi.riseproject.dev`. + builds the Jekyll site, and deploys it to `pypi.riseproject.dev`. Its + `sync-project-status` job then runs `ci_scripts/sync_project_status.py`, which + moves the open ` riscv64 support` issue of every active package in + `docs/packages/*.yaml` to *Available in RISE PyPI* in the Python Wheels + project, unless it already says *Available in RISE PyPI* or *Available Upstream*. + The issue stays open; only a deprecation closes it. The GitHub Release exists before this merge, but pip cannot discover it through the RISE index until the YAML reaches `main` and the website deployment @@ -420,7 +425,8 @@ job must grant `contents: write` and `pull-requests: write`. - `pr-checks.yml` checks commit/PR policy, validates package YAML/workflow pairing, and validates patch `Upstream-Status` headers. - `website.yml` turns `docs/packages/*.yaml` into package documentation and the - Simple API, then deploys GitHub Pages. + Simple API, deploys GitHub Pages, then marks newly published packages' + issues *Available in RISE PyPI* in the Python Wheels project. ## The RISC-V Wheels Dashboard diff --git a/skills/python-project-porting/references/pr-and-publishing.md b/skills/python-project-porting/references/pr-and-publishing.md index 133c31dd3abe..021ed4db4244 100644 --- a/skills/python-project-porting/references/pr-and-publishing.md +++ b/skills/python-project-porting/references/pr-and-publishing.md @@ -19,13 +19,17 @@ builds and publishes it — **do not dispatch anything**. Four steps remain, all 146 already exist, titles are not always the PyPI name (`SGLang`, `LibCST`, `PyNaCl`), and duplicates are already a problem (bcrypt has four). Match on a normalised name, case-insensitively, and when several match, link the **oldest** — that is what the repo already does (#82, #84, #94). -3. **Development link**: PR -> issue. A closing keyword in the PR body also works, but for an - already-merged PR use the mutation the Development panel uses: - `addCloseIssueReferences(input:{issueId:..., pullRequestIds:[...]})`. Read it back via the PR's - `closingIssuesReferences`. -4. **Project**: add the issue to Projects > *Python Wheels* (`PVT_kwDOCRlTBM4BbcwJ`) and set - **Status** (`PVTSSF_lADOCRlTBM4BbcwJzhWNMvs`) to *Available in RISE PyPI* - (option `47fc9ee4`; the others are *Todo* `f75ad846` and *Available Upstream* `98236657`). +3. **Development link**: PR -> issue, on the already-merged PR, with the mutation the + Development panel uses: `addCloseIssueReferences(input:{issueId:..., pullRequestIds:[...]})`. + Read it back via the PR's `closingIssuesReferences`. Never put a closing keyword in the PR + body: the issue must stay open, only a deprecation closes it. +4. **Project**: automatic. Once the package PR is merged and the site deployed, `website.yml`'s + `sync-project-status` job (`ci_scripts/sync_project_status.py`) adds the issue to + Projects > *Python Wheels* (`PVT_kwDOCRlTBM4BbcwJ`) and sets **Status** + (`PVTSSF_lADOCRlTBM4BbcwJzhWNMvs`) to *Available in RISE PyPI* (option `47fc9ee4`; the + others are *Todo* `f75ad846` and *Available Upstream* `98236657`). Do not set it by hand + before the package is on `/simple/`; `ci_scripts/sync_project_status.py --dry-run` shows + what the next run will change. Needs `project` scope on the gh token (`gh auth refresh -h github.com -s project`) on top of `repo`/`workflow`. PR #390 <-> issue #405 is the reference pair to diff a new one against.