diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index c6bef9d5cc7..8cf1b4f271d 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -60,3 +60,33 @@ jobs: with: path: requests-cache.sqlite key: ${{ steps.dashboard-requests-cache.outputs.cache-primary-key }} + + # A package is officially available once deploy has published /simple/, so + # we can now move the issue Status to "Available in RISE PyPI". + # Compares docs/packages with the project rather than diffing this push, + # so pushes the concurrency queue skipped are caught up. + sync-project-status: + needs: deploy + if: ${{ vars.RISEPROJECT_APP_CLIENT_ID != '' }} + runs-on: ubuntu-latest + permissions: + contents: read + steps: + # GITHUB_TOKEN cannot write organization projects; the App needs the + # organization "Projects: read and write" permission. + - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: app-token + with: + client-id: ${{ vars.RISEPROJECT_APP_CLIENT_ID }} + private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: | + ci_scripts + docs/packages + + - name: Mark published packages in the Python Wheels project + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: python3 ci_scripts/sync_project_status.py diff --git a/ci_scripts/check_versions.py b/ci_scripts/check_versions.py index d400c9ff224..17a0e9e759f 100755 --- a/ci_scripts/check_versions.py +++ b/ci_scripts/check_versions.py @@ -16,7 +16,6 @@ import json import re import shutil -import subprocess import sys import time from datetime import datetime, timedelta, timezone @@ -27,14 +26,14 @@ import requests import yaml +from github_issues import fetch_issues, normalize_name, oldest_open_issue from pending_versions import base_versions, RISCV64_PYTHONS -from update_doc import normalize_name, version_blocks +from update_doc import version_blocks ROOT_DIR = Path(__file__).resolve().parent.parent DOCS_DIR = ROOT_DIR / "docs" / "packages" PATCHES_DIR = ROOT_DIR / "patches" -REPO = "riseproject-dev/python-wheels" # Python versions riscv64 can build (cp314t -> "3.14t"); an upstream riscv64 # wheel for any of them deprecates our own build. "3.14t" is the free-threaded @@ -261,31 +260,25 @@ def target_files_lines(pythons: List[str]) -> List[str]: return [f" {line}" for line in dumped.splitlines()] -def find_upstream_issue(package: str) -> Optional[str]: - """ - Find issue for a package in the wheel_builder's Upstream milestone. +_issues: Optional[Dict] = None - All issues in the upstream milestone follow the same title format: - {package} riscv64 support +def find_upstream_issue(package: str) -> Optional[str]: """ - try: - issue_title = f"{package} riscv64 support" - result = subprocess.run([ - "gh", "issue", "list", - "--search", issue_title, - "--json", "number", - "--jq", ".[0].number", - "--repo", REPO, - ], capture_output=True, text=True, timeout=30) - - if result.returncode != 0: - return None + Number of the package's oldest open "{package} riscv64 support" issue. - return result.stdout.strip() or None - - except (subprocess.TimeoutExpired, subprocess.CalledProcessError, Exception): - return None + The issues are fetched once and reused for every package of the run. + """ + global _issues + if _issues is None: + try: + # Project fields are not needed here, and the nightly token cannot read them. + _issues = fetch_issues(with_project=False) + except RuntimeError as e: + print(f"warning: {e}", file=sys.stderr) + _issues = {} + issue = oldest_open_issue(_issues, package) + return str(issue["number"]) if issue else None def deprecate_package(package: str) -> bool: diff --git a/ci_scripts/github_issues.py b/ci_scripts/github_issues.py new file mode 100644 index 00000000000..aec4cc4b360 --- /dev/null +++ b/ci_scripts/github_issues.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +"""Shared helpers for the " riscv64 support" issues and their status in +the "Python Wheels" project. + +Uses the ``gh`` CLI. Reading project fields needs the ``read:project`` scope, +changing them the ``project`` scope (for the GitHub App: the organization +"Projects: read and write" permission). +""" + +import json +import re +import subprocess +import sys + +OWNER = "riseproject-dev" +REPO_NAME = "python-wheels" +REPO = f"{OWNER}/{REPO_NAME}" + +# Projects > "Python Wheels - riseproject-dev/python-wheels". Node IDs rather +# than names, so that the edit works with any gh version (item-edit only +# recently learned --field/--value) and a project rename changes nothing. +PROJECT_NUMBER = 4 +PROJECT_ID = "PVT_kwDOCRlTBM4BbcwJ" +STATUS_FIELD_ID = "PVTSSF_lADOCRlTBM4BbcwJzhWNMvs" +STATUS_TODO = "Todo" +STATUS_RISE = "Available in RISE PyPI" +STATUS_UPSTREAM = "Available Upstream" +STATUS_OPTION_IDS = { + STATUS_TODO: "f75ad846", + STATUS_RISE: "47fc9ee4", + STATUS_UPSTREAM: "98236657", +} + +TITLE_RE = re.compile(r"^\s*(.+?)\s+riscv64\s+support\s*$", re.IGNORECASE) + +QUERY = """ +query($owner: String!, $repo: String!, $cursor: String) { + repository(owner: $owner, name: $repo) { + issues(first: 100, after: $cursor, orderBy: {field: CREATED_AT, direction: ASC}) { + pageInfo { hasNextPage endCursor } + nodes { + number title url state + %s + } + } + } +} +""" + +PROJECT_FIELDS = """ + projectItems(first: 10) { + nodes { + project { id } + status: fieldValueByName(name: "Status") { + ... on ProjectV2ItemFieldSingleSelectValue { name } + } + } + }""" + + +def normalize_name(name): + """ + https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization + """ + return re.sub(r"[-_.]+", "-", name).lower() + + +def fetch_issues(owner=OWNER, repo=REPO_NAME, with_project=True, strict=False): + """ + Return {normalised package name: [issue, ...]} for " riscv64 support" + issues, open and closed, oldest first. + + Without the read:project scope the project fields are dropped with a + warning, unless `strict`, where that and any other GraphQL error raise + RuntimeError instead. + """ + query = QUERY % (PROJECT_FIELDS if with_project else "") + by_name = {} + cursor = None + while True: + cmd = ["gh", "api", "graphql", "-f", f"query={query}", + "-F", f"owner={owner}", "-F", f"repo={repo}"] + if cursor: + cmd += ["-F", f"cursor={cursor}"] + proc = subprocess.run(cmd, capture_output=True, text=True) + try: + data = json.loads(proc.stdout or "{}") + except json.JSONDecodeError: + data = {} + if with_project and not strict and "read:project" in proc.stderr: + print("warning: token lacks the read:project scope, project status unavailable" + " (gh auth refresh -h github.com -s read:project)", file=sys.stderr) + return fetch_issues(owner, repo, with_project=False) + if "data" not in data or data["data"] is None: + raise RuntimeError(f"gh api graphql failed: {proc.stderr.strip() or data}") + if data.get("errors"): + msgs = "; ".join(sorted({e.get("message", "") for e in data["errors"]})) + if strict: + raise RuntimeError(f"gh api graphql failed: {msgs}") + print(f"warning: {msgs}", file=sys.stderr) + issues = data["data"]["repository"]["issues"] + for issue in issues["nodes"]: + m = TITLE_RE.match(issue["title"]) + if m: + by_name.setdefault(normalize_name(m.group(1)), []).append(issue) + if not issues["pageInfo"]["hasNextPage"]: + return by_name + cursor = issues["pageInfo"]["endCursor"] + + +def oldest_open_issue(issues, package): + """The package's oldest open issue in fetch_issues() output, or None.""" + for issue in issues.get(normalize_name(package), []): + if issue["state"] == "OPEN": + return issue + return None + + +def project_status(issue): + if "projectItems" not in issue: + return "(unknown)" + for item in (issue.get("projectItems") or {}).get("nodes") or []: + if item and item["project"]["id"] == PROJECT_ID: + return (item.get("status") or {}).get("name") or "(no status)" + return "(not in project)" + + +def set_issue_status(issue_url, status): + """ + Add the issue to the project (a no-op when it is already there) and set its + Status. Raises subprocess.CalledProcessError when gh fails. + """ + item_id = subprocess.run( + ["gh", "project", "item-add", str(PROJECT_NUMBER), "--owner", OWNER, + "--url", issue_url, "--format", "json", "--jq", ".id"], + capture_output=True, text=True, check=True, + ).stdout.strip() + subprocess.run( + ["gh", "project", "item-edit", "--id", item_id, + "--project-id", PROJECT_ID, "--field-id", STATUS_FIELD_ID, + "--single-select-option-id", STATUS_OPTION_IDS[status]], + capture_output=True, text=True, check=True, + ) diff --git a/ci_scripts/sync_project_status.py b/ci_scripts/sync_project_status.py new file mode 100755 index 00000000000..0ddded6f8e9 --- /dev/null +++ b/ci_scripts/sync_project_status.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +"""Mark every package we publish as "Available in RISE PyPI" in the project. + +For each active package in docs/packages/*.yaml, take its oldest open +" riscv64 support" issue and, unless its Status already says it is +available (in RISE PyPI, or upstream after a deprecation), add it to the +"Python Wheels" project and set Status to "Available in RISE PyPI". + +The website workflow runs this after each deploy, so a package only flips once +its docs PR is merged and /simple/ lists it. It compares state rather than +reacting to one push, so a skipped or failed run is caught up by the next one. + +Needs a token with the ``project`` scope (the GitHub App: organization +"Projects: read and write"): + + gh auth refresh -h github.com -s project + ci_scripts/sync_project_status.py --dry-run +""" + +import argparse +import subprocess +import sys +import time +from pathlib import Path + +from github_issues import ( + STATUS_RISE, + STATUS_UPSTREAM, + fetch_issues, + oldest_open_issue, + project_status, + set_issue_status, +) + +DOCS_DIR = Path(__file__).resolve().parent.parent / "docs" / "packages" + +# Each update is two mutations. Usually only a package or two change per run, +# but a backlog would otherwise trip GitHub's secondary rate limit. +PAUSE_SECONDS = 1.5 + + +def read_packages(): + """Active package names: docs/packages/.yaml not marked `deprecated:`.""" + return [path.stem for path in sorted(DOCS_DIR.glob("*.yaml")) + if not path.read_text().startswith("deprecated:")] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--dry-run", action="store_true", + help="Print the issues that would change, change nothing") + args = parser.parse_args() + + try: + issues = fetch_issues(strict=True) + except RuntimeError as e: + sys.exit(str(e)) + + no_issue, failed, changed = [], [], 0 + for package in read_packages(): + issue = oldest_open_issue(issues, package) + if issue is None: + no_issue.append(package) + continue + status = project_status(issue) + if status in (STATUS_RISE, STATUS_UPSTREAM): + continue + prefix = "[dry-run] would set" if args.dry_run else "[+] set" + print(f"{prefix} {package} #{issue['number']}: {status} -> {STATUS_RISE}") + if args.dry_run: + changed += 1 + continue + try: + set_issue_status(issue["url"], STATUS_RISE) + changed += 1 + time.sleep(PAUSE_SECONDS) + except subprocess.CalledProcessError as e: + print(f"::warning::{package} #{issue['number']}: {e.stderr.strip()}") + failed.append(package) + + print(f"{changed} issue(s) {'to update' if args.dry_run else 'updated'}, " + f"{len(failed)} failed, {len(no_issue)} package(s) without an open issue") + if no_issue: + print("::group::Packages without an open ' riscv64 support' issue") + print("\n".join(no_issue)) + print("::endgroup::") + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/ci_scripts/update_doc.py b/ci_scripts/update_doc.py index e0d0271ba6e..6738292ee13 100644 --- a/ci_scripts/update_doc.py +++ b/ci_scripts/update_doc.py @@ -32,7 +32,7 @@ import yaml -REPO = "riseproject-dev/python-wheels" +from github_issues import REPO, normalize_name CI_SCRIPTS_DIR = Path(os.path.dirname(os.path.abspath(__file__))) DOCS_DIR = (CI_SCRIPTS_DIR / ".." / "docs" / "packages").resolve() ARTIFACTS_PATH = os.environ.get("ARTIFACTS_PATH", "dist") @@ -47,13 +47,6 @@ def find_wheel_files(path): return sorted(Path(path).glob("*.whl")) -def normalize_name(name): - """ - https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization - """ - return re.sub(r"[-_.]+", "-", name).lower() - - # "License :: OSI Approved :: BSD License" -> "BSD" TROVE_LICENSE_PREFIX = "License :: " diff --git a/docs/workflows.md b/docs/workflows.md index 1b82fe36ae0..49c370eb8fc 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -287,7 +287,12 @@ version is no longer pending. That push has two effects: - the package build workflow may be triggered by its YAML path, but `_setup.yml` returns `[]`, so all guarded build/publish jobs skip; - `website.yml` regenerates package pages and the static PEP 503/691 Simple API, - builds the Jekyll site, and deploys it to `pypi.riseproject.dev`. + builds the Jekyll site, and deploys it to `pypi.riseproject.dev`. Its + `sync-project-status` job then runs `ci_scripts/sync_project_status.py`, which + moves the open ` riscv64 support` issue of every active package in + `docs/packages/*.yaml` to *Available in RISE PyPI* in the Python Wheels + project, unless it already says *Available in RISE PyPI* or *Available Upstream*. + The issue stays open; only a deprecation closes it. The GitHub Release exists before this merge, but pip cannot discover it through the RISE index until the YAML reaches `main` and the website deployment @@ -420,7 +425,8 @@ job must grant `contents: write` and `pull-requests: write`. - `pr-checks.yml` checks commit/PR policy, validates package YAML/workflow pairing, and validates patch `Upstream-Status` headers. - `website.yml` turns `docs/packages/*.yaml` into package documentation and the - Simple API, then deploys GitHub Pages. + Simple API, deploys GitHub Pages, then marks newly published packages' + issues *Available in RISE PyPI* in the Python Wheels project. ## The RISC-V Wheels Dashboard diff --git a/skills/python-project-porting/references/pr-and-publishing.md b/skills/python-project-porting/references/pr-and-publishing.md index 133c31dd3ab..021ed4db424 100644 --- a/skills/python-project-porting/references/pr-and-publishing.md +++ b/skills/python-project-porting/references/pr-and-publishing.md @@ -19,13 +19,17 @@ builds and publishes it — **do not dispatch anything**. Four steps remain, all 146 already exist, titles are not always the PyPI name (`SGLang`, `LibCST`, `PyNaCl`), and duplicates are already a problem (bcrypt has four). Match on a normalised name, case-insensitively, and when several match, link the **oldest** — that is what the repo already does (#82, #84, #94). -3. **Development link**: PR -> issue. A closing keyword in the PR body also works, but for an - already-merged PR use the mutation the Development panel uses: - `addCloseIssueReferences(input:{issueId:..., pullRequestIds:[...]})`. Read it back via the PR's - `closingIssuesReferences`. -4. **Project**: add the issue to Projects > *Python Wheels* (`PVT_kwDOCRlTBM4BbcwJ`) and set - **Status** (`PVTSSF_lADOCRlTBM4BbcwJzhWNMvs`) to *Available in RISE PyPI* - (option `47fc9ee4`; the others are *Todo* `f75ad846` and *Available Upstream* `98236657`). +3. **Development link**: PR -> issue, on the already-merged PR, with the mutation the + Development panel uses: `addCloseIssueReferences(input:{issueId:..., pullRequestIds:[...]})`. + Read it back via the PR's `closingIssuesReferences`. Never put a closing keyword in the PR + body: the issue must stay open, only a deprecation closes it. +4. **Project**: automatic. Once the package PR is merged and the site deployed, `website.yml`'s + `sync-project-status` job (`ci_scripts/sync_project_status.py`) adds the issue to + Projects > *Python Wheels* (`PVT_kwDOCRlTBM4BbcwJ`) and sets **Status** + (`PVTSSF_lADOCRlTBM4BbcwJzhWNMvs`) to *Available in RISE PyPI* (option `47fc9ee4`; the + others are *Todo* `f75ad846` and *Available Upstream* `98236657`). Do not set it by hand + before the package is on `/simple/`; `ci_scripts/sync_project_status.py --dry-run` shows + what the next run will change. Needs `project` scope on the gh token (`gh auth refresh -h github.com -s project`) on top of `repo`/`workflow`. PR #390 <-> issue #405 is the reference pair to diff a new one against.