diff --git a/.changelog/5381.removed b/.changelog/5381.removed new file mode 100644 index 00000000000..9d30be6336d --- /dev/null +++ b/.changelog/5381.removed @@ -0,0 +1 @@ +Removed `scripts/eachdist.py` and the `.github/scripts/update-version*.sh` wrappers in favor of tox and small purpose-built scripts: `scripts/version.py`, `scripts/update_version.py`, and `scripts/update_patch_version.py` for release version bumping, and `scripts/repo_targets.py` for distribution discovery. Renamed `eachdist.ini` to `repo.toml`. \ No newline at end of file diff --git a/.github/scripts/update-version-patch.sh b/.github/scripts/update-version-patch.sh deleted file mode 100755 index fec7cd82604..00000000000 --- a/.github/scripts/update-version-patch.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -e - -sed -i "/\[stable\]/{n;s/version=.*/version=$1/}" eachdist.ini -sed -i "/\[prerelease\]/{n;s/version=.*/version=$2/}" eachdist.ini - -./scripts/eachdist.py update_patch_versions \ - --stable_version=$1 \ - --unstable_version=$2 \ - --stable_version_prev=$3 \ - --unstable_version_prev=$4 - diff --git a/.github/scripts/update-version.sh b/.github/scripts/update-version.sh deleted file mode 100755 index ba1bd22955b..00000000000 --- a/.github/scripts/update-version.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -e - -sed -i "/\[stable\]/{n;s/version=.*/version=$1/}" eachdist.ini -sed -i "/\[prerelease\]/{n;s/version=.*/version=$2/}" eachdist.ini - -./scripts/eachdist.py update_versions --versions stable,prerelease diff --git a/.github/workflows/prepare-patch-release.yml b/.github/workflows/prepare-patch-release.yml index 4ba5b9955a9..282d02e3ea1 100644 --- a/.github/workflows/prepare-patch-release.yml +++ b/.github/workflows/prepare-patch-release.yml @@ -1,3 +1,30 @@ +# ============================================================================= +# Prepare patch release workflow +# ============================================================================= +# +# Purpose: prepares a patch release (e.g. 1.44.0 -> 1.44.1) on an existing +# long-term release branch. Opens a PR against that same branch which bumps +# the patch version and regenerates the changelog -- mirroring what +# prepare-release-branch.yml does for a brand new release branch -- and, in +# addition, backports the new changelog entry to main via a second PR, since +# main's own version isn't touched by a patch release but its CHANGELOG.md +# should still document that the patch happened. +# +# Run this manually (workflow_dispatch) on the long-term release branch you +# want to patch (one previously created by prepare-release-branch.yml's +# "normal release" path -- see that file's header comment; one-off +# prerelease branches, e.g. release/v1.45.0rc2-0.66b0, are never patched +# this way). Once the PR this workflow opens is merged, run release.yml on +# that branch again to actually publish the patch. See release.yml's header +# comment for how this fits into the full 3-workflow release pipeline. +# +# Unlike prepare-release-branch.yml/update_version.py, this workflow (via +# scripts/release/update_patch_version.py) can't rely on version pins ending +# in ".dev" to know what to bump -- a release branch's versions are already +# finalized, non-dev values -- so it explicitly computes both the previous +# and the next patch version and passes both to the script, which replaces +# only pins matching the exact previous version. +# ============================================================================= name: Prepare patch release on: workflow_dispatch: @@ -14,20 +41,34 @@ jobs: steps: - uses: actions/checkout@v4 + # tomlkit: read/write repo.toml (via scripts/release/print_version.py + # and update_patch_version.py). towncrier: generate the changelog + # section for this patch below. - name: Install dependencies - run: pip install toml towncrier - + run: pip install tomlkit towncrier + + # Guard rail: this workflow only makes sense on a long-term release + # branch, i.e. one named "release/vX.Y.x-0.ZbX" (the "x" is a literal + # placeholder character in the branch name, standing in for "whatever + # patch number", not a wildcard) -- the kind created by + # prepare-release-branch.yml's non-prerelease path. It refuses to run + # on main, a one-off prerelease branch, or anything else. - run: | if [[ ! $GITHUB_REF_NAME =~ ^release/v[0-9]+\.[0-9]+\.x-0\.[0-9]+bx$ ]]; then echo this workflow should only be run against long-term release branches exit 1 fi + # Reads this branch's current (already-published) version and + # computes both the version being patched (*_PREV) and the new patch + # version (bumping the patch number by 1 on both tracks). - name: Set environment variables run: | - stable_version=$(./scripts/eachdist.py version --mode stable) - unstable_version=$(./scripts/eachdist.py version --mode prerelease) + stable_version=$(./scripts/release/print_version.py --stable) + unstable_version=$(./scripts/release/print_version.py --unstable) + # Split e.g. "1.44.0" into stable_major_minor="1.44" + # stable_patch=0. if [[ $stable_version =~ ^([0-9]+\.[0-9]+)\.([0-9]+)$ ]]; then stable_major_minor="${BASH_REMATCH[1]}" stable_patch="${BASH_REMATCH[2]}" @@ -36,6 +77,7 @@ jobs: exit 1 fi + # Split e.g. "0.65b0" into unstable_minor=65 unstable_patch=0. if [[ $unstable_version =~ ^0\.([0-9]+)b([0-9]+)$ ]]; then unstable_minor="${BASH_REMATCH[1]}" unstable_patch="${BASH_REMATCH[2]}" @@ -44,6 +86,9 @@ jobs: exit 1 fi + # *_prev is just the current (pre-patch) version, restated from + # its parsed pieces; the *_version (without _prev) is that same + # version with its patch number incremented by one. stable_version_prev="$stable_major_minor.$((stable_patch))" unstable_version_prev="0.${unstable_minor}b$((unstable_patch))" stable_version="$stable_major_minor.$((stable_patch + 1))" @@ -54,21 +99,39 @@ jobs: echo "STABLE_VERSION_PREV=$stable_version_prev" >> $GITHUB_ENV echo "UNSTABLE_VERSION_PREV=$unstable_version_prev" >> $GITHUB_ENV + # Rewrites repo.toml, every package's pyproject.toml pins currently + # matching the exact previous version, and every package's + # __version__, to the new patch version. See + # scripts/release/update_patch_version.py's own module docstring for + # a worked before/after example. - name: Update version - run: .github/scripts/update-version-patch.sh $STABLE_VERSION $UNSTABLE_VERSION $STABLE_VERSION_PREV $UNSTABLE_VERSION_PREV + run: ./scripts/release/update_patch_version.py --stable_version=$STABLE_VERSION --unstable_version=$UNSTABLE_VERSION --stable_version_prev=$STABLE_VERSION_PREV --unstable_version_prev=$UNSTABLE_VERSION_PREV + # Consumes the .changelog/*. fragments accumulated since the + # last release into a new "## Version X/Y" section in CHANGELOG.md, + # same as prepare-release-branch.yml does for a normal release. - name: Generate changelog run: towncrier build --yes --version "$STABLE_VERSION/$UNSTABLE_VERSION" + # Configures git's committer identity as the "otelbot" account so the + # commits made below pass this repo's CLA check. - name: Use CLA approved github bot run: .github/scripts/use-cla-approved-github-bot.sh + # Mints a short-lived GitHub App installation token for otelbot, + # reused by both PR-creating steps below (this workflow only needs + # one token for its single job, unlike prepare-release-branch.yml + # which mints one per job). - uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 id: otelbot-token with: app-id: ${{ vars.OTELBOT_APP_ID }} private-key: ${{ secrets.OTELBOT_PRIVATE_KEY }} + # Commits the version bump + changelog changes, pushes them to a new + # otelbot-owned branch, and opens a PR from that branch back into + # *this same release branch* (not into main -- see the backport steps + # below for how main's changelog gets updated separately). - name: Create pull request id: create_pr env: @@ -86,6 +149,8 @@ jobs: --base $GITHUB_REF_NAME) echo "pr_url=$pr_url" >> $GITHUB_OUTPUT + # Tags the PR so it's easy to find/filter in the repo's PR list. Only + # runs if a PR URL was actually produced above. - name: Add prepare-release label to PR if: steps.create_pr.outputs.pr_url != '' env: @@ -93,6 +158,9 @@ jobs: run: | gh pr edit ${{ steps.create_pr.outputs.pr_url }} --add-label "prepare-release" + # Switches the workspace to main, so the remaining steps operate on + # main's git history instead of the release branch's (needed to build + # a PR whose base is main, below). - uses: actions/checkout@v4 with: ref: main @@ -100,6 +168,13 @@ jobs: - name: Use CLA approved github bot run: .github/scripts/use-cla-approved-github-bot.sh + # Since a patch release doesn't touch main's own version, main's + # CHANGELOG.md needs its own separate PR to pick up the new "## + # Version X/Y" section -- this step builds that PR. It fetches the + # otelbot branch pushed above (which already has the correct, + # towncrier-generated CHANGELOG.md from the release branch), copies + # just that one file into a fresh commit on top of main, and opens a + # PR from that commit into main. - name: Backport patch release changelog to main id: backport_pr env: @@ -123,6 +198,9 @@ jobs: --base main) echo "pr_url=$pr_url" >> $GITHUB_OUTPUT + # "Skip Changelog": this PR's only change *is* CHANGELOG.md itself + # (copied verbatim from the release branch above), so it doesn't need + # -- and shouldn't be asked for -- a changelog fragment of its own. - name: Add Skip Changelog label to backport PR if: steps.backport_pr.outputs.pr_url != '' env: diff --git a/.github/workflows/prepare-release-branch.yml b/.github/workflows/prepare-release-branch.yml index bac3dd082c6..ce1b9a07aef 100644 --- a/.github/workflows/prepare-release-branch.yml +++ b/.github/workflows/prepare-release-branch.yml @@ -1,3 +1,38 @@ +# ============================================================================= +# Prepare release branch workflow +# ============================================================================= +# +# Purpose: kicks off a release by cutting a new release branch from main and +# preparing two pull requests: one that bumps the new branch's version to +# the version being released (for a human to merge before running +# release.yml), and a separate one that bumps main forward to the *next* +# dev version, so ongoing development on main isn't left pointing at a +# version that's about to ship. +# +# This is the FIRST of this repo's 3 release workflows, and it must be +# triggered manually (workflow_dispatch) on main. See release.yml's header +# comment for how it fits into the full release pipeline (this workflow -> +# release.yml -> optionally prepare-patch-release.yml -> release.yml again). +# +# Two release shapes are supported, both driven by the optional +# prerelease_version input: +# - Normal release (no input given): cuts a long-term release branch named +# after the *next* stable/unstable version rounded down to a ".0"/"b0" +# patch number (e.g. main at 1.45.0.dev/0.66b0.dev -> branch +# release/v1.45.x-0.66bx), and prepares that branch to release exactly +# that ".0"/"b0" version. Patch releases later reuse this same branch +# (see prepare-patch-release.yml). +# - One-off prerelease (prerelease_version given, e.g. "1.45.0rc2"): cuts a +# branch for a release-candidate-style version instead (e.g. +# release/v1.45.0rc2-0.66b0), which does NOT get reused for patches. +# +# This workflow has 3 jobs: "prereqs" (validation only, no writes), and two +# independent jobs that both depend on it and run in parallel -- one opens +# the PR against the new release branch, the other opens the PR against +# main. Splitting them like this means a failure preparing one PR doesn't +# necessarily stop the other, and each PR's diff stays focused on exactly +# one branch's version bump. +# ============================================================================= name: Prepare release branch on: workflow_dispatch: @@ -10,25 +45,36 @@ permissions: contents: read jobs: + # Validates the workflow was triggered correctly before either of the two + # PR-creating jobs below does any real work. Does not modify anything. prereqs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Install toml - run: pip install toml + # scripts/release/print_version.py needs tomlkit to parse repo.toml. + - name: Install tomlkit + run: pip install tomlkit - name: Verify prerequisites env: PRERELEASE_VERSION: ${{ github.event.inputs.prerelease_version }} run: | + # Guard rail: this workflow cuts a new branch off of the current + # commit and rewrites version files on it, so it must only ever + # run from main, never from a release branch or a fork/PR branch. if [[ $GITHUB_REF_NAME != main ]]; then echo this workflow should only be run against main exit 1 fi + # If a specific prerelease_version was requested (e.g. "1.9.0rc2"), + # sanity-check it's actually a prerelease *of* the version + # currently on main (e.g. main is at "1.9.0.dev" -> stripped to + # "1.9.0" -> "1.9.0rc2" must start with "1.9.0"). This catches an + # operator accidentally typing the wrong version number. if [[ ! -z $PRERELEASE_VERSION ]]; then - stable_version=$(./scripts/eachdist.py version --mode stable) + stable_version=$(./scripts/release/print_version.py --stable) stable_version=${stable_version//.dev/} if [[ $PRERELEASE_VERSION != ${stable_version}* ]]; then echo "$PRERELEASE_VERSION is not a prerelease for the version on main ($stable_version)" @@ -36,6 +82,10 @@ jobs: fi fi + # Cuts the new release branch and opens a PR *against that branch* (not + # against main) which bumps its version to the version being released and + # regenerates the changelog. A human must review and merge this PR before + # release.yml can be run on the new branch. create-pull-request-against-release-branch: permissions: contents: write # required for pushing changes @@ -45,23 +95,38 @@ jobs: steps: - uses: actions/checkout@v4 + # tomlkit: read/write repo.toml. towncrier: generate the changelog + # section for the new release below. - name: Install dependencies - run: pip install toml towncrier + run: pip install tomlkit towncrier - name: Create release branch env: PRERELEASE_VERSION: ${{ github.event.inputs.prerelease_version }} run: | + # Absent an explicit prerelease_version, the version to release is + # main's current stable version with ".dev" stripped (e.g. + # "1.45.0.dev" -> "1.45.0"). With one given, that's the version to + # release outright (already validated against main's version in + # the prereqs job above). if [[ -z $PRERELEASE_VERSION ]]; then - stable_version=$(./scripts/eachdist.py version --mode stable) + stable_version=$(./scripts/release/print_version.py --stable) stable_version=${stable_version//.dev/} else stable_version=$PRERELEASE_VERSION fi - unstable_version=$(./scripts/eachdist.py version --mode prerelease) + # Same idea for the unstable/prerelease-track version. + unstable_version=$(./scripts/release/print_version.py --unstable) unstable_version=${unstable_version//.dev/} + # Names the release branch. A ".0"/"b0" version (a normal, + # non-prerelease release) gets a long-term branch name using "x" + # as a wildcard for the not-yet-known patch number (e.g. + # "1.45.0" + "0.66b0" -> "release/v1.45.x-0.66bx"), since this + # same branch will be reused for future patch releases. Any other + # version (e.g. "1.45.0rc2") is a one-off prerelease, so the + # branch is just named after that exact version instead. if [[ $stable_version =~ ^([0-9]+)\.([0-9]+)\.0$ ]]; then stable_version_branch_part=$(echo $stable_version | sed -E 's/([0-9]+)\.([0-9]+)\.0/\1.\2.x/') unstable_version_branch_part=$(echo $unstable_version | sed -E 's/0\.([0-9]+)b0/0.\1bx/') @@ -74,27 +139,51 @@ jobs: exit 1 fi + # Pushes a new branch pointing at the current commit (main's + # HEAD) under the computed name. No version bump has happened + # yet -- that's done by the "Update version" step below, as a + # separate commit on top of this branch. git push origin HEAD:$release_branch_name echo "STABLE_VERSION=$stable_version" >> $GITHUB_ENV echo "UNSTABLE_VERSION=$unstable_version" >> $GITHUB_ENV echo "RELEASE_BRANCH_NAME=$release_branch_name" >> $GITHUB_ENV + # Rewrites repo.toml, every package's pyproject.toml dependency pins, + # and every package's __version__ from the current ".dev" version to + # the finalized release version computed above. See + # scripts/release/update_version.py's own module docstring for a + # worked before/after example of exactly what it changes. - name: Update version - run: .github/scripts/update-version.sh $STABLE_VERSION $UNSTABLE_VERSION + run: ./scripts/release/update_version.py --stable_version=$STABLE_VERSION --unstable_version=$UNSTABLE_VERSION + # Consumes the .changelog/*. fragment files accumulated since + # the last release and rewrites them into a single new + # "## Version X/Y" section at the top of CHANGELOG.md. - name: Generate changelog run: towncrier build --yes --version "$STABLE_VERSION/$UNSTABLE_VERSION" + # Configures git's committer identity as the "otelbot" account (see + # .github/scripts/use-cla-approved-github-bot.sh) so the commit made + # below passes this repo's CLA check. - name: Use CLA approved github bot run: .github/scripts/use-cla-approved-github-bot.sh + # Mints a short-lived GitHub App installation token for otelbot, used + # instead of the workflow's own GITHUB_TOKEN below specifically so + # that opening the PR can itself trigger other workflows (see the env + # comment on the next step). - uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 id: otelbot-token with: app-id: ${{ vars.OTELBOT_APP_ID }} private-key: ${{ secrets.OTELBOT_PRIVATE_KEY }} + # Commits the version bump + changelog changes made above, pushes + # them to a new otelbot-owned branch, and opens a PR from that branch + # *into the release branch created earlier* -- not into main. Records + # the created PR's URL in $GITHUB_OUTPUT (empty if PR creation is + # ever skipped/fails silently) for the labeling step below. - name: Create pull request against the release branch id: create_release_branch_pr env: @@ -112,6 +201,8 @@ jobs: --base $RELEASE_BRANCH_NAME) echo "pr_url=$pr_url" >> $GITHUB_OUTPUT + # Tags the PR so it's easy to find/filter in the repo's PR list. Only + # runs if a PR URL was actually produced above. - name: Add prepare-release label to PR if: steps.create_release_branch_pr.outputs.pr_url != '' env: @@ -119,6 +210,11 @@ jobs: run: | gh pr edit ${{ steps.create_release_branch_pr.outputs.pr_url }} --add-label "prepare-release" + # Runs in parallel with the job above (both only depend on prereqs, not on + # each other). Bumps main itself forward to the *next* dev version -- so + # that, once the release branch above ships, ongoing work on main is + # already pointed at the version after it -- and opens a separate PR + # against main for that change. create-pull-request-against-main: permissions: contents: write # required for pushing changes @@ -128,23 +224,35 @@ jobs: steps: - uses: actions/checkout@v4 + # tomlkit: read/write repo.toml. towncrier: generate the changelog + # section for the release that's about to happen off of this run's + # current version (see the "Generate changelog" step below). - name: Install dependencies - run: pip install toml towncrier + run: pip install tomlkit towncrier - name: Set environment variables env: PRERELEASE_VERSION: ${{ github.event.inputs.prerelease_version }} run: | + # Same version-resolution logic as the sibling job above: the + # version about to be released is either main's current version + # (".dev" stripped) or the explicit prerelease_version given. if [[ -z $PRERELEASE_VERSION ]]; then - stable_version=$(./scripts/eachdist.py version --mode stable) + stable_version=$(./scripts/release/print_version.py --stable) stable_version=${stable_version//.dev/} else stable_version=$PRERELEASE_VERSION fi - unstable_version=$(./scripts/eachdist.py version --mode prerelease) + unstable_version=$(./scripts/release/print_version.py --unstable) unstable_version=${unstable_version//.dev/} + # Computes the version main should advance to *after* this + # release: for a normal ".0" release, bump the minor version + # (e.g. 1.45.0 -> 1.46.0). For a prerelease/release-candidate + # version, stay on the same "X.Y.0" and just strip the + # prerelease suffix (e.g. 1.45.0rc2 -> 1.45.0), since the real + # 1.45.0 hasn't shipped yet. if [[ $stable_version =~ ^([0-9]+)\.([0-9]+)\.0$ ]]; then stable_major="${BASH_REMATCH[1]}" stable_minor="${BASH_REMATCH[2]}" @@ -159,6 +267,8 @@ jobs: exit 1 fi + # Same idea for the unstable/prerelease-track version: always + # bump its minor ("b" prefix) number forward by one. if [[ $unstable_version =~ ^0\.([0-9]+)b[0-9]+$ ]]; then unstable_minor="${BASH_REMATCH[1]}" else @@ -168,15 +278,27 @@ jobs: unstable_next_version="0.$((unstable_minor + 1))b0" + # STABLE_VERSION/UNSTABLE_VERSION here are the version *being + # released* (used below to generate that release's changelog + # section), while STABLE_NEXT_VERSION/UNSTABLE_NEXT_VERSION (with + # ".dev" appended back on) are what main's own version gets + # bumped to. echo "STABLE_VERSION=${stable_version}" >> $GITHUB_ENV echo "STABLE_NEXT_VERSION=${stable_next_version}.dev" >> $GITHUB_ENV echo "UNSTABLE_VERSION=${unstable_version}" >> $GITHUB_ENV echo "UNSTABLE_NEXT_VERSION=${unstable_next_version}.dev" >> $GITHUB_ENV + # Rewrites main's repo.toml/pyproject.toml pins/version files to the + # *next* dev version (not the version being released -- that happens + # on the release branch, in the sibling job above). - name: Update version - run: .github/scripts/update-version.sh $STABLE_NEXT_VERSION $UNSTABLE_NEXT_VERSION + run: ./scripts/release/update_version.py --stable_version=$STABLE_NEXT_VERSION --unstable_version=$UNSTABLE_NEXT_VERSION + # Generates the changelog section for the version being released + # (STABLE_VERSION/UNSTABLE_VERSION, not the _NEXT_ version), so that + # main's CHANGELOG.md documents the release the same way the release + # branch's own copy will. - name: Generate changelog run: towncrier build --yes --version "$STABLE_VERSION/$UNSTABLE_VERSION" @@ -189,6 +311,9 @@ jobs: app-id: ${{ vars.OTELBOT_APP_ID }} private-key: ${{ secrets.OTELBOT_PRIVATE_KEY }} + # Same pattern as the sibling job: commit, push to an otelbot branch, + # open a PR -- but this one targets main instead of the release + # branch. - name: Create pull request against main id: create_main_pr env: @@ -207,6 +332,10 @@ jobs: --base main) echo "pr_url=$pr_url" >> $GITHUB_OUTPUT + # Same labeling as the sibling job, plus "Skip Changelog": this PR's + # own diff already touches CHANGELOG.md directly (via towncrier + # above), so it doesn't need an *additional* changelog fragment of + # its own the way a normal PR would. - name: Add prepare-release label to PR if: steps.create_main_pr.outputs.pr_url != '' env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3179fc3486e..c5382f59e67 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,3 +1,37 @@ +# ============================================================================= +# Release workflow +# ============================================================================= +# +# Purpose: publishes a release. It builds wheels for every package in this +# repo, uploads them to PyPI (via a TestPyPI dry run first), and creates the +# corresponding GitHub release with release notes pulled straight out of +# CHANGELOG.md. +# +# This is the LAST of this repo's 3 release workflows to run, and it must be +# triggered manually (workflow_dispatch) on a "release/*" branch -- never on +# main. The full release pipeline looks like this: +# +# 1. prepare-release-branch.yml -- run on main. Cuts a new release branch +# (e.g. release/v1.45.x-0.66bx) and opens a pull request *against that +# new branch* which bumps repo.toml/pyproject.toml/version files to the +# version being released and updates CHANGELOG.md. A human reviews and +# merges that PR into the release branch. +# 2. release.yml (this file) -- run manually on the release branch, once +# the PR from step 1 (or step 3, for a patch) has been merged. Actually +# publishes the release described by the branch's current version. +# 3. prepare-patch-release.yml -- for a later patch release on the SAME +# release branch (e.g. 1.44.0 -> 1.44.1), run this first to open a PR +# that bumps the version/changelog again. Once merged, go back to step +# 2 to publish the patch. +# +# This workflow handles both cases -- "first release on this branch" (patch +# number 0) and "patch release" (patch number > 0) -- by reading the version +# already committed to repo.toml (via scripts/release/print_version.py) and +# checking whether the patch number is 0. It refuses to run unless +# CHANGELOG.md already documents the version being released, which is this +# workflow's way of confirming the PR from step 1/3 was actually merged +# before it starts building and publishing anything. +# ============================================================================= name: Release on: workflow_dispatch: @@ -11,22 +45,37 @@ jobs: contents: write # required for creating GitHub releases runs-on: ubuntu-latest steps: + # Guard rail: this job publishes real packages to PyPI and creates a + # real GitHub release, so it must never run against main or a feature + # branch by accident. GITHUB_REF_NAME is the branch that was selected + # when this workflow was manually triggered. - run: | if [[ $GITHUB_REF_NAME != release/* ]]; then echo this workflow should only be run against release branches exit 1 fi + # Checks out the release branch at the commit that triggered this run + # (i.e. GITHUB_REF_NAME, whatever the operator selected when starting + # the workflow). - uses: actions/checkout@v4 - - name: Install toml - run: pip install toml + # scripts/release/print_version.py needs tomlkit to parse repo.toml. + - name: Install tomlkit + run: pip install tomlkit + # Reads the version repo.toml is currently set to on this branch -- + # which prepare-release-branch.yml or prepare-patch-release.yml + # already bumped to the version being released -- and works out + # whether this is the first release on this branch (stable patch + # number == 0) or a patch release (patch number > 0). - name: Set environment variables run: | - stable_version=$(./scripts/eachdist.py version --mode stable) - unstable_version=$(./scripts/eachdist.py version --mode prerelease) + stable_version=$(./scripts/release/print_version.py --stable) + unstable_version=$(./scripts/release/print_version.py --unstable) + # Split e.g. "1.44.1" into stable_major=1 stable_minor=44 + # stable_patch=1 via a regex capture group match. if [[ $stable_version =~ ^([0-9]+)\.([0-9]+)\.([0-9]+) ]]; then stable_major="${BASH_REMATCH[1]}" stable_minor="${BASH_REMATCH[2]}" @@ -35,6 +84,12 @@ jobs: echo "unexpected stable_version: $stable_version" exit 1 fi + # A non-zero patch number means this is a patch release (e.g. + # 1.44.1, not 1.44.0): work out PRIOR_VERSION_WHEN_PATCH, the + # exact version being patched, one patch number back on both the + # stable and unstable tracks. This is only used later to word the + # release notes ("this is a patch release on the previous X + # release, fixing..."). if [[ $stable_patch != 0 ]]; then if [[ $unstable_version =~ ^0\.([0-9]+)b([0-9]+)$ ]]; then unstable_minor="${BASH_REMATCH[1]}" @@ -48,11 +103,21 @@ jobs: fi fi + # $GITHUB_ENV persists these as env vars for every later step in + # this job. echo "STABLE_VERSION=$stable_version" >> $GITHUB_ENV echo "UNSTABLE_VERSION=$unstable_version" >> $GITHUB_ENV + # Empty string when this isn't a patch release; later steps treat + # "is this variable non-empty" as the patch-release check. echo "PRIOR_VERSION_WHEN_PATCH=$prior_version_when_patch" >> $GITHUB_ENV + # Refuses to publish unless CHANGELOG.md already has a "## Version + # X/Y" section for the version being released. That section only + # exists once the PR opened by prepare-release-branch.yml (or + # prepare-patch-release.yml, for a patch) has actually been merged + # into this branch, so this is the check that catches "someone ran + # this workflow before merging that PR". - run: | if [[ -z $PRIOR_VERSION_WHEN_PATCH ]]; then # not making a patch release @@ -64,6 +129,15 @@ jobs: # check out main branch to verify there won't be problems with merging the change log # at the end of this workflow + # + # (Note this file doesn't contain a later step that merges anything + # into main -- the changelog backport for patch releases is instead a + # separate PR opened by prepare-patch-release.yml's own "Backport + # patch release changelog to main" step. This checkout appears to be a + # pre-flight sanity check, run before the expensive wheel-build/ + # publish steps below, though a plain `actions/checkout` doesn't + # itself merge anything, so it's unclear what failure mode it would + # actually catch.) - uses: actions/checkout@v4 with: ref: main @@ -76,6 +150,8 @@ jobs: with: python-version: '3.10' + # Builds sdists/wheels for every package into ./dist, via tox (see + # scripts/build.sh). - name: Build wheels run: ./scripts/build.sh @@ -89,6 +165,10 @@ jobs: # rejected by pypi (e.g "3 - Beta"). This would cause a failure during the # middle of the package upload causing the action to fail, and certain packages # might have already been updated, this would be bad. + # + # --skip-existing makes this (and the real PyPI upload below) safe to + # re-run: if a package version was already uploaded in a prior, + # partially-failed run, twine won't error out trying to re-upload it. - name: Publish to TestPyPI env: TWINE_USERNAME: '__token__' @@ -96,6 +176,8 @@ jobs: run: | twine upload --repository testpypi --skip-existing --verbose dist/* + # The real, production PyPI upload. Only reached if the TestPyPI dry + # run above succeeded. - name: Publish to PyPI env: TWINE_USERNAME: '__token__' @@ -103,11 +185,17 @@ jobs: run: | twine upload --skip-existing --verbose dist/* + # Builds /tmp/release-notes.txt, the body of the GitHub release + # created in the next step, out of CHANGELOG.md. - name: Generate release notes env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | # conditional block not indented because of the heredoc + # + # For a patch release, prefixes the release notes with a sentence + # naming the version being patched (computed above as + # PRIOR_VERSION_WHEN_PATCH). if [[ ! -z $PRIOR_VERSION_WHEN_PATCH ]]; then cat > /tmp/release-notes.txt << EOF This is a patch release on the previous $PRIOR_VERSION_WHEN_PATCH release, fixing the issue(s) below. @@ -117,14 +205,30 @@ jobs: # CHANGELOG_SECTION.md is also used at the end of the release workflow # for copying the change log updates to main + # + # Extracts just this version's section out of CHANGELOG.md: sed + # deletes everything up to and including the "## Version X/Y " + # heading line, then stops (quits) at the next "## Version" line, + # printing everything in between. sed -n "0,/^## Version ${STABLE_VERSION}\/${UNSTABLE_VERSION} /d;/^## Version /q;p" CHANGELOG.md \ > /tmp/CHANGELOG_SECTION.md # the complex perl regex is needed because markdown docs render newlines as soft wraps # while release notes render them as line breaks + # + # Appends the extracted section to release-notes.txt, collapsing + # markdown's soft-wrapped lines (a blank-line-free paragraph split + # across multiple lines) into single lines, but leaving actual + # blank lines and list items (bullets, numbered items) alone, so + # GitHub's release notes renderer doesn't show stray line breaks + # in the middle of a sentence. perl -0pe 's/(?> /tmp/release-notes.txt + # Creates the actual GitHub release: tags the current commit + # v$STABLE_VERSION, titles the release "Version X/Y" (both the stable + # and unstable version numbers, since this repo ships both together), + # and attaches the notes generated above. - name: Create GitHub release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/eachdist.ini b/eachdist.ini deleted file mode 100644 index a9c30cac21e..00000000000 --- a/eachdist.ini +++ /dev/null @@ -1,53 +0,0 @@ -# These will be sorted first in that order. -# All packages that are depended upon by others should be listed here. -[DEFAULT] - -sortfirst= - opentelemetry-api - opentelemetry-sdk - opentelemetry-proto - opentelemetry-distro - tests/opentelemetry-test-utils - exporter/* - -[stable] -version=1.44.0.dev - -packages= - opentelemetry-sdk - opentelemetry-proto - opentelemetry-propagator-jaeger - opentelemetry-propagator-b3 - opentelemetry-exporter-zipkin-proto-http - opentelemetry-exporter-zipkin-json - opentelemetry-exporter-zipkin - opentelemetry-exporter-otlp-proto-grpc - opentelemetry-exporter-otlp-proto-http - opentelemetry-exporter-otlp - opentelemetry-api - -[prerelease] -version=0.65b0.dev - -packages= - opentelemetry-opentracing-shim - opentelemetry-opencensus-shim - opentelemetry-exporter-http-transport - opentelemetry-exporter-opencensus - opentelemetry-exporter-prometheus - opentelemetry-exporter-otlp-json-common - opentelemetry-exporter-otlp-json-file - opentelemetry-exporter-otlp-common - opentelemetry-distro - opentelemetry-proto-json - opentelemetry-semantic-conventions - opentelemetry-test-utils - tests - -[lintroots] -extraroots=examples/*,scripts/ -subglob=*.py,tests/,test/,src/*,examples/* - -[testroots] -extraroots=examples/*,tests/ -subglob=tests/,test/ diff --git a/pyproject.toml b/pyproject.toml index a273dff32da..842bf5cc20e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,6 +102,9 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "docs/**/*.*" = ["PLE"] "opentelemetry-sdk/tests/_configuration/test_models.py" = ["E402", "PLC0415"] +"scripts/release/print_version.py" = ["E402"] +"scripts/release/update_version.py" = ["E402"] +"scripts/release/update_patch_version.py" = ["E402"] "shim/opentelemetry-opentracing-shim/tests/*" = ["TID252"] [tool.ruff.lint.isort] diff --git a/repo.toml b/repo.toml new file mode 100644 index 00000000000..0a324b47893 --- /dev/null +++ b/repo.toml @@ -0,0 +1,45 @@ +# These will be sorted first in that order. +# All packages that are depended upon by others should be listed here. +[DEFAULT] +sortfirst = [ + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-proto", + "opentelemetry-distro", + "tests/opentelemetry-test-utils", + "exporter/*", +] + +[stable] +version = "1.44.0.dev" +packages = [ + "opentelemetry-sdk", + "opentelemetry-proto", + "opentelemetry-propagator-jaeger", + "opentelemetry-propagator-b3", + "opentelemetry-exporter-zipkin-proto-http", + "opentelemetry-exporter-zipkin-json", + "opentelemetry-exporter-zipkin", + "opentelemetry-exporter-otlp-proto-grpc", + "opentelemetry-exporter-otlp-proto-http", + "opentelemetry-exporter-otlp", + "opentelemetry-api", +] + +[prerelease] +version = "0.65b0.dev" +packages = [ + "opentelemetry-opentracing-shim", + "opentelemetry-opencensus-shim", + "opentelemetry-exporter-http-transport", + "opentelemetry-exporter-opencensus", + "opentelemetry-exporter-prometheus", + "opentelemetry-exporter-otlp-json-common", + "opentelemetry-exporter-otlp-json-file", + "opentelemetry-exporter-otlp-common", + "opentelemetry-distro", + "opentelemetry-proto-json", + "opentelemetry-semantic-conventions", + "opentelemetry-test-utils", + "tests", +] diff --git a/scripts/coverage.sh b/scripts/coverage.sh deleted file mode 100755 index 99dce848782..00000000000 --- a/scripts/coverage.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -e - -function cov { - if [ ${TOX_ENV_NAME:0:4} == "py34" ] - then - pytest \ - --ignore-glob=instrumentation/opentelemetry-instrumentation-opentracing-shim/tests/testbed/* \ - --cov ${1} \ - --cov-append \ - --cov-branch \ - --cov-report='' \ - ${1} - else - pytest \ - --cov ${1} \ - --cov-append \ - --cov-branch \ - --cov-report='' \ - ${1} - fi -} - -coverage erase - -cov opentelemetry-api -cov opentelemetry-sdk -cov exporter/opentelemetry-exporter-datadog -cov instrumentation/opentelemetry-instrumentation-flask -cov instrumentation/opentelemetry-instrumentation-requests -cov instrumentation/opentelemetry-instrumentation-opentracing-shim -cov util/opentelemetry-util-http -cov exporter/opentelemetry-exporter-zipkin - - -cov instrumentation/opentelemetry-instrumentation-aiohttp-client -cov instrumentation/opentelemetry-instrumentation-asgi - -coverage report --show-missing -coverage xml diff --git a/scripts/eachdist.py b/scripts/eachdist.py deleted file mode 100755 index c5f5e9880c7..00000000000 --- a/scripts/eachdist.py +++ /dev/null @@ -1,752 +0,0 @@ -#!/usr/bin/env python3 -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -import argparse -import os -import re -import shlex -import shutil -import subprocess -import sys -from configparser import ConfigParser -from inspect import cleandoc -from itertools import chain -from os.path import basename -from pathlib import Path, PurePath - -from toml import load - -DEFAULT_ALLSEP = " " -DEFAULT_ALLFMT = "{rel}" - - -def unique(elems): - seen = set() - for elem in elems: - if elem not in seen: - yield elem - seen.add(elem) - - -subprocess_run = subprocess.run - - -def extraargs_help(calledcmd): - return cleandoc( - f""" - Additional arguments to pass on to {calledcmd}. - - This is collected from any trailing arguments passed to `%(prog)s`. - Use an initial `--` to separate them from regular arguments. - """ - ) - - -def parse_args(args=None): - parser = argparse.ArgumentParser(description="Development helper script.") - parser.set_defaults(parser=parser) - parser.add_argument( - "--dry-run", - action="store_true", - help="Only display what would be done, don't actually do anything.", - ) - subparsers = parser.add_subparsers(metavar="COMMAND") - subparsers.required = True - - excparser = subparsers.add_parser( - "exec", - help="Run a command for each or all targets.", - formatter_class=argparse.RawTextHelpFormatter, - description=cleandoc( - """Run a command according to the `format` argument for each or all targets. - - This is an advanced command that is used internally by other commands. - - For example, to install all distributions in this repository - editable, you could use: - - scripts/eachdist.py exec "python -m pip install -e {}" - - This will run pip for all distributions which is quite slow. It gets - a bit faster if we only invoke pip once but with all the paths - gathered together, which can be achieved by using `--all`: - - scripts/eachdist.py exec "python -m pip install {}" --all "-e {}" - - The sortfirst option in the DEFAULT section of eachdist.ini makes - sure that dependencies are installed before their dependents. - - Search for usages of `parse_subargs` in the source code of this script - to see more examples. - - This command first collects target paths and then executes - commands according to `format` and `--all`. - - Target paths are initially all Python distribution root paths - (as determined by the existence of pyproject.toml, etc. files). - They are then augmented according to the section of the - `PROJECT_ROOT/eachdist.ini` config file specified by the `--mode` option. - - The following config options are available (and processed in that order): - - - `extraroots`: List of project root-relative glob expressions. - The resulting paths will be added. - - `sortfirst`: List of glob expressions. - Any matching paths will be put to the front of the path list, - in the same order they appear in this option. If more than one - glob matches, ordering is according to the first. - - `subglob`: List of glob expressions. Each path added so far is removed - and replaced with the result of all glob expressions relative to it (in - order of the glob expressions). - - After all this, any duplicate paths are removed (the first occurrence remains). - """ - ), - ) - excparser.set_defaults(func=execute_args) - excparser.add_argument( - "format", - help=cleandoc( - """Format string for the command to execute. - - The available replacements depend on whether `--all` is specified. - If `--all` was specified, there is only a single replacement, - `{}`, that is replaced with the string that is generated from - joining all targets formatted with `--all` to a single string - with the value of `--allsep` as separator. - - If `--all` was not specified, the following replacements are available: - - - `{}`: the absolute path to the current target in POSIX format - (with forward slashes) - - `{rel}`: like `{}` but relative to the project root. - - `{raw}`: the absolute path to the current target in native format - (thus exactly the same as `{}` on Unix but with backslashes on Windows). - - `{rawrel}`: like `{raw}` but relative to the project root. - - The resulting string is then split according to POSIX shell rules - (so you can use quotation marks or backslashes to handle arguments - containing spaces). - - The first token is the name of the executable to run, the remaining - tokens are the arguments. - - Note that a shell is *not* involved by default. - You can add bash/sh/cmd/powershell yourself to the format if you want. - - If `--all` was specified, the resulting command is simply executed once. - Otherwise, the command is executed for each found target. In both cases, - the project root is the working directory. - """ - ), - ) - excparser.add_argument( - "--all", - nargs="?", - const=DEFAULT_ALLFMT, - metavar="ALLFORMAT", - help=cleandoc( - """Instead of running the command for each target, join all target - paths together to run a single command. - - This option optionally takes a format string to apply to each path. The - available replacements are the ones that would be available for `format` - if `--all` was not specified. - - Default ALLFORMAT if this flag is specified: `%(const)s`. - """ - ), - ) - excparser.add_argument( - "--allsep", - help=cleandoc( - """Separator string for the strings resulting from `--all`. - Only valid if `--all` is specified. - """ - ), - ) - excparser.add_argument( - "--allowexitcode", - type=int, - action="append", - default=[0], - help=cleandoc( - """The given command exit code is treated as success and does not abort execution. - Can be specified multiple times. - """ - ), - ) - excparser.add_argument( - "--mode", - "-m", - default="DEFAULT", - help=cleandoc( - """Section of config file to use for target selection configuration. - See description of exec for available options.""" - ), - ) - - instparser = subparsers.add_parser( - "install", help="Install all distributions." - ) - - def setup_instparser(instparser): - instparser.set_defaults(func=install_args) - instparser.add_argument( - "pipargs", nargs=argparse.REMAINDER, help=extraargs_help("pip") - ) - - setup_instparser(instparser) - instparser.add_argument("--editable", "-e", action="store_true") - instparser.add_argument("--with-dev-deps", action="store_true") - instparser.add_argument("--eager-upgrades", action="store_true") - - devparser = subparsers.add_parser( - "develop", - help="Install all distributions editable + dev dependencies.", - ) - setup_instparser(devparser) - devparser.set_defaults( - editable=True, - with_dev_deps=True, - eager_upgrades=True, - ) - - lintparser = subparsers.add_parser( - "lint", help="Lint everything, autofixing if possible." - ) - lintparser.add_argument("--check-only", action="store_true") - lintparser.set_defaults(func=lint_args) - - testparser = subparsers.add_parser( - "test", - help="Test everything (run pytest yourself for more complex operations).", - ) - testparser.set_defaults(func=test_args) - testparser.add_argument( - "pytestargs", nargs=argparse.REMAINDER, help=extraargs_help("pytest") - ) - - releaseparser = subparsers.add_parser( - "update_versions", - help="Updates version numbers, used by maintainers and CI", - ) - releaseparser.set_defaults(func=release_args) - releaseparser.add_argument("--versions", required=True) - releaseparser.add_argument( - "releaseargs", nargs=argparse.REMAINDER, help=extraargs_help("pytest") - ) - - patchreleaseparser = subparsers.add_parser( - "update_patch_versions", - help="Updates version numbers during patch release, used by maintainers and CI", - ) - patchreleaseparser.set_defaults(func=patch_release_args) - patchreleaseparser.add_argument("--stable_version", required=True) - patchreleaseparser.add_argument("--unstable_version", required=True) - patchreleaseparser.add_argument("--stable_version_prev", required=True) - patchreleaseparser.add_argument("--unstable_version_prev", required=True) - - fmtparser = subparsers.add_parser( - "format", - help="Formats all source code with black and isort.", - ) - fmtparser.set_defaults(func=format_args) - fmtparser.add_argument( - "--path", - required=False, - help="Format only this path instead of entire repository", - ) - - versionparser = subparsers.add_parser( - "version", - help="Get the version for a release", - ) - versionparser.set_defaults(func=version_args) - versionparser.add_argument( - "--mode", - "-m", - default="DEFAULT", - help=cleandoc( - """Section of config file to use for target selection configuration. - See description of exec for available options.""" - ), - ) - - return parser.parse_args(args) - - -def find_projectroot(search_start=Path(".")): - root = search_start.resolve() - for root in chain((root,), root.parents): - if any((root / marker).exists() for marker in (".git", "tox.ini")): - return root - return None - - -def find_targets_unordered(rootpath): - for subdir in rootpath.iterdir(): - if not subdir.is_dir(): - continue - if subdir.name.startswith(".") or subdir.name.startswith("venv"): - continue - if any( - (subdir / marker).exists() - for marker in ("setup.py", "pyproject.toml") - ): - yield subdir - else: - yield from find_targets_unordered(subdir) - - -def getlistcfg(strval): - return [ - val.strip() - for line in strval.split("\n") - for val in line.split(",") - if val.strip() - ] - - -def find_targets(mode, rootpath): - if not rootpath: - sys.exit("Could not find a root directory.") - - cfg = ConfigParser() - cfg.read(str(rootpath / "eachdist.ini")) - mcfg = cfg[mode] - - targets = list(find_targets_unordered(rootpath)) - if "extraroots" in mcfg: - targets += [ - path - for extraglob in getlistcfg(mcfg["extraroots"]) - for path in rootpath.glob(extraglob) - ] - if "sortfirst" in mcfg: - sortfirst = getlistcfg(mcfg["sortfirst"]) - - def keyfunc(path): - path = path.relative_to(rootpath) - for idx, pattern in enumerate(sortfirst): - if path.match(pattern): - return idx - return float("inf") - - targets.sort(key=keyfunc) - if "ignore" in mcfg: - ignore = getlistcfg(mcfg["ignore"]) - - def filter_func(path): - path = path.relative_to(rootpath) - for pattern in ignore: - if path.match(pattern): - return False - return True - - filtered = filter(filter_func, targets) - targets = list(filtered) - - subglobs = getlistcfg(mcfg.get("subglob", "")) - if subglobs: - targets = [ - newentry - for newentry in ( - target / subdir - for target in targets - for subglob in subglobs - # We need to special-case the dot, because glob fails to parse that with an IndexError. - for subdir in ( - (target,) if subglob == "." else target.glob(subglob) - ) - ) - if ".egg-info" not in str(newentry) and newentry.exists() - ] - - return list(unique(targets)) - - -def runsubprocess(dry_run, params, *args, **kwargs): - cmdstr = join_args(params) - if dry_run: - print(cmdstr) - return None - - # Py < 3.6 compat. - cwd = kwargs.get("cwd") - if cwd and isinstance(cwd, PurePath): - kwargs["cwd"] = str(cwd) - - check = kwargs.pop("check") # Enforce specifying check - - print(">>>", cmdstr, file=sys.stderr, flush=True) - - # This is a workaround for subprocess.run(['python']) leaving the virtualenv on Win32. - # The cause for this is that when running the python.exe in a virtualenv, - # the wrapper executable launches the global python as a subprocess and the search sequence - # for CreateProcessW which subprocess.run and Popen use is a follows - # (https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessw): - # > 1. The directory from which the application loaded. - # This will be the directory of the global python.exe, not the venv directory, due to the suprocess mechanism. - # > 6. The directories that are listed in the PATH environment variable. - # Only this would find the "correct" python.exe. - - params = list(params) - executable = shutil.which(params[0]) - if executable: - params[0] = executable - try: - return subprocess_run(params, *args, check=check, **kwargs) - except OSError as exc: - raise ValueError( - "Failed executing " + repr(params) + ": " + str(exc) - ) from exc - - -def execute_args(args): - if args.allsep and not args.all: - args.parser.error("--allsep specified but not --all.") - - if args.all and not args.allsep: - args.allsep = DEFAULT_ALLSEP - - rootpath = find_projectroot() - targets = find_targets(args.mode, rootpath) - if not targets: - sys.exit(f"Error: No targets selected (root: {rootpath})") - - def fmt_for_path(fmt, path): - return fmt.format( - path.as_posix(), - rel=path.relative_to(rootpath).as_posix(), - raw=path, - rawrel=path.relative_to(rootpath), - ) - - def _runcmd(cmd): - result = runsubprocess( - args.dry_run, shlex.split(cmd), cwd=rootpath, check=False - ) - if result is not None and result.returncode not in args.allowexitcode: - print( - f"'{cmd}' failed with code {result.returncode}", - file=sys.stderr, - ) - sys.exit(result.returncode) - - if args.all: - allstr = args.allsep.join( - fmt_for_path(args.all, path) for path in targets - ) - cmd = args.format.format(allstr) - _runcmd(cmd) - else: - for target in targets: - cmd = fmt_for_path(args.format, target) - _runcmd(cmd) - - -def clean_remainder_args(remainder_args): - if remainder_args and remainder_args[0] == "--": - del remainder_args[0] - - -def join_args(arglist): - return " ".join(map(shlex.quote, arglist)) - - -def install_args(args): - clean_remainder_args(args.pipargs) - if args.eager_upgrades: - args.pipargs += ["--upgrade-strategy=eager"] - - if args.with_dev_deps: - runsubprocess( - args.dry_run, - [ - "python", - "-m", - "pip", - "install", - "--upgrade", - "pip", - "setuptools", - "wheel", - ] - + args.pipargs, - check=True, - ) - - allfmt = "-e 'file://{}'" if args.editable else "'file://{}'" - - execute_args( - parse_subargs( - args, - ( - "exec", - "python -m pip install {} " + join_args(args.pipargs), - "--all", - allfmt, - ), - ) - ) - - if args.with_dev_deps: - rootpath = find_projectroot() - runsubprocess( - args.dry_run, - [ - "python", - "-m", - "pip", - "install", - "--upgrade", - "-r", - str(rootpath / "dev-requirements.txt"), - ] - + args.pipargs, - check=True, - ) - - -def parse_subargs(parentargs, args): - subargs = parse_args(args) - subargs.dry_run = parentargs.dry_run or subargs.dry_run - return subargs - - -def lint_args(args): - rootdir = str(find_projectroot()) - - runsubprocess( - args.dry_run, - ("black", "--config", "pyproject.toml", ".") - + (("--diff", "--check") if args.check_only else ()), - cwd=rootdir, - check=True, - ) - runsubprocess( - args.dry_run, - ("isort", "--settings-path", ".isort.cfg", ".") - + (("--diff", "--check-only") if args.check_only else ()), - cwd=rootdir, - check=True, - ) - runsubprocess( - args.dry_run, ("flake8", "--config", ".flake8", rootdir), check=True - ) - execute_args( - parse_subargs( - args, ("exec", "pylint {}", "--all", "--mode", "lintroots") - ) - ) - execute_args( - parse_subargs( - args, - ( - "exec", - "python scripts/check_for_valid_readme.py {}", - "--all", - ), - ) - ) - - -def find(name, path): - for root, _, files in os.walk(path): - if name in files: - return os.path.join(root, name) - return None - - -def filter_packages(targets, packages): - filtered_packages = [] - for target in targets: - for pkg in packages: - if pkg in str(target): - filtered_packages.append(target) - break - return filtered_packages - - -def update_version_files(targets, version, packages): - print("updating version/__init__.py files") - - search = "__version__ .*" - replace = f'__version__ = "{version}"' - - for target in filter_packages(targets, packages): - version_file_path = target.joinpath( - load(target.joinpath("pyproject.toml"))["tool"]["hatch"][ - "version" - ]["path"] - ) - - with open(version_file_path) as file: - text = file.read() - - if replace in text: - print(f"{version_file_path} already contains {replace}") - continue - - with open(version_file_path, "w", encoding="utf-8") as file: - file.write(re.sub(search, replace, text)) - - -def update_dependencies(targets, version, packages): - print("updating dependencies") - # PEP 508 allowed specifier operators - operators = ["==", "!=", "<=", ">=", "<", ">", "===", "~=", "="] - operators_pattern = "|".join(re.escape(op) for op in operators) - - for pkg in packages: - search = rf"({basename(pkg)}[^,]*)({operators_pattern})(.*\.dev)" - replace = r"\1\2 " + version - update_files( - targets, - "pyproject.toml", - search, - replace, - ) - - -def update_patch_dependencies(targets, version, prev_version, packages): - print("updating patch dependencies") - # PEP 508 allowed specifier operators - operators = ["==", "!=", "<=", ">=", "<", ">", "===", "~=", "="] - operators_pattern = "|".join(re.escape(op) for op in operators) - - for pkg in packages: - search = rf"({basename(pkg)}[^,]*?)(\s?({operators_pattern})\s?)(.*{prev_version})" - replace = r"\g<1>\g<2>" + version - print(f"{search=}\t{replace=}\t{pkg=}") - update_files( - targets, - "pyproject.toml", - search, - replace, - ) - - -def update_files(targets, filename, search, replace): - errors = False - for target in targets: - curr_file = find(filename, target) - if curr_file is None: - print(f"file missing: {target}/{filename}") - continue - - with open(curr_file, encoding="utf-8") as _file: - text = _file.read() - - if replace in text: - print(f"{curr_file} already contains {replace}") - continue - - with open(curr_file, "w", encoding="utf-8") as _file: - _file.write(re.sub(search, replace, text)) - - if errors: - sys.exit(1) - - -def release_args(args): - print("preparing release") - - rootpath = find_projectroot() - targets = list(find_targets_unordered(rootpath)) - cfg = ConfigParser() - cfg.read(str(find_projectroot() / "eachdist.ini")) - versions = args.versions - updated_versions = [] - for group in versions.split(","): - mcfg = cfg[group] - version = mcfg["version"] - updated_versions.append(version) - packages = mcfg["packages"].split() - print(f"update {group} packages to {version}") - update_dependencies(targets, version, packages) - update_version_files(targets, version, packages) - - -def patch_release_args(args): - print("preparing patch release") - - rootpath = find_projectroot() - targets = list(find_targets_unordered(rootpath)) - cfg = ConfigParser() - cfg.read(str(find_projectroot() / "eachdist.ini")) - # stable - mcfg = cfg["stable"] - packages = mcfg["packages"].split() - print(f"update stable packages to {args.stable_version}") - update_patch_dependencies( - targets, args.stable_version, args.stable_version_prev, packages - ) - update_version_files(targets, args.stable_version, packages) - - # prerelease - mcfg = cfg["prerelease"] - packages = mcfg["packages"].split() - print(f"update prerelease packages to {args.unstable_version}") - update_patch_dependencies( - targets, args.unstable_version, args.unstable_version_prev, packages - ) - update_version_files(targets, args.unstable_version, packages) - - -def test_args(args): - clean_remainder_args(args.pytestargs) - execute_args( - parse_subargs( - args, - ( - "exec", - "pytest {} " + join_args(args.pytestargs), - "--mode", - "testroots", - ), - ) - ) - - -def format_args(args): - root_dir = format_dir = str(find_projectroot()) - if args.path: - format_dir = os.path.join(format_dir, args.path) - - runsubprocess( - args.dry_run, - ("black", "--config", f"{root_dir}/pyproject.toml", "."), - cwd=format_dir, - check=True, - ) - runsubprocess( - args.dry_run, - ( - "isort", - "--settings-path", - f"{root_dir}/.isort.cfg", - "--profile", - "black", - ".", - ), - cwd=format_dir, - check=True, - ) - - -def version_args(args): - cfg = ConfigParser() - cfg.read(str(find_projectroot() / "eachdist.ini")) - print(cfg[args.mode]["version"]) - - -def main(): - args = parse_args() - args.func(args) - - -if __name__ == "__main__": - main() diff --git a/scripts/find.py b/scripts/find.py new file mode 100644 index 00000000000..e54d3c87b30 --- /dev/null +++ b/scripts/find.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers for locating this repo's root and its individual +package directories (each one a directory containing setup.py or +pyproject.toml), ordered per repo.toml's [DEFAULT].sortfirst list. + +Used by the release scripts in scripts/release/ and by +scripts/griffe_check.py's public-API breaking-change check.""" + +from collections.abc import Iterator +from itertools import chain +from pathlib import Path + +from tomlkit import load + + +def find_projectroot(search_start: Path = Path(".")) -> Path: + """Walks upward from search_start to the nearest directory containing + .git or tox.ini.""" + root = search_start.resolve() + for root in chain((root,), root.parents): + if any((root / marker).exists() for marker in (".git", "tox.ini")): + return root + raise FileNotFoundError( + "could not find project root (no .git or tox.ini) above " + f"{search_start.resolve()}" + ) + + +def find_package_dirs_unordered(root_path: Path) -> Iterator[Path]: + """Recursively yields every package directory (one containing setup.py + or pyproject.toml) under root_path, in arbitrary order.""" + for subdir in root_path.iterdir(): + if not subdir.is_dir(): + continue + if subdir.name.startswith(".") or subdir.name.startswith("venv"): + continue + if any( + (subdir / marker).exists() + for marker in ("setup.py", "pyproject.toml") + ): + yield subdir + else: + yield from find_package_dirs_unordered(subdir) + + +def find_package_dirs_ordered(root_path: Path) -> list[Path]: + """Returns every package directory under root_path, ordered per + repo.toml's [DEFAULT].sortfirst list.""" + with open(root_path / "repo.toml", encoding="utf-8") as file: + sortfirst = load(file)["DEFAULT"].get("sortfirst", []) + + package_directory_paths = list(find_package_dirs_unordered(root_path)) + + def keyfunc(path: Path) -> float: + """A package directory's index in sortfirst, or infinity if it + isn't listed.""" + path = path.relative_to(root_path) + for idx, pattern in enumerate(sortfirst): + if path.match(pattern): + return idx + return float("inf") + + package_directory_paths.sort(key=keyfunc) + + return list(dict.fromkeys(package_directory_paths)) diff --git a/scripts/griffe_check.py b/scripts/griffe_check.py index db24dec0cc6..167689c47b7 100644 --- a/scripts/griffe_check.py +++ b/scripts/griffe_check.py @@ -5,12 +5,12 @@ import sys import griffe -from eachdist import find_projectroot, find_targets +from find import find_package_dirs_ordered, find_projectroot def get_modules() -> list[str]: - rootpath = find_projectroot() - targets = find_targets("DEFAULT", rootpath) + root_path = find_projectroot() + package_directory_paths = find_package_dirs_ordered(root_path) dirs_to_exclude = [ "docs", @@ -21,8 +21,8 @@ def get_modules() -> list[str]: ] packages = [] - for target in targets: - rel_path = target.relative_to(rootpath) + for package_directory_path in package_directory_paths: + rel_path = package_directory_path.relative_to(root_path) if not any(excluded in str(rel_path) for excluded in dirs_to_exclude): packages.append(str(rel_path / "src")) return packages diff --git a/scripts/release/edit.py b/scripts/release/edit.py new file mode 100644 index 00000000000..8097a2275c6 --- /dev/null +++ b/scripts/release/edit.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Shared file-editing helpers used by update_version.py and +update_patch_version.py: rewriting repo.toml's version fields, bumping +each package's pinned dependencies, and rewriting each package's +__version__.""" + +from logging import getLogger +from os import walk +from os.path import join +from pathlib import Path +from re import escape, sub + +from tomlkit import dump, load + +logger = getLogger(__name__) + +# PEP 508 allowed specifier operators +OPERATORS_PATTERN = "|".join( + escape(op) for op in ["==", "!=", "<=", ">=", "<", ">", "===", "~=", "="] +) + + +def edit_version_files( + package_directory_paths: list[Path], version: str, packages: list[str] +) -> None: + """Rewrites __version__ to version in each package directory's version + file, for package directories matching one of packages.""" + logger.info("updating version/__init__.py files") + + replace = f'__version__ = "{version}"' + + for package_directory_path in package_directory_paths: + if not any(pkg in str(package_directory_path) for pkg in packages): + continue + + with open( + package_directory_path.joinpath("pyproject.toml"), + encoding="utf-8", + ) as file: + version_file_path = package_directory_path.joinpath( + load(file)["tool"]["hatch"]["version"]["path"] + ) + + with open(version_file_path) as file: + text = file.read() + + if replace in text: + logger.info("%s already contains %s", version_file_path, replace) + continue + + with open(version_file_path, "w", encoding="utf-8") as file: + file.write(sub("__version__ .*", replace, text)) + + +def edit_files( + package_directory_paths: list[Path], + filename: str, + search: str, + replace: str, +) -> None: + """Finds filename under each package directory and replaces every + regex match of search with replace.""" + for package_directory_path in package_directory_paths: + curr_file = None + for root, _, files in walk(package_directory_path): + if filename in files: + curr_file = join(root, filename) + break + + if curr_file is None: + logger.warning( + "file missing: %s/%s", package_directory_path, filename + ) + continue + + with open(curr_file, encoding="utf-8") as _file: + text = _file.read() + + if replace in text: + logger.info("%s already contains %s", curr_file, replace) + continue + + with open(curr_file, "w", encoding="utf-8") as _file: + _file.write(sub(search, replace, text)) + + +def edit_repo_toml_version( + root_path: Path, section: str, version: str +) -> None: + """Sets repo.toml's [section].version to version.""" + repo_toml_path = root_path / "repo.toml" + with open(repo_toml_path, encoding="utf-8") as file: + data = load(file) + data[section]["version"] = version + with open(repo_toml_path, "w", encoding="utf-8") as file: + dump(data, file) diff --git a/scripts/release/print_version.py b/scripts/release/print_version.py new file mode 100755 index 00000000000..3e118dcc75f --- /dev/null +++ b/scripts/release/print_version.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Prints repo.toml's current stable or prerelease version to stdout, +for the release workflows to capture (e.g. +`$(./scripts/release/print_version.py --stable)`).""" + +from argparse import ArgumentParser +from pathlib import Path +from sys import path + +path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from find import find_projectroot +from tomlkit import load + +parser = ArgumentParser(description="Get the version for a release") +group = parser.add_mutually_exclusive_group(required=True) +group.add_argument( + "--stable", dest="section", action="store_const", const="stable" +) +group.add_argument( + "--unstable", dest="section", action="store_const", const="prerelease" +) + +with open(find_projectroot() / "repo.toml", encoding="utf-8") as file: + print(load(file)[parser.parse_args().section]["version"]) diff --git a/scripts/release/update_patch_version.py b/scripts/release/update_patch_version.py new file mode 100755 index 00000000000..4bb468203d8 --- /dev/null +++ b/scripts/release/update_patch_version.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Bumps repo.toml's stable/prerelease versions for a patch release, +along with every dependency pin and __version__ file currently pinned +to the exact previous version. Unlike update_version.py, this can't +rely on a ".dev" suffix to find what to replace -- patch releases bump +an already-released version -- so it needs the previous version to +know exactly what to replace. + +Example, given --stable_version=1.44.1 --stable_version_prev=1.44.0: + +repo.toml, before: + [stable] + version = "1.44.0" +after: + [stable] + version = "1.44.1" + +opentelemetry-sdk/pyproject.toml, before: + dependencies = [ + "opentelemetry-api == 1.44.0", + ] +after: + dependencies = [ + "opentelemetry-api == 1.44.1", + ] + +opentelemetry-sdk/src/opentelemetry/sdk/version/__init__.py, before: + __version__ = "1.44.0" +after: + __version__ = "1.44.1" +""" + +from argparse import ArgumentParser +from logging import INFO, basicConfig, getLogger +from os.path import basename +from pathlib import Path +from sys import path + +path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from edit import ( + OPERATORS_PATTERN, + edit_files, + edit_repo_toml_version, + edit_version_files, +) +from find import find_package_dirs_unordered, find_projectroot +from tomlkit import load + +basicConfig(level=INFO, format="%(message)s") +logger = getLogger(__name__) + + +def update_patch_dependencies( + package_directory_paths: list[Path], + version: str, + prev_version: str, + packages: list[str], +) -> None: + """For each of package_directory_paths, updates its pinned dependency + on packages from prev_version to version.""" + logger.info("updating patch dependencies") + for pkg in packages: + search = rf"({basename(pkg)}[^,]*?)(\s?({OPERATORS_PATTERN})\s?)(.*{prev_version})" + replace = r"\g<1>\g<2>" + version + logger.debug("search=%r replace=%r pkg=%r", search, replace, pkg) + edit_files(package_directory_paths, "pyproject.toml", search, replace) + + +parser = ArgumentParser( + description="Updates version numbers during patch release, used by maintainers and CI" +) +parser.add_argument("--stable_version", required=True) +parser.add_argument("--unstable_version", required=True) +parser.add_argument("--stable_version_prev", required=True) +parser.add_argument("--unstable_version_prev", required=True) +args = parser.parse_args() + +logger.info("preparing patch release") + +root_path = find_projectroot() +package_directory_paths = list(find_package_dirs_unordered(root_path)) + +edit_repo_toml_version(root_path, "stable", args.stable_version) +edit_repo_toml_version(root_path, "prerelease", args.unstable_version) + +with open(root_path / "repo.toml", encoding="utf-8") as file: + cfg = load(file) + +packages = cfg["stable"]["packages"] +logger.info("update stable packages to %s", args.stable_version) +update_patch_dependencies( + package_directory_paths, + args.stable_version, + args.stable_version_prev, + packages, +) +edit_version_files(package_directory_paths, args.stable_version, packages) + +packages = cfg["prerelease"]["packages"] +logger.info("update prerelease packages to %s", args.unstable_version) +update_patch_dependencies( + package_directory_paths, + args.unstable_version, + args.unstable_version_prev, + packages, +) +edit_version_files(package_directory_paths, args.unstable_version, packages) diff --git a/scripts/release/update_version.py b/scripts/release/update_version.py new file mode 100755 index 00000000000..f0cfc9f4f32 --- /dev/null +++ b/scripts/release/update_version.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Bumps repo.toml's stable/prerelease versions, along with every +dependency pin and __version__ file currently pinned to a ".dev" +version, to the given new version. Used both to finalize a release +branch's version and, separately, to advance main to the next dev +version right after a release branch is cut. + +Example, given --stable_version=1.45.0: + +repo.toml, before: + [stable] + version = "1.44.0.dev" +after: + [stable] + version = "1.45.0" + +opentelemetry-sdk/pyproject.toml, before: + dependencies = [ + "opentelemetry-api == 1.44.0.dev", + ] +after: + dependencies = [ + "opentelemetry-api == 1.45.0", + ] + +opentelemetry-sdk/src/opentelemetry/sdk/version/__init__.py, before: + __version__ = "1.44.0.dev" +after: + __version__ = "1.45.0" +""" + +from argparse import ArgumentParser +from logging import INFO, basicConfig, getLogger +from os.path import basename +from pathlib import Path +from sys import path + +path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from edit import ( + OPERATORS_PATTERN, + edit_files, + edit_repo_toml_version, + edit_version_files, +) +from find import find_package_dirs_unordered, find_projectroot +from tomlkit import load + +basicConfig(level=INFO, format="%(message)s") +logger = getLogger(__name__) + +parser = ArgumentParser( + description="Updates version numbers, used by maintainers and CI" +) +parser.add_argument("--stable_version", required=True) +parser.add_argument("--unstable_version", required=True) +args = parser.parse_args() + +logger.info("preparing release") + +root_path = find_projectroot() +package_directory_paths = list(find_package_dirs_unordered(root_path)) + +edit_repo_toml_version(root_path, "stable", args.stable_version) +edit_repo_toml_version(root_path, "prerelease", args.unstable_version) + +with open(root_path / "repo.toml", encoding="utf-8") as file: + cfg = load(file) + +for group, version in ( + ("stable", args.stable_version), + ("prerelease", args.unstable_version), +): + packages = cfg[group]["packages"] + logger.info("update %s packages to %s", group, version) + + logger.info("updating dependencies") + for pkg in packages: + edit_files( + package_directory_paths, + "pyproject.toml", + rf"({basename(pkg)}[^,]*)({OPERATORS_PATTERN})(.*\.dev)", + r"\1\2 " + version, + ) + + edit_version_files(package_directory_paths, version, packages) diff --git a/tox.ini b/tox.ini index 1acd90d9764..393f518fd6b 100644 --- a/tox.ini +++ b/tox.ini @@ -128,8 +128,6 @@ envlist = [testenv] deps = lint: -r dev-requirements.txt - coverage: pytest - coverage: pytest-cov api: -r {toxinidir}/opentelemetry-api/test-requirements.txt @@ -216,10 +214,6 @@ setenv = CONTRIB_REPO_INSTRUMENTATION_WSGI={env:CONTRIB_REPO_INSTRUMENTATION_WSGI:{env:CONTRIB_REPO}\#egg=opentelemetry-instrumentation-wsgi&subdirectory=instrumentation/opentelemetry-instrumentation-wsgi} CONTRIB_REPO_INSTRUMENTATION_FLASK={env:CONTRIB_REPO_INSTRUMENTATION_FLASK:{env:CONTRIB_REPO}\#egg=opentelemetry-instrumentation-flask&subdirectory=instrumentation/opentelemetry-instrumentation-flask} UV_CONFIG_FILE={toxinidir}/tox-uv.toml -commands_pre = - ; In order to get a healthy coverage report, - ; we have to install packages in editable mode. - coverage: python {toxinidir}/scripts/eachdist.py install --editable commands = test-opentelemetry-api: pytest {toxinidir}/opentelemetry-api/tests {posargs} @@ -301,8 +295,6 @@ commands = test-opentelemetry-test-utils: pytest {toxinidir}/tests/opentelemetry-test-utils/tests {posargs} lint-opentelemetry-test-utils: sh -c "cd tests && pylint --rcfile ../.pylintrc {toxinidir}/tests/opentelemetry-test-utils" - coverage: {toxinidir}/scripts/coverage.sh - [testenv:spellcheck] recreate = True deps = @@ -393,7 +385,7 @@ recreate = True deps = GitPython==3.1.50 griffe==1.7.3 - toml + tomlkit commands = ; griffe check before to fail fast if there are any issues python {toxinidir}/scripts/griffe_check.py