diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..9ea7e33 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +/.github/ @ZenRows/prod-approvers +/release-please-config.json @ZenRows/prod-approvers diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..b8ab1c3 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,7 @@ +# GitHub's generated notes: "What's Changed", one line per PR. The release PR +# itself is left out. +changelog: + exclude: + labels: + - "autorelease: pending" + - "autorelease: tagged" diff --git a/.github/workflows/commit-hygiene.yml b/.github/workflows/commit-hygiene.yml new file mode 100644 index 0000000..7296a0c --- /dev/null +++ b/.github/workflows/commit-hygiene.yml @@ -0,0 +1,73 @@ +name: Commit hygiene + +on: + pull_request: + types: [opened, edited, synchronize, reopened] + +permissions: + contents: read + pull-requests: read + +jobs: + check: + name: Title, body, branch and commits + if: >- + github.event.pull_request.user.login != 'dependabot[bot]' && + github.event.pull_request.user.login != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + # Untrusted text reaches the shell only through environment variables. + - name: Check the PR title, body, branch name and commit messages + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_TITLE: ${{ github.event.pull_request.title }} + PR_BODY: ${{ github.event.pull_request.body }} + PR_BRANCH: ${{ github.head_ref }} + run: | + set -euo pipefail + trailer='(?i)^(Co-authored-by|Signed-off-by|Claude-Session|Generated-by|Assisted-by|Reviewed-by):' + # Standard identifiers such as CVE-2024-1234, SHA-256, UTF-8, ISO-8601, PEP-8, RFC-9110 and HTTP-2 are not tracker keys. + tracker='\b(?!(?:CVE|GHSA|SHA|UTF|ISO|PEP|RFC|HTTP)-)[A-Z]{2,10}-[0-9]+\b' + # Attribution phrases only: product names such as Claude or OpenAI stay allowed. + attribution='(?i)(\bgenerated (with|by) (claude|chatgpt|copilot|openai|gemini|an? ai)\b|\bai-generated\b|noreply@anthropic\.com)' + conventional='^((fix|feat|perf)\([a-z][a-z0-9-]*\)|(build|chore|ci|docs|refactor|revert|style|test)(\([a-z][a-z0-9-]*\))?)!?: \S' + failed=0 + + # Every printed line starts with a label, so text from the PR cannot start a workflow command. + check() { # $1: where, $2: pattern, $3: rule; the text is on stdin + local hits + if hits=$(grep -nP -- "$2" | sed 's/^/ line /'); then + echo "$1 has $3:" + echo "$hits" + failed=1 + fi + } + scan() { # $1: where; the text is on stdin + local text + text=$(cat) + check "$1" "$trailer" "a trailer line" <<<"$text" + check "$1" "$tracker" "an issue tracker key" <<<"$text" + check "$1" "$attribution" "an AI attribution" <<<"$text" + } + + if ! grep -qP -- "$conventional" <<<"$PR_TITLE"; then + echo "PR title is not a Conventional Commit: type(scope)!: subject, with type one of build, chore, ci, docs, feat, fix, perf, refactor, revert, style, test. fix, feat and perf need a lowercase scope, for example fix(crawl): subject." + failed=1 + fi + scan "PR title" <<<"$PR_TITLE" + scan "PR body" <<<"$PR_BODY" + scan "Branch name" <<<"$PR_BRANCH" + # Read the commits first, so a failed API call fails the check instead of scanning nothing. + commits=$(gh api --paginate "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/commits" \ + --jq '.[] | "\(.sha[0:12]) \(.commit.message | @base64)"') + [ -n "$commits" ] || { echo "::error::The PR has no commits to check."; exit 1; } + while read -r sha message; do + scan "Commit $sha" < <(base64 -d <<<"$message") + done <<<"$commits" + + if [ "$failed" -ne 0 ]; then + echo "::error::Fix the PR title, body, branch name or commit messages listed above." + exit 1 + fi + echo "The PR title, body, branch name and commit messages are clean." diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ace932c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,261 @@ +# npm trusted publishing is bound to this file name. Do not rename it. +name: Release + +on: + push: + branches: [main] + workflow_dispatch: + inputs: + tag: + description: Existing release tag to build and publish again, for example v1.5.3 + required: true + type: string + +permissions: {} + +jobs: + release-pr: + name: Open or update the release PR + if: github.event_name == 'push' + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-release-pr + cancel-in-progress: false + permissions: + contents: write + pull-requests: write + steps: + - uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071 # v4.4.1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + config-file: release-please-config.json + manifest-file: .release-please-manifest.json + skip-github-release: true + + detect: + name: Detect a release + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + release: ${{ steps.detect.outputs.release }} + version: ${{ steps.detect.outputs.version }} + tag: ${{ steps.detect.outputs.tag }} + previous_tag: ${{ steps.detect.outputs.previous_tag }} + ref: ${{ steps.detect.outputs.ref }} + sha: ${{ steps.detect.outputs.sha }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - id: detect + env: + GH_TOKEN: ${{ github.token }} + BEFORE: ${{ github.event.before }} + INPUT_TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + manifest=.release-please-manifest.json + no_release() { echo "release=false" >>"$GITHUB_OUTPUT"; echo "No release: $1"; exit 0; } + + if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + if ! [[ "$INPUT_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::The tag must look like v1.2.3."; exit 1 + fi + git rev-parse -q --verify "refs/tags/$INPUT_TAG" >/dev/null || { + echo "::error::The tag $INPUT_TAG does not exist."; exit 1; } + # Only a tag this workflow released: its commit is on main and it + # has a GitHub release. A hand-made tag on an unreviewed commit is refused. + [ "$GITHUB_REF" = refs/heads/main ] || { + echo "::error::Run the Release workflow from main."; exit 1; } + git merge-base --is-ancestor "refs/tags/$INPUT_TAG" HEAD || { + echo "::error::The tag $INPUT_TAG is not on main."; exit 1; } + gh release view "$INPUT_TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1 || { + echo "::error::The tag $INPUT_TAG has no GitHub release."; exit 1; } + { + echo "release=true" + echo "version=${INPUT_TAG#v}" + echo "tag=$INPUT_TAG" + echo "ref=refs/tags/$INPUT_TAG" + echo "sha=$(git rev-list -n 1 "refs/tags/$INPUT_TAG")" + } >>"$GITHUB_OUTPUT" + exit 0 + fi + + # This push must merge a release PR that release-please opened and + # change the manifest. Branch name alone is not enough: anyone with + # write access can name a branch. "tagged" keeps a full re-run working. + pr=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq ' + [.[] | select(.merged_at != null + and (.head.ref | startswith("release-please--branches--main")) + and .user.login == "github-actions[bot]" + and ([.labels[].name] | any(. == "autorelease: pending" or . == "autorelease: tagged")))][0].number // empty') + [ -n "$pr" ] || no_release "this push does not merge a release PR" + if git diff --quiet "$BEFORE" "$GITHUB_SHA" -- "$manifest"; then no_release "the manifest did not change"; fi + + version=$(jq -r '."."' "$manifest") + previous=$(git show "$BEFORE:$manifest" | jq -r '."."') + echo "Release PR #$pr: v$previous -> v$version" + { + echo "release=true" + echo "version=$version" + echo "tag=v$version" + echo "previous_tag=v$previous" + echo "ref=$GITHUB_SHA" + echo "sha=$GITHUB_SHA" + } >>"$GITHUB_OUTPUT" + + # Checks and build run with read-only rights. The job that can mint an npm + # token runs no project code. + build: + name: Check and build + needs: detect + if: needs.detect.outputs.release == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + env: + VERSION: ${{ needs.detect.outputs.version }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.detect.outputs.ref }} + persist-credentials: false + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + package-manager-cache: false + + - name: Install + run: npm ci + + - name: Run the checks + if: github.event_name == 'push' + run: | + set -euo pipefail + npm run typecheck + npm test + + - name: Build + run: | + set -euo pipefail + # dist/ is the compiled output, so the tarball goes to package/. + mkdir package + npm pack --pack-destination package + # The built version must be the release version. + test -f "package/zenrows-cli-$VERSION.tgz" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: package + path: package/ + if-no-files-found: error + + release: + name: Tag, release and publish + needs: [detect, build] + runs-on: ubuntu-latest + # The release environment deploys only from main. + environment: release + concurrency: + group: ${{ github.workflow }}-release + cancel-in-progress: false + permissions: + contents: write + pull-requests: write + id-token: write + env: + VERSION: ${{ needs.detect.outputs.version }} + TAG: ${{ needs.detect.outputs.tag }} + PREVIOUS_TAG: ${{ needs.detect.outputs.previous_tag }} + COMMIT: ${{ needs.detect.outputs.sha }} + IS_PUSH: ${{ github.event_name == 'push' }} + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: package + path: package/ + + # A re-run after a partial release finds its own tag and goes on to publish. + - name: Check the tag + id: tag + if: env.IS_PUSH == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + ref=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$TAG" 2>/dev/null) || exit 0 + sha=$(jq -r .object.sha <<<"$ref") + if [ "$(jq -r .object.type <<<"$ref")" = tag ]; then + sha=$(gh api "repos/$GITHUB_REPOSITORY/git/tags/$sha" --jq .object.sha) + fi + if [ "$sha" != "$COMMIT" ]; then + echo "::error::The tag $TAG already exists on another commit ($sha)." + exit 1 + fi + echo "::notice::The tag $TAG already exists on this commit. Creating it is skipped." + echo "exists=true" >>"$GITHUB_OUTPUT" + + - name: Check npm + id: npm + run: | + status=$(curl -s -o /dev/null -w '%{http_code}' "https://registry.npmjs.org/@zenrows%2fcli/$VERSION") + if [ "$status" = 200 ]; then + echo "::notice::@zenrows/cli $VERSION is already on npm. Publishing is skipped." + echo "published=true" >>"$GITHUB_OUTPUT" + fi + + - name: Create the tag and GitHub release + if: env.IS_PUSH == 'true' && steps.tag.outputs.exists != 'true' + uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071 # v4.4.1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + config-file: release-please-config.json + manifest-file: .release-please-manifest.json + skip-github-pull-request: true + + - name: Set the release title and notes + if: env.IS_PUSH == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh api "repos/$GITHUB_REPOSITORY/releases/generate-notes" \ + -f "tag_name=$TAG" -f "previous_tag_name=$PREVIOUS_TAG" --jq .body | + gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --title "$TAG" --notes-file - + + # Trusted publishing needs npm 11.5.1 or later, which Node 24 has. + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + if: steps.npm.outputs.published != 'true' + with: + node-version: 24 + package-manager-cache: false + + - name: Publish to npm + if: steps.npm.outputs.published != 'true' + run: npm publish "package/zenrows-cli-$VERSION.tgz" --access public --ignore-scripts + + - name: Verify on npm + run: | + for _ in $(seq 10); do + status=$(curl -s -o /dev/null -w '%{http_code}' "https://registry.npmjs.org/@zenrows%2fcli/$VERSION") + [ "$status" = 200 ] && exit 0 + sleep 15 + done + echo "::error::@zenrows/cli $VERSION is not on npm after 150 seconds." + exit 1 + + - name: Write the summary + run: | + repo="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" + { + echo "## @zenrows/cli $VERSION" + echo "- Tag: [$TAG]($repo/tree/$TAG)" + echo "- Release: [$TAG]($repo/releases/tag/$TAG)" + echo "- npm: [@zenrows/cli $VERSION](https://www.npmjs.com/package/@zenrows/cli/v/$VERSION)" + echo "- Commit: [${COMMIT:0:7}]($repo/commit/$COMMIT)" + } >>"$GITHUB_STEP_SUMMARY" diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000..1adaf63 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "1.5.3" +} diff --git a/docs/contributing.md b/docs/contributing.md index 761bc02..6243da1 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -26,7 +26,7 @@ npm install # dev deps only (typescript, @types/node) npm run dev -- --help # run the CLI from source npm test # node --test npm run typecheck # tsc --noEmit -npm run build # tsc → dist/ (for publishing) +npm run build # tsc → dist/ ``` ## Adding a primitive @@ -70,6 +70,37 @@ default, shared with its Batch jobs), the test waits `retry_after` seconds and retries the create for up to 5 minutes, so run concurrent e2e runs on one account one at a time. +## Releases + +Merge PRs with Conventional Commit titles, for example `fix(crawl): ...` or +`feat(extract): ...`. `fix`, `feat` and `perf` need a scope. + +After each merge to `main`, release-please opens or updates the PR +`chore: release X.Y.Z`. It sets the version in `package.json`, +`package-lock.json` and `CLI_VERSION` in `src/core/config.ts`. Do not change +these by hand. + +To release, merge that PR. The workflow runs the checks, creates the tag +`vX.Y.Z` and the GitHub release, and publishes `@zenrows/cli` to npm. + +To publish an existing tag again, run the Release workflow by hand from +`main` with that tag. + +### If the checks fail after the release PR is merged + +The release PR runs no CI, so its checks first run after the merge. If they +fail, `main` already has the new version, but there is no tag, and the +merged release PR keeps the `autorelease: pending` label, which blocks the +next release PR. To recover: + +1. Fix the problem in a normal PR and merge it. +2. Create the release on that fix commit, with the version from + `.release-please-manifest.json`: + `gh release create vX.Y.Z --target --title vX.Y.Z --generate-notes --notes-start-tag ` +3. On the merged release PR, replace the label `autorelease: pending` with + `autorelease: tagged`. +4. Run the Release workflow by hand from `main` with the tag `vX.Y.Z`. + ## Rules - Never print or persist API keys. Redact secrets in logs and artifacts. diff --git a/package.json b/package.json index 876e54c..d55e455 100644 --- a/package.json +++ b/package.json @@ -51,8 +51,7 @@ "test:e2e": "npm run build && node --test dist/tests/e2e/*.test.js", "typecheck": "tsc --noEmit", "build": "rm -rf dist && tsc && chmod +x dist/bin/zenrows.js", - "prepack": "npm run build", - "version": "node scripts/sync-version.mjs && git add src/core/config.ts" + "prepack": "npm run build" }, "devDependencies": { "@types/node": "^22.10.0", diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..d22e8ba --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,71 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "release-type": "node", + "include-component-in-tag": false, + "skip-changelog": true, + "pull-request-title-pattern": "chore: release ${version}", + "changelog-sections": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "perf", + "section": "Performance Improvements" + }, + { + "type": "revert", + "section": "Reverts" + }, + { + "type": "docs", + "section": "Other", + "hidden": true + }, + { + "type": "chore", + "section": "Other", + "hidden": true + }, + { + "type": "test", + "section": "Other", + "hidden": true + }, + { + "type": "ci", + "section": "Other", + "hidden": true + }, + { + "type": "refactor", + "section": "Other", + "hidden": true + }, + { + "type": "build", + "section": "Other", + "hidden": true + }, + { + "type": "style", + "section": "Other", + "hidden": true + } + ], + "packages": { + ".": { + "package-name": "@zenrows/cli", + "exclude-paths": [ + "acceptance" + ], + "extra-files": [ + "src/core/config.ts" + ] + } + } +} diff --git a/scripts/sync-version.mjs b/scripts/sync-version.mjs deleted file mode 100644 index 0993ee1..0000000 --- a/scripts/sync-version.mjs +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env node -/** - * Keep CLI_VERSION (src/core/config.ts) in lockstep with package.json. - * - * There are two version sources: package.json (npm/publish) and the hardcoded - * CLI_VERSION const, which drives `zenrows --version` and the X-ZR-CLI-Version - * header. This script rewrites CLI_VERSION to match package.json so they never - * drift. It runs automatically via the npm `version` lifecycle (so - * `npm version ` updates both), and is also runnable - * standalone: `node scripts/sync-version.mjs`. - */ -import { readFileSync, writeFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; -import { dirname, join } from "node:path"; - -const root = join(dirname(fileURLToPath(import.meta.url)), ".."); -const { version } = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); -const configPath = join(root, "src", "core", "config.ts"); -const src = readFileSync(configPath, "utf8"); - -const re = /(export const CLI_VERSION = ")[^"]*(";)/; -if (!re.test(src)) { - console.error("sync-version: could not find `export const CLI_VERSION` in src/core/config.ts"); - process.exit(1); -} - -const next = src.replace(re, `$1${version}$2`); -if (next === src) { - console.log(`sync-version: CLI_VERSION already ${version}`); -} else { - writeFileSync(configPath, next); - console.log(`sync-version: CLI_VERSION -> ${version}`); -} diff --git a/src/core/config.ts b/src/core/config.ts index db1e9a0..5aa9833 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -16,7 +16,7 @@ export const CONFIG_VERSION = "0.1.0"; * entrypoint) so core modules — telemetry, signup provenance — can read it * without an import cycle. `VERSION` in `cli/index.ts` re-exports this. */ -export const CLI_VERSION = "1.5.3"; +export const CLI_VERSION = "1.5.3"; // x-release-please-version /** The User-Agent on every request to a Zenrows API. The gateway reads the version from it. */ export const CLI_USER_AGENT = `zenrows-cli/${CLI_VERSION}`; /** Env var to override the Fetch and Extract API base (local/staging testing). */ diff --git a/tests/config.test.ts b/tests/config.test.ts index 326c26f..29f6e2a 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -6,7 +6,7 @@ import { defaultConfig, loadConfig, saveConfig, DEFAULT_API_BASE, CLI_VERSION } import { createWorkspace } from "../src/core/workspace.ts"; import { tempRoot } from "./helpers.ts"; -test("CLI_VERSION stays in sync with package.json (kept aligned by npm `version` → scripts/sync-version.mjs)", () => { +test("CLI_VERSION stays in sync with package.json (release-please updates both)", () => { const pkg = JSON.parse(readFileSync(join(process.cwd(), "package.json"), "utf8")); assert.equal(CLI_VERSION, pkg.version); });