diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4535c0..1e393c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,3 +31,19 @@ jobs: - run: go test -race ./... - name: Build for Windows run: GOOS=windows go build -o /dev/null ./cmd/proof + + # Runs the action the way a server's CI would, against Babashka. As this + # isn't a release, the action builds proof from source. It's a test of + # the action rather than of Babashka, so only the eval checks run, and new + # checks Babashka fails don't break proof's CI. + action: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: DeLaGuardo/setup-clojure@13.7.0 + with: + bb: '1.13.226' + - uses: ./ + with: + profile: babashka + args: -v -only eval.* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ef4f153 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,35 @@ +# Builds proof for every platform when a version is tagged (e.g. v0.1.0) +# and publishes the binaries in a GitHub release, whose notes are the +# version's section of the CHANGELOG. +name: Release + +on: + push: + tags: ['v*'] + +permissions: + contents: write + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version: stable + - name: Build + run: script/build-release "${GITHUB_REF_NAME#v}" dist + - name: Write the release notes + run: | + version="${GITHUB_REF_NAME#v}" + # Everything between the version's heading and the next one. + awk -v heading="## $version " 'index($0, heading) == 1 { on = 1; next } on && /^## / { exit } on' CHANGELOG.md > notes.md + if ! grep -q '[^[:space:]]' notes.md; then + echo "CHANGELOG.md has no section for $version" + exit 1 + fi + - name: Publish + env: + GH_TOKEN: ${{ github.token }} + run: gh release create "$GITHUB_REF_NAME" --title "proof ${GITHUB_REF_NAME#v}" --notes-file notes.md dist/* diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..60b3a3b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,30 @@ +# Changelog + +## main (unreleased) + +Initial release. + +### Features + +- Checks for what clients need from a server's replies to `describe`, + unknown ops, sessions, `eval` and `stdin`, and for every message the + server sends (the `wire.*` checks). +- [#4](https://github.com/nrepl/proof/pull/4): Client profiles, which send + what CIDER, Calva, Conjure, vim-fireplace, REPLy and neorepl send while + connecting and evaluating code + ([#6](https://github.com/nrepl/proof/pull/6), + [#8](https://github.com/nrepl/proof/pull/8)). +- Profiles for nREPL, Babashka, ClojureCLR, Basilisp, jank, dialtone and + repartee, which come with proof, and expected failures that can't go + stale. +- `proof matrix`, which builds a compatibility matrix out of the JSON + reports of several servers. +- [#1](https://github.com/nrepl/proof/pull/1): `proof proxy`, which checks + what a client sends to a server, with `-record` for starting a client + profile ([#5](https://github.com/nrepl/proof/pull/5)). +- [#2](https://github.com/nrepl/proof/pull/2): `proof serve`, a server for + the tests of clients that can behave like other servers + ([#3](https://github.com/nrepl/proof/pull/3)). +- [#10](https://github.com/nrepl/proof/pull/10): Binaries for Linux, macOS + and Windows on every release, and a GitHub Action for running proof in the + CI of a server. diff --git a/PLAN.md b/PLAN.md index a31005f..4882605 100644 --- a/PLAN.md +++ b/PLAN.md @@ -51,15 +51,16 @@ lookups and completions, plus requests that go wrong. ### Releases -- `proof version` takes the version from the build info, so a binary - installed with `go install ...@v0.1.0` knows its version. -- A CHANGELOG. -- Binaries for Linux, macOS and Windows on every release, so the CI of a - server doesn't need Go. -- A GitHub Action, so a server's workflow needs nothing but `uses: +- Done: `proof version` takes the version from the build info, so a + binary installed with `go install ...@v0.1.0` knows its version. +- Done: a CHANGELOG. +- Done: binaries for Linux, macOS and Windows on every release, so the + CI of a server doesn't need Go. The Release workflow builds them when + a version gets tagged. +- Done: a GitHub Action, so a server's workflow needs nothing but `uses: nrepl/proof@v0.1.0` and the path to its profile. -- The CI section of [Usage](doc/usage.md) uses the release instead of - `@latest`. +- Done: the CI section of [Usage](doc/usage.md) uses the release instead + of `@latest`. ### interrupt diff --git a/README.md b/README.md index 7b85674..d272d3a 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,9 @@ eventually. ## Quick Start -proof is a single binary. You can install it with Go 1.23 or newer: +proof is a single binary. You can download it from the +[releases](https://github.com/nrepl/proof/releases) or install it with Go +1.23 or newer: ```shell $ go install github.com/nrepl/proof/cmd/proof@latest diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..65dc3b8 --- /dev/null +++ b/action.yml @@ -0,0 +1,77 @@ +name: proof +description: Check that an nREPL server works with the clients people use +branding: + icon: check-circle + color: green + +inputs: + profile: + description: The profile of the server, or the name of one that comes with proof (e.g. babashka) + required: true + args: + description: More arguments for proof run, separated by spaces (e.g. -v -json report.json) + required: false + default: '' + +runs: + using: composite + steps: + - name: Install proof + shell: bash + env: + GH_TOKEN: ${{ github.token }} + OS: ${{ runner.os }} + ARCH: ${{ runner.arch }} + run: | + bin="$RUNNER_TEMP/proof-bin" + mkdir -p "$bin" + os=$(echo "$OS" | tr '[:upper:]' '[:lower:]') + [ "$os" = macos ] && os=darwin + case "$ARCH" in + X64) arch=amd64 ;; + ARM64) arch=arm64 ;; + *) arch=unknown ;; + esac + # The action lives in .../_actions/OWNER/REPO/REF. Composite + # actions don't always get github.action_ref and + # github.action_repository (actions/runner#2473), so they come from + # the path instead. + path=${GITHUB_ACTION_PATH//\\//} + ref=$(basename "$path") + repo=$(basename "$(dirname "$(dirname "$path")")")/$(basename "$(dirname "$path")") + + # A release (e.g. uses: nrepl/proof@v0.1.0) comes with binaries. + download() { + [[ "$ref" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]] && [ "$arch" != unknown ] || return 1 + local name="proof_${ref#v}_${os}_${arch}" archive dir="$RUNNER_TEMP/proof-download" + archive="$name.tar.gz" + [ "$os" = windows ] && archive="$name.zip" + gh release download "$ref" --repo "$repo" --pattern "$archive" --pattern checksums.txt --dir "$dir" --clobber && + (cd "$dir" && grep " $archive\$" checksums.txt | shasum -a 256 -c -) || return 1 + if [ "$os" = windows ]; then + 7z x -y -o"$dir" "$dir/$archive" > /dev/null && mv "$dir/$name/proof.exe" "$bin/" + else + tar -xzf "$dir/$archive" -C "$dir" && mv "$dir/$name/proof" "$bin/" + fi + } + + # Anything else (a branch, a commit or the action in proof's own + # repository), or a release that can't be downloaded, gets built + # from source, which needs Go. + if ! download; then + if ! command -v go > /dev/null; then + echo "Building proof from $repo@$ref needs Go, so set it up (e.g. with actions/setup-go) before this step." + exit 1 + fi + (cd "$GITHUB_ACTION_PATH" && go build -o "$bin/" ./cmd/proof) + fi + echo "$bin" >> "$GITHUB_PATH" + - name: Run proof + shell: bash + env: + PROFILE: ${{ inputs.profile }} + ARGS: ${{ inputs.args }} + run: | + # The arguments are split into words, but they're not file names. + set -f + proof run $ARGS "$PROFILE" diff --git a/cmd/proof/main.go b/cmd/proof/main.go index aca58b4..8683184 100644 --- a/cmd/proof/main.go +++ b/cmd/proof/main.go @@ -11,6 +11,7 @@ import ( "os" "os/signal" "path" + "runtime/debug" "strings" "syscall" "time" @@ -23,7 +24,27 @@ import ( "github.com/nrepl/proof/internal/server" ) -const version = "0.1.0-dev" +// version is set for releases, with -ldflags "-X main.version=...". +var version string + +// proofVersion returns the version of proof. +func proofVersion() string { + info, _ := debug.ReadBuildInfo() + return versionOf(version, info) +} + +// versionOf returns the version a release set, or else the one go install +// recorded in the binary's build info (which may be nil). Builds that have +// neither (e.g. go run) are 0.0.0-dev, so the version still has numbers. +func versionOf(release string, info *debug.BuildInfo) string { + switch { + case release != "": + return release + case info != nil && info.Main.Version != "" && info.Main.Version != "(devel)": + return strings.TrimPrefix(info.Main.Version, "v") + } + return "0.0.0-dev" +} const usage = `proof checks an nREPL server's compatibility with existing clients. @@ -64,7 +85,7 @@ func main() { case "list": list(os.Stdout) case "version", "--version", "-version": - fmt.Println("proof", version) + fmt.Println("proof", proofVersion()) case "help", "-h", "--help": printUsage(os.Stdout) default: @@ -192,7 +213,7 @@ func run(args []string) int { results := check.Run(env, selected, checks.WireRules()) outcome := check.Apply(results, p.ExpectedFailures, opts.only == "") - r := report.Run{Proof: version, Server: p.Name, Address: addr, Started: started, Results: results} + r := report.Run{Proof: proofVersion(), Server: p.Name, Address: addr, Started: started, Results: results} report.Text(os.Stdout, r, opts.verbose) for _, id := range outcome.Stale { fmt.Printf("%s is listed in expected-failures but didn't fail; remove it from the profile\n", id) diff --git a/cmd/proof/main_test.go b/cmd/proof/main_test.go new file mode 100644 index 0000000..e350a23 --- /dev/null +++ b/cmd/proof/main_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "runtime/debug" + "testing" +) + +func TestVersion(t *testing.T) { + built := func(v string) *debug.BuildInfo { return &debug.BuildInfo{Main: debug.Module{Version: v}} } + cases := []struct { + release string + info *debug.BuildInfo + want string + }{ + {"0.1.0", built("v0.2.0"), "0.1.0"}, + {"", built("v0.2.0"), "0.2.0"}, + {"", built("v0.0.0-20261023130420-411883581cfd"), "0.0.0-20261023130420-411883581cfd"}, + {"", built("(devel)"), "0.0.0-dev"}, + {"", nil, "0.0.0-dev"}, + } + for _, c := range cases { + if got := versionOf(c.release, c.info); got != c.want { + t.Errorf("versionOf(%q, %v) = %q, want %q", c.release, c.info, got, c.want) + } + } +} diff --git a/cmd/proof/proxy.go b/cmd/proof/proxy.go index 03adaba..dfa8c51 100644 --- a/cmd/proof/proxy.go +++ b/cmd/proof/proxy.go @@ -123,7 +123,7 @@ func proxyUntil(ctx context.Context, args []string, stdout, stderr io.Writer, li <-ctx.Done() fmt.Fprintln(stderr) - r := report.Run{Proof: version, Server: "client traffic to " + name, Address: upstream, Started: started} + r := report.Run{Proof: proofVersion(), Server: "client traffic to " + name, Address: upstream, Started: started} return gradeClients(stdout, stderr, r, px.Stop(time.Second), o.clientOptions, srv) } diff --git a/cmd/proof/serve.go b/cmd/proof/serve.go index 1c67e8b..16b2324 100644 --- a/cmd/proof/serve.go +++ b/cmd/proof/serve.go @@ -56,7 +56,7 @@ func serveUntil(ctx context.Context, args []string, stdout, stderr io.Writer, li } shown = append([]string{"like " + name}, shown...) } - srv, err := serve.Listen(o.listen, version, scenarios) + srv, err := serve.Listen(o.listen, proofVersion(), scenarios) if err != nil { fmt.Fprintln(stderr, "proof:", err) return 2 @@ -75,6 +75,6 @@ func serveUntil(ctx context.Context, args []string, stdout, stderr io.Writer, li <-ctx.Done() fmt.Fprintln(stderr) - r := report.Run{Proof: version, Server: "client traffic to " + name, Address: srv.Addr(), Started: started} + r := report.Run{Proof: proofVersion(), Server: "client traffic to " + name, Address: srv.Addr(), Started: started} return gradeClients(stdout, stderr, r, srv.Stop(time.Second), o.clientOptions, nil) } diff --git a/doc/hacking.md b/doc/hacking.md index 3f0d1ca..cc49d21 100644 --- a/doc/hacking.md +++ b/doc/hacking.md @@ -352,13 +352,34 @@ links using it still point to the right lines. ## Continuous Integration -proof uses a couple of GitHub Actions workflows: +proof uses a few GitHub Actions workflows: - CI checks the formatting, runs `go vet` and the tests on Linux and macOS (with both the oldest supported Go version and the latest stable one) and makes sure proof builds on Windows. +- CI also runs proof's GitHub Action (`action.yml`) against Babashka, + the way a server's CI would use it. - Compatibility runs all the profiles against their servers every day (and on every push to `main`) and builds the compatibility matrix in the job summary. As servers failing checks is perfectly normal, this workflow fails only when proof itself can't do its job (e.g. a server can't be installed or doesn't start). +- Release builds the binaries for every platform when a version gets + tagged and publishes them in a GitHub release. + +## Releasing + +Releasing a new version of proof is a very simple process: + +1. Make sure CI and the Compatibility workflow are green. +2. Turn the `main (unreleased)` section of the CHANGELOG into a section + for the new version (e.g. `## 0.2.0 (2027-01-15)`). +3. Update the version in the examples in [Usage](usage.md) (e.g. + `nrepl/proof@v0.2.0`). +4. Commit the changes ("Release 0.2.0"), tag the commit (`v0.2.0`) and + push both. + +The Release workflow takes it from there. It builds the binaries with +`script/build-release` and publishes them in a GitHub release, whose +notes are the version's section of the CHANGELOG. If you edit that +section later, update the notes of the release as well. diff --git a/doc/usage.md b/doc/usage.md index 0e8c26e..03b5a8c 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -7,19 +7,24 @@ profile format are covered separately in [Profiles](profiles.md). ## Installation -proof is a single Go binary, so installing it is a very simple process. -With Go 1.23 or newer you can do: +proof is a single binary, so installing it is a very simple process. +Every [release](https://github.com/nrepl/proof/releases) comes with +binaries for Linux, macOS and Windows. Just download the one for your +platform and put `proof` somewhere on your `PATH`. + +With Go 1.23 or newer you can also install it with `go install`: ```shell $ go install github.com/nrepl/proof/cmd/proof@latest ``` -That puts `proof` in `$(go env GOPATH)/bin`. +That puts `proof` in `$(go env GOPATH)/bin`. `@latest` is the latest +release, and `@main` is whatever is on `main` at the moment. > [!NOTE] -> There are no tagged releases yet, so `@latest` is simply whatever is on -> `main`. If you always need the same build (e.g. in CI), you can install a -> specific commit with `@`. +> proof 0.1.0 is not out yet, so for now `@latest` is simply whatever is +> on `main`. If you always need the same build (e.g. in CI), you can +> install a specific commit with `@`. Alternatively you can build proof from source: @@ -186,7 +191,8 @@ Expected failures still show up in the report. When one of them starts passing, proof will fail the run and ask you to remove it from the list. See [Profiles](profiles.md#expected-failures) for more details. -Here's an example GitHub Actions job: +On GitHub Actions you can use proof's own action, which runs `proof run` +with your profile: ```yaml proof: @@ -194,13 +200,22 @@ proof: steps: - uses: actions/checkout@v7 # set up and build your server here - - uses: actions/setup-go@v7 + - uses: nrepl/proof@v0.1.0 with: - go-version: stable - - run: go install github.com/nrepl/proof/cmd/proof@latest - - run: proof run -v test/proof.toml + profile: test/proof.toml + args: -v ``` +`args` are passed to `proof run` before the profile, so that's where +`-only` and `-json` go as well. The action downloads the binary of the +release you've picked, so it doesn't need Go. If you point it to a branch +or a commit instead (e.g. `nrepl/proof@main`), it builds proof from +source with the Go that comes with the runner. + +Elsewhere you can download proof from its +[releases](https://github.com/nrepl/proof/releases) or install it with +`go install github.com/nrepl/proof/cmd/proof@v0.1.0`. + ## Comparing Servers `proof matrix` can combine several JSON reports into a Markdown table diff --git a/script/build-release b/script/build-release new file mode 100755 index 0000000..de5f1ae --- /dev/null +++ b/script/build-release @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Builds the binaries of a release: an archive for each platform, with the +# license and the README, and their checksums. +# +# Usage: script/build-release VERSION DIR +set -euo pipefail + +version=$1 +dist=$2 +build=$(mktemp -d) +trap 'rm -rf "$build"' EXIT +mkdir -p "$dist" + +for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do + os=${target%/*} + arch=${target#*/} + name="proof_${version}_${os}_${arch}" + exe=proof + [ "$os" = windows ] && exe=proof.exe + mkdir "$build/$name" + CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -trimpath -ldflags "-s -w -X main.version=$version" \ + -o "$build/$name/$exe" ./cmd/proof + cp LICENSE README.md "$build/$name/" + if [ "$os" = windows ]; then + (cd "$build" && zip -qr - "$name") > "$dist/$name.zip" + else + tar -C "$build" -czf "$dist/$name.tar.gz" "$name" + fi +done + +(cd "$dist" && shasum -a 256 proof_* > checksums.txt)