From 4b5e759faf6c2e64173d9bfb9ff164453d9f7f32 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Wed, 16 Sep 2026 10:18:49 +0100 Subject: [PATCH 1/3] test: restore progress conformance baseline Signed-off-by: lucarlig --- .../baselines/2026-07-28/modern/external-data-plane.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml b/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml index b34444f1..d26ef7ab 100644 --- a/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml +++ b/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml @@ -127,6 +127,10 @@ findings: check: sep-2575-server-declares-prompts-in-discover name: ServerDeclaresPromptsInDiscover status: FAILURE +- scenario: tools-call-with-progress + check: tools-call-with-progress + name: ToolsCallWithProgress + status: FAILURE - scenario: tools-list check: tools-list name: ToolsList From d103d54cb577f9ed4dc70e9387f9b65a76142945 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Wed, 16 Sep 2026 10:28:35 +0100 Subject: [PATCH 2/3] ci: update cf-integration to 0.4.0 Signed-off-by: lucarlig --- .github/workflows/conformance.yml | 4 ++-- _context/wiki/getting-started.md | 2 +- _context/wiki/performance.md | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 506217cb..1f01be88 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -61,8 +61,8 @@ jobs: with: tool: cargo-binstall - - name: Install cf-integration 0.3.2 binary - run: cargo binstall cf-integration@0.3.2 --no-confirm + - name: Install cf-integration 0.4.0 binary + run: cargo binstall cf-integration@0.4.0 --no-confirm - name: Prepare the exact data-plane image env: diff --git a/_context/wiki/getting-started.md b/_context/wiki/getting-started.md index 2f4a5e6d..ded05398 100644 --- a/_context/wiki/getting-started.md +++ b/_context/wiki/getting-started.md @@ -270,7 +270,7 @@ Install the same released harness version used by CI. Conformance and Inspector install and run Node/npm only inside Docker images. ```bash -cargo binstall cf-integration@0.3.2 --no-confirm +cargo binstall cf-integration@0.4.0 --no-confirm make conformance ``` diff --git a/_context/wiki/performance.md b/_context/wiki/performance.md index 38655c9a..eb81dab7 100644 --- a/_context/wiki/performance.md +++ b/_context/wiki/performance.md @@ -1,7 +1,7 @@ # Performance and Load Testing Load testing is owned by [`cf-integration`](https://crates.io/crates/cf-integration). -The commands below match **0.3.2**, the version pinned by this repository's +The commands below match **0.4.0**, the version pinned by this repository's conformance workflow. The old `scripts/cf-integration.sh` wrapper is no longer in this repository. @@ -11,7 +11,7 @@ Use Docker with enough resources for the selected topology. Install the pinned CLI and check its command reference: ```bash -cargo binstall cf-integration@0.3.2 --no-confirm +cargo binstall cf-integration@0.4.0 --no-confirm cf-integration load --help ``` @@ -70,7 +70,7 @@ Use equivalent backends, tools, hardware, authentication, policy, cache settings and client metadata for comparisons. A full-stack failure while discovering or publishing the catalog is a setup failure, not a throughput measurement. Standalone measurements omit control-plane publication and must be labeled as -such. See the [pinned harness documentation](https://github.com/contextforge-org/contextforge-dev-tools/blob/v0.3.2/README.md) +such. See the [pinned harness documentation](https://github.com/contextforge-org/contextforge-dev-tools/blob/v0.4.0/README.md) for topology and source selection. ## Benchmark Controls From 17f74ddcc00e2572ed740c280d86c62848963976 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Thu, 17 Sep 2026 12:09:13 +0100 Subject: [PATCH 3/3] ci: run conformance for every pull request Signed-off-by: lucarlig --- .github/workflows/ci.yml | 2 +- .github/workflows/conformance.yml | 60 ++++--------------------------- _context/wiki/getting-started.md | 2 +- _context/wiki/performance.md | 16 ++++----- _context/wiki/testing.md | 16 ++++----- 5 files changed, 23 insertions(+), 73 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4e7bd45..a47d8347 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,7 @@ jobs: steps: - uses: actions/checkout@v6.0.2 with: - # Match the PR revision selected by /conformance. + # Match the PR head revision used by the conformance workflow. ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2.9.1 diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 1f01be88..021927f7 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,9 +1,8 @@ name: Conformance -run-name: "Conformance for PR #${{ github.event.issue.number }}" +run-name: "Conformance for PR #${{ github.event.pull_request.number }}" on: - issue_comment: - types: [created] + pull_request: env: CARGO_TERM_COLOR: always @@ -11,58 +10,28 @@ env: jobs: conformance: name: conformance - if: >- - github.event.issue.pull_request && - github.event.comment.body == '/conformance' && - contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association) concurrency: - group: conformance-pr-${{ github.event.issue.number }} + group: conformance-pr-${{ github.event.pull_request.number }} cancel-in-progress: true permissions: actions: read contents: read - pull-requests: read - statuses: write runs-on: ubuntu-latest timeout-minutes: 60 steps: - - name: Resolve the exact PR revision - id: revision - env: - GH_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.issue.number }} - REPOSITORY: ${{ github.repository }} - run: >- - gh api "repos/$REPOSITORY/pulls/$PR_NUMBER" - --jq '"head_sha=\(.head.sha)"' - >> "$GITHUB_OUTPUT" - - - name: Publish the pending PR status - env: - GH_TOKEN: ${{ github.token }} - HEAD_SHA: ${{ steps.revision.outputs.head_sha }} - REPOSITORY: ${{ github.repository }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: >- - gh api --method POST "repos/$REPOSITORY/statuses/$HEAD_SHA" - -f state=pending - -f context=conformance - -f description='Conformance is running' - -f target_url="$RUN_URL" - - name: Check out data plane uses: actions/checkout@v6.0.2 with: fetch-depth: 0 persist-credentials: false - ref: ${{ steps.revision.outputs.head_sha }} + ref: ${{ github.event.pull_request.head.sha }} - uses: taiki-e/install-action@v2.75.27 with: tool: cargo-binstall - - name: Install cf-integration 0.4.0 binary - run: cargo binstall cf-integration@0.4.0 --no-confirm + - name: Install cf-integration 0.5.0 binary + run: cargo binstall cf-integration@0.5.0 --no-confirm - name: Prepare the exact data-plane image env: @@ -74,25 +43,8 @@ jobs: --image contextforge-data-plane:conformance - name: Run strict modern conformance - id: conformance env: CF_DATAPLANE_IMAGE: contextforge-data-plane:conformance CF_DATAPLANE_PULL_POLICY: never CF_DATAPLANE_REF: "" run: make conformance - - - name: Publish the final PR status - if: always() && steps.revision.outputs.head_sha != '' - env: - DESCRIPTION: Conformance ${{ job.status }} - GH_TOKEN: ${{ github.token }} - HEAD_SHA: ${{ steps.revision.outputs.head_sha }} - REPOSITORY: ${{ github.repository }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - STATE: ${{ job.status == 'success' && 'success' || job.status == 'failure' && 'failure' || 'error' }} - run: >- - gh api --method POST "repos/$REPOSITORY/statuses/$HEAD_SHA" - -f state="$STATE" - -f context=conformance - -f description="$DESCRIPTION" - -f target_url="$RUN_URL" diff --git a/_context/wiki/getting-started.md b/_context/wiki/getting-started.md index ded05398..0dc69da4 100644 --- a/_context/wiki/getting-started.md +++ b/_context/wiki/getting-started.md @@ -270,7 +270,7 @@ Install the same released harness version used by CI. Conformance and Inspector install and run Node/npm only inside Docker images. ```bash -cargo binstall cf-integration@0.4.0 --no-confirm +cargo binstall cf-integration@0.5.0 --no-confirm make conformance ``` diff --git a/_context/wiki/performance.md b/_context/wiki/performance.md index eb81dab7..ab500ec0 100644 --- a/_context/wiki/performance.md +++ b/_context/wiki/performance.md @@ -1,7 +1,7 @@ # Performance and Load Testing Load testing is owned by [`cf-integration`](https://crates.io/crates/cf-integration). -The commands below match **0.4.0**, the version pinned by this repository's +The commands below match **0.5.0**, the version pinned by this repository's conformance workflow. The old `scripts/cf-integration.sh` wrapper is no longer in this repository. @@ -11,8 +11,8 @@ Use Docker with enough resources for the selected topology. Install the pinned CLI and check its command reference: ```bash -cargo binstall cf-integration@0.4.0 --no-confirm -cf-integration load --help +cargo binstall cf-integration@0.5.0 --no-confirm +cf-integration load run --help ``` The standalone lane starts the external dataplane, Redis, nginx, and a fixture @@ -22,7 +22,7 @@ does not depend on the unsupported external `tools/list` method: ```bash CF_INTEGRATION_DIR="$PWD/.integration" \ CF_DATAPLANE_REPO="$PWD" CF_DATAPLANE_REF="$(git rev-parse HEAD)" \ - cf-integration load --lane external --protocol-version modern --standalone \ + cf-integration load run --lane external --client-era modern --standalone \ --users 1 --spawn-rate 1 --run-time 10s ``` @@ -34,7 +34,7 @@ comparing them. All external examples target modern MCP `2026-07-28`. ```bash CF_INTEGRATION_DIR="$PWD/.integration" \ CF_DATAPLANE_REPO="$PWD" CF_DATAPLANE_REF="$(git rev-parse HEAD)" \ - cf-integration load --lane external --protocol-version modern --standalone \ + cf-integration load run --lane external --client-era modern --standalone \ --users 20 --spawn-rate 5 --run-time 2m ``` @@ -60,9 +60,9 @@ versions alongside the report. Omit `--standalone` to include the control plane and its publication path: ```bash -cf-integration load --lane external --protocol-version modern \ +cf-integration load run --lane external --client-era modern \ --users 20 --spawn-rate 5 --run-time 2m -cf-integration load --lane builtin --protocol-version modern \ +cf-integration load run --lane builtin --client-era modern \ --users 20 --spawn-rate 5 --run-time 2m ``` @@ -70,7 +70,7 @@ Use equivalent backends, tools, hardware, authentication, policy, cache settings and client metadata for comparisons. A full-stack failure while discovering or publishing the catalog is a setup failure, not a throughput measurement. Standalone measurements omit control-plane publication and must be labeled as -such. See the [pinned harness documentation](https://github.com/contextforge-org/contextforge-dev-tools/blob/v0.4.0/README.md) +such. See the [pinned harness documentation](https://github.com/contextforge-org/contextforge-dev-tools/blob/v0.5.0/README.md) for topology and source selection. ## Benchmark Controls diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index 44f25a34..79f59260 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -88,15 +88,13 @@ owns the official fixture, control-plane registration, Compose topology, server and client runners, result rendering, and transactional baseline handling. This repository keeps only the CI invocation, Make targets, and expected findings. -Comment exactly `/conformance` on a pull request to run the **Conformance** -Actions workflow. Only repository owners, members, and collaborators can start -it. The workflow sets a pending `conformance` commit status, tests the pull request -head commit, and updates that status with the result. The `issue_comment` -workflow itself is read from the default branch, so a PR edit to that workflow -does not change the command handling until merged. CI builds and names the -conformance binary artifact using that same head SHA and retains it for 90 days, -so changes to `main` do not invalidate the artifact. It runs the modern client -and modern server eras through the external dataplane in standalone mode. This +The **Conformance** Actions workflow runs automatically for every pull request +revision and reports its result as a native pull request check. It checks out +the pull request head commit while CI builds and names the conformance binary +artifact using that same head SHA. The harness waits for that artifact when the +workflows start concurrently, and CI retains it for 90 days, so changes to +`main` do not invalidate the artifact. Conformance runs the modern client and +modern server eras through the external dataplane in standalone mode. This starts Redis, the dataplane, nginx, and the official fixture without the control plane. The harness discovers the fixture's tools, resources, templates, and prompts and publishes their routes and actual tool schemas directly to Redis