Compare built-in and Rust load on FYRE

.github/workflows/images.yml

@@ -9,9 +9,19 @@ on:
       tag:
         type: string
         default: ''
+      publish_sdk_gateway:
+        type: boolean
+        default: false
+  workflow_dispatch:
+    inputs:
+      publish_sdk_gateway:
+        description: Publish the pinned MCP SDK v2 gateway used by FYRE comparisons
+        type: boolean
+        default: false
 
 jobs:
   build:
+    if: ${{ !inputs.publish_sdk_gateway }}
     name: Images (${{ matrix.arch }})
     strategy:
       fail-fast: false
@@ -108,7 +118,7 @@ jobs:
 
   publish:
     name: Publish image manifests
-    if: inputs.publish
+    if: inputs.publish && !inputs.publish_sdk_gateway
     needs: build
     runs-on: ubuntu-24.04
     permissions:
@@ -149,3 +159,96 @@ jobs:
               exit 1
             fi
           done
+
+  sdk-gateway:
+    name: SDK v2 gateway (${{ matrix.arch }})
+    if: inputs.publish_sdk_gateway
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            arch: amd64
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: read
+      packages: write
+    env:
+      GATEWAY_REVISION: 33e2dd93a53a9cc2c5088b731822dfec4852fa2e
+      GATEWAY_TAG: sdk-v2-33e2dd93a
+    steps:
+      - name: Checkout benchmark repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          path: benchmark
+          persist-credentials: false
+
+      - name: Checkout pinned gateway source
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: IBM/mcp-context-forge
+          ref: ${{ env.GATEWAY_REVISION }}
+          path: gateway
+          persist-credentials: false
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push pinned gateway
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
+        with:
+          context: gateway
+          file: gateway/Containerfile
+          platforms: linux/${{ matrix.arch }}
+          push: true
+          tags: ghcr.io/contextforge-org/cf-integration-fixture:gateway-${{ env.GATEWAY_TAG }}-${{ matrix.arch }}
+          labels: |
+            org.opencontainers.image.revision=${{ env.GATEWAY_REVISION }}
+            org.opencontainers.image.source=https://github.com/contextforge-org/contextforge-dev-tools
+          cache-from: type=gha,scope=sdk-gateway-${{ matrix.arch }}
+          cache-to: type=gha,mode=max,scope=sdk-gateway-${{ matrix.arch }}
+          provenance: false
+
+  publish-sdk-gateway:
+    name: Publish SDK v2 gateway manifest
+    if: inputs.publish_sdk_gateway
+    needs: sdk-gateway
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      packages: write
+    env:
+      GATEWAY_TAG: sdk-v2-33e2dd93a
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Publish multi-platform manifest
+        run: |
+          ref=ghcr.io/contextforge-org/cf-integration-fixture
+          docker buildx imagetools create \
+            --tag "$ref:gateway-$GATEWAY_TAG" \
+            "$ref:gateway-$GATEWAY_TAG-amd64" \
+            "$ref:gateway-$GATEWAY_TAG-arm64"
+
+      - name: Verify anonymous image access
+        run: |
+          docker logout ghcr.io
+          docker buildx imagetools inspect \
+            "ghcr.io/contextforge-org/cf-integration-fixture:gateway-$GATEWAY_TAG"

.github/workflows/quality.yml

@@ -36,6 +36,9 @@ jobs:
       - name: Run full test suite
         run: cargo test --all-targets --locked
 
+      - name: Run FYRE campaign tests
+        run: python3 -m unittest discover -s benchmarks/fyre -p 'test_*.py'
+
       - name: Verify standalone lazy runtime state
         shell: bash
         run: |
@@ -61,4 +64,3 @@ jobs:
 
       - name: Validate GitHub Actions workflows
         run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.12
-

CHANGELOG.md

@@ -7,6 +7,123 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
 
 ## [Unreleased]
 
+## [0.5.0] - 2026-09-16
+
+### Added
+
+- Add `load fyre run|status|destroy` (with short aliases) and a packaged FYRE
+  Terraform campaign for matched vertical/horizontal Rust dataplane scaling.
+  The campaign pins provider and container versions, uses dedicated Locust and
+  Fast Time VMs, grows saturated helpers, captures host/container telemetry,
+  preserves raw reports before cleanup, and produces JSON, CSV, and a
+  Slack-ready comparison PNG.
+
+- Preflight FYRE product-group CPU, memory, standalone-VM Ubuntu root-disk, and
+  public-IP quota for the full scaling matrix before provisioning any benchmark
+  VM. Distinguish this standalone-VM limitation from the OCP cluster API's
+  configurable `base_disk_size`.
+
+- Add a repeatable low-memory vertical profile for 2 vCPU / 2 GB and
+  4 vCPU / 4 GB dataplanes, with scenario multipliers derived from each
+  configuration's baseline resources. Allow profiles to start Locust and Fast
+  Time at independently validated helper sizes.
+
+- Add a reusable 2 vCPU / 2 GB FYRE profile for the complete eight-run built-in
+  dataplane versus external dataplane comparison.
+
+- Add `-w/--workers` to distribute load across local Locust processes,
+  `-i/--isolate-cpus` to split Docker CPUs between the target and load
+  generator, and `-m/--builtin-memory-limit` to tune the built-in gateway
+  without external environment setup.
+
+### Changed
+
+- Make bare `load fyre run` execute the CI-ready built-in-versus-Rust
+  comparison by default: the same modern `2026-07-28` client runs at 125, 250,
+  500, and 1,000 users for one measured hour per lane on the same 4 vCPU / 4 GB
+  target VM. Produce combined JSON, CSV, and Slack-ready PNG artifacts with
+  matching-lane throughput ratios, and restart the full comparison with larger
+  helpers when telemetry shows helper saturation.
+
+### Fixed
+
+- Derive FYRE comparison report labels and target resources from the selected
+  profile instead of requiring and displaying a hard-coded 4 vCPU / 4 GB
+  target.
+
+- Quote inventory-derived backend URLs before composing every FYRE remote shell
+  command.
+
+- Pin the built-in comparison lane to the MCP SDK v2 gateway revision that
+  supports the same `2026-07-28` client as Rust, balance replicas across
+  distributed Locust workers, and keep benchmark services off FYRE public
+  interfaces.
+
+- Refine detected throughput plateaus to the configured concurrency boundary
+  before confirming capacity instead of confirming the highest doubled load.
+
+- Publish the dataplane's Redis-backed MCP Host and Origin policy before
+  startup, and isolate client-conformance scenarios from its per-user config
+  cache.
+
+- Discover the FYRE account's default or sole product group when no override is
+  configured, use API-compatible VM descriptions, and log in with the root SSH
+  account provisioned by FYRE's Ubuntu images.
+
+- Bootstrap FYRE hosts in parallel with pinned Ansible, installing Docker Engine
+  and Compose from Docker's Ubuntu repository when the base image lacks them.
+
+- Disable Ubuntu's automatic APT timers and services on FYRE benchmark hosts so
+  package upgrades cannot consume resources or restart services during a run.
+
+- Keep the FYRE dataplane and loopback JWKS helper in a stable shared network
+  namespace so either process can restart without breaking sidecar startup.
+
+- Omit an empty MCP allowed-origin environment value that prevented the Rust
+  dataplane from starting on FYRE.
+
+- Invoke the FYRE smoke script correctly through the Locust image's Python
+  entrypoint before beginning a capacity step.
+
+- Run FYRE smoke and Locust containers with access to the protected benchmark
+  bundle and root-owned report mounts on ephemeral load-generator VMs.
+
+- Use a valid Fast Time conversion timestamp and deploy the packaged Locust
+  workload to the FYRE load-generator VM.
+
+- Fully detach telemetry monitors from their SSH sessions so a benchmark phase
+  starts immediately instead of waiting for its own monitor to exit.
+
+- Accept Docker's empty-container telemetry form when evaluating helper and
+  dataplane pressure after a load phase.
+
+- Allow clean distributed-worker shutdown at the Locust time limit while still
+  stopping the coordinator immediately for nonzero or missing workers.
+
+- Record FYRE measurement boundaries from Locust's keyword-based spawning event
+  so ramp and warmup traffic remain excluded from reported statistics.
+
+- Propagate distributed worker failures to the Locust coordinator and reject
+  reports containing a hidden worker failure.
+
+- Raise the open-file limit for the load generator and external dataplane so
+  high-concurrency tests measure service capacity instead of Docker's low
+  default descriptor limit.
+
+- Remove the Locust client's 50-200 ms think time so load runs measure maximum
+  request throughput.
+
+- Refine below a provisional capacity when a confirmation repetition fails,
+  then confirm the lower zero-error boundary instead of aborting the remaining
+  FYRE comparison scenarios.
+
+- Keep each FYRE capacity step on one continuous Locust user population through
+  ramp, steady-state warmup, and measurement, resetting statistics only after
+  warmup completes.
+
+- Pull the pinned Locust and Redis images through the Google registry mirror and
+  retry transient container-pull failures with bounded backoff.
+
 ## [0.4.0] - 2026-09-14
 
 ### Added
@@ -210,7 +327,9 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
 - Added builtin and external dataplane routing through reusable Docker Compose
   overlays.
 
-[Unreleased]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.3.1...HEAD
+[Unreleased]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.5.0...HEAD
+[0.5.0]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.4.0...v0.5.0
+[0.4.0]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.3.1...v0.4.0
 [0.3.1]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.3.0...v0.3.1
 [0.3.0]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.2.0...v0.3.0
 [0.2.0]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.1.0...v0.2.0

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cf-integration"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2024"
 rust-version = "1.97"
 license = "Apache-2.0"
@@ -18,6 +18,9 @@ include = [
     "/src/**",
     "/docker/**",
     "/scripts/locustfile_mcp.py",
+    "/benchmarks/fyre/**",
+    "!/benchmarks/fyre/**/__pycache__/**",
+    "!/benchmarks/fyre/**/*.pyc",
     "/scripts/live_protocol/sitecustomize.py",
     "/tests/conformance/baselines/**",
     "/README.md",

README.md

@@ -47,7 +47,7 @@ Every public command and option has a short form, shown in `--help`.
 | --- | --- | --- |
 | `stack` | `s` | `up` → `u`, `down` → `d`, `status` → `s`, `logs` → `l`, `config` → `c` |
 | `probe` | `p` | — |
-| `load` | `l` | `run` → `r` |
+| `load` | `l` | `run` → `r`, `fyre` → `f` (`run` → `r`, `status` → `s`, `destroy` → `d`) |
 | `live` | `v` | — |
 | `conformance` | `c` | `run` → `r`, `report` → `p` |
 | `debug` | `d` | `inspect` → `i`, `token` → `t` |
@@ -179,12 +179,26 @@ cf-integration load run --lane external --client-era legacy --standalone \
 # Include telemetry when diagnostic value matters more than benchmark purity
 cf-integration load run --lane external --client-era modern --standalone \
   --observability --users 10 --spawn-rate 2 --run-time 2m
+
+# Spread a high-throughput run across eight local Locust workers
+cf-integration load run --lane external --client-era modern --standalone \
+  --workers 8 --isolate-cpus \
+  --users 1000 --spawn-rate 100 --run-time 30s
+
+# Raise the built-in gateway limit for a high-concurrency comparison
+cf-integration load run --lane builtin --client-era legacy \
+  --builtin-memory-limit 16G --workers 8 \
+  --isolate-cpus --users 1000 --spawn-rate 100 --run-time 30s
 ```
 
 `--smoke` selects a short workload. Durations accept ordered positive `h`, `m`,
 and `s` groups such as `2m30s`. Defaults are `100` users, `10` users/s, and
 `5m`, overridable with `LOCUST_USERS`, `LOCUST_SPAWN_RATE`, and
-`LOCUST_RUN_TIME`. Observability is opt-in for load tests to avoid skew.
+`LOCUST_RUN_TIME`. `-w/--workers` starts that many local Locust worker
+processes and defaults to one. `-m/--builtin-memory-limit` overrides the
+built-in gateway container limit for that run. `-i/--isolate-cpus` splits all
+Docker CPUs evenly between the selected target and Locust. Observability is
+opt-in for load tests to avoid skew.
 
 `--client-era` accepts `legacy` or `modern` (default). The harness owns the
 Locust client: legacy uses initialization and the server's negotiated revision;
@@ -195,15 +209,34 @@ Every load lane uses the Fast Time server with the same `CF_FAST_TIME_EXPECTED_I
 override and the same `echo` payload (`{"message":"cf-integration"}`). Pin that
 image to a digest when comparing lanes. The measured workload contains only
 `tools/call`; initialization/discovery and builtin tool-name discovery happen
-once per user. Compare the `MCP tools/call` statistics to exclude setup traffic.
-A missing echo tool fails the run instead of producing an empty benchmark.
+once per user. Virtual users issue calls without client think time. Compare the
+`MCP tools/call` statistics to exclude setup traffic. A missing echo tool fails
+the run instead of producing an empty benchmark.
 
 There is no server-era selector: the backend must support the selected client
 era. `--standalone` runs Fast Time with the external dataplane and a harness
 routing snapshot in Redis, without the control plane. It discovers Fast Time's
 catalog directly; it never starts the conformance fixture or its proxy.
 Conformance, probes, and Inspector retain their protocol fixtures.
 
+### FYRE scaling campaign
+
+Run the reproducible vertical and horizontal Rust dataplane comparison on FYRE:
+
+```bash
+cf-integration load fyre run
+cf-integration load fyre status --run-id scale-candidate
+cf-integration load fyre destroy --run-id scale-candidate
+```
+
+The short forms are `cf-integration l f r`, `l f s`, and `l f d`; configuration
+and run IDs use `-f` and `-i`. The packaged matrix, infrastructure lifecycle,
+capacity-search rules, recovery behavior, and report layout are documented in
+[`benchmarks/fyre/README.md`](benchmarks/fyre/README.md). FYRE credentials stay
+in provider environment variables. All generated Terraform state, inventories,
+raw reports, telemetry, manifests, and the Slack-ready PNG are kept under
+`CF_INTEGRATION_DIR/fyre/<run-id>/`.
+
 ## Live gateway checks
 
 Groups are `mcp`, `rbac`, `protocol`, and `all` (default):